Try re-enable FF test with webRTC team options

Update all non-major dependencies

.eslintrc.cjs

@@ -8,7 +8,7 @@ Please see LICENSE in the repository root for full details.
 `;
 
 module.exports = {
-  plugins: ["matrix-org", "rxjs"],
+  plugins: ["matrix-org", "rxjs", "jsdoc"],
   extends: [
     "plugin:matrix-org/react",
     "plugin:matrix-org/a11y",
@@ -26,6 +26,13 @@ module.exports = {
     node: true,
   },
   rules: {
+    "jsdoc/no-types": "error",
+    "jsdoc/empty-tags": "error",
+    "jsdoc/check-property-names": "error",
+    "jsdoc/check-values": "error",
+    "jsdoc/check-param-names": "warn",
+    // "jsdoc/require-param": "warn",
+    "jsdoc/require-param-description": "warn",
     "matrix-org/require-copyright-header": ["error", COPYRIGHT_HEADER],
     "jsx-a11y/media-has-caption": "off",
     "react/display-name": "error",
@@ -75,6 +82,23 @@ module.exports = {
         "no-console": ["error"],
       },
     },
+    {
+      files: [
+        "**/*.test.ts",
+        "**/*.test.tsx",
+        "**/test.ts",
+        "**/test.tsx",
+        "**/test-**",
+      ],
+      rules: {
+        "jsdoc/no-types": "off",
+        "jsdoc/empty-tags": "off",
+        "jsdoc/check-property-names": "off",
+        "jsdoc/check-values": "off",
+        "jsdoc/check-param-names": "off",
+        "jsdoc/require-param-description": "off",
+      },
+    },
   ],
   settings: {
     react: {

.github/workflows/publish.yaml

@@ -61,6 +61,7 @@ jobs:
       docker_tags: |
         type=sha,format=short,event=branch
         type=raw,value=${{ github.event.release.tag_name }}
+        type=raw,value=latest
       # Like before, using ${{ env.VERSION }} above doesn't work
   add_docker_release_note:
     needs: publish_docker

config/config.devenv.json

@@ -15,5 +15,6 @@
     "delayed_leave_event_delay_ms": 18000,
     "delayed_leave_event_restart_ms": 4000,
     "network_error_retry_ms": 100
-  }
+  },
+  "trust_localhost_for_mute_state": true
 }

config/otel_dev/README.md

@@ -1,5 +1,12 @@
 # OpenTelemetry Collector for development
 
+## Edit:
+
+Open telemetry has been removed in: https://github.com/element-hq/element-call/pull/3586
+Check this PR to get back the implementation or to use it as reference to add it back.
+
+---
+
 This directory contains a docker compose file that starts a jaeger all-in-one instance
 with an in-memory database, along with a standalone OpenTelemetry collector that forwards
 traces into the jaeger. Jaeger has a built-in OpenTelemetry collector, but it can't be

knip.ts

@@ -9,7 +9,7 @@ import { type KnipConfig } from "knip";
 
 export default {
   vite: {
-    config: ["vite.config.ts", "vite-embedded.config.ts"],
+    config: ["vite.config.ts", "vite-embedded.config.ts", "vite-sdk.config.ts"],
   },
   entry: ["src/main.tsx", "i18next-parser.config.ts"],
   ignoreBinaries: [
@@ -34,10 +34,6 @@ export default {
     // then Knip will flag it as a false positive
     // https://github.com/webpro-nl/knip/issues/766
     "@vector-im/compound-web",
-    // We need this so that TypeScript is happy with @livekit/track-processors.
-    // This might be a bug in the LiveKit repo but for now we fix it on the
-    // Element Call side.
-    "@types/dom-mediacapture-transform",
     "matrix-widget-api",
   ],
   ignoreExportsUsedInFile: true,

package.json

@@ -13,6 +13,8 @@
     "build:embedded": "yarn build:full --config vite-embedded.config.js",
     "build:embedded:production": "yarn build:embedded",
     "build:embedded:development": "yarn build:embedded --mode development",
+    "build:sdk": "yarn build:full --config vite-sdk.config.js",
+    "build:sdk:development": "yarn build:sdk --mode development",
     "serve": "vite preview",
     "prettier:check": "prettier -c .",
     "prettier:format": "prettier -w .",
@@ -40,21 +42,14 @@
     "@codecov/vite-plugin": "^1.3.0",
     "@fontsource/inconsolata": "^5.1.0",
     "@fontsource/inter": "^5.1.0",
-    "@formatjs/intl-durationformat": "^0.7.0",
+    "@formatjs/intl-durationformat": "^0.9.0",
     "@formatjs/intl-segmenter": "^11.7.3",
     "@livekit/components-core": "^0.12.0",
     "@livekit/components-react": "^2.0.0",
     "@livekit/protocol": "^1.42.2",
-    "@livekit/track-processors": "^0.5.5",
+    "@livekit/track-processors": "^0.6.0 || ^0.7.1",
     "@mediapipe/tasks-vision": "^0.10.18",
-    "@opentelemetry/api": "^1.4.0",
-    "@opentelemetry/core": "^2.0.0",
-    "@opentelemetry/exporter-trace-otlp-http": "^0.203.0",
-    "@opentelemetry/resources": "^2.0.0",
-    "@opentelemetry/sdk-trace-base": "^2.0.0",
-    "@opentelemetry/sdk-trace-web": "^2.0.0",
-    "@opentelemetry/semantic-conventions": "^1.25.1",
-    "@playwright/test": "^1.56.1",
+    "@playwright/test": "^1.57.0",
     "@radix-ui/react-dialog": "^1.0.4",
     "@radix-ui/react-slider": "^1.1.2",
     "@radix-ui/react-visually-hidden": "^1.0.3",
@@ -67,7 +62,6 @@
     "@testing-library/react": "^16.0.0",
     "@testing-library/user-event": "^14.5.1",
     "@types/content-type": "^1.1.5",
-    "@types/dom-mediacapture-transform": "^0.1.11",
     "@types/grecaptcha": "^3.0.9",
     "@types/jsdom": "^21.1.7",
     "@types/lodash-es": "^4.17.12",
@@ -93,6 +87,7 @@
     "eslint-config-prettier": "^10.0.0",
     "eslint-plugin-deprecate": "^0.8.2",
     "eslint-plugin-import": "^2.26.0",
+    "eslint-plugin-jsdoc": "^61.5.0",
     "eslint-plugin-jsx-a11y": "^6.5.1",
     "eslint-plugin-matrix-org": "2.1.0",
     "eslint-plugin-react": "^7.29.4",
@@ -101,7 +96,7 @@
     "eslint-plugin-unicorn": "^56.0.0",
     "fetch-mock": "11.1.5",
     "global-jsdom": "^26.0.0",
-    "i18next": "^24.0.0",
+    "i18next": "^25.0.0",
     "i18next-browser-languagedetector": "^8.0.0",
     "i18next-parser": "^9.1.0",
     "jsdom": "^26.0.0",
@@ -109,8 +104,9 @@
     "livekit-client": "^2.13.0",
     "lodash-es": "^4.17.21",
     "loglevel": "^1.9.1",
-    "matrix-js-sdk": "^39.2.0",
+    "matrix-js-sdk": "matrix-org/matrix-js-sdk#develop",
     "matrix-widget-api": "^1.14.0",
+    "node-stdlib-browser": "^1.3.1",
     "normalize.css": "^8.0.1",
     "observable-hooks": "^4.2.3",
     "pako": "^2.0.4",
@@ -121,7 +117,7 @@
     "qrcode": "^1.5.4",
     "react": "19",
     "react-dom": "19",
-    "react-i18next": "^15.0.0",
+    "react-i18next": "^16.0.0 <16.1.0",
     "react-router-dom": "^7.0.0",
     "react-use-measure": "^2.1.1",
     "rxjs": "^7.8.1",
@@ -133,6 +129,7 @@
     "vite": "^7.0.0",
     "vite-plugin-generate-file": "^0.3.0",
     "vite-plugin-html": "^3.2.2",
+    "vite-plugin-node-stdlib-browser": "^0.2.1",
     "vite-plugin-svgr": "^4.0.0",
     "vitest": "^3.0.0",
     "vitest-axe": "^1.0.0-pre.3"

playwright.config.ts

@@ -65,8 +65,27 @@ export default defineConfig({
         ignoreHTTPSErrors: true,
         launchOptions: {
           firefoxUserPrefs: {
+            // Use the same set of options used by WebRTC tests in Firefox
+            // see https://webrtc.org/getting-started/testing
+            "media.navigator.permission.disabled": true,
             "permissions.default.microphone": 1,
             "permissions.default.camera": 1,
+            "browser.cache.disk.enable": false,
+            "browser.cache.disk.capacity": 0,
+            "browser.cache.disk.smart_size.enabled": false,
+            "browser.cache.disk.smart_size.first_run": false,
+            "browser.sessionstore.resume_from_crash": false,
+            "browser.startup.page": 0,
+            "media.navigator.streams.fake": true,
+            "media.navigator.permission.disabled": true,
+            "device.storage.enabled": false,
+            "media.gstreamer.enabled": false,
+            "browser.startup.homepage": "about:blank",
+            "browser.startup.firstrunSkipsHomepage": false,
+            "extensions.update.enabled": false,
+            "app.update.enabled": false,
+            "network.http.use-cache": false,
+            "browser.shell.checkDefaultBrowser": false,
           },
         },
       },

playwright/errors.spec.ts

@@ -7,6 +7,8 @@ Please see LICENSE in the repository root for full details.
 
 import { expect, test } from "@playwright/test";
 
+import { createJTWToken } from "./fixtures/jwt-token";
+
 test("Should show error screen if fails to get JWT token", async ({ page }) => {
   await page.goto("/");
 
@@ -93,7 +95,7 @@ test("Should show error screen if call creation is restricted", async ({
         contentType: "application/json",
         body: JSON.stringify({
           url: "wss://badurltotricktest/livekit/sfu",
-          jwt: "FAKE",
+          jwt: createJTWToken("@fake:user", "!fake:room"),
         }),
       }),
   );

playwright/fixtures/jwt-token.ts

@@ -0,0 +1,22 @@
+/*
+Copyright 2025 New Vector Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+export function createJTWToken(sub: string, room: string): string {
+  return [
+    {}, // header
+    {
+      // payload
+      sub,
+      video: {
+        room,
+      },
+    },
+    {}, // signature
+  ]
+    .map((d) => global.btoa(JSON.stringify(d)))
+    .join(".");
+}

playwright/fixtures/widget-user.ts

@@ -17,6 +17,7 @@ import type { MatrixClient } from "matrix-js-sdk";
 
 export type UserBaseFixture = {
   mxId: string;
+  displayName: string;
   page: Page;
   clientHandle: JSHandle<MatrixClient>;
 };
@@ -28,6 +29,7 @@ export type BaseWidgetSetup = {
 
 export interface MyFixtures {
   asWidget: BaseWidgetSetup;
+  callType: "room" | "dm";
 }
 
 const PASSWORD = "foobarbaz1!";
@@ -67,7 +69,6 @@ const CONFIG_JSON = {
 
 /**
  * Set the Element Call URL in the dev tool settings using `window.mxSettingsStore` via `page.evaluate`.
- * @param page
  */
 const setDevToolElementCallDevUrl = process.env.USE_DOCKER
   ? async (page: Page): Promise<void> => {
@@ -111,19 +112,27 @@ async function registerUser(
   await page.getByRole("textbox", { name: "Confirm password" }).click();
   await page.getByRole("textbox", { name: "Confirm password" }).fill(PASSWORD);
   await page.getByRole("button", { name: "Register" }).click();
-  const continueButton = page.getByRole("button", { name: "Continue" });
-  try {
-    await expect(continueButton).toBeVisible({ timeout: 5000 });
-    await page
-      .getByRole("textbox", { name: "Password", exact: true })
-      .fill(PASSWORD);
-    await continueButton.click();
-  } catch {
-    // continueButton not visible, continue as normal
-  }
+
   await expect(
     page.getByRole("heading", { name: `Welcome ${username}` }),
   ).toBeVisible();
+
+  const browserUnsupportedToast = page
+    .getByText("Element does not support this browser")
+    .locator("..")
+    .locator("..");
+
+  // Dismiss incompatible browser toast
+  const dismissButton = browserUnsupportedToast.getByRole("button", {
+    name: "Dismiss",
+  });
+  try {
+    await expect(dismissButton).toBeVisible({ timeout: 700 });
+    await dismissButton.click();
+  } catch {
+    // dismissButton not visible, continue as normal
+  }
+
   await setDevToolElementCallDevUrl(page);
 
   const clientHandle = await page.evaluateHandle(() =>
@@ -138,25 +147,27 @@ async function registerUser(
 }
 
 export const widgetTest = test.extend<MyFixtures>({
-  asWidget: async ({ browser, context }, pUse) => {
+  // allow per-test override: `widgetTest.use({ callType: "dm" })`
+  callType: ["room", { option: true }],
+  asWidget: async ({ browser, context, callType }, pUse) => {
     await context.route(`http://localhost:8081/config.json*`, async (route) => {
       await route.fulfill({ json: CONFIG_JSON });
     });
 
-    const userA = `brooks_${Date.now()}`;
-    const userB = `whistler_${Date.now()}`;
+    const brooksDisplayName = `brooks_${Date.now()}`;
+    const whistlerDisplayName = `whistler_${Date.now()}`;
 
     // Register users
     const {
       page: ewPage1,
       clientHandle: brooksClientHandle,
       mxId: brooksMxId,
-    } = await registerUser(browser, userA);
+    } = await registerUser(browser, brooksDisplayName);
     const {
       page: ewPage2,
       clientHandle: whistlerClientHandle,
       mxId: whistlerMxId,
-    } = await registerUser(browser, userB);
+    } = await registerUser(browser, whistlerDisplayName);
 
     // Invite the second user
     await ewPage1
@@ -164,37 +175,66 @@ export const widgetTest = test.extend<MyFixtures>({
       .getByRole("button", { name: "New conversation" })
       .click();
 
-    await ewPage1.getByRole("menuitem", { name: "New Room" }).click();
-    await ewPage1.getByRole("textbox", { name: "Name" }).fill("Welcome Room");
-    await ewPage1.getByRole("button", { name: "Create room" }).click();
-    await expect(ewPage1.getByText("You created this room.")).toBeVisible();
-    await expect(ewPage1.getByText("Encryption enabled")).toBeVisible();
+    if (callType === "room") {
+      await ewPage1.getByRole("menuitem", { name: "New Room" }).click();
+      await ewPage1.getByRole("textbox", { name: "Name" }).fill("Welcome Room");
+      await ewPage1.getByRole("button", { name: "Create room" }).click();
+      await expect(ewPage1.getByText("You created this room.")).toBeVisible();
+      await expect(ewPage1.getByText("Encryption enabled")).toBeVisible();
 
-    await ewPage1
-      .getByRole("button", { name: "Invite to this room", exact: true })
-      .click();
-    await expect(
-      ewPage1.getByRole("heading", { name: "Invite to Welcome Room" }),
-    ).toBeVisible();
+      await ewPage1
+        .getByRole("button", { name: "Invite to this room", exact: true })
+        .click();
+      await expect(
+        ewPage1.getByRole("heading", { name: "Invite to Welcome Room" }),
+      ).toBeVisible();
 
-    // To get the invite textbox we need to specifically select within the
-    // dialog, since there is another textbox in the background (the message
-    // composer). In theory the composer shouldn't be visible to Playwright at
-    // all because the invite dialog has trapped focus, but the focus trap
-    // doesn't quite work right on Firefox.
-    await ewPage1.getByRole("dialog").getByRole("textbox").fill(whistlerMxId);
-    await ewPage1.getByRole("dialog").getByRole("textbox").click();
-    await ewPage1.getByRole("button", { name: "Invite" }).click();
+      // To get the invite textbox we need to specifically select within the
+      // dialog, since there is another textbox in the background (the message
+      // composer). In theory the composer shouldn't be visible to Playwright at
+      // all because the invite dialog has trapped focus, but the focus trap
+      // doesn't quite work right on Firefox.
+      await ewPage1.getByRole("dialog").getByRole("textbox").fill(whistlerMxId);
+      await ewPage1.getByRole("dialog").getByRole("textbox").click();
+      await ewPage1.getByRole("button", { name: "Invite" }).click();
 
-    // Accept the invite
-    await expect(
-      ewPage2.getByRole("option", { name: "Welcome Room" }),
-    ).toBeVisible();
-    await ewPage2.getByRole("option", { name: "Welcome Room" }).click();
-    await ewPage2.getByRole("button", { name: "Accept" }).click();
-    await expect(
-      ewPage2.getByRole("main").getByRole("heading", { name: "Welcome Room" }),
-    ).toBeVisible();
+      // Accept the invite
+      await expect(
+        ewPage2.getByRole("option", { name: "Welcome Room" }),
+      ).toBeVisible();
+      await ewPage2.getByRole("option", { name: "Welcome Room" }).click();
+      await ewPage2.getByRole("button", { name: "Accept" }).click();
+      await expect(
+        ewPage2
+          .getByRole("main")
+          .getByRole("heading", { name: "Welcome Room" }),
+      ).toBeVisible();
+    } else if (callType === "dm") {
+      await ewPage1.getByRole("menuitem", { name: "Start chat" }).click();
+      await ewPage1.getByRole("textbox", { name: "Search" }).click();
+      await ewPage1.getByRole("textbox", { name: "Search" }).fill(whistlerMxId);
+      await ewPage1.getByRole("button", { name: "Go" }).click();
+
+      // Wait and send the first message to create the DM
+      await expect(
+        ewPage1.getByText(/Send your first message to invite/),
+      ).toBeVisible();
+
+      await ewPage1.locator(".mx_BasicMessageComposer_input > div").click();
+      await ewPage1
+        .getByRole("textbox", { name: "Send a message…" })
+        .fill("Hello!");
+      await ewPage1.getByRole("button", { name: "Send message" }).click();
+
+      await expect(
+        ewPage1.getByText("This is the beginning of your"),
+      ).toBeVisible();
+
+      // Accept the DM invite from brooks
+      // This how playwright record selects the DM invite in the room list
+      await ewPage2.getByRole("option", { name: "Open room" }).click();
+      await ewPage2.getByRole("button", { name: "Start chatting" }).click();
+    }
 
     // Renamed use to pUse, as a workaround for eslint error that was thinking this use was a react use.
     await pUse({
@@ -202,11 +242,13 @@ export const widgetTest = test.extend<MyFixtures>({
         mxId: brooksMxId,
         page: ewPage1,
         clientHandle: brooksClientHandle,
+        displayName: brooksDisplayName,
       },
       whistler: {
         mxId: whistlerMxId,
         page: ewPage2,
         clientHandle: whistlerClientHandle,
+        displayName: whistlerDisplayName,
       },
     });
   },

playwright/sfu-reconnect-bug.spec.ts

@@ -68,11 +68,6 @@ test("When creator left, avoid reconnect to the same SFU", async ({
     reducedMotion: "reduce",
   });
   const guestCPage = await guestC.newPage();
-  let sfuGetCallCount = 0;
-  await guestCPage.route("**/livekit/jwt/sfu/get", async (route) => {
-    sfuGetCallCount++;
-    await route.continue();
-  });
   // Track WebSocket connections
   let wsConnectionCount = 0;
   await guestCPage.routeWebSocket("**", (ws) => {
@@ -100,5 +95,4 @@ test("When creator left, avoid reconnect to the same SFU", async ({
   // https://github.com/element-hq/element-call/issues/3344
   // The app used to request a new jwt token then to reconnect to the SFU
   expect(wsConnectionCount).toBe(1);
-  expect(sfuGetCallCount).toBe(2 /* the first one is for the warmup */);
 });

playwright/widget/voice-call-dm.spec.ts

@@ -0,0 +1,192 @@
+/*
+Copyright 2026 Element Creations Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+import { expect, test } from "@playwright/test";
+
+import { widgetTest } from "../fixtures/widget-user.ts";
+
+widgetTest.use({ callType: "dm" });
+
+widgetTest("Start a new voice call in DM as widget", async ({ asWidget }) => {
+  test.slow(); // Triples the timeout
+
+  const { brooks, whistler } = asWidget;
+
+  await expect(
+    brooks.page.getByRole("button", { name: "Voice call" }),
+  ).toBeVisible();
+  await brooks.page.getByRole("button", { name: "Voice call" }).click();
+
+  await expect(
+    brooks.page.getByRole("menuitem", { name: "Element Call" }),
+  ).toBeVisible();
+
+  await brooks.page.getByRole("menuitem", { name: "Element Call" }).click();
+
+  await expect(
+    brooks.page.locator('iframe[title="Element Call"]'),
+  ).toBeVisible();
+
+  const brooksFrame = brooks.page
+    .locator('iframe[title="Element Call"]')
+    .contentFrame();
+
+  // We should show a ringing overlay, let's check for that
+  await expect(
+    brooksFrame.getByText(`Waiting for ${whistler.displayName} to join…`),
+  ).toBeVisible();
+
+  await expect(whistler.page.getByText("Incoming voice call")).toBeVisible();
+  await whistler.page.getByRole("button", { name: "Accept" }).click();
+
+  await expect(
+    whistler.page.locator('iframe[title="Element Call"]'),
+  ).toBeVisible();
+
+  const whistlerFrame = whistler.page
+    .locator('iframe[title="Element Call"]')
+    .contentFrame();
+
+  // ASSERT the button states for whistler (the callee)
+  {
+    // The only way to know if it is muted or not is to look at the data-kind attribute..
+    const videoButton = whistlerFrame.getByTestId("incall_videomute");
+    // video should be off by default in a voice call
+    await expect(videoButton).toHaveAttribute("aria-label", /^Start video$/);
+
+    const audioButton = whistlerFrame.getByTestId("incall_mute");
+    // audio should be on for the voice call
+    await expect(audioButton).toHaveAttribute(
+      "aria-label",
+      /^Mute microphone$/,
+    );
+  }
+
+  // ASSERT the button states for brools (the caller)
+  {
+    // The only way to know if it is muted or not is to look at the data-kind attribute..
+    const videoButton = brooksFrame.getByTestId("incall_videomute");
+    // video should be off by default in a voice call
+    await expect(videoButton).toHaveAttribute("aria-label", /^Start video$/);
+
+    const audioButton = brooksFrame.getByTestId("incall_mute");
+    // audio should be on for the voice call
+    await expect(audioButton).toHaveAttribute(
+      "aria-label",
+      /^Mute microphone$/,
+    );
+  }
+
+  // In order to confirm that the call is disconnected we will check that the message composer is shown again.
+  // So first we need to confirm that it is hidden when in the call.
+  await expect(
+    whistler.page.locator(".mx_BasicMessageComposer"),
+  ).not.toBeVisible();
+  await expect(
+    brooks.page.locator(".mx_BasicMessageComposer"),
+  ).not.toBeVisible();
+
+  // ASSERT hanging up on one side ends the call for both
+  {
+    const hangupButton = brooksFrame.getByTestId("incall_leave");
+    await hangupButton.click();
+  }
+
+  // The widget should be closed on both sides and the timeline should be back on screen
+  await expect(whistler.page.locator(".mx_BasicMessageComposer")).toBeVisible();
+  await expect(brooks.page.locator(".mx_BasicMessageComposer")).toBeVisible();
+});
+
+widgetTest("Start a new video call in DM as widget", async ({ asWidget }) => {
+  test.slow(); // Triples the timeout
+
+  const { brooks, whistler } = asWidget;
+
+  await expect(
+    brooks.page.getByRole("button", { name: "Video call" }),
+  ).toBeVisible();
+  await brooks.page.getByRole("button", { name: "Video call" }).click();
+
+  await expect(
+    brooks.page.getByRole("menuitem", { name: "Element Call" }),
+  ).toBeVisible();
+
+  await brooks.page.getByRole("menuitem", { name: "Element Call" }).click();
+
+  await expect(
+    brooks.page.locator('iframe[title="Element Call"]'),
+  ).toBeVisible();
+
+  const brooksFrame = brooks.page
+    .locator('iframe[title="Element Call"]')
+    .contentFrame();
+
+  // We should show a ringing overlay, let's check for that
+  await expect(
+    brooksFrame.getByText(`Waiting for ${whistler.displayName} to join…`),
+  ).toBeVisible();
+
+  await expect(whistler.page.getByText("Incoming video call")).toBeVisible();
+  await whistler.page.getByRole("button", { name: "Accept" }).click();
+
+  await expect(
+    whistler.page.locator('iframe[title="Element Call"]'),
+  ).toBeVisible();
+
+  const whistlerFrame = whistler.page
+    .locator('iframe[title="Element Call"]')
+    .contentFrame();
+
+  // ASSERT the button states for whistler (the callee)
+  {
+    // The only way to know if it is muted or not is to look at the data-kind attribute..
+    const videoButton = whistlerFrame.getByTestId("incall_videomute");
+    // video should be on by default in a voice call
+    await expect(videoButton).toHaveAttribute("aria-label", /^Stop video$/);
+
+    const audioButton = whistlerFrame.getByTestId("incall_mute");
+    // audio should be on for the voice call
+    await expect(audioButton).toHaveAttribute(
+      "aria-label",
+      /^Mute microphone$/,
+    );
+  }
+
+  // ASSERT the button states for brools (the caller)
+  {
+    // The only way to know if it is muted or not is to look at the data-kind attribute..
+    const videoButton = brooksFrame.getByTestId("incall_videomute");
+    // video should be on by default in a voice call
+    await expect(videoButton).toHaveAttribute("aria-label", /^Stop video$/);
+
+    const audioButton = brooksFrame.getByTestId("incall_mute");
+    // audio should be on for the voice call
+    await expect(audioButton).toHaveAttribute(
+      "aria-label",
+      /^Mute microphone$/,
+    );
+  }
+
+  // In order to confirm that the call is disconnected we will check that the message composer is shown again.
+  // So first we need to confirm that it is hidden when in the call.
+  await expect(
+    whistler.page.locator(".mx_BasicMessageComposer"),
+  ).not.toBeVisible();
+  await expect(
+    brooks.page.locator(".mx_BasicMessageComposer"),
+  ).not.toBeVisible();
+
+  // ASSERT hanging up on one side ends the call for both
+  {
+    const hangupButton = brooksFrame.getByTestId("incall_leave");
+    await hangupButton.click();
+  }
+
+  // The widget should be closed on both sides and the timeline should be back on screen
+  await expect(whistler.page.locator(".mx_BasicMessageComposer")).toBeVisible();
+  await expect(brooks.page.locator(".mx_BasicMessageComposer")).toBeVisible();
+});

sdk/README.md

@@ -0,0 +1,35 @@
+# SDK mode
+
+EC can be build in sdk mode. This will result in a compiled js file that can be imported in very simple webapps.
+
+It allows to use matrixRTC in combination with livekit without relying on element call.
+
+This is done by instantiating the call view model and exposing some useful behaviors (observables) and methods.
+
+This folder contains an example index.html file that showcases the sdk in use (hosted on localhost:8123 with a webserver ellowing cors (for example `npx serve -l 81234 --cors`)) as a godot engine HTML export template.
+
+## Widgets
+
+The sdk mode is particularly interesting to be used in widgets where you do not need to pay attention to matrix login/cs api ...
+To create a widget see the example index.html file in this folder. And add it to EW via:
+`/addwidget <widgetUrl>` (see **url parameters** for more details on `<widgetUrl>`)
+
+### url parameters
+
+```
+widgetId = $matrix_widget_id
+perParticipantE2EE = true
+userId = $matrix_user_id
+deviceId = $org.matrix.msc3819.matrix_device_id
+baseUrl = $org.matrix.msc4039.matrix_base_url
+```
+
+`parentUrl = // will be inserted automatically`
+
+Full template use as `<widgetUrl>`:
+
+```
+http://localhost:3000?widgetId=$matrix_widget_id&perParticipantE2EE=true&userId=$matrix_user_id&deviceId=$org.matrix.msc3819.matrix_device_id&baseUrl=$org.matrix.msc4039.matrix_base_url&roomId=$matrix_room_id
+```
+
+the `$` prefixed variables will be replaced by EW on widget instantiation. (e.g. `$matrix_user_id` -> `@user:example.com` (url encoding will also be applied automatically by EW) -> `%40user%3Aexample.com`)

sdk/helper.ts

@@ -0,0 +1,55 @@
+/*
+Copyright 2025 Element Creations Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+/**
+ * This file contains helper functions and types for the MatrixRTC SDK.
+ */
+
+import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
+import { scan } from "rxjs";
+
+import { widget as _widget } from "../src/widget";
+import { type LivekitRoomItem } from "../src/state/CallViewModel/CallViewModel";
+
+export const logger = rootLogger.getChild("[MatrixRTCSdk]");
+
+if (!_widget) throw Error("No widget. This webapp can only start as a widget");
+export const widget = _widget;
+
+export const tryMakeSticky = (): void => {
+  logger.info("try making sticky MatrixRTCSdk");
+  void widget.api
+    .setAlwaysOnScreen(true)
+    .then(() => {
+      logger.info("sticky MatrixRTCSdk");
+    })
+    .catch((error) => {
+      logger.error("failed to make sticky MatrixRTCSdk", error);
+    });
+};
+export const TEXT_LK_TOPIC = "matrixRTC";
+/**
+ * simple helper operator to combine the last emitted and the current emitted value of a rxjs observable
+ *
+ * I think there should be a builtin for this but i did not find it...
+ */
+export const currentAndPrev = scan<
+  LivekitRoomItem[],
+  {
+    prev: LivekitRoomItem[];
+    current: LivekitRoomItem[];
+  }
+>(
+  ({ current: lastCurrentVal }, items) => ({
+    prev: lastCurrentVal,
+    current: items,
+  }),
+  {
+    prev: [],
+    current: [],
+  },
+);

sdk/index.html

@@ -0,0 +1,87 @@
+<!doctype html>
+<html>
+  <head>
+    <title>Godot MatrixRTC Widget</title>
+    <meta charset="utf-8" />
+    <script type="module">
+      // TODO use the url where the matrixrtc-sdk.js file from dist is hosted
+      import { createMatrixRTCSdk } from "http://localhost:8123/matrixrtc-sdk.js";
+
+      try {
+        window.matrixRTCSdk = await createMatrixRTCSdk(
+          "com.github.toger5.godot-game",
+        );
+        console.info("createMatrixRTCSdk was created!");
+      } catch (e) {
+        console.error("createMatrixRTCSdk", e);
+      }
+      const sdk = window.matrixRTCSdk;
+
+      // This is the main bridging interface to godot
+      window.matrixRTCSdkGodot = {
+        dataObs: sdk.data$,
+        memberObs: sdk.members$,
+        // join: sdk.join, // lets stick with autojoin for now
+        sendData: sdk.sendData,
+        leave: sdk.leave,
+        connectedObs: sdk.connected$,
+      };
+
+      console.info("matrixRTCSdk join ", sdk);
+      const connectionState = sdk.join();
+      console.info("matrixRTCSdk joined");
+
+      const div = document.getElementById("data");
+      div.innerHTML = "<h3>Data:</h3>";
+
+      sdk.data$.subscribe((data) => {
+        const child = document.createElement("p");
+        child.innerHTML = JSON.stringify(data);
+        div.appendChild(child);
+        // TODO forward to godot
+      });
+
+      sdk.members$.subscribe((memberObjects) => {
+        // reset div
+        const div = document.getElementById("members");
+        div.innerHTML = "<h3>Members:</h3>";
+
+        // create member list
+        const members = memberObjects.map((member) => member.membership.sender);
+        console.info("members changed", members);
+        for (const m of members) {
+          console.info("member", m);
+          const child = document.createElement("p");
+          child.innerHTML = m;
+          div.appendChild(child);
+        }
+      });
+
+      sdk.connected$.subscribe((connected) => {
+        console.info("connected changed", connected);
+        const div = document.getElementById("connect_status");
+        div.innerHTML = connected ? "Connected" : "Disconnected";
+      });
+
+      let engine = new Engine($GODOT_CONFIG);
+      engine.startGame();
+    </script>
+    <!--// TODO use it as godot HTML template-->
+    <script src="$GODOT_URL"></script>
+  </head>
+  <body>
+    <canvas id="canvas"></canvas>
+    <div
+      id="overlay"
+      style="position: absolute; top: 0; right: 0; background-color: #ffffff10"
+    >
+      <div id="connect_status"></div>
+      <button onclick="window.matrixRTCSdk.leave()">Leave</button>
+      <button onclick="window.matrixRTCSdk.sendData({ prop: 'Hello, world!' })">
+        Send Text
+      </button>
+      <div id="members"></div>
+      <div id="data"></div>
+    </div>
+  </body>
+</html>

sdk/main.ts

@@ -0,0 +1,311 @@
+/*
+Copyright 2025 Element Creations Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+/**
+ * This file is the entrypoint for the sdk build of element call: `yarn build:sdk`
+ * use in widgets.
+ * It exposes the `createMatrixRTCSdk` which creates the `MatrixRTCSdk` interface (see below) that
+ * can be used to join a rtc session and exchange realtime data.
+ * It takes care of all the tricky bits:
+ *  - sending delayed events
+ *  - finding the right sfu
+ *  - handling the media stream
+ *  - sending join/leave state or sticky events
+ *  - setting up encryption and scharing keys
+ */
+
+import {
+  combineLatest,
+  map,
+  type Observable,
+  of,
+  shareReplay,
+  Subject,
+  switchMap,
+  tap,
+} from "rxjs";
+import {
+  type CallMembership,
+  MatrixRTCSession,
+  MatrixRTCSessionEvent,
+} from "matrix-js-sdk/lib/matrixrtc";
+import {
+  type Room as LivekitRoom,
+  type TextStreamReader,
+  type LocalParticipant,
+  type RemoteParticipant,
+} from "livekit-client";
+
+// TODO how can this get fixed? to just be part of `livekit-client`
+// Can this be done in the tsconfig.json
+import { type TextStreamInfo } from "../node_modules/livekit-client/dist/src/room/types";
+import { type Behavior, constant } from "../src/state/Behavior";
+import { createCallViewModel$ } from "../src/state/CallViewModel/CallViewModel";
+import { ObservableScope } from "../src/state/ObservableScope";
+import { getUrlParams } from "../src/UrlParams";
+import { MuteStates } from "../src/state/MuteStates";
+import { MediaDevices } from "../src/state/MediaDevices";
+import { E2eeType } from "../src/e2ee/e2eeType";
+import {
+  currentAndPrev,
+  logger,
+  TEXT_LK_TOPIC,
+  tryMakeSticky,
+  widget,
+} from "./helper";
+import { ElementWidgetActions } from "../src/widget";
+import { type Connection } from "../src/state/CallViewModel/remoteMembers/Connection";
+
+interface MatrixRTCSdk {
+  /**
+   * observe connected$ to track the state.
+   * @returns
+   */
+  join: () => void;
+  /** @throws on leave errors */
+  leave: () => void;
+  data$: Observable<{ sender: string; data: string }>;
+  /**
+   * flattened list of members
+   */
+  members$: Behavior<
+    {
+      connection: Connection | null;
+      membership: CallMembership;
+      participant: LocalParticipant | RemoteParticipant | null;
+    }[]
+  >;
+  /** Use the LocalMemberConnectionState returned from `join` for a more detailed connection state  */
+  connected$: Behavior<boolean>;
+  sendData?: (data: unknown) => Promise<void>;
+}
+
+export async function createMatrixRTCSdk(
+  application: string = "m.call",
+  id: string = "",
+): Promise<MatrixRTCSdk> {
+  logger.info("Hello");
+  const client = await widget.client;
+  logger.info("client created");
+  const scope = new ObservableScope();
+  const { roomId } = getUrlParams();
+  if (roomId === null) throw Error("could not get roomId from url params");
+
+  const room = client.getRoom(roomId);
+  if (room === null) throw Error("could not get room from client");
+
+  const mediaDevices = new MediaDevices(scope);
+  const muteStates = new MuteStates(scope, mediaDevices, {
+    audioEnabled: true,
+    videoEnabled: true,
+  });
+  const slot = { application, id };
+  const rtcSession = new MatrixRTCSession(
+    client,
+    room,
+    MatrixRTCSession.sessionMembershipsForSlot(room, slot),
+    slot,
+  );
+  const callViewModel = createCallViewModel$(
+    scope,
+    rtcSession,
+    room,
+    mediaDevices,
+    muteStates,
+    { encryptionSystem: { kind: E2eeType.PER_PARTICIPANT } },
+    of({}),
+    of({}),
+    constant({ supported: false, processor: undefined }),
+  );
+  logger.info("CallViewModelCreated");
+  // create data listener
+  const data$ = new Subject<{ sender: string; data: string }>();
+
+  const lkTextStreamHandlerFunction = async (
+    reader: TextStreamReader,
+    participantInfo: { identity: string },
+    livekitRoom: LivekitRoom,
+  ): Promise<void> => {
+    const info = reader.info;
+    logger.info(
+      `Received text stream from ${participantInfo.identity}\n` +
+        `  Topic: ${info.topic}\n` +
+        `  Timestamp: ${info.timestamp}\n` +
+        `  ID: ${info.id}\n` +
+        `  Size: ${info.size}`, // Optional, only available if the stream was sent with `sendText`
+    );
+
+    const participants = callViewModel.livekitRoomItems$.value.find(
+      (i) => i.livekitRoom === livekitRoom,
+    )?.participants;
+    if (participants && participants.includes(participantInfo.identity)) {
+      const text = await reader.readAll();
+      logger.info(`Received text: ${text}`);
+      data$.next({ sender: participantInfo.identity, data: text });
+    } else {
+      logger.warn(
+        "Received text from unknown participant",
+        participantInfo.identity,
+      );
+    }
+  };
+
+  const livekitRoomItemsSub = callViewModel.livekitRoomItems$
+    .pipe(
+      tap((beforecurrentAndPrev) => {
+        logger.info(
+          `LiveKit room items updated: ${beforecurrentAndPrev.length}`,
+          beforecurrentAndPrev,
+        );
+      }),
+      currentAndPrev,
+      tap((aftercurrentAndPrev) => {
+        logger.info(
+          `LiveKit room items updated: ${aftercurrentAndPrev.current.length}, ${aftercurrentAndPrev.prev.length}`,
+          aftercurrentAndPrev,
+        );
+      }),
+    )
+    .subscribe({
+      next: ({ prev, current }) => {
+        const prevRooms = prev.map((i) => i.livekitRoom);
+        const currentRooms = current.map((i) => i.livekitRoom);
+        const addedRooms = currentRooms.filter((r) => !prevRooms.includes(r));
+        const removedRooms = prevRooms.filter((r) => !currentRooms.includes(r));
+        addedRooms.forEach((r) => {
+          logger.info(`Registering text stream handler for room `);
+          r.registerTextStreamHandler(
+            TEXT_LK_TOPIC,
+            (reader, participantInfo) =>
+              void lkTextStreamHandlerFunction(reader, participantInfo, r),
+          );
+        });
+        removedRooms.forEach((r) => {
+          logger.info(`Unregistering text stream handler for room `);
+          r.unregisterTextStreamHandler(TEXT_LK_TOPIC);
+        });
+      },
+      complete: () => {
+        logger.info("Livekit room items subscription completed");
+        for (const item of callViewModel.livekitRoomItems$.value) {
+          logger.info("unregistering room item from room", item.url);
+          item.livekitRoom.unregisterTextStreamHandler(TEXT_LK_TOPIC);
+        }
+      },
+    });
+
+  // create sendData function
+  const sendFn: Behavior<(data: string) => Promise<TextStreamInfo>> =
+    scope.behavior(
+      callViewModel.localMatrixLivekitMember$.pipe(
+        switchMap((m) => {
+          if (!m)
+            return of((data: string): never => {
+              throw Error("local membership not yet ready.");
+            });
+          return m.participant.value$.pipe(
+            map((p) => {
+              if (p === null) {
+                return (data: string): never => {
+                  throw Error("local participant not yet ready to send data.");
+                };
+              } else {
+                return async (data: string): Promise<TextStreamInfo> =>
+                  p.sendText(data, { topic: TEXT_LK_TOPIC });
+              }
+            }),
+          );
+        }),
+      ),
+    );
+
+  const sendData = async (data: unknown): Promise<void> => {
+    const dataString = JSON.stringify(data);
+    logger.info("try sending: ", dataString);
+    try {
+      await Promise.resolve();
+      const info = await sendFn.value(dataString);
+      logger.info(`Sent text with stream ID: ${info.id}`);
+    } catch (e) {
+      logger.error("failed sending: ", dataString, e);
+    }
+  };
+
+  // after hangup gets called
+  const leaveSubs = callViewModel.leave$.subscribe(() => {
+    const scheduleWidgetCloseOnLeave = async (): Promise<void> => {
+      const leaveResolver = Promise.withResolvers<void>();
+      logger.info("waiting for RTC leave");
+      rtcSession.on(MatrixRTCSessionEvent.JoinStateChanged, (isJoined) => {
+        logger.info("received RTC join update: ", isJoined);
+        if (!isJoined) leaveResolver.resolve();
+      });
+      await leaveResolver.promise;
+      logger.info("send Unstick");
+      await widget.api
+        .setAlwaysOnScreen(false)
+        .catch((e) =>
+          logger.error(
+            "Failed to set call widget `alwaysOnScreen` to false",
+            e,
+          ),
+        );
+      logger.info("send Close");
+      await widget.api.transport
+        .send(ElementWidgetActions.Close, {})
+        .catch((e) => logger.error("Failed to send close action", e));
+    };
+
+    // schedule close first and then leave (scope.end)
+    void scheduleWidgetCloseOnLeave();
+
+    // actual hangup (ending scope will send the leave event.. its kinda odd. since you might end up closing the widget too fast)
+    scope.end();
+  });
+
+  logger.info("createMatrixRTCSdk done");
+
+  return {
+    join: (): void => {
+      // first lets try making the widget sticky
+      tryMakeSticky();
+      callViewModel.join();
+    },
+    leave: (): void => {
+      callViewModel.hangup();
+      leaveSubs.unsubscribe();
+      livekitRoomItemsSub.unsubscribe();
+    },
+    data$,
+    connected$: callViewModel.connected$,
+    members$: scope.behavior(
+      callViewModel.matrixLivekitMembers$.pipe(
+        switchMap((members) => {
+          const listOfMemberObservables = members.map((member) =>
+            combineLatest([
+              member.connection$,
+              member.membership$,
+              member.participant.value$,
+            ]).pipe(
+              map(([connection, membership, participant]) => ({
+                connection,
+                membership,
+                participant,
+              })),
+              // using shareReplay instead of a Behavior here because the behavior would need
+              // a tricky scope.end() setup.
+              shareReplay({ bufferSize: 1, refCount: true }),
+            ),
+          );
+          return combineLatest(listOfMemberObservables);
+        }),
+      ),
+      [],
+    ),
+    sendData,
+  };
+}

src/@types/dom-mediacapture-transform.d.ts

@@ -0,0 +1,147 @@
+/* eslint-disable */
+// The contents of this file below the line are copied from
+// @types/dom-mediacapture-transform, which is inlined here into Element Call so
+// that we can avoid the package's dependency on @types/dom-webcodecs, which is
+// broken in TypeScript 5.9.
+// (https://github.com/DefinitelyTyped/DefinitelyTyped/discussions/74294)
+// If that issue is resolved, we can remove this file and return to depending on
+// @types/dom-mediacapture-transform.
+// -----------------------------------------------------------------------------
+
+// This project is licensed under the MIT license.
+// Copyrights are respective of each contributor listed at the beginning of each definition file.
+
+// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+// The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+// In general, these types are only available behind a command line flag or an origin trial in
+// Chrome 90+.
+
+// This API depends on WebCodecs.
+
+// Versioning:
+// Until the above-mentioned spec is finalized, the major version number is 0. Although not
+// necessary for version 0, consider incrementing the minor version number for breaking changes.
+
+// The following modify existing DOM types to allow defining type-safe APIs on audio and video tracks.
+
+/** Specialize MediaStreamTrack so that we can refer specifically to an audio track. */
+interface MediaStreamAudioTrack extends MediaStreamTrack {
+  readonly kind: "audio";
+  clone(): MediaStreamAudioTrack;
+}
+
+/** Specialize MediaStreamTrack so that we can refer specifically to a video track. */
+interface MediaStreamVideoTrack extends MediaStreamTrack {
+  readonly kind: "video";
+  clone(): MediaStreamVideoTrack;
+}
+
+/** Assert that getAudioTracks and getVideoTracks return the tracks with the appropriate kind. */
+interface MediaStream {
+  getAudioTracks(): MediaStreamAudioTrack[];
+  getVideoTracks(): MediaStreamVideoTrack[];
+}
+
+// The following were originally generated from the spec using
+// https://github.com/microsoft/TypeScript-DOM-lib-generator, then heavily modified.
+
+/**
+ * A track sink that is capable of exposing the unencoded frames from the track to a
+ * ReadableStream, and exposes a control channel for signals going in the oppposite direction.
+ */
+interface MediaStreamTrackProcessor<T extends AudioData | VideoFrame> {
+  /**
+   * Allows reading the frames flowing through the MediaStreamTrack provided to the constructor.
+   */
+  readonly readable: ReadableStream<T>;
+  /** Allows sending control signals to the MediaStreamTrack provided to the constructor. */
+  readonly writableControl: WritableStream<MediaStreamTrackSignal>;
+}
+
+declare var MediaStreamTrackProcessor: {
+  prototype: MediaStreamTrackProcessor<any>;
+
+  /** Constructor overrides based on the type of track. */
+  new (
+    init: MediaStreamTrackProcessorInit & { track: MediaStreamAudioTrack },
+  ): MediaStreamTrackProcessor<AudioData>;
+  new (
+    init: MediaStreamTrackProcessorInit & { track: MediaStreamVideoTrack },
+  ): MediaStreamTrackProcessor<VideoFrame>;
+};
+
+interface MediaStreamTrackProcessorInit {
+  track: MediaStreamTrack;
+  /**
+   * If media frames are not read from MediaStreamTrackProcessor.readable quickly enough, the
+   * MediaStreamTrackProcessor will internally buffer up to maxBufferSize of the frames produced
+   * by the track. If the internal buffer is full, each time the track produces a new frame, the
+   * oldest frame in the buffer will be dropped and the new frame will be added to the buffer.
+   */
+  maxBufferSize?: number | undefined;
+}
+
+/**
+ * Takes video frames as input, and emits control signals that result from subsequent processing.
+ */
+interface MediaStreamTrackGenerator<
+  T extends AudioData | VideoFrame,
+> extends MediaStreamTrack {
+  /**
+   * Allows writing media frames to the MediaStreamTrackGenerator, which is itself a
+   * MediaStreamTrack. When a frame is written to writable, the frame’s close() method is
+   * automatically invoked, so that its internal resources are no longer accessible from
+   * JavaScript.
+   */
+  readonly writable: WritableStream<T>;
+  /**
+   * Allows reading control signals sent from any sinks connected to the
+   * MediaStreamTrackGenerator.
+   */
+  readonly readableControl: ReadableStream<MediaStreamTrackSignal>;
+}
+
+type MediaStreamAudioTrackGenerator = MediaStreamTrackGenerator<AudioData> &
+  MediaStreamAudioTrack;
+type MediaStreamVideoTrackGenerator = MediaStreamTrackGenerator<VideoFrame> &
+  MediaStreamVideoTrack;
+
+declare var MediaStreamTrackGenerator: {
+  prototype: MediaStreamTrackGenerator<any>;
+
+  /** Constructor overrides based on the type of track. */
+  new (
+    init: MediaStreamTrackGeneratorInit & {
+      kind: "audio";
+      signalTarget?: MediaStreamAudioTrack | undefined;
+    },
+  ): MediaStreamAudioTrackGenerator;
+  new (
+    init: MediaStreamTrackGeneratorInit & {
+      kind: "video";
+      signalTarget?: MediaStreamVideoTrack | undefined;
+    },
+  ): MediaStreamVideoTrackGenerator;
+};
+
+interface MediaStreamTrackGeneratorInit {
+  kind: MediaStreamTrackGeneratorKind;
+  /**
+   * (Optional) track to which the MediaStreamTrackGenerator will automatically forward control
+   * signals. If signalTarget is provided and signalTarget.kind and kind do not match, the
+   * MediaStreamTrackGenerator’s constructor will raise an exception.
+   */
+  signalTarget?: MediaStreamTrack | undefined;
+}
+
+type MediaStreamTrackGeneratorKind = "audio" | "video";
+
+type MediaStreamTrackSignalType = "request-frame";
+
+interface MediaStreamTrackSignal {
+  signalType: MediaStreamTrackSignalType;
+}

src/UrlParams.test.ts

@@ -256,8 +256,6 @@ describe("UrlParams", () => {
       skipLobby: false,
       returnToLobby: false,
       sendNotificationType: "notification",
-      defaultAudioEnabled: true,
-      defaultVideoEnabled: true,
     });
     it("use no-intent-defaults with unknown intent", () => {
       expect(computeUrlParams()).toMatchObject(noIntentDefaults);
@@ -395,8 +393,6 @@ describe("UrlParams", () => {
         expect.any(Object),
         "configuration:",
         expect.any(Object),
-        "intentAndPlatformDerivedConfiguration:",
-        {},
       );
     });
   });

src/UrlParams.ts

@@ -246,24 +246,13 @@ export interface UrlConfiguration {
 
   callIntent?: RTCCallIntent;
 }
-interface IntentAndPlatformDerivedConfiguration {
-  defaultAudioEnabled?: boolean;
-  defaultVideoEnabled?: boolean;
-}
-interface IntentAndPlatformDerivedConfiguration {
-  defaultAudioEnabled?: boolean;
-  defaultVideoEnabled?: boolean;
-}
 
 // If you need to add a new flag to this interface, prefer a name that describes
 // a specific behavior (such as 'confineToRoom'), rather than one that describes
 // the situations that call for this behavior ('isEmbedded'). This makes it
 // clearer what each flag means, and helps us avoid coupling Element Call's
 // behavior to the needs of specific consumers.
-export interface UrlParams
-  extends UrlProperties,
-    UrlConfiguration,
-    IntentAndPlatformDerivedConfiguration {}
+export interface UrlParams extends UrlProperties, UrlConfiguration {}
 
 // This is here as a stopgap, but what would be far nicer is a function that
 // takes a UrlParams and returns a query string. That would enable us to
@@ -463,29 +452,6 @@ export const computeUrlParams = (search = "", hash = ""): UrlParams => {
       };
   }
 
-  const intentAndPlatformDerivedConfiguration: IntentAndPlatformDerivedConfiguration =
-    {};
-  // Desktop also includes web. Its anything that is not mobile.
-  const desktopMobile = platform === "desktop" ? "desktop" : "mobile";
-  switch (desktopMobile) {
-    case "desktop":
-    case "mobile":
-      switch (intent) {
-        case UserIntent.StartNewCall:
-        case UserIntent.JoinExistingCall:
-        case UserIntent.StartNewCallDM:
-        case UserIntent.JoinExistingCallDM:
-          intentAndPlatformDerivedConfiguration.defaultAudioEnabled = true;
-          intentAndPlatformDerivedConfiguration.defaultVideoEnabled = true;
-          break;
-        case UserIntent.StartNewCallDMVoice:
-        case UserIntent.JoinExistingCallDMVoice:
-          intentAndPlatformDerivedConfiguration.defaultAudioEnabled = true;
-          intentAndPlatformDerivedConfiguration.defaultVideoEnabled = false;
-          break;
-      }
-  }
-
   const properties: UrlProperties = {
     widgetId,
     parentUrl,
@@ -550,15 +516,12 @@ export const computeUrlParams = (search = "", hash = ""): UrlParams => {
     properties,
     "configuration:",
     configuration,
-    "intentAndPlatformDerivedConfiguration:",
-    intentAndPlatformDerivedConfiguration,
   );
 
   return {
     ...properties,
     ...intentPreset,
     ...pickBy(configuration, (v?: unknown) => v !== undefined),
-    ...intentAndPlatformDerivedConfiguration,
   };
 };
 

src/__snapshots__/AppBar.test.tsx.snap

@@ -13,7 +13,7 @@ exports[`AppBar > renders 1`] = `
         class="nav leftNav"
       >
         <button
-          aria-labelledby="«r0»"
+          aria-labelledby="_r_0_"
           class="_icon-button_1pz9o_8"
           data-kind="primary"
           role="button"

src/__snapshots__/Modal.test.tsx.snap

@@ -2,10 +2,10 @@
 
 exports[`the content is rendered when the modal is open 1`] = `
 <div
-  aria-labelledby="radix-«r4»"
+  aria-labelledby="radix-_r_4_"
   class="overlay animate modal dialog _glass_sepwu_8"
   data-state="open"
-  id="radix-«r3»"
+  id="radix-_r_3_"
   role="dialog"
   style="pointer-events: auto;"
   tabindex="-1"
@@ -18,7 +18,7 @@ exports[`the content is rendered when the modal is open 1`] = `
     >
       <h2
         class="_typography_6v6n8_153 _font-heading-md-semibold_6v6n8_112"
-        id="radix-«r4»"
+        id="radix-_r_4_"
       >
         My modal
       </h2>
@@ -36,7 +36,7 @@ exports[`the content is rendered when the modal is open 1`] = `
 
 exports[`the modal renders as a drawer in mobile viewports 1`] = `
 <div
-  aria-labelledby="radix-«ra»"
+  aria-labelledby="radix-_r_a_"
   class="overlay modal drawer"
   data-state="open"
   data-vaul-animate="true"
@@ -45,7 +45,7 @@ exports[`the modal renders as a drawer in mobile viewports 1`] = `
   data-vaul-drawer=""
   data-vaul-drawer-direction="bottom"
   data-vaul-snap-points="false"
-  id="radix-«r9»"
+  id="radix-_r_9_"
   role="dialog"
   style="pointer-events: auto;"
   tabindex="-1"
@@ -60,7 +60,7 @@ exports[`the modal renders as a drawer in mobile viewports 1`] = `
         class="handle"
       />
       <h2
-        id="radix-«ra»"
+        id="radix-_r_a_"
         style="position: absolute; border: 0px; width: 1px; height: 1px; padding: 0px; margin: -1px; overflow: hidden; clip: rect(0px, 0px, 0px, 0px); white-space: nowrap; word-wrap: normal;"
       >
         My modal

src/__snapshots__/Toast.test.tsx.snap

@@ -2,10 +2,10 @@
 
 exports[`Toast > renders 1`] = `
 <button
-  aria-labelledby="radix-«r4»"
+  aria-labelledby="radix-_r_4_"
   class="overlay animate toast"
   data-state="open"
-  id="radix-«r3»"
+  id="radix-_r_3_"
   role="dialog"
   style="pointer-events: auto;"
   tabindex="-1"
@@ -13,7 +13,7 @@ exports[`Toast > renders 1`] = `
 >
   <h3
     class="_typography_6v6n8_153 _font-body-sm-semibold_6v6n8_36"
-    id="radix-«r4»"
+    id="radix-_r_4_"
   >
     Hello world!
   </h3>

src/analytics/PosthogSpanProcessor.ts

@@ -1,157 +0,0 @@
-/*
-Copyright 2023, 2024 New Vector Ltd.
-
-SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
-Please see LICENSE in the repository root for full details.
-*/
-
-import {
-  type SpanProcessor,
-  type ReadableSpan,
-  type Span,
-} from "@opentelemetry/sdk-trace-base";
-import { hrTimeToMilliseconds } from "@opentelemetry/core";
-import { logger } from "matrix-js-sdk/lib/logger";
-
-import { PosthogAnalytics } from "./PosthogAnalytics";
-
-interface PrevCall {
-  callId: string;
-  hangupTs: number;
-}
-
-/**
- * The maximum time between hanging up and joining the same call that we would
- * consider a 'rejoin' on the user's part.
- */
-const maxRejoinMs = 2 * 60 * 1000; // 2 minutes
-
-/**
- * Span processor that extracts certain metrics from spans to send to PostHog
- */
-export class PosthogSpanProcessor implements SpanProcessor {
-  public async forceFlush(): Promise<void> {}
-
-  public onStart(span: Span): void {
-    // Hack: Yield to allow attributes to be set before processing
-    try {
-      switch (span.name) {
-        case "matrix.groupCallMembership":
-          this.onGroupCallMembershipStart(span);
-          return;
-        case "matrix.groupCallMembership.summaryReport":
-          this.onSummaryReportStart(span);
-          return;
-      }
-    } catch (e) {
-      // log to avoid tripping @typescript-eslint/no-unused-vars
-      logger.debug(e);
-    }
-  }
-
-  public onEnd(span: ReadableSpan): void {
-    switch (span.name) {
-      case "matrix.groupCallMembership":
-        this.onGroupCallMembershipEnd(span);
-        return;
-    }
-  }
-
-  private get prevCall(): PrevCall | null {
-    // This is stored in localStorage so we can remember the previous call
-    // across app restarts
-    const data = localStorage.getItem("matrix-prev-call");
-    if (data === null) return null;
-
-    try {
-      return JSON.parse(data);
-    } catch (e) {
-      logger.warn("Invalid prev call data", data, "error:", e);
-      return null;
-    }
-  }
-
-  private set prevCall(data: PrevCall | null) {
-    localStorage.setItem("matrix-prev-call", JSON.stringify(data));
-  }
-
-  private onGroupCallMembershipStart(span: ReadableSpan): void {
-    const prevCall = this.prevCall;
-    const newCallId = span.attributes["matrix.confId"] as string;
-
-    // If the user joined the same call within a short time frame, log this as a
-    // rejoin. This is interesting as a call quality metric, since rejoins may
-    // indicate that users had to intervene to make the product work.
-    if (prevCall !== null && newCallId === prevCall.callId) {
-      const duration = hrTimeToMilliseconds(span.startTime) - prevCall.hangupTs;
-      if (duration <= maxRejoinMs) {
-        PosthogAnalytics.instance.trackEvent({
-          eventName: "Rejoin",
-          callId: prevCall.callId,
-          rejoinDuration: duration,
-        });
-      }
-    }
-  }
-
-  private onGroupCallMembershipEnd(span: ReadableSpan): void {
-    this.prevCall = {
-      callId: span.attributes["matrix.confId"] as string,
-      hangupTs: hrTimeToMilliseconds(span.endTime),
-    };
-  }
-
-  private onSummaryReportStart(span: ReadableSpan): void {
-    // Searching for an event like this:
-    //    matrix.stats.summary
-    //    matrix.stats.summary.percentageReceivedAudioMedia: 0.75
-    //    matrix.stats.summary.percentageReceivedMedia: 1
-    //    matrix.stats.summary.percentageReceivedVideoMedia: 0.75
-    //    matrix.stats.summary.maxJitter: 100
-    //    matrix.stats.summary.maxPacketLoss: 20
-    const event = span.events.find((e) => e.name === "matrix.stats.summary");
-    if (event !== undefined) {
-      const attributes = event.attributes;
-      if (attributes) {
-        const mediaReceived = `${attributes["matrix.stats.summary.percentageReceivedMedia"]}`;
-        const videoReceived = `${attributes["matrix.stats.summary.percentageReceivedVideoMedia"]}`;
-        const audioReceived = `${attributes["matrix.stats.summary.percentageReceivedAudioMedia"]}`;
-        const maxJitter = `${attributes["matrix.stats.summary.maxJitter"]}`;
-        const maxPacketLoss = `${attributes["matrix.stats.summary.maxPacketLoss"]}`;
-        const peerConnections = `${attributes["matrix.stats.summary.peerConnections"]}`;
-        const percentageConcealedAudio = `${attributes["matrix.stats.summary.percentageConcealedAudio"]}`;
-        const opponentUsersInCall = `${attributes["matrix.stats.summary.opponentUsersInCall"]}`;
-        const opponentDevicesInCall = `${attributes["matrix.stats.summary.opponentDevicesInCall"]}`;
-        const diffDevicesToPeerConnections = `${attributes["matrix.stats.summary.diffDevicesToPeerConnections"]}`;
-        const ratioPeerConnectionToDevices = `${attributes["matrix.stats.summary.ratioPeerConnectionToDevices"]}`;
-
-        PosthogAnalytics.instance.trackEvent(
-          {
-            eventName: "MediaReceived",
-            callId: span.attributes["matrix.confId"] as string,
-            mediaReceived: mediaReceived,
-            audioReceived: audioReceived,
-            videoReceived: videoReceived,
-            maxJitter: maxJitter,
-            maxPacketLoss: maxPacketLoss,
-            peerConnections: peerConnections,
-            percentageConcealedAudio: percentageConcealedAudio,
-            opponentUsersInCall: opponentUsersInCall,
-            opponentDevicesInCall: opponentDevicesInCall,
-            diffDevicesToPeerConnections: diffDevicesToPeerConnections,
-            ratioPeerConnectionToDevices: ratioPeerConnectionToDevices,
-          },
-          // Send instantly because the window might be closing
-          { send_instantly: true },
-        );
-      }
-    }
-  }
-
-  /**
-   * Shutdown the processor.
-   */
-  public async shutdown(): Promise<void> {
-    return Promise.resolve();
-  }
-}

src/analytics/RageshakeSpanProcessor.ts

@@ -1,135 +0,0 @@
-/*
-Copyright 2023, 2024 New Vector Ltd.
-
-SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
-Please see LICENSE in the repository root for full details.
-*/
-
-import { type AttributeValue, type Attributes } from "@opentelemetry/api";
-import { hrTimeToMicroseconds } from "@opentelemetry/core";
-import {
-  type SpanProcessor,
-  type ReadableSpan,
-  type Span,
-} from "@opentelemetry/sdk-trace-base";
-
-const dumpAttributes = (
-  attr: Attributes,
-): {
-  key: string;
-  type:
-    | "string"
-    | "number"
-    | "bigint"
-    | "boolean"
-    | "symbol"
-    | "undefined"
-    | "object"
-    | "function";
-  value: AttributeValue | undefined;
-}[] =>
-  Object.entries(attr).map(([key, value]) => ({
-    key,
-    type: typeof value,
-    value,
-  }));
-
-/**
- * Exports spans on demand to the Jaeger JSON format, which can be attached to
- * rageshakes and loaded into analysis tools like Jaeger and Stalk.
- */
-export class RageshakeSpanProcessor implements SpanProcessor {
-  private readonly spans: ReadableSpan[] = [];
-
-  public async forceFlush(): Promise<void> {}
-
-  public onStart(span: Span): void {
-    this.spans.push(span);
-  }
-
-  public onEnd(): void {}
-
-  /**
-   * Dumps the spans collected so far as Jaeger-compatible JSON.
-   */
-  public dump(): string {
-    const now = Date.now() * 1000; // Jaeger works in microseconds
-    const traces = new Map<string, ReadableSpan[]>();
-
-    // Organize spans by their trace IDs
-    for (const span of this.spans) {
-      const traceId = span.spanContext().traceId;
-      let trace = traces.get(traceId);
-
-      if (trace === undefined) {
-        trace = [];
-        traces.set(traceId, trace);
-      }
-
-      trace.push(span);
-    }
-
-    const processId = "p1";
-    const processes = {
-      [processId]: {
-        serviceName: "element-call",
-        tags: [],
-      },
-      warnings: null,
-    };
-
-    return JSON.stringify({
-      // Honestly not sure what some of these fields mean, I just know that
-      // they're present in Jaeger JSON exports
-      total: 0,
-      limit: 0,
-      offset: 0,
-      errors: null,
-      data: [...traces.entries()].map(([traceId, spans]) => ({
-        traceID: traceId,
-        warnings: null,
-        processes,
-        spans: spans.map((span) => {
-          const ctx = span.spanContext();
-          const startTime = hrTimeToMicroseconds(span.startTime);
-          // If the span has not yet ended, pretend that it ends now
-          const duration =
-            span.duration[0] === -1
-              ? now - startTime
-              : hrTimeToMicroseconds(span.duration);
-
-          return {
-            traceID: traceId,
-            spanID: ctx.spanId,
-            operationName: span.name,
-            processID: processId,
-            warnings: null,
-            startTime,
-            duration,
-            references:
-              span.parentSpanContext?.spanId === undefined
-                ? []
-                : [
-                    {
-                      refType: "CHILD_OF",
-                      traceID: traceId,
-                      spanID: span.parentSpanContext?.spanId,
-                    },
-                  ],
-            tags: dumpAttributes(span.attributes),
-            logs: span.events.map((event) => ({
-              timestamp: hrTimeToMicroseconds(event.time),
-              // The name of the event is in the "event" field, aparently.
-              fields: [
-                ...dumpAttributes(event.attributes ?? {}),
-                { key: "event", type: "string", value: event.name },
-              ],
-            })),
-          };
-        }),
-      })),
-    });
-  }
-
-  public async shutdown(): Promise<void> {}
-}

src/button/Button.tsx

@@ -35,6 +35,7 @@ export const MicButton: FC<MicButtonProps> = ({ muted, ...props }) => {
     <Tooltip label={label}>
       <CpdButton
         iconOnly
+        aria-label={label}
         Icon={Icon}
         kind={muted ? "primary" : "secondary"}
         {...props}
@@ -58,6 +59,7 @@ export const VideoButton: FC<VideoButtonProps> = ({ muted, ...props }) => {
     <Tooltip label={label}>
       <CpdButton
         iconOnly
+        aria-label={label}
         Icon={Icon}
         kind={muted ? "primary" : "secondary"}
         {...props}
@@ -102,6 +104,7 @@ export const EndCallButton: FC<ComponentPropsWithoutRef<"button">> = ({
       <CpdButton
         className={classNames(className, styles.endCall)}
         iconOnly
+        aria-label={t("hangup_button_label")}
         Icon={EndCallIcon}
         destructive
         {...props}

src/button/__snapshots__/ReactionToggleButton.test.tsx.snap

@@ -9,7 +9,7 @@ exports[`Can close reaction dialog 1`] = `
     aria-disabled="false"
     aria-expanded="true"
     aria-haspopup="true"
-    aria-labelledby="«rbb»"
+    aria-labelledby="_r_bb_"
     class="_button_vczzf_8 _has-icon_vczzf_57 _icon-only_vczzf_50"
     data-kind="primary"
     data-size="lg"
@@ -43,7 +43,7 @@ exports[`Can fully expand emoji picker 1`] = `
     aria-disabled="false"
     aria-expanded="true"
     aria-haspopup="true"
-    aria-labelledby="«r7m»"
+    aria-labelledby="_r_7m_"
     class="_button_vczzf_8 _has-icon_vczzf_57 _icon-only_vczzf_50"
     data-kind="primary"
     data-size="lg"
@@ -74,7 +74,7 @@ exports[`Can lower hand 1`] = `
     aria-disabled="false"
     aria-expanded="false"
     aria-haspopup="true"
-    aria-labelledby="«r36»"
+    aria-labelledby="_r_36_"
     class="_button_vczzf_8 _has-icon_vczzf_57 _icon-only_vczzf_50"
     data-kind="secondary"
     data-size="lg"
@@ -108,7 +108,7 @@ exports[`Can open menu 1`] = `
     aria-disabled="false"
     aria-expanded="true"
     aria-haspopup="true"
-    aria-labelledby="«r0»"
+    aria-labelledby="_r_0_"
     class="_button_vczzf_8 _has-icon_vczzf_57 _icon-only_vczzf_50"
     data-kind="primary"
     data-size="lg"
@@ -139,7 +139,7 @@ exports[`Can raise hand 1`] = `
     aria-disabled="false"
     aria-expanded="false"
     aria-haspopup="true"
-    aria-labelledby="«r1j»"
+    aria-labelledby="_r_1j_"
     class="_button_vczzf_8 raisedButton _has-icon_vczzf_57 _icon-only_vczzf_50"
     data-kind="primary"
     data-size="lg"

src/config/ConfigOptions.ts

@@ -163,11 +163,10 @@ export interface ResolvedConfigOptions extends ConfigOptions {
     };
   };
   ssla: string;
-  media_devices: {
-    enable_audio: boolean;
-    enable_video: boolean;
-  };
   app_prompt: boolean;
+  // For privacy reasons when lobby is skipped we do not unmute by default on full builds
+  // but for development builds we want to unmute by default to speed up testing.
+  trust_localhost_for_mute_state: boolean;
 }
 
 export const DEFAULT_CONFIG: ResolvedConfigOptions = {
@@ -181,9 +180,6 @@ export const DEFAULT_CONFIG: ResolvedConfigOptions = {
     feature_use_device_session_member_events: true,
   },
   ssla: "https://static.element.io/legal/element-software-and-services-license-agreement-uk-1.pdf",
-  media_devices: {
-    enable_audio: true,
-    enable_video: true,
-  },
   app_prompt: true,
+  trust_localhost_for_mute_state: false,
 };

src/e2ee/matrixKeyProvider.ts

@@ -40,7 +40,7 @@ export class MatrixKeyProvider extends BaseKeyProvider {
   }
 
   private onEncryptionKeyChanged = (
-    encryptionKey: Uint8Array,
+    encryptionKey: Uint8Array<ArrayBuffer>,
     encryptionKeyIndex: number,
     participantId: string,
   ): void => {

src/e2ee/sharedKeyManagement.ts

@@ -34,8 +34,8 @@ const getRoomSharedKeyLocalStorageKey = (roomId: string): string =>
   `room-shared-key-${roomId}`;
 
 /**
- * An upto-date shared key for the room. Either from local storage or the value from `setInitialValue`.
- * @param roomId
+ * An up-to-date shared key for the room. Either from local storage or the value from `setInitialValue`.
+ * @param roomId The room ID we want the shared key for.
  * @param setInitialValue The value we get from the URL. The hook will overwrite the local storage value with this.
  * @returns [roomSharedKey, setRoomSharedKey] like a react useState hook.
  */

src/grid/Grid.tsx

@@ -200,8 +200,11 @@ interface Drag {
 
 export type DragCallback = (drag: Drag) => void;
 
-interface LayoutMemoProps<LayoutModel, TileModel, R extends HTMLElement>
-  extends LayoutProps<LayoutModel, TileModel, R> {
+interface LayoutMemoProps<
+  LayoutModel,
+  TileModel,
+  R extends HTMLElement,
+> extends LayoutProps<LayoutModel, TileModel, R> {
   Layout: ComponentType<LayoutProps<LayoutModel, TileModel, R>>;
 }
 

src/initializer.tsx

@@ -16,7 +16,7 @@ import LanguageDetector from "i18next-browser-languagedetector";
 import * as Sentry from "@sentry/react";
 import { logger } from "matrix-js-sdk/lib/logger";
 import { shouldPolyfill as shouldPolyfillSegmenter } from "@formatjs/intl-segmenter/should-polyfill";
-import { shouldPolyfill as shouldPolyfillDurationFormat } from "@formatjs/intl-durationformat/should-polyfill";
+import { shouldPolyfill as shouldPolyfillDurationFormat } from "@formatjs/intl-durationformat/should-polyfill.js";
 import {
   useLocation,
   useNavigationType,
@@ -26,7 +26,6 @@ import {
 
 import { getUrlParams } from "./UrlParams";
 import { Config } from "./config/Config";
-import { ElementCallOpenTelemetry } from "./otel/otel";
 import { platform } from "./Platform";
 import { isFailure } from "./utils/fetch";
 
@@ -101,7 +100,6 @@ enum LoadState {
 class DependencyLoadStates {
   public config: LoadState = LoadState.None;
   public sentry: LoadState = LoadState.None;
-  public openTelemetry: LoadState = LoadState.None;
 
   public allDepsAreLoaded(): boolean {
     return !Object.values(this).some((s) => s !== LoadState.Loaded);
@@ -123,7 +121,7 @@ export class Initializer {
     }
 
     if (shouldPolyfillDurationFormat()) {
-      polyfills.push(import("@formatjs/intl-durationformat/polyfill-force"));
+      polyfills.push(import("@formatjs/intl-durationformat/polyfill-force.js"));
     }
 
     await Promise.all(polyfills);
@@ -266,15 +264,6 @@ export class Initializer {
       this.loadStates.sentry = LoadState.Loaded;
     }
 
-    // OpenTelemetry (also only after config loaded)
-    if (
-      this.loadStates.openTelemetry === LoadState.None &&
-      this.loadStates.config === LoadState.Loaded
-    ) {
-      ElementCallOpenTelemetry.globalInit();
-      this.loadStates.openTelemetry = LoadState.Loaded;
-    }
-
     if (this.loadStates.allDepsAreLoaded()) {
       // resolve if there is no dependency that is not loaded
       resolve();

src/livekit/MatrixAudioRenderer.tsx

@@ -166,7 +166,11 @@ interface StereoPanAudioTrackProps {
  * It main purpose is to remount the AudioTrack component when switching from
  * audioContext to normal audio playback.
  * As of now the AudioTrack component does not support adding audio nodes while being mounted.
- * @param param0
+ * @param props The component props
+ * @param props.trackRef The track reference
+ * @param props.muted If the track should be muted
+ * @param props.audioContext The audio context to use
+ * @param props.audioNodes The audio nodes to use
  * @returns
  */
 function AudioTrackWithAudioNodes({

src/livekit/openIDSFU.test.ts

@@ -0,0 +1,112 @@
+/*
+Copyright 2025 Element Creations Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+import {
+  beforeEach,
+  afterEach,
+  describe,
+  expect,
+  it,
+  type MockedObject,
+  vitest,
+} from "vitest";
+import fetchMock from "fetch-mock";
+
+import { getSFUConfigWithOpenID, type OpenIDClientParts } from "./openIDSFU";
+import { testJWTToken } from "../utils/test-fixtures";
+
+const sfuUrl = "https://sfu.example.org";
+
+describe("getSFUConfigWithOpenID", () => {
+  let matrixClient: MockedObject<OpenIDClientParts>;
+  beforeEach(() => {
+    matrixClient = {
+      getOpenIdToken: vitest.fn(),
+      getDeviceId: vitest.fn(),
+    };
+  });
+  afterEach(() => {
+    vitest.clearAllMocks();
+    fetchMock.reset();
+  });
+  it("should handle fetching a token", async () => {
+    fetchMock.post("https://sfu.example.org/sfu/get", () => {
+      return {
+        status: 200,
+        body: { url: sfuUrl, jwt: testJWTToken },
+      };
+    });
+    const config = await getSFUConfigWithOpenID(
+      matrixClient,
+      "https://sfu.example.org",
+      "!example_room_id",
+    );
+    expect(config).toEqual({
+      jwt: testJWTToken,
+      url: sfuUrl,
+      livekitIdentity: "@me:example.org:ABCDEF",
+      livekitAlias: "!example_room_id",
+    });
+    void (await fetchMock.flush());
+  });
+  it("should fail if the SFU errors", async () => {
+    fetchMock.post("https://sfu.example.org/sfu/get", () => {
+      return {
+        status: 500,
+        body: { error: "Test failure" },
+      };
+    });
+    try {
+      await getSFUConfigWithOpenID(
+        matrixClient,
+        "https://sfu.example.org",
+        "!example_room_id",
+      );
+    } catch (ex) {
+      expect(((ex as Error).cause as Error).message).toEqual(
+        "SFU Config fetch failed with status code 500",
+      );
+      void (await fetchMock.flush());
+      return;
+    }
+    expect.fail("Expected test to throw;");
+  });
+
+  it("should retry fetching the openid token", async () => {
+    let count = 0;
+    matrixClient.getOpenIdToken.mockImplementation(async () => {
+      count++;
+      if (count < 2) {
+        throw Error("Test failure");
+      }
+      return Promise.resolve({
+        token_type: "Bearer",
+        access_token: "foobar",
+        matrix_server_name: "example.org",
+        expires_in: 30,
+      });
+    });
+    fetchMock.post("https://sfu.example.org/sfu/get", () => {
+      return {
+        status: 200,
+        body: { url: sfuUrl, jwt: testJWTToken },
+      };
+    });
+    const config = await getSFUConfigWithOpenID(
+      matrixClient,
+      "https://sfu.example.org",
+      "!example_room_id",
+    );
+    expect(config).toEqual({
+      jwt: testJWTToken,
+      url: sfuUrl,
+      livekitIdentity: "@me:example.org:ABCDEF",
+      livekitAlias: "!example_room_id",
+    });
+    void (await fetchMock.flush());
+  });
+});

src/livekit/openIDSFU.ts

@@ -11,9 +11,47 @@ import { logger } from "matrix-js-sdk/lib/logger";
 import { FailToGetOpenIdToken } from "../utils/errors";
 import { doNetworkOperationWithRetry } from "../utils/matrix";
 
+/**
+ * Configuration and access tokens provided by the SFU on successful authentication.
+ */
 export interface SFUConfig {
   url: string;
   jwt: string;
+  livekitAlias: string;
+  livekitIdentity: string;
+}
+
+/**
+ * Decoded details from the JWT.
+ */
+interface SFUJWTPayload {
+  /**
+   * Expiration time for the JWT.
+   * Note: This value is in seconds since Unix epoch.
+   */
+  exp: number;
+  /**
+   * Name of the instance which authored the JWT
+   */
+  iss: string;
+  /**
+   * Time at which the JWT can start to be used.
+   * Note: This value is in seconds since Unix epoch.
+   */
+  nbf: number;
+  /**
+   * Subject. The Livekit alias in this context.
+   */
+  sub: string;
+  /**
+   * The set of permissions for the user.
+   */
+  video: {
+    canPublish: boolean;
+    canSubscribe: boolean;
+    room: string;
+    roomJoin: boolean;
+  };
 }
 
 // The bits we need from MatrixClient
@@ -25,9 +63,9 @@ export type OpenIDClientParts = Pick<
  * Gets a bearer token from the homeserver and then use it to authenticate
  * to the matrix RTC backend in order to get acces to the SFU.
  * It has built-in retry for calls to the homeserver with a backoff policy.
- * @param client
- * @param serviceUrl
- * @param matrixRoomId
+ * @param client The Matrix client
+ * @param serviceUrl The URL of the livekit SFU service
+ * @param matrixRoomId The Matrix room ID for which to get the SFU config
  * @returns Object containing the token information
  * @throws FailToGetOpenIdToken
  */
@@ -57,7 +95,17 @@ export async function getSFUConfigWithOpenID(
   );
   logger.info(`Got JWT from call's active focus URL.`);
 
-  return sfuConfig;
+  // Pull the details from the JWT
+  const [, payloadStr] = sfuConfig.jwt.split(".");
+  // TODO: Prefer Uint8Array.fromBase64 when widely available
+  const payload = JSON.parse(global.atob(payloadStr)) as SFUJWTPayload;
+  return {
+    jwt: sfuConfig.jwt,
+    url: sfuConfig.url,
+    livekitAlias: payload.video.room,
+    // NOTE: Currently unused.
+    livekitIdentity: payload.sub,
+  };
 }
 
 async function getLiveKitJWT(
@@ -65,7 +113,7 @@ async function getLiveKitJWT(
   livekitServiceURL: string,
   roomName: string,
   openIDToken: IOpenIDToken,
-): Promise<SFUConfig> {
+): Promise<{ url: string; jwt: string }> {
   try {
     const res = await fetch(livekitServiceURL + "/sfu/get", {
       method: "POST",
@@ -83,6 +131,6 @@ async function getLiveKitJWT(
     }
     return await res.json();
   } catch (e) {
-    throw new Error("SFU Config fetch failed with exception " + e);
+    throw new Error("SFU Config fetch failed with exception", { cause: e });
   }
 }

src/otel/OTelCall.ts

@@ -1,188 +0,0 @@
-/*
-Copyright 2023, 2024 New Vector Ltd.
-
-SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
-Please see LICENSE in the repository root for full details.
-*/
-
-import { type Span } from "@opentelemetry/api";
-import { type MatrixCall } from "matrix-js-sdk";
-import { CallEvent } from "matrix-js-sdk/lib/webrtc/call";
-import {
-  type TransceiverStats,
-  type CallFeedStats,
-} from "matrix-js-sdk/lib/webrtc/stats/statsReport";
-
-import { ObjectFlattener } from "./ObjectFlattener";
-import { ElementCallOpenTelemetry } from "./otel";
-import { type OTelCallAbstractMediaStreamSpan } from "./OTelCallAbstractMediaStreamSpan";
-import { OTelCallTransceiverMediaStreamSpan } from "./OTelCallTransceiverMediaStreamSpan";
-import { OTelCallFeedMediaStreamSpan } from "./OTelCallFeedMediaStreamSpan";
-
-type StreamId = string;
-type MID = string;
-
-/**
- * Tracks an individual call within a group call, either to a full-mesh peer or a focus
- */
-export class OTelCall {
-  private readonly trackFeedSpan = new Map<
-    StreamId,
-    OTelCallAbstractMediaStreamSpan
-  >();
-  private readonly trackTransceiverSpan = new Map<
-    MID,
-    OTelCallAbstractMediaStreamSpan
-  >();
-
-  public constructor(
-    public userId: string,
-    public deviceId: string,
-    public call: MatrixCall,
-    public span: Span,
-  ) {
-    if (call.peerConn) {
-      this.addCallPeerConnListeners();
-    } else {
-      this.call.once(
-        CallEvent.PeerConnectionCreated,
-        this.addCallPeerConnListeners,
-      );
-    }
-  }
-
-  public dispose(): void {
-    this.call.peerConn?.removeEventListener(
-      "connectionstatechange",
-      this.onCallConnectionStateChanged,
-    );
-    this.call.peerConn?.removeEventListener(
-      "signalingstatechange",
-      this.onCallSignalingStateChanged,
-    );
-    this.call.peerConn?.removeEventListener(
-      "iceconnectionstatechange",
-      this.onIceConnectionStateChanged,
-    );
-    this.call.peerConn?.removeEventListener(
-      "icegatheringstatechange",
-      this.onIceGatheringStateChanged,
-    );
-    this.call.peerConn?.removeEventListener(
-      "icecandidateerror",
-      this.onIceCandidateError,
-    );
-  }
-
-  private addCallPeerConnListeners = (): void => {
-    this.call.peerConn?.addEventListener(
-      "connectionstatechange",
-      this.onCallConnectionStateChanged,
-    );
-    this.call.peerConn?.addEventListener(
-      "signalingstatechange",
-      this.onCallSignalingStateChanged,
-    );
-    this.call.peerConn?.addEventListener(
-      "iceconnectionstatechange",
-      this.onIceConnectionStateChanged,
-    );
-    this.call.peerConn?.addEventListener(
-      "icegatheringstatechange",
-      this.onIceGatheringStateChanged,
-    );
-    this.call.peerConn?.addEventListener(
-      "icecandidateerror",
-      this.onIceCandidateError,
-    );
-  };
-
-  public onCallConnectionStateChanged = (): void => {
-    this.span.addEvent("matrix.call.callConnectionStateChange", {
-      callConnectionState: this.call.peerConn?.connectionState,
-    });
-  };
-
-  public onCallSignalingStateChanged = (): void => {
-    this.span.addEvent("matrix.call.callSignalingStateChange", {
-      callSignalingState: this.call.peerConn?.signalingState,
-    });
-  };
-
-  public onIceConnectionStateChanged = (): void => {
-    this.span.addEvent("matrix.call.iceConnectionStateChange", {
-      iceConnectionState: this.call.peerConn?.iceConnectionState,
-    });
-  };
-
-  public onIceGatheringStateChanged = (): void => {
-    this.span.addEvent("matrix.call.iceGatheringStateChange", {
-      iceGatheringState: this.call.peerConn?.iceGatheringState,
-    });
-  };
-
-  public onIceCandidateError = (ev: Event): void => {
-    const flatObject = {};
-    ObjectFlattener.flattenObjectRecursive(ev, flatObject, "error.", 0);
-
-    this.span.addEvent("matrix.call.iceCandidateError", flatObject);
-  };
-
-  public onCallFeedStats(callFeeds: CallFeedStats[]): void {
-    let prvFeeds: StreamId[] = [...this.trackFeedSpan.keys()];
-
-    callFeeds.forEach((feed) => {
-      if (!this.trackFeedSpan.has(feed.stream)) {
-        this.trackFeedSpan.set(
-          feed.stream,
-          new OTelCallFeedMediaStreamSpan(
-            ElementCallOpenTelemetry.instance,
-            this.span,
-            feed,
-          ),
-        );
-      }
-      this.trackFeedSpan.get(feed.stream)?.update(feed);
-      prvFeeds = prvFeeds.filter((prvStreamId) => prvStreamId !== feed.stream);
-    });
-
-    prvFeeds.forEach((prvStreamId) => {
-      this.trackFeedSpan.get(prvStreamId)?.end();
-      this.trackFeedSpan.delete(prvStreamId);
-    });
-  }
-
-  public onTransceiverStats(transceiverStats: TransceiverStats[]): void {
-    let prvTransSpan: MID[] = [...this.trackTransceiverSpan.keys()];
-
-    transceiverStats.forEach((transStats) => {
-      if (!this.trackTransceiverSpan.has(transStats.mid)) {
-        this.trackTransceiverSpan.set(
-          transStats.mid,
-          new OTelCallTransceiverMediaStreamSpan(
-            ElementCallOpenTelemetry.instance,
-            this.span,
-            transStats,
-          ),
-        );
-      }
-      this.trackTransceiverSpan.get(transStats.mid)?.update(transStats);
-      prvTransSpan = prvTransSpan.filter(
-        (prvStreamId) => prvStreamId !== transStats.mid,
-      );
-    });
-
-    prvTransSpan.forEach((prvMID) => {
-      this.trackTransceiverSpan.get(prvMID)?.end();
-      this.trackTransceiverSpan.delete(prvMID);
-    });
-  }
-
-  public end(): void {
-    this.trackFeedSpan.forEach((feedSpan) => feedSpan.end());
-    this.trackTransceiverSpan.forEach((transceiverSpan) =>
-      transceiverSpan.end(),
-    );
-    this.span.end();
-  }
-}

src/otel/OTelCallAbstractMediaStreamSpan.ts

@@ -1,69 +0,0 @@
-/*
-Copyright 2023, 2024 New Vector Ltd.
-
-SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
-Please see LICENSE in the repository root for full details.
-*/
-
-import opentelemetry, { type Span } from "@opentelemetry/api";
-import { type TrackStats } from "matrix-js-sdk/lib/webrtc/stats/statsReport";
-
-import { type ElementCallOpenTelemetry } from "./otel";
-import { OTelCallMediaStreamTrackSpan } from "./OTelCallMediaStreamTrackSpan";
-
-type TrackId = string;
-
-export abstract class OTelCallAbstractMediaStreamSpan {
-  protected readonly trackSpans = new Map<
-    TrackId,
-    OTelCallMediaStreamTrackSpan
-  >();
-  public readonly span;
-
-  public constructor(
-    protected readonly oTel: ElementCallOpenTelemetry,
-    protected readonly callSpan: Span,
-    protected readonly type: string,
-  ) {
-    const ctx = opentelemetry.trace.setSpan(
-      opentelemetry.context.active(),
-      callSpan,
-    );
-    const options = {
-      links: [
-        {
-          context: callSpan.spanContext(),
-        },
-      ],
-    };
-    this.span = oTel.tracer.startSpan(this.type, options, ctx);
-  }
-
-  protected upsertTrackSpans(tracks: TrackStats[]): void {
-    let prvTracks: TrackId[] = [...this.trackSpans.keys()];
-    tracks.forEach((t) => {
-      if (!this.trackSpans.has(t.id)) {
-        this.trackSpans.set(
-          t.id,
-          new OTelCallMediaStreamTrackSpan(this.oTel, this.span, t),
-        );
-      }
-      this.trackSpans.get(t.id)?.update(t);
-      prvTracks = prvTracks.filter((prvTrackId) => prvTrackId !== t.id);
-    });
-
-    prvTracks.forEach((prvTrackId) => {
-      this.trackSpans.get(prvTrackId)?.end();
-      this.trackSpans.delete(prvTrackId);
-    });
-  }
-
-  public abstract update(data: object): void;
-
-  public end(): void {
-    this.trackSpans.forEach((tSpan) => {
-      tSpan.end();
-    });
-    this.span.end();
-  }
-}

src/otel/OTelCallFeedMediaStreamSpan.ts

@@ -1,64 +0,0 @@
-/*
-Copyright 2023, 2024 New Vector Ltd.
-
-SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
-Please see LICENSE in the repository root for full details.
-*/
-
-import { type Span } from "@opentelemetry/api";
-import {
-  type CallFeedStats,
-  type TrackStats,
-} from "matrix-js-sdk/lib/webrtc/stats/statsReport";
-
-import { type ElementCallOpenTelemetry } from "./otel";
-import { OTelCallAbstractMediaStreamSpan } from "./OTelCallAbstractMediaStreamSpan";
-
-export class OTelCallFeedMediaStreamSpan extends OTelCallAbstractMediaStreamSpan {
-  private readonly prev: { isAudioMuted: boolean; isVideoMuted: boolean };
-
-  public constructor(
-    protected readonly oTel: ElementCallOpenTelemetry,
-    protected readonly callSpan: Span,
-    callFeed: CallFeedStats,
-  ) {
-    const postFix =
-      callFeed.type === "local" && callFeed.prefix === "from-call-feed"
-        ? "(clone)"
-        : "";
-    super(oTel, callSpan, `matrix.call.feed.${callFeed.type}${postFix}`);
-    this.span.setAttribute("feed.streamId", callFeed.stream);
-    this.span.setAttribute("feed.type", callFeed.type);
-    this.span.setAttribute("feed.readFrom", callFeed.prefix);
-    this.span.setAttribute("feed.purpose", callFeed.purpose);
-    this.prev = {
-      isAudioMuted: callFeed.isAudioMuted,
-      isVideoMuted: callFeed.isVideoMuted,
-    };
-    this.span.addEvent("matrix.call.feed.initState", this.prev);
-  }
-
-  public update(callFeed: CallFeedStats): void {
-    if (this.prev.isAudioMuted !== callFeed.isAudioMuted) {
-      this.span.addEvent("matrix.call.feed.audioMuted", {
-        isAudioMuted: callFeed.isAudioMuted,
-      });
-      this.prev.isAudioMuted = callFeed.isAudioMuted;
-    }
-    if (this.prev.isVideoMuted !== callFeed.isVideoMuted) {
-      this.span.addEvent("matrix.call.feed.isVideoMuted", {
-        isVideoMuted: callFeed.isVideoMuted,
-      });
-      this.prev.isVideoMuted = callFeed.isVideoMuted;
-    }
-
-    const trackStats: TrackStats[] = [];
-    if (callFeed.video) {
-      trackStats.push(callFeed.video);
-    }
-    if (callFeed.audio) {
-      trackStats.push(callFeed.audio);
-    }
-    this.upsertTrackSpans(trackStats);
-  }
-}

src/otel/OTelCallMediaStreamTrackSpan.ts

@@ -1,69 +0,0 @@
-/*
-Copyright 2023, 2024 New Vector Ltd.
-
-SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
-Please see LICENSE in the repository root for full details.
-*/
-
-import { type TrackStats } from "matrix-js-sdk/lib/webrtc/stats/statsReport";
-import opentelemetry, { type Span } from "@opentelemetry/api";
-
-import { type ElementCallOpenTelemetry } from "./otel";
-
-export class OTelCallMediaStreamTrackSpan {
-  private readonly span: Span;
-  private prev: TrackStats;
-
-  public constructor(
-    protected readonly oTel: ElementCallOpenTelemetry,
-    protected readonly streamSpan: Span,
-    data: TrackStats,
-  ) {
-    const ctx = opentelemetry.trace.setSpan(
-      opentelemetry.context.active(),
-      streamSpan,
-    );
-    const options = {
-      links: [
-        {
-          context: streamSpan.spanContext(),
-        },
-      ],
-    };
-    const type = `matrix.call.track.${data.label}.${data.kind}`;
-    this.span = oTel.tracer.startSpan(type, options, ctx);
-    this.span.setAttribute("track.trackId", data.id);
-    this.span.setAttribute("track.kind", data.kind);
-    this.span.setAttribute("track.constrainDeviceId", data.constrainDeviceId);
-    this.span.setAttribute("track.settingDeviceId", data.settingDeviceId);
-    this.span.setAttribute("track.label", data.label);
-
-    this.span.addEvent("matrix.call.track.initState", {
-      readyState: data.readyState,
-      muted: data.muted,
-      enabled: data.enabled,
-    });
-    this.prev = data;
-  }
-
-  public update(data: TrackStats): void {
-    if (this.prev.muted !== data.muted) {
-      this.span.addEvent("matrix.call.track.muted", { muted: data.muted });
-    }
-    if (this.prev.enabled !== data.enabled) {
-      this.span.addEvent("matrix.call.track.enabled", {
-        enabled: data.enabled,
-      });
-    }
-    if (this.prev.readyState !== data.readyState) {
-      this.span.addEvent("matrix.call.track.readyState", {
-        readyState: data.readyState,
-      });
-    }
-    this.prev = data;
-  }
-
-  public end(): void {
-    this.span.end();
-  }
-}

src/otel/OTelCallTransceiverMediaStreamSpan.ts

@@ -1,61 +0,0 @@
-/*
-Copyright 2023, 2024 New Vector Ltd.
-
-SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
-Please see LICENSE in the repository root for full details.
-*/
-
-import { type Span } from "@opentelemetry/api";
-import {
-  type TrackStats,
-  type TransceiverStats,
-} from "matrix-js-sdk/lib/webrtc/stats/statsReport";
-
-import { type ElementCallOpenTelemetry } from "./otel";
-import { OTelCallAbstractMediaStreamSpan } from "./OTelCallAbstractMediaStreamSpan";
-
-export class OTelCallTransceiverMediaStreamSpan extends OTelCallAbstractMediaStreamSpan {
-  private readonly prev: {
-    direction: string;
-    currentDirection: string;
-  };
-
-  public constructor(
-    protected readonly oTel: ElementCallOpenTelemetry,
-    protected readonly callSpan: Span,
-    stats: TransceiverStats,
-  ) {
-    super(oTel, callSpan, `matrix.call.transceiver.${stats.mid}`);
-    this.span.setAttribute("transceiver.mid", stats.mid);
-
-    this.prev = {
-      direction: stats.direction,
-      currentDirection: stats.currentDirection,
-    };
-    this.span.addEvent("matrix.call.transceiver.initState", this.prev);
-  }
-
-  public update(stats: TransceiverStats): void {
-    if (this.prev.currentDirection !== stats.currentDirection) {
-      this.span.addEvent("matrix.call.transceiver.currentDirection", {
-        currentDirection: stats.currentDirection,
-      });
-      this.prev.currentDirection = stats.currentDirection;
-    }
-    if (this.prev.direction !== stats.direction) {
-      this.span.addEvent("matrix.call.transceiver.direction", {
-        direction: stats.direction,
-      });
-      this.prev.direction = stats.direction;
-    }
-
-    const trackStats: TrackStats[] = [];
-    if (stats.sender) {
-      trackStats.push(stats.sender);
-    }
-    if (stats.receiver) {
-      trackStats.push(stats.receiver);
-    }
-    this.upsertTrackSpans(trackStats);
-  }
-}

src/otel/OTelGroupCallMembership.ts

@@ -1,477 +0,0 @@
-/*
-Copyright 2023, 2024 New Vector Ltd.
-
-SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
-Please see LICENSE in the repository root for full details.
-*/
-
-import opentelemetry, {
-  type Span,
-  type Attributes,
-  type Context,
-} from "@opentelemetry/api";
-import {
-  type GroupCall,
-  type MatrixClient,
-  type MatrixEvent,
-  type RoomMember,
-} from "matrix-js-sdk";
-import { logger } from "matrix-js-sdk/lib/logger";
-import {
-  type CallError,
-  type CallState,
-  type MatrixCall,
-  type VoipEvent,
-} from "matrix-js-sdk/lib/webrtc/call";
-import {
-  type CallsByUserAndDevice,
-  type GroupCallError,
-  GroupCallEvent,
-  type GroupCallStatsReport,
-} from "matrix-js-sdk/lib/webrtc/groupCall";
-import {
-  type ConnectionStatsReport,
-  type ByteSentStatsReport,
-  type SummaryStatsReport,
-  type CallFeedReport,
-} from "matrix-js-sdk/lib/webrtc/stats/statsReport";
-
-import { ElementCallOpenTelemetry } from "./otel";
-import { ObjectFlattener } from "./ObjectFlattener";
-import { OTelCall } from "./OTelCall";
-
-/**
- * Represent the span of time which we intend to be joined to a group call
- */
-export class OTelGroupCallMembership {
-  private callMembershipSpan?: Span;
-  private groupCallContext?: Context;
-  private myUserId = "unknown";
-  private myDeviceId: string;
-  private myMember?: RoomMember;
-  private callsByCallId = new Map<string, OTelCall>();
-  private statsReportSpan: {
-    span: Span | undefined;
-    stats: OTelStatsReportEvent[];
-  };
-  private readonly speakingSpans = new Map<RoomMember, Map<string, Span>>();
-
-  public constructor(
-    private groupCall: GroupCall,
-    client: MatrixClient,
-  ) {
-    const clientId = client.getUserId();
-    if (clientId) {
-      this.myUserId = clientId;
-      const myMember = groupCall.room.getMember(clientId);
-      if (myMember) {
-        this.myMember = myMember;
-      }
-    }
-    this.myDeviceId = client.getDeviceId() || "unknown";
-    this.statsReportSpan = { span: undefined, stats: [] };
-    this.groupCall.on(GroupCallEvent.CallsChanged, this.onCallsChanged);
-  }
-
-  public dispose(): void {
-    this.groupCall.removeListener(
-      GroupCallEvent.CallsChanged,
-      this.onCallsChanged,
-    );
-  }
-
-  public onJoinCall(): void {
-    if (!ElementCallOpenTelemetry.instance) return;
-    if (this.callMembershipSpan !== undefined) {
-      logger.warn("Call membership span is already started");
-      return;
-    }
-
-    // Create the main span that tracks the time we intend to be in the call
-    this.callMembershipSpan =
-      ElementCallOpenTelemetry.instance.tracer.startSpan(
-        "matrix.groupCallMembership",
-      );
-    this.callMembershipSpan.setAttribute(
-      "matrix.confId",
-      this.groupCall.groupCallId,
-    );
-    this.callMembershipSpan.setAttribute("matrix.userId", this.myUserId);
-    this.callMembershipSpan.setAttribute("matrix.deviceId", this.myDeviceId);
-    this.callMembershipSpan.setAttribute(
-      "matrix.displayName",
-      this.myMember ? this.myMember.name : "unknown-name",
-    );
-
-    this.groupCallContext = opentelemetry.trace.setSpan(
-      opentelemetry.context.active(),
-      this.callMembershipSpan,
-    );
-
-    this.callMembershipSpan?.addEvent("matrix.joinCall");
-  }
-
-  public onLeaveCall(): void {
-    if (this.callMembershipSpan === undefined) {
-      logger.warn("Call membership span is already ended");
-      return;
-    }
-
-    this.callMembershipSpan.addEvent("matrix.leaveCall");
-    // and end the span to indicate we've left
-    this.callMembershipSpan.end();
-    this.callMembershipSpan = undefined;
-    this.groupCallContext = undefined;
-  }
-
-  public onUpdateRoomState(event: MatrixEvent): void {
-    if (
-      !event ||
-      (!event.getType().startsWith("m.call") &&
-        !event.getType().startsWith("org.matrix.msc3401.call"))
-    ) {
-      return;
-    }
-
-    this.callMembershipSpan?.addEvent(
-      `matrix.roomStateEvent_${event.getType()}`,
-      ObjectFlattener.flattenVoipEvent(event.getContent()),
-    );
-  }
-
-  public onCallsChanged(calls: CallsByUserAndDevice): void {
-    for (const [userId, userCalls] of calls.entries()) {
-      for (const [deviceId, call] of userCalls.entries()) {
-        if (!this.callsByCallId.has(call.callId)) {
-          if (ElementCallOpenTelemetry.instance) {
-            const span = ElementCallOpenTelemetry.instance.tracer.startSpan(
-              `matrix.call`,
-              undefined,
-              this.groupCallContext,
-            );
-            // XXX: anonymity
-            span.setAttribute("matrix.call.target.userId", userId);
-            span.setAttribute("matrix.call.target.deviceId", deviceId);
-            const displayName =
-              this.groupCall.room.getMember(userId)?.name ?? "unknown";
-            span.setAttribute("matrix.call.target.displayName", displayName);
-            this.callsByCallId.set(
-              call.callId,
-              new OTelCall(userId, deviceId, call, span),
-            );
-          }
-        }
-      }
-    }
-
-    for (const callTrackingInfo of this.callsByCallId.values()) {
-      const userCalls = calls.get(callTrackingInfo.userId);
-      if (
-        !userCalls ||
-        !userCalls.has(callTrackingInfo.deviceId) ||
-        userCalls.get(callTrackingInfo.deviceId)?.callId !==
-          callTrackingInfo.call.callId
-      ) {
-        callTrackingInfo.end();
-        this.callsByCallId.delete(callTrackingInfo.call.callId);
-      }
-    }
-  }
-
-  public onCallStateChange(call: MatrixCall, newState: CallState): void {
-    const callTrackingInfo = this.callsByCallId.get(call.callId);
-    if (!callTrackingInfo) {
-      logger.error(`Got call state change for unknown call ID ${call.callId}`);
-      return;
-    }
-
-    callTrackingInfo.span.addEvent("matrix.call.stateChange", {
-      state: newState,
-    });
-  }
-
-  public onSendEvent(call: MatrixCall, event: VoipEvent): void {
-    const eventType = event.eventType as string;
-    if (
-      !eventType.startsWith("m.call") &&
-      !eventType.startsWith("org.matrix.call")
-    )
-      return;
-
-    const callTrackingInfo = this.callsByCallId.get(call.callId);
-    if (!callTrackingInfo) {
-      logger.error(`Got call send event for unknown call ID ${call.callId}`);
-      return;
-    }
-
-    if (event.type === "toDevice") {
-      callTrackingInfo.span.addEvent(
-        `matrix.sendToDeviceEvent_${event.eventType}`,
-        ObjectFlattener.flattenVoipEvent(event),
-      );
-    } else if (event.type === "sendEvent") {
-      callTrackingInfo.span.addEvent(
-        `matrix.sendToRoomEvent_${event.eventType}`,
-        ObjectFlattener.flattenVoipEvent(event),
-      );
-    }
-  }
-
-  public onReceivedVoipEvent(event: MatrixEvent): void {
-    // These come straight from CallEventHandler so don't have
-    // a call already associated (in principle we could receive
-    // events for calls we don't know about).
-    const callId = event.getContent().call_id;
-    if (!callId) {
-      this.callMembershipSpan?.addEvent("matrix.receive_voip_event_no_callid", {
-        "sender.userId": event.getSender(),
-      });
-      logger.error("Received call event with no call ID!");
-      return;
-    }
-
-    const call = this.callsByCallId.get(callId);
-    if (!call) {
-      this.callMembershipSpan?.addEvent(
-        "matrix.receive_voip_event_unknown_callid",
-        {
-          "sender.userId": event.getSender(),
-        },
-      );
-      logger.error("Received call event for unknown call ID " + callId);
-      return;
-    }
-
-    call.span.addEvent("matrix.receive_voip_event", {
-      "sender.userId": event.getSender(),
-      ...ObjectFlattener.flattenVoipEvent(event.getContent()),
-    });
-  }
-
-  public onToggleMicrophoneMuted(newValue: boolean): void {
-    this.callMembershipSpan?.addEvent("matrix.toggleMicMuted", {
-      "matrix.microphone.muted": newValue,
-    });
-  }
-
-  public onSetMicrophoneMuted(setMuted: boolean): void {
-    this.callMembershipSpan?.addEvent("matrix.setMicMuted", {
-      "matrix.microphone.muted": setMuted,
-    });
-  }
-
-  public onToggleLocalVideoMuted(newValue: boolean): void {
-    this.callMembershipSpan?.addEvent("matrix.toggleVidMuted", {
-      "matrix.video.muted": newValue,
-    });
-  }
-
-  public onSetLocalVideoMuted(setMuted: boolean): void {
-    this.callMembershipSpan?.addEvent("matrix.setVidMuted", {
-      "matrix.video.muted": setMuted,
-    });
-  }
-
-  public onToggleScreensharing(newValue: boolean): void {
-    this.callMembershipSpan?.addEvent("matrix.setVidMuted", {
-      "matrix.screensharing.enabled": newValue,
-    });
-  }
-
-  public onSpeaking(
-    member: RoomMember,
-    deviceId: string,
-    speaking: boolean,
-  ): void {
-    if (speaking) {
-      // Ensure that there's an audio activity span for this speaker
-      let deviceMap = this.speakingSpans.get(member);
-      if (deviceMap === undefined) {
-        deviceMap = new Map();
-        this.speakingSpans.set(member, deviceMap);
-      }
-
-      if (!deviceMap.has(deviceId)) {
-        const span = ElementCallOpenTelemetry.instance.tracer.startSpan(
-          "matrix.audioActivity",
-          undefined,
-          this.groupCallContext,
-        );
-        span.setAttribute("matrix.userId", member.userId);
-        span.setAttribute("matrix.displayName", member.rawDisplayName);
-
-        deviceMap.set(deviceId, span);
-      }
-    } else {
-      // End the audio activity span for this speaker, if any
-      const deviceMap = this.speakingSpans.get(member);
-      deviceMap?.get(deviceId)?.end();
-      deviceMap?.delete(deviceId);
-
-      if (deviceMap?.size === 0) this.speakingSpans.delete(member);
-    }
-  }
-
-  public onCallError(error: CallError, call: MatrixCall): void {
-    const callTrackingInfo = this.callsByCallId.get(call.callId);
-    if (!callTrackingInfo) {
-      logger.error(`Got error for unknown call ID ${call.callId}`);
-      return;
-    }
-
-    callTrackingInfo.span.recordException(error);
-  }
-
-  public onGroupCallError(error: GroupCallError): void {
-    this.callMembershipSpan?.recordException(error);
-  }
-
-  public onUndecryptableToDevice(event: MatrixEvent): void {
-    this.callMembershipSpan?.addEvent("matrix.toDevice.undecryptable", {
-      "sender.userId": event.getSender(),
-    });
-  }
-
-  public onCallFeedStatsReport(
-    report: GroupCallStatsReport<CallFeedReport>,
-  ): void {
-    if (!ElementCallOpenTelemetry.instance) return;
-    let call: OTelCall | undefined;
-    const callId = report.report?.callId;
-
-    if (callId) {
-      call = this.callsByCallId.get(callId);
-    }
-
-    if (!call) {
-      this.callMembershipSpan?.addEvent(
-        OTelStatsReportType.CallFeedReport + "_unknown_callId",
-        {
-          "call.callId": callId,
-          "call.opponentMemberId": report.report?.opponentMemberId
-            ? report.report?.opponentMemberId
-            : "unknown",
-        },
-      );
-      logger.error(
-        `Received ${OTelStatsReportType.CallFeedReport} with unknown call ID: ${callId}`,
-      );
-      return;
-    } else {
-      call.onCallFeedStats(report.report.callFeeds);
-      call.onTransceiverStats(report.report.transceiver);
-    }
-  }
-
-  public onConnectionStatsReport(
-    statsReport: GroupCallStatsReport<ConnectionStatsReport>,
-  ): void {
-    this.buildCallStatsSpan(
-      OTelStatsReportType.ConnectionReport,
-      statsReport.report,
-    );
-  }
-
-  public onByteSentStatsReport(
-    statsReport: GroupCallStatsReport<ByteSentStatsReport>,
-  ): void {
-    this.buildCallStatsSpan(
-      OTelStatsReportType.ByteSentReport,
-      statsReport.report,
-    );
-  }
-
-  public buildCallStatsSpan(
-    type: OTelStatsReportType,
-    report: ByteSentStatsReport | ConnectionStatsReport,
-  ): void {
-    if (!ElementCallOpenTelemetry.instance) return;
-    let call: OTelCall | undefined;
-    const callId = report?.callId;
-
-    if (callId) {
-      call = this.callsByCallId.get(callId);
-    }
-
-    if (!call) {
-      this.callMembershipSpan?.addEvent(type + "_unknown_callid", {
-        "call.callId": callId,
-        "call.opponentMemberId": report.opponentMemberId
-          ? report.opponentMemberId
-          : "unknown",
-      });
-      logger.error(`Received ${type} with unknown call ID: ${callId}`);
-      return;
-    }
-    const data = ObjectFlattener.flattenReportObject(type, report);
-    const ctx = opentelemetry.trace.setSpan(
-      opentelemetry.context.active(),
-      call.span,
-    );
-
-    const options = {
-      links: [
-        {
-          context: call.span.spanContext(),
-        },
-      ],
-    };
-
-    const span = ElementCallOpenTelemetry.instance.tracer.startSpan(
-      type,
-      options,
-      ctx,
-    );
-
-    span.setAttribute("matrix.callId", callId ?? "unknown");
-    span.setAttribute(
-      "matrix.opponentMemberId",
-      report.opponentMemberId ? report.opponentMemberId : "unknown",
-    );
-    span.addEvent("matrix.call.connection_stats_event", data);
-    span.end();
-  }
-
-  public onSummaryStatsReport(
-    statsReport: GroupCallStatsReport<SummaryStatsReport>,
-  ): void {
-    if (!ElementCallOpenTelemetry.instance) return;
-
-    const type = OTelStatsReportType.SummaryReport;
-    const data = ObjectFlattener.flattenSummaryStatsReportObject(statsReport);
-    if (this.statsReportSpan.span === undefined && this.callMembershipSpan) {
-      const ctx = opentelemetry.trace.setSpan(
-        opentelemetry.context.active(),
-        this.callMembershipSpan,
-      );
-      const span = ElementCallOpenTelemetry.instance?.tracer.startSpan(
-        "matrix.groupCallMembership.summaryReport",
-        undefined,
-        ctx,
-      );
-      if (span === undefined) {
-        return;
-      }
-      span.setAttribute("matrix.confId", this.groupCall.groupCallId);
-      span.setAttribute("matrix.userId", this.myUserId);
-      span.setAttribute(
-        "matrix.displayName",
-        this.myMember ? this.myMember.name : "unknown-name",
-      );
-      span.addEvent(type, data);
-      span.end();
-    }
-  }
-}
-
-interface OTelStatsReportEvent {
-  type: OTelStatsReportType;
-  data: Attributes;
-}
-
-enum OTelStatsReportType {
-  ConnectionReport = "matrix.call.stats.connection",
-  ByteSentReport = "matrix.call.stats.byteSent",
-  SummaryReport = "matrix.stats.summary",
-  CallFeedReport = "matrix.stats.call_feed",
-}

src/otel/ObjectFlattener.test.ts

@@ -1,266 +0,0 @@
-/*
-Copyright 2023, 2024 New Vector Ltd.
-
-SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
-Please see LICENSE in the repository root for full details.
-*/
-
-import { type GroupCallStatsReport } from "matrix-js-sdk/lib/webrtc/groupCall";
-import {
-  type AudioConcealment,
-  type ByteSentStatsReport,
-  type ConnectionStatsReport,
-} from "matrix-js-sdk/lib/webrtc/stats/statsReport";
-import { describe, expect, it } from "vitest";
-
-import { ObjectFlattener } from "../../src/otel/ObjectFlattener";
-
-describe("ObjectFlattener", () => {
-  const noConcealment: AudioConcealment = {
-    concealedAudio: 0,
-    totalAudioDuration: 0,
-  };
-
-  const statsReport: GroupCallStatsReport<ConnectionStatsReport> = {
-    report: {
-      callId: "callId",
-      opponentMemberId: "opponentMemberId",
-      bandwidth: { upload: 426, download: 0 },
-      bitrate: {
-        upload: 426,
-        download: 0,
-        audio: {
-          upload: 124,
-          download: 0,
-        },
-        video: {
-          upload: 302,
-          download: 0,
-        },
-      },
-      packetLoss: {
-        total: 0,
-        download: 0,
-        upload: 0,
-      },
-      framerate: {
-        local: new Map([
-          ["LOCAL_AUDIO_TRACK_ID", 0],
-          ["LOCAL_VIDEO_TRACK_ID", 30],
-        ]),
-        remote: new Map([
-          ["REMOTE_AUDIO_TRACK_ID", 0],
-          ["REMOTE_VIDEO_TRACK_ID", 60],
-        ]),
-      },
-      resolution: {
-        local: new Map([
-          ["LOCAL_AUDIO_TRACK_ID", { height: -1, width: -1 }],
-          ["LOCAL_VIDEO_TRACK_ID", { height: 460, width: 780 }],
-        ]),
-        remote: new Map([
-          ["REMOTE_AUDIO_TRACK_ID", { height: -1, width: -1 }],
-          ["REMOTE_VIDEO_TRACK_ID", { height: 960, width: 1080 }],
-        ]),
-      },
-      jitter: new Map([
-        ["REMOTE_AUDIO_TRACK_ID", 2],
-        ["REMOTE_VIDEO_TRACK_ID", 50],
-      ]),
-      codec: {
-        local: new Map([
-          ["LOCAL_AUDIO_TRACK_ID", "opus"],
-          ["LOCAL_VIDEO_TRACK_ID", "v8"],
-        ]),
-        remote: new Map([
-          ["REMOTE_AUDIO_TRACK_ID", "opus"],
-          ["REMOTE_VIDEO_TRACK_ID", "v9"],
-        ]),
-      },
-      transport: [
-        {
-          ip: "ff11::5fa:abcd:999c:c5c5:50000",
-          type: "udp",
-          localIp: "2aaa:9999:2aaa:999:8888:2aaa:2aaa:7777:50000",
-          isFocus: true,
-          localCandidateType: "host",
-          remoteCandidateType: "host",
-          networkType: "ethernet",
-          rtt: NaN,
-        },
-        {
-          ip: "10.10.10.2:22222",
-          type: "tcp",
-          localIp: "10.10.10.100:33333",
-          isFocus: true,
-          localCandidateType: "srfx",
-          remoteCandidateType: "srfx",
-          networkType: "ethernet",
-          rtt: 0,
-        },
-      ],
-      audioConcealment: new Map([
-        ["REMOTE_AUDIO_TRACK_ID", noConcealment],
-        ["REMOTE_VIDEO_TRACK_ID", noConcealment],
-      ]),
-      totalAudioConcealment: noConcealment,
-    },
-  };
-
-  describe("on flattenObjectRecursive", () => {
-    it("should flatter an Map object", () => {
-      const flatObject = {};
-      ObjectFlattener.flattenObjectRecursive(
-        statsReport.report.resolution,
-        flatObject,
-        "matrix.call.stats.connection.resolution.",
-        0,
-      );
-      expect(flatObject).toEqual({
-        "matrix.call.stats.connection.resolution.local.LOCAL_AUDIO_TRACK_ID.height":
-          -1,
-        "matrix.call.stats.connection.resolution.local.LOCAL_AUDIO_TRACK_ID.width":
-          -1,
-
-        "matrix.call.stats.connection.resolution.local.LOCAL_VIDEO_TRACK_ID.height": 460,
-        "matrix.call.stats.connection.resolution.local.LOCAL_VIDEO_TRACK_ID.width": 780,
-
-        "matrix.call.stats.connection.resolution.remote.REMOTE_AUDIO_TRACK_ID.height":
-          -1,
-        "matrix.call.stats.connection.resolution.remote.REMOTE_AUDIO_TRACK_ID.width":
-          -1,
-
-        "matrix.call.stats.connection.resolution.remote.REMOTE_VIDEO_TRACK_ID.height": 960,
-        "matrix.call.stats.connection.resolution.remote.REMOTE_VIDEO_TRACK_ID.width": 1080,
-      });
-    });
-    it("should flatter an Array object", () => {
-      const flatObject = {};
-      ObjectFlattener.flattenObjectRecursive(
-        statsReport.report.transport,
-        flatObject,
-        "matrix.call.stats.connection.transport.",
-        0,
-      );
-      expect(flatObject).toEqual({
-        "matrix.call.stats.connection.transport.0.ip":
-          "ff11::5fa:abcd:999c:c5c5:50000",
-        "matrix.call.stats.connection.transport.0.type": "udp",
-        "matrix.call.stats.connection.transport.0.localIp":
-          "2aaa:9999:2aaa:999:8888:2aaa:2aaa:7777:50000",
-        "matrix.call.stats.connection.transport.0.isFocus": true,
-        "matrix.call.stats.connection.transport.0.localCandidateType": "host",
-        "matrix.call.stats.connection.transport.0.remoteCandidateType": "host",
-        "matrix.call.stats.connection.transport.0.networkType": "ethernet",
-        "matrix.call.stats.connection.transport.0.rtt": "NaN",
-        "matrix.call.stats.connection.transport.1.ip": "10.10.10.2:22222",
-        "matrix.call.stats.connection.transport.1.type": "tcp",
-        "matrix.call.stats.connection.transport.1.localIp":
-          "10.10.10.100:33333",
-        "matrix.call.stats.connection.transport.1.isFocus": true,
-        "matrix.call.stats.connection.transport.1.localCandidateType": "srfx",
-        "matrix.call.stats.connection.transport.1.remoteCandidateType": "srfx",
-        "matrix.call.stats.connection.transport.1.networkType": "ethernet",
-        "matrix.call.stats.connection.transport.1.rtt": 0,
-      });
-    });
-  });
-
-  describe("on flattenReportObject Connection Stats", () => {
-    it("should flatten a Report to otel Attributes Object", () => {
-      expect(
-        ObjectFlattener.flattenReportObject(
-          "matrix.call.stats.connection",
-          statsReport.report,
-        ),
-      ).toEqual({
-        "matrix.call.stats.connection.callId": "callId",
-        "matrix.call.stats.connection.opponentMemberId": "opponentMemberId",
-        "matrix.call.stats.connection.bandwidth.download": 0,
-        "matrix.call.stats.connection.bandwidth.upload": 426,
-        "matrix.call.stats.connection.bitrate.audio.download": 0,
-        "matrix.call.stats.connection.bitrate.audio.upload": 124,
-        "matrix.call.stats.connection.bitrate.download": 0,
-        "matrix.call.stats.connection.bitrate.upload": 426,
-        "matrix.call.stats.connection.bitrate.video.download": 0,
-        "matrix.call.stats.connection.bitrate.video.upload": 302,
-        "matrix.call.stats.connection.codec.local.LOCAL_AUDIO_TRACK_ID": "opus",
-        "matrix.call.stats.connection.codec.local.LOCAL_VIDEO_TRACK_ID": "v8",
-        "matrix.call.stats.connection.codec.remote.REMOTE_AUDIO_TRACK_ID":
-          "opus",
-        "matrix.call.stats.connection.codec.remote.REMOTE_VIDEO_TRACK_ID": "v9",
-        "matrix.call.stats.connection.framerate.local.LOCAL_AUDIO_TRACK_ID": 0,
-        "matrix.call.stats.connection.framerate.local.LOCAL_VIDEO_TRACK_ID": 30,
-        "matrix.call.stats.connection.framerate.remote.REMOTE_AUDIO_TRACK_ID": 0,
-        "matrix.call.stats.connection.framerate.remote.REMOTE_VIDEO_TRACK_ID": 60,
-        "matrix.call.stats.connection.jitter.REMOTE_AUDIO_TRACK_ID": 2,
-        "matrix.call.stats.connection.jitter.REMOTE_VIDEO_TRACK_ID": 50,
-        "matrix.call.stats.connection.packetLoss.download": 0,
-        "matrix.call.stats.connection.packetLoss.total": 0,
-        "matrix.call.stats.connection.packetLoss.upload": 0,
-        "matrix.call.stats.connection.resolution.local.LOCAL_AUDIO_TRACK_ID.height":
-          -1,
-        "matrix.call.stats.connection.resolution.local.LOCAL_AUDIO_TRACK_ID.width":
-          -1,
-        "matrix.call.stats.connection.resolution.local.LOCAL_VIDEO_TRACK_ID.height": 460,
-        "matrix.call.stats.connection.resolution.local.LOCAL_VIDEO_TRACK_ID.width": 780,
-        "matrix.call.stats.connection.resolution.remote.REMOTE_AUDIO_TRACK_ID.height":
-          -1,
-        "matrix.call.stats.connection.resolution.remote.REMOTE_AUDIO_TRACK_ID.width":
-          -1,
-        "matrix.call.stats.connection.resolution.remote.REMOTE_VIDEO_TRACK_ID.height": 960,
-        "matrix.call.stats.connection.resolution.remote.REMOTE_VIDEO_TRACK_ID.width": 1080,
-        "matrix.call.stats.connection.transport.0.ip":
-          "ff11::5fa:abcd:999c:c5c5:50000",
-        "matrix.call.stats.connection.transport.0.type": "udp",
-        "matrix.call.stats.connection.transport.0.localIp":
-          "2aaa:9999:2aaa:999:8888:2aaa:2aaa:7777:50000",
-        "matrix.call.stats.connection.transport.0.isFocus": true,
-        "matrix.call.stats.connection.transport.0.localCandidateType": "host",
-        "matrix.call.stats.connection.transport.0.remoteCandidateType": "host",
-        "matrix.call.stats.connection.transport.0.networkType": "ethernet",
-        "matrix.call.stats.connection.transport.0.rtt": "NaN",
-        "matrix.call.stats.connection.transport.1.ip": "10.10.10.2:22222",
-        "matrix.call.stats.connection.transport.1.type": "tcp",
-        "matrix.call.stats.connection.transport.1.localIp":
-          "10.10.10.100:33333",
-        "matrix.call.stats.connection.transport.1.isFocus": true,
-        "matrix.call.stats.connection.transport.1.localCandidateType": "srfx",
-        "matrix.call.stats.connection.transport.1.remoteCandidateType": "srfx",
-        "matrix.call.stats.connection.transport.1.networkType": "ethernet",
-        "matrix.call.stats.connection.transport.1.rtt": 0,
-        "matrix.call.stats.connection.audioConcealment.REMOTE_AUDIO_TRACK_ID.concealedAudio": 0,
-        "matrix.call.stats.connection.audioConcealment.REMOTE_AUDIO_TRACK_ID.totalAudioDuration": 0,
-        "matrix.call.stats.connection.audioConcealment.REMOTE_VIDEO_TRACK_ID.concealedAudio": 0,
-        "matrix.call.stats.connection.audioConcealment.REMOTE_VIDEO_TRACK_ID.totalAudioDuration": 0,
-        "matrix.call.stats.connection.totalAudioConcealment.concealedAudio": 0,
-        "matrix.call.stats.connection.totalAudioConcealment.totalAudioDuration": 0,
-      });
-    });
-  });
-
-  describe("on flattenByteSendStatsReportObject", () => {
-    const byteSentStatsReport = new Map<
-      string,
-      number
-    >() as ByteSentStatsReport;
-    byteSentStatsReport.callId = "callId";
-    byteSentStatsReport.opponentMemberId = "opponentMemberId";
-    byteSentStatsReport.set("4aa92608-04c6-428e-8312-93e17602a959", 132093);
-    byteSentStatsReport.set("a08e4237-ee30-4015-a932-b676aec894b1", 913448);
-
-    it("should flatten a Report to otel Attributes Object", () => {
-      expect(
-        ObjectFlattener.flattenReportObject(
-          "matrix.call.stats.bytesSend",
-          byteSentStatsReport,
-        ),
-      ).toEqual({
-        "matrix.call.stats.bytesSend.4aa92608-04c6-428e-8312-93e17602a959": 132093,
-        "matrix.call.stats.bytesSend.a08e4237-ee30-4015-a932-b676aec894b1": 913448,
-      });
-      expect(byteSentStatsReport.callId).toEqual("callId");
-      expect(byteSentStatsReport.opponentMemberId).toEqual("opponentMemberId");
-    });
-  });
-});

src/otel/ObjectFlattener.ts

@@ -1,100 +0,0 @@
-/*
-Copyright 2023, 2024 New Vector Ltd.
-
-SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
-Please see LICENSE in the repository root for full details.
-*/
-import { type Attributes } from "@opentelemetry/api";
-import { type VoipEvent } from "matrix-js-sdk/lib/webrtc/call";
-import { type GroupCallStatsReport } from "matrix-js-sdk/lib/webrtc/groupCall";
-import {
-  type ByteSentStatsReport,
-  type ConnectionStatsReport,
-  type SummaryStatsReport,
-} from "matrix-js-sdk/lib/webrtc/stats/statsReport";
-
-export class ObjectFlattener {
-  public static flattenReportObject(
-    prefix: string,
-    report: ConnectionStatsReport | ByteSentStatsReport,
-  ): Attributes {
-    const flatObject = {};
-    ObjectFlattener.flattenObjectRecursive(report, flatObject, `${prefix}.`, 0);
-    return flatObject;
-  }
-
-  public static flattenByteSentStatsReportObject(
-    statsReport: GroupCallStatsReport<ByteSentStatsReport>,
-  ): Attributes {
-    const flatObject = {};
-    ObjectFlattener.flattenObjectRecursive(
-      statsReport.report,
-      flatObject,
-      "matrix.stats.bytesSent.",
-      0,
-    );
-    return flatObject;
-  }
-
-  public static flattenSummaryStatsReportObject(
-    statsReport: GroupCallStatsReport<SummaryStatsReport>,
-  ): Attributes {
-    const flatObject = {};
-    ObjectFlattener.flattenObjectRecursive(
-      statsReport.report,
-      flatObject,
-      "matrix.stats.summary.",
-      0,
-    );
-    return flatObject;
-  }
-
-  /* Flattens out an object into a single layer with components
-   * of the key separated by dots
-   */
-  public static flattenVoipEvent(event: VoipEvent): Attributes {
-    const flatObject = {};
-    ObjectFlattener.flattenObjectRecursive(
-      event as unknown as Record<string, unknown>, // XXX Types
-      flatObject,
-      "matrix.event.",
-      0,
-    );
-
-    return flatObject;
-  }
-
-  public static flattenObjectRecursive(
-    obj: object,
-    flatObject: Attributes,
-    prefix: string,
-    depth: number,
-  ): void {
-    if (depth > 10)
-      throw new Error(
-        "Depth limit exceeded: aborting VoipEvent recursion. Prefix is " +
-          prefix,
-      );
-    let entries;
-    if (obj instanceof Map) {
-      entries = obj.entries();
-    } else {
-      entries = Object.entries(obj);
-    }
-    for (const [k, v] of entries) {
-      if (["string", "number", "boolean"].includes(typeof v) || v === null) {
-        let value;
-        value = v === null ? "null" : v;
-        value = typeof v === "number" && Number.isNaN(v) ? "NaN" : value;
-        flatObject[prefix + k] = value;
-      } else if (typeof v === "object") {
-        ObjectFlattener.flattenObjectRecursive(
-          v,
-          flatObject,
-          prefix + k + ".",
-          depth + 1,
-        );
-      }
-    }
-  }
-}

src/otel/otel.test.ts

@@ -1,79 +0,0 @@
-/*
-Copyright 2025 New Vector Ltd.
-
-SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
-Please see LICENSE in the repository root for full details.
-*/
-
-import {
-  expect,
-  describe,
-  it,
-  vi,
-  beforeEach,
-  beforeAll,
-  afterAll,
-} from "vitest";
-
-import { ElementCallOpenTelemetry } from "./otel";
-import { mockConfig } from "../utils/test";
-
-describe("ElementCallOpenTelemetry", () => {
-  describe("embedded package", () => {
-    beforeAll(() => {
-      vi.stubEnv("VITE_PACKAGE", "embedded");
-    });
-
-    beforeEach(() => {
-      mockConfig({});
-    });
-
-    afterAll(() => {
-      vi.unstubAllEnvs();
-    });
-
-    it("does not create instance without config value", () => {
-      ElementCallOpenTelemetry.globalInit();
-      expect(ElementCallOpenTelemetry.instance?.isOtlpEnabled).toBe(false);
-    });
-
-    it("ignores config value and does not create instance", () => {
-      mockConfig({
-        opentelemetry: {
-          collector_url: "https://collector.example.com.localhost",
-        },
-      });
-      ElementCallOpenTelemetry.globalInit();
-      expect(ElementCallOpenTelemetry.instance?.isOtlpEnabled).toBe(false);
-    });
-  });
-
-  describe("full package", () => {
-    beforeAll(() => {
-      vi.stubEnv("VITE_PACKAGE", "full");
-    });
-
-    beforeEach(() => {
-      mockConfig({});
-    });
-
-    afterAll(() => {
-      vi.unstubAllEnvs();
-    });
-
-    it("does not create instance without config value", () => {
-      ElementCallOpenTelemetry.globalInit();
-      expect(ElementCallOpenTelemetry.instance?.isOtlpEnabled).toBe(false);
-    });
-
-    it("creates instance with config value", () => {
-      mockConfig({
-        opentelemetry: {
-          collector_url: "https://collector.example.com.localhost",
-        },
-      });
-      ElementCallOpenTelemetry.globalInit();
-      expect(ElementCallOpenTelemetry.instance?.isOtlpEnabled).toBe(true);
-    });
-  });
-});

src/otel/otel.ts

@@ -1,117 +0,0 @@
-/*
-Copyright 2023, 2024 New Vector Ltd.
-
-SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
-Please see LICENSE in the repository root for full details.
-*/
-
-import {
-  SimpleSpanProcessor,
-  type SpanProcessor,
-} from "@opentelemetry/sdk-trace-base";
-import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
-import { WebTracerProvider } from "@opentelemetry/sdk-trace-web";
-import opentelemetry, { type Tracer } from "@opentelemetry/api";
-import { resourceFromAttributes } from "@opentelemetry/resources";
-import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions";
-import { logger } from "matrix-js-sdk/lib/logger";
-
-import { PosthogSpanProcessor } from "../analytics/PosthogSpanProcessor";
-import { Config } from "../config/Config";
-import { RageshakeSpanProcessor } from "../analytics/RageshakeSpanProcessor";
-import { getRageshakeSubmitUrl } from "../settings/submit-rageshake";
-
-const SERVICE_NAME = "element-call";
-
-let sharedInstance: ElementCallOpenTelemetry;
-
-export class ElementCallOpenTelemetry {
-  private _provider: WebTracerProvider;
-  private _tracer: Tracer;
-  private otlpExporter?: OTLPTraceExporter;
-  public readonly rageshakeProcessor?: RageshakeSpanProcessor;
-
-  public static globalInit(): void {
-    // this is only supported in the full package as the is currently no support for passing in the collector URL from the widget host
-    const collectorUrl =
-      import.meta.env.VITE_PACKAGE === "full"
-        ? Config.get().opentelemetry?.collector_url
-        : undefined;
-    // we always enable opentelemetry in general. We only enable the OTLP
-    // collector if a URL is defined (and in future if another setting is defined)
-    // Posthog reporting is enabled or disabled
-    // within the posthog code.
-    const shouldEnableOtlp = Boolean(collectorUrl);
-
-    if (!sharedInstance || sharedInstance.isOtlpEnabled !== shouldEnableOtlp) {
-      logger.info("(Re)starting OpenTelemetry debug reporting");
-      sharedInstance?.dispose();
-
-      sharedInstance = new ElementCallOpenTelemetry(
-        collectorUrl,
-        getRageshakeSubmitUrl(),
-      );
-    }
-  }
-
-  public static get instance(): ElementCallOpenTelemetry {
-    return sharedInstance;
-  }
-
-  private constructor(
-    collectorUrl: string | undefined,
-    rageshakeUrl: string | undefined,
-  ) {
-    const spanProcessors: SpanProcessor[] = [];
-
-    if (collectorUrl) {
-      logger.info("Enabling OTLP collector with URL " + collectorUrl);
-      this.otlpExporter = new OTLPTraceExporter({
-        url: collectorUrl,
-      });
-      spanProcessors.push(new SimpleSpanProcessor(this.otlpExporter));
-    } else {
-      logger.info("OTLP collector disabled");
-    }
-
-    if (rageshakeUrl) {
-      this.rageshakeProcessor = new RageshakeSpanProcessor();
-      spanProcessors.push(this.rageshakeProcessor);
-    }
-
-    spanProcessors.push(new PosthogSpanProcessor());
-
-    this._provider = new WebTracerProvider({
-      resource: resourceFromAttributes({
-        // This is how we can make Jaeger show a reasonable service in the dropdown on the left.
-        [ATTR_SERVICE_NAME]: SERVICE_NAME,
-      }),
-      spanProcessors,
-    });
-
-    opentelemetry.trace.setGlobalTracerProvider(this._provider);
-    this._tracer = opentelemetry.trace.getTracer(
-      // This is not the serviceName shown in jaeger
-      "my-element-call-otl-tracer",
-    );
-  }
-
-  public dispose(): void {
-    opentelemetry.trace.disable();
-    this._provider?.shutdown().catch((e) => {
-      logger.error("Failed to shutdown OpenTelemetry", e);
-    });
-  }
-
-  public get isOtlpEnabled(): boolean {
-    return Boolean(this.otlpExporter);
-  }
-
-  public get tracer(): Tracer {
-    return this._tracer;
-  }
-
-  public get provider(): WebTracerProvider {
-    return this._provider;
-  }
-}

src/reactions/ReactionsReader.ts

@@ -135,10 +135,10 @@ export class ReactionsReader {
   }
 
   /**
-   * Fetchest any hand wave reactions by the given sender on the given
+   * Fetches any hand wave reactions by the given sender on the given
    * membership event.
-   * @param membershipEventId
-   * @param expectedSender
+   * @param membershipEventId - The user membership event id.
+   * @param expectedSender - The expected sender of the reaction.
    * @returns A MatrixEvent if one was found.
    */
   private getLastReactionEvent(

src/room/GroupCallView.tsx

@@ -160,6 +160,7 @@ export const GroupCallView: FC<Props> = ({
   }, [rtcSession]);
 
   // TODO move this into the callViewModel LocalMembership.ts
+  // We might actually not need this at all. Since we get into fatalError on those errors already?
   useTypedEventEmitter(
     rtcSession,
     MatrixRTCSessionEvent.MembershipManagerError,
@@ -313,6 +314,7 @@ export const GroupCallView: FC<Props> = ({
 
   const navigate = useNavigate();
 
+  // TODO split this into leave and onDisconnect
   const onLeft = useCallback(
     (
       reason: "timeout" | "user" | "allOthersLeft" | "decline" | "error",

src/room/InCallView.tsx

@@ -24,7 +24,7 @@ import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc";
 import classNames from "classnames";
 import { BehaviorSubject, map } from "rxjs";
 import { useObservable } from "observable-hooks";
-import { logger } from "matrix-js-sdk/lib/logger";
+import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
 import {
   VoiceCallSolidIcon,
   VolumeOnSolidIcon,
@@ -48,7 +48,6 @@ import { useCallViewKeyboardShortcuts } from "../useCallViewKeyboardShortcuts";
 import { ElementWidgetActions, widget } from "../widget";
 import styles from "./InCallView.module.css";
 import { GridTile } from "../tile/GridTile";
-import { type OTelGroupCallMembership } from "../otel/OTelGroupCallMembership";
 import { SettingsModal, defaultSettingsTab } from "../settings/SettingsModal";
 import { useRageshakeRequestModal } from "../settings/submit-rageshake";
 import { RageshakeRequestModal } from "./RageshakeRequestModal";
@@ -88,6 +87,7 @@ import { ReactionsOverlay } from "./ReactionsOverlay";
 import { CallEventAudioRenderer } from "./CallEventAudioRenderer";
 import {
   debugTileLayout as debugTileLayoutSetting,
+  matrixRTCMode as matrixRTCModeSetting,
   useSetting,
 } from "../settings/settings";
 import { ReactionsReader } from "../reactions/ReactionsReader";
@@ -109,10 +109,14 @@ import { useTrackProcessorObservable$ } from "../livekit/TrackProcessorContext.t
 import { type Layout } from "../state/layout-types.ts";
 import { ObservableScope } from "../state/ObservableScope.ts";
 
+const logger = rootLogger.getChild("[InCallView]");
+
 const maxTapDurationMs = 400;
 
-export interface ActiveCallProps
-  extends Omit<InCallViewProps, "vm" | "livekitRoom" | "connState"> {
+export interface ActiveCallProps extends Omit<
+  InCallViewProps,
+  "vm" | "livekitRoom" | "connState"
+> {
   e2eeSystem: EncryptionSystem;
   // TODO refactor those reasons into an enum
   onLeft: (
@@ -127,6 +131,7 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
   const mediaDevices = useMediaDevices();
   const trackProcessorState$ = useTrackProcessorObservable$();
   useEffect(() => {
+    logger.info("START CALL VIEW SCOPE");
     const scope = new ObservableScope();
     const reactionsReader = new ReactionsReader(scope, props.rtcSession);
     const { autoLeaveWhenOthersLeft, waitForCallPickup, sendNotificationType } =
@@ -141,6 +146,7 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
         encryptionSystem: props.e2eeSystem,
         autoLeaveWhenOthersLeft,
         waitForCallPickup: waitForCallPickup && sendNotificationType === "ring",
+        matrixRTCMode$: matrixRTCModeSetting.value$,
       },
       reactionsReader.raisedHands$,
       reactionsReader.reactions$,
@@ -151,7 +157,9 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
     setVm(vm);
 
     vm.leave$.pipe(scope.bind()).subscribe(props.onLeft);
+
     return (): void => {
+      logger.info("END CALL VIEW SCOPE");
       scope.end();
     };
   }, [
@@ -182,7 +190,6 @@ export interface InCallViewProps {
   matrixRoom: MatrixRoom;
   muteStates: MuteStates;
   header: HeaderStyle;
-  otelGroupCallMembership?: OTelGroupCallMembership;
   onShareClick: (() => void) | null;
 }
 
@@ -253,7 +260,7 @@ export const InCallView: FC<InCallViewProps> = ({
     () => void toggleRaisedHand(),
   );
 
-  const audioParticipants = useBehavior(vm.audioParticipants$);
+  const audioParticipants = useBehavior(vm.livekitRoomItems$);
   const participantCount = useBehavior(vm.participantCount$);
   const reconnecting = useBehavior(vm.reconnecting$);
   const windowMode = useBehavior(vm.windowMode$);
@@ -270,7 +277,10 @@ export const InCallView: FC<InCallViewProps> = ({
   const ringOverlay = useBehavior(vm.ringOverlay$);
   const fatalCallError = useBehavior(vm.fatalError$);
   // Stop the rendering and throw for the error boundary
-  if (fatalCallError) throw fatalCallError;
+  if (fatalCallError) {
+    logger.debug("fatalCallError stop rendering", fatalCallError);
+    throw fatalCallError;
+  }
 
   // We need to set the proper timings on the animation based upon the sound length.
   const ringDuration = pickupPhaseAudio?.soundDuration["waiting"] ?? 1;

src/room/LobbyView.tsx

@@ -79,9 +79,9 @@ export const LobbyView: FC<Props> = ({
   waitingForInvite,
 }) => {
   useEffect(() => {
-    logger.info("[Lifecycle] GroupCallView Component mounted");
+    logger.info("[Lifecycle] LobbyView Component mounted");
     return (): void => {
-      logger.info("[Lifecycle] GroupCallView Component unmounted");
+      logger.info("[Lifecycle] LobbyView Component unmounted");
     };
   }, []);
 

src/room/RoomPage.tsx

@@ -20,8 +20,6 @@ import {
   CheckIcon,
   UnknownSolidIcon,
 } from "@vector-im/compound-design-tokens/assets/web/icons";
-import { useObservable } from "observable-hooks";
-import { map } from "rxjs";
 
 import { useClientLegacy } from "../ClientContext";
 import { ErrorPage, FullScreenView, LoadingPage } from "../FullScreenView";
@@ -44,10 +42,12 @@ import { ErrorView } from "../ErrorView";
 import { useMediaDevices } from "../MediaDevicesContext";
 import { MuteStates } from "../state/MuteStates";
 import { ObservableScope } from "../state/ObservableScope";
+import { calculateInitialMuteState } from "../state/initialMuteState.ts";
 
 export const RoomPage: FC = () => {
+  const urlParams = useUrlParams();
   const { confineToRoom, appPrompt, preload, header, displayName, skipLobby } =
-    useUrlParams();
+    urlParams;
   const { t } = useTranslation();
   const { roomAlias, roomId, viaServers } = useRoomIdentifier();
 
@@ -68,15 +68,23 @@ export const RoomPage: FC = () => {
 
   const devices = useMediaDevices();
   const [muteStates, setMuteStates] = useState<MuteStates | null>(null);
-  const joined$ = useObservable(
-    (inputs$) => inputs$.pipe(map(([joined]) => joined)),
-    [joined],
-  );
+
   useEffect(() => {
     const scope = new ObservableScope();
-    setMuteStates(new MuteStates(scope, devices, joined$));
+    setMuteStates(
+      new MuteStates(
+        scope,
+        devices,
+        calculateInitialMuteState(
+          urlParams,
+          import.meta.env.VITE_PACKAGE,
+          window.location.hostname,
+          Config.get().trust_localhost_for_mute_state,
+        ),
+      ),
+    );
     return (): void => scope.end();
-  }, [devices, joined$]);
+  }, [devices, urlParams]);
 
   useEffect(() => {
     // If we've finished loading, are not already authed and we've been given a display name as

src/room/__snapshots__/InCallView.test.tsx.snap

@@ -34,7 +34,7 @@ exports[`InCallView > rendering > renders 1`] = `
               tabindex="0"
             >
               <svg
-                aria-labelledby="«r0»"
+                aria-labelledby="_r_0_"
                 class="lock"
                 data-encrypted="false"
                 fill="currentColor"
@@ -286,7 +286,8 @@ exports[`InCallView > rendering > renders 1`] = `
       >
         <button
           aria-disabled="true"
-          aria-labelledby="«r8»"
+          aria-label="Unmute microphone"
+          aria-labelledby="_r_8_"
           class="_button_vczzf_8 _has-icon_vczzf_57 _icon-only_vczzf_50"
           data-kind="primary"
           data-size="lg"
@@ -309,7 +310,8 @@ exports[`InCallView > rendering > renders 1`] = `
         </button>
         <button
           aria-disabled="true"
-          aria-labelledby="«rd»"
+          aria-label="Start video"
+          aria-labelledby="_r_d_"
           class="_button_vczzf_8 _has-icon_vczzf_57 _icon-only_vczzf_50"
           data-kind="primary"
           data-size="lg"
@@ -331,7 +333,7 @@ exports[`InCallView > rendering > renders 1`] = `
           </svg>
         </button>
         <button
-          aria-labelledby="«ri»"
+          aria-labelledby="_r_i_"
           class="_button_vczzf_8 _has-icon_vczzf_57 _icon-only_vczzf_50"
           data-kind="secondary"
           data-size="lg"
@@ -352,7 +354,8 @@ exports[`InCallView > rendering > renders 1`] = `
           </svg>
         </button>
         <button
-          aria-labelledby="«rn»"
+          aria-label="End call"
+          aria-labelledby="_r_n_"
           class="_button_vczzf_8 endCall _has-icon_vczzf_57 _icon-only_vczzf_50 _destructive_vczzf_107"
           data-kind="primary"
           data-size="lg"
@@ -378,7 +381,7 @@ exports[`InCallView > rendering > renders 1`] = `
         class="toggle layout"
       >
         <input
-          aria-labelledby="«rs»"
+          aria-labelledby="_r_s_"
           name="layout"
           type="radio"
           value="spotlight"
@@ -396,7 +399,7 @@ exports[`InCallView > rendering > renders 1`] = `
           />
         </svg>
         <input
-          aria-labelledby="«r11»"
+          aria-labelledby="_r_11_"
           checked=""
           name="layout"
           type="radio"

src/room/useLoadGroupCall.ts

@@ -106,22 +106,18 @@ async function joinRoomAfterInvite(
 
 export class CallTerminatedMessage extends Error {
   /**
+   * Creates a new CallTerminatedMessage.
+   *
+   * @param icon The icon to display with the message
    * @param messageTitle The title of the call ended screen message (translated)
+   * @param messageBody The message explaining the kind of termination
+   * (kick, ban, knock reject, etc.) (translated)
+   * @param reason  The user-provided reason for the termination (kick/ban)
    */
   public constructor(
-    /**
-     * The icon to display with the message.
-     */
     public readonly icon: ComponentType<SVGAttributes<SVGElement>>,
     messageTitle: string,
-    /**
-     * The message explaining the kind of termination (kick, ban, knock reject,
-     * etc.) (translated)
-     */
     public readonly messageBody: string,
-    /**
-     * The user-provided reason for the termination (kick/ban)
-     */
     public readonly reason?: string,
   ) {
     super(messageTitle);

src/settings/__snapshots__/DeveloperSettingsTab.test.tsx.snap

@@ -24,7 +24,7 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
       class="field inputField"
     >
       <input
-        aria-describedby="«r1»"
+        aria-describedby="_r_1_"
         id="duplicateTiles"
         min="0"
         type="number"
@@ -44,7 +44,7 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
       class="field checkboxField"
     >
       <input
-        aria-describedby="«r2»"
+        aria-describedby="_r_2_"
         id="debugTileLayout"
         type="checkbox"
       />
@@ -81,7 +81,7 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
       class="field checkboxField"
     >
       <input
-        aria-describedby="«r3»"
+        aria-describedby="_r_3_"
         id="showConnectionStats"
         type="checkbox"
       />
@@ -118,7 +118,7 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
       class="field checkboxField"
     >
       <input
-        aria-describedby="«r4»"
+        aria-describedby="_r_4_"
         id="muteAllAudio"
         type="checkbox"
       />
@@ -156,7 +156,7 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
       class="field checkboxField"
     >
       <input
-        aria-describedby="«r5»"
+        aria-describedby="_r_5_"
         id="alwaysShowIphoneEarpiece"
         type="checkbox"
       />
@@ -195,7 +195,7 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
     >
       <label
         class="_label_19upo_59"
-        for="radix-«r6»"
+        for="radix-_r_6_"
       >
         Custom Livekit-url
       </label>
@@ -203,9 +203,9 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
         class="_controls_17lij_8"
       >
         <input
-          aria-describedby="radix-«r7»"
+          aria-describedby="radix-_r_7_"
           class="_control_sqdq4_10"
-          id="radix-«r6»"
+          id="radix-_r_6_"
           name="input"
           title=""
           value=""
@@ -213,7 +213,7 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
       </div>
       <span
         class="_message_19upo_85 _help-message_19upo_91"
-        id="radix-«r7»"
+        id="radix-_r_7_"
       >
         Currently, no overwrite is set. Url from well-known or config is used.
       </span>
@@ -237,11 +237,11 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
           class="_container_1e0uz_10"
         >
           <input
-            aria-describedby="radix-«r9» radix-«rb» radix-«rd»"
+            aria-describedby="radix-_r_9_ radix-_r_b_ radix-_r_d_"
             checked=""
             class="_input_1e0uz_18"
-            id="radix-«r8»"
-            name="«r0»"
+            id="radix-_r_8_"
+            name="_r_0_"
             title=""
             type="radio"
             value="legacy"
@@ -256,13 +256,13 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
       >
         <label
           class="_label_19upo_59"
-          for="radix-«r8»"
+          for="radix-_r_8_"
         >
           Legacy: state events & oldest membership SFU
         </label>
         <span
           class="_message_19upo_85 _help-message_19upo_91"
-          id="radix-«r9»"
+          id="radix-_r_9_"
         >
           Compatible with old versions of EC that do not support multi SFU
         </span>
@@ -278,10 +278,10 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
           class="_container_1e0uz_10"
         >
           <input
-            aria-describedby="radix-«r9» radix-«rb» radix-«rd»"
+            aria-describedby="radix-_r_9_ radix-_r_b_ radix-_r_d_"
             class="_input_1e0uz_18"
-            id="radix-«ra»"
-            name="«r0»"
+            id="radix-_r_a_"
+            name="_r_0_"
             title=""
             type="radio"
             value="compatibil"
@@ -296,13 +296,13 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
       >
         <label
           class="_label_19upo_59"
-          for="radix-«ra»"
+          for="radix-_r_a_"
         >
           Compatibility: state events & multi SFU
         </label>
         <span
           class="_message_19upo_85 _help-message_19upo_91"
-          id="radix-«rb»"
+          id="radix-_r_b_"
         >
           Compatible with homeservers that do not support sticky events (but all other EC clients are v0.17.0 or later)
         </span>
@@ -318,10 +318,10 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
           class="_container_1e0uz_10"
         >
           <input
-            aria-describedby="radix-«r9» radix-«rb» radix-«rd»"
+            aria-describedby="radix-_r_9_ radix-_r_b_ radix-_r_d_"
             class="_input_1e0uz_18"
-            id="radix-«rc»"
-            name="«r0»"
+            id="radix-_r_c_"
+            name="_r_0_"
             title=""
             type="radio"
             value="matrix_2_0"
@@ -336,13 +336,13 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
       >
         <label
           class="_label_19upo_59"
-          for="radix-«rc»"
+          for="radix-_r_c_"
         >
           Matrix 2.0: sticky events & multi SFU
         </label>
         <span
           class="_message_19upo_85 _help-message_19upo_91"
-          id="radix-«rd»"
+          id="radix-_r_d_"
         >
           Compatible only with homservers supporting sticky events and all EC clients v0.17.0 or later
         </span>

src/settings/rageshake.ts

@@ -99,7 +99,7 @@ class ConsoleLogger extends EventEmitter {
 
   /**
    * Returns the log lines to flush to disk and empties the internal log buffer
-   * @return {string} \n delimited log lines
+   * @return \n delimited log lines
    */
   public popLogs(): string {
     const logsToFlush = this.logs;
@@ -109,7 +109,7 @@ class ConsoleLogger extends EventEmitter {
 
   /**
    * Returns lines currently in the log buffer without removing them
-   * @return {string} \n delimited log lines
+   * @return \n delimited log lines
    */
   public peekLogs(): string {
     return this.logs;
@@ -139,7 +139,7 @@ class IndexedDBLogStore {
   }
 
   /**
-   * @return {Promise} Resolves when the store is ready.
+   * @return Resolves when the store is ready.
    */
   public async connect(): Promise<void> {
     const req = this.indexedDB.open("logs");
@@ -219,7 +219,7 @@ class IndexedDBLogStore {
    * This guarantees that we will always eventually do a flush when flush() is
    * called.
    *
-   * @return {Promise} Resolved when the logs have been flushed.
+   * @return Resolved when the logs have been flushed.
    */
   public flush = async (): Promise<void> => {
     // check if a flush() operation is ongoing
@@ -270,7 +270,7 @@ class IndexedDBLogStore {
    * returned are deleted at the same time, so this can be called at startup
    * to do house-keeping to keep the logs from growing too large.
    *
-   * @return {Promise<Object[]>} Resolves to an array of objects. The array is
+   * @return Resolves to an array of objects. The array is
    * sorted in time (oldest first) based on when the log file was created (the
    * log ID). The objects have said log ID in an "id" field and "lines" which
    * is a big string with all the new-line delimited logs.
@@ -421,12 +421,12 @@ class IndexedDBLogStore {
 
 /**
  * Helper method to collect results from a Cursor and promiseify it.
- * @param {ObjectStore|Index} store The store to perform openCursor on.
- * @param {IDBKeyRange=} keyRange Optional key range to apply on the cursor.
- * @param {Function} resultMapper A function which is repeatedly called with a
+ * @param store - The store to perform openCursor on.
+ * @param keyRange - Optional key range to apply on the cursor.
+ * @param resultMapper - A function which is repeatedly called with a
  * Cursor.
  * Return the data you want to keep.
- * @return {Promise<T[]>} Resolves to an array of whatever you returned from
+ * @return Resolves to an array of whatever you returned from
  * resultMapper.
  */
 async function selectQuery<T>(
@@ -464,9 +464,7 @@ declare global {
 /**
  * Configure rage shaking support for sending bug reports.
  * Modifies globals.
- * @param {boolean} setUpPersistence When true (default), the persistence will
- * be set up immediately for the logs.
- * @return {Promise} Resolves when set up.
+ * @return Resolves when set up.
  */
 export async function init(): Promise<void> {
   global.mx_rage_logger = new ConsoleLogger();
@@ -503,7 +501,7 @@ export async function init(): Promise<void> {
 /**
  * Try to start up the rageshake storage for logs. If not possible (client unsupported)
  * then this no-ops.
- * @return {Promise} Resolves when complete.
+ * @return Resolves when complete.
  */
 async function tryInitStorage(): Promise<void> {
   if (global.mx_rage_initStoragePromise) {
@@ -536,7 +534,7 @@ async function tryInitStorage(): Promise<void> {
 /**
  * Get a recent snapshot of the logs, ready for attaching to a bug report
  *
- * @return {LogEntry[]}  list of log data
+ * @return list of log data
  */
 export async function getLogsForReport(): Promise<LogEntry[]> {
   if (!global.mx_rage_logger) {

src/settings/submit-rageshake.ts

@@ -17,7 +17,6 @@ import { type CryptoApi } from "matrix-js-sdk/lib/crypto-api";
 import { getLogsForReport } from "./rageshake";
 import { useClient } from "../ClientContext";
 import { Config } from "../config/Config";
-import { ElementCallOpenTelemetry } from "../otel/otel";
 import { type RageshakeRequestModal } from "../room/RageshakeRequestModal";
 import { getUrlParams } from "../UrlParams";
 
@@ -274,14 +273,6 @@ export function useSubmitRageshake(
           for (const entry of logs) {
             body.append("compressed-log", await gzip(entry.lines), entry.id);
           }
-
-          body.append(
-            "file",
-            await gzip(
-              ElementCallOpenTelemetry.instance.rageshakeProcessor!.dump(),
-            ),
-            "traces.json.gz",
-          );
         }
 
         if (opts.rageshakeRequestId) {

src/state/CallViewModel/CallNotificationLifecycle.ts

@@ -81,7 +81,7 @@ export interface Props {
   localUser: { deviceId: string; userId: string };
 }
 /**
- * @returns {callPickupState$, autoLeave$}
+ * @returns two observables:
  * `callPickupState$` The current call pickup state of the call.
  *  - "unknown": The client has not yet sent the notification event. We don't know if it will because it first needs to send its own membership.
  *     Then we can conclude if we were the first one to join or not.

src/state/CallViewModel/CallViewModel.test.ts

@@ -60,7 +60,8 @@ import {
 import { MediaDevices } from "../MediaDevices.ts";
 import { getValue } from "../../utils/observable.ts";
 import { type Behavior, constant } from "../Behavior.ts";
-import { withCallViewModel } from "./CallViewModelTestUtils.ts";
+import { withCallViewModel as withCallViewModelInMode } from "./CallViewModelTestUtils.ts";
+import { MatrixRTCMode } from "../../settings/settings.ts";
 
 vi.mock("rxjs", async (importOriginal) => ({
   ...(await importOriginal()),
@@ -229,7 +230,13 @@ function mockRingEvent(
 // need a value to fill in for them when emitting notifications
 const mockLegacyRingEvent = {} as { event_id: string } & ICallNotifyContent;
 
-describe("CallViewModel", () => {
+describe.each([
+  [MatrixRTCMode.Legacy],
+  [MatrixRTCMode.Compatibil],
+  [MatrixRTCMode.Matrix_2_0],
+])("CallViewModel (%s mode)", (mode) => {
+  const withCallViewModel = withCallViewModelInMode(mode);
+
   test("participants are retained during a focus switch", () => {
     withTestScheduler(({ behavior, expectObservable }) => {
       // Participants disappear on frame 2 and come back on frame 3
@@ -1249,7 +1256,9 @@ describe("CallViewModel", () => {
               rtcSession.membershipStatus = Status.Connected;
             },
             n: () => {
-              rtcSession.membershipStatus = Status.Reconnecting;
+              // NOTE: This was removed in https://github.com/matrix-org/matrix-js-sdk/pull/5103 accidentally.
+              // eslint-disable-next-line @typescript-eslint/no-explicit-any
+              rtcSession.membershipStatus = "Reconnecting" as any;
             },
           });
           schedule(probablyLeftMarbles, {

src/state/CallViewModel/CallViewModel.ts

@@ -15,6 +15,7 @@ import {
 } from "livekit-client";
 import { type Room as MatrixRoom } from "matrix-js-sdk";
 import {
+  catchError,
   combineLatest,
   distinctUntilChanged,
   filter,
@@ -52,11 +53,15 @@ import {
   ScreenShareViewModel,
   type UserMediaViewModel,
 } from "../MediaViewModel";
-import { accumulate, generateItems, pauseWhen } from "../../utils/observable";
+import {
+  accumulate,
+  filterBehavior,
+  generateItems,
+  pauseWhen,
+} from "../../utils/observable";
 import {
   duplicateTiles,
   MatrixRTCMode,
-  matrixRTCMode,
   playReactionsSound,
   showReactions,
 } from "../../settings/settings";
@@ -75,7 +80,7 @@ import {
 } from "../../reactions";
 import { shallowEquals } from "../../utils/array";
 import { type MediaDevices } from "../MediaDevices";
-import { type Behavior } from "../Behavior";
+import { constant, type Behavior } from "../Behavior";
 import { E2eeType } from "../../e2ee/e2eeType";
 import { MatrixKeyProvider } from "../../e2ee/matrixKeyProvider";
 import { type MuteStates } from "../MuteStates";
@@ -93,14 +98,14 @@ import {
   type SpotlightLandscapeLayoutMedia,
   type SpotlightPortraitLayoutMedia,
 } from "../layout-types.ts";
-import { type ElementCallError } from "../../utils/errors.ts";
+import { ElementCallError } from "../../utils/errors.ts";
 import { type ObservableScope } from "../ObservableScope.ts";
 import { createHomeserverConnected$ } from "./localMember/HomeserverConnected.ts";
 import {
   createLocalMembership$,
   enterRTCSession,
-  RTCBackendState,
-} from "./localMember/LocalMembership.ts";
+  TransportState,
+} from "./localMember/LocalMember.ts";
 import { createLocalTransport$ } from "./localMember/LocalTransport.ts";
 import {
   createMemberships$,
@@ -110,7 +115,9 @@ import { ECConnectionFactory } from "./remoteMembers/ConnectionFactory.ts";
 import { createConnectionManager$ } from "./remoteMembers/ConnectionManager.ts";
 import {
   createMatrixLivekitMembers$,
-  type MatrixLivekitMember,
+  type TaggedParticipant,
+  type LocalMatrixLivekitMember,
+  type RemoteMatrixLivekitMember,
 } from "./remoteMembers/MatrixLivekitMembers.ts";
 import {
   type AutoLeaveReason,
@@ -149,6 +156,8 @@ export interface CallViewModelOptions {
   connectionState$?: Behavior<ConnectionState>;
   /** Optional behavior overriding the computed window size, mainly for testing purposes. */
   windowSize$?: Behavior<{ width: number; height: number }>;
+  /** The version & compatibility mode of MatrixRTC that we should use. */
+  matrixRTCMode$?: Behavior<MatrixRTCMode>;
 }
 
 // Do not play any sounds if the participant count has exceeded this
@@ -174,7 +183,7 @@ interface LayoutScanState {
 }
 
 type MediaItem = UserMedia | ScreenShare;
-type AudioLivekitItem = {
+export type LivekitRoomItem = {
   livekitRoom: LivekitRoom;
   participants: string[];
   url: string;
@@ -197,8 +206,11 @@ export interface CallViewModel {
   callPickupState$: Behavior<
     "unknown" | "ringing" | "timeout" | "decline" | "success" | null
   >;
+  /** Observable that emits when the user should leave the call (hangup pressed, widget action, error).
+   * THIS DOES NOT LEAVE THE CALL YET. The only way to leave the call (send the hangup event) is by ending the scope.
+   */
   leave$: Observable<"user" | AutoLeaveReason>;
-  /** Call to initiate hangup. Use in conbination with connectino state track the async hangup process. */
+  /** Call to initiate hangup. Use in conbination with reconnectino state track the async hangup process. */
   hangup: () => void;
 
   // joining
@@ -250,7 +262,11 @@ export interface CallViewModel {
    */
   participantCount$: Behavior<number>;
   /** Participants sorted by livekit room so they can be used in the audio rendering */
-  audioParticipants$: Behavior<AudioLivekitItem[]>;
+  livekitRoomItems$: Behavior<LivekitRoomItem[]>;
+  userMedia$: Behavior<UserMedia[]>;
+  /** use the layout instead, this is just for the sdk export. */
+  matrixLivekitMembers$: Behavior<RemoteMatrixLivekitMember[]>;
+  localMatrixLivekitMember$: Behavior<LocalMatrixLivekitMember | null>;
   /** List of participants raising their hand */
   handsRaised$: Behavior<Record<string, RaisedHandInfo>>;
   /** List of reactions. Keys are: membership.membershipId (currently predefined as: `${membershipEvent.userId}:${membershipEvent.deviceId}`)*/
@@ -333,17 +349,15 @@ export interface CallViewModel {
     switch: () => void;
   } | null>;
 
-  // connection state
   /**
-   * Whether various media/event sources should pretend to be disconnected from
-   * all network input, even if their connection still technically works.
+   * Whether the app is currently reconnecting to the LiveKit server and/or setting the matrix rtc room state.
    */
-  // We do this when the app is in the 'reconnecting' state, because it might be
-  // that the LiveKit connection is still functional while the homeserver is
-  // down, for example, and we want to avoid making people worry that the app is
-  // in a split-brained state.
-  // DISCUSSION own membership manager ALSO this probably can be simplifis
   reconnecting$: Behavior<boolean>;
+
+  /**
+   * Shortcut for not requireing to parse and combine connectionState.matrix and connectionState.livekit
+   */
+  connected$: Behavior<boolean>;
 }
 
 /**
@@ -373,6 +387,8 @@ export function createCallViewModel$(
     options.encryptionSystem,
     matrixRTCSession,
   );
+  const matrixRTCMode$ =
+    options.matrixRTCMode$ ?? constant(MatrixRTCMode.Legacy);
 
   // Each hbar seperates a block of input variables required for the CallViewModel to function.
   // The outputs of this block is written under the hbar.
@@ -405,7 +421,7 @@ export function createCallViewModel$(
     client,
     roomId: matrixRoom.roomId,
     useOldestMember$: scope.behavior(
-      matrixRTCMode.value$.pipe(map((v) => v === MatrixRTCMode.Legacy)),
+      matrixRTCMode$.pipe(map((v) => v === MatrixRTCMode.Legacy)),
     ),
   });
 
@@ -425,7 +441,18 @@ export function createCallViewModel$(
     connectionFactory: connectionFactory,
     inputTransports$: scope.behavior(
       combineLatest(
-        [localTransport$, membershipsAndTransports.transports$],
+        [
+          localTransport$.pipe(
+            catchError((e: unknown) => {
+              logger.info(
+                "dont pass local transport to createConnectionManager$. localTransport$ threw an error",
+                e,
+              );
+              return of(null);
+            }),
+          ),
+          membershipsAndTransports.transports$,
+        ],
         (localTransport, transports) => {
           const localTransportAsArray = localTransport ? [localTransport] : [];
           return transports.mapInner((transports) => [
@@ -435,7 +462,7 @@ export function createCallViewModel$(
         },
       ),
     ),
-    logger: logger,
+    logger,
   });
 
   const matrixLivekitMembers$ = createMatrixLivekitMembers$({
@@ -446,7 +473,7 @@ export function createCallViewModel$(
   });
 
   const connectOptions$ = scope.behavior(
-    matrixRTCMode.value$.pipe(
+    matrixRTCMode$.pipe(
       map((mode) => ({
         encryptMedia: livekitKeyProvider !== undefined,
         // TODO. This might need to get called again on each change of matrixRTCMode...
@@ -457,13 +484,13 @@ export function createCallViewModel$(
 
   const localMembership = createLocalMembership$({
     scope: scope,
-    homeserverConnected$: createHomeserverConnected$(
+    homeserverConnected: createHomeserverConnected$(
       scope,
       client,
       matrixRTCSession,
     ),
     muteStates: muteStates,
-    joinMatrixRTC: async (transport: LivekitTransport) => {
+    joinMatrixRTC: (transport: LivekitTransport) => {
       return enterRTCSession(
         matrixRTCSession,
         transport,
@@ -478,7 +505,7 @@ export function createCallViewModel$(
         muteStates,
         trackProcessorState$,
         logger.getChild(
-          "[Publisher" + connection.transport.livekit_service_url + "]",
+          "[Publisher " + connection.transport.livekit_service_url + "]",
         ),
       );
     },
@@ -500,22 +527,21 @@ export function createCallViewModel$(
     ),
   );
 
-  const localMatrixLivekitMemberUninitialized = {
-    membership$: localRtcMembership$,
-    participant$: localMembership.participant$,
-    connection$: localMembership.connection$,
-    userId: userId,
-  };
-
-  const localMatrixLivekitMember$: Behavior<MatrixLivekitMember | null> =
+  const localMatrixLivekitMember$: Behavior<LocalMatrixLivekitMember | null> =
     scope.behavior(
       localRtcMembership$.pipe(
-        switchMap((membership) => {
-          if (!membership) return of(null);
-          return of(
-            // casting is save here since we know that localRtcMembership$ is !== null since we reached this case.
-            localMatrixLivekitMemberUninitialized as MatrixLivekitMember,
-          );
+        filterBehavior((membership) => membership !== null),
+        map((membership$) => {
+          if (membership$ === null) return null;
+          return {
+            membership$,
+            participant: {
+              type: "local" as const,
+              value$: localMembership.participant$,
+            },
+            connection$: localMembership.connection$,
+            userId,
+          };
         }),
       ),
     );
@@ -578,24 +604,16 @@ export function createCallViewModel$(
     ),
   );
 
-  /**
-   * Whether various media/event sources should pretend to be disconnected from
-   * all network input, even if their connection still technically works.
-   */
-  // We do this when the app is in the 'reconnecting' state, because it might be
-  // that the LiveKit connection is still functional while the homeserver is
-  // down, for example, and we want to avoid making people worry that the app is
-  // in a split-brained state.
-  // DISCUSSION own membership manager ALSO this probably can be simplifis
-  const reconnecting$ = localMembership.reconnecting$;
-
-  const audioParticipants$ = scope.behavior(
+  const livekitRoomItems$ = scope.behavior(
     matrixLivekitMembers$.pipe(
+      tap((val) => {
+        logger.debug("matrixLivekitMembers$ updated", val.value);
+      }),
       switchMap((membersWithEpoch) => {
         const members = membersWithEpoch.value;
         const a$ = combineLatest(
           members.map((member) =>
-            combineLatest([member.connection$, member.participant$]).pipe(
+            combineLatest([member.connection$, member.participant.value$]).pipe(
               map(([connection, participant]) => {
                 // do not render audio for local participant
                 if (!connection || !participant || participant.isLocal)
@@ -615,7 +633,7 @@ export function createCallViewModel$(
         return a$;
       }),
       map((members) =>
-        members.reduce<AudioLivekitItem[]>((acc, curr) => {
+        members.reduce<LivekitRoomItem[]>((acc, curr) => {
           if (!curr) return acc;
 
           const existing = acc.find((item) => item.url === curr.url);
@@ -636,7 +654,7 @@ export function createCallViewModel$(
   );
 
   const handsRaised$ = scope.behavior(
-    handsRaisedSubject$.pipe(pauseWhen(reconnecting$)),
+    handsRaisedSubject$.pipe(pauseWhen(localMembership.reconnecting$)),
   );
 
   const reactions$ = scope.behavior(
@@ -649,7 +667,7 @@ export function createCallViewModel$(
           ]),
         ),
       ),
-      pauseWhen(reconnecting$),
+      pauseWhen(localMembership.reconnecting$),
     ),
   );
 
@@ -673,7 +691,7 @@ export function createCallViewModel$(
           let localParticipantId: string | undefined = undefined;
           // add local member if available
           if (localMatrixLivekitMember) {
-            const { userId, participant$, connection$, membership$ } =
+            const { userId, participant, connection$, membership$ } =
               localMatrixLivekitMember;
             localParticipantId = `${userId}:${membership$.value.deviceId}`; // should be membership$.value.membershipID which is not optional
             // const participantId = membership$.value.membershipID;
@@ -684,7 +702,7 @@ export function createCallViewModel$(
                     dup,
                     localParticipantId,
                     userId,
-                    participant$,
+                    participant satisfies TaggedParticipant as TaggedParticipant, // Widen the type safely
                     connection$,
                   ],
                   data: undefined,
@@ -695,7 +713,7 @@ export function createCallViewModel$(
           // add remote members that are available
           for (const {
             userId,
-            participant$,
+            participant,
             connection$,
             membership$,
           } of matrixLivekitMembers) {
@@ -704,7 +722,7 @@ export function createCallViewModel$(
             // const participantId = membership$.value?.identity;
             for (let dup = 0; dup < 1 + duplicateTiles; dup++) {
               yield {
-                keys: [dup, participantId, userId, participant$, connection$],
+                keys: [dup, participantId, userId, participant, connection$],
                 data: undefined,
               };
             }
@@ -716,7 +734,7 @@ export function createCallViewModel$(
           dup,
           participantId,
           userId,
-          participant$,
+          participant,
           connection$,
         ) => {
           const livekitRoom$ = scope.behavior(
@@ -735,12 +753,12 @@ export function createCallViewModel$(
             scope,
             `${participantId}:${dup}`,
             userId,
-            participant$,
+            participant,
             options.encryptionSystem,
             livekitRoom$,
             focusUrl$,
             mediaDevices,
-            reconnecting$,
+            localMembership.reconnecting$,
             displayName$,
             matrixMemberMetadataStore.createAvatarUrlBehavior$(userId),
             handsRaised$.pipe(map((v) => v[participantId]?.time ?? null)),
@@ -945,11 +963,12 @@ export function createCallViewModel$(
     ),
   );
 
-  const hasRemoteScreenShares$: Observable<boolean> = spotlight$.pipe(
-    map((spotlight) =>
-      spotlight.some((vm) => !vm.local && vm instanceof ScreenShareViewModel),
+  const hasRemoteScreenShares$ = scope.behavior<boolean>(
+    spotlight$.pipe(
+      map((spotlight) =>
+        spotlight.some((vm) => !vm.local && vm instanceof ScreenShareViewModel),
+      ),
     ),
-    distinctUntilChanged(),
   );
 
   const pipEnabled$ = scope.behavior(setPipEnabled$, false);
@@ -1423,13 +1442,44 @@ export function createCallViewModel$(
   // reassigned here to make it publicly accessible
   const toggleScreenSharing = localMembership.toggleScreenSharing;
 
+  const errors$ = scope.behavior<{
+    transportError?: ElementCallError;
+    matrixError?: ElementCallError;
+    connectionError?: ElementCallError;
+    publishError?: ElementCallError;
+  } | null>(
+    localMembership.localMemberState$.pipe(
+      map((value) => {
+        const returnObject: {
+          transportError?: ElementCallError;
+          matrixError?: ElementCallError;
+          connectionError?: ElementCallError;
+          publishError?: ElementCallError;
+        } = {};
+        if (value instanceof ElementCallError) return { transportError: value };
+        if (value === TransportState.Waiting) return null;
+        if (value.matrix instanceof ElementCallError)
+          returnObject.matrixError = value.matrix;
+        if (value.media instanceof ElementCallError)
+          returnObject.publishError = value.media;
+        else if (
+          typeof value.media === "object" &&
+          value.media.connection instanceof ElementCallError
+        )
+          returnObject.connectionError = value.media.connection;
+        return returnObject;
+      }),
+    ),
+    null,
+  );
+
   return {
     autoLeave$: autoLeave$,
     callPickupState$: callPickupState$,
     ringOverlay$: ringOverlay$,
     leave$: leave$,
     hangup: (): void => userHangup$.next(),
-    join: localMembership.requestConnect,
+    join: localMembership.requestJoinAndPublish,
     toggleScreenSharing: toggleScreenSharing,
     sharingScreen$: sharingScreen$,
 
@@ -1439,16 +1489,21 @@ export function createCallViewModel$(
     unhoverScreen: (): void => screenUnhover$.next(),
 
     fatalError$: scope.behavior(
-      localMembership.connectionState.livekit$.pipe(
-        filter((v) => v.state === RTCBackendState.Error),
-        map((s) => s.error),
+      errors$.pipe(
+        map((errors) => {
+          logger.debug("errors$ to compute any fatal errors:", errors);
+          return (
+            errors?.transportError ??
+            errors?.matrixError ??
+            errors?.connectionError ??
+            null
+          );
+        }),
+        filter((error) => error !== null),
       ),
       null,
     ),
-
     participantCount$: participantCount$,
-    audioParticipants$: audioParticipants$,
-
     handsRaised$: handsRaised$,
     reactions$: reactions$,
     joinSoundEffect$: joinSoundEffect$,
@@ -1467,6 +1522,16 @@ export function createCallViewModel$(
     spotlight$: spotlight$,
     pip$: pip$,
     layout$: layout$,
+    userMedia$,
+    localMatrixLivekitMember$,
+    matrixLivekitMembers$: scope.behavior(
+      matrixLivekitMembers$.pipe(
+        map((members) => members.value),
+        tap((v) => {
+          logger.debug("matrixLivekitMembers$ updated (exported)", v);
+        }),
+      ),
+    ),
     tileStoreGeneration$: tileStoreGeneration$,
     showSpotlightIndicators$: showSpotlightIndicators$,
     showSpeakingIndicators$: showSpeakingIndicators$,
@@ -1474,7 +1539,9 @@ export function createCallViewModel$(
     showFooter$: showFooter$,
     earpieceMode$: earpieceMode$,
     audioOutputSwitcher$: audioOutputSwitcher$,
-    reconnecting$: reconnecting$,
+    reconnecting$: localMembership.reconnecting$,
+    livekitRoomItems$,
+    connected$: localMembership.connected$,
   };
 }
 

src/state/CallViewModel/CallViewModelTestUtils.ts

@@ -53,6 +53,7 @@ import {
 import { type Behavior, constant } from "../Behavior";
 import { type ProcessorState } from "../../livekit/TrackProcessorContext";
 import { type MediaDevices } from "../MediaDevices";
+import { type MatrixRTCMode } from "../../settings/settings";
 
 mockConfig({
   livekit: { livekit_service_url: "http://my-default-service-url.com" },
@@ -80,117 +81,125 @@ export interface CallViewModelInputs {
 
 const localParticipant = mockLocalParticipant({ identity: "" });
 
-export function withCallViewModel(
-  {
-    remoteParticipants$ = constant([]),
-    rtcMembers$ = constant([localRtcMember]),
-    livekitConnectionState$: connectionState$ = constant(
-      ConnectionState.Connected,
-    ),
-    speaking = new Map(),
-    mediaDevices = mockMediaDevices({}),
-    initialSyncState = SyncState.Syncing,
-    windowSize$ = constant({ width: 1000, height: 800 }),
-  }: Partial<CallViewModelInputs> = {},
-  continuation: (
-    vm: CallViewModel,
-    rtcSession: MockRTCSession,
-    subjects: { raisedHands$: BehaviorSubject<Record<string, RaisedHandInfo>> },
-    setSyncState: (value: SyncState) => void,
-  ) => void,
-  options: CallViewModelOptions = {
-    encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
-    autoLeaveWhenOthersLeft: false,
-  },
-): void {
-  let syncState = initialSyncState;
-  const setSyncState = (value: SyncState): void => {
-    const prev = syncState;
-    syncState = value;
-    room.client.emit(ClientEvent.Sync, value, prev);
-  };
-  const room = mockMatrixRoom({
-    client: new (class extends EventEmitter {
-      public getUserId(): string | undefined {
-        return localRtcMember.userId;
-      }
+export function withCallViewModel(mode: MatrixRTCMode) {
+  return (
+    {
+      remoteParticipants$ = constant([]),
+      rtcMembers$ = constant([localRtcMember]),
+      livekitConnectionState$: connectionState$ = constant(
+        ConnectionState.Connected,
+      ),
+      speaking = new Map(),
+      mediaDevices = mockMediaDevices({}),
+      initialSyncState = SyncState.Syncing,
+      windowSize$ = constant({ width: 1000, height: 800 }),
+    }: Partial<CallViewModelInputs> = {},
+    continuation: (
+      vm: CallViewModel,
+      rtcSession: MockRTCSession,
+      subjects: {
+        raisedHands$: BehaviorSubject<Record<string, RaisedHandInfo>>;
+      },
+      setSyncState: (value: SyncState) => void,
+    ) => void,
+    options: Partial<CallViewModelOptions> = {},
+  ): void => {
+    let syncState = initialSyncState;
+    const setSyncState = (value: SyncState): void => {
+      const prev = syncState;
+      syncState = value;
+      room.client.emit(ClientEvent.Sync, value, prev);
+    };
+    const room = mockMatrixRoom({
+      client: new (class extends EventEmitter {
+        public getUserId(): string | undefined {
+          return localRtcMember.userId;
+        }
 
-      public getDeviceId(): string {
-        return localRtcMember.deviceId;
-      }
+        public getDeviceId(): string {
+          return localRtcMember.deviceId;
+        }
 
-      public getDomain(): string {
-        return "example.com";
-      }
+        public getDomain(): string {
+          return "example.com";
+        }
 
-      public getSyncState(): SyncState {
-        return syncState;
-      }
-    })() as Partial<MatrixClient> as MatrixClient,
-    getMembers: () => Array.from(roomMembers.values()),
-    getMembersWithMembership: () => Array.from(roomMembers.values()),
-  });
-  const rtcSession = new MockRTCSession(room, []).withMemberships(rtcMembers$);
-  const participantsSpy = vi
-    .spyOn(ComponentsCore, "connectedParticipantsObserver")
-    .mockReturnValue(remoteParticipants$);
-  const mediaSpy = vi
-    .spyOn(ComponentsCore, "observeParticipantMedia")
-    .mockImplementation((p) =>
-      of({ participant: p } as Partial<
-        ComponentsCore.ParticipantMedia<LocalParticipant>
-      > as ComponentsCore.ParticipantMedia<LocalParticipant>),
+        public getSyncState(): SyncState {
+          return syncState;
+        }
+      })() as Partial<MatrixClient> as MatrixClient,
+      getMembers: () => Array.from(roomMembers.values()),
+      getMembersWithMembership: () => Array.from(roomMembers.values()),
+    });
+    const rtcSession = new MockRTCSession(room, []).withMemberships(
+      rtcMembers$,
     );
-  const eventsSpy = vi
-    .spyOn(ComponentsCore, "observeParticipantEvents")
-    .mockImplementation((p, ...eventTypes) => {
-      if (eventTypes.includes(ParticipantEvent.IsSpeakingChanged)) {
-        return (speaking.get(p) ?? of(false)).pipe(
-          map((s): Participant => ({ ...p, isSpeaking: s }) as Participant),
-        );
-      } else {
-        return of(p);
-      }
+    const participantsSpy = vi
+      .spyOn(ComponentsCore, "connectedParticipantsObserver")
+      .mockReturnValue(remoteParticipants$);
+    const mediaSpy = vi
+      .spyOn(ComponentsCore, "observeParticipantMedia")
+      .mockImplementation((p) =>
+        of({ participant: p } as Partial<
+          ComponentsCore.ParticipantMedia<LocalParticipant>
+        > as ComponentsCore.ParticipantMedia<LocalParticipant>),
+      );
+    const eventsSpy = vi
+      .spyOn(ComponentsCore, "observeParticipantEvents")
+      .mockImplementation((p, ...eventTypes) => {
+        if (eventTypes.includes(ParticipantEvent.IsSpeakingChanged)) {
+          return (speaking.get(p) ?? of(false)).pipe(
+            map((s): Participant => ({ ...p, isSpeaking: s }) as Participant),
+          );
+        } else {
+          return of(p);
+        }
+      });
+
+    const roomEventSelectorSpy = vi
+      .spyOn(ComponentsCore, "roomEventSelector")
+      .mockImplementation((_room, _eventType) => of());
+    const muteStates = mockMuteStates();
+    const raisedHands$ = new BehaviorSubject<Record<string, RaisedHandInfo>>(
+      {},
+    );
+    const reactions$ = new BehaviorSubject<Record<string, ReactionInfo>>({});
+
+    const vm = createCallViewModel$(
+      testScope(),
+      rtcSession.asMockedSession(),
+      room,
+      mediaDevices,
+      muteStates,
+      {
+        encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
+        autoLeaveWhenOthersLeft: false,
+        livekitRoomFactory: (): LivekitRoom =>
+          mockLivekitRoom({
+            localParticipant,
+            disconnect: async () => Promise.resolve(),
+            setE2EEEnabled: async () => Promise.resolve(),
+          }),
+        connectionState$,
+        windowSize$,
+        matrixRTCMode$: constant(mode),
+        ...options,
+      },
+      raisedHands$,
+      reactions$,
+      new BehaviorSubject<ProcessorState>({
+        processor: undefined,
+        supported: undefined,
+      }),
+    );
+
+    onTestFinished(() => {
+      participantsSpy.mockRestore();
+      mediaSpy.mockRestore();
+      eventsSpy.mockRestore();
+      roomEventSelectorSpy.mockRestore();
     });
 
-  const roomEventSelectorSpy = vi
-    .spyOn(ComponentsCore, "roomEventSelector")
-    .mockImplementation((_room, _eventType) => of());
-  const muteStates = mockMuteStates();
-  const raisedHands$ = new BehaviorSubject<Record<string, RaisedHandInfo>>({});
-  const reactions$ = new BehaviorSubject<Record<string, ReactionInfo>>({});
-
-  const vm = createCallViewModel$(
-    testScope(),
-    rtcSession.asMockedSession(),
-    room,
-    mediaDevices,
-    muteStates,
-    {
-      ...options,
-      livekitRoomFactory: (): LivekitRoom =>
-        mockLivekitRoom({
-          localParticipant,
-          disconnect: async () => Promise.resolve(),
-          setE2EEEnabled: async () => Promise.resolve(),
-        }),
-      connectionState$,
-      windowSize$,
-    },
-    raisedHands$,
-    reactions$,
-    new BehaviorSubject<ProcessorState>({
-      processor: undefined,
-      supported: undefined,
-    }),
-  );
-
-  onTestFinished(() => {
-    participantsSpy.mockRestore();
-    mediaSpy.mockRestore();
-    eventsSpy.mockRestore();
-    roomEventSelectorSpy.mockRestore();
-  });
-
-  continuation(vm, rtcSession, { raisedHands$: raisedHands$ }, setSyncState);
+    continuation(vm, rtcSession, { raisedHands$: raisedHands$ }, setSyncState);
+  };
 }

src/state/CallViewModel/LayoutSwitch.test.ts

@@ -5,198 +5,128 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
 Please see LICENSE in the repository root for full details.
 */
 
-import { afterEach, beforeEach, describe, expect, test } from "vitest";
-import { firstValueFrom, of } from "rxjs";
+import { describe, test } from "vitest";
 
 import { createLayoutModeSwitch } from "./LayoutSwitch";
-import { ObservableScope } from "../ObservableScope";
-import { constant } from "../Behavior";
-import { withTestScheduler } from "../../utils/test";
+import { testScope, withTestScheduler } from "../../utils/test";
 
-let scope: ObservableScope;
-beforeEach(() => {
-  scope = new ObservableScope();
-});
-afterEach(() => {
-  scope.end();
-});
-
-describe("Default mode", () => {
-  test("Should be in grid layout by default", async () => {
-    const { gridMode$ } = createLayoutModeSwitch(
-      scope,
-      constant("normal"),
-      of(false),
-    );
-
-    const mode = await firstValueFrom(gridMode$);
-    expect(mode).toBe("grid");
-  });
-
-  test("Should switch to spotlight mode when window mode is flat", async () => {
-    const { gridMode$ } = createLayoutModeSwitch(
-      scope,
-      constant("flat"),
-      of(false),
-    );
-
-    const mode = await firstValueFrom(gridMode$);
-    expect(mode).toBe("spotlight");
-  });
-});
-
-test("Should allow switching modes manually", () => {
-  withTestScheduler(({ cold, behavior, expectObservable, schedule }): void => {
+function testLayoutSwitch({
+  windowMode = "n",
+  hasScreenShares = "n",
+  userSelection = "",
+  expectedGridMode,
+}: {
+  windowMode?: string;
+  hasScreenShares?: string;
+  userSelection?: string;
+  expectedGridMode: string;
+}): void {
+  withTestScheduler(({ behavior, schedule, expectObservable }) => {
     const { gridMode$, setGridMode } = createLayoutModeSwitch(
-      scope,
-      behavior("n", { n: "normal" }),
-      cold("f", { f: false, t: true }),
+      testScope(),
+      behavior(windowMode, { n: "normal", N: "narrow", f: "flat" }),
+      behavior(hasScreenShares, { y: true, n: false }),
     );
-
-    schedule("--sgs", {
-      s: () => setGridMode("spotlight"),
-      g: () => setGridMode("grid"),
-    });
-
-    expectObservable(gridMode$).toBe("g-sgs", {
-      g: "grid",
-      s: "spotlight",
-    });
-  });
-});
-
-test("Should switch to spotlight mode when there is a remote screen share", () => {
-  withTestScheduler(({ cold, behavior, expectObservable }): void => {
-    const shareMarble = "f--t";
-    const gridsMarble = "g--s";
-    const { gridMode$ } = createLayoutModeSwitch(
-      scope,
-      behavior("n", { n: "normal" }),
-      cold(shareMarble, { f: false, t: true }),
-    );
-
-    expectObservable(gridMode$).toBe(gridsMarble, {
-      g: "grid",
-      s: "spotlight",
-    });
-  });
-});
-
-test("Can manually force grid when there is a screenshare", () => {
-  withTestScheduler(({ cold, behavior, expectObservable, schedule }): void => {
-    const { gridMode$, setGridMode } = createLayoutModeSwitch(
-      scope,
-      behavior("n", { n: "normal" }),
-      cold("-ft", { f: false, t: true }),
-    );
-
-    schedule("---g", {
-      g: () => setGridMode("grid"),
-    });
-
-    expectObservable(gridMode$).toBe("ggsg", {
-      g: "grid",
-      s: "spotlight",
-    });
-  });
-});
-
-test("Should auto-switch after manually selected grid", () => {
-  withTestScheduler(({ cold, behavior, expectObservable, schedule }): void => {
-    const { gridMode$, setGridMode } = createLayoutModeSwitch(
-      scope,
-      behavior("n", { n: "normal" }),
-      // Two screenshares will happen in sequence
-      cold("-ft-ft", { f: false, t: true }),
-    );
-
-    // There was a screen-share that forced spotlight, then
-    // the user manually switch back to grid
-    schedule("---g", {
-      g: () => setGridMode("grid"),
-    });
-
-    // If we did want to respect manual selection, the expectation would be:
-    // const expectation = "ggsg";
-    const expectation = "ggsg-s";
-
-    expectObservable(gridMode$).toBe(expectation, {
-      g: "grid",
-      s: "spotlight",
-    });
-  });
-});
-
-test("Should switch back to grid mode when the remote screen share ends", () => {
-  withTestScheduler(({ cold, behavior, expectObservable }): void => {
-    const shareMarble = "f--t--f-";
-    const gridsMarble = "g--s--g-";
-    const { gridMode$ } = createLayoutModeSwitch(
-      scope,
-      behavior("n", { n: "normal" }),
-      cold(shareMarble, { f: false, t: true }),
-    );
-
-    expectObservable(gridMode$).toBe(gridsMarble, {
-      g: "grid",
-      s: "spotlight",
-    });
-  });
-});
-
-test("can auto-switch to spotlight again after first screen share ends", () => {
-  withTestScheduler(({ cold, behavior, expectObservable }): void => {
-    const shareMarble = "ftft";
-    const gridsMarble = "gsgs";
-    const { gridMode$ } = createLayoutModeSwitch(
-      scope,
-      behavior("n", { n: "normal" }),
-      cold(shareMarble, { f: false, t: true }),
-    );
-
-    expectObservable(gridMode$).toBe(gridsMarble, {
-      g: "grid",
-      s: "spotlight",
-    });
-  });
-});
-
-test("can switch manually to grid after screen share while manually in spotlight", () => {
-  withTestScheduler(({ cold, behavior, schedule, expectObservable }): void => {
-    // Initially, no one is sharing. Then the user manually switches to
-    // spotlight. After a screen share starts, the user manually switches to
-    // grid.
-    const shareMarbles = "  f-t-";
-    const setModeMarbles = "-s-g";
-    const expectation = "   gs-g";
-    const { gridMode$, setGridMode } = createLayoutModeSwitch(
-      scope,
-      behavior("n", { n: "normal" }),
-      cold(shareMarbles, { f: false, t: true }),
-    );
-    schedule(setModeMarbles, {
+    schedule(userSelection, {
       g: () => setGridMode("grid"),
       s: () => setGridMode("spotlight"),
     });
-
-    expectObservable(gridMode$).toBe(expectation, {
+    expectObservable(gridMode$).toBe(expectedGridMode, {
       g: "grid",
       s: "spotlight",
     });
   });
+}
+
+describe("default mode", () => {
+  test("uses grid layout by default", () =>
+    testLayoutSwitch({
+      expectedGridMode: "g",
+    }));
+
+  test("uses spotlight mode when window mode is flat", () =>
+    testLayoutSwitch({
+      windowMode: "      f",
+      expectedGridMode: "s",
+    }));
 });
 
-test("Should auto-switch to spotlight when in flat window mode", () => {
-  withTestScheduler(({ cold, behavior, expectObservable }): void => {
-    const { gridMode$ } = createLayoutModeSwitch(
-      scope,
-      behavior("naf", { n: "normal", a: "narrow", f: "flat" }),
-      cold("f", { f: false, t: true }),
-    );
+test("allows switching modes manually", () =>
+  testLayoutSwitch({
+    userSelection: "   --sgs",
+    expectedGridMode: "g-sgs",
+  }));
 
-    expectObservable(gridMode$).toBe("g-s-", {
-      g: "grid",
-      s: "spotlight",
-    });
-  });
-});
+test("switches to spotlight mode when there is a remote screen share", () =>
+  testLayoutSwitch({
+    hasScreenShares: " n--y",
+    expectedGridMode: "g--s",
+  }));
+
+test("can manually switch to grid when there is a screenshare", () =>
+  testLayoutSwitch({
+    hasScreenShares: " n-y",
+    userSelection: "   ---g",
+    expectedGridMode: "g-sg",
+  }));
+
+test("auto-switches after manually selecting grid", () =>
+  testLayoutSwitch({
+    // Two screenshares will happen in sequence. There is a screen share that
+    // forces spotlight, then the user manually switches back to grid.
+    hasScreenShares: " n-y-ny",
+    userSelection: "   ---g",
+    expectedGridMode: "g-sg-s",
+    // If we did want to respect manual selection, the expectation would be: g-sg
+  }));
+
+test("switches back to grid mode when the remote screen share ends", () =>
+  testLayoutSwitch({
+    hasScreenShares: " n--y--n",
+    expectedGridMode: "g--s--g",
+  }));
+
+test("auto-switches to spotlight again after first screen share ends", () =>
+  testLayoutSwitch({
+    hasScreenShares: " nyny",
+    expectedGridMode: "gsgs",
+  }));
+
+test("switches manually to grid after screen share while manually in spotlight", () =>
+  testLayoutSwitch({
+    // Initially, no one is sharing. Then the user manually switches to spotlight.
+    // After a screen share starts, the user manually switches to grid.
+    hasScreenShares: " n-y",
+    userSelection: "   -s-g",
+    expectedGridMode: "gs-g",
+  }));
+
+test("auto-switches to spotlight when in flat window mode", () =>
+  testLayoutSwitch({
+    // First normal, then narrow, then flat.
+    windowMode: "      nNf",
+    expectedGridMode: "g-s",
+  }));
+
+test("allows switching modes manually when in flat window mode", () =>
+  testLayoutSwitch({
+    // Window becomes flat, then user switches to grid and back.
+    // Finally the window returns to a normal shape.
+    windowMode: "      nf--n",
+    userSelection: "   --gs",
+    expectedGridMode: "gsgsg",
+  }));
+
+test("stays in spotlight while there are screen shares even when window mode changes", () =>
+  testLayoutSwitch({
+    windowMode: "      nfn",
+    hasScreenShares: " y",
+    expectedGridMode: "s",
+  }));
+
+test("ignores end of screen share until window mode returns to normal", () =>
+  testLayoutSwitch({
+    windowMode: "      nf-n",
+    hasScreenShares: " y-n",
+    expectedGridMode: "s--g",
+  }));

src/state/CallViewModel/LayoutSwitch.ts

@@ -6,122 +6,85 @@ Please see LICENSE in the repository root for full details.
 */
 
 import {
-  BehaviorSubject,
   combineLatest,
   map,
-  type Observable,
-  scan,
+  Subject,
+  startWith,
+  skipWhile,
+  switchMap,
 } from "rxjs";
-import { logger } from "matrix-js-sdk/lib/logger";
 
 import { type GridMode, type WindowMode } from "./CallViewModel.ts";
-import { type Behavior } from "../Behavior.ts";
+import { constant, type Behavior } from "../Behavior.ts";
 import { type ObservableScope } from "../ObservableScope.ts";
 
 /**
  * Creates a layout mode switch that allows switching between grid and spotlight modes.
- * The actual layout mode can be overridden to spotlight mode if there is a remote screen share active
- * or if the window mode is flat.
+ * The actual layout mode might switch automatically to spotlight if there is a
+ * remote screen share active or if the window mode is flat.
  *
  * @param scope - The observable scope to manage subscriptions.
- * @param windowMode$ - The current window mode observable.
- * @param hasRemoteScreenShares$ - An observable indicating if there are remote screen shares active.
+ * @param windowMode$ - The current window mode.
+ * @param hasRemoteScreenShares$ - A behavior indicating if there are remote screen shares active.
  */
 export function createLayoutModeSwitch(
   scope: ObservableScope,
   windowMode$: Behavior<WindowMode>,
-  hasRemoteScreenShares$: Observable<boolean>,
+  hasRemoteScreenShares$: Behavior<boolean>,
 ): {
   gridMode$: Behavior<GridMode>;
   setGridMode: (value: GridMode) => void;
 } {
-  const gridModeUserSelection$ = new BehaviorSubject<GridMode>("grid");
-
+  const userSelection$ = new Subject<GridMode>();
   // Callback to set the grid mode desired by the user.
   // Notice that this is only a preference, the actual grid mode can be overridden
   // if there is a remote screen share active.
-  const setGridMode = (value: GridMode): void => {
-    gridModeUserSelection$.next(value);
-  };
+  const setGridMode = (value: GridMode): void => userSelection$.next(value);
+
+  /**
+   * The natural grid mode - the mode that the grid would prefer to be in,
+   * not accounting for the user's manual selections.
+   */
+  const naturalGridMode$ = scope.behavior<GridMode>(
+    combineLatest(
+      [hasRemoteScreenShares$, windowMode$],
+      (hasRemoteScreenShares, windowMode) =>
+        // When there are screen shares or the window is flat (as with a phone
+        // in landscape orientation), spotlight is a better experience.
+        // We want screen shares to be big and readable, and we want flipping
+        // your phone into landscape to be a quick way of maximising the
+        // spotlight tile.
+        hasRemoteScreenShares || windowMode === "flat" ? "spotlight" : "grid",
+    ),
+  );
+
   /**
    * The layout mode of the media tile grid.
    */
-  const gridMode$ =
-    // If the user hasn't selected spotlight and somebody starts screen sharing,
-    // automatically switch to spotlight mode and reset when screen sharing ends
-    scope.behavior<GridMode>(
-      combineLatest([
-        gridModeUserSelection$,
-        hasRemoteScreenShares$,
-        windowMode$,
-      ]).pipe(
-        // Scan to keep track if we have auto-switched already or not.
-        // To allow the user to override the auto-switch by selecting grid mode again.
-        scan<
-          [GridMode, boolean, WindowMode],
-          {
-            mode: GridMode;
-            /** Remember if the change was user driven or not */
-            hasAutoSwitched: boolean;
-            /** To know if it is new screen share or an already handled */
-            hasScreenShares: boolean;
-          }
-        >(
-          (prev, [userSelection, hasScreenShares, windowMode]) => {
-            const isFlatMode = windowMode === "flat";
+  const gridMode$ = scope.behavior<GridMode>(
+    // Whenever the user makes a selection, we enter a new mode of behavior:
+    userSelection$.pipe(
+      map((selection) => {
+        if (selection === "grid")
+          // The user has selected grid mode. Start by respecting their choice,
+          // but then follow the natural mode again as soon as it matches.
+          return naturalGridMode$.pipe(
+            skipWhile((naturalMode) => naturalMode !== selection),
+            startWith(selection),
+          );
 
-            // Always force spotlight in flat mode, grid layout is not supported
-            // in that mode.
-            // TODO: strange that we do that for flat mode but not for other modes?
-            // TODO: Why is this not handled in layoutMedia$ like other window modes?
-            if (isFlatMode) {
-              logger.debug(`Forcing spotlight mode, windowMode=${windowMode}`);
-              return {
-                mode: "spotlight",
-                hasAutoSwitched: prev.hasAutoSwitched,
-                hasScreenShares,
-              };
-            }
-
-            // User explicitly chose spotlight.
-            // Respect that choice.
-            if (userSelection === "spotlight") {
-              return {
-                mode: "spotlight",
-                hasAutoSwitched: prev.hasAutoSwitched,
-                hasScreenShares,
-              };
-            }
-
-            // User has chosen grid mode. If a screen share starts, we will
-            // auto-switch to spotlight mode for better experience.
-            // But we only do it once, if the user switches back to grid mode,
-            // we respect that choice until they explicitly change it again.
-            const isNewShare = hasScreenShares && !prev.hasScreenShares;
-            if (isNewShare && !prev.hasAutoSwitched) {
-              return {
-                mode: "spotlight",
-                hasAutoSwitched: true,
-                hasScreenShares: true,
-              };
-            }
-
-            // Respect user's grid choice
-            // XXX If we want to forbid switching automatically again after we can
-            // return hasAutoSwitched: acc.hasAutoSwitched here instead of setting to false.
-            return {
-              mode: "grid",
-              hasAutoSwitched: false,
-              hasScreenShares,
-            };
-          },
-          // initial value
-          { mode: "grid", hasAutoSwitched: false, hasScreenShares: false },
-        ),
-        map(({ mode }) => mode),
-      ),
-      "grid",
-    );
+        // The user has selected spotlight mode. If this matches the natural
+        // mode, then follow the natural mode going forward.
+        return selection === naturalGridMode$.value
+          ? naturalGridMode$
+          : constant(selection);
+      }),
+      // Initially the mode of behavior is to just follow the natural grid mode.
+      startWith(naturalGridMode$),
+      // Switch between each mode of behavior.
+      switchMap((mode$) => mode$),
+    ),
+  );
 
   return {
     gridMode$,

src/state/CallViewModel/localMember/HomeserverConnected.test.ts

@@ -97,106 +97,106 @@ describe("createHomeserverConnected$", () => {
   // LLM generated test cases. They are a bit overkill but I improved the mocking so it is
   // easy enough to read them so I think they can stay.
   it("is false when sync state is not Syncing", () => {
-    const hsConnected$ = createHomeserverConnected$(scope, client, session);
-    expect(hsConnected$.value).toBe(false);
+    const hsConnected = createHomeserverConnected$(scope, client, session);
+    expect(hsConnected.combined$.value).toBe(false);
   });
 
   it("remains false while membership status is not Connected even if sync is Syncing", () => {
-    const hsConnected$ = createHomeserverConnected$(scope, client, session);
+    const hsConnected = createHomeserverConnected$(scope, client, session);
     client.setSyncState(SyncState.Syncing);
-    expect(hsConnected$.value).toBe(false); // membership still disconnected
+    expect(hsConnected.combined$.value).toBe(false); // membership still disconnected
   });
 
   it("is false when membership status transitions to Connected but ProbablyLeft is true", () => {
-    const hsConnected$ = createHomeserverConnected$(scope, client, session);
+    const hsConnected = createHomeserverConnected$(scope, client, session);
     // Make sync loop OK
     client.setSyncState(SyncState.Syncing);
     // Indicate probable leave before connection
     session.setProbablyLeft(true);
     session.setMembershipStatus(Status.Connected);
-    expect(hsConnected$.value).toBe(false);
+    expect(hsConnected.combined$.value).toBe(false);
   });
 
   it("becomes true only when all three conditions are satisfied", () => {
-    const hsConnected$ = createHomeserverConnected$(scope, client, session);
+    const hsConnected = createHomeserverConnected$(scope, client, session);
     // 1. Sync loop connected
     client.setSyncState(SyncState.Syncing);
-    expect(hsConnected$.value).toBe(false); // not yet membership connected
+    expect(hsConnected.combined$.value).toBe(false); // not yet membership connected
     // 2. Membership connected
     session.setMembershipStatus(Status.Connected);
-    expect(hsConnected$.value).toBe(true); // probablyLeft is false
+    expect(hsConnected.combined$.value).toBe(true); // probablyLeft is false
   });
 
   it("drops back to false when sync loop leaves Syncing", () => {
-    const hsConnected$ = createHomeserverConnected$(scope, client, session);
+    const hsConnected = createHomeserverConnected$(scope, client, session);
     // Reach connected state
     client.setSyncState(SyncState.Syncing);
     session.setMembershipStatus(Status.Connected);
-    expect(hsConnected$.value).toBe(true);
+    expect(hsConnected.combined$.value).toBe(true);
 
     // Sync loop error => should flip false
     client.setSyncState(SyncState.Error);
-    expect(hsConnected$.value).toBe(false);
+    expect(hsConnected.combined$.value).toBe(false);
   });
 
   it("drops back to false when membership status becomes disconnected", () => {
-    const hsConnected$ = createHomeserverConnected$(scope, client, session);
+    const hsConnected = createHomeserverConnected$(scope, client, session);
     client.setSyncState(SyncState.Syncing);
     session.setMembershipStatus(Status.Connected);
-    expect(hsConnected$.value).toBe(true);
+    expect(hsConnected.combined$.value).toBe(true);
 
     session.setMembershipStatus(Status.Disconnected);
-    expect(hsConnected$.value).toBe(false);
+    expect(hsConnected.combined$.value).toBe(false);
   });
 
   it("drops to false when ProbablyLeft is emitted after being true", () => {
-    const hsConnected$ = createHomeserverConnected$(scope, client, session);
+    const hsConnected = createHomeserverConnected$(scope, client, session);
     client.setSyncState(SyncState.Syncing);
     session.setMembershipStatus(Status.Connected);
-    expect(hsConnected$.value).toBe(true);
+    expect(hsConnected.combined$.value).toBe(true);
 
     session.setProbablyLeft(true);
-    expect(hsConnected$.value).toBe(false);
+    expect(hsConnected.combined$.value).toBe(false);
   });
 
   it("recovers to true if ProbablyLeft becomes false again while other conditions remain true", () => {
-    const hsConnected$ = createHomeserverConnected$(scope, client, session);
+    const hsConnected = createHomeserverConnected$(scope, client, session);
     client.setSyncState(SyncState.Syncing);
     session.setMembershipStatus(Status.Connected);
-    expect(hsConnected$.value).toBe(true);
+    expect(hsConnected.combined$.value).toBe(true);
 
     session.setProbablyLeft(true);
-    expect(hsConnected$.value).toBe(false);
+    expect(hsConnected.combined$.value).toBe(false);
 
     // Simulate clearing the flag (in realistic scenario membership manager would update)
     session.setProbablyLeft(false);
-    expect(hsConnected$.value).toBe(true);
+    expect(hsConnected.combined$.value).toBe(true);
   });
 
   it("composite sequence reflects each individual failure reason", () => {
-    const hsConnected$ = createHomeserverConnected$(scope, client, session);
+    const hsConnected = createHomeserverConnected$(scope, client, session);
 
     // Initially false (sync error + disconnected + not probably left)
-    expect(hsConnected$.value).toBe(false);
+    expect(hsConnected.combined$.value).toBe(false);
 
     // Fix sync only
     client.setSyncState(SyncState.Syncing);
-    expect(hsConnected$.value).toBe(false);
+    expect(hsConnected.combined$.value).toBe(false);
 
     // Fix membership
     session.setMembershipStatus(Status.Connected);
-    expect(hsConnected$.value).toBe(true);
+    expect(hsConnected.combined$.value).toBe(true);
 
     // Introduce probablyLeft -> false
     session.setProbablyLeft(true);
-    expect(hsConnected$.value).toBe(false);
+    expect(hsConnected.combined$.value).toBe(false);
 
     // Restore notProbablyLeft -> true again
     session.setProbablyLeft(false);
-    expect(hsConnected$.value).toBe(true);
+    expect(hsConnected.combined$.value).toBe(true);
 
     // Drop sync -> false
     client.setSyncState(SyncState.Error);
-    expect(hsConnected$.value).toBe(false);
+    expect(hsConnected.combined$.value).toBe(false);
   });
 });

src/state/CallViewModel/localMember/HomeserverConnected.ts

@@ -25,6 +25,11 @@ import { type NodeStyleEventEmitter } from "../../../utils/test";
  */
 const logger = rootLogger.getChild("[HomeserverConnected]");
 
+export interface HomeserverConnected {
+  combined$: Behavior<boolean>;
+  rtsSession$: Behavior<Status>;
+}
+
 /**
  * Behavior representing whether we consider ourselves connected to the Matrix homeserver
  * for the purposes of a MatrixRTC session.
@@ -39,7 +44,7 @@ export function createHomeserverConnected$(
   client: NodeStyleEventEmitter & Pick<MatrixClient, "getSyncState">,
   matrixRTCSession: NodeStyleEventEmitter &
     Pick<MatrixRTCSession, "membershipStatus" | "probablyLeft">,
-): Behavior<boolean> {
+): HomeserverConnected {
   const syncing$ = (
     fromEvent(client, ClientEvent.Sync) as Observable<[SyncState]>
   ).pipe(
@@ -47,12 +52,15 @@ export function createHomeserverConnected$(
     map(([state]) => state === SyncState.Syncing),
   );
 
-  const membershipConnected$ = fromEvent(
-    matrixRTCSession,
-    MembershipManagerEvent.StatusChanged,
-  ).pipe(
-    startWith(null),
-    map(() => matrixRTCSession.membershipStatus === Status.Connected),
+  const rtsSession$ = scope.behavior<Status>(
+    fromEvent(matrixRTCSession, MembershipManagerEvent.StatusChanged).pipe(
+      map(() => matrixRTCSession.membershipStatus ?? Status.Unknown),
+    ),
+    Status.Unknown,
+  );
+
+  const membershipConnected$ = rtsSession$.pipe(
+    map((status) => status === Status.Connected),
   );
 
   // This is basically notProbablyLeft$
@@ -71,15 +79,13 @@ export function createHomeserverConnected$(
     map(() => matrixRTCSession.probablyLeft !== true),
   );
 
-  const connectedCombined$ = and$(
-    syncing$,
-    membershipConnected$,
-    certainlyConnected$,
-  ).pipe(
-    tap((connected) => {
-      logger.info(`Homeserver connected update: ${connected}`);
-    }),
+  const combined$ = scope.behavior(
+    and$(syncing$, membershipConnected$, certainlyConnected$).pipe(
+      tap((connected) => {
+        logger.info(`Homeserver connected update: ${connected}`);
+      }),
+    ),
   );
 
-  return scope.behavior(connectedCombined$);
+  return { combined$, rtsSession$ };
 }

src/state/CallViewModel/localMember/LocalMember.test.ts

@@ -7,6 +7,7 @@ Please see LICENSE in the repository root for full details.
 */
 
 import {
+  Status as RTCMemberStatus,
   type LivekitTransport,
   type MatrixRTCSession,
 } from "matrix-js-sdk/lib/matrixrtc";
@@ -14,11 +15,7 @@ import { describe, expect, it, vi } from "vitest";
 import { AutoDiscovery } from "matrix-js-sdk/lib/autodiscovery";
 import { BehaviorSubject, map, of } from "rxjs";
 import { logger } from "matrix-js-sdk/lib/logger";
-import {
-  ConnectionState as LivekitConnectionState,
-  type LocalParticipant,
-  type LocalTrack,
-} from "livekit-client";
+import { type LocalParticipant, type LocalTrack } from "livekit-client";
 
 import { MatrixRTCMode } from "../../../settings/settings";
 import {
@@ -29,15 +26,17 @@ import {
   withTestScheduler,
 } from "../../../utils/test";
 import {
+  TransportState,
   createLocalMembership$,
   enterRTCSession,
-  RTCBackendState,
-} from "./LocalMembership";
+  PublishState,
+  TrackState,
+} from "./LocalMember";
 import { MatrixRTCTransportMissingError } from "../../../utils/errors";
 import { Epoch, ObservableScope } from "../../ObservableScope";
 import { constant } from "../../Behavior";
 import { ConnectionManagerData } from "../remoteMembers/ConnectionManager";
-import { type Connection } from "../remoteMembers/Connection";
+import { ConnectionState, type Connection } from "../remoteMembers/Connection";
 import { type Publisher } from "./Publisher";
 
 const MATRIX_RTC_MODE = MatrixRTCMode.Legacy;
@@ -51,7 +50,7 @@ vi.mock("@livekit/components-core", () => ({
 
 describe("LocalMembership", () => {
   describe("enterRTCSession", () => {
-    it("It joins the correct Session", async () => {
+    it("It joins the correct Session", () => {
       const focusFromOlderMembership = {
         type: "livekit",
         livekit_service_url: "http://my-oldest-member-service-url.com",
@@ -107,7 +106,7 @@ describe("LocalMembership", () => {
         joinRoomSession: vi.fn(),
       }) as unknown as MatrixRTCSession;
 
-      await enterRTCSession(
+      enterRTCSession(
         mockedSession,
         {
           livekit_alias: "roomId",
@@ -136,7 +135,7 @@ describe("LocalMembership", () => {
       );
     });
 
-    it("It should not fail with configuration error if homeserver config has livekit url but not fallback", async () => {
+    it("It should not fail with configuration error if homeserver config has livekit url but not fallback", () => {
       mockConfig({});
       vi.spyOn(AutoDiscovery, "getRawClientConfig").mockResolvedValue({
         "org.matrix.msc4143.rtc_foci": [
@@ -165,7 +164,7 @@ describe("LocalMembership", () => {
         joinRoomSession: vi.fn(),
       }) as unknown as MatrixRTCSession;
 
-      await enterRTCSession(
+      enterRTCSession(
         mockedSession,
         {
           livekit_alias: "roomId",
@@ -190,7 +189,6 @@ describe("LocalMembership", () => {
       leaveRoomSession: () => {},
     } as unknown as MatrixRTCSession,
     muteStates: mockMuteStates(),
-    isHomeserverConnected: constant(true),
     trackProcessorState$: constant({
       supported: false,
       processor: undefined,
@@ -198,20 +196,20 @@ describe("LocalMembership", () => {
     logger: logger,
     createPublisherFactory: vi.fn(),
     joinMatrixRTC: async (): Promise<void> => {},
-    homeserverConnected$: constant(true),
+    homeserverConnected: {
+      combined$: constant(true),
+      rtsSession$: constant(RTCMemberStatus.Connected),
+    },
   };
 
   it("throws error on missing RTC config error", () => {
     withTestScheduler(({ scope, hot, expectObservable }) => {
-      const goodTransport = {
-        livekit_service_url: "other",
-      } as LivekitTransport;
-
-      const localTransport$ = scope.behavior<LivekitTransport>(
+      const localTransport$ = scope.behavior<null | LivekitTransport>(
         hot("1ms #", {}, new MatrixRTCTransportMissingError("domain.com")),
-        goodTransport,
+        null,
       );
 
+      // we do not need any connection data since we want to fail before reaching that.
       const mockConnectionManager = {
         transports$: scope.behavior(
           localTransport$.pipe(map((t) => new Epoch([t]))),
@@ -227,15 +225,11 @@ describe("LocalMembership", () => {
         connectionManager: mockConnectionManager,
         localTransport$,
       });
+      localMembership.requestJoinAndPublish();
 
-      expectObservable(localMembership.connectionState.livekit$).toBe("ne", {
-        n: { state: RTCBackendState.WaitingForConnection },
-        e: {
-          state: RTCBackendState.Error,
-          error: expect.toSatisfy(
-            (e) => e instanceof MatrixRTCTransportMissingError,
-          ),
-        },
+      expectObservable(localMembership.localMemberState$).toBe("ne", {
+        n: TransportState.Waiting,
+        e: expect.toSatisfy((e) => e instanceof MatrixRTCTransportMissingError),
       });
     });
   });
@@ -247,33 +241,26 @@ describe("LocalMembership", () => {
     livekit_service_url: "b",
   } as LivekitTransport;
 
-  const connectionManagerData = new ConnectionManagerData();
-
-  connectionManagerData.add(
-    {
-      livekitRoom: mockLivekitRoom({
-        localParticipant: {
-          isScreenShareEnabled: false,
-          trackPublications: [],
-        } as unknown as LocalParticipant,
-      }),
-      state$: constant({
-        state: "ConnectedToLkRoom",
-        livekitConnectionState$: constant(LivekitConnectionState.Connected),
-      }),
-      transport: aTransport,
-    } as unknown as Connection,
-    [],
-  );
-  connectionManagerData.add(
-    {
-      state$: constant({
-        state: "ConnectedToLkRoom",
-      }),
-      transport: bTransport,
-    } as unknown as Connection,
-    [],
-  );
+  const connectionTransportAConnected = {
+    livekitRoom: mockLivekitRoom({
+      localParticipant: {
+        isScreenShareEnabled: false,
+        trackPublications: [],
+      } as unknown as LocalParticipant,
+    }),
+    state$: constant(ConnectionState.LivekitConnected),
+    transport: aTransport,
+  } as unknown as Connection;
+  const connectionTransportAConnecting = {
+    ...connectionTransportAConnected,
+    state$: constant(ConnectionState.LivekitConnecting),
+    livekitRoom: mockLivekitRoom({}),
+  } as unknown as Connection;
+  const connectionTransportBConnected = {
+    state$: constant(ConnectionState.LivekitConnected),
+    transport: bTransport,
+    livekitRoom: mockLivekitRoom({}),
+  } as unknown as Connection;
 
   it("recreates publisher if new connection is used and ENDS always unpublish and end tracks", async () => {
     const scope = new ObservableScope();
@@ -281,13 +268,17 @@ describe("LocalMembership", () => {
     const localTransport$ = new BehaviorSubject(aTransport);
 
     const publishers: Publisher[] = [];
-
+    let seed = 0;
     defaultCreateLocalMemberValues.createPublisherFactory.mockImplementation(
       () => {
+        const a = seed;
+        seed += 1;
+        logger.info(`creating [${a}]`);
         const p = {
-          stopPublishing: vi.fn(),
+          stopPublishing: vi.fn().mockImplementation(() => {
+            logger.info(`stopPublishing [${a}]`);
+          }),
           stopTracks: vi.fn(),
-          publishing$: constant(false),
         };
         publishers.push(p as unknown as Publisher);
         return p;
@@ -298,6 +289,9 @@ describe("LocalMembership", () => {
         typeof vi.fn
       >;
 
+    const connectionManagerData = new ConnectionManagerData();
+    connectionManagerData.add(connectionTransportAConnected, []);
+    connectionManagerData.add(connectionTransportBConnected, []);
     createLocalMembership$({
       scope,
       ...defaultCreateLocalMemberValues,
@@ -322,7 +316,7 @@ describe("LocalMembership", () => {
     await flushPromises();
     // stop all tracks after ending scopes
     expect(publishers[1].stopPublishing).toHaveBeenCalled();
-    expect(publishers[1].stopTracks).toHaveBeenCalled();
+    // expect(publishers[1].stopTracks).toHaveBeenCalled();
 
     defaultCreateLocalMemberValues.createPublisherFactory.mockReset();
   });
@@ -357,6 +351,9 @@ describe("LocalMembership", () => {
         typeof vi.fn
       >;
 
+    const connectionManagerData = new ConnectionManagerData();
+    connectionManagerData.add(connectionTransportAConnected, []);
+    // connectionManagerData.add(connectionTransportB, []);
     const localMembership = createLocalMembership$({
       scope,
       ...defaultCreateLocalMemberValues,
@@ -367,15 +364,17 @@ describe("LocalMembership", () => {
     });
     await flushPromises();
     expect(publisherFactory).toHaveBeenCalledOnce();
-    expect(localMembership.tracks$.value.length).toBe(0);
+    // expect(localMembership.tracks$.value.length).toBe(0);
+    expect(publishers[0].createAndSetupTracks).not.toHaveBeenCalled();
     localMembership.startTracks();
     await flushPromises();
-    expect(localMembership.tracks$.value.length).toBe(2);
+    expect(publishers[0].createAndSetupTracks).toHaveBeenCalled();
+    // expect(localMembership.tracks$.value.length).toBe(2);
     scope.end();
     await flushPromises();
     // stop all tracks after ending scopes
     expect(publishers[0].stopPublishing).toHaveBeenCalled();
-    expect(publishers[0].stopTracks).toHaveBeenCalled();
+    // expect(publishers[0].stopTracks).toHaveBeenCalled();
     publisherFactory.mockClear();
   });
   // TODO add an integration test combining publisher and localMembership
@@ -383,10 +382,11 @@ describe("LocalMembership", () => {
   it("tracks livekit state correctly", async () => {
     const scope = new ObservableScope();
 
+    const connectionManagerData = new ConnectionManagerData();
     const localTransport$ = new BehaviorSubject<null | LivekitTransport>(null);
-    const connectionManagerData$ = new BehaviorSubject<
-      Epoch<ConnectionManagerData>
-    >(new Epoch(new ConnectionManagerData()));
+    const connectionManagerData$ = new BehaviorSubject(
+      new Epoch(connectionManagerData),
+    );
     const publishers: Publisher[] = [];
 
     const tracks$ = new BehaviorSubject<LocalTrack[]>([]);
@@ -432,61 +432,96 @@ describe("LocalMembership", () => {
     });
 
     await flushPromises();
-    expect(localMembership.connectionState.livekit$.value).toStrictEqual({
-      state: RTCBackendState.WaitingForTransport,
-    });
+    expect(localMembership.localMemberState$.value).toStrictEqual(
+      TransportState.Waiting,
+    );
     localTransport$.next(aTransport);
     await flushPromises();
-    expect(localMembership.connectionState.livekit$.value).toStrictEqual({
-      state: RTCBackendState.WaitingForConnection,
+    expect(localMembership.localMemberState$.value).toStrictEqual({
+      matrix: RTCMemberStatus.Connected,
+      media: { connection: null, tracks: TrackState.WaitingForUser },
     });
-    connectionManagerData$.next(new Epoch(connectionManagerData));
+
+    const connectionManagerData2 = new ConnectionManagerData();
+    connectionManagerData2.add(
+      // clone because we will mutate this later.
+      { ...connectionTransportAConnecting } as unknown as Connection,
+      [],
+    );
+
+    connectionManagerData$.next(new Epoch(connectionManagerData2));
     await flushPromises();
-    expect(localMembership.connectionState.livekit$.value).toStrictEqual({
-      state: RTCBackendState.Initialized,
+    expect(localMembership.localMemberState$.value).toStrictEqual({
+      matrix: RTCMemberStatus.Connected,
+      media: {
+        connection: ConnectionState.LivekitConnecting,
+        tracks: TrackState.WaitingForUser,
+      },
     });
+
+    (
+      connectionManagerData2.getConnectionForTransport(aTransport)!
+        .state$ as BehaviorSubject<ConnectionState>
+    ).next(ConnectionState.LivekitConnected);
+    expect(localMembership.localMemberState$.value).toStrictEqual({
+      matrix: RTCMemberStatus.Connected,
+      media: {
+        connection: ConnectionState.LivekitConnected,
+        tracks: TrackState.WaitingForUser,
+      },
+    });
+
     expect(publisherFactory).toHaveBeenCalledOnce();
-    expect(localMembership.tracks$.value.length).toBe(0);
+    // expect(localMembership.tracks$.value.length).toBe(0);
 
     // -------
     localMembership.startTracks();
     // -------
 
     await flushPromises();
-    expect(localMembership.connectionState.livekit$.value).toStrictEqual({
-      state: RTCBackendState.CreatingTracks,
-    });
+    // expect(localMembership.localMemberState$.value).toStrictEqual({
+    //   matrix: RTCMemberStatus.Connected,
+    //   media: {
+    //     tracks: TrackState.Creating,
+    //     connection: ConnectionState.LivekitConnected,
+    //   },
+    // });
     createTrackResolver.resolve();
     await flushPromises();
-    expect(localMembership.connectionState.livekit$.value).toStrictEqual({
-      state: RTCBackendState.ReadyToPublish,
-    });
+    expect(
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      (localMembership.localMemberState$.value as any).media,
+    ).toStrictEqual(PublishState.WaitingForUser);
 
     // -------
-    localMembership.requestConnect();
+    localMembership.requestJoinAndPublish();
     // -------
 
-    expect(localMembership.connectionState.livekit$.value).toStrictEqual({
-      state: RTCBackendState.WaitingToPublish,
-    });
+    expect(
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      (localMembership.localMemberState$.value as any).media,
+    ).toStrictEqual(PublishState.Publishing);
 
     publishResolver.resolve();
     await flushPromises();
-    expect(localMembership.connectionState.livekit$.value).toStrictEqual({
-      state: RTCBackendState.Connected,
-    });
+    expect(
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      (localMembership.localMemberState$.value as any).media,
+    ).toStrictEqual(PublishState.Publishing);
+
     expect(publishers[0].stopPublishing).not.toHaveBeenCalled();
 
-    expect(localMembership.connectionState.livekit$.isStopped).toBe(false);
+    expect(localMembership.localMemberState$.isStopped).toBe(false);
     scope.end();
     await flushPromises();
     // stays in connected state because it is stopped before the update to tracks update the state.
-    expect(localMembership.connectionState.livekit$.value).toStrictEqual({
-      state: RTCBackendState.Connected,
-    });
+    expect(
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      (localMembership.localMemberState$.value as any).media,
+    ).toStrictEqual(PublishState.Publishing);
     // stop all tracks after ending scopes
     expect(publishers[0].stopPublishing).toHaveBeenCalled();
-    expect(publishers[0].stopTracks).toHaveBeenCalled();
+    // expect(publishers[0].stopTracks).toHaveBeenCalled();
   });
   // TODO add tests for matrix local matrix participation.
 });

src/state/CallViewModel/localMember/LocalMember.ts

@@ -6,15 +6,16 @@ Please see LICENSE in the repository root for full details.
 */
 
 import {
-  type LocalTrack,
   type Participant,
   ParticipantEvent,
   type LocalParticipant,
   type ScreenShareCaptureOptions,
-  ConnectionState,
+  RoomEvent,
+  MediaDeviceFailure,
 } from "livekit-client";
 import { observeParticipantEvents } from "@livekit/components-core";
 import {
+  Status as RTCSessionStatus,
   type LivekitTransport,
   type MatrixRTCSession,
 } from "matrix-js-sdk/lib/matrixrtc";
@@ -24,10 +25,11 @@ import {
   combineLatest,
   distinctUntilChanged,
   from,
+  fromEvent,
   map,
   type Observable,
   of,
-  scan,
+  pairwise,
   startWith,
   switchMap,
   tap,
@@ -35,74 +37,73 @@ import {
 import { type Logger } from "matrix-js-sdk/lib/logger";
 import { deepCompare } from "matrix-js-sdk/lib/utils";
 
-import { constant, type Behavior } from "../../Behavior";
-import { type IConnectionManager } from "../remoteMembers/ConnectionManager";
-import { ObservableScope } from "../../ObservableScope";
-import { type Publisher } from "./Publisher";
-import { type MuteStates } from "../../MuteStates";
-import { and$ } from "../../../utils/observable";
+import { type Behavior } from "../../Behavior.ts";
+import { type IConnectionManager } from "../remoteMembers/ConnectionManager.ts";
+import { type ObservableScope } from "../../ObservableScope.ts";
+import { type Publisher } from "./Publisher.ts";
+import { type MuteStates } from "../../MuteStates.ts";
 import {
   ElementCallError,
+  FailToStartLivekitConnection,
   MembershipManagerError,
   UnknownCallError,
-} from "../../../utils/errors";
-import { ElementWidgetActions, widget } from "../../../widget";
+} from "../../../utils/errors.ts";
+import { ElementWidgetActions, widget } from "../../../widget.ts";
 import { getUrlParams } from "../../../UrlParams.ts";
 import { PosthogAnalytics } from "../../../analytics/PosthogAnalytics.ts";
 import { MatrixRTCMode } from "../../../settings/settings.ts";
 import { Config } from "../../../config/Config.ts";
-import { type Connection } from "../remoteMembers/Connection.ts";
+import {
+  ConnectionState,
+  type Connection,
+  type FailedToStartError,
+} from "../remoteMembers/Connection.ts";
+import { type HomeserverConnected } from "./HomeserverConnected.ts";
+import { and$ } from "../../../utils/observable.ts";
 
-export enum RTCBackendState {
-  Error = "error",
+export enum TransportState {
   /** Not even a transport is available to the LocalMembership */
-  WaitingForTransport = "waiting_for_transport",
-  /** A connection appeared so we can initialise the publisher */
-  WaitingForConnection = "waiting_for_connection",
-  /** Connection and transport arrived, publisher Initialized */
-  Initialized = "Initialized",
-  CreatingTracks = "creating_tracks",
-  ReadyToPublish = "ready_to_publish",
-  WaitingToPublish = "waiting_to_publish",
-  Connected = "connected",
-  Disconnected = "disconnected",
-  Disconnecting = "disconnecting",
+  Waiting = "transport_waiting",
 }
 
-type LocalMemberRtcBackendState =
-  | { state: RTCBackendState.Error; error: ElementCallError }
-  | { state: RTCBackendState.WaitingForTransport }
-  | { state: RTCBackendState.WaitingForConnection }
-  | { state: RTCBackendState.Initialized }
-  | { state: RTCBackendState.CreatingTracks }
-  | { state: RTCBackendState.ReadyToPublish }
-  | { state: RTCBackendState.WaitingToPublish }
-  | { state: RTCBackendState.Connected }
-  | { state: RTCBackendState.Disconnected }
-  | { state: RTCBackendState.Disconnecting };
-
-export enum MatrixState {
-  WaitingForTransport = "waiting_for_transport",
-  Ready = "ready",
-  Connecting = "connecting",
-  Connected = "connected",
-  Disconnected = "disconnected",
-  Error = "Error",
+export enum PublishState {
+  WaitingForUser = "publish_waiting_for_user",
+  // XXX: This state is removed for now since we do not have full control over
+  // track publication anymore with the publisher abstraction, might come back in the future?
+  // /** Implies lk connection is connected */
+  // Starting = "publish_start_publishing",
+  /** Implies lk connection is connected */
+  Publishing = "publish_publishing",
 }
 
-type LocalMemberMatrixState =
-  | { state: MatrixState.Connected }
-  | { state: MatrixState.WaitingForTransport }
-  | { state: MatrixState.Ready }
-  | { state: MatrixState.Connecting }
-  | { state: MatrixState.Disconnected }
-  | { state: MatrixState.Error; error: Error };
-
-export interface LocalMemberConnectionState {
-  livekit$: Behavior<LocalMemberRtcBackendState>;
-  matrix$: Behavior<LocalMemberMatrixState>;
+// TODO not sure how to map that correctly with the
+// new publisher that does not manage tracks itself anymore
+export enum TrackState {
+  /** The track is waiting for user input to create tracks (waiting to call `startTracks()`) */
+  WaitingForUser = "tracks_waiting_for_user",
+  // XXX: This state is removed for now since we do not have full control over
+  // track creation anymore with the publisher abstraction, might come back in the future?
+  // /** Implies lk connection is connected */
+  // Creating = "tracks_creating",
+  /** Implies lk connection is connected */
+  Ready = "tracks_ready",
 }
 
+export type LocalMemberMediaState =
+  | {
+      tracks: TrackState;
+      connection: ConnectionState | FailedToStartError;
+    }
+  | PublishState
+  | ElementCallError;
+export type LocalMemberState =
+  | ElementCallError
+  | TransportState.Waiting
+  | {
+      media: LocalMemberMediaState;
+      matrix: ElementCallError | RTCSessionStatus;
+    };
+
 /*
  * - get well known
  * - get oldest membership
@@ -122,8 +123,8 @@ interface Props {
   muteStates: MuteStates;
   connectionManager: IConnectionManager;
   createPublisherFactory: (connection: Connection) => Publisher;
-  joinMatrixRTC: (transport: LivekitTransport) => Promise<void>;
-  homeserverConnected$: Behavior<boolean>;
+  joinMatrixRTC: (transport: LivekitTransport) => void;
+  homeserverConnected: HomeserverConnected;
   localTransport$: Behavior<LivekitTransport | null>;
   matrixRTCSession: Pick<
     MatrixRTCSession,
@@ -137,7 +138,16 @@ interface Props {
  * We want
  *  - a publisher
  *  -
- * @param param0
+ * @param props The properties required to create the local membership.
+ * @param props.scope The observable scope to use.
+ * @param props.connectionManager The connection manager to get connections from.
+ * @param props.createPublisherFactory Factory to create a publisher once we have a connection.
+ * @param props.joinMatrixRTC Callback to join the matrix RTC session once we have a transport.
+ * @param props.homeserverConnected The homeserver connected state.
+ * @param props.localTransport$ The local transport to use for publishing.
+ * @param props.logger The logger to use.
+ * @param props.muteStates The mute states for video and audio.
+ * @param props.matrixRTCSession The matrix RTC session to join.
  * @returns
  *  - publisher: The handle to create tracks and publish them to the room.
  *  - connected$: the current connection state. Including matrix server and livekit server connection. (only considering the livekit server we are using for our own media publication)
@@ -149,7 +159,7 @@ export const createLocalMembership$ = ({
   scope,
   connectionManager,
   localTransport$: localTransportCanThrow$,
-  homeserverConnected$,
+  homeserverConnected,
   createPublisherFactory,
   joinMatrixRTC,
   logger: parentLogger,
@@ -157,28 +167,37 @@ export const createLocalMembership$ = ({
   matrixRTCSession,
 }: Props): {
   /**
-   * This starts audio and video tracks. They will be reused when calling `requestConnect`.
+   * This request to start audio and video tracks.
+   * Can be called early to pre-emptively get media permissions and start devices.
    */
-  startTracks: () => Behavior<LocalTrack[]>;
+  startTracks: () => void;
   /**
-   * This sets a inner state (shouldConnect) to true and instructs the js-sdk and livekit to keep the user
+   * This sets a inner state (shouldPublish) to true and instructs the js-sdk and livekit to keep the user
    * connected to matrix and livekit.
    */
-  requestConnect: () => void;
+  requestJoinAndPublish: () => void;
   requestDisconnect: () => void;
-  connectionState: LocalMemberConnectionState;
+  localMemberState$: Behavior<LocalMemberState>;
   sharingScreen$: Behavior<boolean>;
   /**
    * Callback to toggle screen sharing. If null, screen sharing is not possible.
    */
   toggleScreenSharing: (() => void) | null;
-  tracks$: Behavior<LocalTrack[]>;
+  // tracks$: Behavior<LocalTrack[]>;
   participant$: Behavior<LocalParticipant | null>;
   connection$: Behavior<Connection | null>;
-  homeserverConnected$: Behavior<boolean>;
-  // this needs to be discussed
-  /** @deprecated use state instead*/
+  /**
+   * Tracks the homserver and livekit connected state and based on that computes reconnecting.
+   */
   reconnecting$: Behavior<boolean>;
+  /** Shorthand for homeserverConnected.rtcSession === Status.Disconnected
+   * Direct translation to the js-sdk membership manager connection `Status`.
+   */
+  disconnected$: Behavior<boolean>;
+  /**
+   * Fully connected
+   */
+  connected$: Behavior<boolean>;
 } => {
   const logger = parentLogger.getChild("[LocalMembership]");
   logger.debug(`Creating local membership..`);
@@ -197,7 +216,7 @@ export const createLocalMembership$ = ({
               : new Error("Unknown error from localTransport"),
           );
         }
-        setLivekitError(error);
+        setTransportError(error);
         return of(null);
       }),
     ),
@@ -224,94 +243,59 @@ export const createLocalMembership$ = ({
     ),
   );
 
-  const localConnectionState$ = localConnection$.pipe(
-    switchMap((connection) => (connection ? connection.state$ : of(null))),
-  );
-
-  // /**
-  //  * Whether we are "fully" connected to the call. Accounts for both the
-  //  * connection to the MatrixRTC session and the LiveKit publish connection.
-  //  */
-  const connected$ = scope.behavior(
-    and$(
-      homeserverConnected$.pipe(
-        tap((v) => logger.debug("matrix: Connected state changed", v)),
-      ),
-      localConnectionState$.pipe(
-        switchMap((state) => {
-          logger.debug("livekit: Connected state changed", state);
-          if (!state) return of(false);
-          if (state.state === "ConnectedToLkRoom") {
-            logger.debug(
-              "livekit: Connected state changed (inner livekitConnectionState$)",
-              state.livekitConnectionState$.value,
-            );
-            return state.livekitConnectionState$.pipe(
-              map((lkState) => lkState === ConnectionState.Connected),
-            );
-          }
-          return of(false);
-        }),
-      ),
-    ).pipe(tap((v) => logger.debug("combined: Connected state changed", v))),
+  // Tracks error that happen when creating the local tracks.
+  const mediaErrors$ = localConnection$.pipe(
+    switchMap((connection) => {
+      if (!connection) {
+        return of(null);
+      } else {
+        return fromEvent(
+          connection.livekitRoom,
+          RoomEvent.MediaDevicesError,
+          (error: Error) => {
+            return MediaDeviceFailure.getFailure(error) ?? null;
+          },
+        );
+      }
+    }),
   );
 
+  mediaErrors$.pipe(scope.bind()).subscribe((error) => {
+    if (error) {
+      logger.error(`Failed to create local tracks:`, error);
+      setMatrixError(
+        // TODO is it fatal? Do we need to create a new Specialized Error?
+        new UnknownCallError(new Error(`Media device error: ${error}`)),
+      );
+    }
+  });
   // MATRIX RELATED
 
-  // /**
-  //  * Whether we should tell the user that we're reconnecting to the call.
-  //  */
-  // DISCUSSION is there a better way to do this?
-  // sth that is more deriectly implied from the membership manager of the js sdk. (fromEvent(matrixRTCSession, Reconnecting)) ??? or similar
-  const reconnecting$ = scope.behavior(
-    connected$.pipe(
-      // We are reconnecting if we previously had some successful initial
-      // connection but are now disconnected
-      scan(
-        ({ connectedPreviously }, connectedNow) => ({
-          connectedPreviously: connectedPreviously || connectedNow,
-          reconnecting: connectedPreviously && !connectedNow,
-        }),
-        { connectedPreviously: false, reconnecting: false },
-      ),
-      map(({ reconnecting }) => reconnecting),
-    ),
-  );
-
   // This should be used in a combineLatest with publisher$ to connect.
   // to make it possible to call startTracks before the preferredTransport$ has resolved.
   const trackStartRequested = Promise.withResolvers<void>();
 
   // This should be used in a combineLatest with publisher$ to connect.
   // to make it possible to call startTracks before the preferredTransport$ has resolved.
-  const connectRequested$ = new BehaviorSubject(false);
+  const joinAndPublishRequested$ = new BehaviorSubject(false);
 
   /**
    * The publisher is stored in here an abstracts creating and publishing tracks.
    */
   const publisher$ = new BehaviorSubject<Publisher | null>(null);
-  /**
-   * Extract the tracks from the published. Also reacts to changing publishers.
-   */
-  const tracks$ = scope.behavior(
-    publisher$.pipe(switchMap((p) => (p?.tracks$ ? p.tracks$ : constant([])))),
-  );
-  const publishing$ = scope.behavior(
-    publisher$.pipe(switchMap((p) => p?.publishing$ ?? constant(false))),
-  );
 
-  const startTracks = (): Behavior<LocalTrack[]> => {
+  const startTracks = (): void => {
     trackStartRequested.resolve();
-    return tracks$;
+    // This used to return the tracks, but now they are only accessible via the publisher.
   };
 
-  const requestConnect = (): void => {
+  const requestJoinAndPublish = (): void => {
     trackStartRequested.resolve();
-    connectRequested$.next(true);
+    joinAndPublishRequested$.next(true);
   };
 
   const requestDisconnect = (): void => {
-    connectRequested$.next(false);
+    joinAndPublishRequested$.next(false);
   };
 
   // Take care of the publisher$
@@ -323,25 +307,30 @@ export const createLocalMembership$ = ({
   //  - overwrite current publisher
   scope.reconcile(localConnection$, async (connection) => {
     if (connection !== null) {
-      publisher$.next(createPublisherFactory(connection));
+      const publisher = createPublisherFactory(connection);
+      publisher$.next(publisher);
+      // Clean-up callback
+      return Promise.resolve(async (): Promise<void> => {
+        await publisher.stopPublishing();
+        await publisher.stopTracks();
+      });
     }
-    return Promise.resolve(async (): Promise<void> => {
-      await publisher$?.value?.stopPublishing();
-      publisher$?.value?.stopTracks();
-    });
   });
 
   // Use reconcile here to not run concurrent createAndSetupTracks calls
   // `tracks$` will update once they are ready.
   scope.reconcile(
     scope.behavior(
-      combineLatest([publisher$, tracks$, from(trackStartRequested.promise)]),
+      combineLatest([
+        publisher$ /*, tracks$*/,
+        from(trackStartRequested.promise),
+      ]),
       null,
     ),
     async (valueIfReady) => {
       if (!valueIfReady) return;
-      const [publisher, tracks] = valueIfReady;
-      if (publisher && tracks.length === 0) {
+      const [publisher] = valueIfReady;
+      if (publisher) {
         await publisher.createAndSetupTracks().catch((e) => logger.error(e));
       }
     },
@@ -349,140 +338,215 @@ export const createLocalMembership$ = ({
 
   // Based on `connectRequested$` we start publishing tracks. (once they are there!)
   scope.reconcile(
-    scope.behavior(combineLatest([publisher$, tracks$, connectRequested$])),
-    async ([publisher, tracks, shouldConnect]) => {
-      if (shouldConnect === publisher?.publishing$.value) return;
-      if (tracks.length !== 0 && shouldConnect) {
+    scope.behavior(combineLatest([publisher$, joinAndPublishRequested$])),
+    async ([publisher, shouldJoinAndPublish]) => {
+      // Get the current publishing state to avoid redundant calls.
+      const isPublishing = publisher?.shouldPublish === true;
+      if (shouldJoinAndPublish && !isPublishing) {
         try {
           await publisher?.startPublishing();
         } catch (error) {
-          setLivekitError(error as ElementCallError);
+          const message =
+            error instanceof Error ? error.message : String(error);
+          setPublishError(new FailToStartLivekitConnection(message));
         }
-      } else if (tracks.length !== 0 && !shouldConnect) {
+      } else if (isPublishing) {
         try {
           await publisher?.stopPublishing();
         } catch (error) {
-          setLivekitError(new UnknownCallError(error as Error));
+          setPublishError(new UnknownCallError(error as Error));
         }
       }
     },
   );
 
-  const fatalLivekitError$ = new BehaviorSubject<ElementCallError | null>(null);
-  const setLivekitError = (e: ElementCallError): void => {
-    if (fatalLivekitError$.value !== null)
-      logger.error("Multiple Livkit Errors:", e);
-    else fatalLivekitError$.next(e);
+  // STATE COMPUTATION
+
+  // These are non fatal since we can join a room and concume media even though publishing failed.
+  const publishError$ = new BehaviorSubject<ElementCallError | null>(null);
+  const setPublishError = (e: ElementCallError): void => {
+    if (publishError$.value !== null) {
+      logger.error("Multiple Media Errors:", e);
+    } else {
+      publishError$.next(e);
+    }
   };
-  const livekitState$: Behavior<LocalMemberRtcBackendState> = scope.behavior(
+
+  const fatalTransportError$ = new BehaviorSubject<ElementCallError | null>(
+    null,
+  );
+
+  const setTransportError = (e: ElementCallError): void => {
+    if (fatalTransportError$.value !== null) {
+      logger.error("Multiple Transport Errors:", e);
+    } else {
+      fatalTransportError$.next(e);
+    }
+  };
+
+  const localConnectionState$ = localConnection$.pipe(
+    switchMap((connection) => (connection ? connection.state$ : of(null))),
+  );
+
+  const mediaState$: Behavior<LocalMemberMediaState> = scope.behavior(
     combineLatest([
-      publisher$,
+      localConnectionState$,
       localTransport$,
-      tracks$.pipe(
-        tap((t) => {
-          logger.info("tracks$: ", t);
-        }),
-      ),
-      publishing$,
-      connectRequested$,
+      joinAndPublishRequested$,
       from(trackStartRequested.promise).pipe(
         map(() => true),
         startWith(false),
       ),
-      fatalLivekitError$,
     ]).pipe(
       map(
         ([
-          publisher,
+          localConnectionState,
           localTransport,
-          tracks,
-          publishing,
-          shouldConnect,
+          shouldPublish,
           shouldStartTracks,
-          error,
         ]) => {
-          // read this:
-          // if(!<A>) return {state: ...}
-          // if(!<B>) return {state: <MyState>}
-          //
-          // as:
-          // We do have <A> but not yet <B> so we are in <MyState>
-          if (error !== null) return { state: RTCBackendState.Error, error };
-          const hasTracks = tracks.length > 0;
-          if (!localTransport)
-            return { state: RTCBackendState.WaitingForTransport };
-          if (!publisher)
-            return { state: RTCBackendState.WaitingForConnection };
-          if (!shouldStartTracks) return { state: RTCBackendState.Initialized };
-          if (!hasTracks) return { state: RTCBackendState.CreatingTracks };
-          if (!shouldConnect) return { state: RTCBackendState.ReadyToPublish };
-          if (!publishing) return { state: RTCBackendState.WaitingToPublish };
-          return { state: RTCBackendState.Connected };
+          if (!localTransport) return null;
+          const trackState: TrackState = shouldStartTracks
+            ? TrackState.Ready
+            : TrackState.WaitingForUser;
+
+          if (
+            localConnectionState !== ConnectionState.LivekitConnected ||
+            trackState !== TrackState.Ready
+          )
+            return {
+              connection: localConnectionState,
+              tracks: trackState,
+            };
+          if (!shouldPublish) return PublishState.WaitingForUser;
+          // if (!publishing) return PublishState.Starting;
+          return PublishState.Publishing;
         },
       ),
       distinctUntilChanged(deepCompare),
     ),
   );
-
   const fatalMatrixError$ = new BehaviorSubject<ElementCallError | null>(null);
   const setMatrixError = (e: ElementCallError): void => {
-    if (fatalMatrixError$.value !== null)
+    if (fatalMatrixError$.value !== null) {
       logger.error("Multiple Matrix Errors:", e);
-    else fatalMatrixError$.next(e);
+    } else {
+      fatalMatrixError$.next(e);
+    }
   };
-  const matrixState$: Behavior<LocalMemberMatrixState> = scope.behavior(
+
+  const localMemberState$ = scope.behavior<LocalMemberState>(
     combineLatest([
-      localTransport$,
-      connectRequested$,
-      homeserverConnected$,
+      mediaState$,
+      homeserverConnected.rtsSession$,
+      fatalMatrixError$,
+      fatalTransportError$,
+      publishError$,
     ]).pipe(
-      map(([localTransport, connectRequested, homeserverConnected]) => {
-        if (!localTransport) return { state: MatrixState.WaitingForTransport };
-        if (!connectRequested) return { state: MatrixState.Ready };
-        if (!homeserverConnected) return { state: MatrixState.Connecting };
-        return { state: MatrixState.Connected };
-      }),
+      map(
+        ([
+          mediaState,
+          rtcSessionStatus,
+          fatalMatrixError,
+          fatalTransportError,
+          publishError,
+        ]) => {
+          if (fatalTransportError !== null) return fatalTransportError;
+          // `mediaState` will be 'null' until the transport/connection appears.
+          if (mediaState && rtcSessionStatus)
+            return {
+              matrix: fatalMatrixError ?? rtcSessionStatus,
+              media: publishError ?? mediaState,
+            };
+          return TransportState.Waiting;
+        },
+      ),
     ),
   );
 
-  // Keep matrix rtc session in sync with localTransport$, connectRequested$ and muteStates.video.enabled$
+  /**
+   * Whether we are "fully" connected to the call. Accounts for both the
+   * connection to the MatrixRTC session and the LiveKit publish connection.
+   */
+  const matrixAndLivekitConnected$ = scope.behavior(
+    and$(
+      homeserverConnected.combined$,
+      localConnectionState$.pipe(
+        map((state) => state === ConnectionState.LivekitConnected),
+      ),
+    ).pipe(
+      tap((v) => logger.debug("livekit+matrix: Connected state changed", v)),
+    ),
+  );
+
+  /**
+   * Whether we should tell the user that we're reconnecting to the call.
+   */
+  const reconnecting$ = scope.behavior(
+    matrixAndLivekitConnected$.pipe(
+      pairwise(),
+      map(([prev, current]) => prev === true && current === false),
+    ),
+    false,
+  );
+
+  // inform the widget about the connect and disconnect intent from the user.
+  scope
+    .behavior(joinAndPublishRequested$.pipe(pairwise(), scope.bind()), [
+      undefined,
+      joinAndPublishRequested$.value,
+    ])
+    .subscribe(([prev, current]) => {
+      if (!widget) return;
+      // JOIN prev=false (was left) => current-true (now joiend)
+      if (!prev && current) {
+        widget.api.transport
+          .send(ElementWidgetActions.JoinCall, {})
+          .catch((e) => {
+            logger.error("Failed to send join action", e);
+          });
+      }
+      // LEAVE prev=false (was joined) => current-true (now left)
+      if (prev && !current) {
+        widget.api.transport
+          .send(ElementWidgetActions.HangupCall, {})
+          .catch((e) => {
+            logger.error("Failed to send hangup action", e);
+          });
+      }
+    });
+
+  combineLatest([muteStates.video.enabled$, homeserverConnected.combined$])
+    .pipe(scope.bind())
+    .subscribe(([videoEnabled, connected]) => {
+      if (!connected) return;
+      void matrixRTCSession.updateCallIntent(videoEnabled ? "video" : "audio");
+    });
+
+  // Keep matrix rtc session in sync with localTransport$, connectRequested$
   scope.reconcile(
-    scope.behavior(combineLatest([localTransport$, connectRequested$])),
+    scope.behavior(combineLatest([localTransport$, joinAndPublishRequested$])),
     async ([transport, shouldConnect]) => {
+      if (!transport) return;
+      // if shouldConnect=false we will do the disconnect as the cleanup from the previous reconcile iteration.
       if (!shouldConnect) return;
 
-      if (!transport) return;
       try {
-        await joinMatrixRTC(transport);
+        joinMatrixRTC(transport);
       } catch (error) {
         logger.error("Error entering RTC session", error);
         if (error instanceof Error)
           setMatrixError(new MembershipManagerError(error));
       }
 
-      // Update our member event when our mute state changes.
-      const callIntentScope = new ObservableScope();
-      // because this uses its own scope, we can start another reconciliation for the duration of one connection.
-      callIntentScope.reconcile(
-        muteStates.video.enabled$,
-        async (videoEnabled) =>
-          matrixRTCSession.updateCallIntent(videoEnabled ? "video" : "audio"),
-      );
-
-      return async (): Promise<void> => {
-        callIntentScope.end();
+      return Promise.resolve(async (): Promise<void> => {
         try {
-          // Update matrixRTCSession to allow udpating the transport without leaving the session!
-          await matrixRTCSession.leaveRoomSession();
+          // TODO Update matrixRTCSession to allow udpating the transport without leaving the session!
+          await matrixRTCSession.leaveRoomSession(1000);
         } catch (e) {
           logger.error("Error leaving RTC session", e);
         }
-        try {
-          await widget?.api.transport.send(ElementWidgetActions.HangupCall, {});
-        } catch (e) {
-          logger.error("Failed to send hangup action", e);
-        }
-      };
+      });
     },
   );
 
@@ -497,7 +561,7 @@ export const createLocalMembership$ = ({
   // pause tracks during the initial joining sequence too until we're sure
   // that our own media is displayed on screen.
   // TODO refactor this based no livekitState$
-  combineLatest([participant$, homeserverConnected$])
+  combineLatest([participant$, homeserverConnected.combined$])
     .pipe(scope.bind())
     .subscribe(([participant, connected]) => {
       if (!participant) return;
@@ -582,16 +646,17 @@ export const createLocalMembership$ = ({
 
   return {
     startTracks,
-    requestConnect,
+    requestJoinAndPublish,
     requestDisconnect,
-    connectionState: {
-      livekit$: livekitState$,
-      matrix$: matrixState$,
-    },
-    tracks$,
+    localMemberState$,
     participant$,
-    homeserverConnected$,
     reconnecting$,
+    connected$: matrixAndLivekitConnected$,
+    disconnected$: scope.behavior(
+      homeserverConnected.rtsSession$.pipe(
+        map((state) => state === RTCSessionStatus.Disconnected),
+      ),
+    ),
     sharingScreen$,
     toggleScreenSharing,
     connection$: localConnection$,
@@ -620,17 +685,19 @@ interface EnterRTCSessionOptions {
  *      - Delay events management
  *      - Handles retries (fails only after several attempts)
  *
- * @param rtcSession
- * @param transport
- * @param options
+ * @param rtcSession - The MatrixRTCSession to join.
+ * @param transport - The LivekitTransport to use for this session.
+ * @param options - Options for entering the RTC session.
+ * @param options.encryptMedia - Whether to encrypt media.
+ * @param options.matrixRTCMode - The Matrix RTC mode to use.
  * @throws If the widget could not send ElementWidgetActions.JoinCall action.
  */
 // Exported for unit testing
-export async function enterRTCSession(
+export function enterRTCSession(
   rtcSession: MatrixRTCSession,
   transport: LivekitTransport,
   { encryptMedia, matrixRTCMode }: EnterRTCSessionOptions,
-): Promise<void> {
+): void {
   PosthogAnalytics.instance.eventCallEnded.cacheStartCall(new Date());
   PosthogAnalytics.instance.eventCallStarted.track(rtcSession.room.roomId);
 
@@ -669,7 +736,4 @@ export async function enterRTCSession(
       unstableSendStickyEvents: matrixRTCMode === MatrixRTCMode.Matrix_2_0,
     },
   );
-  if (widget) {
-    await widget.api.transport.send(ElementWidgetActions.JoinCall, {});
-  }
 }

src/state/CallViewModel/localMember/LocalTransport.test.ts

@@ -5,9 +5,18 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
 Please see LICENSE in the repository root for full details.
 */
 
-import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import {
+  afterEach,
+  beforeEach,
+  describe,
+  expect,
+  it,
+  type MockedObject,
+  vi,
+} from "vitest";
 import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc";
-import { BehaviorSubject } from "rxjs";
+import { BehaviorSubject, lastValueFrom } from "rxjs";
+import fetchMock from "fetch-mock";
 
 import { mockConfig, flushPromises } from "../../../utils/test";
 import { createLocalTransport$ } from "./LocalTransport";
@@ -18,31 +27,22 @@ import {
   FailToGetOpenIdToken,
 } from "../../../utils/errors";
 import * as openIDSFU from "../../../livekit/openIDSFU";
+import { customLivekitUrl } from "../../../settings/settings";
+import { testJWTToken } from "../../../utils/test-fixtures";
 
 describe("LocalTransport", () => {
+  const openIdResponse: openIDSFU.SFUConfig = {
+    url: "https://lk.example.org",
+    jwt: testJWTToken,
+    livekitAlias: "!example_room_id",
+    livekitIdentity: "@lk_user:ABCDEF",
+  };
+
   let scope: ObservableScope;
-  beforeEach(() => (scope = new ObservableScope()));
-  afterEach(() => scope.end());
-
-  it("throws if config is missing", async () => {
-    const localTransport$ = createLocalTransport$({
-      scope,
-      roomId: "!room:example.org",
-      useOldestMember$: constant(false),
-      memberships$: constant(new Epoch<CallMembership[]>([])),
-      client: {
-        getDomain: () => "",
-        // These won't be called in this error path but satisfy the type
-        getOpenIdToken: vi.fn(),
-        getDeviceId: vi.fn(),
-      },
-    });
-    await flushPromises();
-
-    expect(() => localTransport$.value).toThrow(
-      new MatrixRTCTransportMissingError(""),
-    );
+  beforeEach(() => {
+    scope = new ObservableScope();
   });
+  afterEach(() => scope.end());
 
   it("throws FailToGetOpenIdToken when OpenID fetch fails", async () => {
     // Provide a valid config so makeTransportInternal resolves a transport
@@ -61,12 +61,14 @@ describe("LocalTransport", () => {
     const errors: Error[] = [];
     const localTransport$ = createLocalTransport$({
       scope,
-      roomId: "!room:example.org",
+      roomId: "!example_room_id",
       useOldestMember$: constant(false),
       memberships$: constant(new Epoch<CallMembership[]>([])),
       client: {
         // Use empty domain to skip .well-known and use config directly
         getDomain: () => "",
+        // eslint-disable-next-line @typescript-eslint/naming-convention
+        _unstable_getRTCTransports: async () => Promise.resolve([]),
         getOpenIdToken: vi.fn(),
         getDeviceId: vi.fn(),
       },
@@ -84,41 +86,6 @@ describe("LocalTransport", () => {
     expect(() => localTransport$.value).toThrow(expectedError);
   });
 
-  it("emits preferred transport after OpenID resolves", async () => {
-    // Use config so transport discovery succeeds, but delay OpenID JWT fetch
-    mockConfig({
-      livekit: { livekit_service_url: "https://lk.example.org" },
-    });
-
-    const openIdResolver = Promise.withResolvers<openIDSFU.SFUConfig>();
-
-    vi.spyOn(openIDSFU, "getSFUConfigWithOpenID").mockReturnValue(
-      openIdResolver.promise,
-    );
-
-    const localTransport$ = createLocalTransport$({
-      scope,
-      roomId: "!room:example.org",
-      useOldestMember$: constant(false),
-      memberships$: constant(new Epoch<CallMembership[]>([])),
-      client: {
-        getDomain: () => "",
-        getOpenIdToken: vi.fn(),
-        getDeviceId: vi.fn(),
-      },
-    });
-
-    openIdResolver.resolve?.({ url: "https://lk.example.org", jwt: "jwt" });
-    expect(localTransport$.value).toBe(null);
-    await flushPromises();
-    // final
-    expect(localTransport$.value).toStrictEqual({
-      livekit_alias: "!room:example.org",
-      livekit_service_url: "https://lk.example.org",
-      type: "livekit",
-    });
-  });
-
   it("updates local transport when oldest member changes", async () => {
     // Use config so transport discovery succeeds, but delay OpenID JWT fetch
     mockConfig({
@@ -133,24 +100,171 @@ describe("LocalTransport", () => {
 
     const localTransport$ = createLocalTransport$({
       scope,
-      roomId: "!room:example.org",
+      roomId: "!example_room_id",
       useOldestMember$: constant(true),
       memberships$,
       client: {
         getDomain: () => "",
+        // eslint-disable-next-line @typescript-eslint/naming-convention
+        _unstable_getRTCTransports: async () => Promise.resolve([]),
         getOpenIdToken: vi.fn(),
         getDeviceId: vi.fn(),
       },
     });
 
-    openIdResolver.resolve?.({ url: "https://lk.example.org", jwt: "jwt" });
+    openIdResolver.resolve?.(openIdResponse);
     expect(localTransport$.value).toBe(null);
     await flushPromises();
     // final
     expect(localTransport$.value).toStrictEqual({
-      livekit_alias: "!room:example.org",
+      livekit_alias: "!example_room_id",
       livekit_service_url: "https://lk.example.org",
       type: "livekit",
     });
   });
+
+  type LocalTransportProps = Parameters<typeof createLocalTransport$>[0];
+
+  describe("transport configuration mechanisms", () => {
+    let localTransportOpts: LocalTransportProps & {
+      client: MockedObject<LocalTransportProps["client"]>;
+    };
+    let openIdResolver: PromiseWithResolvers<openIDSFU.SFUConfig>;
+    beforeEach(() => {
+      mockConfig({});
+      customLivekitUrl.setValue(customLivekitUrl.defaultValue);
+      localTransportOpts = {
+        scope,
+        roomId: "!example_room_id",
+        useOldestMember$: constant(false),
+        memberships$: constant(new Epoch<CallMembership[]>([])),
+        client: {
+          getDomain: vi.fn().mockReturnValue(""),
+          // eslint-disable-next-line @typescript-eslint/naming-convention
+          _unstable_getRTCTransports: vi.fn().mockResolvedValue([]),
+          getOpenIdToken: vi.fn(),
+          getDeviceId: vi.fn(),
+        },
+      };
+      openIdResolver = Promise.withResolvers<openIDSFU.SFUConfig>();
+      vi.spyOn(openIDSFU, "getSFUConfigWithOpenID").mockReturnValue(
+        openIdResolver.promise,
+      );
+    });
+
+    afterEach(() => {
+      fetchMock.reset();
+    });
+
+    it("supports getting transport via application config", async () => {
+      mockConfig({
+        livekit: { livekit_service_url: "https://lk.example.org" },
+      });
+      const localTransport$ = createLocalTransport$(localTransportOpts);
+      openIdResolver.resolve?.(openIdResponse);
+      expect(localTransport$.value).toBe(null);
+      await flushPromises();
+      expect(localTransport$.value).toStrictEqual({
+        livekit_alias: "!example_room_id",
+        livekit_service_url: "https://lk.example.org",
+        type: "livekit",
+      });
+    });
+    it("supports getting transport via user settings", async () => {
+      customLivekitUrl.setValue("https://lk.example.org");
+      const localTransport$ = createLocalTransport$(localTransportOpts);
+      openIdResolver.resolve?.(openIdResponse);
+      expect(localTransport$.value).toBe(null);
+      await flushPromises();
+      expect(localTransport$.value).toStrictEqual({
+        livekit_alias: "!example_room_id",
+        livekit_service_url: "https://lk.example.org",
+        type: "livekit",
+      });
+    });
+    it("supports getting transport via backend", async () => {
+      localTransportOpts.client._unstable_getRTCTransports.mockResolvedValue([
+        { type: "livekit", livekit_service_url: "https://lk.example.org" },
+      ]);
+      const localTransport$ = createLocalTransport$(localTransportOpts);
+      openIdResolver.resolve?.(openIdResponse);
+      expect(localTransport$.value).toBe(null);
+      await flushPromises();
+      expect(localTransport$.value).toStrictEqual({
+        livekit_alias: "!example_room_id",
+        livekit_service_url: "https://lk.example.org",
+        type: "livekit",
+      });
+    });
+    it("fails fast if the openID request fails for backend config", async () => {
+      localTransportOpts.client._unstable_getRTCTransports.mockResolvedValue([
+        { type: "livekit", livekit_service_url: "https://lk.example.org" },
+      ]);
+      openIdResolver.reject(
+        new FailToGetOpenIdToken(new Error("Test driven error")),
+      );
+      try {
+        await lastValueFrom(createLocalTransport$(localTransportOpts));
+        throw Error("Expected test to throw");
+      } catch (ex) {
+        expect(ex).toBeInstanceOf(FailToGetOpenIdToken);
+      }
+    });
+    it("supports getting transport via well-known", async () => {
+      localTransportOpts.client.getDomain.mockReturnValue("example.org");
+      fetchMock.getOnce("https://example.org/.well-known/matrix/client", {
+        "org.matrix.msc4143.rtc_foci": [
+          { type: "livekit", livekit_service_url: "https://lk.example.org" },
+        ],
+      });
+      const localTransport$ = createLocalTransport$(localTransportOpts);
+      openIdResolver.resolve?.(openIdResponse);
+      expect(localTransport$.value).toBe(null);
+      await flushPromises();
+      expect(localTransport$.value).toStrictEqual({
+        livekit_alias: "!example_room_id",
+        livekit_service_url: "https://lk.example.org",
+        type: "livekit",
+      });
+      expect(fetchMock.done()).toEqual(true);
+    });
+    it("fails fast if the openId request fails for the well-known config", async () => {
+      localTransportOpts.client.getDomain.mockReturnValue("example.org");
+      fetchMock.getOnce("https://example.org/.well-known/matrix/client", {
+        "org.matrix.msc4143.rtc_foci": [
+          { type: "livekit", livekit_service_url: "https://lk.example.org" },
+        ],
+      });
+      openIdResolver.reject(
+        new FailToGetOpenIdToken(new Error("Test driven error")),
+      );
+      try {
+        await lastValueFrom(createLocalTransport$(localTransportOpts));
+        throw Error("Expected test to throw");
+      } catch (ex) {
+        expect(ex).toBeInstanceOf(FailToGetOpenIdToken);
+      }
+    });
+    it("throws if no options are available", async () => {
+      const localTransport$ = createLocalTransport$({
+        scope,
+        roomId: "!example_room_id",
+        useOldestMember$: constant(false),
+        memberships$: constant(new Epoch<CallMembership[]>([])),
+        client: {
+          getDomain: () => "",
+          // eslint-disable-next-line @typescript-eslint/naming-convention
+          _unstable_getRTCTransports: async () => Promise.resolve([]),
+          // These won't be called in this error path but satisfy the type
+          getOpenIdToken: vi.fn(),
+          getDeviceId: vi.fn(),
+        },
+      });
+      await flushPromises();
+
+      expect(() => localTransport$.value).toThrow(
+        new MatrixRTCTransportMissingError(""),
+      );
+    });
+  });
 });

src/state/CallViewModel/localMember/LocalTransport.ts

@@ -8,11 +8,11 @@ Please see LICENSE in the repository root for full details.
 import {
   type CallMembership,
   isLivekitTransport,
-  type LivekitTransportConfig,
   type LivekitTransport,
   isLivekitTransportConfig,
+  type Transport,
 } from "matrix-js-sdk/lib/matrixrtc";
-import { type MatrixClient } from "matrix-js-sdk";
+import { MatrixError, type MatrixClient } from "matrix-js-sdk";
 import {
   combineLatest,
   distinctUntilChanged,
@@ -27,7 +27,10 @@ import { AutoDiscovery } from "matrix-js-sdk/lib/autodiscovery";
 import { type Behavior } from "../../Behavior.ts";
 import { type Epoch, type ObservableScope } from "../../ObservableScope.ts";
 import { Config } from "../../../config/Config.ts";
-import { MatrixRTCTransportMissingError } from "../../../utils/errors.ts";
+import {
+  FailToGetOpenIdToken,
+  MatrixRTCTransportMissingError,
+} from "../../../utils/errors.ts";
 import {
   getSFUConfigWithOpenID,
   type OpenIDClientParts,
@@ -45,7 +48,8 @@ const logger = rootLogger.getChild("[LocalTransport]");
 interface Props {
   scope: ObservableScope;
   memberships$: Behavior<Epoch<CallMembership[]>>;
-  client: Pick<MatrixClient, "getDomain"> & OpenIDClientParts;
+  client: Pick<MatrixClient, "getDomain" | "_unstable_getRTCTransports"> &
+    OpenIDClientParts;
   roomId: string;
   useOldestMember$: Behavior<boolean>;
 }
@@ -85,7 +89,7 @@ export const createLocalTransport$ = ({
    * The transport that we would personally prefer to publish on (if not for the
    * transport preferences of others, perhaps).
    *
-   * @throws
+   * @throws MatrixRTCTransportMissingError | FailToGetOpenIdToken
    */
   const preferredTransport$: Behavior<LivekitTransport | null> = scope.behavior(
     customLivekitUrl.value$.pipe(
@@ -116,73 +120,150 @@ export const createLocalTransport$ = ({
 const FOCI_WK_KEY = "org.matrix.msc4143.rtc_foci";
 
 /**
+ * Determine the correct Transport for the current session, including
+ * validating auth against the service to ensure it's correct.
+ * Prefers in order:
  *
- * @param client
- * @param roomId
- * @returns
+ * 1. The `urlFromDevSettings` value. If this cannot be validated, the function will throw.
+ * 2. The transports returned via the homeserver.
+ * 3. The transports returned via .well-known.
+ * 4. The transport configured in Element Call's config.
+ *
+ * @param client The authenticated Matrix client for the current user
+ * @param roomId The ID of the room to be connected to.
+ * @param urlFromDevSettings Override URL provided by the user's local config.
+ * @returns A fully validated transport config.
  * @throws MatrixRTCTransportMissingError | FailToGetOpenIdToken
  */
 async function makeTransport(
-  client: Pick<MatrixClient, "getDomain"> & OpenIDClientParts,
+  client: Pick<MatrixClient, "getDomain" | "_unstable_getRTCTransports"> &
+    OpenIDClientParts,
   roomId: string,
   urlFromDevSettings: string | null,
 ): Promise<LivekitTransport> {
-  let transport: LivekitTransport | undefined;
   logger.trace("Searching for a preferred transport");
-  //TODO refactor this to use the jwt service returned alias.
-  const livekitAlias = roomId;
+
+  // We will call `getSFUConfigWithOpenID` once per transport here as it's our
+  // only mechanism of valiation. This means we will also ask the
+  // homeserver for a OpenID token a few times. Since OpenID tokens are single
+  // use we don't want to risk any issues by re-using a token.
+  //
+  // If the OpenID request were to fail then it's acceptable for us to fail
+  // this function early, as we assume the homeserver has got some problems.
 
   // DEVTOOL: Highest priority: Load from devtool setting
   if (urlFromDevSettings !== null) {
-    const transportFromStorage: LivekitTransport = {
+    logger.info("Using LiveKit transport from dev tools: ", urlFromDevSettings);
+    // Validate that the SFU is up. Otherwise, we want to fail on this
+    // as we don't permit other SFUs.
+    const config = await getSFUConfigWithOpenID(
+      client,
+      urlFromDevSettings,
+      roomId,
+    );
+    return {
       type: "livekit",
       livekit_service_url: urlFromDevSettings,
-      livekit_alias: livekitAlias,
+      livekit_alias: config.livekitAlias,
     };
-    logger.info(
-      "Using LiveKit transport from dev tools: ",
-      transportFromStorage,
-    );
-    transport = transportFromStorage;
   }
 
-  // WELL_KNOWN: Prioritize the .well-known/matrix/client, if available, over the configured SFU
+  async function getFirstUsableTransport(
+    transports: Transport[],
+  ): Promise<LivekitTransport | null> {
+    for (const potentialTransport of transports) {
+      if (isLivekitTransportConfig(potentialTransport)) {
+        try {
+          const { livekitAlias } = await getSFUConfigWithOpenID(
+            client,
+            potentialTransport.livekit_service_url,
+            roomId,
+          );
+          return {
+            ...potentialTransport,
+            livekit_alias: livekitAlias,
+          };
+        } catch (ex) {
+          if (ex instanceof FailToGetOpenIdToken) {
+            // Explictly throw these
+            throw ex;
+          }
+          logger.debug(
+            `Could not use SFU service "${potentialTransport.livekit_service_url}" as SFU`,
+            ex,
+          );
+        }
+      }
+    }
+    return null;
+  }
+
+  // MSC4143: Attempt to fetch transports from backend.
+  if ("_unstable_getRTCTransports" in client) {
+    try {
+      const selectedTransport = await getFirstUsableTransport(
+        await client._unstable_getRTCTransports(),
+      );
+      if (selectedTransport) {
+        logger.info("Using backend-configured SFU", selectedTransport);
+        return selectedTransport;
+      }
+    } catch (ex) {
+      if (ex instanceof MatrixError && ex.httpStatus === 404) {
+        // Expected, this is an unstable endpoint and it's not required.
+        logger.debug("Backend does not provide any RTC transports", ex);
+      } else if (ex instanceof FailToGetOpenIdToken) {
+        throw ex;
+      } else {
+        // We got an error that wasn't just missing support for the feature, so log it loudly.
+        logger.error(
+          "Unexpected error fetching RTC transports from backend",
+          ex,
+        );
+      }
+    }
+  }
+
+  // Legacy MSC4143 (to be removed) WELL_KNOWN: Prioritize the .well-known/matrix/client, if available.
   const domain = client.getDomain();
-  if (domain && transport === undefined) {
+  if (domain) {
     // we use AutoDiscovery instead of relying on the MatrixClient having already
     // been fully configured and started
     const wellKnownFoci = (await AutoDiscovery.getRawClientConfig(domain))?.[
       FOCI_WK_KEY
     ];
-    if (Array.isArray(wellKnownFoci)) {
-      const wellKnownTransport: LivekitTransportConfig | undefined =
-        wellKnownFoci.find((f) => f && isLivekitTransportConfig(f));
-      if (wellKnownTransport !== undefined) {
-        logger.info("Using LiveKit transport from .well-known: ", transport);
-        transport = { ...wellKnownTransport, livekit_alias: livekitAlias };
-      }
+    const selectedTransport = Array.isArray(wellKnownFoci)
+      ? await getFirstUsableTransport(wellKnownFoci)
+      : null;
+    if (selectedTransport) {
+      logger.info("Using .well-known SFU", selectedTransport);
+      return selectedTransport;
     }
   }
 
   // CONFIG: Least prioritized; Load from config file
   const urlFromConf = Config.get().livekit?.livekit_service_url;
-  if (urlFromConf && transport === undefined) {
-    const transportFromConf: LivekitTransport = {
-      type: "livekit",
-      livekit_service_url: urlFromConf,
-      livekit_alias: livekitAlias,
-    };
-    logger.info("Using LiveKit transport from config: ", transportFromConf);
-    transport = transportFromConf;
+  if (urlFromConf) {
+    try {
+      const { livekitAlias } = await getSFUConfigWithOpenID(
+        client,
+        urlFromConf,
+        roomId,
+      );
+      const selectedTransport: LivekitTransport = {
+        type: "livekit",
+        livekit_service_url: urlFromConf,
+        livekit_alias: livekitAlias,
+      };
+      logger.info("Using config SFU", selectedTransport);
+      return selectedTransport;
+    } catch (ex) {
+      if (ex instanceof FailToGetOpenIdToken) {
+        throw ex;
+      }
+      logger.error("Failed to validate config SFU", ex);
+    }
   }
 
-  if (!transport) throw new MatrixRTCTransportMissingError(domain ?? ""); // this will call the jwt/sfu/get endpoint to pre create the livekit room.
-
-  await getSFUConfigWithOpenID(
-    client,
-    transport.livekit_service_url,
-    transport.livekit_alias,
-  );
-
-  return transport;
+  throw new MatrixRTCTransportMissingError(domain ?? "");
 }

src/state/CallViewModel/localMember/Publisher.test.ts

@@ -5,66 +5,320 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
 Please see LICENSE in the repository root for full details.
 */
 
+import { afterEach, beforeEach, describe, expect, it, test, vi } from "vitest";
 import {
-  afterEach,
-  beforeEach,
-  describe,
-  expect,
-  it,
-  type Mock,
-  vi,
-} from "vitest";
-import { ConnectionState as LivekitConenctionState } from "livekit-client";
-import { type BehaviorSubject } from "rxjs";
+  ConnectionState as LivekitConnectionState,
+  LocalParticipant,
+  type LocalTrack,
+  type LocalTrackPublication,
+  ParticipantEvent,
+  Track,
+} from "livekit-client";
+import { BehaviorSubject } from "rxjs";
 import { logger } from "matrix-js-sdk/lib/logger";
 
 import { ObservableScope } from "../../ObservableScope";
 import { constant } from "../../Behavior";
 import {
+  flushPromises,
   mockLivekitRoom,
-  mockLocalParticipant,
   mockMediaDevices,
 } from "../../../utils/test";
 import { Publisher } from "./Publisher";
-import {
-  type Connection,
-  type ConnectionState,
-} from "../remoteMembers/Connection";
+import { type Connection } from "../remoteMembers/Connection";
 import { type MuteStates } from "../../MuteStates";
-import { FailToStartLivekitConnection } from "../../../utils/errors";
 
-describe("Publisher", () => {
-  let scope: ObservableScope;
-  let connection: Connection;
-  let muteStates: MuteStates;
-  beforeEach(() => {
-    muteStates = {
-      audio: {
-        enabled$: constant(false),
-        unsetHandler: vi.fn(),
-        setHandler: vi.fn(),
-      },
-      video: {
-        enabled$: constant(false),
-        unsetHandler: vi.fn(),
-        setHandler: vi.fn(),
-      },
-    } as unknown as MuteStates;
-    scope = new ObservableScope();
-    connection = {
-      state$: constant({
-        state: "ConnectedToLkRoom",
-        livekitConnectionState$: constant(LivekitConenctionState.Connected),
-      }),
-      livekitRoom: mockLivekitRoom({
-        localParticipant: mockLocalParticipant({}),
-      }),
-    } as unknown as Connection;
+let scope: ObservableScope;
+
+beforeEach(() => {
+  scope = new ObservableScope();
+});
+
+afterEach(() => scope.end());
+
+function createMockLocalTrack(source: Track.Source): LocalTrack {
+  const track = {
+    source,
+    isMuted: false,
+    isUpstreamPaused: false,
+  } as Partial<LocalTrack> as LocalTrack;
+
+  vi.mocked(track).mute = vi.fn().mockImplementation(() => {
+    track.isMuted = true;
+  });
+  vi.mocked(track).unmute = vi.fn().mockImplementation(() => {
+    track.isMuted = false;
+  });
+  vi.mocked(track).pauseUpstream = vi.fn().mockImplementation(() => {
+    // @ts-expect-error - for that test we want to set isUpstreamPaused directly
+    track.isUpstreamPaused = true;
+  });
+  vi.mocked(track).resumeUpstream = vi.fn().mockImplementation(() => {
+    // @ts-expect-error - for that test we want to set isUpstreamPaused directly
+    track.isUpstreamPaused = false;
   });
 
-  afterEach(() => scope.end());
+  return track;
+}
 
-  it("throws if livekit room could not publish", async () => {
+function createMockMuteState(enabled$: BehaviorSubject<boolean>): {
+  enabled$: BehaviorSubject<boolean>;
+  setHandler: (h: (enabled: boolean) => void) => void;
+  unsetHandler: () => void;
+} {
+  let currentHandler = (enabled: boolean): void => {};
+
+  const ms = {
+    enabled$,
+    setHandler: vi.fn().mockImplementation((h: (enabled: boolean) => void) => {
+      currentHandler = h;
+    }),
+    unsetHandler: vi.fn().mockImplementation(() => {
+      currentHandler = (enabled: boolean): void => {};
+    }),
+  };
+  // forward enabled$ emissions to the current handler
+  enabled$.subscribe((enabled) => {
+    logger.info(`MockMuteState: enabled changed to ${enabled}`);
+    currentHandler(enabled);
+  });
+
+  return ms;
+}
+
+let connection: Connection;
+let muteStates: MuteStates;
+let localParticipant: LocalParticipant;
+let audioEnabled$: BehaviorSubject<boolean>;
+let videoEnabled$: BehaviorSubject<boolean>;
+let trackPublications: LocalTrackPublication[];
+// use it to control when track creation resolves, default to resolved
+let createTrackLock: Promise<void>;
+
+beforeEach(() => {
+  trackPublications = [];
+  audioEnabled$ = new BehaviorSubject(false);
+  videoEnabled$ = new BehaviorSubject(false);
+  createTrackLock = Promise.resolve();
+
+  muteStates = {
+    audio: createMockMuteState(audioEnabled$),
+    video: createMockMuteState(videoEnabled$),
+  } as unknown as MuteStates;
+
+  const mockSendDataPacket = vi.fn();
+  const mockEngine = {
+    client: {
+      sendUpdateLocalMetadata: vi.fn(),
+    },
+    on: vi.fn().mockReturnThis(),
+    sendDataPacket: mockSendDataPacket,
+  };
+
+  localParticipant = new LocalParticipant(
+    "local-sid",
+    "local-identity",
+    // @ts-expect-error - for that test we want a real LocalParticipant to have the pending publications logic
+    mockEngine,
+    {
+      adaptiveStream: true,
+      dynacase: false,
+      audioCaptureDefaults: {},
+      videoCaptureDefaults: {},
+      stopLocalTrackOnUnpublish: true,
+      reconnectPolicy: "always",
+      disconnectOnPageLeave: true,
+    },
+    new Map(),
+    {},
+  );
+
+  vi.mocked(localParticipant).createTracks = vi
+    .fn()
+    .mockImplementation(async (opts) => {
+      const tracks: LocalTrack[] = [];
+      if (opts.audio) {
+        tracks.push(createMockLocalTrack(Track.Source.Microphone));
+      }
+      if (opts.video) {
+        tracks.push(createMockLocalTrack(Track.Source.Camera));
+      }
+      await createTrackLock;
+      return tracks;
+    });
+
+  vi.mocked(localParticipant).publishTrack = vi
+    .fn()
+    .mockImplementation(async (track: LocalTrack) => {
+      const pub = {
+        track,
+        source: track.source,
+        mute: track.mute,
+        unmute: track.unmute,
+      } as Partial<LocalTrackPublication> as LocalTrackPublication;
+      trackPublications.push(pub);
+      localParticipant.emit(ParticipantEvent.LocalTrackPublished, pub);
+      return Promise.resolve(pub);
+    });
+
+  vi.mocked(localParticipant).getTrackPublication = vi
+    .fn()
+    .mockImplementation((source: Track.Source) => {
+      return trackPublications.find((pub) => pub.track?.source === source);
+    });
+
+  connection = {
+    state$: constant({
+      state: "ConnectedToLkRoom",
+      livekitConnectionState$: constant(LivekitConnectionState.Connected),
+    }),
+    livekitRoom: mockLivekitRoom({
+      localParticipant: localParticipant,
+    }),
+  } as unknown as Connection;
+});
+
+describe("Publisher", () => {
+  let publisher: Publisher;
+
+  beforeEach(() => {
+    publisher = new Publisher(
+      scope,
+      connection,
+      mockMediaDevices({}),
+      muteStates,
+      constant({ supported: false, processor: undefined }),
+      logger,
+    );
+  });
+
+  afterEach(() => {});
+
+  it("Should not create tracks if started muted to avoid unneeded permission requests", async () => {
+    const createTracksSpy = vi.spyOn(
+      connection.livekitRoom.localParticipant,
+      "createTracks",
+    );
+
+    audioEnabled$.next(false);
+    videoEnabled$.next(false);
+    await publisher.createAndSetupTracks();
+
+    expect(createTracksSpy).not.toHaveBeenCalled();
+  });
+
+  it("Should minimize permission request by querying create at once", async () => {
+    const enableCameraAndMicrophoneSpy = vi.spyOn(
+      localParticipant,
+      "enableCameraAndMicrophone",
+    );
+    const createTracksSpy = vi.spyOn(localParticipant, "createTracks");
+
+    audioEnabled$.next(true);
+    videoEnabled$.next(true);
+    await publisher.createAndSetupTracks();
+    await flushPromises();
+
+    expect(enableCameraAndMicrophoneSpy).toHaveBeenCalled();
+
+    // It should create both at once
+    expect(createTracksSpy).toHaveBeenCalledWith({
+      audio: true,
+      video: true,
+    });
+  });
+
+  it("Ensure no data is streamed until publish has been called", async () => {
+    audioEnabled$.next(true);
+    await publisher.createAndSetupTracks();
+
+    // The track should be created and paused
+    expect(localParticipant.createTracks).toHaveBeenCalledWith({
+      audio: true,
+      video: undefined,
+    });
+    await flushPromises();
+    expect(localParticipant.publishTrack).toHaveBeenCalled();
+
+    await flushPromises();
+    const track = localParticipant.getTrackPublication(
+      Track.Source.Microphone,
+    )?.track;
+    expect(track).toBeDefined();
+    expect(track!.pauseUpstream).toHaveBeenCalled();
+    expect(track!.isUpstreamPaused).toBe(true);
+  });
+
+  it("Ensure resume upstream when published is called", async () => {
+    videoEnabled$.next(true);
+    await publisher.createAndSetupTracks();
+    // await flushPromises();
+    await publisher.startPublishing();
+
+    const track = localParticipant.getTrackPublication(
+      Track.Source.Camera,
+    )?.track;
+    expect(track).toBeDefined();
+    // expect(track.pauseUpstream).toHaveBeenCalled();
+    expect(track!.isUpstreamPaused).toBe(false);
+  });
+
+  describe("Mute states", () => {
+    let publisher: Publisher;
+    beforeEach(() => {
+      publisher = new Publisher(
+        scope,
+        connection,
+        mockMediaDevices({}),
+        muteStates,
+        constant({ supported: false, processor: undefined }),
+        logger,
+      );
+    });
+
+    test.each([
+      { mutes: { audioEnabled: true, videoEnabled: false } },
+      { mutes: { audioEnabled: true, videoEnabled: false } },
+    ])("only create the tracks that are unmuted $mutes", async ({ mutes }) => {
+      // Ensure all muted
+      audioEnabled$.next(mutes.audioEnabled);
+      videoEnabled$.next(mutes.videoEnabled);
+
+      vi.mocked(connection.livekitRoom.localParticipant).createTracks = vi
+        .fn()
+        .mockResolvedValue([]);
+
+      await publisher.createAndSetupTracks();
+
+      expect(
+        connection.livekitRoom.localParticipant.createTracks,
+      ).toHaveBeenCalledOnce();
+
+      expect(
+        connection.livekitRoom.localParticipant.createTracks,
+      ).toHaveBeenCalledWith({
+        audio: mutes.audioEnabled ? true : undefined,
+        video: mutes.videoEnabled ? true : undefined,
+      });
+    });
+  });
+
+  it("does mute unmute audio", async () => {});
+});
+
+describe("Bug fix", () => {
+  // There is a race condition when creating and publishing tracks while the mute state changes.
+  // This race condition could cause tracks to be published even though they are muted at the
+  // beginning of a call coming from lobby.
+  // This is caused by our stack using manually the low level API to create and publish tracks,
+  // but also using the higher level setMicrophoneEnabled and setCameraEnabled functions that also create
+  // and publish tracks, and managing pending publications.
+  // Race is as follow, on creation of the Publisher we create the tracks then publish them.
+  // If in the middle of that process the mute state changes:
+  //  - the `setMicrophoneEnabled` will be no-op because it is not aware of our created track and can't see any pending publication
+  //  - If start publication is requested it will publish the track even though there was a mute request.
+  it("wrongly publish tracks while muted", async () => {
+    // setLogLevel(`debug`);
     const publisher = new Publisher(
       scope,
       connection,
@@ -73,68 +327,34 @@ describe("Publisher", () => {
       constant({ supported: false, processor: undefined }),
       logger,
     );
+    audioEnabled$.next(true);
 
-    // should do nothing if no tracks have been created yet.
-    await publisher.startPublishing();
-    expect(
-      connection.livekitRoom.localParticipant.publishTrack,
-    ).not.toHaveBeenCalled();
+    const resolvers = Promise.withResolvers<void>();
+    createTrackLock = resolvers.promise;
 
-    await expect(publisher.createAndSetupTracks()).rejects.toThrow(
-      Error("audio and video is false"),
-    );
-
-    (muteStates.audio.enabled$ as BehaviorSubject<boolean>).next(true);
-
-    (
-      connection.livekitRoom.localParticipant.createTracks as Mock
-    ).mockResolvedValue([{}, {}]);
-
-    await expect(publisher.createAndSetupTracks()).resolves.not.toThrow();
-    expect(
-      connection.livekitRoom.localParticipant.createTracks,
-    ).toHaveBeenCalledOnce();
-
-    // failiour due to localParticipant.publishTrack
-    (
-      connection.livekitRoom.localParticipant.publishTrack as Mock
-    ).mockRejectedValue(Error("testError"));
-
-    await expect(publisher.startPublishing()).rejects.toThrow(
-      new FailToStartLivekitConnection("testError"),
-    );
-
-    // does not try other conenction after the first one failed
-    expect(
-      connection.livekitRoom.localParticipant.publishTrack,
-    ).toHaveBeenCalledTimes(1);
-
-    // failiour due to connection.state$
-    const beforeState = connection.state$.value;
-    (connection.state$ as BehaviorSubject<ConnectionState>).next({
-      state: "FailedToStart",
-      error: Error("testStartError"),
+    // Initially the audio is unmuted, so creating tracks should publish the audio track
+    const createTracks = publisher.createAndSetupTracks();
+    void publisher.startPublishing();
+    void createTracks.then(() => {
+      void publisher.startPublishing();
     });
+    // now mute the audio before allowing track creation to complete
+    audioEnabled$.next(false);
+    resolvers.resolve(undefined);
+    await createTracks;
 
-    await expect(publisher.startPublishing()).rejects.toThrow(
-      new FailToStartLivekitConnection("testStartError"),
-    );
-    (connection.state$ as BehaviorSubject<ConnectionState>).next(beforeState);
+    await flushPromises();
 
-    // does not try other conenction after the first one failed
-    expect(
-      connection.livekitRoom.localParticipant.publishTrack,
-    ).toHaveBeenCalledTimes(1);
+    const track = localParticipant.getTrackPublication(
+      Track.Source.Microphone,
+    )?.track;
+    expect(track).toBeDefined();
 
-    // success case
-    (
-      connection.livekitRoom.localParticipant.publishTrack as Mock
-    ).mockResolvedValue({});
-
-    await expect(publisher.startPublishing()).resolves.not.toThrow();
-
-    expect(
-      connection.livekitRoom.localParticipant.publishTrack,
-    ).toHaveBeenCalledTimes(3);
+    try {
+      expect(localParticipant.publishTrack).not.toHaveBeenCalled();
+    } catch {
+      expect(track!.mute).toHaveBeenCalled();
+      expect(track!.isMuted).toBe(true);
+    }
   });
 });

src/state/CallViewModel/localMember/Publisher.ts

@@ -6,15 +6,14 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
 Please see LICENSE in the repository root for full details.
 */
 import {
+  ConnectionState as LivekitConnectionState,
+  type LocalTrackPublication,
   LocalVideoTrack,
+  ParticipantEvent,
   type Room as LivekitRoom,
   Track,
-  type LocalTrack,
-  type LocalTrackPublication,
-  ConnectionState as LivekitConnectionState,
 } from "livekit-client";
 import {
-  BehaviorSubject,
   map,
   NEVER,
   type Observable,
@@ -34,10 +33,6 @@ import { getUrlParams } from "../../../UrlParams.ts";
 import { observeTrackReference$ } from "../../MediaViewModel.ts";
 import { type Connection } from "../remoteMembers/Connection.ts";
 import { type ObservableScope } from "../../ObservableScope.ts";
-import {
-  ElementCallError,
-  FailToStartLivekitConnection,
-} from "../../../utils/errors.ts";
 
 /**
  * A wrapper for a Connection object.
@@ -45,14 +40,21 @@ import {
  * The Publisher is also responsible for creating the media tracks.
  */
 export class Publisher {
+  /**
+   * By default, livekit will start publishing tracks as soon as they are created.
+   * In the matrix RTC world, we want to control when tracks are published based
+   * on whether the user is part of the RTC session or not.
+   */
+  public shouldPublish = false;
+
   /**
    * Creates a new Publisher.
    * @param scope - The observable scope to use for managing the publisher.
    * @param connection - The connection to use for publishing.
    * @param devices - The media devices to use for audio and video input.
    * @param muteStates - The mute states for audio and video.
-   * @param e2eeLivekitOptions - The E2EE options to use for the LiveKit room. Use to share the same key provider across connections!.
    * @param trackerProcessorState$ - The processor state for the video track processor (e.g. background blur).
+   * @param logger - The logger to use for logging :D.
    */
   public constructor(
     private scope: ObservableScope,
@@ -62,7 +64,6 @@ export class Publisher {
     trackerProcessorState$: Behavior<ProcessorState>,
     private logger: Logger,
   ) {
-    this.logger.info("Create LiveKit room");
     const { controlledAudioDevices } = getUrlParams();
 
     const room = connection.livekitRoom;
@@ -80,161 +81,185 @@ export class Publisher {
     this.scope.onEnd(() => {
       this.logger.info("Scope ended -> stop publishing all tracks");
       void this.stopPublishing();
+      muteStates.audio.unsetHandler();
+      muteStates.video.unsetHandler();
     });
 
-    // TODO move mute state handling here using reconcile (instead of inside the mute state class)
-    // this.scope.reconcile(
-    //   this.scope.behavior(
-    //     combineLatest([this.muteStates.video.enabled$, this.tracks$]),
-    //   ),
-    //   async ([videoEnabled, tracks]) => {
-    //     const track = tracks.find((t) => t.kind == Track.Kind.Video);
-    //     if (!track) return;
-
-    //     if (videoEnabled) {
-    //       await track.unmute();
-    //     } else {
-    //       await track.mute();
-    //     }
-    //   },
-    // );
+    this.connection.livekitRoom.localParticipant.on(
+      ParticipantEvent.LocalTrackPublished,
+      this.onLocalTrackPublished.bind(this),
+    );
   }
 
-  private _tracks$ = new BehaviorSubject<LocalTrack<Track.Kind>[]>([]);
-  public tracks$ = this._tracks$ as Behavior<LocalTrack<Track.Kind>[]>;
-
+  // LiveKit will publish the tracks as soon as they are created
+  // but we want to control when tracks are published.
+  // We cannot just mute the tracks, even if this will effectively stop the publishing,
+  // it would also prevent the user from seeing their own video/audio preview.
+  // So for that we use pauseUpStream():  Stops sending media to the server by replacing
+  // the sender track with null, but keeps the local MediaStreamTrack active.
+  // The user can still see/hear themselves locally, but remote participants see nothing.
+  private onLocalTrackPublished(
+    localTrackPublication: LocalTrackPublication,
+  ): void {
+    this.logger.info("Local track published", localTrackPublication);
+    const lkRoom = this.connection.livekitRoom;
+    if (!this.shouldPublish) {
+      this.pauseUpstreams(lkRoom, [localTrackPublication.source]).catch((e) => {
+        this.logger.error(`Failed to pause upstreams`, e);
+      });
+    }
+    // also check the mute state and apply it
+    if (localTrackPublication.source === Track.Source.Microphone) {
+      const enabled = this.muteStates.audio.enabled$.value;
+      lkRoom.localParticipant.setMicrophoneEnabled(enabled).catch((e) => {
+        this.logger.error(
+          `Failed to enable microphone track, enabled:${enabled}`,
+          e,
+        );
+      });
+    } else if (localTrackPublication.source === Track.Source.Camera) {
+      const enabled = this.muteStates.video.enabled$.value;
+      lkRoom.localParticipant.setCameraEnabled(enabled).catch((e) => {
+        this.logger.error(
+          `Failed to enable camera track, enabled:${enabled}`,
+          e,
+        );
+      });
+    }
+  }
   /**
-   * Start the connection to LiveKit and publish local tracks.
+   * Create and setup local audio and video tracks based on the current mute states.
+   * It creates the tracks only if audio and/or video is enabled, to avoid unnecessary
+   * permission prompts.
    *
-   * This will:
-   * wait for the connection to be ready.
-   // * 1. Request an OpenId token `request_token` (allows matrix users to verify their identity with a third-party service.)
-   // * 2. Use this token to request the SFU config to the MatrixRtc authentication service.
-   // * 3. Connect to the configured LiveKit room.
-   // * 4. Create local audio and video tracks based on the current mute states and publish them to the room.
+   * It also observes mute state changes to update LiveKit microphone/camera states accordingly.
+   * If a track is not created initially because disabled, it will be created when unmuting.
+   *
+   * This call is not blocking anymore, instead callers can listen to the
+   * `RoomEvent.MediaDevicesError` event in the LiveKit room to be notified of any errors.
    *
-   * @throws {InsufficientCapacityError} if the LiveKit server indicates that it has insufficient capacity to accept the connection.
-   * @throws {SFURoomCreationRestrictedError} if the LiveKit server indicates that the room does not exist and cannot be created.
    */
   public async createAndSetupTracks(): Promise<void> {
     this.logger.debug("createAndSetupTracks called");
     const lkRoom = this.connection.livekitRoom;
     // Observe mute state changes and update LiveKit microphone/camera states accordingly
-    this.observeMuteStates(this.scope);
+    this.observeMuteStates();
 
-    // TODO-MULTI-SFU: Prepublish a microphone track
+    // Check if audio and/or video is enabled. We only create tracks if enabled,
+    // because it could prompt for permission, and we don't want to do that unnecessarily.
     const audio = this.muteStates.audio.enabled$.value;
     const video = this.muteStates.video.enabled$.value;
-    // createTracks throws if called with audio=false and video=false
-    if (audio || video) {
-      // TODO this can still throw errors? It will also prompt for permissions if not already granted
-      return lkRoom.localParticipant
-        .createTracks({
-          audio,
-          video,
-        })
-        .then((tracks) => {
-          this.logger.info(
-            "created track",
-            tracks.map((t) => t.kind + ", " + t.id),
-          );
-          this._tracks$.next(tracks);
-        })
-        .catch((error) => {
-          this.logger.error("Failed to create tracks", error);
-        });
+
+    // We don't await the creation, because livekit could block until the tracks
+    // are fully published, and not only that they are created.
+    // We don't have control on that, localParticipant creates and publishes the tracks
+    // asap.
+    // We are using the `ParticipantEvent.LocalTrackPublished` to be notified
+    // when tracks are actually published, and at that point
+    // we can pause upstream if needed (depending on if startPublishing has been called).
+    if (audio && video) {
+      // Enable both at once in order to have a single permission prompt!
+      void lkRoom.localParticipant.enableCameraAndMicrophone();
+    } else if (audio) {
+      void lkRoom.localParticipant.setMicrophoneEnabled(true);
+    } else if (video) {
+      void lkRoom.localParticipant.setCameraEnabled(true);
+    }
+
+    return Promise.resolve();
+  }
+
+  private async pauseUpstreams(
+    lkRoom: LivekitRoom,
+    sources: Track.Source[],
+  ): Promise<void> {
+    for (const source of sources) {
+      const track = lkRoom.localParticipant.getTrackPublication(source)?.track;
+      if (track) {
+        await track.pauseUpstream();
+      } else {
+        this.logger.warn(
+          `No track found for source ${source} to pause upstream`,
+        );
+      }
+    }
+  }
+
+  private async resumeUpstreams(
+    lkRoom: LivekitRoom,
+    sources: Track.Source[],
+  ): Promise<void> {
+    for (const source of sources) {
+      const track = lkRoom.localParticipant.getTrackPublication(source)?.track;
+      if (track) {
+        await track.resumeUpstream();
+      } else {
+        this.logger.warn(
+          `No track found for source ${source} to resume upstream`,
+        );
+      }
     }
-    throw Error("audio and video is false");
   }
 
-  private _publishing$ = new BehaviorSubject<boolean>(false);
-  public publishing$ = this.scope.behavior(this._publishing$);
   /**
+   *
+   * Request to publish local tracks to the LiveKit room.
+   * This will wait for the connection to be ready before publishing.
+   * Livekit also have some local retry logic for publishing tracks.
+   * Can be called multiple times, localparticipant manages the state of published tracks (or pending publications).
    *
    * @returns
-   * @throws ElementCallError
    */
-  public async startPublishing(): Promise<LocalTrack[]> {
+  public async startPublishing(): Promise<void> {
+    if (this.shouldPublish) {
+      this.logger.debug(`Already publishing, ignoring startPublishing call`);
+      return;
+    }
+    this.shouldPublish = true;
     this.logger.debug("startPublishing called");
+
     const lkRoom = this.connection.livekitRoom;
-    const { promise, resolve, reject } = Promise.withResolvers<void>();
-    const sub = this.connection.state$.subscribe((s) => {
-      switch (s.state) {
-        case "ConnectedToLkRoom":
-          resolve();
-          break;
-        case "FailedToStart":
-          reject(
-            s.error instanceof ElementCallError
-              ? s.error
-              : new FailToStartLivekitConnection(s.error.message),
-          );
-          break;
-        default:
-          this.logger.info("waiting for connection: ", s.state);
-      }
-    });
+
+    // Resume upstream for both audio and video tracks
+    // We need to call it explicitly because call setTrackEnabled does not always
+    // resume upstream. It will only if you switch the track from disabled to enabled,
+    // but if the track is already enabled but upstream is paused, it won't resume it.
+    // TODO what about screen share?
     try {
-      await promise;
+      await this.resumeUpstreams(lkRoom, [
+        Track.Source.Microphone,
+        Track.Source.Camera,
+      ]);
     } catch (e) {
-      throw e;
-    } finally {
-      sub.unsubscribe();
+      this.logger.error(`Failed to resume upstreams`, e);
     }
-
-    for (const track of this.tracks$.value) {
-      this.logger.info("publish ", this.tracks$.value.length, "tracks");
-      // TODO: handle errors? Needs the signaling connection to be up, but it has some retries internally
-      // with a timeout.
-      await lkRoom.localParticipant.publishTrack(track).catch((error) => {
-        this.logger.error("Failed to publish track", error);
-        throw new FailToStartLivekitConnection(
-          error instanceof Error ? error.message : error,
-        );
-      });
-      this.logger.info("published track ", track.kind, track.id);
-
-      // TODO: check if the connection is still active? and break the loop if not?
-    }
-    this._publishing$.next(true);
-    return this.tracks$.value;
   }
 
   public async stopPublishing(): Promise<void> {
     this.logger.debug("stopPublishing called");
-    // TODO-MULTI-SFU: Move these calls back to ObservableScope.onEnd once scope
-    // actually has the right lifetime
-    this.muteStates.audio.unsetHandler();
-    this.muteStates.video.unsetHandler();
-
-    const localParticipant = this.connection.livekitRoom.localParticipant;
-    const tracks: LocalTrack[] = [];
-    const addToTracksIfDefined = (p: LocalTrackPublication): void => {
-      if (p.track !== undefined) tracks.push(p.track);
-    };
-    localParticipant.trackPublications.forEach(addToTracksIfDefined);
-    this.logger.debug(
-      "list of tracks to unpublish:",
-      tracks.map((t) => t.kind + ", " + t.id),
-      "start unpublishing now",
-    );
-    await localParticipant.unpublishTracks(tracks).catch((error) => {
-      this.logger.error("Failed to unpublish tracks", error);
-      throw error;
-    });
-    this.logger.debug(
-      "unpublished tracks",
-      tracks.map((t) => t.kind + ", " + t.id),
-    );
-    this._publishing$.next(false);
+    this.shouldPublish = false;
+    // Pause upstream will stop sending media to the server, while keeping
+    // the local MediaStreamTrack active, so the user can still see themselves.
+    await this.pauseUpstreams(this.connection.livekitRoom, [
+      Track.Source.Microphone,
+      Track.Source.Camera,
+      Track.Source.ScreenShare,
+    ]);
   }
 
-  /**
-   * Stops all tracks that are currently running
-   */
-  public stopTracks(): void {
-    this.tracks$.value.forEach((t) => t.stop());
-    this._tracks$.next([]);
+  public async stopTracks(): Promise<void> {
+    const lkRoom = this.connection.livekitRoom;
+    for (const source of [
+      Track.Source.Microphone,
+      Track.Source.Camera,
+      Track.Source.ScreenShare,
+    ]) {
+      const localPub = lkRoom.localParticipant.getTrackPublication(source);
+      if (localPub?.track) {
+        // stops and unpublishes the track
+        await lkRoom.localParticipant.unpublishTrack(localPub!.track, true);
+      }
+    }
   }
 
   /// Private methods
@@ -331,22 +356,35 @@ export class Publisher {
 
   /**
    * Observe changes in the mute states and update the LiveKit room accordingly.
-   * @param scope
    * @private
    */
-  private observeMuteStates(scope: ObservableScope): void {
+  private observeMuteStates(): void {
     const lkRoom = this.connection.livekitRoom;
-    this.muteStates.audio.setHandler(async (desired) => {
+    this.muteStates.audio.setHandler(async (enable) => {
       try {
-        await lkRoom.localParticipant.setMicrophoneEnabled(desired);
+        this.logger.debug(
+          `handler: Setting LiveKit microphone enabled: ${enable}`,
+        );
+        await lkRoom.localParticipant.setMicrophoneEnabled(enable);
+        // Unmute will restart the track if it was paused upstream,
+        // but until explicitly requested, we want to keep it paused.
+        if (!this.shouldPublish && enable) {
+          await this.pauseUpstreams(lkRoom, [Track.Source.Microphone]);
+        }
       } catch (e) {
         this.logger.error("Failed to update LiveKit audio input mute state", e);
       }
       return lkRoom.localParticipant.isMicrophoneEnabled;
     });
-    this.muteStates.video.setHandler(async (desired) => {
+    this.muteStates.video.setHandler(async (enable) => {
       try {
-        await lkRoom.localParticipant.setCameraEnabled(desired);
+        this.logger.debug(`handler: Setting LiveKit camera enabled: ${enable}`);
+        await lkRoom.localParticipant.setCameraEnabled(enable);
+        // Unmute will restart the track if it was paused upstream,
+        // but until explicitly requested, we want to keep it paused.
+        if (!this.shouldPublish && enable) {
+          await this.pauseUpstreams(lkRoom, [Track.Source.Camera]);
+        }
       } catch (e) {
         this.logger.error("Failed to update LiveKit video input mute state", e);
       }
@@ -362,7 +400,7 @@ export class Publisher {
     const track$ = scope.behavior(
       observeTrackReference$(room.localParticipant, Track.Source.Camera).pipe(
         map((trackRef) => {
-          const track = trackRef?.publication?.track;
+          const track = trackRef?.publication.track;
           return track instanceof LocalVideoTrack ? track : null;
         }),
       ),

src/state/CallViewModel/remoteMembers/Connection.test.ts

@@ -30,13 +30,17 @@ import { logger } from "matrix-js-sdk/lib/logger";
 import type { LivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
 import {
   Connection,
+  ConnectionState,
   type ConnectionOpts,
-  type ConnectionState,
-  type PublishingParticipant,
 } from "./Connection.ts";
 import { ObservableScope } from "../../ObservableScope.ts";
 import { type OpenIDClientParts } from "../../../livekit/openIDSFU.ts";
-import { FailToGetOpenIdToken } from "../../../utils/errors.ts";
+import {
+  ElementCallError,
+  FailToGetOpenIdToken,
+} from "../../../utils/errors.ts";
+import { testJWTToken } from "../../../utils/test-fixtures.ts";
+import { mockRemoteParticipant } from "../../../utils/test.ts";
 
 let testScope: ObservableScope;
 
@@ -47,11 +51,6 @@ let fakeLivekitRoom: MockedObject<LivekitRoom>;
 let localParticipantEventEmiter: EventEmitter;
 let fakeLocalParticipant: MockedObject<LocalParticipant>;
 
-let fakeRoomEventEmiter: EventEmitter;
-// let fakeMembershipsFocusMap$: BehaviorSubject<
-//   { membership: CallMembership; transport: LivekitTransport }[]
-// >;
-
 const livekitFocus: LivekitTransport = {
   livekit_alias: "!roomID:example.org",
   livekit_service_url: "https://matrix-rtc.example.org/livekit/jwt",
@@ -88,22 +87,25 @@ function setupTest(): void {
       localParticipantEventEmiter,
     ),
   } as unknown as LocalParticipant);
-  fakeRoomEventEmiter = new EventEmitter();
 
+  const fakeRoomEventEmitter = new EventEmitter();
   fakeLivekitRoom = vi.mocked<LivekitRoom>({
     connect: vi.fn(),
     disconnect: vi.fn(),
     remoteParticipants: new Map(),
     localParticipant: fakeLocalParticipant,
     state: LivekitConnectionState.Disconnected,
-    on: fakeRoomEventEmiter.on.bind(fakeRoomEventEmiter),
-    off: fakeRoomEventEmiter.off.bind(fakeRoomEventEmiter),
-    addListener: fakeRoomEventEmiter.addListener.bind(fakeRoomEventEmiter),
+    on: fakeRoomEventEmitter.on.bind(fakeRoomEventEmitter),
+    off: fakeRoomEventEmitter.off.bind(fakeRoomEventEmitter),
+    addListener: fakeRoomEventEmitter.addListener.bind(fakeRoomEventEmitter),
     removeListener:
-      fakeRoomEventEmiter.removeListener.bind(fakeRoomEventEmiter),
+      fakeRoomEventEmitter.removeListener.bind(fakeRoomEventEmitter),
     removeAllListeners:
-      fakeRoomEventEmiter.removeAllListeners.bind(fakeRoomEventEmiter),
+      fakeRoomEventEmitter.removeAllListeners.bind(fakeRoomEventEmitter),
     setE2EEEnabled: vi.fn().mockResolvedValue(undefined),
+    emit: (eventName: string | symbol, ...args: unknown[]) => {
+      fakeRoomEventEmitter.emit(eventName, ...args);
+    },
   } as unknown as LivekitRoom);
 }
 
@@ -120,12 +122,21 @@ function setupRemoteConnection(): Connection {
       status: 200,
       body: {
         url: "wss://matrix-rtc.m.localhost/livekit/sfu",
-        jwt: "ATOKEN",
+        jwt: testJWTToken,
       },
     };
   });
 
-  fakeLivekitRoom.connect.mockResolvedValue(undefined);
+  fakeLivekitRoom.connect.mockImplementation(async (): Promise<void> => {
+    const changeEv = RoomEvent.ConnectionStateChanged;
+
+    fakeLivekitRoom.state = LivekitConnectionState.Connecting;
+    fakeLivekitRoom.emit(changeEv, fakeLivekitRoom.state);
+    fakeLivekitRoom.state = LivekitConnectionState.Connected;
+    fakeLivekitRoom.emit(changeEv, fakeLivekitRoom.state);
+
+    return Promise.resolve();
+  });
 
   return new Connection(opts, logger);
 }
@@ -148,7 +159,7 @@ describe("Start connection states", () => {
     };
     const connection = new Connection(opts, logger);
 
-    expect(connection.state$.getValue().state).toEqual("Initialized");
+    expect(connection.state$.getValue()).toEqual("Initialized");
   });
 
   it("fail to getOpenId token then error state", async () => {
@@ -164,7 +175,7 @@ describe("Start connection states", () => {
 
     const connection = new Connection(opts, logger);
 
-    const capturedStates: ConnectionState[] = [];
+    const capturedStates: (ConnectionState | Error)[] = [];
     const s = connection.state$.subscribe((value) => {
       capturedStates.push(value);
     });
@@ -184,22 +195,20 @@ describe("Start connection states", () => {
 
     let capturedState = capturedStates.pop();
     expect(capturedState).toBeDefined();
-    expect(capturedState!.state).toEqual("FetchingConfig");
+    expect(capturedState!).toEqual("FetchingConfig");
 
     deferred.reject(new FailToGetOpenIdToken(new Error("Failed to get token")));
 
     await vi.runAllTimersAsync();
 
     capturedState = capturedStates.pop();
-    if (capturedState!.state === "FailedToStart") {
-      expect(capturedState!.error.message).toEqual("Something went wrong");
+    if (capturedState instanceof Error) {
+      expect(capturedState.message).toEqual("Something went wrong");
       expect(connection.transport.livekit_alias).toEqual(
         livekitFocus.livekit_alias,
       );
     } else {
-      expect.fail(
-        "Expected FailedToStart state but got " + capturedState?.state,
-      );
+      expect.fail("Expected FailedToStart state but got " + capturedState);
     }
   });
 
@@ -216,7 +225,7 @@ describe("Start connection states", () => {
 
     const connection = new Connection(opts, logger);
 
-    const capturedStates: ConnectionState[] = [];
+    const capturedStates: (ConnectionState | Error)[] = [];
     const s = connection.state$.subscribe((value) => {
       capturedStates.push(value);
     });
@@ -238,24 +247,25 @@ describe("Start connection states", () => {
 
     let capturedState = capturedStates.pop();
     expect(capturedState).toBeDefined();
-    expect(capturedState?.state).toEqual("FetchingConfig");
+    expect(capturedState).toEqual(ConnectionState.FetchingConfig);
 
     deferredSFU.resolve();
     await vi.runAllTimersAsync();
 
     capturedState = capturedStates.pop();
 
-    if (capturedState?.state === "FailedToStart") {
-      expect(capturedState?.error.message).toContain(
-        "SFU Config fetch failed with exception Error",
+    if (
+      capturedState instanceof ElementCallError &&
+      capturedState.cause instanceof Error
+    ) {
+      expect(capturedState.cause.message).toContain(
+        "SFU Config fetch failed with exception",
       );
       expect(connection.transport.livekit_alias).toEqual(
         livekitFocus.livekit_alias,
       );
     } else {
-      expect.fail(
-        "Expected FailedToStart state but got " + capturedState?.state,
-      );
+      expect.fail("Expected FailedToStart state but got " + capturedState);
     }
   });
 
@@ -272,7 +282,7 @@ describe("Start connection states", () => {
 
     const connection = new Connection(opts, logger);
 
-    const capturedStates: ConnectionState[] = [];
+    const capturedStates: (ConnectionState | Error)[] = [];
     const s = connection.state$.subscribe((value) => {
       capturedStates.push(value);
     });
@@ -285,7 +295,7 @@ describe("Start connection states", () => {
         status: 200,
         body: {
           url: "wss://matrix-rtc.m.localhost/livekit/sfu",
-          jwt: "ATOKEN",
+          jwt: testJWTToken,
         },
       };
     });
@@ -302,15 +312,18 @@ describe("Start connection states", () => {
     let capturedState = capturedStates.pop();
     expect(capturedState).toBeDefined();
 
-    expect(capturedState?.state).toEqual("FetchingConfig");
+    expect(capturedState).toEqual(ConnectionState.FetchingConfig);
 
     deferredSFU.resolve();
     await vi.runAllTimersAsync();
 
     capturedState = capturedStates.pop();
 
-    if (capturedState && capturedState?.state === "FailedToStart") {
-      expect(capturedState.error.message).toContain(
+    if (
+      capturedState instanceof ElementCallError &&
+      capturedState.cause instanceof Error
+    ) {
+      expect(capturedState.cause.message).toContain(
         "Failed to connect to livekit",
       );
       expect(connection.transport.livekit_alias).toEqual(
@@ -329,7 +342,7 @@ describe("Start connection states", () => {
 
     const connection = setupRemoteConnection();
 
-    const capturedStates: ConnectionState[] = [];
+    const capturedStates: (ConnectionState | Error)[] = [];
     const s = connection.state$.subscribe((value) => {
       capturedStates.push(value);
     });
@@ -339,13 +352,15 @@ describe("Start connection states", () => {
     await vi.runAllTimersAsync();
 
     const initialState = capturedStates.shift();
-    expect(initialState?.state).toEqual("Initialized");
+    expect(initialState).toEqual(ConnectionState.Initialized);
     const fetchingState = capturedStates.shift();
-    expect(fetchingState?.state).toEqual("FetchingConfig");
+    expect(fetchingState).toEqual(ConnectionState.FetchingConfig);
+    const disconnectedState = capturedStates.shift();
+    expect(disconnectedState).toEqual(ConnectionState.LivekitDisconnected);
     const connectingState = capturedStates.shift();
-    expect(connectingState?.state).toEqual("ConnectingToLkRoom");
+    expect(connectingState).toEqual(ConnectionState.LivekitConnecting);
     const connectedState = capturedStates.shift();
-    expect(connectedState?.state).toEqual("ConnectedToLkRoom");
+    expect(connectedState).toEqual(ConnectionState.LivekitConnected);
   });
 
   it("shutting down the scope should stop the connection", async () => {
@@ -363,46 +378,32 @@ describe("Start connection states", () => {
   });
 });
 
-function fakeRemoteLivekitParticipant(
-  id: string,
-  publications: number = 1,
-): RemoteParticipant {
-  return {
-    identity: id,
-    getTrackPublications: () => Array(publications),
-  } as unknown as RemoteParticipant;
-}
-
-describe("Publishing participants observations", () => {
-  it("should emit the list of publishing participants", () => {
+describe("remote participants", () => {
+  it("emits the list of remote participants", () => {
     setupTest();
 
     const connection = setupRemoteConnection();
 
-    const bobIsAPublisher = Promise.withResolvers<void>();
-    const danIsAPublisher = Promise.withResolvers<void>();
-    const observedPublishers: PublishingParticipant[][] = [];
-    const s = connection.remoteParticipantsWithTracks$.subscribe(
-      (publishers) => {
-        observedPublishers.push(publishers);
-        if (publishers.some((p) => p.identity === "@bob:example.org:DEV111")) {
-          bobIsAPublisher.resolve();
-        }
-        if (publishers.some((p) => p.identity === "@dan:example.org:DEV333")) {
-          danIsAPublisher.resolve();
-        }
-      },
-    );
+    const observedParticipants: RemoteParticipant[][] = [];
+    const s = connection.remoteParticipants$.subscribe((participants) => {
+      observedParticipants.push(participants);
+    });
     onTestFinished(() => s.unsubscribe());
-    // The publishingParticipants$ observable is derived from the current members of the
+    // The remoteParticipants$ observable is derived from the current members of the
     // livekitRoom and the rtc membership in order to publish the members that are publishing
     // on this connection.
 
     let participants: RemoteParticipant[] = [
-      fakeRemoteLivekitParticipant("@alice:example.org:DEV000", 0),
-      fakeRemoteLivekitParticipant("@bob:example.org:DEV111", 0),
-      fakeRemoteLivekitParticipant("@carol:example.org:DEV222", 0),
-      fakeRemoteLivekitParticipant("@dan:example.org:DEV333", 0),
+      mockRemoteParticipant({ identity: "@alice:example.org:DEV000" }),
+      mockRemoteParticipant({ identity: "@bob:example.org:DEV111" }),
+      mockRemoteParticipant({ identity: "@carol:example.org:DEV222" }),
+      // Mock Dan to have no published tracks. We want him to still show show up
+      // in the participants list.
+      mockRemoteParticipant({
+        identity: "@dan:example.org:DEV333",
+        getTrackPublication: () => undefined,
+        getTrackPublications: () => [],
+      }),
     ];
 
     // Let's simulate 3 members on the livekitRoom
@@ -411,24 +412,26 @@ describe("Publishing participants observations", () => {
     );
 
     participants.forEach((p) =>
-      fakeRoomEventEmiter.emit(RoomEvent.ParticipantConnected, p),
+      fakeLivekitRoom.emit(RoomEvent.ParticipantConnected, p),
     );
 
-    // At this point there should be no publishers
-    expect(observedPublishers.pop()!.length).toEqual(0);
+    // At this point there should be ~~no~~ publishers
+    // We do have publisher now, since we do not filter for publishers anymore (to also have participants with only data tracks)
+    // The filtering we do is just based on the matrixRTC member events.
+    expect(observedParticipants.pop()!.length).toEqual(4);
 
     participants = [
-      fakeRemoteLivekitParticipant("@alice:example.org:DEV000", 1),
-      fakeRemoteLivekitParticipant("@bob:example.org:DEV111", 1),
-      fakeRemoteLivekitParticipant("@carol:example.org:DEV222", 1),
-      fakeRemoteLivekitParticipant("@dan:example.org:DEV333", 2),
+      mockRemoteParticipant({ identity: "@alice:example.org:DEV000" }),
+      mockRemoteParticipant({ identity: "@bob:example.org:DEV111" }),
+      mockRemoteParticipant({ identity: "@carol:example.org:DEV222" }),
+      mockRemoteParticipant({ identity: "@dan:example.org:DEV333" }),
     ];
     participants.forEach((p) =>
-      fakeRoomEventEmiter.emit(RoomEvent.ParticipantConnected, p),
+      fakeLivekitRoom.emit(RoomEvent.ParticipantConnected, p),
     );
 
     // At this point there should be no publishers
-    expect(observedPublishers.pop()!.length).toEqual(4);
+    expect(observedParticipants.pop()!.length).toEqual(4);
   });
 
   it("should be scoped to parent scope", (): void => {
@@ -436,16 +439,14 @@ describe("Publishing participants observations", () => {
 
     const connection = setupRemoteConnection();
 
-    let observedPublishers: PublishingParticipant[][] = [];
-    const s = connection.remoteParticipantsWithTracks$.subscribe(
-      (publishers) => {
-        observedPublishers.push(publishers);
-      },
-    );
+    let observedParticipants: RemoteParticipant[][] = [];
+    const s = connection.remoteParticipants$.subscribe((participants) => {
+      observedParticipants.push(participants);
+    });
     onTestFinished(() => s.unsubscribe());
 
     let participants: RemoteParticipant[] = [
-      fakeRemoteLivekitParticipant("@bob:example.org:DEV111", 0),
+      mockRemoteParticipant({ identity: "@bob:example.org:DEV111" }),
     ];
 
     // Let's simulate 3 members on the livekitRoom
@@ -454,38 +455,29 @@ describe("Publishing participants observations", () => {
     );
 
     for (const participant of participants) {
-      fakeRoomEventEmiter.emit(RoomEvent.ParticipantConnected, participant);
+      fakeLivekitRoom.emit(RoomEvent.ParticipantConnected, participant);
     }
 
-    // At this point there should be no publishers
-    expect(observedPublishers.pop()!.length).toEqual(0);
-
-    participants = [fakeRemoteLivekitParticipant("@bob:example.org:DEV111", 1)];
-
-    for (const participant of participants) {
-      fakeRoomEventEmiter.emit(RoomEvent.ParticipantConnected, participant);
-    }
-
-    // We should have bob has a publisher now
-    const publishers = observedPublishers.pop();
-    expect(publishers?.length).toEqual(1);
-    expect(publishers?.[0]?.identity).toEqual("@bob:example.org:DEV111");
+    // We should have bob as a participant now
+    const ps = observedParticipants.pop();
+    expect(ps?.length).toEqual(1);
+    expect(ps?.[0]?.identity).toEqual("@bob:example.org:DEV111");
 
     // end the parent scope
     testScope.end();
-    observedPublishers = [];
+    observedParticipants = [];
 
-    // SHOULD NOT emit any more publishers as the scope is ended
+    // SHOULD NOT emit any more participants as the scope is ended
     participants = participants.filter(
       (p) => p.identity !== "@bob:example.org:DEV111",
     );
 
-    fakeRoomEventEmiter.emit(
+    fakeLivekitRoom.emit(
       RoomEvent.ParticipantDisconnected,
-      fakeRemoteLivekitParticipant("@bob:example.org:DEV111"),
+      mockRemoteParticipant({ identity: "@bob:example.org:DEV111" }),
     );
 
-    expect(observedPublishers.length).toEqual(0);
+    expect(observedParticipants.length).toEqual(0);
   });
 });
 

src/state/CallViewModel/remoteMembers/Connection.ts

@@ -12,11 +12,8 @@ import {
 } from "@livekit/components-core";
 import {
   ConnectionError,
-  type ConnectionState as LivekitConenctionState,
   type Room as LivekitRoom,
-  type LocalParticipant,
   type RemoteParticipant,
-  RoomEvent,
 } from "livekit-client";
 import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
 import { BehaviorSubject, map } from "rxjs";
@@ -30,12 +27,12 @@ import {
 import { type Behavior } from "../../Behavior.ts";
 import { type ObservableScope } from "../../ObservableScope.ts";
 import {
+  ElementCallError,
   InsufficientCapacityError,
   SFURoomCreationRestrictedError,
+  UnknownCallError,
 } from "../../../utils/errors.ts";
 
-export type PublishingParticipant = LocalParticipant | RemoteParticipant;
-
 export interface ConnectionOpts {
   /** The media transport to connect to. */
   transport: LivekitTransport;
@@ -47,17 +44,30 @@ export interface ConnectionOpts {
   /** Optional factory to create the LiveKit room, mainly for testing purposes. */
   livekitRoomFactory: () => LivekitRoom;
 }
+export class FailedToStartError extends Error {
+  public constructor(message: string) {
+    super(message);
+    this.name = "FailedToStartError";
+  }
+}
 
-export type ConnectionState =
-  | { state: "Initialized" }
-  | { state: "FetchingConfig" }
-  | { state: "ConnectingToLkRoom" }
-  | {
-      state: "ConnectedToLkRoom";
-      livekitConnectionState$: Behavior<LivekitConenctionState>;
-    }
-  | { state: "FailedToStart"; error: Error }
-  | { state: "Stopped" };
+export enum ConnectionState {
+  /** The start state of a connection. It has been created but nothing has loaded yet. */
+  Initialized = "Initialized",
+  /** `start` has been called on the connection. It aquires the jwt info to conenct to the LK Room  */
+  FetchingConfig = "FetchingConfig",
+  Stopped = "Stopped",
+  /** The same as ConnectionState.Disconnected from `livekit-client` */
+  LivekitDisconnected = "disconnected",
+  /** The same as ConnectionState.Connecting from `livekit-client` */
+  LivekitConnecting = "connecting",
+  /** The same as ConnectionState.Connected from `livekit-client` */
+  LivekitConnected = "connected",
+  /** The same as ConnectionState.Reconnecting from `livekit-client` */
+  LivekitReconnecting = "reconnecting",
+  /** The same as ConnectionState.SignalReconnecting from `livekit-client` */
+  LivekitSignalReconnecting = "signalReconnecting",
+}
 
 /**
  * A connection to a Matrix RTC LiveKit backend.
@@ -66,14 +76,14 @@ export type ConnectionState =
  */
 export class Connection {
   // Private Behavior
-  private readonly _state$ = new BehaviorSubject<ConnectionState>({
-    state: "Initialized",
-  });
+  private readonly _state$ = new BehaviorSubject<
+    ConnectionState | ElementCallError
+  >(ConnectionState.Initialized);
 
   /**
    * The current state of the connection to the media transport.
    */
-  public readonly state$: Behavior<ConnectionState> = this._state$;
+  public readonly state$: Behavior<ConnectionState | Error> = this._state$;
 
   /**
    * The media transport to connect to.
@@ -85,13 +95,13 @@ export class Connection {
   private scope: ObservableScope;
 
   /**
-   * An observable of the participants that are publishing on this connection. (Excluding our local participant)
-   * This is derived from `participantsIncludingSubscribers$` and `remoteTransports$`.
-   * It filters the participants to only those that are associated with a membership that claims to publish on this connection.
+   * The remote LiveKit participants that are visible on this connection.
+   *
+   * Note that this may include participants that are connected only to
+   * subscribe, or publishers that are otherwise unattested in MatrixRTC state.
+   * It is therefore more low-level than what should be presented to the user.
    */
-  public readonly remoteParticipantsWithTracks$: Behavior<
-    PublishingParticipant[]
-  >;
+  public readonly remoteParticipants$: Behavior<RemoteParticipant[]>;
 
   /**
    * Whether the connection has been stopped.
@@ -117,16 +127,24 @@ export class Connection {
     this.logger.debug("Starting Connection");
     this.stopped = false;
     try {
-      this._state$.next({
-        state: "FetchingConfig",
-      });
+      this._state$.next(ConnectionState.FetchingConfig);
+      // We should already have this information after creating the localTransport.
+      // It would probably be better to forward this here.
       const { url, jwt } = await this.getSFUConfigWithOpenID();
       // If we were stopped while fetching the config, don't proceed to connect
       if (this.stopped) return;
 
-      this._state$.next({
-        state: "ConnectingToLkRoom",
-      });
+      // Setup observer once we are done with getSFUConfigWithOpenID
+      connectionStateObserver(this.livekitRoom)
+        .pipe(
+          this.scope.bind(),
+          map((s) => s as unknown as ConnectionState),
+        )
+        .subscribe((lkState) => {
+          // It is save to cast lkState to ConnectionState as they are fully overlapping.
+          this._state$.next(lkState);
+        });
+
       try {
         await this.livekitRoom.connect(url, jwt);
       } catch (e) {
@@ -141,7 +159,8 @@ export class Connection {
             throw new InsufficientCapacityError();
           }
           if (e.status === 404) {
-            // error msg is "Could not establish signal connection: requested room does not exist"
+            // error msg is "Failed to create call"
+            // error description is "Call creation might be restricted to authorized users only. Try again later, or contact your server admin if the problem persists."
             // The room does not exist. There are two different modes of operation for the SFU:
             // - the room is created on the fly when connecting (livekit `auto_create` option)
             // - Only authorized users can create rooms, so the room must exist before connecting (done by the auth jwt service)
@@ -153,19 +172,16 @@ export class Connection {
       }
       // If we were stopped while connecting, don't proceed to update state.
       if (this.stopped) return;
-
-      this._state$.next({
-        state: "ConnectedToLkRoom",
-        livekitConnectionState$: this.scope.behavior(
-          connectionStateObserver(this.livekitRoom),
-        ),
-      });
     } catch (error) {
       this.logger.debug(`Failed to connect to LiveKit room: ${error}`);
-      this._state$.next({
-        state: "FailedToStart",
-        error: error instanceof Error ? error : new Error(`${error}`),
-      });
+      this._state$.next(
+        error instanceof ElementCallError
+          ? error
+          : error instanceof Error
+            ? new UnknownCallError(error)
+            : new UnknownCallError(new Error(`${error}`)),
+      );
+      // Its okay to ignore the throw. The error is part of the state.
       throw error;
     }
   }
@@ -190,9 +206,7 @@ export class Connection {
     );
     if (this.stopped) return;
     await this.livekitRoom.disconnect();
-    this._state$.next({
-      state: "Stopped",
-    });
+    this._state$.next(ConnectionState.Stopped);
     this.stopped = true;
   }
 
@@ -204,12 +218,12 @@ export class Connection {
    *
    * @param opts - Connection options {@link ConnectionOpts}.
    *
-   * @param logger
+   * @param logger - The logger to use.
    */
   public constructor(opts: ConnectionOpts, logger: Logger) {
     this.logger = logger.getChild("[Connection]");
     this.logger.info(
-      `[Connection] Creating new connection to ${opts.transport.livekit_service_url} ${opts.transport.livekit_alias}`,
+      `Creating new connection to ${opts.transport.livekit_service_url} ${opts.transport.livekit_alias}`,
     );
     const { transport, client, scope } = opts;
 
@@ -218,23 +232,9 @@ export class Connection {
     this.transport = transport;
     this.client = client;
 
-    // REMOTE participants with track!!!
-    // this.remoteParticipantsWithTracks$
-    this.remoteParticipantsWithTracks$ = scope.behavior(
-      // only tracks remote participants
-      connectedParticipantsObserver(this.livekitRoom, {
-        additionalRoomEvents: [
-          RoomEvent.TrackPublished,
-          RoomEvent.TrackUnpublished,
-        ],
-      }).pipe(
-        map((participants) => {
-          return participants.filter(
-            (participant) => participant.getTrackPublications().length > 0,
-          );
-        }),
-      ),
-      [],
+    this.remoteParticipants$ = scope.behavior(
+      // Only tracks remote participants
+      connectedParticipantsObserver(this.livekitRoom),
     );
 
     scope.onEnd(() => {

src/state/CallViewModel/remoteMembers/ConnectionFactory.ts

@@ -14,7 +14,8 @@ import {
   type BaseE2EEManager,
 } from "livekit-client";
 import { type Logger } from "matrix-js-sdk/lib/logger";
-import E2EEWorker from "livekit-client/e2ee-worker?worker";
+// imported as inline to support worker when loaded from a cdn (cross domain)
+import E2EEWorker from "livekit-client/e2ee-worker?worker&inline";
 
 import { type ObservableScope } from "../../ObservableScope.ts";
 import { Connection } from "./Connection.ts";
@@ -42,11 +43,11 @@ export class ECConnectionFactory implements ConnectionFactory {
    * @param client - The OpenID client parts for authentication, needed to get openID and JWT tokens.
    * @param devices - Used for video/audio out/in capture options.
    * @param processorState$ - Effects like background blur (only for publishing connection?)
-   * @param livekitKeyProvider
+   * @param livekitKeyProvider - Optional key provider for end-to-end encryption.
    * @param controlledAudioDevices - Option to indicate whether audio output device is controlled externally (native mobile app).
+   * @param livekitRoomFactory - Optional factory function (for testing) to create LivekitRoom instances. If not provided, a default factory is used.
    * @param echoCancellation - Whether to enable echo cancellation for audio capture.
    * @param noiseSuppression - Whether to enable noise suppression for audio capture.
-   * @param livekitRoomFactory - Optional factory function (for testing) to create LivekitRoom instances. If not provided, a default factory is used.
    */
   public constructor(
     private client: OpenIDClientParts,

src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts

@@ -8,7 +8,7 @@ Please see LICENSE in the repository root for full details.
 import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";
 import { BehaviorSubject } from "rxjs";
 import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
-import { type Participant as LivekitParticipant } from "livekit-client";
+import { type RemoteParticipant } from "livekit-client";
 import { logger } from "matrix-js-sdk/lib/logger";
 
 import { Epoch, mapEpoch, ObservableScope } from "../../ObservableScope.ts";
@@ -52,7 +52,7 @@ beforeEach(() => {
       (transport: LivekitTransport, scope: ObservableScope) => {
         const mockConnection = {
           transport,
-          remoteParticipantsWithTracks$: new BehaviorSubject([]),
+          remoteParticipants$: new BehaviorSubject([]),
         } as unknown as Connection;
         vi.mocked(mockConnection).start = vi.fn();
         vi.mocked(mockConnection).stop = vi.fn();
@@ -200,24 +200,21 @@ describe("connections$ stream", () => {
 });
 
 describe("connectionManagerData$ stream", () => {
-  // Used in test to control fake connections' remoteParticipantsWithTracks$ streams
-  let fakePublishingParticipantsStreams: Map<
-    string,
-    Behavior<LivekitParticipant[]>
-  >;
+  // Used in test to control fake connections' remoteParticipants$ streams
+  let fakeRemoteParticipantsStreams: Map<string, Behavior<RemoteParticipant[]>>;
 
   function keyForTransport(transport: LivekitTransport): string {
     return `${transport.livekit_service_url}|${transport.livekit_alias}`;
   }
 
   beforeEach(() => {
-    fakePublishingParticipantsStreams = new Map();
+    fakeRemoteParticipantsStreams = new Map();
 
-    function getPublishingParticipantsFor(
+    function getRemoteParticipantsFor(
       transport: LivekitTransport,
-    ): Behavior<LivekitParticipant[]> {
+    ): Behavior<RemoteParticipant[]> {
       return (
-        fakePublishingParticipantsStreams.get(keyForTransport(transport)) ??
+        fakeRemoteParticipantsStreams.get(keyForTransport(transport)) ??
         new BehaviorSubject([])
       );
     }
@@ -227,13 +224,12 @@ describe("connectionManagerData$ stream", () => {
       .fn()
       .mockImplementation(
         (transport: LivekitTransport, scope: ObservableScope) => {
-          const fakePublishingParticipants$ = new BehaviorSubject<
-            LivekitParticipant[]
+          const fakeRemoteParticipants$ = new BehaviorSubject<
+            RemoteParticipant[]
           >([]);
           const mockConnection = {
             transport,
-            remoteParticipantsWithTracks$:
-              getPublishingParticipantsFor(transport),
+            remoteParticipants$: getRemoteParticipantsFor(transport),
           } as unknown as Connection;
           vi.mocked(mockConnection).start = vi.fn();
           vi.mocked(mockConnection).stop = vi.fn();
@@ -242,36 +238,36 @@ describe("connectionManagerData$ stream", () => {
             void mockConnection.stop();
           });
 
-          fakePublishingParticipantsStreams.set(
+          fakeRemoteParticipantsStreams.set(
             keyForTransport(transport),
-            fakePublishingParticipants$,
+            fakeRemoteParticipants$,
           );
           return mockConnection;
         },
       );
   });
 
-  test("Should report connections with the publishing participants", () => {
+  test("Should report connections with the remote participants", () => {
     withTestScheduler(({ expectObservable, schedule, behavior }) => {
       // Setup the fake participants streams behavior
       // ==============================
-      fakePublishingParticipantsStreams.set(
+      fakeRemoteParticipantsStreams.set(
         keyForTransport(TRANSPORT_1),
         behavior("oa-b", {
           o: [],
-          a: [{ identity: "user1A" } as LivekitParticipant],
+          a: [{ identity: "user1A" } as RemoteParticipant],
           b: [
-            { identity: "user1A" } as LivekitParticipant,
-            { identity: "user1B" } as LivekitParticipant,
+            { identity: "user1A" } as RemoteParticipant,
+            { identity: "user1B" } as RemoteParticipant,
           ],
         }),
       );
 
-      fakePublishingParticipantsStreams.set(
+      fakeRemoteParticipantsStreams.set(
         keyForTransport(TRANSPORT_2),
         behavior("o-a", {
           o: [],
-          a: [{ identity: "user2A" } as LivekitParticipant],
+          a: [{ identity: "user2A" } as RemoteParticipant],
         }),
       );
       // ==============================
@@ -289,47 +285,47 @@ describe("connectionManagerData$ stream", () => {
         a: expect.toSatisfy((e) => {
           const data: ConnectionManagerData = e.value;
           expect(data.getConnections().length).toBe(2);
-          expect(data.getParticipantForTransport(TRANSPORT_1).length).toBe(0);
-          expect(data.getParticipantForTransport(TRANSPORT_2).length).toBe(0);
+          expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(0);
+          expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(0);
           return true;
         }),
         b: expect.toSatisfy((e) => {
           const data: ConnectionManagerData = e.value;
           expect(data.getConnections().length).toBe(2);
-          expect(data.getParticipantForTransport(TRANSPORT_1).length).toBe(1);
-          expect(data.getParticipantForTransport(TRANSPORT_2).length).toBe(0);
-          expect(data.getParticipantForTransport(TRANSPORT_1)[0].identity).toBe(
-            "user1A",
-          );
+          expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(1);
+          expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(0);
+          expect(
+            data.getParticipantsForTransport(TRANSPORT_1)[0].identity,
+          ).toBe("user1A");
           return true;
         }),
         c: expect.toSatisfy((e) => {
           const data: ConnectionManagerData = e.value;
           expect(data.getConnections().length).toBe(2);
-          expect(data.getParticipantForTransport(TRANSPORT_1).length).toBe(1);
-          expect(data.getParticipantForTransport(TRANSPORT_2).length).toBe(1);
-          expect(data.getParticipantForTransport(TRANSPORT_1)[0].identity).toBe(
-            "user1A",
-          );
-          expect(data.getParticipantForTransport(TRANSPORT_2)[0].identity).toBe(
-            "user2A",
-          );
+          expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(1);
+          expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(1);
+          expect(
+            data.getParticipantsForTransport(TRANSPORT_1)[0].identity,
+          ).toBe("user1A");
+          expect(
+            data.getParticipantsForTransport(TRANSPORT_2)[0].identity,
+          ).toBe("user2A");
           return true;
         }),
         d: expect.toSatisfy((e) => {
           const data: ConnectionManagerData = e.value;
           expect(data.getConnections().length).toBe(2);
-          expect(data.getParticipantForTransport(TRANSPORT_1).length).toBe(2);
-          expect(data.getParticipantForTransport(TRANSPORT_2).length).toBe(1);
-          expect(data.getParticipantForTransport(TRANSPORT_1)[0].identity).toBe(
-            "user1A",
-          );
-          expect(data.getParticipantForTransport(TRANSPORT_1)[1].identity).toBe(
-            "user1B",
-          );
-          expect(data.getParticipantForTransport(TRANSPORT_2)[0].identity).toBe(
-            "user2A",
-          );
+          expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(2);
+          expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(1);
+          expect(
+            data.getParticipantsForTransport(TRANSPORT_1)[0].identity,
+          ).toBe("user1A");
+          expect(
+            data.getParticipantsForTransport(TRANSPORT_1)[1].identity,
+          ).toBe("user1B");
+          expect(
+            data.getParticipantsForTransport(TRANSPORT_2)[0].identity,
+          ).toBe("user2A");
           return true;
         }),
       });

src/state/CallViewModel/remoteMembers/ConnectionManager.ts

@@ -6,13 +6,10 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
 Please see LICENSE in the repository root for full details.
 */
 
-import {
-  type LivekitTransport,
-  type ParticipantId,
-} from "matrix-js-sdk/lib/matrixrtc";
-import { BehaviorSubject, combineLatest, map, of, switchMap, tap } from "rxjs";
+import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
+import { combineLatest, map, of, switchMap, tap } from "rxjs";
 import { type Logger } from "matrix-js-sdk/lib/logger";
-import { type LocalParticipant, type RemoteParticipant } from "livekit-client";
+import { type RemoteParticipant } from "livekit-client";
 
 import { type Behavior } from "../../Behavior.ts";
 import { type Connection } from "./Connection.ts";
@@ -24,21 +21,18 @@ import { type ConnectionFactory } from "./ConnectionFactory.ts";
 export class ConnectionManagerData {
   private readonly store: Map<
     string,
-    [Connection, (LocalParticipant | RemoteParticipant)[]]
+    { connection: Connection; participants: RemoteParticipant[] }
   > = new Map();
 
   public constructor() {}
 
-  public add(
-    connection: Connection,
-    participants: (LocalParticipant | RemoteParticipant)[],
-  ): void {
+  public add(connection: Connection, participants: RemoteParticipant[]): void {
     const key = this.getKey(connection.transport);
     const existing = this.store.get(key);
     if (!existing) {
-      this.store.set(key, [connection, participants]);
+      this.store.set(key, { connection, participants });
     } else {
-      existing[1].push(...participants);
+      existing.participants.push(...participants);
     }
   }
 
@@ -47,58 +41,46 @@ export class ConnectionManagerData {
   }
 
   public getConnections(): Connection[] {
-    return Array.from(this.store.values()).map(([connection]) => connection);
+    return Array.from(this.store.values()).map(({ connection }) => connection);
   }
 
   public getConnectionForTransport(
     transport: LivekitTransport,
   ): Connection | null {
-    return this.store.get(this.getKey(transport))?.[0] ?? null;
+    return this.store.get(this.getKey(transport))?.connection ?? null;
   }
 
-  public getParticipantForTransport(
+  public getParticipantsForTransport(
     transport: LivekitTransport,
-  ): (LocalParticipant | RemoteParticipant)[] {
+  ): RemoteParticipant[] {
     const key = transport.livekit_service_url + "|" + transport.livekit_alias;
     const existing = this.store.get(key);
     if (existing) {
-      return existing[1];
+      return existing.participants;
     }
     return [];
   }
-  /**
-   * Get all connections where the given participant is publishing.
-   * In theory, there could be several connections where the same participant is publishing but with
-   * only well behaving clients a participant should only be publishing on a single connection.
-   * @param participantId
-   */
-  public getConnectionsForParticipant(
-    participantId: ParticipantId,
-  ): Connection[] {
-    const connections: Connection[] = [];
-    for (const [connection, participants] of this.store.values()) {
-      if (participants.some((p) => p.identity === participantId)) {
-        connections.push(connection);
-      }
-    }
-    return connections;
-  }
 }
+
 interface Props {
   scope: ObservableScope;
   connectionFactory: ConnectionFactory;
   inputTransports$: Behavior<Epoch<LivekitTransport[]>>;
   logger: Logger;
 }
+
 // TODO - write test for scopes (do we really need to bind scope)
 export interface IConnectionManager {
   connectionManagerData$: Behavior<Epoch<ConnectionManagerData>>;
 }
+
 /**
  * Crete a `ConnectionManager`
- * @param scope the observable scope used by this object.
- * @param connectionFactory used to create new connections.
- * @param _transportsSubscriptions$ A list of Behaviors each containing a LIST of LivekitTransport.
+ * @param props - Configuration object
+ * @param props.scope - The observable scope used by this object
+ * @param props.connectionFactory - Used to create new connections
+ * @param props.inputTransports$ - A list of Behaviors each containing a LIST of LivekitTransport.
+ * @param props.logger - The logger to use
  *   Each of these behaviors can be interpreted as subscribed list of transports.
  *
  *   Using `registerTransports` independent external modules can control what connections
@@ -115,9 +97,6 @@ export function createConnectionManager$({
   logger: parentLogger,
 }: Props): IConnectionManager {
   const logger = parentLogger.getChild("[ConnectionManager]");
-
-  const running$ = new BehaviorSubject(true);
-  scope.onEnd(() => running$.next(false));
   // TODO logger: only construct one logger from the client and make it compatible via a EC specific sing
 
   /**
@@ -129,10 +108,7 @@ export function createConnectionManager$({
    * externally this is modified via `registerTransports()`.
    */
   const transports$ = scope.behavior(
-    combineLatest([running$, inputTransports$]).pipe(
-      map(([running, transports]) =>
-        transports.mapInner((transport) => (running ? transport : [])),
-      ),
+    inputTransports$.pipe(
       map((transports) => transports.mapInner(removeDuplicateTransports)),
       tap(({ value: transports }) => {
         logger.trace(
@@ -182,23 +158,25 @@ export function createConnectionManager$({
         const epoch = connections.epoch;
 
         // Map the connections to list of {connection, participants}[]
-        const listOfConnectionsWithPublishingParticipants =
-          connections.value.map((connection) => {
-            return connection.remoteParticipantsWithTracks$.pipe(
+        const listOfConnectionsWithRemoteParticipants = connections.value.map(
+          (connection) => {
+            return connection.remoteParticipants$.pipe(
               map((participants) => ({
                 connection,
                 participants,
               })),
             );
-          });
+          },
+        );
 
         // probably not required
-        if (listOfConnectionsWithPublishingParticipants.length === 0) {
+
+        if (listOfConnectionsWithRemoteParticipants.length === 0) {
           return of(new Epoch(new ConnectionManagerData(), epoch));
         }
 
         // combineLatest the several streams into a single stream with the ConnectionManagerData
-        return combineLatest(listOfConnectionsWithPublishingParticipants).pipe(
+        return combineLatest(listOfConnectionsWithRemoteParticipants).pipe(
           map(
             (lists) =>
               new Epoch(

src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts

@@ -15,7 +15,7 @@ import { combineLatest, map, type Observable } from "rxjs";
 
 import { type IConnectionManager } from "./ConnectionManager.ts";
 import {
-  type MatrixLivekitMember,
+  type RemoteMatrixLivekitMember,
   createMatrixLivekitMembers$,
 } from "./MatrixLivekitMembers.ts";
 import {
@@ -91,7 +91,7 @@ test("should signal participant not yet connected to livekit", () => {
       }),
     );
 
-    const matrixLivekitMember$ = createMatrixLivekitMembers$({
+    const matrixLivekitMembers$ = createMatrixLivekitMembers$({
       scope: testScope,
       membershipsWithTransport$: testScope.behavior(membershipsWithTransport$),
       connectionManager: {
@@ -99,21 +99,24 @@ test("should signal participant not yet connected to livekit", () => {
       } as unknown as IConnectionManager,
     });
 
-    expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe("a", {
-      a: expect.toSatisfy((data: MatrixLivekitMember[]) => {
-        expect(data.length).toEqual(1);
-        expectObservable(data[0].membership$).toBe("a", {
-          a: bobMembership,
-        });
-        expectObservable(data[0].participant$).toBe("a", {
-          a: null,
-        });
-        expectObservable(data[0].connection$).toBe("a", {
-          a: null,
-        });
-        return true;
-      }),
-    });
+    expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe(
+      "a",
+      {
+        a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => {
+          expect(data.length).toEqual(1);
+          expectObservable(data[0].membership$).toBe("a", {
+            a: bobMembership,
+          });
+          expectObservable(data[0].participant.value$).toBe("a", {
+            a: null,
+          });
+          expectObservable(data[0].connection$).toBe("a", {
+            a: null,
+          });
+          return true;
+        }),
+      },
+    );
   });
 });
 
@@ -171,7 +174,7 @@ test("should signal participant on a connection that is publishing", () => {
       }),
     );
 
-    const matrixLivekitMember$ = createMatrixLivekitMembers$({
+    const matrixLivekitMembers$ = createMatrixLivekitMembers$({
       scope: testScope,
       membershipsWithTransport$: testScope.behavior(membershipsWithTransport$),
       connectionManager: {
@@ -179,25 +182,28 @@ test("should signal participant on a connection that is publishing", () => {
       } as unknown as IConnectionManager,
     });
 
-    expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe("a", {
-      a: expect.toSatisfy((data: MatrixLivekitMember[]) => {
-        expect(data.length).toEqual(1);
-        expectObservable(data[0].membership$).toBe("a", {
-          a: bobMembership,
-        });
-        expectObservable(data[0].participant$).toBe("a", {
-          a: expect.toSatisfy((participant) => {
-            expect(participant).toBeDefined();
-            expect(participant!.identity).toEqual(bobParticipantId);
-            return true;
-          }),
-        });
-        expectObservable(data[0].connection$).toBe("a", {
-          a: connection,
-        });
-        return true;
-      }),
-    });
+    expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe(
+      "a",
+      {
+        a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => {
+          expect(data.length).toEqual(1);
+          expectObservable(data[0].membership$).toBe("a", {
+            a: bobMembership,
+          });
+          expectObservable(data[0].participant.value$).toBe("a", {
+            a: expect.toSatisfy((participant) => {
+              expect(participant).toBeDefined();
+              expect(participant!.identity).toEqual(bobParticipantId);
+              return true;
+            }),
+          });
+          expectObservable(data[0].connection$).toBe("a", {
+            a: connection,
+          });
+          return true;
+        }),
+      },
+    );
   });
 });
 
@@ -222,7 +228,7 @@ test("should signal participant on a connection that is not publishing", () => {
       }),
     );
 
-    const matrixLivekitMember$ = createMatrixLivekitMembers$({
+    const matrixLivekitMembers$ = createMatrixLivekitMembers$({
       scope: testScope,
       membershipsWithTransport$: testScope.behavior(membershipsWithTransport$),
       connectionManager: {
@@ -230,21 +236,24 @@ test("should signal participant on a connection that is not publishing", () => {
       } as unknown as IConnectionManager,
     });
 
-    expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe("a", {
-      a: expect.toSatisfy((data: MatrixLivekitMember[]) => {
-        expect(data.length).toEqual(1);
-        expectObservable(data[0].membership$).toBe("a", {
-          a: bobMembership,
-        });
-        expectObservable(data[0].participant$).toBe("a", {
-          a: null,
-        });
-        expectObservable(data[0].connection$).toBe("a", {
-          a: connection,
-        });
-        return true;
-      }),
-    });
+    expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe(
+      "a",
+      {
+        a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => {
+          expect(data.length).toEqual(1);
+          expectObservable(data[0].membership$).toBe("a", {
+            a: bobMembership,
+          });
+          expectObservable(data[0].participant.value$).toBe("a", {
+            a: null,
+          });
+          expectObservable(data[0].connection$).toBe("a", {
+            a: connection,
+          });
+          return true;
+        }),
+      },
+    );
   });
 });
 
@@ -283,7 +292,7 @@ describe("Publication edge case", () => {
         }),
       );
 
-      const matrixLivekitMember$ = createMatrixLivekitMembers$({
+      const matrixLivekitMembers$ = createMatrixLivekitMembers$({
         scope: testScope,
         membershipsWithTransport$: testScope.behavior(
           membershipsWithTransport$,
@@ -293,10 +302,10 @@ describe("Publication edge case", () => {
         } as unknown as IConnectionManager,
       });
 
-      expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe(
+      expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe(
         "a",
         {
-          a: expect.toSatisfy((data: MatrixLivekitMember[]) => {
+          a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => {
             expect(data.length).toEqual(2);
             expectObservable(data[0].membership$).toBe("a", {
               a: bobMembership,
@@ -305,7 +314,7 @@ describe("Publication edge case", () => {
               // The real connection should be from transportA as per the membership
               a: connectionA,
             });
-            expectObservable(data[0].participant$).toBe("a", {
+            expectObservable(data[0].participant.value$).toBe("a", {
               a: expect.toSatisfy((participant) => {
                 expect(participant).toBeDefined();
                 expect(participant!.identity).toEqual(bobParticipantId);
@@ -349,7 +358,7 @@ describe("Publication edge case", () => {
         }),
       );
 
-      const matrixLivekitMember$ = createMatrixLivekitMembers$({
+      const matrixLivekitMembers$ = createMatrixLivekitMembers$({
         scope: testScope,
         membershipsWithTransport$: testScope.behavior(
           membershipsWithTransport$,
@@ -359,10 +368,10 @@ describe("Publication edge case", () => {
         } as unknown as IConnectionManager,
       });
 
-      expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe(
+      expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe(
         "a",
         {
-          a: expect.toSatisfy((data: MatrixLivekitMember[]) => {
+          a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => {
             expect(data.length).toEqual(2);
             expectObservable(data[0].membership$).toBe("a", {
               a: bobMembership,
@@ -371,7 +380,7 @@ describe("Publication edge case", () => {
               // The real connection should be from transportA as per the membership
               a: connectionA,
             });
-            expectObservable(data[0].participant$).toBe("a", {
+            expectObservable(data[0].participant.value$).toBe("a", {
               // No participant as Bob is not publishing on his membership transport
               a: null,
             });

src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts

@@ -5,10 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
 Please see LICENSE in the repository root for full details.
 */
 
-import {
-  type LocalParticipant as LocalLivekitParticipant,
-  type RemoteParticipant as RemoteLivekitParticipant,
-} from "livekit-client";
+import { type LocalParticipant, type RemoteParticipant } from "livekit-client";
 import {
   type LivekitTransport,
   type CallMembership,
@@ -24,22 +21,44 @@ import { generateItemsWithEpoch } from "../../../utils/observable";
 
 const logger = rootLogger.getChild("[MatrixLivekitMembers]");
 
-/**
- * Represents a Matrix call member and their associated LiveKit participation.
- * `livekitParticipant` can be undefined if the member is not yet connected to the livekit room
- * or if it has no livekit transport at all.
- */
-export interface MatrixLivekitMember {
+interface LocalTaggedParticipant {
+  type: "local";
+  value$: Behavior<LocalParticipant | null>;
+}
+interface RemoteTaggedParticipant {
+  type: "remote";
+  value$: Behavior<RemoteParticipant | null>;
+}
+export type TaggedParticipant =
+  | LocalTaggedParticipant
+  | RemoteTaggedParticipant;
+
+interface MatrixLivekitMember {
   membership$: Behavior<CallMembership>;
-  participant$: Behavior<
-    LocalLivekitParticipant | RemoteLivekitParticipant | null
-  >;
   connection$: Behavior<Connection | null>;
   // participantId: string; We do not want a participantId here since it will be generated by the jwt
   // TODO decide if we can also drop the userId. Its in the matrix membership anyways.
   userId: string;
 }
 
+/**
+ * Represents the local Matrix call member and their associated LiveKit participation.
+ * `livekitParticipant` can be null if the member is not yet connected to the livekit room
+ * or if it has no livekit transport at all.
+ */
+export interface LocalMatrixLivekitMember extends MatrixLivekitMember {
+  participant: LocalTaggedParticipant;
+}
+
+/**
+ * Represents a remote Matrix call member and their associated LiveKit participation.
+ * `livekitParticipant` can be null if the member is not yet connected to the livekit room
+ * or if it has no livekit transport at all.
+ */
+export interface RemoteMatrixLivekitMember extends MatrixLivekitMember {
+  participant: RemoteTaggedParticipant;
+}
+
 interface Props {
   scope: ObservableScope;
   membershipsWithTransport$: Behavior<
@@ -61,7 +80,7 @@ export function createMatrixLivekitMembers$({
   scope,
   membershipsWithTransport$,
   connectionManager,
-}: Props): Behavior<Epoch<MatrixLivekitMember[]>> {
+}: Props): Behavior<Epoch<RemoteMatrixLivekitMember[]>> {
   /**
    * Stream of all the call members and their associated livekit data (if available).
    */
@@ -91,7 +110,7 @@ export function createMatrixLivekitMembers$({
             const participantId = /*membership.membershipID*/ `${membership.userId}:${membership.deviceId}`;
 
             const participants = transport
-              ? managerData.getParticipantForTransport(transport)
+              ? managerData.getParticipantsForTransport(transport)
               : [];
             const participant =
               participants.find((p) => p.identity == participantId) ?? null;
@@ -108,14 +127,16 @@ export function createMatrixLivekitMembers$({
         // Each update where the key of the generator array do not change will result in updates to the `data$` observable in the factory.
         (scope, data$, participantId, userId) => {
           logger.debug(
-            `Updating data$ for participantId: ${participantId}, userId: ${userId}`,
+            `Generating member for participantId: ${participantId}, userId: ${userId}`,
           );
+          const { participant$, ...rest } = scope.splitBehavior(data$);
           // will only get called once per `participantId, userId` pair.
           // updates to data$ and as a result to displayName$ and mxcAvatarUrl$ are more frequent.
           return {
             participantId,
             userId,
-            ...scope.splitBehavior(data$),
+            participant: { type: "remote" as const, value$: participant$ },
+            ...rest,
           };
         },
       ),

src/state/CallViewModel/remoteMembers/integration.test.ts

@@ -29,10 +29,11 @@ import { type ProcessorState } from "../../../livekit/TrackProcessorContext.tsx"
 import {
   areLivekitTransportsEqual,
   createMatrixLivekitMembers$,
-  type MatrixLivekitMember,
+  type RemoteMatrixLivekitMember,
 } from "./MatrixLivekitMembers.ts";
 import { createConnectionManager$ } from "./ConnectionManager.ts";
 import { membershipsAndTransports$ } from "../../SessionBehaviors.ts";
+import { testJWTToken } from "../../../utils/test-fixtures.ts";
 
 // Test the integration of ConnectionManager and MatrixLivekitMerger
 
@@ -85,7 +86,7 @@ beforeEach(() => {
       status: 200,
       body: {
         url: `wss://${domain}/livekit/sfu`,
-        jwt: "ATOKEN",
+        jwt: testJWTToken,
       },
     };
   });
@@ -124,15 +125,15 @@ test("bob, carl, then bob joining no tracks yet", () => {
       logger: logger,
     });
 
-    const matrixLivekitItems$ = createMatrixLivekitMembers$({
+    const matrixLivekitMembers$ = createMatrixLivekitMembers$({
       scope: testScope,
       membershipsWithTransport$:
         membershipsAndTransports.membershipsWithTransport$,
       connectionManager,
     });
 
-    expectObservable(matrixLivekitItems$).toBe(vMarble, {
-      a: expect.toSatisfy((e: Epoch<MatrixLivekitMember[]>) => {
+    expectObservable(matrixLivekitMembers$).toBe(vMarble, {
+      a: expect.toSatisfy((e: Epoch<RemoteMatrixLivekitMember[]>) => {
         const items = e.value;
         expect(items.length).toBe(1);
         const item = items[0]!;
@@ -147,12 +148,12 @@ test("bob, carl, then bob joining no tracks yet", () => {
             ),
           ),
         });
-        expectObservable(item.participant$).toBe("a", {
+        expectObservable(item.participant.value$).toBe("a", {
           a: null,
         });
         return true;
       }),
-      b: expect.toSatisfy((e: Epoch<MatrixLivekitMember[]>) => {
+      b: expect.toSatisfy((e: Epoch<RemoteMatrixLivekitMember[]>) => {
         const items = e.value;
         expect(items.length).toBe(2);
 
@@ -161,7 +162,7 @@ test("bob, carl, then bob joining no tracks yet", () => {
           expectObservable(item.membership$).toBe("a", {
             a: bobMembership,
           });
-          expectObservable(item.participant$).toBe("a", {
+          expectObservable(item.participant.value$).toBe("a", {
             a: null,
           });
         }
@@ -172,7 +173,7 @@ test("bob, carl, then bob joining no tracks yet", () => {
           expectObservable(item.membership$).toBe("a", {
             a: carlMembership,
           });
-          expectObservable(item.participant$).toBe("a", {
+          expectObservable(item.participant.value$).toBe("a", {
             a: null,
           });
           expectObservable(item.connection$).toBe("a", {
@@ -189,7 +190,7 @@ test("bob, carl, then bob joining no tracks yet", () => {
         }
         return true;
       }),
-      c: expect.toSatisfy((e: Epoch<MatrixLivekitMember[]>) => {
+      c: expect.toSatisfy((e: Epoch<RemoteMatrixLivekitMember[]>) => {
         const items = e.value;
         expect(items.length).toBe(3);
 
@@ -216,7 +217,7 @@ test("bob, carl, then bob joining no tracks yet", () => {
               return true;
             }),
           });
-          expectObservable(item.participant$).toBe("a", {
+          expectObservable(item.participant.value$).toBe("a", {
             a: null,
           });
         }

src/state/CallViewModelWidget.test.ts

@@ -15,6 +15,7 @@ import { constant } from "./Behavior.ts";
 import { aliceParticipant, localRtcMember } from "../utils/test-fixtures.ts";
 import { ElementWidgetActions, widget } from "../widget.ts";
 import { E2eeType } from "../e2ee/e2eeType.ts";
+import { MatrixRTCMode } from "../settings/settings.ts";
 
 vi.mock("@livekit/components-core", { spy: true });
 
@@ -34,36 +35,43 @@ vi.mock("../widget", () => ({
   },
 }));
 
-it("expect leave when ElementWidgetActions.HangupCall is called", async () => {
-  const pr = Promise.withResolvers<string>();
-  withCallViewModel(
-    {
-      remoteParticipants$: constant([aliceParticipant]),
-      rtcMembers$: constant([localRtcMember]),
-    },
-    (vm: CallViewModel) => {
-      vm.leave$.subscribe((s: string) => {
-        pr.resolve(s);
-      });
+it.each([
+  [MatrixRTCMode.Legacy],
+  [MatrixRTCMode.Compatibil],
+  [MatrixRTCMode.Matrix_2_0],
+])(
+  "expect leave when ElementWidgetActions.HangupCall is called (%s mode)",
+  async (mode) => {
+    const pr = Promise.withResolvers<string>();
+    withCallViewModel(mode)(
+      {
+        remoteParticipants$: constant([aliceParticipant]),
+        rtcMembers$: constant([localRtcMember]),
+      },
+      (vm: CallViewModel) => {
+        vm.leave$.subscribe((s: string) => {
+          pr.resolve(s);
+        });
 
-      widget!.lazyActions!.emit(
-        ElementWidgetActions.HangupCall,
-        new CustomEvent(ElementWidgetActions.HangupCall, {
-          detail: {
-            action: "im.vector.hangup",
-            api: "toWidget",
-            data: {},
-            requestId: "widgetapi-1761237395918",
-            widgetId: "mrUjS9T6uKUOWHMxXvLbSv0F",
-          },
-        }),
-      );
-    },
-    {
-      encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
-    },
-  );
+        widget!.lazyActions!.emit(
+          ElementWidgetActions.HangupCall,
+          new CustomEvent(ElementWidgetActions.HangupCall, {
+            detail: {
+              action: "im.vector.hangup",
+              api: "toWidget",
+              data: {},
+              requestId: "widgetapi-1761237395918",
+              widgetId: "mrUjS9T6uKUOWHMxXvLbSv0F",
+            },
+          }),
+        );
+      },
+      {
+        encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
+      },
+    );
 
-  const source = await pr.promise;
-  expect(source).toBe("user");
-});
+    const source = await pr.promise;
+    expect(source).toBe("user");
+  },
+);

src/state/MediaDevices.ts

@@ -212,9 +212,10 @@ class AudioInput implements MediaDevice<DeviceLabel, SelectedAudioInputDevice> {
   }
 }
 
-class AudioOutput
-  implements MediaDevice<AudioOutputDeviceLabel, SelectedAudioOutputDevice>
-{
+class AudioOutput implements MediaDevice<
+  AudioOutputDeviceLabel,
+  SelectedAudioOutputDevice
+> {
   private logger = rootLogger.getChild("[MediaDevices AudioOutput]");
   public readonly available$ = this.scope.behavior(
     availableRawDevices$(
@@ -274,9 +275,10 @@ class AudioOutput
   }
 }
 
-class ControlledAudioOutput
-  implements MediaDevice<AudioOutputDeviceLabel, SelectedAudioOutputDevice>
-{
+class ControlledAudioOutput implements MediaDevice<
+  AudioOutputDeviceLabel,
+  SelectedAudioOutputDevice
+> {
   private logger = rootLogger.getChild("[MediaDevices ControlledAudioOutput]");
   // We need to subscribe to the raw devices so that the OS does update the input
   // back to what it was before. otherwise we will switch back to the default

src/state/MediaViewModel.test.ts

@@ -20,6 +20,7 @@ import {
   createLocalMedia,
   createRemoteMedia,
   withTestScheduler,
+  mockRemoteParticipant,
 } from "../utils/test";
 import { getValue } from "../utils/observable";
 import { constant } from "./Behavior";
@@ -44,7 +45,11 @@ const rtcMembership = mockRtcMembership("@alice:example.org", "AAAA");
 
 test("control a participant's volume", () => {
   const setVolumeSpy = vi.fn();
-  const vm = createRemoteMedia(rtcMembership, {}, { setVolume: setVolumeSpy });
+  const vm = createRemoteMedia(
+    rtcMembership,
+    {},
+    mockRemoteParticipant({ setVolume: setVolumeSpy }),
+  );
   withTestScheduler(({ expectObservable, schedule }) => {
     schedule("-ab---c---d|", {
       a() {
@@ -88,7 +93,7 @@ test("control a participant's volume", () => {
 });
 
 test("toggle fit/contain for a participant's video", () => {
-  const vm = createRemoteMedia(rtcMembership, {}, {});
+  const vm = createRemoteMedia(rtcMembership, {}, mockRemoteParticipant({}));
   withTestScheduler(({ expectObservable, schedule }) => {
     schedule("-ab|", {
       a: () => vm.toggleFitContain(),
@@ -199,3 +204,35 @@ test("switch cameras", async () => {
   });
   expect(deviceId).toBe("front camera");
 });
+
+test("remote media is in waiting state when participant has not yet connected", () => {
+  const vm = createRemoteMedia(rtcMembership, {}, null); // null participant
+  expect(vm.waitingForMedia$.value).toBe(true);
+});
+
+test("remote media is not in waiting state when participant is connected", () => {
+  const vm = createRemoteMedia(rtcMembership, {}, mockRemoteParticipant({}));
+  expect(vm.waitingForMedia$.value).toBe(false);
+});
+
+test("remote media is not in waiting state when participant is connected with no publications", () => {
+  const vm = createRemoteMedia(
+    rtcMembership,
+    {},
+    mockRemoteParticipant({
+      getTrackPublication: () => undefined,
+      getTrackPublications: () => [],
+    }),
+  );
+  expect(vm.waitingForMedia$.value).toBe(false);
+});
+
+test("remote media is not in waiting state when user does not intend to publish anywhere", () => {
+  const vm = createRemoteMedia(
+    rtcMembership,
+    {},
+    mockRemoteParticipant({}),
+    undefined, // No room (no advertised transport)
+  );
+  expect(vm.waitingForMedia$.value).toBe(false);
+});

src/state/MediaViewModel.ts

@@ -7,8 +7,8 @@ Please see LICENSE in the repository root for full details.
 
 import {
   type AudioSource,
-  type TrackReferenceOrPlaceholder,
   type VideoSource,
+  type TrackReference,
   observeParticipantEvents,
   observeParticipantMedia,
   roomEventSelector,
@@ -33,7 +33,6 @@ import {
   type Observable,
   Subject,
   combineLatest,
-  distinctUntilKeyChanged,
   filter,
   fromEvent,
   interval,
@@ -60,14 +59,11 @@ import { type ObservableScope } from "./ObservableScope";
 export function observeTrackReference$(
   participant: Participant,
   source: Track.Source,
-): Observable<TrackReferenceOrPlaceholder> {
+): Observable<TrackReference | undefined> {
   return observeParticipantMedia(participant).pipe(
-    map(() => ({
-      participant: participant,
-      publication: participant.getTrackPublication(source),
-      source,
-    })),
-    distinctUntilKeyChanged("publication"),
+    map(() => participant.getTrackPublication(source)),
+    distinctUntilChanged(),
+    map((publication) => publication && { participant, publication, source }),
   );
 }
 
@@ -226,7 +222,7 @@ abstract class BaseMediaViewModel {
   /**
    * The LiveKit video track for this media.
    */
-  public readonly video$: Behavior<TrackReferenceOrPlaceholder | null>;
+  public readonly video$: Behavior<TrackReference | undefined>;
   /**
    * Whether there should be a warning that this media is unencrypted.
    */
@@ -241,10 +237,12 @@ abstract class BaseMediaViewModel {
 
   private observeTrackReference$(
     source: Track.Source,
-  ): Behavior<TrackReferenceOrPlaceholder | null> {
+  ): Behavior<TrackReference | undefined> {
     return this.scope.behavior(
       this.participant$.pipe(
-        switchMap((p) => (!p ? of(null) : observeTrackReference$(p, source))),
+        switchMap((p) =>
+          !p ? of(undefined) : observeTrackReference$(p, source),
+        ),
       ),
     );
   }
@@ -268,7 +266,7 @@ abstract class BaseMediaViewModel {
     encryptionSystem: EncryptionSystem,
     audioSource: AudioSource,
     videoSource: VideoSource,
-    livekitRoom$: Behavior<LivekitRoom | undefined>,
+    protected readonly livekitRoom$: Behavior<LivekitRoom | undefined>,
     public readonly focusUrl$: Behavior<string | undefined>,
     public readonly displayName$: Behavior<string>,
     public readonly mxcAvatarUrl$: Behavior<string | undefined>,
@@ -281,8 +279,8 @@ abstract class BaseMediaViewModel {
         [audio$, this.video$],
         (a, v) =>
           encryptionSystem.kind !== E2eeType.NONE &&
-          (a?.publication?.isEncrypted === false ||
-            v?.publication?.isEncrypted === false),
+          (a?.publication.isEncrypted === false ||
+            v?.publication.isEncrypted === false),
       ),
     );
 
@@ -471,7 +469,7 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel {
   private readonly videoTrack$: Observable<LocalVideoTrack | null> =
     this.video$.pipe(
       switchMap((v) => {
-        const track = v?.publication?.track;
+        const track = v?.publication.track;
         if (!(track instanceof LocalVideoTrack)) return of(null);
         return merge(
           // Watch for track restarts because they indicate a camera switch.
@@ -596,6 +594,21 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel {
  * A remote participant's user media.
  */
 export class RemoteUserMediaViewModel extends BaseUserMediaViewModel {
+  /**
+   * Whether we are waiting for this user's LiveKit participant to exist. This
+   * could be because either we or the remote party are still connecting.
+   */
+  public readonly waitingForMedia$ = this.scope.behavior<boolean>(
+    combineLatest(
+      [this.livekitRoom$, this.participant$],
+      (livekitRoom, participant) =>
+        // If livekitRoom is undefined, the user is not attempting to publish on
+        // any transport and so we shouldn't expect a participant. (They might
+        // be a subscribe-only bot for example.)
+        livekitRoom !== undefined && participant === null,
+    ),
+  );
+
   // This private field is used to override the value from the superclass
   private __speaking$: Behavior<boolean>;
   public get speaking$(): Behavior<boolean> {

src/state/MuteStates.test.ts

@@ -48,13 +48,7 @@ describe("MuteState", () => {
       select(): void {},
     } as unknown as MediaDevice<DeviceLabel, SelectedDevice>;
 
-    const muteState = new MuteState(
-      testScope,
-      deviceStub,
-      constant(true),
-      true,
-      forceMute$,
-    );
+    const muteState = new MuteState(testScope, deviceStub, true, forceMute$);
     let lastEnabled: boolean = false;
     muteState.enabled$.subscribe((enabled) => {
       lastEnabled = enabled;
@@ -163,12 +157,10 @@ describe("MuteStates", () => {
       videoInput: aVideoInput(),
       // other devices are not relevant for this test
     });
-    const muteStates = new MuteStates(
-      testScope,
-      mediaDevices,
-      // consider joined
-      constant(true),
-    );
+    const muteStates = new MuteStates(testScope, mediaDevices, {
+      audioEnabled: false,
+      videoEnabled: false,
+    });
 
     let latestSyncedState: boolean | null = null;
     muteStates.video.setHandler(async (enabled: boolean): Promise<boolean> => {

src/state/MuteStates.ts

@@ -24,8 +24,6 @@ import {
 
 import { type MediaDevices, type MediaDevice } from "../state/MediaDevices";
 import { ElementWidgetActions, widget } from "../widget";
-import { Config } from "../config/Config";
-import { getUrlParams } from "../UrlParams";
 import { type ObservableScope } from "./ObservableScope";
 import { type Behavior, constant } from "./Behavior";
 
@@ -43,12 +41,6 @@ const defaultHandler: Handler = async (desired) => Promise.resolve(desired);
  * Do not use directly outside of tests.
  */
 export class MuteState<Label, Selected> {
-  // TODO: rewrite this to explain behavior, it is not understandable, and cannot add logging
-  private readonly enabledByDefault$ =
-    this.enabledByConfig && !getUrlParams().skipLobby
-      ? this.joined$.pipe(map((isJoined) => !isJoined))
-      : of(false);
-
   private readonly handler$ = new BehaviorSubject(defaultHandler);
 
   public setHandler(handler: Handler): void {
@@ -73,76 +65,73 @@ export class MuteState<Label, Selected> {
   private readonly data$ = this.scope.behavior<MuteStateData>(
     this.canControlDevices$.pipe(
       distinctUntilChanged(),
-      withLatestFrom(
-        this.enabledByDefault$,
-        (canControlDevices, enabledByDefault) => {
+      map((canControlDevices) => {
+        logger.info(
+          `MuteState: canControlDevices: ${canControlDevices}, enabled by default: ${this.enabledByDefault}`,
+        );
+        if (!canControlDevices) {
           logger.info(
-            `MuteState: canControlDevices: ${canControlDevices}, enabled by default: ${enabledByDefault}`,
+            `MuteState: devices connected: ${canControlDevices}, disabling`,
           );
-          if (!canControlDevices) {
-            logger.info(
-              `MuteState: devices connected: ${canControlDevices}, disabling`,
-            );
-            // We need to sync the mute state with the handler
-            // to ensure nothing is beeing published.
-            this.handler$.value(false).catch((err) => {
-              logger.error("MuteState-disable: handler error", err);
-            });
-            return { enabled$: of(false), set: null, toggle: null };
-          }
+          // We need to sync the mute state with the handler
+          // to ensure nothing is beeing published.
+          this.handler$.value(false).catch((err) => {
+            logger.error("MuteState-disable: handler error", err);
+          });
+          return { enabled$: of(false), set: null, toggle: null };
+        }
 
-          // Assume the default value only once devices are actually connected
-          let enabled = enabledByDefault;
-          const set$ = new Subject<boolean>();
-          const toggle$ = new Subject<void>();
-          const desired$ = merge(set$, toggle$.pipe(map(() => !enabled)));
-          const enabled$ = new Observable<boolean>((subscriber) => {
-            subscriber.next(enabled);
-            let latestDesired = enabledByDefault;
-            let syncing = false;
+        // Assume the default value only once devices are actually connected
+        let enabled = this.enabledByDefault;
+        const set$ = new Subject<boolean>();
+        const toggle$ = new Subject<void>();
+        const desired$ = merge(set$, toggle$.pipe(map(() => !enabled)));
+        const enabled$ = new Observable<boolean>((subscriber) => {
+          subscriber.next(enabled);
+          let latestDesired = this.enabledByDefault;
+          let syncing = false;
 
-            const sync = async (): Promise<void> => {
-              if (enabled === latestDesired) syncing = false;
-              else {
-                const previouslyEnabled = enabled;
-                enabled = await firstValueFrom(
-                  this.handler$.pipe(
-                    switchMap(async (handler) => handler(latestDesired)),
-                  ),
-                );
-                if (enabled === previouslyEnabled) {
-                  syncing = false;
-                } else {
-                  subscriber.next(enabled);
-                  syncing = true;
-                  sync().catch((err) => {
-                    // TODO: better error handling
-                    logger.error("MuteState: handler error", err);
-                  });
-                }
-              }
-            };
-
-            const s = desired$.subscribe((desired) => {
-              latestDesired = desired;
-              if (syncing === false) {
+          const sync = async (): Promise<void> => {
+            if (enabled === latestDesired) syncing = false;
+            else {
+              const previouslyEnabled = enabled;
+              enabled = await firstValueFrom(
+                this.handler$.pipe(
+                  switchMap(async (handler) => handler(latestDesired)),
+                ),
+              );
+              if (enabled === previouslyEnabled) {
+                syncing = false;
+              } else {
+                subscriber.next(enabled);
                 syncing = true;
                 sync().catch((err) => {
                   // TODO: better error handling
                   logger.error("MuteState: handler error", err);
                 });
               }
-            });
-            return (): void => s.unsubscribe();
-          });
-
-          return {
-            set: (enabled: boolean): void => set$.next(enabled),
-            toggle: (): void => toggle$.next(),
-            enabled$,
+            }
           };
-        },
-      ),
+
+          const s = desired$.subscribe((desired) => {
+            latestDesired = desired;
+            if (syncing === false) {
+              syncing = true;
+              sync().catch((err) => {
+                // TODO: better error handling
+                logger.error("MuteState: handler error", err);
+              });
+            }
+          });
+          return (): void => s.unsubscribe();
+        });
+
+        return {
+          set: (enabled: boolean): void => set$.next(enabled),
+          toggle: (): void => toggle$.next(),
+          enabled$,
+        };
+      }),
     ),
   );
 
@@ -160,8 +149,7 @@ export class MuteState<Label, Selected> {
   public constructor(
     private readonly scope: ObservableScope,
     private readonly device: MediaDevice<Label, Selected>,
-    private readonly joined$: Observable<boolean>,
-    private readonly enabledByConfig: boolean,
+    private readonly enabledByDefault: boolean,
     /**
      * An optional observable which, when it emits `true`, will force the mute.
      * Used for video to stop camera when earpiece mode is on.
@@ -176,10 +164,10 @@ export class MuteStates {
    *  True if the selected audio output device is an earpiece.
    *  Used to force-disable video when on earpiece.
    */
-  private readonly isEarpiece$ = combineLatest(
+  private readonly isEarpiece$ = combineLatest([
     this.mediaDevices.audioOutput.available$,
     this.mediaDevices.audioOutput.selected$,
-  ).pipe(
+  ]).pipe(
     map(([available, selected]) => {
       if (!selected?.id) return false;
       const device = available.get(selected.id);
@@ -191,22 +179,23 @@ export class MuteStates {
   public readonly audio = new MuteState(
     this.scope,
     this.mediaDevices.audioInput,
-    this.joined$,
-    Config.get().media_devices.enable_audio,
+    this.initialMuteState.audioEnabled,
     constant(false),
   );
   public readonly video = new MuteState(
     this.scope,
     this.mediaDevices.videoInput,
-    this.joined$,
-    Config.get().media_devices.enable_video,
+    this.initialMuteState.videoEnabled,
     this.isEarpiece$,
   );
 
   public constructor(
     private readonly scope: ObservableScope,
     private readonly mediaDevices: MediaDevices,
-    private readonly joined$: Observable<boolean>,
+    private readonly initialMuteState: {
+      audioEnabled: boolean;
+      videoEnabled: boolean;
+    },
   ) {
     if (widget !== null) {
       // Sync our mute states with the hosting client

src/state/UserMedia.ts

@@ -27,6 +27,7 @@ import type { ReactionOption } from "../reactions";
 import { observeSpeaker$ } from "./observeSpeaker.ts";
 import { generateItems } from "../utils/observable.ts";
 import { ScreenShare } from "./ScreenShare.ts";
+import { type TaggedParticipant } from "./CallViewModel/remoteMembers/MatrixLivekitMembers.ts";
 
 /**
  * Sorting bins defining the order in which media tiles appear in the layout.
@@ -68,40 +69,46 @@ enum SortingBin {
  * for inclusion in the call layout and tracks associated screen shares.
  */
 export class UserMedia {
-  public readonly vm: UserMediaViewModel = this.participant$.value?.isLocal
-    ? new LocalUserMediaViewModel(
-        this.scope,
-        this.id,
-        this.userId,
-        this.participant$ as Behavior<LocalParticipant | null>,
-        this.encryptionSystem,
-        this.livekitRoom$,
-        this.focusUrl$,
-        this.mediaDevices,
-        this.displayName$,
-        this.mxcAvatarUrl$,
-        this.scope.behavior(this.handRaised$),
-        this.scope.behavior(this.reaction$),
-      )
-    : new RemoteUserMediaViewModel(
-        this.scope,
-        this.id,
-        this.userId,
-        this.participant$ as Behavior<RemoteParticipant | null>,
-        this.encryptionSystem,
-        this.livekitRoom$,
-        this.focusUrl$,
-        this.pretendToBeDisconnected$,
-        this.displayName$,
-        this.mxcAvatarUrl$,
-        this.scope.behavior(this.handRaised$),
-        this.scope.behavior(this.reaction$),
-      );
+  public readonly vm: UserMediaViewModel =
+    this.participant.type === "local"
+      ? new LocalUserMediaViewModel(
+          this.scope,
+          this.id,
+          this.userId,
+          this.participant.value$,
+          this.encryptionSystem,
+          this.livekitRoom$,
+          this.focusUrl$,
+          this.mediaDevices,
+          this.displayName$,
+          this.mxcAvatarUrl$,
+          this.scope.behavior(this.handRaised$),
+          this.scope.behavior(this.reaction$),
+        )
+      : new RemoteUserMediaViewModel(
+          this.scope,
+          this.id,
+          this.userId,
+          this.participant.value$,
+          this.encryptionSystem,
+          this.livekitRoom$,
+          this.focusUrl$,
+          this.pretendToBeDisconnected$,
+          this.displayName$,
+          this.mxcAvatarUrl$,
+          this.scope.behavior(this.handRaised$),
+          this.scope.behavior(this.reaction$),
+        );
 
   private readonly speaker$ = this.scope.behavior(
     observeSpeaker$(this.vm.speaking$),
   );
 
+  // TypeScript needs this widening of the type to happen in a separate statement
+  private readonly participant$: Behavior<
+    LocalParticipant | RemoteParticipant | null
+  > = this.participant.value$;
+
   /**
    * All screen share media associated with this user media.
    */
@@ -184,9 +191,7 @@ export class UserMedia {
     private readonly scope: ObservableScope,
     public readonly id: string,
     private readonly userId: string,
-    private readonly participant$: Behavior<
-      LocalParticipant | RemoteParticipant | null
-    >,
+    private readonly participant: TaggedParticipant,
     private readonly encryptionSystem: EncryptionSystem,
     private readonly livekitRoom$: Behavior<LivekitRoom | undefined>,
     private readonly focusUrl$: Behavior<string | undefined>,

src/state/initialMuteState.test.ts

@@ -0,0 +1,97 @@
+/*
+Copyright 2026 Element Creations Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+import { test, expect } from "vitest";
+import { type RTCCallIntent } from "matrix-js-sdk/lib/matrixrtc";
+
+import { calculateInitialMuteState } from "./initialMuteState";
+
+test.each<{
+  callIntent: RTCCallIntent;
+  packageType: "full" | "embedded";
+}>([
+  { callIntent: "audio", packageType: "full" },
+  { callIntent: "audio", packageType: "embedded" },
+  { callIntent: "video", packageType: "full" },
+  { callIntent: "video", packageType: "embedded" },
+  { callIntent: "unknown", packageType: "full" },
+  { callIntent: "unknown", packageType: "embedded" },
+])(
+  "Should allow to unmute on start if not skipping lobby (callIntent: $callIntent, packageType: $packageType)",
+  ({ callIntent, packageType }) => {
+    const { audioEnabled, videoEnabled } = calculateInitialMuteState(
+      { skipLobby: false, callIntent },
+      packageType,
+    );
+    expect(audioEnabled).toBe(true);
+    expect(videoEnabled).toBe(callIntent !== "audio");
+  },
+);
+
+test.each<{
+  callIntent: RTCCallIntent;
+}>([
+  { callIntent: "audio" },
+  { callIntent: "video" },
+  { callIntent: "unknown" },
+])(
+  "Should always mute on start if skipping lobby on non embedded build (callIntent: $callIntent)",
+  ({ callIntent }) => {
+    const { audioEnabled, videoEnabled } = calculateInitialMuteState(
+      { skipLobby: true, callIntent },
+      "full",
+    );
+    expect(audioEnabled).toBe(false);
+    expect(videoEnabled).toBe(false);
+  },
+);
+
+test.each<{
+  callIntent: RTCCallIntent;
+}>([
+  { callIntent: "audio" },
+  { callIntent: "video" },
+  { callIntent: "unknown" },
+])(
+  "Can start unmuted if skipping lobby on embedded build (callIntent: $callIntent)",
+  ({ callIntent }) => {
+    const { audioEnabled, videoEnabled } = calculateInitialMuteState(
+      { skipLobby: true, callIntent },
+      "embedded",
+    );
+    expect(audioEnabled).toBe(true);
+    expect(videoEnabled).toBe(callIntent !== "audio");
+  },
+);
+
+test.each<{
+  isDevBuild: boolean;
+  currentHost: string;
+  expectedEnabled: boolean;
+}>([
+  { isDevBuild: true, currentHost: "localhost", expectedEnabled: true },
+  { isDevBuild: false, currentHost: "localhost", expectedEnabled: false },
+  { isDevBuild: true, currentHost: "call.example.com", expectedEnabled: false },
+  {
+    isDevBuild: false,
+    currentHost: "call.example.com",
+    expectedEnabled: false,
+  },
+])(
+  "Should trust localhost domain when in dev mode isDevBuild($isDevBuild) host($currentHost)",
+  ({ isDevBuild, currentHost, expectedEnabled }) => {
+    const { audioEnabled, videoEnabled } = calculateInitialMuteState(
+      { skipLobby: true, callIntent: "video" },
+      "full",
+      currentHost,
+      isDevBuild,
+    );
+
+    expect(audioEnabled).toBe(expectedEnabled);
+    expect(videoEnabled).toBe(expectedEnabled);
+  },
+);

src/state/initialMuteState.ts

@@ -0,0 +1,52 @@
+/*
+Copyright 2026 Element Creations Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+import { logger } from "matrix-js-sdk/lib/logger";
+
+import { type UrlParams } from "../UrlParams.ts";
+
+/**
+ * Calculates the initial mute state for media devices based on configuration.
+ *
+ * It is not always possible to start the widget with audio/video unmuted due to privacy concerns.
+ * This function encapsulates the logic to determine the appropriate initial state.
+ */
+export function calculateInitialMuteState(
+  urlParams: Pick<UrlParams, "skipLobby" | "callIntent">,
+  packageType: "full" | "embedded",
+  hostname: string | undefined = undefined,
+  trustLocalhost: boolean = false,
+): { audioEnabled: boolean; videoEnabled: boolean } {
+  const { skipLobby, callIntent } = urlParams;
+
+  logger.debug(
+    `calculateInitialMuteState: skipLobby=${skipLobby}, callIntent=${callIntent}, hostname=${hostname}, trustLocalhost=${trustLocalhost}`,
+  );
+
+  const isTrustedHost =
+    packageType == "embedded" ||
+    // Trust local hosts in dev mode to make local testing easier
+    (hostname == "localhost" && trustLocalhost);
+
+  if (skipLobby && !isTrustedHost) {
+    // If host not trusted and lobby skipped, default to muted to protect user privacy.
+    // This prevents users from inadvertently joining with active audio/video
+    // when browser permissions were previously granted in a different context.
+    return {
+      audioEnabled: false,
+      videoEnabled: false,
+    };
+  }
+
+  // Embedded contexts are trusted environments, so they allow unmuted by default.
+  // Same for when showing a lobby, as users can adjust their settings there.
+  // Additionally, if the call intent is "audio", we disable video by default.
+  return {
+    audioEnabled: true,
+    videoEnabled: callIntent != "audio",
+  };
+}

src/tile/GridTile.test.tsx

@@ -12,7 +12,11 @@ import { axe } from "vitest-axe";
 import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc";
 
 import { GridTile } from "./GridTile";
-import { mockRtcMembership, createRemoteMedia } from "../utils/test";
+import {
+  mockRtcMembership,
+  createRemoteMedia,
+  mockRemoteParticipant,
+} from "../utils/test";
 import { GridTileViewModel } from "../state/TileViewModel";
 import { ReactionsSenderProvider } from "../reactions/useReactionsSender";
 import type { CallViewModel } from "../state/CallViewModel/CallViewModel";
@@ -31,11 +35,11 @@ test("GridTile is accessible", async () => {
       rawDisplayName: "Alice",
       getMxcAvatarUrl: () => "mxc://adfsg",
     },
-    {
+    mockRemoteParticipant({
       setVolume() {},
       getTrackPublication: () =>
         ({}) as Partial<RemoteTrackPublication> as RemoteTrackPublication,
-    },
+    }),
   );
 
   const fakeRtcSession = {

src/tile/GridTile.tsx

@@ -69,6 +69,7 @@ interface UserMediaTileProps extends TileProps {
   vm: UserMediaViewModel;
   mirror: boolean;
   locallyMuted: boolean;
+  waitingForMedia?: boolean;
   primaryButton?: ReactNode;
   menuStart?: ReactNode;
   menuEnd?: ReactNode;
@@ -79,6 +80,7 @@ const UserMediaTile: FC<UserMediaTileProps> = ({
   vm,
   showSpeakingIndicators,
   locallyMuted,
+  waitingForMedia,
   primaryButton,
   menuStart,
   menuEnd,
@@ -148,7 +150,7 @@ const UserMediaTile: FC<UserMediaTileProps> = ({
   const tile = (
     <MediaView
       ref={ref}
-      video={video ?? undefined}
+      video={video}
       userId={vm.userId}
       unencryptedWarning={unencryptedWarning}
       encryptionStatus={encryptionStatus}
@@ -194,7 +196,7 @@ const UserMediaTile: FC<UserMediaTileProps> = ({
       raisedHandTime={handRaised ?? undefined}
       currentReaction={reaction ?? undefined}
       raisedHandOnClick={raisedHandOnClick}
-      localParticipant={vm.local}
+      waitingForMedia={waitingForMedia}
       focusUrl={focusUrl}
       audioStreamStats={audioStreamStats}
       videoStreamStats={videoStreamStats}
@@ -290,6 +292,7 @@ const RemoteUserMediaTile: FC<RemoteUserMediaTileProps> = ({
   ...props
 }) => {
   const { t } = useTranslation();
+  const waitingForMedia = useBehavior(vm.waitingForMedia$);
   const locallyMuted = useBehavior(vm.locallyMuted$);
   const localVolume = useBehavior(vm.localVolume$);
   const onSelectMute = useCallback(
@@ -311,6 +314,7 @@ const RemoteUserMediaTile: FC<RemoteUserMediaTileProps> = ({
     <UserMediaTile
       ref={ref}
       vm={vm}
+      waitingForMedia={waitingForMedia}
       locallyMuted={locallyMuted}
       mirror={false}
       menuStart={

src/tile/MediaView.test.tsx

@@ -47,7 +47,6 @@ describe("MediaView", () => {
     video: trackReference,
     userId: "@alice:example.com",
     mxcAvatarUrl: undefined,
-    localParticipant: false,
     focusable: true,
   };
 
@@ -66,24 +65,13 @@ describe("MediaView", () => {
     });
   });
 
-  describe("with no participant", () => {
-    it("shows avatar for local user", () => {
-      render(
-        <MediaView {...baseProps} video={undefined} localParticipant={true} />,
-      );
+  describe("with no video", () => {
+    it("shows avatar", () => {
+      render(<MediaView {...baseProps} video={undefined} />);
       expect(
         screen.getByRole("img", { name: "@alice:example.com" }),
       ).toBeVisible();
-      expect(screen.queryAllByText("Waiting for media...").length).toBe(0);
-    });
-    it("shows avatar and label for remote user", () => {
-      render(
-        <MediaView {...baseProps} video={undefined} localParticipant={false} />,
-      );
-      expect(
-        screen.getByRole("img", { name: "@alice:example.com" }),
-      ).toBeVisible();
-      expect(screen.getByText("Waiting for media...")).toBeVisible();
+      expect(screen.queryByTestId("video")).toBe(null);
     });
   });
 
@@ -94,6 +82,22 @@ describe("MediaView", () => {
     });
   });
 
+  describe("waitingForMedia", () => {
+    test("defaults to false", () => {
+      render(<MediaView {...baseProps} />);
+      expect(screen.queryAllByText("Waiting for media...").length).toBe(0);
+    });
+    test("shows and is accessible", async () => {
+      const { container } = render(
+        <TooltipProvider>
+          <MediaView {...baseProps} waitingForMedia={true} />
+        </TooltipProvider>,
+      );
+      expect(await axe(container)).toHaveNoViolations();
+      expect(screen.getByText("Waiting for media...")).toBeVisible();
+    });
+  });
+
   describe("unencryptedWarning", () => {
     test("is shown and accessible", async () => {
       const { container } = render(

src/tile/MediaView.tsx

@@ -43,7 +43,7 @@ interface Props extends ComponentProps<typeof animated.div> {
   raisedHandTime?: Date;
   currentReaction?: ReactionOption;
   raisedHandOnClick?: () => void;
-  localParticipant: boolean;
+  waitingForMedia?: boolean;
   audioStreamStats?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats;
   videoStreamStats?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats;
   // The focus url, mainly for debugging purposes
@@ -71,7 +71,7 @@ export const MediaView: FC<Props> = ({
   raisedHandTime,
   currentReaction,
   raisedHandOnClick,
-  localParticipant,
+  waitingForMedia,
   audioStreamStats,
   videoStreamStats,
   focusUrl,
@@ -129,7 +129,7 @@ export const MediaView: FC<Props> = ({
             />
           )}
         </div>
-        {!video && !localParticipant && (
+        {waitingForMedia && (
           <div className={styles.status}>
             {t("video_tile.waiting_for_media")}
           </div>

src/tile/SpotlightTile.test.tsx

@@ -17,6 +17,7 @@ import {
   mockRtcMembership,
   createLocalMedia,
   createRemoteMedia,
+  mockRemoteParticipant,
 } from "../utils/test";
 import { SpotlightTileViewModel } from "../state/TileViewModel";
 import { constant } from "../state/Behavior";
@@ -33,7 +34,7 @@ test("SpotlightTile is accessible", async () => {
       rawDisplayName: "Alice",
       getMxcAvatarUrl: () => "mxc://adfsg",
     },
-    {},
+    mockRemoteParticipant({}),
   );
 
   const vm2 = createLocalMedia(

src/tile/SpotlightTile.tsx

@@ -38,6 +38,7 @@ import {
   type MediaViewModel,
   ScreenShareViewModel,
   type UserMediaViewModel,
+  type RemoteUserMediaViewModel,
 } from "../state/MediaViewModel";
 import { useInitial } from "../useInitial";
 import { useMergedRefs } from "../useMergedRefs";
@@ -69,8 +70,7 @@ interface SpotlightUserMediaItemBaseProps extends SpotlightItemBaseProps {
   videoFit: "contain" | "cover";
 }
 
-interface SpotlightLocalUserMediaItemProps
-  extends SpotlightUserMediaItemBaseProps {
+interface SpotlightLocalUserMediaItemProps extends SpotlightUserMediaItemBaseProps {
   vm: LocalUserMediaViewModel;
 }
 
@@ -84,6 +84,20 @@ const SpotlightLocalUserMediaItem: FC<SpotlightLocalUserMediaItemProps> = ({
 
 SpotlightLocalUserMediaItem.displayName = "SpotlightLocalUserMediaItem";
 
+interface SpotlightRemoteUserMediaItemProps extends SpotlightUserMediaItemBaseProps {
+  vm: RemoteUserMediaViewModel;
+}
+
+const SpotlightRemoteUserMediaItem: FC<SpotlightRemoteUserMediaItemProps> = ({
+  vm,
+  ...props
+}) => {
+  const waitingForMedia = useBehavior(vm.waitingForMedia$);
+  return (
+    <MediaView waitingForMedia={waitingForMedia} mirror={false} {...props} />
+  );
+};
+
 interface SpotlightUserMediaItemProps extends SpotlightItemBaseProps {
   vm: UserMediaViewModel;
 }
@@ -103,7 +117,7 @@ const SpotlightUserMediaItem: FC<SpotlightUserMediaItemProps> = ({
   return vm instanceof LocalUserMediaViewModel ? (
     <SpotlightLocalUserMediaItem vm={vm} {...baseProps} />
   ) : (
-    <MediaView mirror={false} {...baseProps} />
+    <SpotlightRemoteUserMediaItem vm={vm} {...baseProps} />
   );
 };
 

src/useAudioContext.tsx

@@ -22,9 +22,12 @@ import * as controls from "./controls";
  * Play a sound though a given AudioContext. Will take
  * care of connecting the correct buffer and gating
  * through gain.
- * @param volume The volume to play at.
  * @param ctx The context to play through.
  * @param buffer The buffer to play.
+ * @param volume The volume to play at.
+ * @param stereoPan The stereo pan to apply.
+ * @param delayS Delay in seconds before starting playing.
+ * @param abort Optional AbortController that can be used to stop playback.
  * @returns A promise that resolves when the sound has finished playing.
  */
 async function playSound(
@@ -55,9 +58,11 @@ async function playSound(
  * Play a sound though a given AudioContext, looping until stopped. Will take
  * care of connecting the correct buffer and gating
  * through gain.
- * @param volume The volume to play at.
  * @param ctx The context to play through.
  * @param buffer The buffer to play.
+ * @param volume The volume to play at.
+ * @param stereoPan The stereo pan to apply.
+ * @param delayS Delay in seconds between each loop.
  * @returns A function used to end the sound. This function will return a promise when the sound has stopped.
  */
 function playSoundLooping(
@@ -120,7 +125,7 @@ interface UseAudioContext<S extends string> {
 /**
  * Add an audio context which can be used to play
  * a set of preloaded sounds.
- * @param props
+ * @param props The properties for the audio context.
  * @returns Either an instance that can be used to play sounds, or null if not ready.
  */
 export function useAudioContext<S extends string>(

src/utils/displayname.ts

@@ -77,6 +77,13 @@ export function shouldDisambiguate(
   );
 }
 
+/**
+ * Calculates a display name for a member, optionally disambiguating it.
+ * @param member - The member to calculate the display name for.
+ * @param member.rawDisplayName - The raw display name of the member
+ * @param member.userId - The user ID of the member
+ * @param disambiguate - Whether to disambiguate the display name.
+ */
 export function calculateDisplayName(
   member: { rawDisplayName?: string; userId: string },
   disambiguate: boolean,

src/utils/errors.ts

@@ -57,9 +57,16 @@ export class ElementCallError extends Error {
   }
 }
 
+/**
+ * Configuration problem due to no MatrixRTC backend/SFU is exposed via .well-known and no fallback configured.
+ */
 export class MatrixRTCTransportMissingError extends ElementCallError {
   public domain: string;
 
+  /**
+   * Creates an instance of MatrixRTCTransportMissingError.
+   * @param domain - The domain where the MatrixRTC transport is missing.
+   */
   public constructor(domain: string) {
     super(
       t("error.call_is_not_supported"),
@@ -75,6 +82,9 @@ export class MatrixRTCTransportMissingError extends ElementCallError {
   }
 }
 
+/**
+ * Error indicating that the connection to the call was lost and could not be re-established.
+ */
 export class ConnectionLostError extends ElementCallError {
   public constructor() {
     super(
@@ -86,7 +96,16 @@ export class ConnectionLostError extends ElementCallError {
   }
 }
 
+/**
+ * Error indicating a failure in the membership manager causing the join call
+ * operation to fail.
+ */
 export class MembershipManagerError extends ElementCallError {
+  /**
+   * Creates an instance of MembershipManagerError.
+   *
+   * @param error - The underlying error that caused the membership manager failure.
+   */
   public constructor(error: Error) {
     super(
       t("error.membership_manager"),
@@ -98,6 +117,9 @@ export class MembershipManagerError extends ElementCallError {
   }
 }
 
+/**
+ * Error indicating that end-to-end encryption is not supported in the current environment.
+ */
 export class E2EENotSupportedError extends ElementCallError {
   public constructor() {
     super(
@@ -109,7 +131,14 @@ export class E2EENotSupportedError extends ElementCallError {
   }
 }
 
+/**
+ * Error indicating an unknown issue occurred during a call operation.
+ */
 export class UnknownCallError extends ElementCallError {
+  /**
+   * Creates an instance of UnknownCallError.
+   * @param error - The underlying error that caused the unknown issue.
+   */
   public constructor(error: Error) {
     super(
       t("error.generic"),
@@ -122,7 +151,14 @@ export class UnknownCallError extends ElementCallError {
   }
 }
 
+/**
+ * Error indicating a failure to obtain an OpenID token.
+ */
 export class FailToGetOpenIdToken extends ElementCallError {
+  /**
+   * Creates an instance of FailToGetOpenIdToken.
+   * @param error - The underlying error that caused the failure.
+   */
   public constructor(error: Error) {
     super(
       t("error.generic"),
@@ -135,7 +171,14 @@ export class FailToGetOpenIdToken extends ElementCallError {
   }
 }
 
+/**
+ * Error indicating a failure to start publishing on a LiveKit connection.
+ */
 export class FailToStartLivekitConnection extends ElementCallError {
+  /**
+   * Creates an instance of FailToStartLivekitConnection.
+   * @param e - An optional error message providing additional context.
+   */
   public constructor(e?: string) {
     super(
       t("error.failed_to_start_livekit"),
@@ -146,6 +189,9 @@ export class FailToStartLivekitConnection extends ElementCallError {
   }
 }
 
+/**
+ * Error indicating that a LiveKit's server has hit its track limits.
+ */
 export class InsufficientCapacityError extends ElementCallError {
   public constructor() {
     super(
@@ -157,6 +203,10 @@ export class InsufficientCapacityError extends ElementCallError {
   }
 }
 
+/**
+ * Error indicating that room creation is restricted by the SFU.
+ * Only authorized users can create rooms, so the room must exist before connecting (done by the auth jwt service)
+ */
 export class SFURoomCreationRestrictedError extends ElementCallError {
   public constructor() {
     super(

src/utils/matrix.ts

@@ -188,7 +188,6 @@ function fullAliasFromRoomName(roomName: string, client: MatrixClient): string {
  * Applies some basic sanitisation to a room name that the user
  * has given us
  * @param input The room name from the user
- * @param client A matrix client object
  */
 export function sanitiseRoomNameInput(input: string): string {
   // check to see if the user has entered a fully qualified room
@@ -304,8 +303,9 @@ export async function createRoom(
 /**
  * Returns an absolute URL to that will load Element Call with the given room
  * @param roomId ID of the room
- * @param roomName Name of the room
  * @param encryptionSystem what encryption (or EncryptionSystem.Unencrypted) the room uses
+ * @param roomName Name of the room
+ * @param viaServers Optional list of servers to include as 'via' parameters in the URL
  */
 export function getAbsoluteRoomUrl(
   roomId: string,
@@ -321,8 +321,9 @@ export function getAbsoluteRoomUrl(
 /**
  * Returns a relative URL to that will load Element Call with the given room
  * @param roomId ID of the room
- * @param roomName Name of the room
  * @param encryptionSystem what encryption (or EncryptionSystem.Unencrypted) the room uses
+ * @param roomName Name of the room
+ * @param viaServers Optional list of servers to include as 'via' parameters in the URL
  */
 export function getRelativeRoomUrl(
   roomId: string,

src/utils/media.ts

@@ -8,6 +8,7 @@ Please see LICENSE in the repository root for full details.
 /**
  * Finds a media device with label matching 'deviceName'
  * @param deviceName The label of the device to look for
+ * @param kind The kind of media device to look for
  * @param devices The list of devices to search
  * @returns A matching media device or undefined if no matching device was found
  */

src/utils/observable.test.ts

@@ -5,11 +5,12 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
 Please see LICENSE in the repository root for full details.
 */
 
-import { test } from "vitest";
-import { Subject } from "rxjs";
+import { expect, test } from "vitest";
+import { type Observable, of, Subject, switchMap } from "rxjs";
 
 import { withTestScheduler } from "./test";
-import { generateItems, pauseWhen } from "./observable";
+import { filterBehavior, generateItems, pauseWhen } from "./observable";
+import { type Behavior } from "../state/Behavior";
 
 test("pauseWhen", () => {
   withTestScheduler(({ behavior, expectObservable }) => {
@@ -72,3 +73,31 @@ test("generateItems", () => {
     expectObservable(scope4$).toBe(scope4Marbles);
   });
 });
+
+test("filterBehavior", () => {
+  withTestScheduler(({ behavior, expectObservable }) => {
+    // Filtering the input should segment it into 2 modes of non-null behavior.
+    const inputMarbles = "   abcxabx";
+    const filteredMarbles = "a--xa-x";
+
+    const input$ = behavior(inputMarbles, {
+      a: "a",
+      b: "b",
+      c: "c",
+      x: null,
+    });
+    const filtered$: Observable<Behavior<string> | null> = input$.pipe(
+      filterBehavior((value) => typeof value === "string"),
+    );
+
+    expectObservable(filtered$).toBe(filteredMarbles, {
+      a: expect.any(Object),
+      x: null,
+    });
+    expectObservable(
+      filtered$.pipe(
+        switchMap((value$) => (value$ === null ? of(null) : value$)),
+      ),
+    ).toBe(inputMarbles, { a: "a", b: "b", c: "c", x: null });
+  });
+});

src/utils/observable.ts

@@ -22,6 +22,7 @@ import {
   withLatestFrom,
   BehaviorSubject,
   type OperatorFunction,
+  distinctUntilChanged,
 } from "rxjs";
 
 import { type Behavior } from "../state/Behavior";
@@ -134,7 +135,6 @@ interface ItemHandle<Data, Item> {
  * requested at a later time, and destroyed (have their scope ended) when the
  * key is no longer requested.
  *
- * @param input$ The input value to be mapped.
  * @param generator A generator function yielding a tuple of keys and the
  *   currently associated data for each item that it wants to exist.
  * @param factory A function constructing an individual item, given the item's key,
@@ -185,6 +185,28 @@ export function generateItemsWithEpoch<
   );
 }
 
+/**
+ * Segments a behavior into periods during which its value matches the filter
+ * (outputting a behavior with a narrowed type) and periods during which it does
+ * not match (outputting null).
+ */
+export function filterBehavior<T, S extends T>(
+  predicate: (value: T) => value is S,
+): OperatorFunction<T, Behavior<S> | null> {
+  return (input$) =>
+    input$.pipe(
+      scan<T, BehaviorSubject<S> | null>((acc$, input) => {
+        if (predicate(input)) {
+          const output$ = acc$ ?? new BehaviorSubject(input);
+          output$.next(input);
+          return output$;
+        }
+        return null;
+      }, null),
+      distinctUntilChanged(),
+    );
+}
+
 function generateItemsInternal<
   Input,
   Keys extends [unknown, ...unknown[]],

src/utils/test-fixtures.ts

@@ -59,3 +59,17 @@ export const daveRTLRtcMember = mockRtcMembership("@dave2:example.org", "DDDD");
 export const daveRTL = mockMatrixRoomMember(daveRTLRtcMember, {
   rawDisplayName: "\u202eevaD",
 });
+
+export const testJWTToken = [
+  {}, // header
+  {
+    // payload
+    sub: "@me:example.org:ABCDEF",
+    video: {
+      room: "!example_room_id",
+    },
+  },
+  {}, // signature
+]
+  .map((d) => global.btoa(JSON.stringify(d)))
+  .join(".");

src/utils/test-viewmodel.ts

@@ -37,6 +37,7 @@ import {
 import { aliceRtcMember, localRtcMember } from "./test-fixtures";
 import { type RaisedHandInfo, type ReactionInfo } from "../reactions";
 import { constant } from "../state/Behavior";
+import { MatrixRTCMode } from "../settings/settings";
 
 mockConfig({ livekit: { livekit_service_url: "https://example.com" } });
 
@@ -162,6 +163,7 @@ export function getBasicCallViewModelEnvironment(
           setE2EEEnabled: async () => Promise.resolve(),
         }),
       connectionState$: constant(ConnectionState.Connected),
+      matrixRTCMode$: constant(MatrixRTCMode.Legacy),
       ...callViewModelOptions,
     },
     handRaisedSubject$,