Compare commits

..
Author SHA1 Message Date
Johannes Marbach cb0490e3f0 Render tiles for unmappable LiveKit identities 2026-09-14 10:30:45 +02:00
21 changed files with 553 additions and 467 deletions
+2 -2
View File
@@ -19,7 +19,7 @@ services:
- ./backend/playwright_homeserver-othersite.yaml:/data/cfg/homeserver.yaml:Z
element-web:
# Pin to a SHA so that upstream cannot break our tests. Renovate handles regular updates.
image: ghcr.io/element-hq/element-web:develop@sha256:c76d29903090eeb08ff625272277ede02ea8ee93cbaa067b77adfa89c89a494d
image: ghcr.io/element-hq/element-web:develop@sha256:6ed62b2963ae098a1f79521eb516834dcf29452168e281cc359d9cfb4380adf6
element-web-1:
# Pin to a SHA so that upstream cannot break our tests. Renovate handles regular updates.
image: ghcr.io/element-hq/element-web:develop@sha256:c76d29903090eeb08ff625272277ede02ea8ee93cbaa067b77adfa89c89a494d
image: ghcr.io/element-hq/element-web:develop@sha256:6ed62b2963ae098a1f79521eb516834dcf29452168e281cc359d9cfb4380adf6
+1
View File
@@ -273,6 +273,7 @@
"mute_for_me": "Mute for me",
"muted_for_me": "Muted for me",
"screen_share_volume": "Screen share volume",
"unknown_participant": "Unknown participant",
"volume": "Volume",
"waiting_for_media": "Waiting for media..."
}
+4 -13
View File
@@ -109,15 +109,10 @@ export const widgetTest = test.extend<MyFixtures>({
await TestHelpers.dismissInviteUnknownUserModal(ewPage1);
// Accept the invite
await TestHelpers.closeReleaseAnnouncement(
ewPage2,
"Introducing Sections",
);
await TestHelpers.expandAllSections(ewPage2);
await expect(
TestHelpers.roomListItem(ewPage2, "Welcome Room"),
ewPage2.getByRole("option", { name: "Welcome Room" }),
).toBeVisible();
await TestHelpers.roomListItem(ewPage2, "Welcome Room").click();
await ewPage2.getByRole("option", { name: "Welcome Room" }).click();
await ewPage2.getByRole("button", { name: "Accept" }).click();
await expect(
ewPage2
@@ -157,12 +152,8 @@ export const widgetTest = test.extend<MyFixtures>({
).toBeVisible();
// Accept the DM invite from brooks
await TestHelpers.closeReleaseAnnouncement(
ewPage2,
"Introducing Sections",
);
await TestHelpers.expandAllSections(ewPage2);
await TestHelpers.roomListItem(ewPage2, brooksDisplayName).click();
// This how playwright record selects the DM invite in the room list
await ewPage2.getByRole("button", { name: "Open room" }).click();
await ewPage2.getByRole("button", { name: "Start chatting" }).click();
}
+4 -27
View File
@@ -11,7 +11,6 @@ import {
type JSHandle,
type Page,
type FrameLocator,
type Locator,
} from "@playwright/test";
import { type MatrixClient } from "matrix-js-sdk";
@@ -204,16 +203,6 @@ export class TestHelpers {
}
}
public static async expandAllSections(page: Page): Promise<void> {
try {
await page
.getByRole("button", { name: "Expand all sections" })
.click({ timeout: 2000 });
} catch {
// Already expanded or button not present
}
}
public static async createRoom(
name: string,
page: Page,
@@ -264,9 +253,9 @@ export class TestHelpers {
roomName: string,
page: Page,
): Promise<void> {
await TestHelpers.closeReleaseAnnouncement(page, "Introducing Sections");
await TestHelpers.expandAllSections(page);
await TestHelpers.roomListItem(page, roomName).click({ timeout: 10000 });
await page.getByRole("option", { name: roomName }).click({
timeout: 10000,
});
await page.getByRole("button", { name: "Accept" }).click({
timeout: 5000,
});
@@ -361,17 +350,6 @@ export class TestHelpers {
.click();
}
/**
* Locates a room in the room list by its name.
*
* Matches on the aria-label prefix because the item's role differs between
* the flat room list (`option`) and the sectioned room list (`button`), and
* the label may carry a suffix such as " invitation.".
*/
public static roomListItem(page: Page, roomName: string): Locator {
return page.locator(`[aria-label^="Open room ${roomName}"]`);
}
/**
* Switches to a room in the room list by its name.
* @param page - The EW page
@@ -381,8 +359,7 @@ export class TestHelpers {
page: Page,
roomName: string,
): Promise<void> {
await TestHelpers.expandAllSections(page);
await TestHelpers.roomListItem(page, roomName).click();
await page.getByRole("option", { name: `Open room ${roomName}` }).click();
}
public static async dismissInviteUnknownUserModal(page: Page): Promise<void> {
+2 -5
View File
@@ -25,11 +25,11 @@ import { useTranslation } from "react-i18next";
import styles from "./MediaMuteAndSwitchButton.module.css";
import { MicButton, VideoButton } from "../button";
import { type AudioInputDeviceLabel } from "../state/MediaDevices";
import { type DeviceLabel } from "../state/MediaDevices";
import { useMediaDevices } from "../MediaDevicesContext";
export interface MenuOptions {
label: AudioInputDeviceLabel;
label: DeviceLabel;
id: string;
}
@@ -180,9 +180,6 @@ export const MediaMuteAndSwitchButton: FC<MediaMuteAndSwitchButtonProps> = ({
case "number":
labelText = numberedLabel(label.number);
break;
case "default":
labelText = t("settings.devices.default");
break;
}
return (
<MenuItem
+1 -2
View File
@@ -135,8 +135,7 @@ export const LobbyView: FC<Props> = ({
const initialAudioOptions = useInitial(
() =>
audioEnabled && {
// "" is the virtual browser default: no deviceId constraint
deviceId: getValue(devices.audioInput.selected$)?.id || undefined,
deviceId: getValue(devices.audioInput.selected$)?.id,
},
);
-116
View File
@@ -1,116 +0,0 @@
/*
Copyright 2026 Element Corp.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { afterEach, beforeEach, describe, vi, it, expect } from "vitest";
import * as ComponentsCore from "@livekit/components-core";
import { ObservableScope } from "./ObservableScope";
import { MediaDevices } from "./MediaDevices";
import { audioInput as audioInputSetting } from "../settings/settings";
import { withTestScheduler } from "../utils/test";
const BUILT_IN_MIC = {
deviceId: "b3d4cb5e0c8e1d7a2f9b6c4e5d8a7f1c3e2b9d6a5c4f8e7d1b2a3c4d5e6f7a8b",
kind: "audioinput",
label: "iPhone Microphone",
groupId: "1",
} as unknown as MediaDeviceInfo;
const HEADSET_MIC = {
deviceId: "c4e5d6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5",
kind: "audioinput",
label: "Headset Microphone",
groupId: "2",
} as unknown as MediaDeviceInfo;
// Chrome exposes a synthetic "default" device that follows the OS default.
const CHROME_DEFAULT = {
...HEADSET_MIC,
deviceId: "default",
label: `Default - ${HEADSET_MIC.label}`,
} as unknown as MediaDeviceInfo;
vi.mock("@livekit/components-core", () => ({
createMediaDeviceObserver: vi.fn(),
}));
describe("AudioInput", () => {
let testScope: ObservableScope;
beforeEach(() => {
testScope = new ObservableScope();
// Device preferences persist in localStorage across tests
audioInputSetting.setValue(undefined);
});
afterEach(() => {
testScope.end();
});
it("uses the browser default input when there is no default pseudo-device", () => {
withTestScheduler(({ behavior, cold, expectObservable }) => {
vi.mocked(ComponentsCore.createMediaDeviceObserver).mockReturnValue(
// iOS / Firefox: physical devices only, built-in first
cold("a", { a: [BUILT_IN_MIC, HEADSET_MIC] }),
);
const { audioInput } = new MediaDevices(testScope, {
controlledAudioDevices: false,
});
expectObservable(audioInput.available$).toBe("a", {
a: new Map([
["", { type: "default", name: null }],
[BUILT_IN_MIC.deviceId, { type: "name", name: BUILT_IN_MIC.label }],
[HEADSET_MIC.deviceId, { type: "name", name: HEADSET_MIC.label }],
]),
});
expectObservable(audioInput.selected$.pipe()).toBe("a", {
a: expect.objectContaining({ id: "" }),
});
});
});
it("keeps using Chrome's default device", () => {
withTestScheduler(({ behavior, cold, expectObservable }) => {
vi.mocked(ComponentsCore.createMediaDeviceObserver).mockReturnValue(
cold("a", { a: [CHROME_DEFAULT, BUILT_IN_MIC, HEADSET_MIC] }),
);
const { audioInput } = new MediaDevices(testScope, {
controlledAudioDevices: false,
});
expectObservable(audioInput.selected$).toBe("a", {
a: expect.objectContaining({ id: "default" }),
});
});
});
it("honours an explicit choice and falls back to the browser default when it disappears", () => {
withTestScheduler(({ behavior, cold, schedule, expectObservable }) => {
vi.mocked(ComponentsCore.createMediaDeviceObserver).mockReturnValue(
cold("a---b", {
a: [BUILT_IN_MIC, HEADSET_MIC],
b: [BUILT_IN_MIC],
}),
);
const { audioInput } = new MediaDevices(testScope, {
controlledAudioDevices: false,
});
schedule("--a", { a: () => audioInput.select(HEADSET_MIC.deviceId) });
expectObservable(audioInput.selected$).toBe("a-b-c", {
a: expect.objectContaining({ id: "" }),
b: expect.objectContaining({ id: HEADSET_MIC.deviceId }),
c: expect.objectContaining({ id: "" }),
});
});
});
});
+143 -7
View File
@@ -52,8 +52,10 @@ import {
aliceRtcMember,
aliceUserId,
bob,
bobDeviceId,
bobId,
bobRtcMember,
bobUserId,
local,
localId,
localRtcMember,
@@ -118,6 +120,19 @@ const daveId = `${dave.userId}:${daveRtcMember.deviceId}`;
const bobParticipant = mockRemoteParticipant({ identity: bobId });
const daveParticipant = mockRemoteParticipant({ identity: daveId });
// A LiveKit participant that no MatrixRTC membership accounts for
const rogueParticipant = mockRemoteParticipant({ identity: "rogue" });
const rogueId = `unknown:${exampleTransport.livekit_service_url}:rogue`;
const otherTransport: LivekitTransport = {
type: "livekit",
livekit_service_url: "https://lk.other.example.org",
livekit_alias: "!alias:other.example.org",
};
const bobOnOtherFocusRtcMember = mockRtcMembership(bobUserId, bobDeviceId, {
fociPreferred: [otherTransport],
});
export interface GridLayoutSummary {
type: "grid";
spotlight?: string[];
@@ -450,6 +465,113 @@ describe.each(modes)("CallViewModel (%s mode)", (mode) => {
});
});
test("LiveKit participants without a membership get an unknown participant tile", () => {
withTestScheduler(({ behavior, expectObservable }) => {
// A participant nobody's membership accounts for connects on frame 1 and
// disconnects on frame 2
const participantInputMarbles = "aba";
// It gets a tile at the end of the grid for as long as it is connected
const expectedLayoutMarbles = " aba";
withCallViewModel(
{
remoteParticipants$: behavior(participantInputMarbles, {
a: [aliceParticipant, bobParticipant],
b: [aliceParticipant, bobParticipant, rogueParticipant],
}),
rtcMembers$: constant([localRtcMember, aliceRtcMember, bobRtcMember]),
},
(vm) => {
expectObservable(summarizeLayout$(vm.layout$)).toBe(
expectedLayoutMarbles,
{
a: {
type: "grid",
spotlight: undefined,
grid: [`${localId}:0`, `${aliceId}:0`, `${bobId}:0`],
},
b: {
type: "grid",
spotlight: undefined,
grid: [`${localId}:0`, `${aliceId}:0`, `${bobId}:0`, rogueId],
},
},
);
},
);
});
});
test("an unknown participant takes the call out of one-on-one layout", () => {
withTestScheduler(({ behavior, expectObservable }) => {
// A participant nobody's membership accounts for connects on frame 1 and
// disconnects on frame 2
const participantInputMarbles = "aba";
// The one-on-one layout would hide the unknown participant, so we must
// fall back to the grid while they are present
const expectedLayoutMarbles = " aba";
withCallViewModel(
{
remoteParticipants$: behavior(participantInputMarbles, {
a: [aliceParticipant],
b: [aliceParticipant, rogueParticipant],
}),
rtcMembers$: constant([localRtcMember, aliceRtcMember]),
},
(vm) => {
expectObservable(summarizeLayout$(vm.layout$)).toBe(
expectedLayoutMarbles,
{
a: {
type: "one-on-one-desktop",
pip: `${localId}:0`,
spotlight: `${aliceId}:0`,
},
b: {
type: "grid",
spotlight: undefined,
grid: [`${localId}:0`, `${aliceId}:0`, rogueId],
},
},
);
},
);
});
});
test("a member seen on a focus other than their own is not an unknown participant", () => {
withTestScheduler(({ expectObservable }) => {
// Bob publishes to another focus, but also connects to ours to subscribe
// (the mock reports the same participants on every connection). Matching
// by identity rather than per transport keeps him from being flagged.
const expectedLayoutMarbles = "a";
withCallViewModel(
{
remoteParticipants$: constant([aliceParticipant, bobParticipant]),
rtcMembers$: constant([
localRtcMember,
aliceRtcMember,
bobOnOtherFocusRtcMember,
]),
},
(vm) => {
expectObservable(summarizeLayout$(vm.layout$)).toBe(
expectedLayoutMarbles,
{
a: {
type: "grid",
spotlight: undefined,
grid: [`${localId}:0`, `${aliceId}:0`, `${bobId}:0`],
},
},
);
},
);
});
});
test("one-on-one mobile layout shows local tile when video is enabled", () => {
withTestScheduler(({ behavior, schedule, expectObservable }) => {
// Local participant enables their video, then disables it
@@ -1211,15 +1333,18 @@ describe.each(modes)("CallViewModel (%s mode)", (mode) => {
});
});
test("participants must have a MatrixRTCSession to be visible", () => {
test("participants without a MatrixRTC membership are shown as unknown, without media", () => {
withTestScheduler(({ behavior, expectObservable }) => {
// iterate through a number of combinations of participants and MatrixRTC memberships
// Bob never has an MatrixRTC membership
const participantInputMarbles = "abcd-c";
// Bob even tries to share his screen at the end
const bobSharingInputMarbles = " n---yn";
// Bob should never be visible
const expectedLayoutMarbles = " a-bc-b";
// Bob gets an unknown participant tile (MSC4143) rather than a tile of
// his own, and his screen share never shows up. His presence also keeps
// the call out of the one-on-one layout, which would hide him.
const expectedLayoutMarbles = " abcd-c";
const bobUnknownId = `unknown:${exampleTransport.livekit_service_url}:${bobId}`;
withCallViewModel(
{
@@ -1251,14 +1376,25 @@ describe.each(modes)("CallViewModel (%s mode)", (mode) => {
grid: [`${localId}:0`],
},
b: {
type: "one-on-one-desktop",
pip: `${localId}:0`,
spotlight: `${aliceId}:0`,
type: "grid",
spotlight: undefined,
grid: [`${localId}:0`, bobUnknownId],
},
// Tiles keep their positions, so Bob stays ahead of the others
c: {
type: "grid",
spotlight: undefined,
grid: [`${localId}:0`, `${aliceId}:0`, `${daveId}:0`],
grid: [`${localId}:0`, bobUnknownId, `${aliceId}:0`],
},
d: {
type: "grid",
spotlight: undefined,
grid: [
`${localId}:0`,
bobUnknownId,
`${aliceId}:0`,
`${daveId}:0`,
],
},
},
);
+92 -19
View File
@@ -95,6 +95,7 @@ import {
layoutShallowEquals,
type Alignment,
type GridLayoutMedia,
type GridMediaViewModel,
type Layout,
type LayoutMedia,
type OneOnOneDesktopLayoutMedia,
@@ -162,6 +163,10 @@ import {
createRingingMedia,
type RingingMediaViewModel,
} from "../media/RingingMediaViewModel.ts";
import {
createUnknownParticipantMedia,
type UnknownParticipantMediaViewModel,
} from "../media/UnknownParticipantMediaViewModel.ts";
import { type GridTileViewModel } from "../TileViewModel.ts";
//TODO
@@ -869,6 +874,61 @@ export function createCallViewModel$(
),
);
/**
* Media for LiveKit participants that cannot be mapped to any MatrixRTC
* membership. MSC4143 asks us to surface these rather than ignore them, as
* they can signal an eavesdropping or impersonation attack.
*
* Participants are matched to memberships by identity alone rather than per
* transport, since a member legitimately shows up as a subscribe-only
* participant on every focus other than the one they publish to.
*/
const unknownParticipantMedia$ = scope.behavior<
UnknownParticipantMediaViewModel[]
>(
combineLatest([
memberships$,
connectionManager.connectionManagerData$,
]).pipe(
// Memberships and connections derive from the same epoch of session
// state; comparing them across epochs would flag members whose
// connection hasn't caught up yet.
filter(([memberships, data]) => memberships.epoch === data.epoch),
map(([memberships, data]) => {
const knownIdentities = new Set(
memberships.value.map((m) => m.rtcBackendIdentity),
);
return data.value.getConnections().flatMap((connection) =>
data.value
.getParticipantsForTransport(connection.transport)
.filter((p) => !knownIdentities.has(p.identity))
.map((p) => ({
focusUrl: connection.transport.livekit_service_url,
identity: p.identity,
})),
);
}),
generateItems(
"CallViewModel unknownParticipantMedia$",
function* (unknownParticipants) {
for (const { focusUrl, identity } of unknownParticipants)
yield { keys: [focusUrl, identity], data: undefined };
},
(_scope, _data$, focusUrl, identity) => {
logger.warn(
`LiveKit participant ${identity} on ${focusUrl} matches no MatrixRTC membership`,
);
return createUnknownParticipantMedia({
id: `unknown:${focusUrl}:${identity}`,
rtcBackendIdentity: identity,
focusUrl,
});
},
),
),
[],
);
const ringingMedia$ = scope.behavior<RingingMediaViewModel | null>(
ringAttempts$.pipe(
switchMap(({ intent, recipient, outcome$ }) =>
@@ -1015,21 +1075,29 @@ export function createCallViewModel$(
),
);
const grid$ = scope.behavior<UserMediaViewModel[]>(
userMedia$.pipe(
switchMap((mediaItems) => {
const bins = mediaItems.map((m) =>
m.bin$.pipe(map((bin) => [m, bin] as const)),
);
// Sort the media by bin order and generate a tile for each one
return bins.length === 0
? of([])
: combineLatest(bins, (...bins) =>
bins.sort(([, bin1], [, bin2]) => bin1 - bin2).map(([m]) => m),
);
}),
distinctUntilChanged(shallowArrayEquals),
),
const sortedUserMedia$: Observable<UserMediaViewModel[]> = userMedia$.pipe(
switchMap((mediaItems) => {
const bins = mediaItems.map((m) =>
m.bin$.pipe(map((bin) => [m, bin] as const)),
);
// Sort the media by bin order and generate a tile for each one
return bins.length === 0
? of([])
: combineLatest(bins, (...bins) =>
bins.sort(([, bin1], [, bin2]) => bin1 - bin2).map(([m]) => m),
);
}),
);
const grid$ = scope.behavior<GridMediaViewModel[]>(
combineLatest(
[sortedUserMedia$, unknownParticipantMedia$],
// Unknown participants have no media to sort by, so they go last
(userMedia, unknownParticipants) => [
...userMedia,
...unknownParticipants,
],
).pipe(distinctUntilChanged(shallowArrayEquals)),
);
/**
@@ -1189,10 +1257,15 @@ export function createCallViewModel$(
local: LocalUserMediaViewModel;
remote: UserMediaViewModel | RingingMediaViewModel;
} | null> = scope.behavior(
combineLatest([userMedia$, screenShares$]).pipe(
switchMap(([userMedia, screenShares]) => {
// One-on-one layout only supports 2 user media, no screen shares
if (userMedia.length <= 2 && screenShares.length === 0) {
combineLatest([userMedia$, screenShares$, unknownParticipantMedia$]).pipe(
switchMap(([userMedia, screenShares, unknownParticipants]) => {
// One-on-one layout only supports 2 user media, no screen shares, and
// must never hide an unknown participant
if (
userMedia.length <= 2 &&
screenShares.length === 0 &&
unknownParticipants.length === 0
) {
const local = userMedia.find(
(vm): vm is WrappedUserMediaViewModel & LocalUserMediaViewModel =>
vm.type === "user" && vm.local,
@@ -5,26 +5,16 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
afterEach,
beforeEach,
describe,
expect,
it,
type Mock,
test,
vi,
} from "vitest";
import { afterEach, beforeEach, describe, expect, it, test, vi } from "vitest";
import {
ConnectionState as LivekitConnectionState,
LocalParticipant,
type LocalTrack,
type LocalTrackPublication,
ParticipantEvent,
type Room as LivekitRoom,
Track,
} from "livekit-client";
import { BehaviorSubject, NEVER } from "rxjs";
import { BehaviorSubject } from "rxjs";
import { logger } from "matrix-js-sdk/lib/logger";
import { ObservableScope } from "../../ObservableScope";
@@ -33,12 +23,10 @@ import {
flushPromises,
mockLivekitRoom,
mockMediaDevices,
deviceStub,
} from "../../../utils/test";
import { Publisher } from "./Publisher";
import { type Connection } from "../remoteMembers/Connection";
import { type MuteStates } from "../../MuteStates";
import { type MediaDevices } from "../../MediaDevices";
let scope: ObservableScope;
@@ -194,159 +182,6 @@ beforeEach(() => {
} as unknown as Connection;
});
describe("Publisher device sync", () => {
type SelectedInput =
| { id: string; hardwareDeviceChange$: typeof NEVER }
| undefined;
function selectedInput(id: string): SelectedInput {
return { id, hardwareDeviceChange$: NEVER };
}
/**
* Creates a Publisher whose LiveKit room starts out capturing from
* capturedDeviceId (undefined meaning no deviceId constraint, i.e. the
* browser default) and reports activeDeviceId as its active audio input.
*/
function createDeviceSyncPublisher({
selected$,
capturedDeviceId,
activeDeviceId,
switchActiveDevice = vi.fn().mockResolvedValue(true),
}: {
selected$: BehaviorSubject<SelectedInput>;
capturedDeviceId?: string;
activeDeviceId: string;
switchActiveDevice?: Mock;
}): { publisher: Publisher; switchActiveDevice: Mock } {
const livekitRoom = mockLivekitRoom({
localParticipant,
state: LivekitConnectionState.Connected,
// ConnectionFactory leaves the deviceId out for the browser default
options: { audioCaptureDefaults: { deviceId: capturedDeviceId } },
switchActiveDevice,
getActiveDevice: () => activeDeviceId,
} as unknown as Partial<LivekitRoom>);
const publisher = new Publisher(
{ ...connection, livekitRoom },
mockMediaDevices({
audioInput: { ...deviceStub, selected$ },
} as unknown as Partial<MediaDevices>),
muteStates,
constant({ supported: false, processor: undefined }),
logger,
false,
);
return { publisher, switchActiveDevice };
}
it("does not pin a device for the virtual browser default input", async () => {
const selected$ = new BehaviorSubject<SelectedInput>(selectedInput(""));
const { publisher, switchActiveDevice } = createDeviceSyncPublisher({
selected$,
activeDeviceId: "hardware-id-of-whatever-the-browser-chose",
});
// Already capturing from the browser default: nothing to switch, even
// though LiveKit reports the physical device it ended up with.
expect(switchActiveDevice).not.toHaveBeenCalled();
selected$.next(selectedInput("headset"));
expect(switchActiveDevice).toHaveBeenLastCalledWith(
"audioinput",
"headset",
);
selected$.next(selectedInput(""));
expect(switchActiveDevice).toHaveBeenLastCalledWith(
"audioinput",
"default",
false,
);
expect(switchActiveDevice).toHaveBeenCalledTimes(2);
await publisher.destroy();
});
it("pins the device the browser default resolved to when picked explicitly", async () => {
const selected$ = new BehaviorSubject<SelectedInput>(selectedInput(""));
const { publisher, switchActiveDevice } = createDeviceSyncPublisher({
selected$,
// The browser default happens to capture from the built-in microphone
activeDeviceId: "built-in-mic",
});
expect(switchActiveDevice).not.toHaveBeenCalled();
// Picking that same microphone has to replace the loose constraint with a
// pin, or the capture would keep following the OS default
selected$.next(selectedInput("built-in-mic"));
expect(switchActiveDevice).toHaveBeenCalledTimes(1);
expect(switchActiveDevice).toHaveBeenLastCalledWith(
"audioinput",
"built-in-mic",
);
// Once pinned, re-emitting the same selection is a no-op again
selected$.next(selectedInput("built-in-mic"));
expect(switchActiveDevice).toHaveBeenCalledTimes(1);
await publisher.destroy();
});
it("retries the same selection after a failed switch", async () => {
const selected$ = new BehaviorSubject<SelectedInput>(selectedInput(""));
const switchActiveDevice = vi
.fn()
.mockRejectedValueOnce(new Error("could not open the device"))
.mockResolvedValue(true);
const { publisher } = createDeviceSyncPublisher({
selected$,
// Pinned to the headset, so moving to the browser default is a switch
capturedDeviceId: "headset",
activeDeviceId: "headset",
switchActiveDevice,
});
expect(switchActiveDevice).toHaveBeenCalledTimes(1);
await flushPromises();
// The failed switch left us on the headset, so selecting the browser
// default again has to try once more rather than count as already done
selected$.next(selectedInput(""));
expect(switchActiveDevice).toHaveBeenCalledTimes(2);
expect(switchActiveDevice).toHaveBeenLastCalledWith(
"audioinput",
"default",
false,
);
await flushPromises();
// This time it worked, so there is nothing left to do
selected$.next(selectedInput(""));
expect(switchActiveDevice).toHaveBeenCalledTimes(2);
await publisher.destroy();
});
it("does not switch when the room already captures from the selected device", async () => {
const selected$ = new BehaviorSubject<SelectedInput>(
selectedInput("headset"),
);
const { publisher, switchActiveDevice } = createDeviceSyncPublisher({
selected$,
capturedDeviceId: "headset",
activeDeviceId: "headset",
});
expect(switchActiveDevice).not.toHaveBeenCalled();
await publisher.destroy();
});
});
describe("Publisher", () => {
let publisher: Publisher;
@@ -355,19 +355,8 @@ export class Publisher {
const syncDevice = (
kind: MediaDeviceKind,
selected$: Observable<SelectedDevice | undefined>,
): Subscription => {
// The ID we last asked LiveKit for. Needed for the virtual browser
// default input ("") because LiveKit reports the physical device it
// ended up capturing from, which never equals "".
// ConnectionFactory already applied the selection when the room was
// created; a browser-default input is represented there by an absent
// deviceId.
let requestedId: string | undefined =
kind === "audioinput" &&
lkRoom.options.audioCaptureDefaults?.deviceId === undefined
? ""
: undefined;
return selected$.pipe(scope.bind()).subscribe((device) => {
): Subscription =>
selected$.pipe(scope.bind()).subscribe((device) => {
if (lkRoom.state != LivekitConnectionState.Connected) return;
// if (this.connectionState$.value !== ConnectionState.Connected) return;
this.logger.info(
@@ -376,41 +365,20 @@ export class Publisher {
" !== ",
device?.id,
);
if (device === undefined) return;
const browserDefaultInput = device.id === "" && kind !== "audiooutput";
// While we capture from the browser default, LiveKit reports the
// physical device it resolved to, which can be any of the devices the
// user could also pick explicitly. getActiveDevice therefore can't
// tell an explicit pin apart from the default resolving to the same
// device, and only requestedId can: without this, picking the device
// the default happens to use would leave the loose constraint in
// place, and the capture would keep following the OS default.
const capturingBrowserDefault = requestedId === "";
if (browserDefaultInput) {
if (capturingBrowserDefault) return;
} else if (
!capturingBrowserDefault &&
lkRoom.getActiveDevice(kind) === device.id
if (
device !== undefined &&
lkRoom.getActiveDevice(kind) !== device.id
) {
return;
lkRoom
.switchActiveDevice(kind, device.id)
.catch((e: Error) =>
this.logger.error(
`Failed to sync ${kind} device with LiveKit`,
e,
),
);
}
const previousRequestedId = requestedId;
requestedId = device.id;
// For the browser default input, ask for "default" as a non-exact
// constraint: browsers that have no such device ignore it and capture
// from the OS default input.
(browserDefaultInput
? lkRoom.switchActiveDevice(kind, "default", false)
: lkRoom.switchActiveDevice(kind, device.id)
).catch((e: Error) => {
// LiveKit keeps capturing from the previous device, so record that
// again to leave the next selection of this device something to do.
// A selection made while the switch was in flight wins, though.
if (requestedId === device.id) requestedId = previousRequestedId;
this.logger.error(`Failed to sync ${kind} device with LiveKit`, e);
});
});
};
syncDevice("audioinput", devices.audioInput.selected$);
if (!controlledAudioDevices)
@@ -174,9 +174,7 @@ function generateRoomOption({
publishDefaults,
audioCaptureDefaults: {
...liveKitOptions.audioCaptureDefaults,
// "" is the virtual browser default: leave the constraint out so the
// browser captures from the OS default input.
deviceId: devices.audioInput.selected$.value?.id || undefined,
deviceId: devices.audioInput.selected$.value?.id,
echoCancellation: echoCancellationSetting.getValue(),
noiseSuppression: noiseSuppressionSetting.getValue(),
autoGainControl: autoGainControlSetting.getValue(),
+9 -34
View File
@@ -36,14 +36,11 @@ export type DeviceLabel =
| { type: "name"; name: string }
| { type: "number"; number: number };
export type AudioInputDeviceLabel =
| DeviceLabel
| { type: "default"; name: string | null };
export type AudioOutputDeviceLabel =
| AudioInputDeviceLabel
| DeviceLabel
| { type: "speaker" }
| { type: "earpiece" };
| { type: "earpiece" }
| { type: "default"; name: string | null };
/**
* Base selected-device value shared by all media kinds.
@@ -194,10 +191,7 @@ function selectDevice$<Label>(
});
}
class AudioInput implements MediaDevice<
AudioInputDeviceLabel,
SelectedAudioInputDevice
> {
class AudioInput implements MediaDevice<DeviceLabel, SelectedAudioInputDevice> {
private logger = rootLogger.getChild("[MediaDevices AudioInput]");
private readonly availableRaw$: Behavior<MediaDeviceInfo[]> =
@@ -209,29 +203,7 @@ class AudioInput implements MediaDevice<
);
public readonly available$ = this.scope.behavior(
this.availableRaw$.pipe(
map((availableRaw) => {
const available: Map<string, AudioInputDeviceLabel> =
buildDeviceMap(availableRaw);
// Browsers without a "default" pseudo-device (Firefox, Safari, and
// in particular iOS) get a virtual default entry, listed first so
// that it is what we use when the user has not chosen a microphone.
// Its ID is the empty string, which consumers translate into "no
// deviceId constraint": the browser then captures from whatever the
// OS routes as the default input (a wired or Bluetooth headset when
// one is connected) instead of EC pinning whichever device happens
// to be enumerated first, which on iOS is the built-in microphone.
// Unlike Chrome's "default" device, a stream opened this way does
// not follow later changes of the OS default on desktop browsers;
// iOS re-routes the audio session itself.
if (available.size && !available.has("") && !available.has("default"))
return new Map<string, AudioInputDeviceLabel>([
["", { type: "default", name: null }],
...available,
]);
return available;
}),
),
this.availableRaw$.pipe(map(buildDeviceMap)),
);
public readonly selected$ = this.scope.behavior(
@@ -299,6 +271,9 @@ export class AudioOutput implements MediaDevice<
// set to empty map if we are on Safari, because it does not support setSinkId
available = new Map();
}
// Note: creating virtual default input devices would be another problem
// entirely, because requesting a media stream from deviceId "" won't
// automatically track the default device.
return available;
}),
),
@@ -402,7 +377,7 @@ export class MediaDevices {
false,
);
public readonly audioInput: MediaDevice<
AudioInputDeviceLabel,
DeviceLabel,
SelectedAudioInputDevice
> = new AudioInput(this.usingNames$, this.scope);
+13 -12
View File
@@ -8,12 +8,15 @@ Please see LICENSE in the repository root for full details.
import { BehaviorSubject } from "rxjs";
import { logger } from "matrix-js-sdk/lib/logger";
import { GridTileViewModel, SpotlightTileViewModel } from "./TileViewModel";
import {
type GridTileMediaViewModel,
GridTileViewModel,
SpotlightTileViewModel,
} from "./TileViewModel";
import { fillGaps } from "../utils/iter";
import { debugTileLayout } from "../settings/settings";
import { type MediaViewModel } from "./media/MediaViewModel";
import { type UserMediaViewModel } from "./media/UserMediaViewModel";
import { type RingingMediaViewModel } from "./media/RingingMediaViewModel";
type SpotlightBackground = "solid" | "transparent";
@@ -68,10 +71,8 @@ class SpotlightTileData {
}
class GridTileData {
private readonly media$: BehaviorSubject<
UserMediaViewModel | RingingMediaViewModel
>;
public get media(): UserMediaViewModel | RingingMediaViewModel {
private readonly media$: BehaviorSubject<GridTileMediaViewModel>;
public get media(): GridTileMediaViewModel {
return this.media$.value;
}
public set media(value: UserMediaViewModel) {
@@ -80,7 +81,7 @@ class GridTileData {
public readonly vm: GridTileViewModel;
public constructor(media: UserMediaViewModel | RingingMediaViewModel) {
public constructor(media: GridTileMediaViewModel) {
this.media$ = new BehaviorSubject(media);
this.vm = new GridTileViewModel(this.media$);
}
@@ -140,7 +141,7 @@ export class TileStoreBuilder {
: null;
private readonly prevGridByMedia: Map<
MediaViewModel,
GridTileMediaViewModel,
[GridTileData, number]
> = new Map(
this.prevGrid.map((entry, i) => [entry.media, [entry, i]] as const),
@@ -205,9 +206,7 @@ export class TileStoreBuilder {
* Sets up a grid tile for the given media. If this is never called for some
* media, then that media will have no grid tile.
*/
public registerGridTile(
media: UserMediaViewModel | RingingMediaViewModel,
): void {
public registerGridTile(media: GridTileMediaViewModel): void {
if (DEBUG_ENABLED)
logger.debug(
`[TileStore, ${this.generation}] register grid tile: ${media.displayName$.value}`,
@@ -215,8 +214,10 @@ export class TileStoreBuilder {
if (this.spotlight !== null) {
// We actually *don't* want spotlight speakers to appear in both the
// spotlight and the grid, so they're filtered out here
// spotlight and the grid, so they're filtered out here. (Unknown
// participants never make it into the spotlight.)
if (
media.type !== "unknown participant" &&
!(media.type === "user" && media.local) &&
this.spotlight.media.includes(media)
)
+7 -3
View File
@@ -10,8 +10,14 @@ import { BehaviorSubject } from "rxjs";
import { type Behavior } from "./Behavior";
import { type MediaViewModel } from "./media/MediaViewModel";
import { type RingingMediaViewModel } from "./media/RingingMediaViewModel";
import { type UnknownParticipantMediaViewModel } from "./media/UnknownParticipantMediaViewModel";
import { type UserMediaViewModel } from "./media/UserMediaViewModel";
export type GridTileMediaViewModel =
| UserMediaViewModel
| RingingMediaViewModel
| UnknownParticipantMediaViewModel;
let nextId = 0;
function createId(): string {
return (nextId++).toString();
@@ -23,9 +29,7 @@ export class GridTileViewModel {
public readonly showOutline$: Behavior<boolean> = this._showOutline$;
public constructor(
public readonly media$: Behavior<
UserMediaViewModel | RingingMediaViewModel
>,
public readonly media$: Behavior<GridTileMediaViewModel>,
) {}
public setShowOutline(value: boolean): void {
+8 -3
View File
@@ -10,6 +10,7 @@ import { type BehaviorSubject } from "rxjs";
import { type LocalUserMediaViewModel } from "./media/LocalUserMediaViewModel.ts";
import { type MediaViewModel } from "./media/MediaViewModel.ts";
import { type RingingMediaViewModel } from "./media/RingingMediaViewModel.ts";
import { type UnknownParticipantMediaViewModel } from "./media/UnknownParticipantMediaViewModel.ts";
import { type UserMediaViewModel } from "./media/UserMediaViewModel.ts";
import {
type GridTileViewModel,
@@ -18,25 +19,29 @@ import {
import { type Behavior } from "./Behavior.ts";
import { shallowEquals as arrayShallowEquals } from "../utils/array.ts";
export type GridMediaViewModel =
| UserMediaViewModel
| UnknownParticipantMediaViewModel;
export interface GridLayoutMedia {
type: "grid";
edgeToEdge: false;
spotlight?: MediaViewModel[];
grid: UserMediaViewModel[];
grid: GridMediaViewModel[];
}
export interface SpotlightLandscapeLayoutMedia {
type: "spotlight-landscape";
edgeToEdge: boolean;
spotlight: MediaViewModel[];
grid: UserMediaViewModel[];
grid: GridMediaViewModel[];
}
export interface SpotlightPortraitLayoutMedia {
type: "spotlight-portrait";
edgeToEdge: false;
spotlight: MediaViewModel[];
grid: UserMediaViewModel[];
grid: GridMediaViewModel[];
}
export interface SpotlightExpandedLayoutMedia {
@@ -0,0 +1,57 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { constant } from "../Behavior";
import { type BaseMediaViewModel } from "./MediaViewModel";
/**
* Media representing a LiveKit participant that cannot be mapped to any
* MatrixRTC member of the session.
*
* MSC4143 asks clients to surface such streams rather than silently ignore
* them, because they can signal an eavesdropping or impersonation attack. We
* therefore give them a tile, but never render their audio or video.
*
* There is no Matrix user behind this media, so `userId` and `displayName$`
* carry the LiveKit identity instead, purely so that debugging output stays
* meaningful.
*/
export interface UnknownParticipantMediaViewModel extends BaseMediaViewModel {
type: "unknown participant";
/**
* The LiveKit identity under which the participant connected. Exposed for
* debugging.
*/
rtcBackendIdentity: string;
/**
* The URL of the LiveKit focus on which the participant was seen. Exposed for
* debugging.
*/
focusUrl: string;
}
export interface UnknownParticipantMediaInputs {
id: string;
rtcBackendIdentity: string;
focusUrl: string;
}
export function createUnknownParticipantMedia({
id,
rtcBackendIdentity,
focusUrl,
}: UnknownParticipantMediaInputs): UnknownParticipantMediaViewModel {
return {
type: "unknown participant",
id,
userId: rtcBackendIdentity,
displayName$: constant(rtcBackendIdentity),
mxcAvatarUrl$: constant(undefined),
rtcBackendIdentity,
focusUrl,
};
}
+98
View File
@@ -0,0 +1,98 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { expect, within } from "storybook/test";
import { type JSX, useMemo } from "react";
import type { Meta, StoryObj } from "@storybook/react-vite";
import { GridTile } from "./GridTile";
import { GridTileViewModel } from "../state/TileViewModel";
import { constant } from "../state/Behavior";
import { createUnknownParticipantMedia } from "../state/media/UnknownParticipantMediaViewModel";
interface UnknownParticipantTileStoryProps {
width: number;
height: number;
rtcBackendIdentity: string;
showNameTags: boolean;
showOutline: boolean;
}
/**
* Renders a GridTile for a LiveKit participant that maps to no MatrixRTC
* member, driven by primitive props so that Storybook can document them.
*/
function UnknownParticipantTile({
width,
height,
rtcBackendIdentity,
...props
}: UnknownParticipantTileStoryProps): JSX.Element {
const vm = useMemo(
() =>
new GridTileViewModel(
constant(
createUnknownParticipantMedia({
id: `unknown:https://rtc.example.org:${rtcBackendIdentity}`,
rtcBackendIdentity,
focusUrl: "https://rtc.example.org",
}),
),
),
[rtcBackendIdentity],
);
return (
<GridTile
vm={vm}
onOpenProfile={null}
targetWidth={width}
targetHeight={height}
style={{ width, height }}
showSpeakingIndicators={false}
showRingingStatus={false}
focusable
{...props}
/>
);
}
const meta = {
component: UnknownParticipantTile,
argTypes: {
width: { control: { type: "range", min: 80, max: 800, step: 10 } },
height: { control: { type: "range", min: 80, max: 600, step: 10 } },
},
} satisfies Meta<typeof UnknownParticipantTile>;
export default meta;
type Story = StoryObj<typeof meta>;
export const UnknownParticipant: Story = {
args: {
width: 400,
height: 300,
rtcBackendIdentity: "@rogue:example.org:DEVICE",
showNameTags: true,
showOutline: false,
},
play: async ({ canvasElement }) => {
const canvas = within(canvasElement);
await expect(canvas.getByText("Unknown participant")).toBeInTheDocument();
await expect(
canvas.queryByText("@rogue:example.org:DEVICE"),
).not.toBeInTheDocument();
},
};
/** Narrow tiles hide the name tag, so the label goes with it. */
export const UnknownParticipantNarrow: Story = {
args: {
...UnknownParticipant.args,
width: 90,
height: 90,
},
};
+31
View File
@@ -32,6 +32,7 @@ import {
createRingingMedia,
type RingingMediaViewModel,
} from "../state/media/RingingMediaViewModel";
import { createUnknownParticipantMedia } from "../state/media/UnknownParticipantMediaViewModel";
global.IntersectionObserver = class MockIntersectionObserver {
public observe(): void {}
@@ -165,3 +166,33 @@ test("GridTile displays ringing media", async () => {
act(() => pickupState$.next("decline"));
screen.getByText("Call ended");
});
test("GridTile displays an unknown participant", async () => {
const vm = createUnknownParticipantMedia({
id: "unknown:https://rtc-example.org:rogue",
rtcBackendIdentity: "rogue",
focusUrl: "https://rtc-example.org",
});
const { container } = render(
<ReactionsSenderProvider vm={callVm} rtcSession={fakeRtcSession}>
<GridTile
vm={new GridTileViewModel(constant(vm))}
onOpenProfile={() => {}}
targetWidth={300}
targetHeight={200}
showSpeakingIndicators
showNameTags
showRingingStatus
showOutline
focusable
/>
</ReactionsSenderProvider>,
);
expect(await axe(container)).toHaveNoViolations();
// The tile is labelled as unknown rather than after its LiveKit identity
screen.getByText("Unknown participant");
expect(screen.queryByText("rogue")).toBeNull();
// There is no user to show an avatar for
expect(container.querySelector("[data-style]")).toBeNull();
});
+48
View File
@@ -51,6 +51,7 @@ import { type LocalUserMediaViewModel } from "../state/media/LocalUserMediaViewM
import { type RemoteUserMediaViewModel } from "../state/media/RemoteUserMediaViewModel";
import { type UserMediaViewModel } from "../state/media/UserMediaViewModel";
import { type RingingMediaViewModel } from "../state/media/RingingMediaViewModel";
import { type UnknownParticipantMediaViewModel } from "../state/media/UnknownParticipantMediaViewModel";
import { RingingStatus } from "./RingingStatus";
interface TileProps {
@@ -97,6 +98,44 @@ const RingingMediaTile: FC<RingingMediaTileProps> = ({
);
};
interface UnknownParticipantTileProps extends Omit<
TileProps,
"displayName" | "mxcAvatarUrl"
> {
vm: UnknownParticipantMediaViewModel;
}
/**
* A blank tile standing in for a LiveKit participant that cannot be mapped to
* any MatrixRTC member. MSC4143 asks clients to surface such streams rather
* than ignore them. No media is rendered for them.
*/
const UnknownParticipantTile: FC<UnknownParticipantTileProps> = ({
vm,
className,
...props
}) => {
const { t } = useTranslation();
return (
<MediaView
className={classNames(className, styles.tile)}
video={undefined}
userId={vm.userId}
unencryptedWarning={false}
videoEnabled={false}
mirror={false}
showAvatar={false}
displayName={t("video_tile.unknown_participant")}
mxcAvatarUrl={undefined}
rtcBackendIdentity={vm.rtcBackendIdentity}
focusUrl={vm.focusUrl}
{...props}
/>
);
};
UnknownParticipantTile.displayName = "UnknownParticipantTile";
interface UserMediaTileProps extends TileProps {
vm: UserMediaViewModel;
showSpeakingIndicators: boolean;
@@ -454,6 +493,15 @@ export const GridTile: FC<GridTileProps> = ({
{...props}
/>
);
} else if (media.type === "unknown participant") {
return (
<UnknownParticipantTile
ref={ref}
vm={media}
className={classNames(className, { [styles.outline]: showOutline })}
{...props}
/>
);
} else if (media.local) {
return (
<LocalUserMediaTile
+17 -9
View File
@@ -56,6 +56,11 @@ interface Props extends ComponentProps<typeof animated.div> {
displayName: string;
mxcAvatarUrl: string | undefined;
avatarStyle?: "solid" | "translucent";
/**
* Whether to show an avatar when there is no video. Off for media that has no
* Matrix user behind it.
*/
showAvatar?: boolean;
background?: "solid" | "transparent";
focusable: boolean;
primaryButton?: ReactNode;
@@ -95,6 +100,7 @@ export const MediaView: FC<Props> = ({
displayName,
mxcAvatarUrl,
avatarStyle = "solid",
showAvatar = true,
background = "solid",
focusable,
primaryButton,
@@ -176,15 +182,17 @@ export const MediaView: FC<Props> = ({
<div className={styles.speakingBorder} />
</div>
)}
<Avatar
id={userId}
name={displayName}
size={avatarSize}
src={mxcAvatarUrl}
data-style={avatarStyle}
className={styles.avatar}
style={{ display: video && videoEnabled ? "none" : "initial" }}
/>
{showAvatar && (
<Avatar
id={userId}
name={displayName}
size={avatarSize}
src={mxcAvatarUrl}
data-style={avatarStyle}
className={styles.avatar}
style={{ display: video && videoEnabled ? "none" : "initial" }}
/>
)}
{video?.publication !== undefined && (
<VideoTrack
trackRef={video}