mirror of
https://github.com/vector-im/element-call.git
synced 2026-10-23 00:02:26 +00:00
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
cb0490e3f0 |
@@ -19,7 +19,7 @@ services:
|
||||
- ./backend/playwright_homeserver-othersite.yaml:/data/cfg/homeserver.yaml:Z
|
||||
element-web:
|
||||
# Pin to a SHA so that upstream cannot break our tests. Renovate handles regular updates.
|
||||
image: ghcr.io/element-hq/element-web:develop@sha256:c76d29903090eeb08ff625272277ede02ea8ee93cbaa067b77adfa89c89a494d
|
||||
image: ghcr.io/element-hq/element-web:develop@sha256:6ed62b2963ae098a1f79521eb516834dcf29452168e281cc359d9cfb4380adf6
|
||||
element-web-1:
|
||||
# Pin to a SHA so that upstream cannot break our tests. Renovate handles regular updates.
|
||||
image: ghcr.io/element-hq/element-web:develop@sha256:c76d29903090eeb08ff625272277ede02ea8ee93cbaa067b77adfa89c89a494d
|
||||
image: ghcr.io/element-hq/element-web:develop@sha256:6ed62b2963ae098a1f79521eb516834dcf29452168e281cc359d9cfb4380adf6
|
||||
|
||||
@@ -273,6 +273,7 @@
|
||||
"mute_for_me": "Mute for me",
|
||||
"muted_for_me": "Muted for me",
|
||||
"screen_share_volume": "Screen share volume",
|
||||
"unknown_participant": "Unknown participant",
|
||||
"volume": "Volume",
|
||||
"waiting_for_media": "Waiting for media..."
|
||||
}
|
||||
|
||||
@@ -109,15 +109,10 @@ export const widgetTest = test.extend<MyFixtures>({
|
||||
await TestHelpers.dismissInviteUnknownUserModal(ewPage1);
|
||||
|
||||
// Accept the invite
|
||||
await TestHelpers.closeReleaseAnnouncement(
|
||||
ewPage2,
|
||||
"Introducing Sections",
|
||||
);
|
||||
await TestHelpers.expandAllSections(ewPage2);
|
||||
await expect(
|
||||
TestHelpers.roomListItem(ewPage2, "Welcome Room"),
|
||||
ewPage2.getByRole("option", { name: "Welcome Room" }),
|
||||
).toBeVisible();
|
||||
await TestHelpers.roomListItem(ewPage2, "Welcome Room").click();
|
||||
await ewPage2.getByRole("option", { name: "Welcome Room" }).click();
|
||||
await ewPage2.getByRole("button", { name: "Accept" }).click();
|
||||
await expect(
|
||||
ewPage2
|
||||
@@ -157,12 +152,8 @@ export const widgetTest = test.extend<MyFixtures>({
|
||||
).toBeVisible();
|
||||
|
||||
// Accept the DM invite from brooks
|
||||
await TestHelpers.closeReleaseAnnouncement(
|
||||
ewPage2,
|
||||
"Introducing Sections",
|
||||
);
|
||||
await TestHelpers.expandAllSections(ewPage2);
|
||||
await TestHelpers.roomListItem(ewPage2, brooksDisplayName).click();
|
||||
// This how playwright record selects the DM invite in the room list
|
||||
await ewPage2.getByRole("button", { name: "Open room" }).click();
|
||||
await ewPage2.getByRole("button", { name: "Start chatting" }).click();
|
||||
}
|
||||
|
||||
|
||||
@@ -11,7 +11,6 @@ import {
|
||||
type JSHandle,
|
||||
type Page,
|
||||
type FrameLocator,
|
||||
type Locator,
|
||||
} from "@playwright/test";
|
||||
import { type MatrixClient } from "matrix-js-sdk";
|
||||
|
||||
@@ -204,16 +203,6 @@ export class TestHelpers {
|
||||
}
|
||||
}
|
||||
|
||||
public static async expandAllSections(page: Page): Promise<void> {
|
||||
try {
|
||||
await page
|
||||
.getByRole("button", { name: "Expand all sections" })
|
||||
.click({ timeout: 2000 });
|
||||
} catch {
|
||||
// Already expanded or button not present
|
||||
}
|
||||
}
|
||||
|
||||
public static async createRoom(
|
||||
name: string,
|
||||
page: Page,
|
||||
@@ -264,9 +253,9 @@ export class TestHelpers {
|
||||
roomName: string,
|
||||
page: Page,
|
||||
): Promise<void> {
|
||||
await TestHelpers.closeReleaseAnnouncement(page, "Introducing Sections");
|
||||
await TestHelpers.expandAllSections(page);
|
||||
await TestHelpers.roomListItem(page, roomName).click({ timeout: 10000 });
|
||||
await page.getByRole("option", { name: roomName }).click({
|
||||
timeout: 10000,
|
||||
});
|
||||
await page.getByRole("button", { name: "Accept" }).click({
|
||||
timeout: 5000,
|
||||
});
|
||||
@@ -361,17 +350,6 @@ export class TestHelpers {
|
||||
.click();
|
||||
}
|
||||
|
||||
/**
|
||||
* Locates a room in the room list by its name.
|
||||
*
|
||||
* Matches on the aria-label prefix because the item's role differs between
|
||||
* the flat room list (`option`) and the sectioned room list (`button`), and
|
||||
* the label may carry a suffix such as " invitation.".
|
||||
*/
|
||||
public static roomListItem(page: Page, roomName: string): Locator {
|
||||
return page.locator(`[aria-label^="Open room ${roomName}"]`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Switches to a room in the room list by its name.
|
||||
* @param page - The EW page
|
||||
@@ -381,8 +359,7 @@ export class TestHelpers {
|
||||
page: Page,
|
||||
roomName: string,
|
||||
): Promise<void> {
|
||||
await TestHelpers.expandAllSections(page);
|
||||
await TestHelpers.roomListItem(page, roomName).click();
|
||||
await page.getByRole("option", { name: `Open room ${roomName}` }).click();
|
||||
}
|
||||
|
||||
public static async dismissInviteUnknownUserModal(page: Page): Promise<void> {
|
||||
|
||||
@@ -25,11 +25,11 @@ import { useTranslation } from "react-i18next";
|
||||
|
||||
import styles from "./MediaMuteAndSwitchButton.module.css";
|
||||
import { MicButton, VideoButton } from "../button";
|
||||
import { type AudioInputDeviceLabel } from "../state/MediaDevices";
|
||||
import { type DeviceLabel } from "../state/MediaDevices";
|
||||
import { useMediaDevices } from "../MediaDevicesContext";
|
||||
|
||||
export interface MenuOptions {
|
||||
label: AudioInputDeviceLabel;
|
||||
label: DeviceLabel;
|
||||
id: string;
|
||||
}
|
||||
|
||||
@@ -180,9 +180,6 @@ export const MediaMuteAndSwitchButton: FC<MediaMuteAndSwitchButtonProps> = ({
|
||||
case "number":
|
||||
labelText = numberedLabel(label.number);
|
||||
break;
|
||||
case "default":
|
||||
labelText = t("settings.devices.default");
|
||||
break;
|
||||
}
|
||||
return (
|
||||
<MenuItem
|
||||
|
||||
@@ -135,8 +135,7 @@ export const LobbyView: FC<Props> = ({
|
||||
const initialAudioOptions = useInitial(
|
||||
() =>
|
||||
audioEnabled && {
|
||||
// "" is the virtual browser default: no deviceId constraint
|
||||
deviceId: getValue(devices.audioInput.selected$)?.id || undefined,
|
||||
deviceId: getValue(devices.audioInput.selected$)?.id,
|
||||
},
|
||||
);
|
||||
|
||||
|
||||
@@ -1,116 +0,0 @@
|
||||
/*
|
||||
Copyright 2026 Element Corp.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { afterEach, beforeEach, describe, vi, it, expect } from "vitest";
|
||||
import * as ComponentsCore from "@livekit/components-core";
|
||||
|
||||
import { ObservableScope } from "./ObservableScope";
|
||||
import { MediaDevices } from "./MediaDevices";
|
||||
import { audioInput as audioInputSetting } from "../settings/settings";
|
||||
import { withTestScheduler } from "../utils/test";
|
||||
|
||||
const BUILT_IN_MIC = {
|
||||
deviceId: "b3d4cb5e0c8e1d7a2f9b6c4e5d8a7f1c3e2b9d6a5c4f8e7d1b2a3c4d5e6f7a8b",
|
||||
kind: "audioinput",
|
||||
label: "iPhone Microphone",
|
||||
groupId: "1",
|
||||
} as unknown as MediaDeviceInfo;
|
||||
|
||||
const HEADSET_MIC = {
|
||||
deviceId: "c4e5d6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5",
|
||||
kind: "audioinput",
|
||||
label: "Headset Microphone",
|
||||
groupId: "2",
|
||||
} as unknown as MediaDeviceInfo;
|
||||
|
||||
// Chrome exposes a synthetic "default" device that follows the OS default.
|
||||
const CHROME_DEFAULT = {
|
||||
...HEADSET_MIC,
|
||||
deviceId: "default",
|
||||
label: `Default - ${HEADSET_MIC.label}`,
|
||||
} as unknown as MediaDeviceInfo;
|
||||
|
||||
vi.mock("@livekit/components-core", () => ({
|
||||
createMediaDeviceObserver: vi.fn(),
|
||||
}));
|
||||
|
||||
describe("AudioInput", () => {
|
||||
let testScope: ObservableScope;
|
||||
|
||||
beforeEach(() => {
|
||||
testScope = new ObservableScope();
|
||||
// Device preferences persist in localStorage across tests
|
||||
audioInputSetting.setValue(undefined);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
testScope.end();
|
||||
});
|
||||
|
||||
it("uses the browser default input when there is no default pseudo-device", () => {
|
||||
withTestScheduler(({ behavior, cold, expectObservable }) => {
|
||||
vi.mocked(ComponentsCore.createMediaDeviceObserver).mockReturnValue(
|
||||
// iOS / Firefox: physical devices only, built-in first
|
||||
cold("a", { a: [BUILT_IN_MIC, HEADSET_MIC] }),
|
||||
);
|
||||
|
||||
const { audioInput } = new MediaDevices(testScope, {
|
||||
controlledAudioDevices: false,
|
||||
});
|
||||
|
||||
expectObservable(audioInput.available$).toBe("a", {
|
||||
a: new Map([
|
||||
["", { type: "default", name: null }],
|
||||
[BUILT_IN_MIC.deviceId, { type: "name", name: BUILT_IN_MIC.label }],
|
||||
[HEADSET_MIC.deviceId, { type: "name", name: HEADSET_MIC.label }],
|
||||
]),
|
||||
});
|
||||
expectObservable(audioInput.selected$.pipe()).toBe("a", {
|
||||
a: expect.objectContaining({ id: "" }),
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps using Chrome's default device", () => {
|
||||
withTestScheduler(({ behavior, cold, expectObservable }) => {
|
||||
vi.mocked(ComponentsCore.createMediaDeviceObserver).mockReturnValue(
|
||||
cold("a", { a: [CHROME_DEFAULT, BUILT_IN_MIC, HEADSET_MIC] }),
|
||||
);
|
||||
|
||||
const { audioInput } = new MediaDevices(testScope, {
|
||||
controlledAudioDevices: false,
|
||||
});
|
||||
|
||||
expectObservable(audioInput.selected$).toBe("a", {
|
||||
a: expect.objectContaining({ id: "default" }),
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it("honours an explicit choice and falls back to the browser default when it disappears", () => {
|
||||
withTestScheduler(({ behavior, cold, schedule, expectObservable }) => {
|
||||
vi.mocked(ComponentsCore.createMediaDeviceObserver).mockReturnValue(
|
||||
cold("a---b", {
|
||||
a: [BUILT_IN_MIC, HEADSET_MIC],
|
||||
b: [BUILT_IN_MIC],
|
||||
}),
|
||||
);
|
||||
|
||||
const { audioInput } = new MediaDevices(testScope, {
|
||||
controlledAudioDevices: false,
|
||||
});
|
||||
|
||||
schedule("--a", { a: () => audioInput.select(HEADSET_MIC.deviceId) });
|
||||
|
||||
expectObservable(audioInput.selected$).toBe("a-b-c", {
|
||||
a: expect.objectContaining({ id: "" }),
|
||||
b: expect.objectContaining({ id: HEADSET_MIC.deviceId }),
|
||||
c: expect.objectContaining({ id: "" }),
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -52,8 +52,10 @@ import {
|
||||
aliceRtcMember,
|
||||
aliceUserId,
|
||||
bob,
|
||||
bobDeviceId,
|
||||
bobId,
|
||||
bobRtcMember,
|
||||
bobUserId,
|
||||
local,
|
||||
localId,
|
||||
localRtcMember,
|
||||
@@ -118,6 +120,19 @@ const daveId = `${dave.userId}:${daveRtcMember.deviceId}`;
|
||||
const bobParticipant = mockRemoteParticipant({ identity: bobId });
|
||||
const daveParticipant = mockRemoteParticipant({ identity: daveId });
|
||||
|
||||
// A LiveKit participant that no MatrixRTC membership accounts for
|
||||
const rogueParticipant = mockRemoteParticipant({ identity: "rogue" });
|
||||
const rogueId = `unknown:${exampleTransport.livekit_service_url}:rogue`;
|
||||
|
||||
const otherTransport: LivekitTransport = {
|
||||
type: "livekit",
|
||||
livekit_service_url: "https://lk.other.example.org",
|
||||
livekit_alias: "!alias:other.example.org",
|
||||
};
|
||||
const bobOnOtherFocusRtcMember = mockRtcMembership(bobUserId, bobDeviceId, {
|
||||
fociPreferred: [otherTransport],
|
||||
});
|
||||
|
||||
export interface GridLayoutSummary {
|
||||
type: "grid";
|
||||
spotlight?: string[];
|
||||
@@ -450,6 +465,113 @@ describe.each(modes)("CallViewModel (%s mode)", (mode) => {
|
||||
});
|
||||
});
|
||||
|
||||
test("LiveKit participants without a membership get an unknown participant tile", () => {
|
||||
withTestScheduler(({ behavior, expectObservable }) => {
|
||||
// A participant nobody's membership accounts for connects on frame 1 and
|
||||
// disconnects on frame 2
|
||||
const participantInputMarbles = "aba";
|
||||
// It gets a tile at the end of the grid for as long as it is connected
|
||||
const expectedLayoutMarbles = " aba";
|
||||
|
||||
withCallViewModel(
|
||||
{
|
||||
remoteParticipants$: behavior(participantInputMarbles, {
|
||||
a: [aliceParticipant, bobParticipant],
|
||||
b: [aliceParticipant, bobParticipant, rogueParticipant],
|
||||
}),
|
||||
rtcMembers$: constant([localRtcMember, aliceRtcMember, bobRtcMember]),
|
||||
},
|
||||
(vm) => {
|
||||
expectObservable(summarizeLayout$(vm.layout$)).toBe(
|
||||
expectedLayoutMarbles,
|
||||
{
|
||||
a: {
|
||||
type: "grid",
|
||||
spotlight: undefined,
|
||||
grid: [`${localId}:0`, `${aliceId}:0`, `${bobId}:0`],
|
||||
},
|
||||
b: {
|
||||
type: "grid",
|
||||
spotlight: undefined,
|
||||
grid: [`${localId}:0`, `${aliceId}:0`, `${bobId}:0`, rogueId],
|
||||
},
|
||||
},
|
||||
);
|
||||
},
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
test("an unknown participant takes the call out of one-on-one layout", () => {
|
||||
withTestScheduler(({ behavior, expectObservable }) => {
|
||||
// A participant nobody's membership accounts for connects on frame 1 and
|
||||
// disconnects on frame 2
|
||||
const participantInputMarbles = "aba";
|
||||
// The one-on-one layout would hide the unknown participant, so we must
|
||||
// fall back to the grid while they are present
|
||||
const expectedLayoutMarbles = " aba";
|
||||
|
||||
withCallViewModel(
|
||||
{
|
||||
remoteParticipants$: behavior(participantInputMarbles, {
|
||||
a: [aliceParticipant],
|
||||
b: [aliceParticipant, rogueParticipant],
|
||||
}),
|
||||
rtcMembers$: constant([localRtcMember, aliceRtcMember]),
|
||||
},
|
||||
(vm) => {
|
||||
expectObservable(summarizeLayout$(vm.layout$)).toBe(
|
||||
expectedLayoutMarbles,
|
||||
{
|
||||
a: {
|
||||
type: "one-on-one-desktop",
|
||||
pip: `${localId}:0`,
|
||||
spotlight: `${aliceId}:0`,
|
||||
},
|
||||
b: {
|
||||
type: "grid",
|
||||
spotlight: undefined,
|
||||
grid: [`${localId}:0`, `${aliceId}:0`, rogueId],
|
||||
},
|
||||
},
|
||||
);
|
||||
},
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
test("a member seen on a focus other than their own is not an unknown participant", () => {
|
||||
withTestScheduler(({ expectObservable }) => {
|
||||
// Bob publishes to another focus, but also connects to ours to subscribe
|
||||
// (the mock reports the same participants on every connection). Matching
|
||||
// by identity rather than per transport keeps him from being flagged.
|
||||
const expectedLayoutMarbles = "a";
|
||||
|
||||
withCallViewModel(
|
||||
{
|
||||
remoteParticipants$: constant([aliceParticipant, bobParticipant]),
|
||||
rtcMembers$: constant([
|
||||
localRtcMember,
|
||||
aliceRtcMember,
|
||||
bobOnOtherFocusRtcMember,
|
||||
]),
|
||||
},
|
||||
(vm) => {
|
||||
expectObservable(summarizeLayout$(vm.layout$)).toBe(
|
||||
expectedLayoutMarbles,
|
||||
{
|
||||
a: {
|
||||
type: "grid",
|
||||
spotlight: undefined,
|
||||
grid: [`${localId}:0`, `${aliceId}:0`, `${bobId}:0`],
|
||||
},
|
||||
},
|
||||
);
|
||||
},
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
test("one-on-one mobile layout shows local tile when video is enabled", () => {
|
||||
withTestScheduler(({ behavior, schedule, expectObservable }) => {
|
||||
// Local participant enables their video, then disables it
|
||||
@@ -1211,15 +1333,18 @@ describe.each(modes)("CallViewModel (%s mode)", (mode) => {
|
||||
});
|
||||
});
|
||||
|
||||
test("participants must have a MatrixRTCSession to be visible", () => {
|
||||
test("participants without a MatrixRTC membership are shown as unknown, without media", () => {
|
||||
withTestScheduler(({ behavior, expectObservable }) => {
|
||||
// iterate through a number of combinations of participants and MatrixRTC memberships
|
||||
// Bob never has an MatrixRTC membership
|
||||
const participantInputMarbles = "abcd-c";
|
||||
// Bob even tries to share his screen at the end
|
||||
const bobSharingInputMarbles = " n---yn";
|
||||
// Bob should never be visible
|
||||
const expectedLayoutMarbles = " a-bc-b";
|
||||
// Bob gets an unknown participant tile (MSC4143) rather than a tile of
|
||||
// his own, and his screen share never shows up. His presence also keeps
|
||||
// the call out of the one-on-one layout, which would hide him.
|
||||
const expectedLayoutMarbles = " abcd-c";
|
||||
const bobUnknownId = `unknown:${exampleTransport.livekit_service_url}:${bobId}`;
|
||||
|
||||
withCallViewModel(
|
||||
{
|
||||
@@ -1251,14 +1376,25 @@ describe.each(modes)("CallViewModel (%s mode)", (mode) => {
|
||||
grid: [`${localId}:0`],
|
||||
},
|
||||
b: {
|
||||
type: "one-on-one-desktop",
|
||||
pip: `${localId}:0`,
|
||||
spotlight: `${aliceId}:0`,
|
||||
type: "grid",
|
||||
spotlight: undefined,
|
||||
grid: [`${localId}:0`, bobUnknownId],
|
||||
},
|
||||
// Tiles keep their positions, so Bob stays ahead of the others
|
||||
c: {
|
||||
type: "grid",
|
||||
spotlight: undefined,
|
||||
grid: [`${localId}:0`, `${aliceId}:0`, `${daveId}:0`],
|
||||
grid: [`${localId}:0`, bobUnknownId, `${aliceId}:0`],
|
||||
},
|
||||
d: {
|
||||
type: "grid",
|
||||
spotlight: undefined,
|
||||
grid: [
|
||||
`${localId}:0`,
|
||||
bobUnknownId,
|
||||
`${aliceId}:0`,
|
||||
`${daveId}:0`,
|
||||
],
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
@@ -95,6 +95,7 @@ import {
|
||||
layoutShallowEquals,
|
||||
type Alignment,
|
||||
type GridLayoutMedia,
|
||||
type GridMediaViewModel,
|
||||
type Layout,
|
||||
type LayoutMedia,
|
||||
type OneOnOneDesktopLayoutMedia,
|
||||
@@ -162,6 +163,10 @@ import {
|
||||
createRingingMedia,
|
||||
type RingingMediaViewModel,
|
||||
} from "../media/RingingMediaViewModel.ts";
|
||||
import {
|
||||
createUnknownParticipantMedia,
|
||||
type UnknownParticipantMediaViewModel,
|
||||
} from "../media/UnknownParticipantMediaViewModel.ts";
|
||||
import { type GridTileViewModel } from "../TileViewModel.ts";
|
||||
|
||||
//TODO
|
||||
@@ -869,6 +874,61 @@ export function createCallViewModel$(
|
||||
),
|
||||
);
|
||||
|
||||
/**
|
||||
* Media for LiveKit participants that cannot be mapped to any MatrixRTC
|
||||
* membership. MSC4143 asks us to surface these rather than ignore them, as
|
||||
* they can signal an eavesdropping or impersonation attack.
|
||||
*
|
||||
* Participants are matched to memberships by identity alone rather than per
|
||||
* transport, since a member legitimately shows up as a subscribe-only
|
||||
* participant on every focus other than the one they publish to.
|
||||
*/
|
||||
const unknownParticipantMedia$ = scope.behavior<
|
||||
UnknownParticipantMediaViewModel[]
|
||||
>(
|
||||
combineLatest([
|
||||
memberships$,
|
||||
connectionManager.connectionManagerData$,
|
||||
]).pipe(
|
||||
// Memberships and connections derive from the same epoch of session
|
||||
// state; comparing them across epochs would flag members whose
|
||||
// connection hasn't caught up yet.
|
||||
filter(([memberships, data]) => memberships.epoch === data.epoch),
|
||||
map(([memberships, data]) => {
|
||||
const knownIdentities = new Set(
|
||||
memberships.value.map((m) => m.rtcBackendIdentity),
|
||||
);
|
||||
return data.value.getConnections().flatMap((connection) =>
|
||||
data.value
|
||||
.getParticipantsForTransport(connection.transport)
|
||||
.filter((p) => !knownIdentities.has(p.identity))
|
||||
.map((p) => ({
|
||||
focusUrl: connection.transport.livekit_service_url,
|
||||
identity: p.identity,
|
||||
})),
|
||||
);
|
||||
}),
|
||||
generateItems(
|
||||
"CallViewModel unknownParticipantMedia$",
|
||||
function* (unknownParticipants) {
|
||||
for (const { focusUrl, identity } of unknownParticipants)
|
||||
yield { keys: [focusUrl, identity], data: undefined };
|
||||
},
|
||||
(_scope, _data$, focusUrl, identity) => {
|
||||
logger.warn(
|
||||
`LiveKit participant ${identity} on ${focusUrl} matches no MatrixRTC membership`,
|
||||
);
|
||||
return createUnknownParticipantMedia({
|
||||
id: `unknown:${focusUrl}:${identity}`,
|
||||
rtcBackendIdentity: identity,
|
||||
focusUrl,
|
||||
});
|
||||
},
|
||||
),
|
||||
),
|
||||
[],
|
||||
);
|
||||
|
||||
const ringingMedia$ = scope.behavior<RingingMediaViewModel | null>(
|
||||
ringAttempts$.pipe(
|
||||
switchMap(({ intent, recipient, outcome$ }) =>
|
||||
@@ -1015,21 +1075,29 @@ export function createCallViewModel$(
|
||||
),
|
||||
);
|
||||
|
||||
const grid$ = scope.behavior<UserMediaViewModel[]>(
|
||||
userMedia$.pipe(
|
||||
switchMap((mediaItems) => {
|
||||
const bins = mediaItems.map((m) =>
|
||||
m.bin$.pipe(map((bin) => [m, bin] as const)),
|
||||
);
|
||||
// Sort the media by bin order and generate a tile for each one
|
||||
return bins.length === 0
|
||||
? of([])
|
||||
: combineLatest(bins, (...bins) =>
|
||||
bins.sort(([, bin1], [, bin2]) => bin1 - bin2).map(([m]) => m),
|
||||
);
|
||||
}),
|
||||
distinctUntilChanged(shallowArrayEquals),
|
||||
),
|
||||
const sortedUserMedia$: Observable<UserMediaViewModel[]> = userMedia$.pipe(
|
||||
switchMap((mediaItems) => {
|
||||
const bins = mediaItems.map((m) =>
|
||||
m.bin$.pipe(map((bin) => [m, bin] as const)),
|
||||
);
|
||||
// Sort the media by bin order and generate a tile for each one
|
||||
return bins.length === 0
|
||||
? of([])
|
||||
: combineLatest(bins, (...bins) =>
|
||||
bins.sort(([, bin1], [, bin2]) => bin1 - bin2).map(([m]) => m),
|
||||
);
|
||||
}),
|
||||
);
|
||||
|
||||
const grid$ = scope.behavior<GridMediaViewModel[]>(
|
||||
combineLatest(
|
||||
[sortedUserMedia$, unknownParticipantMedia$],
|
||||
// Unknown participants have no media to sort by, so they go last
|
||||
(userMedia, unknownParticipants) => [
|
||||
...userMedia,
|
||||
...unknownParticipants,
|
||||
],
|
||||
).pipe(distinctUntilChanged(shallowArrayEquals)),
|
||||
);
|
||||
|
||||
/**
|
||||
@@ -1189,10 +1257,15 @@ export function createCallViewModel$(
|
||||
local: LocalUserMediaViewModel;
|
||||
remote: UserMediaViewModel | RingingMediaViewModel;
|
||||
} | null> = scope.behavior(
|
||||
combineLatest([userMedia$, screenShares$]).pipe(
|
||||
switchMap(([userMedia, screenShares]) => {
|
||||
// One-on-one layout only supports 2 user media, no screen shares
|
||||
if (userMedia.length <= 2 && screenShares.length === 0) {
|
||||
combineLatest([userMedia$, screenShares$, unknownParticipantMedia$]).pipe(
|
||||
switchMap(([userMedia, screenShares, unknownParticipants]) => {
|
||||
// One-on-one layout only supports 2 user media, no screen shares, and
|
||||
// must never hide an unknown participant
|
||||
if (
|
||||
userMedia.length <= 2 &&
|
||||
screenShares.length === 0 &&
|
||||
unknownParticipants.length === 0
|
||||
) {
|
||||
const local = userMedia.find(
|
||||
(vm): vm is WrappedUserMediaViewModel & LocalUserMediaViewModel =>
|
||||
vm.type === "user" && vm.local,
|
||||
|
||||
@@ -5,26 +5,16 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import {
|
||||
afterEach,
|
||||
beforeEach,
|
||||
describe,
|
||||
expect,
|
||||
it,
|
||||
type Mock,
|
||||
test,
|
||||
vi,
|
||||
} from "vitest";
|
||||
import { afterEach, beforeEach, describe, expect, it, test, vi } from "vitest";
|
||||
import {
|
||||
ConnectionState as LivekitConnectionState,
|
||||
LocalParticipant,
|
||||
type LocalTrack,
|
||||
type LocalTrackPublication,
|
||||
ParticipantEvent,
|
||||
type Room as LivekitRoom,
|
||||
Track,
|
||||
} from "livekit-client";
|
||||
import { BehaviorSubject, NEVER } from "rxjs";
|
||||
import { BehaviorSubject } from "rxjs";
|
||||
import { logger } from "matrix-js-sdk/lib/logger";
|
||||
|
||||
import { ObservableScope } from "../../ObservableScope";
|
||||
@@ -33,12 +23,10 @@ import {
|
||||
flushPromises,
|
||||
mockLivekitRoom,
|
||||
mockMediaDevices,
|
||||
deviceStub,
|
||||
} from "../../../utils/test";
|
||||
import { Publisher } from "./Publisher";
|
||||
import { type Connection } from "../remoteMembers/Connection";
|
||||
import { type MuteStates } from "../../MuteStates";
|
||||
import { type MediaDevices } from "../../MediaDevices";
|
||||
|
||||
let scope: ObservableScope;
|
||||
|
||||
@@ -194,159 +182,6 @@ beforeEach(() => {
|
||||
} as unknown as Connection;
|
||||
});
|
||||
|
||||
describe("Publisher device sync", () => {
|
||||
type SelectedInput =
|
||||
| { id: string; hardwareDeviceChange$: typeof NEVER }
|
||||
| undefined;
|
||||
|
||||
function selectedInput(id: string): SelectedInput {
|
||||
return { id, hardwareDeviceChange$: NEVER };
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a Publisher whose LiveKit room starts out capturing from
|
||||
* capturedDeviceId (undefined meaning no deviceId constraint, i.e. the
|
||||
* browser default) and reports activeDeviceId as its active audio input.
|
||||
*/
|
||||
function createDeviceSyncPublisher({
|
||||
selected$,
|
||||
capturedDeviceId,
|
||||
activeDeviceId,
|
||||
switchActiveDevice = vi.fn().mockResolvedValue(true),
|
||||
}: {
|
||||
selected$: BehaviorSubject<SelectedInput>;
|
||||
capturedDeviceId?: string;
|
||||
activeDeviceId: string;
|
||||
switchActiveDevice?: Mock;
|
||||
}): { publisher: Publisher; switchActiveDevice: Mock } {
|
||||
const livekitRoom = mockLivekitRoom({
|
||||
localParticipant,
|
||||
state: LivekitConnectionState.Connected,
|
||||
// ConnectionFactory leaves the deviceId out for the browser default
|
||||
options: { audioCaptureDefaults: { deviceId: capturedDeviceId } },
|
||||
switchActiveDevice,
|
||||
getActiveDevice: () => activeDeviceId,
|
||||
} as unknown as Partial<LivekitRoom>);
|
||||
|
||||
const publisher = new Publisher(
|
||||
{ ...connection, livekitRoom },
|
||||
mockMediaDevices({
|
||||
audioInput: { ...deviceStub, selected$ },
|
||||
} as unknown as Partial<MediaDevices>),
|
||||
muteStates,
|
||||
constant({ supported: false, processor: undefined }),
|
||||
logger,
|
||||
false,
|
||||
);
|
||||
|
||||
return { publisher, switchActiveDevice };
|
||||
}
|
||||
|
||||
it("does not pin a device for the virtual browser default input", async () => {
|
||||
const selected$ = new BehaviorSubject<SelectedInput>(selectedInput(""));
|
||||
const { publisher, switchActiveDevice } = createDeviceSyncPublisher({
|
||||
selected$,
|
||||
activeDeviceId: "hardware-id-of-whatever-the-browser-chose",
|
||||
});
|
||||
|
||||
// Already capturing from the browser default: nothing to switch, even
|
||||
// though LiveKit reports the physical device it ended up with.
|
||||
expect(switchActiveDevice).not.toHaveBeenCalled();
|
||||
|
||||
selected$.next(selectedInput("headset"));
|
||||
expect(switchActiveDevice).toHaveBeenLastCalledWith(
|
||||
"audioinput",
|
||||
"headset",
|
||||
);
|
||||
|
||||
selected$.next(selectedInput(""));
|
||||
expect(switchActiveDevice).toHaveBeenLastCalledWith(
|
||||
"audioinput",
|
||||
"default",
|
||||
false,
|
||||
);
|
||||
expect(switchActiveDevice).toHaveBeenCalledTimes(2);
|
||||
|
||||
await publisher.destroy();
|
||||
});
|
||||
|
||||
it("pins the device the browser default resolved to when picked explicitly", async () => {
|
||||
const selected$ = new BehaviorSubject<SelectedInput>(selectedInput(""));
|
||||
const { publisher, switchActiveDevice } = createDeviceSyncPublisher({
|
||||
selected$,
|
||||
// The browser default happens to capture from the built-in microphone
|
||||
activeDeviceId: "built-in-mic",
|
||||
});
|
||||
|
||||
expect(switchActiveDevice).not.toHaveBeenCalled();
|
||||
|
||||
// Picking that same microphone has to replace the loose constraint with a
|
||||
// pin, or the capture would keep following the OS default
|
||||
selected$.next(selectedInput("built-in-mic"));
|
||||
expect(switchActiveDevice).toHaveBeenCalledTimes(1);
|
||||
expect(switchActiveDevice).toHaveBeenLastCalledWith(
|
||||
"audioinput",
|
||||
"built-in-mic",
|
||||
);
|
||||
|
||||
// Once pinned, re-emitting the same selection is a no-op again
|
||||
selected$.next(selectedInput("built-in-mic"));
|
||||
expect(switchActiveDevice).toHaveBeenCalledTimes(1);
|
||||
|
||||
await publisher.destroy();
|
||||
});
|
||||
|
||||
it("retries the same selection after a failed switch", async () => {
|
||||
const selected$ = new BehaviorSubject<SelectedInput>(selectedInput(""));
|
||||
const switchActiveDevice = vi
|
||||
.fn()
|
||||
.mockRejectedValueOnce(new Error("could not open the device"))
|
||||
.mockResolvedValue(true);
|
||||
const { publisher } = createDeviceSyncPublisher({
|
||||
selected$,
|
||||
// Pinned to the headset, so moving to the browser default is a switch
|
||||
capturedDeviceId: "headset",
|
||||
activeDeviceId: "headset",
|
||||
switchActiveDevice,
|
||||
});
|
||||
|
||||
expect(switchActiveDevice).toHaveBeenCalledTimes(1);
|
||||
await flushPromises();
|
||||
|
||||
// The failed switch left us on the headset, so selecting the browser
|
||||
// default again has to try once more rather than count as already done
|
||||
selected$.next(selectedInput(""));
|
||||
expect(switchActiveDevice).toHaveBeenCalledTimes(2);
|
||||
expect(switchActiveDevice).toHaveBeenLastCalledWith(
|
||||
"audioinput",
|
||||
"default",
|
||||
false,
|
||||
);
|
||||
await flushPromises();
|
||||
|
||||
// This time it worked, so there is nothing left to do
|
||||
selected$.next(selectedInput(""));
|
||||
expect(switchActiveDevice).toHaveBeenCalledTimes(2);
|
||||
|
||||
await publisher.destroy();
|
||||
});
|
||||
|
||||
it("does not switch when the room already captures from the selected device", async () => {
|
||||
const selected$ = new BehaviorSubject<SelectedInput>(
|
||||
selectedInput("headset"),
|
||||
);
|
||||
const { publisher, switchActiveDevice } = createDeviceSyncPublisher({
|
||||
selected$,
|
||||
capturedDeviceId: "headset",
|
||||
activeDeviceId: "headset",
|
||||
});
|
||||
|
||||
expect(switchActiveDevice).not.toHaveBeenCalled();
|
||||
|
||||
await publisher.destroy();
|
||||
});
|
||||
});
|
||||
|
||||
describe("Publisher", () => {
|
||||
let publisher: Publisher;
|
||||
|
||||
|
||||
@@ -355,19 +355,8 @@ export class Publisher {
|
||||
const syncDevice = (
|
||||
kind: MediaDeviceKind,
|
||||
selected$: Observable<SelectedDevice | undefined>,
|
||||
): Subscription => {
|
||||
// The ID we last asked LiveKit for. Needed for the virtual browser
|
||||
// default input ("") because LiveKit reports the physical device it
|
||||
// ended up capturing from, which never equals "".
|
||||
// ConnectionFactory already applied the selection when the room was
|
||||
// created; a browser-default input is represented there by an absent
|
||||
// deviceId.
|
||||
let requestedId: string | undefined =
|
||||
kind === "audioinput" &&
|
||||
lkRoom.options.audioCaptureDefaults?.deviceId === undefined
|
||||
? ""
|
||||
: undefined;
|
||||
return selected$.pipe(scope.bind()).subscribe((device) => {
|
||||
): Subscription =>
|
||||
selected$.pipe(scope.bind()).subscribe((device) => {
|
||||
if (lkRoom.state != LivekitConnectionState.Connected) return;
|
||||
// if (this.connectionState$.value !== ConnectionState.Connected) return;
|
||||
this.logger.info(
|
||||
@@ -376,41 +365,20 @@ export class Publisher {
|
||||
" !== ",
|
||||
device?.id,
|
||||
);
|
||||
if (device === undefined) return;
|
||||
const browserDefaultInput = device.id === "" && kind !== "audiooutput";
|
||||
// While we capture from the browser default, LiveKit reports the
|
||||
// physical device it resolved to, which can be any of the devices the
|
||||
// user could also pick explicitly. getActiveDevice therefore can't
|
||||
// tell an explicit pin apart from the default resolving to the same
|
||||
// device, and only requestedId can: without this, picking the device
|
||||
// the default happens to use would leave the loose constraint in
|
||||
// place, and the capture would keep following the OS default.
|
||||
const capturingBrowserDefault = requestedId === "";
|
||||
if (browserDefaultInput) {
|
||||
if (capturingBrowserDefault) return;
|
||||
} else if (
|
||||
!capturingBrowserDefault &&
|
||||
lkRoom.getActiveDevice(kind) === device.id
|
||||
if (
|
||||
device !== undefined &&
|
||||
lkRoom.getActiveDevice(kind) !== device.id
|
||||
) {
|
||||
return;
|
||||
lkRoom
|
||||
.switchActiveDevice(kind, device.id)
|
||||
.catch((e: Error) =>
|
||||
this.logger.error(
|
||||
`Failed to sync ${kind} device with LiveKit`,
|
||||
e,
|
||||
),
|
||||
);
|
||||
}
|
||||
const previousRequestedId = requestedId;
|
||||
requestedId = device.id;
|
||||
// For the browser default input, ask for "default" as a non-exact
|
||||
// constraint: browsers that have no such device ignore it and capture
|
||||
// from the OS default input.
|
||||
(browserDefaultInput
|
||||
? lkRoom.switchActiveDevice(kind, "default", false)
|
||||
: lkRoom.switchActiveDevice(kind, device.id)
|
||||
).catch((e: Error) => {
|
||||
// LiveKit keeps capturing from the previous device, so record that
|
||||
// again to leave the next selection of this device something to do.
|
||||
// A selection made while the switch was in flight wins, though.
|
||||
if (requestedId === device.id) requestedId = previousRequestedId;
|
||||
this.logger.error(`Failed to sync ${kind} device with LiveKit`, e);
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
syncDevice("audioinput", devices.audioInput.selected$);
|
||||
if (!controlledAudioDevices)
|
||||
|
||||
@@ -174,9 +174,7 @@ function generateRoomOption({
|
||||
publishDefaults,
|
||||
audioCaptureDefaults: {
|
||||
...liveKitOptions.audioCaptureDefaults,
|
||||
// "" is the virtual browser default: leave the constraint out so the
|
||||
// browser captures from the OS default input.
|
||||
deviceId: devices.audioInput.selected$.value?.id || undefined,
|
||||
deviceId: devices.audioInput.selected$.value?.id,
|
||||
echoCancellation: echoCancellationSetting.getValue(),
|
||||
noiseSuppression: noiseSuppressionSetting.getValue(),
|
||||
autoGainControl: autoGainControlSetting.getValue(),
|
||||
|
||||
@@ -36,14 +36,11 @@ export type DeviceLabel =
|
||||
| { type: "name"; name: string }
|
||||
| { type: "number"; number: number };
|
||||
|
||||
export type AudioInputDeviceLabel =
|
||||
| DeviceLabel
|
||||
| { type: "default"; name: string | null };
|
||||
|
||||
export type AudioOutputDeviceLabel =
|
||||
| AudioInputDeviceLabel
|
||||
| DeviceLabel
|
||||
| { type: "speaker" }
|
||||
| { type: "earpiece" };
|
||||
| { type: "earpiece" }
|
||||
| { type: "default"; name: string | null };
|
||||
|
||||
/**
|
||||
* Base selected-device value shared by all media kinds.
|
||||
@@ -194,10 +191,7 @@ function selectDevice$<Label>(
|
||||
});
|
||||
}
|
||||
|
||||
class AudioInput implements MediaDevice<
|
||||
AudioInputDeviceLabel,
|
||||
SelectedAudioInputDevice
|
||||
> {
|
||||
class AudioInput implements MediaDevice<DeviceLabel, SelectedAudioInputDevice> {
|
||||
private logger = rootLogger.getChild("[MediaDevices AudioInput]");
|
||||
|
||||
private readonly availableRaw$: Behavior<MediaDeviceInfo[]> =
|
||||
@@ -209,29 +203,7 @@ class AudioInput implements MediaDevice<
|
||||
);
|
||||
|
||||
public readonly available$ = this.scope.behavior(
|
||||
this.availableRaw$.pipe(
|
||||
map((availableRaw) => {
|
||||
const available: Map<string, AudioInputDeviceLabel> =
|
||||
buildDeviceMap(availableRaw);
|
||||
// Browsers without a "default" pseudo-device (Firefox, Safari, and
|
||||
// in particular iOS) get a virtual default entry, listed first so
|
||||
// that it is what we use when the user has not chosen a microphone.
|
||||
// Its ID is the empty string, which consumers translate into "no
|
||||
// deviceId constraint": the browser then captures from whatever the
|
||||
// OS routes as the default input (a wired or Bluetooth headset when
|
||||
// one is connected) instead of EC pinning whichever device happens
|
||||
// to be enumerated first, which on iOS is the built-in microphone.
|
||||
// Unlike Chrome's "default" device, a stream opened this way does
|
||||
// not follow later changes of the OS default on desktop browsers;
|
||||
// iOS re-routes the audio session itself.
|
||||
if (available.size && !available.has("") && !available.has("default"))
|
||||
return new Map<string, AudioInputDeviceLabel>([
|
||||
["", { type: "default", name: null }],
|
||||
...available,
|
||||
]);
|
||||
return available;
|
||||
}),
|
||||
),
|
||||
this.availableRaw$.pipe(map(buildDeviceMap)),
|
||||
);
|
||||
|
||||
public readonly selected$ = this.scope.behavior(
|
||||
@@ -299,6 +271,9 @@ export class AudioOutput implements MediaDevice<
|
||||
// set to empty map if we are on Safari, because it does not support setSinkId
|
||||
available = new Map();
|
||||
}
|
||||
// Note: creating virtual default input devices would be another problem
|
||||
// entirely, because requesting a media stream from deviceId "" won't
|
||||
// automatically track the default device.
|
||||
return available;
|
||||
}),
|
||||
),
|
||||
@@ -402,7 +377,7 @@ export class MediaDevices {
|
||||
false,
|
||||
);
|
||||
public readonly audioInput: MediaDevice<
|
||||
AudioInputDeviceLabel,
|
||||
DeviceLabel,
|
||||
SelectedAudioInputDevice
|
||||
> = new AudioInput(this.usingNames$, this.scope);
|
||||
|
||||
|
||||
+13
-12
@@ -8,12 +8,15 @@ Please see LICENSE in the repository root for full details.
|
||||
import { BehaviorSubject } from "rxjs";
|
||||
import { logger } from "matrix-js-sdk/lib/logger";
|
||||
|
||||
import { GridTileViewModel, SpotlightTileViewModel } from "./TileViewModel";
|
||||
import {
|
||||
type GridTileMediaViewModel,
|
||||
GridTileViewModel,
|
||||
SpotlightTileViewModel,
|
||||
} from "./TileViewModel";
|
||||
import { fillGaps } from "../utils/iter";
|
||||
import { debugTileLayout } from "../settings/settings";
|
||||
import { type MediaViewModel } from "./media/MediaViewModel";
|
||||
import { type UserMediaViewModel } from "./media/UserMediaViewModel";
|
||||
import { type RingingMediaViewModel } from "./media/RingingMediaViewModel";
|
||||
|
||||
type SpotlightBackground = "solid" | "transparent";
|
||||
|
||||
@@ -68,10 +71,8 @@ class SpotlightTileData {
|
||||
}
|
||||
|
||||
class GridTileData {
|
||||
private readonly media$: BehaviorSubject<
|
||||
UserMediaViewModel | RingingMediaViewModel
|
||||
>;
|
||||
public get media(): UserMediaViewModel | RingingMediaViewModel {
|
||||
private readonly media$: BehaviorSubject<GridTileMediaViewModel>;
|
||||
public get media(): GridTileMediaViewModel {
|
||||
return this.media$.value;
|
||||
}
|
||||
public set media(value: UserMediaViewModel) {
|
||||
@@ -80,7 +81,7 @@ class GridTileData {
|
||||
|
||||
public readonly vm: GridTileViewModel;
|
||||
|
||||
public constructor(media: UserMediaViewModel | RingingMediaViewModel) {
|
||||
public constructor(media: GridTileMediaViewModel) {
|
||||
this.media$ = new BehaviorSubject(media);
|
||||
this.vm = new GridTileViewModel(this.media$);
|
||||
}
|
||||
@@ -140,7 +141,7 @@ export class TileStoreBuilder {
|
||||
: null;
|
||||
|
||||
private readonly prevGridByMedia: Map<
|
||||
MediaViewModel,
|
||||
GridTileMediaViewModel,
|
||||
[GridTileData, number]
|
||||
> = new Map(
|
||||
this.prevGrid.map((entry, i) => [entry.media, [entry, i]] as const),
|
||||
@@ -205,9 +206,7 @@ export class TileStoreBuilder {
|
||||
* Sets up a grid tile for the given media. If this is never called for some
|
||||
* media, then that media will have no grid tile.
|
||||
*/
|
||||
public registerGridTile(
|
||||
media: UserMediaViewModel | RingingMediaViewModel,
|
||||
): void {
|
||||
public registerGridTile(media: GridTileMediaViewModel): void {
|
||||
if (DEBUG_ENABLED)
|
||||
logger.debug(
|
||||
`[TileStore, ${this.generation}] register grid tile: ${media.displayName$.value}`,
|
||||
@@ -215,8 +214,10 @@ export class TileStoreBuilder {
|
||||
|
||||
if (this.spotlight !== null) {
|
||||
// We actually *don't* want spotlight speakers to appear in both the
|
||||
// spotlight and the grid, so they're filtered out here
|
||||
// spotlight and the grid, so they're filtered out here. (Unknown
|
||||
// participants never make it into the spotlight.)
|
||||
if (
|
||||
media.type !== "unknown participant" &&
|
||||
!(media.type === "user" && media.local) &&
|
||||
this.spotlight.media.includes(media)
|
||||
)
|
||||
|
||||
@@ -10,8 +10,14 @@ import { BehaviorSubject } from "rxjs";
|
||||
import { type Behavior } from "./Behavior";
|
||||
import { type MediaViewModel } from "./media/MediaViewModel";
|
||||
import { type RingingMediaViewModel } from "./media/RingingMediaViewModel";
|
||||
import { type UnknownParticipantMediaViewModel } from "./media/UnknownParticipantMediaViewModel";
|
||||
import { type UserMediaViewModel } from "./media/UserMediaViewModel";
|
||||
|
||||
export type GridTileMediaViewModel =
|
||||
| UserMediaViewModel
|
||||
| RingingMediaViewModel
|
||||
| UnknownParticipantMediaViewModel;
|
||||
|
||||
let nextId = 0;
|
||||
function createId(): string {
|
||||
return (nextId++).toString();
|
||||
@@ -23,9 +29,7 @@ export class GridTileViewModel {
|
||||
public readonly showOutline$: Behavior<boolean> = this._showOutline$;
|
||||
|
||||
public constructor(
|
||||
public readonly media$: Behavior<
|
||||
UserMediaViewModel | RingingMediaViewModel
|
||||
>,
|
||||
public readonly media$: Behavior<GridTileMediaViewModel>,
|
||||
) {}
|
||||
|
||||
public setShowOutline(value: boolean): void {
|
||||
|
||||
@@ -10,6 +10,7 @@ import { type BehaviorSubject } from "rxjs";
|
||||
import { type LocalUserMediaViewModel } from "./media/LocalUserMediaViewModel.ts";
|
||||
import { type MediaViewModel } from "./media/MediaViewModel.ts";
|
||||
import { type RingingMediaViewModel } from "./media/RingingMediaViewModel.ts";
|
||||
import { type UnknownParticipantMediaViewModel } from "./media/UnknownParticipantMediaViewModel.ts";
|
||||
import { type UserMediaViewModel } from "./media/UserMediaViewModel.ts";
|
||||
import {
|
||||
type GridTileViewModel,
|
||||
@@ -18,25 +19,29 @@ import {
|
||||
import { type Behavior } from "./Behavior.ts";
|
||||
import { shallowEquals as arrayShallowEquals } from "../utils/array.ts";
|
||||
|
||||
export type GridMediaViewModel =
|
||||
| UserMediaViewModel
|
||||
| UnknownParticipantMediaViewModel;
|
||||
|
||||
export interface GridLayoutMedia {
|
||||
type: "grid";
|
||||
edgeToEdge: false;
|
||||
spotlight?: MediaViewModel[];
|
||||
grid: UserMediaViewModel[];
|
||||
grid: GridMediaViewModel[];
|
||||
}
|
||||
|
||||
export interface SpotlightLandscapeLayoutMedia {
|
||||
type: "spotlight-landscape";
|
||||
edgeToEdge: boolean;
|
||||
spotlight: MediaViewModel[];
|
||||
grid: UserMediaViewModel[];
|
||||
grid: GridMediaViewModel[];
|
||||
}
|
||||
|
||||
export interface SpotlightPortraitLayoutMedia {
|
||||
type: "spotlight-portrait";
|
||||
edgeToEdge: false;
|
||||
spotlight: MediaViewModel[];
|
||||
grid: UserMediaViewModel[];
|
||||
grid: GridMediaViewModel[];
|
||||
}
|
||||
|
||||
export interface SpotlightExpandedLayoutMedia {
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { constant } from "../Behavior";
|
||||
import { type BaseMediaViewModel } from "./MediaViewModel";
|
||||
|
||||
/**
|
||||
* Media representing a LiveKit participant that cannot be mapped to any
|
||||
* MatrixRTC member of the session.
|
||||
*
|
||||
* MSC4143 asks clients to surface such streams rather than silently ignore
|
||||
* them, because they can signal an eavesdropping or impersonation attack. We
|
||||
* therefore give them a tile, but never render their audio or video.
|
||||
*
|
||||
* There is no Matrix user behind this media, so `userId` and `displayName$`
|
||||
* carry the LiveKit identity instead, purely so that debugging output stays
|
||||
* meaningful.
|
||||
*/
|
||||
export interface UnknownParticipantMediaViewModel extends BaseMediaViewModel {
|
||||
type: "unknown participant";
|
||||
/**
|
||||
* The LiveKit identity under which the participant connected. Exposed for
|
||||
* debugging.
|
||||
*/
|
||||
rtcBackendIdentity: string;
|
||||
/**
|
||||
* The URL of the LiveKit focus on which the participant was seen. Exposed for
|
||||
* debugging.
|
||||
*/
|
||||
focusUrl: string;
|
||||
}
|
||||
|
||||
export interface UnknownParticipantMediaInputs {
|
||||
id: string;
|
||||
rtcBackendIdentity: string;
|
||||
focusUrl: string;
|
||||
}
|
||||
|
||||
export function createUnknownParticipantMedia({
|
||||
id,
|
||||
rtcBackendIdentity,
|
||||
focusUrl,
|
||||
}: UnknownParticipantMediaInputs): UnknownParticipantMediaViewModel {
|
||||
return {
|
||||
type: "unknown participant",
|
||||
id,
|
||||
userId: rtcBackendIdentity,
|
||||
displayName$: constant(rtcBackendIdentity),
|
||||
mxcAvatarUrl$: constant(undefined),
|
||||
rtcBackendIdentity,
|
||||
focusUrl,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { expect, within } from "storybook/test";
|
||||
import { type JSX, useMemo } from "react";
|
||||
|
||||
import type { Meta, StoryObj } from "@storybook/react-vite";
|
||||
import { GridTile } from "./GridTile";
|
||||
import { GridTileViewModel } from "../state/TileViewModel";
|
||||
import { constant } from "../state/Behavior";
|
||||
import { createUnknownParticipantMedia } from "../state/media/UnknownParticipantMediaViewModel";
|
||||
|
||||
interface UnknownParticipantTileStoryProps {
|
||||
width: number;
|
||||
height: number;
|
||||
rtcBackendIdentity: string;
|
||||
showNameTags: boolean;
|
||||
showOutline: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Renders a GridTile for a LiveKit participant that maps to no MatrixRTC
|
||||
* member, driven by primitive props so that Storybook can document them.
|
||||
*/
|
||||
function UnknownParticipantTile({
|
||||
width,
|
||||
height,
|
||||
rtcBackendIdentity,
|
||||
...props
|
||||
}: UnknownParticipantTileStoryProps): JSX.Element {
|
||||
const vm = useMemo(
|
||||
() =>
|
||||
new GridTileViewModel(
|
||||
constant(
|
||||
createUnknownParticipantMedia({
|
||||
id: `unknown:https://rtc.example.org:${rtcBackendIdentity}`,
|
||||
rtcBackendIdentity,
|
||||
focusUrl: "https://rtc.example.org",
|
||||
}),
|
||||
),
|
||||
),
|
||||
[rtcBackendIdentity],
|
||||
);
|
||||
return (
|
||||
<GridTile
|
||||
vm={vm}
|
||||
onOpenProfile={null}
|
||||
targetWidth={width}
|
||||
targetHeight={height}
|
||||
style={{ width, height }}
|
||||
showSpeakingIndicators={false}
|
||||
showRingingStatus={false}
|
||||
focusable
|
||||
{...props}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
const meta = {
|
||||
component: UnknownParticipantTile,
|
||||
argTypes: {
|
||||
width: { control: { type: "range", min: 80, max: 800, step: 10 } },
|
||||
height: { control: { type: "range", min: 80, max: 600, step: 10 } },
|
||||
},
|
||||
} satisfies Meta<typeof UnknownParticipantTile>;
|
||||
|
||||
export default meta;
|
||||
type Story = StoryObj<typeof meta>;
|
||||
|
||||
export const UnknownParticipant: Story = {
|
||||
args: {
|
||||
width: 400,
|
||||
height: 300,
|
||||
rtcBackendIdentity: "@rogue:example.org:DEVICE",
|
||||
showNameTags: true,
|
||||
showOutline: false,
|
||||
},
|
||||
play: async ({ canvasElement }) => {
|
||||
const canvas = within(canvasElement);
|
||||
await expect(canvas.getByText("Unknown participant")).toBeInTheDocument();
|
||||
await expect(
|
||||
canvas.queryByText("@rogue:example.org:DEVICE"),
|
||||
).not.toBeInTheDocument();
|
||||
},
|
||||
};
|
||||
|
||||
/** Narrow tiles hide the name tag, so the label goes with it. */
|
||||
export const UnknownParticipantNarrow: Story = {
|
||||
args: {
|
||||
...UnknownParticipant.args,
|
||||
width: 90,
|
||||
height: 90,
|
||||
},
|
||||
};
|
||||
@@ -32,6 +32,7 @@ import {
|
||||
createRingingMedia,
|
||||
type RingingMediaViewModel,
|
||||
} from "../state/media/RingingMediaViewModel";
|
||||
import { createUnknownParticipantMedia } from "../state/media/UnknownParticipantMediaViewModel";
|
||||
|
||||
global.IntersectionObserver = class MockIntersectionObserver {
|
||||
public observe(): void {}
|
||||
@@ -165,3 +166,33 @@ test("GridTile displays ringing media", async () => {
|
||||
act(() => pickupState$.next("decline"));
|
||||
screen.getByText("Call ended");
|
||||
});
|
||||
|
||||
test("GridTile displays an unknown participant", async () => {
|
||||
const vm = createUnknownParticipantMedia({
|
||||
id: "unknown:https://rtc-example.org:rogue",
|
||||
rtcBackendIdentity: "rogue",
|
||||
focusUrl: "https://rtc-example.org",
|
||||
});
|
||||
|
||||
const { container } = render(
|
||||
<ReactionsSenderProvider vm={callVm} rtcSession={fakeRtcSession}>
|
||||
<GridTile
|
||||
vm={new GridTileViewModel(constant(vm))}
|
||||
onOpenProfile={() => {}}
|
||||
targetWidth={300}
|
||||
targetHeight={200}
|
||||
showSpeakingIndicators
|
||||
showNameTags
|
||||
showRingingStatus
|
||||
showOutline
|
||||
focusable
|
||||
/>
|
||||
</ReactionsSenderProvider>,
|
||||
);
|
||||
expect(await axe(container)).toHaveNoViolations();
|
||||
// The tile is labelled as unknown rather than after its LiveKit identity
|
||||
screen.getByText("Unknown participant");
|
||||
expect(screen.queryByText("rogue")).toBeNull();
|
||||
// There is no user to show an avatar for
|
||||
expect(container.querySelector("[data-style]")).toBeNull();
|
||||
});
|
||||
|
||||
@@ -51,6 +51,7 @@ import { type LocalUserMediaViewModel } from "../state/media/LocalUserMediaViewM
|
||||
import { type RemoteUserMediaViewModel } from "../state/media/RemoteUserMediaViewModel";
|
||||
import { type UserMediaViewModel } from "../state/media/UserMediaViewModel";
|
||||
import { type RingingMediaViewModel } from "../state/media/RingingMediaViewModel";
|
||||
import { type UnknownParticipantMediaViewModel } from "../state/media/UnknownParticipantMediaViewModel";
|
||||
import { RingingStatus } from "./RingingStatus";
|
||||
|
||||
interface TileProps {
|
||||
@@ -97,6 +98,44 @@ const RingingMediaTile: FC<RingingMediaTileProps> = ({
|
||||
);
|
||||
};
|
||||
|
||||
interface UnknownParticipantTileProps extends Omit<
|
||||
TileProps,
|
||||
"displayName" | "mxcAvatarUrl"
|
||||
> {
|
||||
vm: UnknownParticipantMediaViewModel;
|
||||
}
|
||||
|
||||
/**
|
||||
* A blank tile standing in for a LiveKit participant that cannot be mapped to
|
||||
* any MatrixRTC member. MSC4143 asks clients to surface such streams rather
|
||||
* than ignore them. No media is rendered for them.
|
||||
*/
|
||||
const UnknownParticipantTile: FC<UnknownParticipantTileProps> = ({
|
||||
vm,
|
||||
className,
|
||||
...props
|
||||
}) => {
|
||||
const { t } = useTranslation();
|
||||
return (
|
||||
<MediaView
|
||||
className={classNames(className, styles.tile)}
|
||||
video={undefined}
|
||||
userId={vm.userId}
|
||||
unencryptedWarning={false}
|
||||
videoEnabled={false}
|
||||
mirror={false}
|
||||
showAvatar={false}
|
||||
displayName={t("video_tile.unknown_participant")}
|
||||
mxcAvatarUrl={undefined}
|
||||
rtcBackendIdentity={vm.rtcBackendIdentity}
|
||||
focusUrl={vm.focusUrl}
|
||||
{...props}
|
||||
/>
|
||||
);
|
||||
};
|
||||
|
||||
UnknownParticipantTile.displayName = "UnknownParticipantTile";
|
||||
|
||||
interface UserMediaTileProps extends TileProps {
|
||||
vm: UserMediaViewModel;
|
||||
showSpeakingIndicators: boolean;
|
||||
@@ -454,6 +493,15 @@ export const GridTile: FC<GridTileProps> = ({
|
||||
{...props}
|
||||
/>
|
||||
);
|
||||
} else if (media.type === "unknown participant") {
|
||||
return (
|
||||
<UnknownParticipantTile
|
||||
ref={ref}
|
||||
vm={media}
|
||||
className={classNames(className, { [styles.outline]: showOutline })}
|
||||
{...props}
|
||||
/>
|
||||
);
|
||||
} else if (media.local) {
|
||||
return (
|
||||
<LocalUserMediaTile
|
||||
|
||||
+17
-9
@@ -56,6 +56,11 @@ interface Props extends ComponentProps<typeof animated.div> {
|
||||
displayName: string;
|
||||
mxcAvatarUrl: string | undefined;
|
||||
avatarStyle?: "solid" | "translucent";
|
||||
/**
|
||||
* Whether to show an avatar when there is no video. Off for media that has no
|
||||
* Matrix user behind it.
|
||||
*/
|
||||
showAvatar?: boolean;
|
||||
background?: "solid" | "transparent";
|
||||
focusable: boolean;
|
||||
primaryButton?: ReactNode;
|
||||
@@ -95,6 +100,7 @@ export const MediaView: FC<Props> = ({
|
||||
displayName,
|
||||
mxcAvatarUrl,
|
||||
avatarStyle = "solid",
|
||||
showAvatar = true,
|
||||
background = "solid",
|
||||
focusable,
|
||||
primaryButton,
|
||||
@@ -176,15 +182,17 @@ export const MediaView: FC<Props> = ({
|
||||
<div className={styles.speakingBorder} />
|
||||
</div>
|
||||
)}
|
||||
<Avatar
|
||||
id={userId}
|
||||
name={displayName}
|
||||
size={avatarSize}
|
||||
src={mxcAvatarUrl}
|
||||
data-style={avatarStyle}
|
||||
className={styles.avatar}
|
||||
style={{ display: video && videoEnabled ? "none" : "initial" }}
|
||||
/>
|
||||
{showAvatar && (
|
||||
<Avatar
|
||||
id={userId}
|
||||
name={displayName}
|
||||
size={avatarSize}
|
||||
src={mxcAvatarUrl}
|
||||
data-style={avatarStyle}
|
||||
className={styles.avatar}
|
||||
style={{ display: video && videoEnabled ? "none" : "initial" }}
|
||||
/>
|
||||
)}
|
||||
{video?.publication !== undefined && (
|
||||
<VideoTrack
|
||||
trackRef={video}
|
||||
|
||||
Reference in New Issue
Block a user