Compare commits

..
Author SHA1 Message Date
Johannes Marbach cb0490e3f0 Render tiles for unmappable LiveKit identities 2026-09-14 10:30:45 +02:00
23 changed files with 545 additions and 200 deletions
+2 -2
View File
@@ -19,7 +19,7 @@ services:
- ./backend/playwright_homeserver-othersite.yaml:/data/cfg/homeserver.yaml:Z
element-web:
# Pin to a SHA so that upstream cannot break our tests. Renovate handles regular updates.
image: ghcr.io/element-hq/element-web:develop@sha256:3993d53bc2c784c0376b86e00230aef0ee2c9a705b2166e9229555117e8db167
image: ghcr.io/element-hq/element-web:develop@sha256:6ed62b2963ae098a1f79521eb516834dcf29452168e281cc359d9cfb4380adf6
element-web-1:
# Pin to a SHA so that upstream cannot break our tests. Renovate handles regular updates.
image: ghcr.io/element-hq/element-web:develop@sha256:3993d53bc2c784c0376b86e00230aef0ee2c9a705b2166e9229555117e8db167
image: ghcr.io/element-hq/element-web:develop@sha256:6ed62b2963ae098a1f79521eb516834dcf29452168e281cc359d9cfb4380adf6
+1
View File
@@ -273,6 +273,7 @@
"mute_for_me": "Mute for me",
"muted_for_me": "Muted for me",
"screen_share_volume": "Screen share volume",
"unknown_participant": "Unknown participant",
"volume": "Volume",
"waiting_for_media": "Waiting for media..."
}
+1 -1
View File
@@ -83,7 +83,7 @@
"@types/sdp-transform": "^2.4.5",
"@typescript-eslint/utils": "^8.61.0",
"@use-gesture/react": "^10.2.11",
"@vector-im/compound-design-tokens": "^11.0.0",
"@vector-im/compound-design-tokens": "^10.0.0",
"@vector-im/compound-web": "^10.0.0",
"@vitejs/plugin-react": "^6.0.2",
"@vitest/browser-playwright": "^4.1.5",
+4 -13
View File
@@ -109,15 +109,10 @@ export const widgetTest = test.extend<MyFixtures>({
await TestHelpers.dismissInviteUnknownUserModal(ewPage1);
// Accept the invite
await TestHelpers.closeReleaseAnnouncement(
ewPage2,
"Introducing Sections",
);
await TestHelpers.expandAllSections(ewPage2);
await expect(
TestHelpers.roomListItem(ewPage2, "Welcome Room"),
ewPage2.getByRole("option", { name: "Welcome Room" }),
).toBeVisible();
await TestHelpers.roomListItem(ewPage2, "Welcome Room").click();
await ewPage2.getByRole("option", { name: "Welcome Room" }).click();
await ewPage2.getByRole("button", { name: "Accept" }).click();
await expect(
ewPage2
@@ -157,12 +152,8 @@ export const widgetTest = test.extend<MyFixtures>({
).toBeVisible();
// Accept the DM invite from brooks
await TestHelpers.closeReleaseAnnouncement(
ewPage2,
"Introducing Sections",
);
await TestHelpers.expandAllSections(ewPage2);
await TestHelpers.roomListItem(ewPage2, brooksDisplayName).click();
// This how playwright record selects the DM invite in the room list
await ewPage2.getByRole("button", { name: "Open room" }).click();
await ewPage2.getByRole("button", { name: "Start chatting" }).click();
}
+4 -27
View File
@@ -11,7 +11,6 @@ import {
type JSHandle,
type Page,
type FrameLocator,
type Locator,
} from "@playwright/test";
import { type MatrixClient } from "matrix-js-sdk";
@@ -204,16 +203,6 @@ export class TestHelpers {
}
}
public static async expandAllSections(page: Page): Promise<void> {
try {
await page
.getByRole("button", { name: "Expand all sections" })
.click({ timeout: 2000 });
} catch {
// Already expanded or button not present
}
}
public static async createRoom(
name: string,
page: Page,
@@ -264,9 +253,9 @@ export class TestHelpers {
roomName: string,
page: Page,
): Promise<void> {
await TestHelpers.closeReleaseAnnouncement(page, "Introducing Sections");
await TestHelpers.expandAllSections(page);
await TestHelpers.roomListItem(page, roomName).click({ timeout: 10000 });
await page.getByRole("option", { name: roomName }).click({
timeout: 10000,
});
await page.getByRole("button", { name: "Accept" }).click({
timeout: 5000,
});
@@ -361,17 +350,6 @@ export class TestHelpers {
.click();
}
/**
* Locates a room in the room list by its name.
*
* Matches on the aria-label prefix because the item's role differs between
* the flat room list (`option`) and the sectioned room list (`button`), and
* the label may carry a suffix such as " invitation.".
*/
public static roomListItem(page: Page, roomName: string): Locator {
return page.locator(`[aria-label^="Open room ${roomName}"]`);
}
/**
* Switches to a room in the room list by its name.
* @param page - The EW page
@@ -381,8 +359,7 @@ export class TestHelpers {
page: Page,
roomName: string,
): Promise<void> {
await TestHelpers.expandAllSections(page);
await TestHelpers.roomListItem(page, roomName).click();
await page.getByRole("option", { name: `Open room ${roomName}` }).click();
}
public static async dismissInviteUnknownUserModal(page: Page): Promise<void> {
+8 -8
View File
@@ -131,11 +131,11 @@ importers:
specifier: ^10.2.11
version: 10.3.1(react@19.2.8)
'@vector-im/compound-design-tokens':
specifier: ^11.0.0
version: 11.0.0(@types/react@19.2.18)(react@19.2.8)
specifier: ^10.0.0
version: 10.2.4(@types/react@19.2.18)(react@19.2.8)
'@vector-im/compound-web':
specifier: ^10.0.0
version: 10.0.1(@fontsource/inconsolata@5.3.0)(@fontsource/inter@5.3.0)(@types/react-dom@19.2.5(@types/react@19.2.18))(@types/react@19.2.18)(@vector-im/compound-design-tokens@11.0.0(@types/react@19.2.18)(react@19.2.8))(react-dom@19.2.8(react@19.2.8))(react@19.2.8)
version: 10.0.1(@fontsource/inconsolata@5.3.0)(@fontsource/inter@5.3.0)(@types/react-dom@19.2.5(@types/react@19.2.18))(@types/react@19.2.18)(@vector-im/compound-design-tokens@10.2.4(@types/react@19.2.18)(react@19.2.8))(react-dom@19.2.8(react@19.2.8))(react@19.2.8)
'@vitejs/plugin-react':
specifier: ^6.0.2
version: 6.1.1(@rolldown/plugin-babel@0.2.3(@babel/core@7.29.7(supports-color@7.2.0))(@babel/runtime@8.0.0)(rolldown@1.2.7)(vite@8.2.2(@types/node@24.13.3)(esbuild@0.28.2)(jiti@2.7.0)(sass@1.103.1)(terser@5.46.1)(yaml@2.9.0)))(babel-plugin-react-compiler@1.0.0)(vite@8.2.2(@types/node@24.13.3)(esbuild@0.28.2)(jiti@2.7.0)(sass@1.103.1)(terser@5.46.1)(yaml@2.9.0))
@@ -3229,8 +3229,8 @@ packages:
peerDependencies:
react: '>= 16.8.0'
'@vector-im/compound-design-tokens@11.0.0':
resolution: {integrity: sha512-pWz0Q67Xbo8gwzaTeFbTTcM6i29ju6PR7SqyuExr2nIV3j9K785cOInft8FA2CM7PEapkr6TRtJKR/4xtWh2RA==}
'@vector-im/compound-design-tokens@10.2.4':
resolution: {integrity: sha512-2aCjhSvJxktPWnvh5s9Xdc0Rd76AfHqbDbN4QjOxo2VIac6zYdLyQbxMUQ5ZeoR9G3vAxILd0H1UvApRfMilpQ==}
peerDependencies:
'@adobe/leonardo-contrast-colors': ^1.0.0
'@types/react': '*'
@@ -8695,12 +8695,12 @@ snapshots:
'@use-gesture/core': 10.3.1
react: 19.2.8
'@vector-im/compound-design-tokens@11.0.0(@types/react@19.2.18)(react@19.2.8)':
'@vector-im/compound-design-tokens@10.2.4(@types/react@19.2.18)(react@19.2.8)':
optionalDependencies:
'@types/react': 19.2.18
react: 19.2.8
'@vector-im/compound-web@10.0.1(@fontsource/inconsolata@5.3.0)(@fontsource/inter@5.3.0)(@types/react-dom@19.2.5(@types/react@19.2.18))(@types/react@19.2.18)(@vector-im/compound-design-tokens@11.0.0(@types/react@19.2.18)(react@19.2.8))(react-dom@19.2.8(react@19.2.8))(react@19.2.8)':
'@vector-im/compound-web@10.0.1(@fontsource/inconsolata@5.3.0)(@fontsource/inter@5.3.0)(@types/react-dom@19.2.5(@types/react@19.2.18))(@types/react@19.2.18)(@vector-im/compound-design-tokens@10.2.4(@types/react@19.2.18)(react@19.2.8))(react-dom@19.2.8(react@19.2.8))(react@19.2.8)':
dependencies:
'@floating-ui/react': 0.27.20(react-dom@19.2.8(react@19.2.8))(react@19.2.8)
'@floating-ui/react-dom': 2.1.9(react-dom@19.2.8(react@19.2.8))(react@19.2.8)
@@ -8712,7 +8712,7 @@ snapshots:
'@radix-ui/react-progress': 1.1.16(@types/react-dom@19.2.5(@types/react@19.2.18))(@types/react@19.2.18)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)
'@radix-ui/react-separator': 1.1.15(@types/react-dom@19.2.5(@types/react@19.2.18))(@types/react@19.2.18)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)
'@radix-ui/react-slot': 1.3.3(@types/react@19.2.18)(react@19.2.8)
'@vector-im/compound-design-tokens': 11.0.0(@types/react@19.2.18)(react@19.2.8)
'@vector-im/compound-design-tokens': 10.2.4(@types/react@19.2.18)(react@19.2.8)
classnames: 2.5.1
react: 19.2.8
vaul: 1.1.2(@types/react-dom@19.2.5(@types/react@19.2.18))(@types/react@19.2.18)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)
@@ -9,7 +9,7 @@ exports[`Can close reaction dialog 1`] = `
aria-disabled="false"
aria-expanded="true"
aria-haspopup="true"
aria-labelledby="_r_bh_"
aria-labelledby="_r_bb_"
class="_button_1nw83_8 _has-icon_1nw83_60 _icon-only_1nw83_53"
data-kind="primary"
data-size="lg"
-11
View File
@@ -29,8 +29,6 @@ import waveSoundOgg from "../sound/reactions/wave.ogg?url";
import waveSoundMp3 from "../sound/reactions/wave.mp3?url";
import baduntssSoundOgg from "../sound/reactions/baduntss.ogg?url";
import baduntssSoundMp3 from "../sound/reactions/baduntss.mp3?url";
import bellSoundOgg from "../sound/reactions/bell.ogg?url";
import bellSoundMp3 from "../sound/reactions/bell.mp3?url";
export const ElementCallReactionEventType = "io.element.call.reaction";
@@ -204,15 +202,6 @@ export const ReactionSet: ReactionOption[] = [
mp3: baduntssSoundMp3,
},
},
{
emoji: "🛎️",
name: "bell",
alias: ["ding"],
sound: {
ogg: bellSoundOgg,
mp3: bellSoundMp3,
},
},
];
export interface RaisedHandInfo {
-1
View File
@@ -10,7 +10,6 @@ The following sound effects have been licensed from Pixabay, under https://pixab
- `reactions/generic`
- `reactions/lightbulb`
- `reactions/party`
- `reactions/bell`
### Other
Binary file not shown.
Binary file not shown.
+1 -61
View File
@@ -10,7 +10,6 @@ import * as ComponentsCore from "@livekit/components-core";
import { ObservableScope } from "./ObservableScope";
import { AudioOutput } from "./MediaDevices";
import { audioOutput as audioOutputSetting } from "../settings/settings";
import { withTestScheduler } from "../utils/test";
const BT_SPEAKER = {
@@ -94,8 +93,6 @@ describe("AudioOutput Tests", () => {
beforeEach(() => {
testScope = new ObservableScope();
// Device preferences persist in localStorage across tests
audioOutputSetting.setValue(undefined);
});
afterEach(() => {
@@ -153,9 +150,7 @@ describe("AudioOutput Tests", () => {
expectObservable(audioOutput.selected$).toBe("abcde", {
a: undefined,
// No "default" pseudo-device (Firefox): the virtual browser default is
// selected until the user picks something.
b: { id: "", virtualEarpiece: false },
b: { id: LAPTOP_SPEAKER.deviceId, virtualEarpiece: false },
c: { id: MONITOR_SPEAKER.deviceId, virtualEarpiece: false },
d: { id: LAPTOP_SPEAKER.deviceId, virtualEarpiece: false },
e: { id: MONITOR_SPEAKER.deviceId, virtualEarpiece: false },
@@ -163,61 +158,6 @@ describe("AudioOutput Tests", () => {
});
});
it("falls back to the browser default when the selected device disappears", () => {
withTestScheduler(({ behavior, cold, schedule, expectObservable }) => {
vi.mocked(ComponentsCore.createMediaDeviceObserver).mockReturnValue(
cold("a---b", {
a: DEVICE_LIST_B,
// The monitor is unplugged (or a Bluetooth sink changes profile)
b: [LAPTOP_SPEAKER],
}),
);
const audioOutput = new AudioOutput(
behavior("a", { a: true }),
testScope,
);
schedule("--a", {
a: () => audioOutput.select(MONITOR_SPEAKER.deviceId),
});
expectObservable(audioOutput.selected$).toBe("a-b-c", {
a: { id: "", virtualEarpiece: false },
b: { id: MONITOR_SPEAKER.deviceId, virtualEarpiece: false },
// Rather than pinning some other physical device, let the browser route
c: { id: "", virtualEarpiece: false },
});
});
});
it("lists the virtual browser default first when there is no default pseudo-device", () => {
withTestScheduler(({ behavior, cold, expectObservable }) => {
vi.mocked(ComponentsCore.createMediaDeviceObserver).mockReturnValue(
cold("a", { a: DEVICE_LIST_B }),
);
const audioOutput = new AudioOutput(
behavior("a", { a: true }),
testScope,
);
expectObservable(audioOutput.available$).toBe("a", {
a: new Map([
["", { type: "default", name: null }],
[
LAPTOP_SPEAKER.deviceId,
{ type: "name", name: LAPTOP_SPEAKER.label },
],
[
MONITOR_SPEAKER.deviceId,
{ type: "name", name: MONITOR_SPEAKER.label },
],
]),
});
});
});
it("Test mappings", () => {
// In a real life setup there would be first a blanked list
// then the real one.
+143 -7
View File
@@ -52,8 +52,10 @@ import {
aliceRtcMember,
aliceUserId,
bob,
bobDeviceId,
bobId,
bobRtcMember,
bobUserId,
local,
localId,
localRtcMember,
@@ -118,6 +120,19 @@ const daveId = `${dave.userId}:${daveRtcMember.deviceId}`;
const bobParticipant = mockRemoteParticipant({ identity: bobId });
const daveParticipant = mockRemoteParticipant({ identity: daveId });
// A LiveKit participant that no MatrixRTC membership accounts for
const rogueParticipant = mockRemoteParticipant({ identity: "rogue" });
const rogueId = `unknown:${exampleTransport.livekit_service_url}:rogue`;
const otherTransport: LivekitTransport = {
type: "livekit",
livekit_service_url: "https://lk.other.example.org",
livekit_alias: "!alias:other.example.org",
};
const bobOnOtherFocusRtcMember = mockRtcMembership(bobUserId, bobDeviceId, {
fociPreferred: [otherTransport],
});
export interface GridLayoutSummary {
type: "grid";
spotlight?: string[];
@@ -450,6 +465,113 @@ describe.each(modes)("CallViewModel (%s mode)", (mode) => {
});
});
test("LiveKit participants without a membership get an unknown participant tile", () => {
withTestScheduler(({ behavior, expectObservable }) => {
// A participant nobody's membership accounts for connects on frame 1 and
// disconnects on frame 2
const participantInputMarbles = "aba";
// It gets a tile at the end of the grid for as long as it is connected
const expectedLayoutMarbles = " aba";
withCallViewModel(
{
remoteParticipants$: behavior(participantInputMarbles, {
a: [aliceParticipant, bobParticipant],
b: [aliceParticipant, bobParticipant, rogueParticipant],
}),
rtcMembers$: constant([localRtcMember, aliceRtcMember, bobRtcMember]),
},
(vm) => {
expectObservable(summarizeLayout$(vm.layout$)).toBe(
expectedLayoutMarbles,
{
a: {
type: "grid",
spotlight: undefined,
grid: [`${localId}:0`, `${aliceId}:0`, `${bobId}:0`],
},
b: {
type: "grid",
spotlight: undefined,
grid: [`${localId}:0`, `${aliceId}:0`, `${bobId}:0`, rogueId],
},
},
);
},
);
});
});
test("an unknown participant takes the call out of one-on-one layout", () => {
withTestScheduler(({ behavior, expectObservable }) => {
// A participant nobody's membership accounts for connects on frame 1 and
// disconnects on frame 2
const participantInputMarbles = "aba";
// The one-on-one layout would hide the unknown participant, so we must
// fall back to the grid while they are present
const expectedLayoutMarbles = " aba";
withCallViewModel(
{
remoteParticipants$: behavior(participantInputMarbles, {
a: [aliceParticipant],
b: [aliceParticipant, rogueParticipant],
}),
rtcMembers$: constant([localRtcMember, aliceRtcMember]),
},
(vm) => {
expectObservable(summarizeLayout$(vm.layout$)).toBe(
expectedLayoutMarbles,
{
a: {
type: "one-on-one-desktop",
pip: `${localId}:0`,
spotlight: `${aliceId}:0`,
},
b: {
type: "grid",
spotlight: undefined,
grid: [`${localId}:0`, `${aliceId}:0`, rogueId],
},
},
);
},
);
});
});
test("a member seen on a focus other than their own is not an unknown participant", () => {
withTestScheduler(({ expectObservable }) => {
// Bob publishes to another focus, but also connects to ours to subscribe
// (the mock reports the same participants on every connection). Matching
// by identity rather than per transport keeps him from being flagged.
const expectedLayoutMarbles = "a";
withCallViewModel(
{
remoteParticipants$: constant([aliceParticipant, bobParticipant]),
rtcMembers$: constant([
localRtcMember,
aliceRtcMember,
bobOnOtherFocusRtcMember,
]),
},
(vm) => {
expectObservable(summarizeLayout$(vm.layout$)).toBe(
expectedLayoutMarbles,
{
a: {
type: "grid",
spotlight: undefined,
grid: [`${localId}:0`, `${aliceId}:0`, `${bobId}:0`],
},
},
);
},
);
});
});
test("one-on-one mobile layout shows local tile when video is enabled", () => {
withTestScheduler(({ behavior, schedule, expectObservable }) => {
// Local participant enables their video, then disables it
@@ -1211,15 +1333,18 @@ describe.each(modes)("CallViewModel (%s mode)", (mode) => {
});
});
test("participants must have a MatrixRTCSession to be visible", () => {
test("participants without a MatrixRTC membership are shown as unknown, without media", () => {
withTestScheduler(({ behavior, expectObservable }) => {
// iterate through a number of combinations of participants and MatrixRTC memberships
// Bob never has an MatrixRTC membership
const participantInputMarbles = "abcd-c";
// Bob even tries to share his screen at the end
const bobSharingInputMarbles = " n---yn";
// Bob should never be visible
const expectedLayoutMarbles = " a-bc-b";
// Bob gets an unknown participant tile (MSC4143) rather than a tile of
// his own, and his screen share never shows up. His presence also keeps
// the call out of the one-on-one layout, which would hide him.
const expectedLayoutMarbles = " abcd-c";
const bobUnknownId = `unknown:${exampleTransport.livekit_service_url}:${bobId}`;
withCallViewModel(
{
@@ -1251,14 +1376,25 @@ describe.each(modes)("CallViewModel (%s mode)", (mode) => {
grid: [`${localId}:0`],
},
b: {
type: "one-on-one-desktop",
pip: `${localId}:0`,
spotlight: `${aliceId}:0`,
type: "grid",
spotlight: undefined,
grid: [`${localId}:0`, bobUnknownId],
},
// Tiles keep their positions, so Bob stays ahead of the others
c: {
type: "grid",
spotlight: undefined,
grid: [`${localId}:0`, `${aliceId}:0`, `${daveId}:0`],
grid: [`${localId}:0`, bobUnknownId, `${aliceId}:0`],
},
d: {
type: "grid",
spotlight: undefined,
grid: [
`${localId}:0`,
bobUnknownId,
`${aliceId}:0`,
`${daveId}:0`,
],
},
},
);
+94 -27
View File
@@ -95,6 +95,7 @@ import {
layoutShallowEquals,
type Alignment,
type GridLayoutMedia,
type GridMediaViewModel,
type Layout,
type LayoutMedia,
type OneOnOneDesktopLayoutMedia,
@@ -162,6 +163,10 @@ import {
createRingingMedia,
type RingingMediaViewModel,
} from "../media/RingingMediaViewModel.ts";
import {
createUnknownParticipantMedia,
type UnknownParticipantMediaViewModel,
} from "../media/UnknownParticipantMediaViewModel.ts";
import { type GridTileViewModel } from "../TileViewModel.ts";
//TODO
@@ -563,14 +568,8 @@ export function createCallViewModel$(
const ownMembershipIdentity: CallMembershipIdentityParts = {
userId,
deviceId,
// Consumed by the sticky membership manager as `member.id`, *and* stamped
// into every to-device key event by the key transport. A pre-sticky
// membership advertises `${userId}:${deviceId}` as its `membershipID`
// instead, so a uuid there names a member no peer can resolve.
memberId:
matrixRTCMode === MatrixRTCMode.Matrix_2_0
? uuidv4()
: `${userId}:${deviceId}`,
// This will only be consumed by the sticky membership manager. So it has no impact on legacy calls.
memberId: uuidv4(),
};
const localTransport =
@@ -875,6 +874,61 @@ export function createCallViewModel$(
),
);
/**
* Media for LiveKit participants that cannot be mapped to any MatrixRTC
* membership. MSC4143 asks us to surface these rather than ignore them, as
* they can signal an eavesdropping or impersonation attack.
*
* Participants are matched to memberships by identity alone rather than per
* transport, since a member legitimately shows up as a subscribe-only
* participant on every focus other than the one they publish to.
*/
const unknownParticipantMedia$ = scope.behavior<
UnknownParticipantMediaViewModel[]
>(
combineLatest([
memberships$,
connectionManager.connectionManagerData$,
]).pipe(
// Memberships and connections derive from the same epoch of session
// state; comparing them across epochs would flag members whose
// connection hasn't caught up yet.
filter(([memberships, data]) => memberships.epoch === data.epoch),
map(([memberships, data]) => {
const knownIdentities = new Set(
memberships.value.map((m) => m.rtcBackendIdentity),
);
return data.value.getConnections().flatMap((connection) =>
data.value
.getParticipantsForTransport(connection.transport)
.filter((p) => !knownIdentities.has(p.identity))
.map((p) => ({
focusUrl: connection.transport.livekit_service_url,
identity: p.identity,
})),
);
}),
generateItems(
"CallViewModel unknownParticipantMedia$",
function* (unknownParticipants) {
for (const { focusUrl, identity } of unknownParticipants)
yield { keys: [focusUrl, identity], data: undefined };
},
(_scope, _data$, focusUrl, identity) => {
logger.warn(
`LiveKit participant ${identity} on ${focusUrl} matches no MatrixRTC membership`,
);
return createUnknownParticipantMedia({
id: `unknown:${focusUrl}:${identity}`,
rtcBackendIdentity: identity,
focusUrl,
});
},
),
),
[],
);
const ringingMedia$ = scope.behavior<RingingMediaViewModel | null>(
ringAttempts$.pipe(
switchMap(({ intent, recipient, outcome$ }) =>
@@ -1021,21 +1075,29 @@ export function createCallViewModel$(
),
);
const grid$ = scope.behavior<UserMediaViewModel[]>(
userMedia$.pipe(
switchMap((mediaItems) => {
const bins = mediaItems.map((m) =>
m.bin$.pipe(map((bin) => [m, bin] as const)),
);
// Sort the media by bin order and generate a tile for each one
return bins.length === 0
? of([])
: combineLatest(bins, (...bins) =>
bins.sort(([, bin1], [, bin2]) => bin1 - bin2).map(([m]) => m),
);
}),
distinctUntilChanged(shallowArrayEquals),
),
const sortedUserMedia$: Observable<UserMediaViewModel[]> = userMedia$.pipe(
switchMap((mediaItems) => {
const bins = mediaItems.map((m) =>
m.bin$.pipe(map((bin) => [m, bin] as const)),
);
// Sort the media by bin order and generate a tile for each one
return bins.length === 0
? of([])
: combineLatest(bins, (...bins) =>
bins.sort(([, bin1], [, bin2]) => bin1 - bin2).map(([m]) => m),
);
}),
);
const grid$ = scope.behavior<GridMediaViewModel[]>(
combineLatest(
[sortedUserMedia$, unknownParticipantMedia$],
// Unknown participants have no media to sort by, so they go last
(userMedia, unknownParticipants) => [
...userMedia,
...unknownParticipants,
],
).pipe(distinctUntilChanged(shallowArrayEquals)),
);
/**
@@ -1195,10 +1257,15 @@ export function createCallViewModel$(
local: LocalUserMediaViewModel;
remote: UserMediaViewModel | RingingMediaViewModel;
} | null> = scope.behavior(
combineLatest([userMedia$, screenShares$]).pipe(
switchMap(([userMedia, screenShares]) => {
// One-on-one layout only supports 2 user media, no screen shares
if (userMedia.length <= 2 && screenShares.length === 0) {
combineLatest([userMedia$, screenShares$, unknownParticipantMedia$]).pipe(
switchMap(([userMedia, screenShares, unknownParticipants]) => {
// One-on-one layout only supports 2 user media, no screen shares, and
// must never hide an unknown participant
if (
userMedia.length <= 2 &&
screenShares.length === 0 &&
unknownParticipants.length === 0
) {
const local = userMedia.find(
(vm): vm is WrappedUserMediaViewModel & LocalUserMediaViewModel =>
vm.type === "user" && vm.local,
+7 -14
View File
@@ -257,21 +257,14 @@ export class AudioOutput implements MediaDevice<
map((availableRaw) => {
let available: Map<string, AudioOutputDeviceLabel> =
buildDeviceMap(availableRaw);
// Create a virtual default audio output for browsers that don't have one
// (Firefox, Safari). Its device ID must be the empty string because
// that's what setSinkId recognizes. It goes first so that it is the
// fallback when no output has been explicitly chosen (or the chosen
// one disappears), rather than pinning the first physical device
// with setSinkId: pinned sinks are not re-routed by the browser, and
// Firefox leaves the audio elements silent when a pinned sink goes
// away (e.g. a Bluetooth headset switching profile when its
// microphone is opened). We can't know which physical device the
// browser default resolves to, so the entry carries no name.
// Create a virtual default audio output for browsers that don't have one.
// Its device ID must be the empty string because that's what setSinkId
// recognizes.
if (available.size && !available.has("") && !available.has("default"))
available = new Map<string, AudioOutputDeviceLabel>([
["", { type: "default", name: null }],
...available,
]);
available.set("", {
type: "default",
name: availableRaw[0]?.label || null,
});
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const isSafari = !!(window as any).GestureEvent; // non standard api only found on Safari. https://developer.mozilla.org/en-US/docs/Web/API/GestureEvent#browser_compatibility
if (isSafari) {
+13 -12
View File
@@ -8,12 +8,15 @@ Please see LICENSE in the repository root for full details.
import { BehaviorSubject } from "rxjs";
import { logger } from "matrix-js-sdk/lib/logger";
import { GridTileViewModel, SpotlightTileViewModel } from "./TileViewModel";
import {
type GridTileMediaViewModel,
GridTileViewModel,
SpotlightTileViewModel,
} from "./TileViewModel";
import { fillGaps } from "../utils/iter";
import { debugTileLayout } from "../settings/settings";
import { type MediaViewModel } from "./media/MediaViewModel";
import { type UserMediaViewModel } from "./media/UserMediaViewModel";
import { type RingingMediaViewModel } from "./media/RingingMediaViewModel";
type SpotlightBackground = "solid" | "transparent";
@@ -68,10 +71,8 @@ class SpotlightTileData {
}
class GridTileData {
private readonly media$: BehaviorSubject<
UserMediaViewModel | RingingMediaViewModel
>;
public get media(): UserMediaViewModel | RingingMediaViewModel {
private readonly media$: BehaviorSubject<GridTileMediaViewModel>;
public get media(): GridTileMediaViewModel {
return this.media$.value;
}
public set media(value: UserMediaViewModel) {
@@ -80,7 +81,7 @@ class GridTileData {
public readonly vm: GridTileViewModel;
public constructor(media: UserMediaViewModel | RingingMediaViewModel) {
public constructor(media: GridTileMediaViewModel) {
this.media$ = new BehaviorSubject(media);
this.vm = new GridTileViewModel(this.media$);
}
@@ -140,7 +141,7 @@ export class TileStoreBuilder {
: null;
private readonly prevGridByMedia: Map<
MediaViewModel,
GridTileMediaViewModel,
[GridTileData, number]
> = new Map(
this.prevGrid.map((entry, i) => [entry.media, [entry, i]] as const),
@@ -205,9 +206,7 @@ export class TileStoreBuilder {
* Sets up a grid tile for the given media. If this is never called for some
* media, then that media will have no grid tile.
*/
public registerGridTile(
media: UserMediaViewModel | RingingMediaViewModel,
): void {
public registerGridTile(media: GridTileMediaViewModel): void {
if (DEBUG_ENABLED)
logger.debug(
`[TileStore, ${this.generation}] register grid tile: ${media.displayName$.value}`,
@@ -215,8 +214,10 @@ export class TileStoreBuilder {
if (this.spotlight !== null) {
// We actually *don't* want spotlight speakers to appear in both the
// spotlight and the grid, so they're filtered out here
// spotlight and the grid, so they're filtered out here. (Unknown
// participants never make it into the spotlight.)
if (
media.type !== "unknown participant" &&
!(media.type === "user" && media.local) &&
this.spotlight.media.includes(media)
)
+7 -3
View File
@@ -10,8 +10,14 @@ import { BehaviorSubject } from "rxjs";
import { type Behavior } from "./Behavior";
import { type MediaViewModel } from "./media/MediaViewModel";
import { type RingingMediaViewModel } from "./media/RingingMediaViewModel";
import { type UnknownParticipantMediaViewModel } from "./media/UnknownParticipantMediaViewModel";
import { type UserMediaViewModel } from "./media/UserMediaViewModel";
export type GridTileMediaViewModel =
| UserMediaViewModel
| RingingMediaViewModel
| UnknownParticipantMediaViewModel;
let nextId = 0;
function createId(): string {
return (nextId++).toString();
@@ -23,9 +29,7 @@ export class GridTileViewModel {
public readonly showOutline$: Behavior<boolean> = this._showOutline$;
public constructor(
public readonly media$: Behavior<
UserMediaViewModel | RingingMediaViewModel
>,
public readonly media$: Behavior<GridTileMediaViewModel>,
) {}
public setShowOutline(value: boolean): void {
+8 -3
View File
@@ -10,6 +10,7 @@ import { type BehaviorSubject } from "rxjs";
import { type LocalUserMediaViewModel } from "./media/LocalUserMediaViewModel.ts";
import { type MediaViewModel } from "./media/MediaViewModel.ts";
import { type RingingMediaViewModel } from "./media/RingingMediaViewModel.ts";
import { type UnknownParticipantMediaViewModel } from "./media/UnknownParticipantMediaViewModel.ts";
import { type UserMediaViewModel } from "./media/UserMediaViewModel.ts";
import {
type GridTileViewModel,
@@ -18,25 +19,29 @@ import {
import { type Behavior } from "./Behavior.ts";
import { shallowEquals as arrayShallowEquals } from "../utils/array.ts";
export type GridMediaViewModel =
| UserMediaViewModel
| UnknownParticipantMediaViewModel;
export interface GridLayoutMedia {
type: "grid";
edgeToEdge: false;
spotlight?: MediaViewModel[];
grid: UserMediaViewModel[];
grid: GridMediaViewModel[];
}
export interface SpotlightLandscapeLayoutMedia {
type: "spotlight-landscape";
edgeToEdge: boolean;
spotlight: MediaViewModel[];
grid: UserMediaViewModel[];
grid: GridMediaViewModel[];
}
export interface SpotlightPortraitLayoutMedia {
type: "spotlight-portrait";
edgeToEdge: false;
spotlight: MediaViewModel[];
grid: UserMediaViewModel[];
grid: GridMediaViewModel[];
}
export interface SpotlightExpandedLayoutMedia {
@@ -0,0 +1,57 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { constant } from "../Behavior";
import { type BaseMediaViewModel } from "./MediaViewModel";
/**
* Media representing a LiveKit participant that cannot be mapped to any
* MatrixRTC member of the session.
*
* MSC4143 asks clients to surface such streams rather than silently ignore
* them, because they can signal an eavesdropping or impersonation attack. We
* therefore give them a tile, but never render their audio or video.
*
* There is no Matrix user behind this media, so `userId` and `displayName$`
* carry the LiveKit identity instead, purely so that debugging output stays
* meaningful.
*/
export interface UnknownParticipantMediaViewModel extends BaseMediaViewModel {
type: "unknown participant";
/**
* The LiveKit identity under which the participant connected. Exposed for
* debugging.
*/
rtcBackendIdentity: string;
/**
* The URL of the LiveKit focus on which the participant was seen. Exposed for
* debugging.
*/
focusUrl: string;
}
export interface UnknownParticipantMediaInputs {
id: string;
rtcBackendIdentity: string;
focusUrl: string;
}
export function createUnknownParticipantMedia({
id,
rtcBackendIdentity,
focusUrl,
}: UnknownParticipantMediaInputs): UnknownParticipantMediaViewModel {
return {
type: "unknown participant",
id,
userId: rtcBackendIdentity,
displayName$: constant(rtcBackendIdentity),
mxcAvatarUrl$: constant(undefined),
rtcBackendIdentity,
focusUrl,
};
}
+98
View File
@@ -0,0 +1,98 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { expect, within } from "storybook/test";
import { type JSX, useMemo } from "react";
import type { Meta, StoryObj } from "@storybook/react-vite";
import { GridTile } from "./GridTile";
import { GridTileViewModel } from "../state/TileViewModel";
import { constant } from "../state/Behavior";
import { createUnknownParticipantMedia } from "../state/media/UnknownParticipantMediaViewModel";
interface UnknownParticipantTileStoryProps {
width: number;
height: number;
rtcBackendIdentity: string;
showNameTags: boolean;
showOutline: boolean;
}
/**
* Renders a GridTile for a LiveKit participant that maps to no MatrixRTC
* member, driven by primitive props so that Storybook can document them.
*/
function UnknownParticipantTile({
width,
height,
rtcBackendIdentity,
...props
}: UnknownParticipantTileStoryProps): JSX.Element {
const vm = useMemo(
() =>
new GridTileViewModel(
constant(
createUnknownParticipantMedia({
id: `unknown:https://rtc.example.org:${rtcBackendIdentity}`,
rtcBackendIdentity,
focusUrl: "https://rtc.example.org",
}),
),
),
[rtcBackendIdentity],
);
return (
<GridTile
vm={vm}
onOpenProfile={null}
targetWidth={width}
targetHeight={height}
style={{ width, height }}
showSpeakingIndicators={false}
showRingingStatus={false}
focusable
{...props}
/>
);
}
const meta = {
component: UnknownParticipantTile,
argTypes: {
width: { control: { type: "range", min: 80, max: 800, step: 10 } },
height: { control: { type: "range", min: 80, max: 600, step: 10 } },
},
} satisfies Meta<typeof UnknownParticipantTile>;
export default meta;
type Story = StoryObj<typeof meta>;
export const UnknownParticipant: Story = {
args: {
width: 400,
height: 300,
rtcBackendIdentity: "@rogue:example.org:DEVICE",
showNameTags: true,
showOutline: false,
},
play: async ({ canvasElement }) => {
const canvas = within(canvasElement);
await expect(canvas.getByText("Unknown participant")).toBeInTheDocument();
await expect(
canvas.queryByText("@rogue:example.org:DEVICE"),
).not.toBeInTheDocument();
},
};
/** Narrow tiles hide the name tag, so the label goes with it. */
export const UnknownParticipantNarrow: Story = {
args: {
...UnknownParticipant.args,
width: 90,
height: 90,
},
};
+31
View File
@@ -32,6 +32,7 @@ import {
createRingingMedia,
type RingingMediaViewModel,
} from "../state/media/RingingMediaViewModel";
import { createUnknownParticipantMedia } from "../state/media/UnknownParticipantMediaViewModel";
global.IntersectionObserver = class MockIntersectionObserver {
public observe(): void {}
@@ -165,3 +166,33 @@ test("GridTile displays ringing media", async () => {
act(() => pickupState$.next("decline"));
screen.getByText("Call ended");
});
test("GridTile displays an unknown participant", async () => {
const vm = createUnknownParticipantMedia({
id: "unknown:https://rtc-example.org:rogue",
rtcBackendIdentity: "rogue",
focusUrl: "https://rtc-example.org",
});
const { container } = render(
<ReactionsSenderProvider vm={callVm} rtcSession={fakeRtcSession}>
<GridTile
vm={new GridTileViewModel(constant(vm))}
onOpenProfile={() => {}}
targetWidth={300}
targetHeight={200}
showSpeakingIndicators
showNameTags
showRingingStatus
showOutline
focusable
/>
</ReactionsSenderProvider>,
);
expect(await axe(container)).toHaveNoViolations();
// The tile is labelled as unknown rather than after its LiveKit identity
screen.getByText("Unknown participant");
expect(screen.queryByText("rogue")).toBeNull();
// There is no user to show an avatar for
expect(container.querySelector("[data-style]")).toBeNull();
});
+48
View File
@@ -51,6 +51,7 @@ import { type LocalUserMediaViewModel } from "../state/media/LocalUserMediaViewM
import { type RemoteUserMediaViewModel } from "../state/media/RemoteUserMediaViewModel";
import { type UserMediaViewModel } from "../state/media/UserMediaViewModel";
import { type RingingMediaViewModel } from "../state/media/RingingMediaViewModel";
import { type UnknownParticipantMediaViewModel } from "../state/media/UnknownParticipantMediaViewModel";
import { RingingStatus } from "./RingingStatus";
interface TileProps {
@@ -97,6 +98,44 @@ const RingingMediaTile: FC<RingingMediaTileProps> = ({
);
};
interface UnknownParticipantTileProps extends Omit<
TileProps,
"displayName" | "mxcAvatarUrl"
> {
vm: UnknownParticipantMediaViewModel;
}
/**
* A blank tile standing in for a LiveKit participant that cannot be mapped to
* any MatrixRTC member. MSC4143 asks clients to surface such streams rather
* than ignore them. No media is rendered for them.
*/
const UnknownParticipantTile: FC<UnknownParticipantTileProps> = ({
vm,
className,
...props
}) => {
const { t } = useTranslation();
return (
<MediaView
className={classNames(className, styles.tile)}
video={undefined}
userId={vm.userId}
unencryptedWarning={false}
videoEnabled={false}
mirror={false}
showAvatar={false}
displayName={t("video_tile.unknown_participant")}
mxcAvatarUrl={undefined}
rtcBackendIdentity={vm.rtcBackendIdentity}
focusUrl={vm.focusUrl}
{...props}
/>
);
};
UnknownParticipantTile.displayName = "UnknownParticipantTile";
interface UserMediaTileProps extends TileProps {
vm: UserMediaViewModel;
showSpeakingIndicators: boolean;
@@ -454,6 +493,15 @@ export const GridTile: FC<GridTileProps> = ({
{...props}
/>
);
} else if (media.type === "unknown participant") {
return (
<UnknownParticipantTile
ref={ref}
vm={media}
className={classNames(className, { [styles.outline]: showOutline })}
{...props}
/>
);
} else if (media.local) {
return (
<LocalUserMediaTile
+17 -9
View File
@@ -56,6 +56,11 @@ interface Props extends ComponentProps<typeof animated.div> {
displayName: string;
mxcAvatarUrl: string | undefined;
avatarStyle?: "solid" | "translucent";
/**
* Whether to show an avatar when there is no video. Off for media that has no
* Matrix user behind it.
*/
showAvatar?: boolean;
background?: "solid" | "transparent";
focusable: boolean;
primaryButton?: ReactNode;
@@ -95,6 +100,7 @@ export const MediaView: FC<Props> = ({
displayName,
mxcAvatarUrl,
avatarStyle = "solid",
showAvatar = true,
background = "solid",
focusable,
primaryButton,
@@ -176,15 +182,17 @@ export const MediaView: FC<Props> = ({
<div className={styles.speakingBorder} />
</div>
)}
<Avatar
id={userId}
name={displayName}
size={avatarSize}
src={mxcAvatarUrl}
data-style={avatarStyle}
className={styles.avatar}
style={{ display: video && videoEnabled ? "none" : "initial" }}
/>
{showAvatar && (
<Avatar
id={userId}
name={displayName}
size={avatarSize}
src={mxcAvatarUrl}
data-style={avatarStyle}
className={styles.avatar}
style={{ display: video && videoEnabled ? "none" : "initial" }}
/>
)}
{video?.publication !== undefined && (
<VideoTrack
trackRef={video}