mirror of
https://github.com/vector-im/element-call.git
synced 2026-09-22 22:29:30 +00:00
Render tiles for unmappable LiveKit identities
This commit is contained in:
@@ -273,6 +273,7 @@
|
|||||||
"mute_for_me": "Mute for me",
|
"mute_for_me": "Mute for me",
|
||||||
"muted_for_me": "Muted for me",
|
"muted_for_me": "Muted for me",
|
||||||
"screen_share_volume": "Screen share volume",
|
"screen_share_volume": "Screen share volume",
|
||||||
|
"unknown_participant": "Unknown participant",
|
||||||
"volume": "Volume",
|
"volume": "Volume",
|
||||||
"waiting_for_media": "Waiting for media..."
|
"waiting_for_media": "Waiting for media..."
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -52,8 +52,10 @@ import {
|
|||||||
aliceRtcMember,
|
aliceRtcMember,
|
||||||
aliceUserId,
|
aliceUserId,
|
||||||
bob,
|
bob,
|
||||||
|
bobDeviceId,
|
||||||
bobId,
|
bobId,
|
||||||
bobRtcMember,
|
bobRtcMember,
|
||||||
|
bobUserId,
|
||||||
local,
|
local,
|
||||||
localId,
|
localId,
|
||||||
localRtcMember,
|
localRtcMember,
|
||||||
@@ -118,6 +120,19 @@ const daveId = `${dave.userId}:${daveRtcMember.deviceId}`;
|
|||||||
const bobParticipant = mockRemoteParticipant({ identity: bobId });
|
const bobParticipant = mockRemoteParticipant({ identity: bobId });
|
||||||
const daveParticipant = mockRemoteParticipant({ identity: daveId });
|
const daveParticipant = mockRemoteParticipant({ identity: daveId });
|
||||||
|
|
||||||
|
// A LiveKit participant that no MatrixRTC membership accounts for
|
||||||
|
const rogueParticipant = mockRemoteParticipant({ identity: "rogue" });
|
||||||
|
const rogueId = `unknown:${exampleTransport.livekit_service_url}:rogue`;
|
||||||
|
|
||||||
|
const otherTransport: LivekitTransport = {
|
||||||
|
type: "livekit",
|
||||||
|
livekit_service_url: "https://lk.other.example.org",
|
||||||
|
livekit_alias: "!alias:other.example.org",
|
||||||
|
};
|
||||||
|
const bobOnOtherFocusRtcMember = mockRtcMembership(bobUserId, bobDeviceId, {
|
||||||
|
fociPreferred: [otherTransport],
|
||||||
|
});
|
||||||
|
|
||||||
export interface GridLayoutSummary {
|
export interface GridLayoutSummary {
|
||||||
type: "grid";
|
type: "grid";
|
||||||
spotlight?: string[];
|
spotlight?: string[];
|
||||||
@@ -450,6 +465,113 @@ describe.each(modes)("CallViewModel (%s mode)", (mode) => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("LiveKit participants without a membership get an unknown participant tile", () => {
|
||||||
|
withTestScheduler(({ behavior, expectObservable }) => {
|
||||||
|
// A participant nobody's membership accounts for connects on frame 1 and
|
||||||
|
// disconnects on frame 2
|
||||||
|
const participantInputMarbles = "aba";
|
||||||
|
// It gets a tile at the end of the grid for as long as it is connected
|
||||||
|
const expectedLayoutMarbles = " aba";
|
||||||
|
|
||||||
|
withCallViewModel(
|
||||||
|
{
|
||||||
|
remoteParticipants$: behavior(participantInputMarbles, {
|
||||||
|
a: [aliceParticipant, bobParticipant],
|
||||||
|
b: [aliceParticipant, bobParticipant, rogueParticipant],
|
||||||
|
}),
|
||||||
|
rtcMembers$: constant([localRtcMember, aliceRtcMember, bobRtcMember]),
|
||||||
|
},
|
||||||
|
(vm) => {
|
||||||
|
expectObservable(summarizeLayout$(vm.layout$)).toBe(
|
||||||
|
expectedLayoutMarbles,
|
||||||
|
{
|
||||||
|
a: {
|
||||||
|
type: "grid",
|
||||||
|
spotlight: undefined,
|
||||||
|
grid: [`${localId}:0`, `${aliceId}:0`, `${bobId}:0`],
|
||||||
|
},
|
||||||
|
b: {
|
||||||
|
type: "grid",
|
||||||
|
spotlight: undefined,
|
||||||
|
grid: [`${localId}:0`, `${aliceId}:0`, `${bobId}:0`, rogueId],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
);
|
||||||
|
},
|
||||||
|
);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test("an unknown participant takes the call out of one-on-one layout", () => {
|
||||||
|
withTestScheduler(({ behavior, expectObservable }) => {
|
||||||
|
// A participant nobody's membership accounts for connects on frame 1 and
|
||||||
|
// disconnects on frame 2
|
||||||
|
const participantInputMarbles = "aba";
|
||||||
|
// The one-on-one layout would hide the unknown participant, so we must
|
||||||
|
// fall back to the grid while they are present
|
||||||
|
const expectedLayoutMarbles = " aba";
|
||||||
|
|
||||||
|
withCallViewModel(
|
||||||
|
{
|
||||||
|
remoteParticipants$: behavior(participantInputMarbles, {
|
||||||
|
a: [aliceParticipant],
|
||||||
|
b: [aliceParticipant, rogueParticipant],
|
||||||
|
}),
|
||||||
|
rtcMembers$: constant([localRtcMember, aliceRtcMember]),
|
||||||
|
},
|
||||||
|
(vm) => {
|
||||||
|
expectObservable(summarizeLayout$(vm.layout$)).toBe(
|
||||||
|
expectedLayoutMarbles,
|
||||||
|
{
|
||||||
|
a: {
|
||||||
|
type: "one-on-one-desktop",
|
||||||
|
pip: `${localId}:0`,
|
||||||
|
spotlight: `${aliceId}:0`,
|
||||||
|
},
|
||||||
|
b: {
|
||||||
|
type: "grid",
|
||||||
|
spotlight: undefined,
|
||||||
|
grid: [`${localId}:0`, `${aliceId}:0`, rogueId],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
);
|
||||||
|
},
|
||||||
|
);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a member seen on a focus other than their own is not an unknown participant", () => {
|
||||||
|
withTestScheduler(({ expectObservable }) => {
|
||||||
|
// Bob publishes to another focus, but also connects to ours to subscribe
|
||||||
|
// (the mock reports the same participants on every connection). Matching
|
||||||
|
// by identity rather than per transport keeps him from being flagged.
|
||||||
|
const expectedLayoutMarbles = "a";
|
||||||
|
|
||||||
|
withCallViewModel(
|
||||||
|
{
|
||||||
|
remoteParticipants$: constant([aliceParticipant, bobParticipant]),
|
||||||
|
rtcMembers$: constant([
|
||||||
|
localRtcMember,
|
||||||
|
aliceRtcMember,
|
||||||
|
bobOnOtherFocusRtcMember,
|
||||||
|
]),
|
||||||
|
},
|
||||||
|
(vm) => {
|
||||||
|
expectObservable(summarizeLayout$(vm.layout$)).toBe(
|
||||||
|
expectedLayoutMarbles,
|
||||||
|
{
|
||||||
|
a: {
|
||||||
|
type: "grid",
|
||||||
|
spotlight: undefined,
|
||||||
|
grid: [`${localId}:0`, `${aliceId}:0`, `${bobId}:0`],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
);
|
||||||
|
},
|
||||||
|
);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
test("one-on-one mobile layout shows local tile when video is enabled", () => {
|
test("one-on-one mobile layout shows local tile when video is enabled", () => {
|
||||||
withTestScheduler(({ behavior, schedule, expectObservable }) => {
|
withTestScheduler(({ behavior, schedule, expectObservable }) => {
|
||||||
// Local participant enables their video, then disables it
|
// Local participant enables their video, then disables it
|
||||||
@@ -1211,15 +1333,18 @@ describe.each(modes)("CallViewModel (%s mode)", (mode) => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
test("participants must have a MatrixRTCSession to be visible", () => {
|
test("participants without a MatrixRTC membership are shown as unknown, without media", () => {
|
||||||
withTestScheduler(({ behavior, expectObservable }) => {
|
withTestScheduler(({ behavior, expectObservable }) => {
|
||||||
// iterate through a number of combinations of participants and MatrixRTC memberships
|
// iterate through a number of combinations of participants and MatrixRTC memberships
|
||||||
// Bob never has an MatrixRTC membership
|
// Bob never has an MatrixRTC membership
|
||||||
const participantInputMarbles = "abcd-c";
|
const participantInputMarbles = "abcd-c";
|
||||||
// Bob even tries to share his screen at the end
|
// Bob even tries to share his screen at the end
|
||||||
const bobSharingInputMarbles = " n---yn";
|
const bobSharingInputMarbles = " n---yn";
|
||||||
// Bob should never be visible
|
// Bob gets an unknown participant tile (MSC4143) rather than a tile of
|
||||||
const expectedLayoutMarbles = " a-bc-b";
|
// his own, and his screen share never shows up. His presence also keeps
|
||||||
|
// the call out of the one-on-one layout, which would hide him.
|
||||||
|
const expectedLayoutMarbles = " abcd-c";
|
||||||
|
const bobUnknownId = `unknown:${exampleTransport.livekit_service_url}:${bobId}`;
|
||||||
|
|
||||||
withCallViewModel(
|
withCallViewModel(
|
||||||
{
|
{
|
||||||
@@ -1251,14 +1376,25 @@ describe.each(modes)("CallViewModel (%s mode)", (mode) => {
|
|||||||
grid: [`${localId}:0`],
|
grid: [`${localId}:0`],
|
||||||
},
|
},
|
||||||
b: {
|
b: {
|
||||||
type: "one-on-one-desktop",
|
type: "grid",
|
||||||
pip: `${localId}:0`,
|
spotlight: undefined,
|
||||||
spotlight: `${aliceId}:0`,
|
grid: [`${localId}:0`, bobUnknownId],
|
||||||
},
|
},
|
||||||
|
// Tiles keep their positions, so Bob stays ahead of the others
|
||||||
c: {
|
c: {
|
||||||
type: "grid",
|
type: "grid",
|
||||||
spotlight: undefined,
|
spotlight: undefined,
|
||||||
grid: [`${localId}:0`, `${aliceId}:0`, `${daveId}:0`],
|
grid: [`${localId}:0`, bobUnknownId, `${aliceId}:0`],
|
||||||
|
},
|
||||||
|
d: {
|
||||||
|
type: "grid",
|
||||||
|
spotlight: undefined,
|
||||||
|
grid: [
|
||||||
|
`${localId}:0`,
|
||||||
|
bobUnknownId,
|
||||||
|
`${aliceId}:0`,
|
||||||
|
`${daveId}:0`,
|
||||||
|
],
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -95,6 +95,7 @@ import {
|
|||||||
layoutShallowEquals,
|
layoutShallowEquals,
|
||||||
type Alignment,
|
type Alignment,
|
||||||
type GridLayoutMedia,
|
type GridLayoutMedia,
|
||||||
|
type GridMediaViewModel,
|
||||||
type Layout,
|
type Layout,
|
||||||
type LayoutMedia,
|
type LayoutMedia,
|
||||||
type OneOnOneDesktopLayoutMedia,
|
type OneOnOneDesktopLayoutMedia,
|
||||||
@@ -162,6 +163,10 @@ import {
|
|||||||
createRingingMedia,
|
createRingingMedia,
|
||||||
type RingingMediaViewModel,
|
type RingingMediaViewModel,
|
||||||
} from "../media/RingingMediaViewModel.ts";
|
} from "../media/RingingMediaViewModel.ts";
|
||||||
|
import {
|
||||||
|
createUnknownParticipantMedia,
|
||||||
|
type UnknownParticipantMediaViewModel,
|
||||||
|
} from "../media/UnknownParticipantMediaViewModel.ts";
|
||||||
import { type GridTileViewModel } from "../TileViewModel.ts";
|
import { type GridTileViewModel } from "../TileViewModel.ts";
|
||||||
|
|
||||||
//TODO
|
//TODO
|
||||||
@@ -869,6 +874,61 @@ export function createCallViewModel$(
|
|||||||
),
|
),
|
||||||
);
|
);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Media for LiveKit participants that cannot be mapped to any MatrixRTC
|
||||||
|
* membership. MSC4143 asks us to surface these rather than ignore them, as
|
||||||
|
* they can signal an eavesdropping or impersonation attack.
|
||||||
|
*
|
||||||
|
* Participants are matched to memberships by identity alone rather than per
|
||||||
|
* transport, since a member legitimately shows up as a subscribe-only
|
||||||
|
* participant on every focus other than the one they publish to.
|
||||||
|
*/
|
||||||
|
const unknownParticipantMedia$ = scope.behavior<
|
||||||
|
UnknownParticipantMediaViewModel[]
|
||||||
|
>(
|
||||||
|
combineLatest([
|
||||||
|
memberships$,
|
||||||
|
connectionManager.connectionManagerData$,
|
||||||
|
]).pipe(
|
||||||
|
// Memberships and connections derive from the same epoch of session
|
||||||
|
// state; comparing them across epochs would flag members whose
|
||||||
|
// connection hasn't caught up yet.
|
||||||
|
filter(([memberships, data]) => memberships.epoch === data.epoch),
|
||||||
|
map(([memberships, data]) => {
|
||||||
|
const knownIdentities = new Set(
|
||||||
|
memberships.value.map((m) => m.rtcBackendIdentity),
|
||||||
|
);
|
||||||
|
return data.value.getConnections().flatMap((connection) =>
|
||||||
|
data.value
|
||||||
|
.getParticipantsForTransport(connection.transport)
|
||||||
|
.filter((p) => !knownIdentities.has(p.identity))
|
||||||
|
.map((p) => ({
|
||||||
|
focusUrl: connection.transport.livekit_service_url,
|
||||||
|
identity: p.identity,
|
||||||
|
})),
|
||||||
|
);
|
||||||
|
}),
|
||||||
|
generateItems(
|
||||||
|
"CallViewModel unknownParticipantMedia$",
|
||||||
|
function* (unknownParticipants) {
|
||||||
|
for (const { focusUrl, identity } of unknownParticipants)
|
||||||
|
yield { keys: [focusUrl, identity], data: undefined };
|
||||||
|
},
|
||||||
|
(_scope, _data$, focusUrl, identity) => {
|
||||||
|
logger.warn(
|
||||||
|
`LiveKit participant ${identity} on ${focusUrl} matches no MatrixRTC membership`,
|
||||||
|
);
|
||||||
|
return createUnknownParticipantMedia({
|
||||||
|
id: `unknown:${focusUrl}:${identity}`,
|
||||||
|
rtcBackendIdentity: identity,
|
||||||
|
focusUrl,
|
||||||
|
});
|
||||||
|
},
|
||||||
|
),
|
||||||
|
),
|
||||||
|
[],
|
||||||
|
);
|
||||||
|
|
||||||
const ringingMedia$ = scope.behavior<RingingMediaViewModel | null>(
|
const ringingMedia$ = scope.behavior<RingingMediaViewModel | null>(
|
||||||
ringAttempts$.pipe(
|
ringAttempts$.pipe(
|
||||||
switchMap(({ intent, recipient, outcome$ }) =>
|
switchMap(({ intent, recipient, outcome$ }) =>
|
||||||
@@ -1015,21 +1075,29 @@ export function createCallViewModel$(
|
|||||||
),
|
),
|
||||||
);
|
);
|
||||||
|
|
||||||
const grid$ = scope.behavior<UserMediaViewModel[]>(
|
const sortedUserMedia$: Observable<UserMediaViewModel[]> = userMedia$.pipe(
|
||||||
userMedia$.pipe(
|
switchMap((mediaItems) => {
|
||||||
switchMap((mediaItems) => {
|
const bins = mediaItems.map((m) =>
|
||||||
const bins = mediaItems.map((m) =>
|
m.bin$.pipe(map((bin) => [m, bin] as const)),
|
||||||
m.bin$.pipe(map((bin) => [m, bin] as const)),
|
);
|
||||||
);
|
// Sort the media by bin order and generate a tile for each one
|
||||||
// Sort the media by bin order and generate a tile for each one
|
return bins.length === 0
|
||||||
return bins.length === 0
|
? of([])
|
||||||
? of([])
|
: combineLatest(bins, (...bins) =>
|
||||||
: combineLatest(bins, (...bins) =>
|
bins.sort(([, bin1], [, bin2]) => bin1 - bin2).map(([m]) => m),
|
||||||
bins.sort(([, bin1], [, bin2]) => bin1 - bin2).map(([m]) => m),
|
);
|
||||||
);
|
}),
|
||||||
}),
|
);
|
||||||
distinctUntilChanged(shallowArrayEquals),
|
|
||||||
),
|
const grid$ = scope.behavior<GridMediaViewModel[]>(
|
||||||
|
combineLatest(
|
||||||
|
[sortedUserMedia$, unknownParticipantMedia$],
|
||||||
|
// Unknown participants have no media to sort by, so they go last
|
||||||
|
(userMedia, unknownParticipants) => [
|
||||||
|
...userMedia,
|
||||||
|
...unknownParticipants,
|
||||||
|
],
|
||||||
|
).pipe(distinctUntilChanged(shallowArrayEquals)),
|
||||||
);
|
);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -1189,10 +1257,15 @@ export function createCallViewModel$(
|
|||||||
local: LocalUserMediaViewModel;
|
local: LocalUserMediaViewModel;
|
||||||
remote: UserMediaViewModel | RingingMediaViewModel;
|
remote: UserMediaViewModel | RingingMediaViewModel;
|
||||||
} | null> = scope.behavior(
|
} | null> = scope.behavior(
|
||||||
combineLatest([userMedia$, screenShares$]).pipe(
|
combineLatest([userMedia$, screenShares$, unknownParticipantMedia$]).pipe(
|
||||||
switchMap(([userMedia, screenShares]) => {
|
switchMap(([userMedia, screenShares, unknownParticipants]) => {
|
||||||
// One-on-one layout only supports 2 user media, no screen shares
|
// One-on-one layout only supports 2 user media, no screen shares, and
|
||||||
if (userMedia.length <= 2 && screenShares.length === 0) {
|
// must never hide an unknown participant
|
||||||
|
if (
|
||||||
|
userMedia.length <= 2 &&
|
||||||
|
screenShares.length === 0 &&
|
||||||
|
unknownParticipants.length === 0
|
||||||
|
) {
|
||||||
const local = userMedia.find(
|
const local = userMedia.find(
|
||||||
(vm): vm is WrappedUserMediaViewModel & LocalUserMediaViewModel =>
|
(vm): vm is WrappedUserMediaViewModel & LocalUserMediaViewModel =>
|
||||||
vm.type === "user" && vm.local,
|
vm.type === "user" && vm.local,
|
||||||
|
|||||||
+13
-12
@@ -8,12 +8,15 @@ Please see LICENSE in the repository root for full details.
|
|||||||
import { BehaviorSubject } from "rxjs";
|
import { BehaviorSubject } from "rxjs";
|
||||||
import { logger } from "matrix-js-sdk/lib/logger";
|
import { logger } from "matrix-js-sdk/lib/logger";
|
||||||
|
|
||||||
import { GridTileViewModel, SpotlightTileViewModel } from "./TileViewModel";
|
import {
|
||||||
|
type GridTileMediaViewModel,
|
||||||
|
GridTileViewModel,
|
||||||
|
SpotlightTileViewModel,
|
||||||
|
} from "./TileViewModel";
|
||||||
import { fillGaps } from "../utils/iter";
|
import { fillGaps } from "../utils/iter";
|
||||||
import { debugTileLayout } from "../settings/settings";
|
import { debugTileLayout } from "../settings/settings";
|
||||||
import { type MediaViewModel } from "./media/MediaViewModel";
|
import { type MediaViewModel } from "./media/MediaViewModel";
|
||||||
import { type UserMediaViewModel } from "./media/UserMediaViewModel";
|
import { type UserMediaViewModel } from "./media/UserMediaViewModel";
|
||||||
import { type RingingMediaViewModel } from "./media/RingingMediaViewModel";
|
|
||||||
|
|
||||||
type SpotlightBackground = "solid" | "transparent";
|
type SpotlightBackground = "solid" | "transparent";
|
||||||
|
|
||||||
@@ -68,10 +71,8 @@ class SpotlightTileData {
|
|||||||
}
|
}
|
||||||
|
|
||||||
class GridTileData {
|
class GridTileData {
|
||||||
private readonly media$: BehaviorSubject<
|
private readonly media$: BehaviorSubject<GridTileMediaViewModel>;
|
||||||
UserMediaViewModel | RingingMediaViewModel
|
public get media(): GridTileMediaViewModel {
|
||||||
>;
|
|
||||||
public get media(): UserMediaViewModel | RingingMediaViewModel {
|
|
||||||
return this.media$.value;
|
return this.media$.value;
|
||||||
}
|
}
|
||||||
public set media(value: UserMediaViewModel) {
|
public set media(value: UserMediaViewModel) {
|
||||||
@@ -80,7 +81,7 @@ class GridTileData {
|
|||||||
|
|
||||||
public readonly vm: GridTileViewModel;
|
public readonly vm: GridTileViewModel;
|
||||||
|
|
||||||
public constructor(media: UserMediaViewModel | RingingMediaViewModel) {
|
public constructor(media: GridTileMediaViewModel) {
|
||||||
this.media$ = new BehaviorSubject(media);
|
this.media$ = new BehaviorSubject(media);
|
||||||
this.vm = new GridTileViewModel(this.media$);
|
this.vm = new GridTileViewModel(this.media$);
|
||||||
}
|
}
|
||||||
@@ -140,7 +141,7 @@ export class TileStoreBuilder {
|
|||||||
: null;
|
: null;
|
||||||
|
|
||||||
private readonly prevGridByMedia: Map<
|
private readonly prevGridByMedia: Map<
|
||||||
MediaViewModel,
|
GridTileMediaViewModel,
|
||||||
[GridTileData, number]
|
[GridTileData, number]
|
||||||
> = new Map(
|
> = new Map(
|
||||||
this.prevGrid.map((entry, i) => [entry.media, [entry, i]] as const),
|
this.prevGrid.map((entry, i) => [entry.media, [entry, i]] as const),
|
||||||
@@ -205,9 +206,7 @@ export class TileStoreBuilder {
|
|||||||
* Sets up a grid tile for the given media. If this is never called for some
|
* Sets up a grid tile for the given media. If this is never called for some
|
||||||
* media, then that media will have no grid tile.
|
* media, then that media will have no grid tile.
|
||||||
*/
|
*/
|
||||||
public registerGridTile(
|
public registerGridTile(media: GridTileMediaViewModel): void {
|
||||||
media: UserMediaViewModel | RingingMediaViewModel,
|
|
||||||
): void {
|
|
||||||
if (DEBUG_ENABLED)
|
if (DEBUG_ENABLED)
|
||||||
logger.debug(
|
logger.debug(
|
||||||
`[TileStore, ${this.generation}] register grid tile: ${media.displayName$.value}`,
|
`[TileStore, ${this.generation}] register grid tile: ${media.displayName$.value}`,
|
||||||
@@ -215,8 +214,10 @@ export class TileStoreBuilder {
|
|||||||
|
|
||||||
if (this.spotlight !== null) {
|
if (this.spotlight !== null) {
|
||||||
// We actually *don't* want spotlight speakers to appear in both the
|
// We actually *don't* want spotlight speakers to appear in both the
|
||||||
// spotlight and the grid, so they're filtered out here
|
// spotlight and the grid, so they're filtered out here. (Unknown
|
||||||
|
// participants never make it into the spotlight.)
|
||||||
if (
|
if (
|
||||||
|
media.type !== "unknown participant" &&
|
||||||
!(media.type === "user" && media.local) &&
|
!(media.type === "user" && media.local) &&
|
||||||
this.spotlight.media.includes(media)
|
this.spotlight.media.includes(media)
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -10,8 +10,14 @@ import { BehaviorSubject } from "rxjs";
|
|||||||
import { type Behavior } from "./Behavior";
|
import { type Behavior } from "./Behavior";
|
||||||
import { type MediaViewModel } from "./media/MediaViewModel";
|
import { type MediaViewModel } from "./media/MediaViewModel";
|
||||||
import { type RingingMediaViewModel } from "./media/RingingMediaViewModel";
|
import { type RingingMediaViewModel } from "./media/RingingMediaViewModel";
|
||||||
|
import { type UnknownParticipantMediaViewModel } from "./media/UnknownParticipantMediaViewModel";
|
||||||
import { type UserMediaViewModel } from "./media/UserMediaViewModel";
|
import { type UserMediaViewModel } from "./media/UserMediaViewModel";
|
||||||
|
|
||||||
|
export type GridTileMediaViewModel =
|
||||||
|
| UserMediaViewModel
|
||||||
|
| RingingMediaViewModel
|
||||||
|
| UnknownParticipantMediaViewModel;
|
||||||
|
|
||||||
let nextId = 0;
|
let nextId = 0;
|
||||||
function createId(): string {
|
function createId(): string {
|
||||||
return (nextId++).toString();
|
return (nextId++).toString();
|
||||||
@@ -23,9 +29,7 @@ export class GridTileViewModel {
|
|||||||
public readonly showOutline$: Behavior<boolean> = this._showOutline$;
|
public readonly showOutline$: Behavior<boolean> = this._showOutline$;
|
||||||
|
|
||||||
public constructor(
|
public constructor(
|
||||||
public readonly media$: Behavior<
|
public readonly media$: Behavior<GridTileMediaViewModel>,
|
||||||
UserMediaViewModel | RingingMediaViewModel
|
|
||||||
>,
|
|
||||||
) {}
|
) {}
|
||||||
|
|
||||||
public setShowOutline(value: boolean): void {
|
public setShowOutline(value: boolean): void {
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ import { type BehaviorSubject } from "rxjs";
|
|||||||
import { type LocalUserMediaViewModel } from "./media/LocalUserMediaViewModel.ts";
|
import { type LocalUserMediaViewModel } from "./media/LocalUserMediaViewModel.ts";
|
||||||
import { type MediaViewModel } from "./media/MediaViewModel.ts";
|
import { type MediaViewModel } from "./media/MediaViewModel.ts";
|
||||||
import { type RingingMediaViewModel } from "./media/RingingMediaViewModel.ts";
|
import { type RingingMediaViewModel } from "./media/RingingMediaViewModel.ts";
|
||||||
|
import { type UnknownParticipantMediaViewModel } from "./media/UnknownParticipantMediaViewModel.ts";
|
||||||
import { type UserMediaViewModel } from "./media/UserMediaViewModel.ts";
|
import { type UserMediaViewModel } from "./media/UserMediaViewModel.ts";
|
||||||
import {
|
import {
|
||||||
type GridTileViewModel,
|
type GridTileViewModel,
|
||||||
@@ -18,25 +19,29 @@ import {
|
|||||||
import { type Behavior } from "./Behavior.ts";
|
import { type Behavior } from "./Behavior.ts";
|
||||||
import { shallowEquals as arrayShallowEquals } from "../utils/array.ts";
|
import { shallowEquals as arrayShallowEquals } from "../utils/array.ts";
|
||||||
|
|
||||||
|
export type GridMediaViewModel =
|
||||||
|
| UserMediaViewModel
|
||||||
|
| UnknownParticipantMediaViewModel;
|
||||||
|
|
||||||
export interface GridLayoutMedia {
|
export interface GridLayoutMedia {
|
||||||
type: "grid";
|
type: "grid";
|
||||||
edgeToEdge: false;
|
edgeToEdge: false;
|
||||||
spotlight?: MediaViewModel[];
|
spotlight?: MediaViewModel[];
|
||||||
grid: UserMediaViewModel[];
|
grid: GridMediaViewModel[];
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface SpotlightLandscapeLayoutMedia {
|
export interface SpotlightLandscapeLayoutMedia {
|
||||||
type: "spotlight-landscape";
|
type: "spotlight-landscape";
|
||||||
edgeToEdge: boolean;
|
edgeToEdge: boolean;
|
||||||
spotlight: MediaViewModel[];
|
spotlight: MediaViewModel[];
|
||||||
grid: UserMediaViewModel[];
|
grid: GridMediaViewModel[];
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface SpotlightPortraitLayoutMedia {
|
export interface SpotlightPortraitLayoutMedia {
|
||||||
type: "spotlight-portrait";
|
type: "spotlight-portrait";
|
||||||
edgeToEdge: false;
|
edgeToEdge: false;
|
||||||
spotlight: MediaViewModel[];
|
spotlight: MediaViewModel[];
|
||||||
grid: UserMediaViewModel[];
|
grid: GridMediaViewModel[];
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface SpotlightExpandedLayoutMedia {
|
export interface SpotlightExpandedLayoutMedia {
|
||||||
|
|||||||
@@ -0,0 +1,57 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2026 Element Creations Ltd.
|
||||||
|
|
||||||
|
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||||
|
Please see LICENSE in the repository root for full details.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { constant } from "../Behavior";
|
||||||
|
import { type BaseMediaViewModel } from "./MediaViewModel";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Media representing a LiveKit participant that cannot be mapped to any
|
||||||
|
* MatrixRTC member of the session.
|
||||||
|
*
|
||||||
|
* MSC4143 asks clients to surface such streams rather than silently ignore
|
||||||
|
* them, because they can signal an eavesdropping or impersonation attack. We
|
||||||
|
* therefore give them a tile, but never render their audio or video.
|
||||||
|
*
|
||||||
|
* There is no Matrix user behind this media, so `userId` and `displayName$`
|
||||||
|
* carry the LiveKit identity instead, purely so that debugging output stays
|
||||||
|
* meaningful.
|
||||||
|
*/
|
||||||
|
export interface UnknownParticipantMediaViewModel extends BaseMediaViewModel {
|
||||||
|
type: "unknown participant";
|
||||||
|
/**
|
||||||
|
* The LiveKit identity under which the participant connected. Exposed for
|
||||||
|
* debugging.
|
||||||
|
*/
|
||||||
|
rtcBackendIdentity: string;
|
||||||
|
/**
|
||||||
|
* The URL of the LiveKit focus on which the participant was seen. Exposed for
|
||||||
|
* debugging.
|
||||||
|
*/
|
||||||
|
focusUrl: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface UnknownParticipantMediaInputs {
|
||||||
|
id: string;
|
||||||
|
rtcBackendIdentity: string;
|
||||||
|
focusUrl: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createUnknownParticipantMedia({
|
||||||
|
id,
|
||||||
|
rtcBackendIdentity,
|
||||||
|
focusUrl,
|
||||||
|
}: UnknownParticipantMediaInputs): UnknownParticipantMediaViewModel {
|
||||||
|
return {
|
||||||
|
type: "unknown participant",
|
||||||
|
id,
|
||||||
|
userId: rtcBackendIdentity,
|
||||||
|
displayName$: constant(rtcBackendIdentity),
|
||||||
|
mxcAvatarUrl$: constant(undefined),
|
||||||
|
rtcBackendIdentity,
|
||||||
|
focusUrl,
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -0,0 +1,98 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2026 Element Creations Ltd.
|
||||||
|
|
||||||
|
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||||
|
Please see LICENSE in the repository root for full details.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { expect, within } from "storybook/test";
|
||||||
|
import { type JSX, useMemo } from "react";
|
||||||
|
|
||||||
|
import type { Meta, StoryObj } from "@storybook/react-vite";
|
||||||
|
import { GridTile } from "./GridTile";
|
||||||
|
import { GridTileViewModel } from "../state/TileViewModel";
|
||||||
|
import { constant } from "../state/Behavior";
|
||||||
|
import { createUnknownParticipantMedia } from "../state/media/UnknownParticipantMediaViewModel";
|
||||||
|
|
||||||
|
interface UnknownParticipantTileStoryProps {
|
||||||
|
width: number;
|
||||||
|
height: number;
|
||||||
|
rtcBackendIdentity: string;
|
||||||
|
showNameTags: boolean;
|
||||||
|
showOutline: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Renders a GridTile for a LiveKit participant that maps to no MatrixRTC
|
||||||
|
* member, driven by primitive props so that Storybook can document them.
|
||||||
|
*/
|
||||||
|
function UnknownParticipantTile({
|
||||||
|
width,
|
||||||
|
height,
|
||||||
|
rtcBackendIdentity,
|
||||||
|
...props
|
||||||
|
}: UnknownParticipantTileStoryProps): JSX.Element {
|
||||||
|
const vm = useMemo(
|
||||||
|
() =>
|
||||||
|
new GridTileViewModel(
|
||||||
|
constant(
|
||||||
|
createUnknownParticipantMedia({
|
||||||
|
id: `unknown:https://rtc.example.org:${rtcBackendIdentity}`,
|
||||||
|
rtcBackendIdentity,
|
||||||
|
focusUrl: "https://rtc.example.org",
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
),
|
||||||
|
[rtcBackendIdentity],
|
||||||
|
);
|
||||||
|
return (
|
||||||
|
<GridTile
|
||||||
|
vm={vm}
|
||||||
|
onOpenProfile={null}
|
||||||
|
targetWidth={width}
|
||||||
|
targetHeight={height}
|
||||||
|
style={{ width, height }}
|
||||||
|
showSpeakingIndicators={false}
|
||||||
|
showRingingStatus={false}
|
||||||
|
focusable
|
||||||
|
{...props}
|
||||||
|
/>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const meta = {
|
||||||
|
component: UnknownParticipantTile,
|
||||||
|
argTypes: {
|
||||||
|
width: { control: { type: "range", min: 80, max: 800, step: 10 } },
|
||||||
|
height: { control: { type: "range", min: 80, max: 600, step: 10 } },
|
||||||
|
},
|
||||||
|
} satisfies Meta<typeof UnknownParticipantTile>;
|
||||||
|
|
||||||
|
export default meta;
|
||||||
|
type Story = StoryObj<typeof meta>;
|
||||||
|
|
||||||
|
export const UnknownParticipant: Story = {
|
||||||
|
args: {
|
||||||
|
width: 400,
|
||||||
|
height: 300,
|
||||||
|
rtcBackendIdentity: "@rogue:example.org:DEVICE",
|
||||||
|
showNameTags: true,
|
||||||
|
showOutline: false,
|
||||||
|
},
|
||||||
|
play: async ({ canvasElement }) => {
|
||||||
|
const canvas = within(canvasElement);
|
||||||
|
await expect(canvas.getByText("Unknown participant")).toBeInTheDocument();
|
||||||
|
await expect(
|
||||||
|
canvas.queryByText("@rogue:example.org:DEVICE"),
|
||||||
|
).not.toBeInTheDocument();
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
/** Narrow tiles hide the name tag, so the label goes with it. */
|
||||||
|
export const UnknownParticipantNarrow: Story = {
|
||||||
|
args: {
|
||||||
|
...UnknownParticipant.args,
|
||||||
|
width: 90,
|
||||||
|
height: 90,
|
||||||
|
},
|
||||||
|
};
|
||||||
@@ -32,6 +32,7 @@ import {
|
|||||||
createRingingMedia,
|
createRingingMedia,
|
||||||
type RingingMediaViewModel,
|
type RingingMediaViewModel,
|
||||||
} from "../state/media/RingingMediaViewModel";
|
} from "../state/media/RingingMediaViewModel";
|
||||||
|
import { createUnknownParticipantMedia } from "../state/media/UnknownParticipantMediaViewModel";
|
||||||
|
|
||||||
global.IntersectionObserver = class MockIntersectionObserver {
|
global.IntersectionObserver = class MockIntersectionObserver {
|
||||||
public observe(): void {}
|
public observe(): void {}
|
||||||
@@ -165,3 +166,33 @@ test("GridTile displays ringing media", async () => {
|
|||||||
act(() => pickupState$.next("decline"));
|
act(() => pickupState$.next("decline"));
|
||||||
screen.getByText("Call ended");
|
screen.getByText("Call ended");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("GridTile displays an unknown participant", async () => {
|
||||||
|
const vm = createUnknownParticipantMedia({
|
||||||
|
id: "unknown:https://rtc-example.org:rogue",
|
||||||
|
rtcBackendIdentity: "rogue",
|
||||||
|
focusUrl: "https://rtc-example.org",
|
||||||
|
});
|
||||||
|
|
||||||
|
const { container } = render(
|
||||||
|
<ReactionsSenderProvider vm={callVm} rtcSession={fakeRtcSession}>
|
||||||
|
<GridTile
|
||||||
|
vm={new GridTileViewModel(constant(vm))}
|
||||||
|
onOpenProfile={() => {}}
|
||||||
|
targetWidth={300}
|
||||||
|
targetHeight={200}
|
||||||
|
showSpeakingIndicators
|
||||||
|
showNameTags
|
||||||
|
showRingingStatus
|
||||||
|
showOutline
|
||||||
|
focusable
|
||||||
|
/>
|
||||||
|
</ReactionsSenderProvider>,
|
||||||
|
);
|
||||||
|
expect(await axe(container)).toHaveNoViolations();
|
||||||
|
// The tile is labelled as unknown rather than after its LiveKit identity
|
||||||
|
screen.getByText("Unknown participant");
|
||||||
|
expect(screen.queryByText("rogue")).toBeNull();
|
||||||
|
// There is no user to show an avatar for
|
||||||
|
expect(container.querySelector("[data-style]")).toBeNull();
|
||||||
|
});
|
||||||
|
|||||||
@@ -51,6 +51,7 @@ import { type LocalUserMediaViewModel } from "../state/media/LocalUserMediaViewM
|
|||||||
import { type RemoteUserMediaViewModel } from "../state/media/RemoteUserMediaViewModel";
|
import { type RemoteUserMediaViewModel } from "../state/media/RemoteUserMediaViewModel";
|
||||||
import { type UserMediaViewModel } from "../state/media/UserMediaViewModel";
|
import { type UserMediaViewModel } from "../state/media/UserMediaViewModel";
|
||||||
import { type RingingMediaViewModel } from "../state/media/RingingMediaViewModel";
|
import { type RingingMediaViewModel } from "../state/media/RingingMediaViewModel";
|
||||||
|
import { type UnknownParticipantMediaViewModel } from "../state/media/UnknownParticipantMediaViewModel";
|
||||||
import { RingingStatus } from "./RingingStatus";
|
import { RingingStatus } from "./RingingStatus";
|
||||||
|
|
||||||
interface TileProps {
|
interface TileProps {
|
||||||
@@ -97,6 +98,44 @@ const RingingMediaTile: FC<RingingMediaTileProps> = ({
|
|||||||
);
|
);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
interface UnknownParticipantTileProps extends Omit<
|
||||||
|
TileProps,
|
||||||
|
"displayName" | "mxcAvatarUrl"
|
||||||
|
> {
|
||||||
|
vm: UnknownParticipantMediaViewModel;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A blank tile standing in for a LiveKit participant that cannot be mapped to
|
||||||
|
* any MatrixRTC member. MSC4143 asks clients to surface such streams rather
|
||||||
|
* than ignore them. No media is rendered for them.
|
||||||
|
*/
|
||||||
|
const UnknownParticipantTile: FC<UnknownParticipantTileProps> = ({
|
||||||
|
vm,
|
||||||
|
className,
|
||||||
|
...props
|
||||||
|
}) => {
|
||||||
|
const { t } = useTranslation();
|
||||||
|
return (
|
||||||
|
<MediaView
|
||||||
|
className={classNames(className, styles.tile)}
|
||||||
|
video={undefined}
|
||||||
|
userId={vm.userId}
|
||||||
|
unencryptedWarning={false}
|
||||||
|
videoEnabled={false}
|
||||||
|
mirror={false}
|
||||||
|
showAvatar={false}
|
||||||
|
displayName={t("video_tile.unknown_participant")}
|
||||||
|
mxcAvatarUrl={undefined}
|
||||||
|
rtcBackendIdentity={vm.rtcBackendIdentity}
|
||||||
|
focusUrl={vm.focusUrl}
|
||||||
|
{...props}
|
||||||
|
/>
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
|
UnknownParticipantTile.displayName = "UnknownParticipantTile";
|
||||||
|
|
||||||
interface UserMediaTileProps extends TileProps {
|
interface UserMediaTileProps extends TileProps {
|
||||||
vm: UserMediaViewModel;
|
vm: UserMediaViewModel;
|
||||||
showSpeakingIndicators: boolean;
|
showSpeakingIndicators: boolean;
|
||||||
@@ -454,6 +493,15 @@ export const GridTile: FC<GridTileProps> = ({
|
|||||||
{...props}
|
{...props}
|
||||||
/>
|
/>
|
||||||
);
|
);
|
||||||
|
} else if (media.type === "unknown participant") {
|
||||||
|
return (
|
||||||
|
<UnknownParticipantTile
|
||||||
|
ref={ref}
|
||||||
|
vm={media}
|
||||||
|
className={classNames(className, { [styles.outline]: showOutline })}
|
||||||
|
{...props}
|
||||||
|
/>
|
||||||
|
);
|
||||||
} else if (media.local) {
|
} else if (media.local) {
|
||||||
return (
|
return (
|
||||||
<LocalUserMediaTile
|
<LocalUserMediaTile
|
||||||
|
|||||||
+17
-9
@@ -56,6 +56,11 @@ interface Props extends ComponentProps<typeof animated.div> {
|
|||||||
displayName: string;
|
displayName: string;
|
||||||
mxcAvatarUrl: string | undefined;
|
mxcAvatarUrl: string | undefined;
|
||||||
avatarStyle?: "solid" | "translucent";
|
avatarStyle?: "solid" | "translucent";
|
||||||
|
/**
|
||||||
|
* Whether to show an avatar when there is no video. Off for media that has no
|
||||||
|
* Matrix user behind it.
|
||||||
|
*/
|
||||||
|
showAvatar?: boolean;
|
||||||
background?: "solid" | "transparent";
|
background?: "solid" | "transparent";
|
||||||
focusable: boolean;
|
focusable: boolean;
|
||||||
primaryButton?: ReactNode;
|
primaryButton?: ReactNode;
|
||||||
@@ -95,6 +100,7 @@ export const MediaView: FC<Props> = ({
|
|||||||
displayName,
|
displayName,
|
||||||
mxcAvatarUrl,
|
mxcAvatarUrl,
|
||||||
avatarStyle = "solid",
|
avatarStyle = "solid",
|
||||||
|
showAvatar = true,
|
||||||
background = "solid",
|
background = "solid",
|
||||||
focusable,
|
focusable,
|
||||||
primaryButton,
|
primaryButton,
|
||||||
@@ -176,15 +182,17 @@ export const MediaView: FC<Props> = ({
|
|||||||
<div className={styles.speakingBorder} />
|
<div className={styles.speakingBorder} />
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
<Avatar
|
{showAvatar && (
|
||||||
id={userId}
|
<Avatar
|
||||||
name={displayName}
|
id={userId}
|
||||||
size={avatarSize}
|
name={displayName}
|
||||||
src={mxcAvatarUrl}
|
size={avatarSize}
|
||||||
data-style={avatarStyle}
|
src={mxcAvatarUrl}
|
||||||
className={styles.avatar}
|
data-style={avatarStyle}
|
||||||
style={{ display: video && videoEnabled ? "none" : "initial" }}
|
className={styles.avatar}
|
||||||
/>
|
style={{ display: video && videoEnabled ? "none" : "initial" }}
|
||||||
|
/>
|
||||||
|
)}
|
||||||
{video?.publication !== undefined && (
|
{video?.publication !== undefined && (
|
||||||
<VideoTrack
|
<VideoTrack
|
||||||
trackRef={video}
|
trackRef={video}
|
||||||
|
|||||||
Reference in New Issue
Block a user