Matrix/element-call-Github
1
0
Fork 0
mirror of https://github.com/vector-im/element-call.git synced 2026-06-12 12:02:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'livekit' into toger5/tiles_based_on_rtc_member

Browse Source
This commit is contained in:
Hugh Nimmo-Smith
2024-11-06 13:42:45 +00:00
parent fecd789d02 c45f724279
commit 733b685e96
57 changed files with 2159 additions and 1101 deletions
Download Patch File Download Diff File Expand all files Collapse all files

513
src/state/CallViewModel.test.ts
View File

@@ -6,19 +6,25 @@ Please see LICENSE in the repository root for full details.
*/
import { test, vi, onTestFinished } from "vitest";
import { map, Observable, of } from "rxjs";
import {
combineLatest,
debounceTime,
distinctUntilChanged,
map,
Observable,
of,
switchMap,
} from "rxjs";
import { MatrixClient } from "matrix-js-sdk/src/matrix";
import {
ConnectionState,
LocalParticipant,
Participant,
RemoteParticipant,
} from "livekit-client";
import * as ComponentsCore from "@livekit/components-core";
import {
CallMembership,
MatrixRTCSession,
MatrixRTCSessionEvent,
} from "matrix-js-sdk/src/matrixrtc";
import { isEqual } from "lodash";
import { CallMembership, MatrixRTCSession, MatrixRTCSessionEvent } from "matrix-js-sdk/src/matrixrtc";
import { CallViewModel, Layout } from "./CallViewModel";
import {
@@ -27,7 +33,6 @@ import {
mockMatrixRoom,
mockRoomMember,
mockRemoteParticipant,
OurRunHelpers,
withTestScheduler,
mockMembership,
} from "../utils/test";
@@ -40,41 +45,40 @@ import { MockRoom, MockRTCSession } from "../useReactions.test";
vi.mock("@livekit/components-core");
const carolId = "@carol:example.org";
const carolDev = "CCCC";
const aliceId = "@alice:example.org";
const alice = mockRoomMember({ userId: "@alice:example.org" });
const bob = mockRoomMember({ userId: "@bob:example.org" });
const carol = mockRoomMember({ userId: "@carol:example.org" });
const dave = mockRoomMember({ userId: "@dave:example.org" });
const aliceDev = "AAAA";
const aliceRTCId = aliceId + ":" + aliceDev;
const bobId = "@bob:example.org";
const bobDev = "BBBB";
const bobRTCId = bobId + ":" + bobDev;
const alice = mockRoomMember({ userId: aliceId });
const bob = mockRoomMember({ userId: bobId });
const carol = mockRoomMember({ userId: carolId });
const carolDev = "CCCC";
const daveDev = "DDDD";
const aliceId = `${alice.userId}:${aliceDev}`;
const bobId = `${bob.userId}:${bobDev}`;
const carolId = `${carol.userId}:${carolDev}`;
const daveId = `${dave.userId}:${daveDev}`;
const localParticipant = mockLocalParticipant({ identity: "" });
const aliceParticipant = mockRemoteParticipant({ identity: aliceRTCId });
const aliceParticipant = mockRemoteParticipant({ identity: aliceId });
const aliceSharingScreen = mockRemoteParticipant({
identity: aliceRTCId,
identity: aliceId,
isScreenShareEnabled: true,
});
const bobParticipant = mockRemoteParticipant({ identity: bobRTCId });
const bobParticipant = mockRemoteParticipant({ identity: bobId });
const bobSharingScreen = mockRemoteParticipant({
identity: bobRTCId,
identity: bobId,
isScreenShareEnabled: true,
});
const daveParticipant = mockRemoteParticipant({ identity: daveId });
const members = new Map([
[alice.userId, alice],
[bob.userId, bob],
[carol.userId, carol],
]);
const members = new Map([alice, bob, carol, dave].map((p) => [p.userId, p]));
const rtcMemberAlice = mockMembership(aliceId, aliceDev);
const rtcMemberBob = mockMembership(bobId, bobDev);
const rtcMemberCarol = mockMembership(carolId, carolDev);
const aliceRtcMember = mockMembership(aliceId, aliceDev);
const bobRtcMember = mockMembership(bobId, bobDev);
const carolRtcMember = mockMembership(carolId, carolDev);
const daveRtcMember = mockMembership(daveId, daveDev);
export interface GridLayoutSummary {
type: "grid";
@@ -119,39 +123,72 @@ export type LayoutSummary =
| OneOnOneLayoutSummary
| PipLayoutSummary;
function summarizeLayout(l: Layout): LayoutSummary {
switch (l.type) {
case "grid":
return {
type: l.type,
spotlight: l.spotlight?.map((vm) => vm.id),
grid: l.grid.map((vm) => vm.id),
};
case "spotlight-landscape":
case "spotlight-portrait":
return {
type: l.type,
spotlight: l.spotlight.map((vm) => vm.id),
grid: l.grid.map((vm) => vm.id),
};
case "spotlight-expanded":
return {
type: l.type,
spotlight: l.spotlight.map((vm) => vm.id),
pip: l.pip?.id,
};
case "one-on-one":
return { type: l.type, local: l.local.id, remote: l.remote.id };
case "pip":
return { type: l.type, spotlight: l.spotlight.map((vm) => vm.id) };
}
function summarizeLayout(l: Observable<Layout>): Observable<LayoutSummary> {
return l.pipe(
switchMap((l) => {
switch (l.type) {
case "grid":
return combineLatest(
[
l.spotlight?.media ?? of(undefined),
...l.grid.map((vm) => vm.media),
],
(spotlight, ...grid) => ({
type: l.type,
spotlight: spotlight?.map((vm) => vm.id),
grid: grid.map((vm) => vm.id),
}),
);
case "spotlight-landscape":
case "spotlight-portrait":
return combineLatest(
[l.spotlight.media, ...l.grid.map((vm) => vm.media)],
(spotlight, ...grid) => ({
type: l.type,
spotlight: spotlight.map((vm) => vm.id),
grid: grid.map((vm) => vm.id),
}),
);
case "spotlight-expanded":
return combineLatest(
[l.spotlight.media, l.pip?.media ?? of(undefined)],
(spotlight, pip) => ({
type: l.type,
spotlight: spotlight.map((vm) => vm.id),
pip: pip?.id,
}),
);
case "one-on-one":
return combineLatest(
[l.local.media, l.remote.media],
(local, remote) => ({
type: l.type,
local: local.id,
remote: remote.id,
}),
);
case "pip":
return l.spotlight.media.pipe(
map((spotlight) => ({
type: l.type,
spotlight: spotlight.map((vm) => vm.id),
})),
);
}
}),
// Sometimes there can be multiple (synchronous) updates per frame. We only
// care about the most recent value for each time step, so discard these
// extra values.
debounceTime(0),
distinctUntilChanged(isEqual),
);
}
function withCallViewModel(
{ cold }: OurRunHelpers,
remoteParticipants: Observable<RemoteParticipant[]>,
rtcMembers: Observable<Partial<CallMembership>[]>,
connectionState: Observable<ECConnectionState>,
speaking: Map<Participant, Observable<boolean>>,
continuation: (vm: CallViewModel) => void,
): void {
const room = mockMatrixRoom({
@@ -172,19 +209,30 @@ function withCallViewModel(
const mediaSpy = vi
.spyOn(ComponentsCore, "observeParticipantMedia")
.mockImplementation((p) =>
cold("a", {
a: { participant: p } as Partial<
ComponentsCore.ParticipantMedia<LocalParticipant>
> as ComponentsCore.ParticipantMedia<LocalParticipant>,
}),
of({ participant: p } as Partial<
ComponentsCore.ParticipantMedia<LocalParticipant>
> as ComponentsCore.ParticipantMedia<LocalParticipant>),
);
const eventsSpy = vi
.spyOn(ComponentsCore, "observeParticipantEvents")
.mockImplementation((p) => cold("a", { a: p }));
.mockImplementation((p) =>
(speaking.get(p) ?? of(false)).pipe(
map((s) => ({ ...p, isSpeaking: s }) as Participant),
),
);
const roomEventSelectorSpy = vi
.spyOn(ComponentsCore, "roomEventSelector")
.mockImplementation((room, eventType) => of());
const liveKitRoom = mockLivekitRoom(
{ localParticipant },
{ remoteParticipants },
);
const vm = new CallViewModel(
rtcSession as unknown as MatrixRTCSession,
mockLivekitRoom({ localParticipant }),
liveKitRoom,
{
kind: E2eeType.PER_PARTICIPANT,
},
@@ -196,115 +244,316 @@ function withCallViewModel(
participantsSpy!.mockRestore();
mediaSpy!.mockRestore();
eventsSpy!.mockRestore();
roomEventSelectorSpy!.mockRestore();
});
continuation(vm);
}
test("participants are retained during a focus switch", () => {
withTestScheduler((helpers) => {
const { hot, expectObservable } = helpers;
withTestScheduler(({ cold, expectObservable }) => {
// Participants disappear on frame 2 and come back on frame 3
const partMarbles = "a-ba";
// The RTC members never disappear
const rtcMemberMarbles = "a---";
const participantMarbles = "a-ba";
// Start switching focus on frame 1 and reconnect on frame 3
const connMarbles = "ab-a";
const connectionMarbles = " cs-c";
// The visible participants should remain the same throughout the switch
const laytMarbles = "aaaa 2997ms a 56998ms a";
const layoutMarbles = " a";
withCallViewModel(
helpers,
hot(partMarbles, {
cold(participantMarbles, {
a: [aliceParticipant, bobParticipant],
b: [],
}),
hot(rtcMemberMarbles, {
a: [rtcMemberAlice, rtcMemberBob, rtcMemberCarol],
}),
hot(connMarbles, {
a: ConnectionState.Connected,
b: ECAddonConnectionState.ECSwitchingFocus,
of([aliceRtcMember, bobRtcMember]),
cold(connectionMarbles, {
c: ConnectionState.Connected,
s: ECAddonConnectionState.ECSwitchingFocus,
}),
new Map(),
(vm) => {
expectObservable(vm.layout.pipe(map(summarizeLayout))).toBe(
laytMarbles,
{
a: {
type: "grid",
spotlight: undefined,
grid: ["local:0", `${aliceRTCId}:0`, `${bobRTCId}:0`],
},
expectObservable(summarizeLayout(vm.layout)).toBe(layoutMarbles, {
a: {
type: "grid",
spotlight: undefined,
grid: ["local:0", `${aliceId}:0`, `${bobId}:0`],
},
);
});
},
);
});
});
test("screen sharing activates spotlight layout", () => {
withTestScheduler((helpers) => {
const { hot, schedule, expectObservable } = helpers;
withTestScheduler(({ cold, schedule, expectObservable }) => {
// Start with no screen shares, then have Alice and Bob share their screens,
// then return to no screen shares, then have just Alice share for a bit
const partMarbles = "abc---d---a-b---a";
const participantMarbles = " abcda-ba";
// While there are no screen shares, switch to spotlight manually, and then
// switch back to grid at the end
const modeMarbles = "-----------a--------b";
const modeMarbles = " -----s--g";
// We should automatically enter spotlight for the first round of screen
// sharing, then return to grid, then manually go into spotlight, and
// remain in spotlight until we manually go back to grid
const laytMarbles = "ab(cc)(dd)ae(bb)(ee)a 59979ms a";
const layoutMarbles = " abcdaefeg";
const showSpeakingMarbles = "y----nyny";
withCallViewModel(
helpers,
hot(partMarbles, {
cold(participantMarbles, {
a: [aliceParticipant, bobParticipant],
b: [aliceSharingScreen, bobParticipant],
c: [aliceSharingScreen, bobSharingScreen],
d: [aliceParticipant, bobSharingScreen],
}),
of([rtcMemberAlice, rtcMemberAlice]),
hot("a", { a: ConnectionState.Connected }),
of([aliceRtcMember, bobRtcMember]),
of(ConnectionState.Connected),
new Map(),
(vm) => {
schedule(modeMarbles, {
a: () => vm.setGridMode("spotlight"),
b: () => vm.setGridMode("grid"),
s: () => vm.setGridMode("spotlight"),
g: () => vm.setGridMode("grid"),
});
expectObservable(vm.layout.pipe(map(summarizeLayout))).toBe(
laytMarbles,
{
a: {
type: "grid",
spotlight: undefined,
grid: ["local:0", `${aliceId}:0`, `${bobId}:0`],
},
b: {
type: "spotlight-landscape",
spotlight: [`${aliceId}:0:screen-share`],
grid: ["local:0", `${aliceId}:0`, `${bobId}:0`],
},
c: {
type: "spotlight-landscape",
spotlight: [
`${aliceId}:0:screen-share`,
`${bobId}:0:screen-share`,
],
grid: ["local:0", `${aliceId}:0`, `${bobId}:0`],
},
d: {
type: "spotlight-landscape",
spotlight: [`${bobId}:0:screen-share`],
grid: ["local:0", `${aliceId}:0`, `${bobId}:0`],
},
e: {
type: "spotlight-landscape",
spotlight: [`${aliceId}:0`],
grid: ["local:0", `${aliceId}:0`, `${bobId}:0`],
},
expectObservable(summarizeLayout(vm.layout)).toBe(layoutMarbles, {
a: {
type: "grid",
spotlight: undefined,
grid: ["local:0", `${aliceId}:0`, `${bobId}:0`],
},
);
b: {
type: "spotlight-landscape",
spotlight: [`${aliceId}:0:screen-share`],
grid: ["local:0", `${aliceId}:0`, `${bobId}:0`],
},
c: {
type: "spotlight-landscape",
spotlight: [`${aliceId}:0:screen-share`, `${bobId}:0:screen-share`],
grid: ["local:0", `${aliceId}:0`, `${bobId}:0`],
},
d: {
type: "spotlight-landscape",
spotlight: [`${bobId}:0:screen-share`],
grid: ["local:0", `${aliceId}:0`, `${bobId}:0`],
},
e: {
type: "spotlight-landscape",
spotlight: [`${aliceId}:0`],
grid: ["local:0", `${bobId}:0`],
},
f: {
type: "spotlight-landscape",
spotlight: [`${aliceId}:0:screen-share`],
grid: ["local:0", `${bobId}:0`, `${aliceId}:0`],
},
g: {
type: "grid",
spotlight: undefined,
grid: ["local:0", `${bobId}:0`, `${aliceId}:0`],
},
});
expectObservable(vm.showSpeakingIndicators).toBe(showSpeakingMarbles, {
y: true,
n: false,
});
},
);
});
});
test("participants stay in the same order unless to appear/disappear", () => {
withTestScheduler(({ cold, schedule, expectObservable }) => {
const modeMarbles = "a";
// First Bob speaks, then Dave, then Alice
const aSpeakingMarbles = "n- 1998ms - 1999ms y";
const bSpeakingMarbles = "ny 1998ms n 1999ms ";
const dSpeakingMarbles = "n- 1998ms y 1999ms n";
// Nothing should change when Bob speaks, because Bob is already on screen.
// When Dave speaks he should switch with Alice because she's the one who
// hasn't spoken at all. Then when Alice speaks, she should return to her
// place at the top.
const layoutMarbles = " a 1999ms b 1999ms a 57999ms c 1999ms a";
withCallViewModel(
of([aliceParticipant, bobParticipant, daveParticipant]),
of([aliceRtcMember, bobRtcMember, daveRtcMember]),
of(ConnectionState.Connected),
new Map([
[aliceParticipant, cold(aSpeakingMarbles, { y: true, n: false })],
[bobParticipant, cold(bSpeakingMarbles, { y: true, n: false })],
[daveParticipant, cold(dSpeakingMarbles, { y: true, n: false })],
]),
(vm) => {
schedule(modeMarbles, {
a: () => {
// We imagine that only three tiles (the first three) will be visible
// on screen at a time
vm.layout.subscribe((layout) => {
if (layout.type === "grid") {
for (let i = 0; i < layout.grid.length; i++)
layout.grid[i].setVisible(i < 3);
}
});
},
});
expectObservable(summarizeLayout(vm.layout)).toBe(layoutMarbles, {
a: {
type: "grid",
spotlight: undefined,
grid: ["local:0", `${aliceId}:0`, `${bobId}:0`, `${daveId}:0`],
},
b: {
type: "grid",
spotlight: undefined,
grid: ["local:0", `${daveId}:0`, `${bobId}:0`, `${aliceId}:0`],
},
c: {
type: "grid",
spotlight: undefined,
grid: ["local:0", `${aliceId}:0`, `${daveId}:0`, `${bobId}:0`],
},
});
},
);
});
});
test("spotlight speakers swap places", () => {
withTestScheduler(({ cold, schedule, expectObservable }) => {
// Go immediately into spotlight mode for the test
const modeMarbles = " s";
// First Bob speaks, then Dave, then Alice
const aSpeakingMarbles = "n--y";
const bSpeakingMarbles = "nyn";
const dSpeakingMarbles = "n-yn";
// Alice should start in the spotlight, then Bob, then Dave, then Alice
// again. However, the positions of Dave and Bob in the grid should be
// reversed by the end because they've been swapped in and out of the
// spotlight.
const layoutMarbles = " abcd";
withCallViewModel(
of([aliceParticipant, bobParticipant, daveParticipant]),
of([aliceRtcMember, bobRtcMember, daveRtcMember]),
of(ConnectionState.Connected),
new Map([
[aliceParticipant, cold(aSpeakingMarbles, { y: true, n: false })],
[bobParticipant, cold(bSpeakingMarbles, { y: true, n: false })],
[daveParticipant, cold(dSpeakingMarbles, { y: true, n: false })],
]),
(vm) => {
schedule(modeMarbles, { s: () => vm.setGridMode("spotlight") });
expectObservable(summarizeLayout(vm.layout)).toBe(layoutMarbles, {
a: {
type: "spotlight-landscape",
spotlight: [`${aliceId}:0`],
grid: ["local:0", `${bobId}:0`, `${daveId}:0`],
},
b: {
type: "spotlight-landscape",
spotlight: [`${bobId}:0`],
grid: ["local:0", `${aliceId}:0`, `${daveId}:0`],
},
c: {
type: "spotlight-landscape",
spotlight: [`${daveId}:0`],
grid: ["local:0", `${aliceId}:0`, `${bobId}:0`],
},
d: {
type: "spotlight-landscape",
spotlight: [`${aliceId}:0`],
grid: ["local:0", `${daveId}:0`, `${bobId}:0`],
},
});
},
);
});
});
test("layout enters picture-in-picture mode when requested", () => {
withTestScheduler(({ schedule, expectObservable }) => {
// Enable then disable picture-in-picture
const pipControlMarbles = "-ed";
// Should go into picture-in-picture layout then back to grid
const layoutMarbles = " aba";
withCallViewModel(
of([aliceParticipant, bobParticipant]),
of([aliceRtcMember, bobRtcMember]),
of(ConnectionState.Connected),
new Map(),
(vm) => {
schedule(pipControlMarbles, {
e: () => window.controls.enablePip(),
d: () => window.controls.disablePip(),
});
expectObservable(summarizeLayout(vm.layout)).toBe(layoutMarbles, {
a: {
type: "grid",
spotlight: undefined,
grid: ["local:0", `${aliceId}:0`, `${bobId}:0`],
},
b: {
type: "pip",
spotlight: [`${aliceId}:0`],
},
});
},
);
});
});
test("spotlight remembers whether it's expanded", () => {
withTestScheduler(({ schedule, expectObservable }) => {
// Start in spotlight mode, then switch to grid and back to spotlight a
// couple times
const modeMarbles = " s-gs-gs";
// Expand and collapse the spotlight
const expandMarbles = "-a--a";
// Spotlight should stay expanded during the first mode switch, and stay
// collapsed during the second mode switch
const layoutMarbles = "abcbada";
withCallViewModel(
of([aliceParticipant, bobParticipant]),
of([aliceRtcMember, bobRtcMember]),
of(ConnectionState.Connected),
new Map(),
(vm) => {
schedule(modeMarbles, {
s: () => vm.setGridMode("spotlight"),
g: () => vm.setGridMode("grid"),
});
schedule(expandMarbles, {
a: () => {
let toggle: () => void;
vm.toggleSpotlightExpanded.subscribe((val) => (toggle = val!));
toggle!();
},
});
expectObservable(summarizeLayout(vm.layout)).toBe(layoutMarbles, {
a: {
type: "spotlight-landscape",
spotlight: [`${aliceId}:0`],
grid: ["local:0", `${bobId}:0`],
},
b: {
type: "spotlight-expanded",
spotlight: [`${aliceId}:0`],
pip: "local:0",
},
c: {
type: "grid",
spotlight: undefined,
grid: ["local:0", `${aliceId}:0`, `${bobId}:0`],
},
d: {
type: "grid",
spotlight: undefined,
grid: ["local:0", `${bobId}:0`, `${aliceId}:0`],
},
});
},
);
});

241
src/state/CallViewModel.ts
View File

@@ -71,6 +71,12 @@ import { ObservableScope } from "./ObservableScope";
import { duplicateTiles, nonMemberTiles } from "../settings/settings";
import { isFirefox } from "../Platform";
import { setPipEnabled } from "../controls";
import { GridTileViewModel, SpotlightTileViewModel } from "./TileViewModel";
import { TileStore } from "./TileStore";
import { gridLikeLayout } from "./GridLikeLayout";
import { spotlightExpandedLayout } from "./SpotlightExpandedLayout";
import { oneOnOneLayout } from "./OneOnOneLayout";
import { pipLayout } from "./PipLayout";
import { EncryptionSystem } from "../e2ee/sharedKeyManagement";
// How long we wait after a focus switch before showing the real participant
@@ -81,39 +87,82 @@ const POST_FOCUS_PARTICIPANT_UPDATE_DELAY_MS = 3000;
// on mobile. No spotlight tile should be shown below this threshold.
const smallMobileCallThreshold = 3;
export interface GridLayout {
export interface GridLayoutMedia {
type: "grid";
spotlight?: MediaViewModel[];
grid: UserMediaViewModel[];
}
export interface SpotlightLandscapeLayout {
export interface SpotlightLandscapeLayoutMedia {
type: "spotlight-landscape";
spotlight: MediaViewModel[];
grid: UserMediaViewModel[];
}
export interface SpotlightPortraitLayout {
export interface SpotlightPortraitLayoutMedia {
type: "spotlight-portrait";
spotlight: MediaViewModel[];
grid: UserMediaViewModel[];
}
export interface SpotlightExpandedLayout {
export interface SpotlightExpandedLayoutMedia {
type: "spotlight-expanded";
spotlight: MediaViewModel[];
pip?: UserMediaViewModel;
}
export interface OneOnOneLayoutMedia {
type: "one-on-one";
local: UserMediaViewModel;
remote: UserMediaViewModel;
}
export interface PipLayoutMedia {
type: "pip";
spotlight: MediaViewModel[];
}
export type LayoutMedia =
| GridLayoutMedia
| SpotlightLandscapeLayoutMedia
| SpotlightPortraitLayoutMedia
| SpotlightExpandedLayoutMedia
| OneOnOneLayoutMedia
| PipLayoutMedia;
export interface GridLayout {
type: "grid";
spotlight?: SpotlightTileViewModel;
grid: GridTileViewModel[];
}
export interface SpotlightLandscapeLayout {
type: "spotlight-landscape";
spotlight: SpotlightTileViewModel;
grid: GridTileViewModel[];
}
export interface SpotlightPortraitLayout {
type: "spotlight-portrait";
spotlight: SpotlightTileViewModel;
grid: GridTileViewModel[];
}
export interface SpotlightExpandedLayout {
type: "spotlight-expanded";
spotlight: SpotlightTileViewModel;
pip?: GridTileViewModel;
}
export interface OneOnOneLayout {
type: "one-on-one";
local: LocalUserMediaViewModel;
remote: RemoteUserMediaViewModel;
local: GridTileViewModel;
remote: GridTileViewModel;
}
export interface PipLayout {
type: "pip";
spotlight: MediaViewModel[];
spotlight: SpotlightTileViewModel;
}
/**
@@ -162,6 +211,12 @@ enum SortingBin {
SelfNotAlwaysShown,
}
interface LayoutScanState {
layout: Layout | null;
tiles: TileStore;
visibleTiles: Set<GridTileViewModel>;
}
class UserMedia {
private readonly scope = new ObservableScope();
public readonly vm: UserMediaViewModel;
@@ -176,6 +231,7 @@ class UserMedia {
member: RoomMember | undefined,
participant: LocalParticipant | RemoteParticipant | undefined,
encryptionSystem: EncryptionSystem,
livekitRoom: LivekitRoom,
) {
this.participant = new BehaviorSubject(participant);
@@ -185,6 +241,7 @@ class UserMedia {
member,
this.participant.asObservable() as Observable<LocalParticipant>,
encryptionSystem,
livekitRoom,
);
} else {
this.vm = new RemoteUserMediaViewModel(
@@ -194,6 +251,7 @@ class UserMedia {
RemoteParticipant | undefined
>,
encryptionSystem,
livekitRoom,
);
}
@@ -247,6 +305,7 @@ class ScreenShare {
member: RoomMember | undefined,
participant: LocalParticipant | RemoteParticipant,
encryptionSystem: EncryptionSystem,
liveKitRoom: LivekitRoom,
) {
this.participant = new BehaviorSubject(participant);
@@ -255,6 +314,7 @@ class ScreenShare {
member,
this.participant.asObservable(),
encryptionSystem,
liveKitRoom,
);
}
@@ -437,6 +497,7 @@ export class CallViewModel extends ViewModel {
member,
participant,
this.encryptionSystem,
this.livekitRoom
),
];
}
@@ -450,6 +511,7 @@ export class CallViewModel extends ViewModel {
member,
participant,
this.encryptionSystem,
this.livekitRoom,
),
];
}
@@ -662,6 +724,14 @@ export class CallViewModel extends ViewModel {
this.scope.state(),
);
private readonly hasRemoteScreenShares: Observable<boolean> =
this.spotlight.pipe(
map((spotlight) =>
spotlight.some((vm) => !vm.local && vm instanceof ScreenShareViewModel),
),
distinctUntilChanged(),
);
private readonly pip: Observable<UserMediaViewModel | null> =
this.spotlightAndPip.pipe(switchMap(([, pip]) => pip));
@@ -742,7 +812,7 @@ export class CallViewModel extends ViewModel {
screenShares.length === 0,
);
private readonly gridLayout: Observable<Layout> = combineLatest(
private readonly gridLayout: Observable<LayoutMedia> = combineLatest(
[this.grid, this.spotlight],
(grid, spotlight) => ({
type: "grid",
@@ -753,38 +823,44 @@ export class CallViewModel extends ViewModel {
}),
);
private readonly spotlightLandscapeLayout: Observable<Layout> = combineLatest(
[this.grid, this.spotlight],
(grid, spotlight) => ({ type: "spotlight-landscape", spotlight, grid }),
);
private readonly spotlightLandscapeLayout: Observable<LayoutMedia> =
combineLatest([this.grid, this.spotlight], (grid, spotlight) => ({
type: "spotlight-landscape",
spotlight,
grid,
}));
private readonly spotlightPortraitLayout: Observable<Layout> = combineLatest(
[this.grid, this.spotlight],
(grid, spotlight) => ({ type: "spotlight-portrait", spotlight, grid }),
);
private readonly spotlightPortraitLayout: Observable<LayoutMedia> =
combineLatest([this.grid, this.spotlight], (grid, spotlight) => ({
type: "spotlight-portrait",
spotlight,
grid,
}));
private readonly spotlightExpandedLayout: Observable<Layout> = combineLatest(
[this.spotlight, this.pip],
(spotlight, pip) => ({
private readonly spotlightExpandedLayout: Observable<LayoutMedia> =
combineLatest([this.spotlight, this.pip], (spotlight, pip) => ({
type: "spotlight-expanded",
spotlight,
pip: pip ?? undefined,
}),
);
}));
private readonly oneOnOneLayout: Observable<Layout> = this.grid.pipe(
map((grid) => ({
type: "one-on-one",
local: grid.find((vm) => vm.local) as LocalUserMediaViewModel,
remote: grid.find((vm) => !vm.local) as RemoteUserMediaViewModel,
})),
);
private readonly oneOnOneLayout: Observable<LayoutMedia> =
this.mediaItems.pipe(
map((grid) => ({
type: "one-on-one",
local: grid.find((vm) => vm.vm.local)!.vm as LocalUserMediaViewModel,
remote: grid.find((vm) => !vm.vm.local)!.vm as RemoteUserMediaViewModel,
})),
);
private readonly pipLayout: Observable<Layout> = this.spotlight.pipe(
private readonly pipLayout: Observable<LayoutMedia> = this.spotlight.pipe(
map((spotlight) => ({ type: "pip", spotlight })),
);
public readonly layout: Observable<Layout> = this.windowMode.pipe(
/**
* The media to be used to produce a layout.
*/
private readonly layoutMedia: Observable<LayoutMedia> = this.windowMode.pipe(
switchMap((windowMode) => {
switch (windowMode) {
case "normal":
@@ -845,32 +921,97 @@ export class CallViewModel extends ViewModel {
this.scope.state(),
);
/**
* The layout of tiles in the call interface.
*/
public readonly layout: Observable<Layout> = this.layoutMedia.pipe(
// Each layout will produce a set of tiles, and these tiles have an
// observable indicating whether they're visible. We loop this information
// back into the layout process by using switchScan.
switchScan<
LayoutMedia,
LayoutScanState,
Observable<LayoutScanState & { layout: Layout }>
>(
({ tiles: prevTiles, visibleTiles }, media) => {
let layout: Layout;
let newTiles: TileStore;
switch (media.type) {
case "grid":
case "spotlight-landscape":
case "spotlight-portrait":
[layout, newTiles] = gridLikeLayout(media, visibleTiles, prevTiles);
break;
case "spotlight-expanded":
[layout, newTiles] = spotlightExpandedLayout(
media,
visibleTiles,
prevTiles,
);
break;
case "one-on-one":
[layout, newTiles] = oneOnOneLayout(media, visibleTiles, prevTiles);
break;
case "pip":
[layout, newTiles] = pipLayout(media, visibleTiles, prevTiles);
break;
}
// Take all of the 'visible' observables and combine them into one big
// observable array
const visibilities =
newTiles.gridTiles.length === 0
? of([])
: combineLatest(newTiles.gridTiles.map((tile) => tile.visible));
return visibilities.pipe(
map((visibilities) => ({
layout: layout,
tiles: newTiles,
visibleTiles: new Set(
newTiles.gridTiles.filter((_tile, i) => visibilities[i]),
),
})),
);
},
{
layout: null,
tiles: TileStore.empty(),
visibleTiles: new Set(),
},
),
map(({ layout }) => layout),
this.scope.state(),
);
public showSpotlightIndicators: Observable<boolean> = this.layout.pipe(
map((l) => l.type !== "grid"),
this.scope.state(),
);
/**
* Determines whether video should be shown for a certain piece of media
* appearing in the grid.
*/
public showGridVideo(vm: MediaViewModel): Observable<boolean> {
return this.layout.pipe(
map(
(l) =>
!(
(l.type === "spotlight-landscape" ||
l.type === "spotlight-portrait") &&
// This media is already visible in the spotlight; avoid duplication
l.spotlight.some((spotlightVm) => spotlightVm === vm)
),
),
distinctUntilChanged(),
);
}
public showSpeakingIndicators: Observable<boolean> = this.layout.pipe(
map((l) => l.type !== "one-on-one" && !l.type.startsWith("spotlight-")),
switchMap((l) => {
switch (l.type) {
case "spotlight-landscape":
case "spotlight-portrait":
// If the spotlight is showing the active speaker, we can do without
// speaking indicators as they're a redundant visual cue. But if
// screen sharing feeds are in the spotlight we still need them.
return l.spotlight.media.pipe(
map((models: MediaViewModel[]) =>
models.some((m) => m instanceof ScreenShareViewModel),
),
);
// In expanded spotlight layout, the active speaker is always shown in
// the picture-in-picture tile so there is no need for speaking
// indicators. And in one-on-one layout there's no question as to who is
// speaking.
case "spotlight-expanded":
case "one-on-one":
return of(false);
default:
return of(true);
}
}),
this.scope.state(),
);

43
src/state/GridLikeLayout.ts Normal file
View File

@@ -0,0 +1,43 @@
/*
Copyright 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { Layout, LayoutMedia } from "./CallViewModel";
import { TileStore } from "./TileStore";
import { GridTileViewModel } from "./TileViewModel";
export type GridLikeLayoutType =
| "grid"
| "spotlight-landscape"
| "spotlight-portrait";
/**
* Produces a grid-like layout (any layout with a grid and possibly a spotlight)
* with the given media.
*/
export function gridLikeLayout(
media: LayoutMedia & { type: GridLikeLayoutType },
visibleTiles: Set<GridTileViewModel>,
prevTiles: TileStore,
): [Layout & { type: GridLikeLayoutType }, TileStore] {
const update = prevTiles.from(visibleTiles);
if (media.spotlight !== undefined)
update.registerSpotlight(
media.spotlight,
media.type === "spotlight-portrait",
);
for (const mediaVm of media.grid) update.registerGridTile(mediaVm);
const tiles = update.build();
return [
{
type: media.type,
spotlight: tiles.spotlightTile,
grid: tiles.gridTiles,
} as Layout & { type: GridLikeLayoutType },
tiles,
];
}

190
src/state/MediaViewModel.ts
View File

@@ -11,6 +11,7 @@ import {
VideoSource,
observeParticipantEvents,
observeParticipantMedia,
roomEventSelector,
} from "@livekit/components-core";
import {
LocalParticipant,
@@ -21,6 +22,9 @@ import {
Track,
TrackEvent,
facingModeFromLocalTrack,
Room as LivekitRoom,
RoomEvent as LivekitRoomEvent,
RemoteTrack,
} from "livekit-client";
import { RoomMember, RoomMemberEvent } from "matrix-js-sdk/src/matrix";
import {
@@ -28,13 +32,18 @@ import {
Observable,
Subject,
combineLatest,
distinctUntilChanged,
distinctUntilKeyChanged,
filter,
fromEvent,
interval,
map,
merge,
of,
shareReplay,
startWith,
switchMap,
throttleTime,
} from "rxjs";
import { useEffect } from "react";
@@ -90,6 +99,115 @@ export function observeTrackReference(
return obs;
}
function observeRemoteTrackReceivingOkay(
participant: Participant,
source: Track.Source,
): Observable<boolean | undefined> {
let lastStats: {
framesDecoded: number | undefined;
framesDropped: number | undefined;
framesReceived: number | undefined;
} = {
framesDecoded: undefined,
framesDropped: undefined,
framesReceived: undefined,
};
return combineLatest([
observeTrackReference(participant, source),
interval(1000).pipe(startWith(0)),
]).pipe(
switchMap(async ([trackReference]) => {
const track = trackReference.publication?.track;
if (!track || !(track instanceof RemoteTrack)) {
return undefined;
}
const report = await track.getRTCStatsReport();
if (!report) {
return undefined;
}
for (const v of report.values()) {
if (v.type === "inbound-rtp") {
const { framesDecoded, framesDropped, framesReceived } =
v as RTCInboundRtpStreamStats;
return {
framesDecoded,
framesDropped,
framesReceived,
};
}
}
return undefined;
}),
filter((newStats) => !!newStats),
map((newStats): boolean | undefined => {
const oldStats = lastStats;
lastStats = newStats;
if (
typeof newStats.framesReceived === "number" &&
typeof oldStats.framesReceived === "number" &&
typeof newStats.framesDecoded === "number" &&
typeof oldStats.framesDecoded === "number"
) {
const framesReceivedDelta =
newStats.framesReceived - oldStats.framesReceived;
const framesDecodedDelta =
newStats.framesDecoded - oldStats.framesDecoded;
// if we received >0 frames and managed to decode >0 frames then we treat that as success
if (framesReceivedDelta > 0) {
return framesDecodedDelta > 0;
}
}
// no change
return undefined;
}),
filter((x) => typeof x === "boolean"),
startWith(undefined),
);
}
function encryptionErrorObservable(
room: LivekitRoom,
participant: Participant,
encryptionSystem: EncryptionSystem,
criteria: string,
): Observable<boolean> {
return roomEventSelector(room, LivekitRoomEvent.EncryptionError).pipe(
map((e) => {
const [err] = e;
if (encryptionSystem.kind === E2eeType.PER_PARTICIPANT) {
return (
// Ideally we would pull the participant identity from the field on the error.
// However, it gets lost in the serialization process between workers.
// So, instead we do a string match
(err?.message.includes(participant.identity) &&
err?.message.includes(criteria)) ??
false
);
} else if (encryptionSystem.kind === E2eeType.SHARED_KEY) {
return !!err?.message.includes(criteria);
}
return false;
}),
throttleTime(1000), // Throttle to avoid spamming the UI
startWith(false),
);
}
export enum EncryptionStatus {
Connecting,
Okay,
KeyMissing,
KeyInvalid,
PasswordInvalid,
}
abstract class BaseMediaViewModel extends ViewModel {
/**
* Whether the media belongs to the local user.
@@ -112,6 +230,8 @@ abstract class BaseMediaViewModel extends ViewModel {
map((p) => !!p),
);
public readonly encryptionStatus: Observable<EncryptionStatus>;
public constructor(
/**
* An opaque identifier for this media.
@@ -132,6 +252,7 @@ abstract class BaseMediaViewModel extends ViewModel {
encryptionSystem: EncryptionSystem,
audioSource: AudioSource,
videoSource: VideoSource,
livekitRoom: LivekitRoom,
) {
super();
const audio = observeTrackReference(participant, audioSource).pipe(
@@ -146,7 +267,64 @@ abstract class BaseMediaViewModel extends ViewModel {
encryptionSystem.kind !== E2eeType.NONE &&
(a?.publication?.isEncrypted === false ||
v?.publication?.isEncrypted === false),
).pipe(this.scope.state());
).pipe(this.scope.state());
if (participant.isLocal || encryptionSystem.kind === E2eeType.NONE) {
this.encryptionStatus = of(EncryptionStatus.Okay).pipe(
this.scope.state(),
);
} else if (encryptionSystem.kind === E2eeType.PER_PARTICIPANT) {
this.encryptionStatus = combineLatest([
encryptionErrorObservable(
livekitRoom,
participant,
encryptionSystem,
"MissingKey",
),
encryptionErrorObservable(
livekitRoom,
participant,
encryptionSystem,
"InvalidKey",
),
observeRemoteTrackReceivingOkay(participant, audioSource),
observeRemoteTrackReceivingOkay(participant, videoSource),
]).pipe(
map(([keyMissing, keyInvalid, audioOkay, videoOkay]) => {
if (keyMissing) return EncryptionStatus.KeyMissing;
if (keyInvalid) return EncryptionStatus.KeyInvalid;
if (audioOkay || videoOkay) return EncryptionStatus.Okay;
return undefined; // no change
}),
filter((x) => !!x),
startWith(EncryptionStatus.Connecting),
this.scope.state(),
);
} else {
this.encryptionStatus = combineLatest([
encryptionErrorObservable(
livekitRoom,
participant,
encryptionSystem,
"InvalidKey",
),
observeRemoteTrackReceivingOkay(participant, audioSource),
observeRemoteTrackReceivingOkay(participant, videoSource),
]).pipe(
map(
([keyInvalid, audioOkay, videoOkay]):
| EncryptionStatus
| undefined => {
if (keyInvalid) return EncryptionStatus.PasswordInvalid;
if (audioOkay || videoOkay) return EncryptionStatus.Okay;
return undefined; // no change
},
),
filter((x) => !!x),
startWith(EncryptionStatus.Connecting),
this.scope.state(),
);
}
}
}
@@ -201,6 +379,7 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel {
member: RoomMember | undefined,
participant: Observable<LocalParticipant | RemoteParticipant | undefined>,
encryptionSystem: EncryptionSystem,
livekitRoom: LivekitRoom,
) {
super(
id,
@@ -209,6 +388,7 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel {
encryptionSystem,
Track.Source.Microphone,
Track.Source.Camera,
livekitRoom,
);
const media = participant.pipe(
@@ -261,8 +441,9 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel {
member: RoomMember | undefined,
participant: Observable<LocalParticipant | undefined>,
encryptionSystem: EncryptionSystem,
livekitRoom: LivekitRoom,
) {
super(id, member, participant, encryptionSystem);
super(id, member, participant, encryptionSystem, livekitRoom);
}
}
@@ -321,8 +502,9 @@ export class RemoteUserMediaViewModel extends BaseUserMediaViewModel {
member: RoomMember | undefined,
participant: Observable<RemoteParticipant | undefined>,
encryptionSystem: EncryptionSystem,
livekitRoom: LivekitRoom,
) {
super(id, member, participant, encryptionSystem);
super(id, member, participant, encryptionSystem, livekitRoom);
// Sync the local volume with LiveKit
combineLatest([
@@ -353,6 +535,7 @@ export class ScreenShareViewModel extends BaseMediaViewModel {
member: RoomMember | undefined,
participant: Observable<LocalParticipant | RemoteParticipant>,
encryptionSystem: EncryptionSystem,
livekitRoom: LivekitRoom,
) {
super(
id,
@@ -361,6 +544,7 @@ export class ScreenShareViewModel extends BaseMediaViewModel {
encryptionSystem,
Track.Source.ScreenShareAudio,
Track.Source.ScreenShare,
livekitRoom,
);
}
}

32
src/state/OneOnOneLayout.ts Normal file
View File

@@ -0,0 +1,32 @@
/*
Copyright 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { OneOnOneLayout, OneOnOneLayoutMedia } from "./CallViewModel";
import { TileStore } from "./TileStore";
import { GridTileViewModel } from "./TileViewModel";
/**
* Produces a one-on-one layout with the given media.
*/
export function oneOnOneLayout(
media: OneOnOneLayoutMedia,
visibleTiles: Set<GridTileViewModel>,
prevTiles: TileStore,
): [OneOnOneLayout, TileStore] {
const update = prevTiles.from(visibleTiles);
update.registerGridTile(media.local);
update.registerGridTile(media.remote);
const tiles = update.build();
return [
{
type: media.type,
local: tiles.gridTilesByMedia.get(media.local)!,
remote: tiles.gridTilesByMedia.get(media.remote)!,
},
tiles,
];
}

30
src/state/PipLayout.ts Normal file
View File

@@ -0,0 +1,30 @@
/*
Copyright 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { PipLayout, PipLayoutMedia } from "./CallViewModel";
import { TileStore } from "./TileStore";
import { GridTileViewModel } from "./TileViewModel";
/**
* Produces a picture-in-picture layout with the given media.
*/
export function pipLayout(
media: PipLayoutMedia,
visibleTiles: Set<GridTileViewModel>,
prevTiles: TileStore,
): [PipLayout, TileStore] {
const update = prevTiles.from(visibleTiles);
update.registerSpotlight(media.spotlight, true);
const tiles = update.build();
return [
{
type: media.type,
spotlight: tiles.spotlightTile!,
},
tiles,
];
}

36
src/state/SpotlightExpandedLayout.ts Normal file
View File

@@ -0,0 +1,36 @@
/*
Copyright 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import {
SpotlightExpandedLayout,
SpotlightExpandedLayoutMedia,
} from "./CallViewModel";
import { TileStore } from "./TileStore";
import { GridTileViewModel } from "./TileViewModel";
/**
* Produces an expanded spotlight layout with the given media.
*/
export function spotlightExpandedLayout(
media: SpotlightExpandedLayoutMedia,
visibleTiles: Set<GridTileViewModel>,
prevTiles: TileStore,
): [SpotlightExpandedLayout, TileStore] {
const update = prevTiles.from(visibleTiles);
update.registerSpotlight(media.spotlight, true);
if (media.pip !== undefined) update.registerGridTile(media.pip);
const tiles = update.build();
return [
{
type: media.type,
spotlight: tiles.spotlightTile!,
pip: tiles.gridTiles[0],
},
tiles,
];
}

259
src/state/TileStore.ts Normal file
View File

@@ -0,0 +1,259 @@
/*
Copyright 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { BehaviorSubject } from "rxjs";
import { MediaViewModel, UserMediaViewModel } from "./MediaViewModel";
import { GridTileViewModel, SpotlightTileViewModel } from "./TileViewModel";
import { fillGaps } from "../utils/iter";
class SpotlightTileData {
private readonly media_: BehaviorSubject<MediaViewModel[]>;
public get media(): MediaViewModel[] {
return this.media_.value;
}
public set media(value: MediaViewModel[]) {
this.media_.next(value);
}
private readonly maximised_: BehaviorSubject<boolean>;
public get maximised(): boolean {
return this.maximised_.value;
}
public set maximised(value: boolean) {
this.maximised_.next(value);
}
public readonly vm: SpotlightTileViewModel;
public constructor(media: MediaViewModel[], maximised: boolean) {
this.media_ = new BehaviorSubject(media);
this.maximised_ = new BehaviorSubject(maximised);
this.vm = new SpotlightTileViewModel(this.media_, this.maximised_);
}
public destroy(): void {
this.vm.destroy();
}
}
class GridTileData {
private readonly media_: BehaviorSubject<UserMediaViewModel>;
public get media(): UserMediaViewModel {
return this.media_.value;
}
public set media(value: UserMediaViewModel) {
this.media_.next(value);
}
public readonly vm: GridTileViewModel;
public constructor(media: UserMediaViewModel) {
this.media_ = new BehaviorSubject(media);
this.vm = new GridTileViewModel(this.media_);
}
public destroy(): void {
this.vm.destroy();
}
}
/**
* A collection of tiles to be mapped to a layout.
*/
export class TileStore {
private constructor(
private readonly spotlight: SpotlightTileData | null,
private readonly grid: GridTileData[],
) {}
public readonly spotlightTile = this.spotlight?.vm;
public readonly gridTiles = this.grid.map(({ vm }) => vm);
public readonly gridTilesByMedia = new Map(
this.grid.map(({ vm, media }) => [media, vm]),
);
/**
* Creates an an empty collection of tiles.
*/
public static empty(): TileStore {
return new TileStore(null, []);
}
/**
* Creates a builder which can be used to update the collection, passing
* ownership of the tiles to the updated collection.
*/
public from(visibleTiles: Set<GridTileViewModel>): TileStoreBuilder {
return new TileStoreBuilder(
this.spotlight,
this.grid,
(spotlight, grid) => new TileStore(spotlight, grid),
visibleTiles,
);
}
}
/**
* A builder for a new collection of tiles. Will reuse tiles and destroy unused
* tiles from a previous collection where appropriate.
*/
export class TileStoreBuilder {
private spotlight: SpotlightTileData | null = null;
private readonly prevSpotlightSpeaker =
this.prevSpotlight?.media.length === 1 &&
"speaking" in this.prevSpotlight.media[0] &&
this.prevSpotlight.media[0];
private readonly prevGridByMedia = new Map(
this.prevGrid.map((entry, i) => [entry.media, [entry, i]] as const),
);
// The total number of grid entries that we have so far
private numGridEntries = 0;
// A sparse array of grid entries which should be kept in the same spots as
// which they appeared in the previous grid
private readonly stationaryGridEntries: GridTileData[] = new Array(
this.prevGrid.length,
);
// Grid entries which should now enter the visible section of the grid
private readonly visibleGridEntries: GridTileData[] = [];
// Grid entries which should now enter the invisible section of the grid
private readonly invisibleGridEntries: GridTileData[] = [];
public constructor(
private readonly prevSpotlight: SpotlightTileData | null,
private readonly prevGrid: GridTileData[],
private readonly construct: (
spotlight: SpotlightTileData | null,
grid: GridTileData[],
) => TileStore,
private readonly visibleTiles: Set<GridTileViewModel>,
) {}
/**
* Sets the contents of the spotlight tile. If this is never called, there
* will be no spotlight tile.
*/
public registerSpotlight(media: MediaViewModel[], maximised: boolean): void {
if (this.spotlight !== null) throw new Error("Spotlight already set");
if (this.numGridEntries > 0)
throw new Error("Spotlight must be registered before grid tiles");
// Reuse the previous spotlight tile if it exists
if (this.prevSpotlight === null) {
this.spotlight = new SpotlightTileData(media, maximised);
} else {
this.spotlight = this.prevSpotlight;
this.spotlight.media = media;
this.spotlight.maximised = maximised;
}
}
/**
* Sets up a grid tile for the given media. If this is never called for some
* media, then that media will have no grid tile.
*/
public registerGridTile(media: UserMediaViewModel): void {
if (this.spotlight !== null) {
// We actually *don't* want spotlight speakers to appear in both the
// spotlight and the grid, so they're filtered out here
if (!media.local && this.spotlight.media.includes(media)) return;
// When the spotlight speaker changes, we would see one grid tile appear
// and another grid tile disappear. This would be an undesirable layout
// shift, so instead what we do is take the speaker's grid tile and swap
// the media out, so it can remain where it is in the layout.
if (
media === this.prevSpotlightSpeaker &&
this.spotlight.media.length === 1 &&
"speaking" in this.spotlight.media[0] &&
this.prevSpotlightSpeaker !== this.spotlight.media[0]
) {
const prev = this.prevGridByMedia.get(this.spotlight.media[0]);
if (prev !== undefined) {
const [entry, prevIndex] = prev;
const previouslyVisible = this.visibleTiles.has(entry.vm);
const nowVisible = this.visibleTiles.has(
this.prevGrid[this.numGridEntries]?.vm,
);
// If it doesn't need to move between the visible/invisible sections of
// the grid, then we can keep it where it was and swap the media
if (previouslyVisible === nowVisible) {
this.stationaryGridEntries[prevIndex] = entry;
// Do the media swap
entry.media = media;
this.prevGridByMedia.delete(this.spotlight.media[0]);
this.prevGridByMedia.set(media, prev);
} else {
// Create a new tile; this will cause a layout shift but I'm not
// sure there's any other straightforward option in this case
(nowVisible
? this.visibleGridEntries
: this.invisibleGridEntries
).push(new GridTileData(media));
}
this.numGridEntries++;
return;
}
}
}
// Was there previously a tile with this same media?
const prev = this.prevGridByMedia.get(media);
if (prev === undefined) {
// Create a new tile
(this.visibleTiles.has(this.prevGrid[this.numGridEntries]?.vm)
? this.visibleGridEntries
: this.invisibleGridEntries
).push(new GridTileData(media));
} else {
// Reuse the existing tile
const [entry, prevIndex] = prev;
const previouslyVisible = this.visibleTiles.has(entry.vm);
const nowVisible = this.visibleTiles.has(
this.prevGrid[this.numGridEntries]?.vm,
);
// If it doesn't need to move between the visible/invisible sections of
// the grid, then we can keep it exactly where it was previously
if (previouslyVisible === nowVisible)
this.stationaryGridEntries[prevIndex] = entry;
// Otherwise, queue this tile to be moved
else
(nowVisible ? this.visibleGridEntries : this.invisibleGridEntries).push(
entry,
);
}
this.numGridEntries++;
}
/**
* Constructs a new collection of all registered tiles, transferring ownership
* of the tiles to the new collection. Any tiles present in the previous
* collection but not the new collection will be destroyed.
*/
public build(): TileStore {
// Piece together the grid
const grid = [
...fillGaps(this.stationaryGridEntries, [
...this.visibleGridEntries,
...this.invisibleGridEntries,
]),
];
// Destroy unused tiles
if (this.spotlight === null && this.prevSpotlight !== null)
this.prevSpotlight.destroy();
const gridEntries = new Set(grid);
for (const entry of this.prevGrid)
if (!gridEntries.has(entry)) entry.destroy();
return this.construct(this.spotlight, grid);
}
}

43
src/state/TileViewModel.ts Normal file
View File

@@ -0,0 +1,43 @@
/*
Copyright 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { BehaviorSubject, Observable } from "rxjs";
import { ViewModel } from "./ViewModel";
import { MediaViewModel, UserMediaViewModel } from "./MediaViewModel";
let nextId = 0;
function createId(): string {
return (nextId++).toString();
}
export class GridTileViewModel extends ViewModel {
public readonly id = createId();
private readonly visible_ = new BehaviorSubject(false);
/**
* Whether the tile is visible within the current viewport.
*/
public readonly visible: Observable<boolean> = this.visible_;
public setVisible = (value: boolean): void => this.visible_.next(value);
public constructor(public readonly media: Observable<UserMediaViewModel>) {
super();
}
}
export class SpotlightTileViewModel extends ViewModel {
public constructor(
public readonly media: Observable<MediaViewModel[]>,
public readonly maximised: Observable<boolean>,
) {
super();
}
}
export type TileViewModel = GridTileViewModel | SpotlightTileViewModel;

34
src/state/useObservable.ts
View File

@@ -1,34 +0,0 @@
/*
Copyright 2023, 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { Ref, useCallback, useRef } from "react";
import { BehaviorSubject, Observable } from "rxjs";
import { useInitial } from "../useInitial";
/**
* React hook that creates an Observable from a changing value. The Observable
* replays its current value upon subscription and emits whenever the value
* changes.
*/
export function useObservable<T>(value: T): Observable<T> {
const subject = useRef<BehaviorSubject<T>>();
subject.current ??= new BehaviorSubject(value);
if (value !== subject.current.value) subject.current.next(value);
return subject.current;
}
/**
* React hook that creates a ref and an Observable that emits any values
* stored in the ref. The Observable replays the value currently stored in the
* ref upon subscription.
*/
export function useObservableRef<T>(initialValue: T): [Observable<T>, Ref<T>] {
const subject = useInitial(() => new BehaviorSubject(initialValue));
const ref = useCallback((value: T) => subject.next(value), [subject]);
return [subject, ref];
}