Matrix/element-call-Github
1
0
Fork 0
mirror of https://github.com/vector-im/element-call.git synced 2026-02-02 04:05:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Keep tiles in a stable order (#2670)

Browse Source 
* Keep tiles in a stable order

This introduces a new layer of abstraction on top of MediaViewModel: TileViewModel, which gives us a place to store data relating to tiles rather than their media, and also generally makes it easier to reason about tiles as they move about the call layout. I have created a class called TileStore to keep track of these tiles.

This allows us to swap out the media shown on a tile as the spotlight speaker changes, and avoid moving tiles around unless they really need to jump between the visible/invisible regions of the layout.

* Don't throttle spotlight updates

Since we now assume that the spotlight and grid will be in sync (i.e. an active speaker in one will behave as an active speaker in the other), we don't want the spotlight to ever lag behind due to throttling. If this causes usability issues we should maybe look into making LiveKit's 'speaking' indicators less erratic first.

* Make layout shifts due to a change in speaker less surprising

Although we try now to avoid layout shifts due to the spotlight speaker changing wherever possible, a spotlight speaker coming from off screen can still trigger one. Let's shift the layout a bit more gracefully in this case.

* Improve the tile ordering tests

* Maximize the spotlight tile in portrait layout

* Tell tiles whether they're actually visible in a more timely manner

* Fix test

* Fix speaking indicators logic

* Improve readability of marbles

* Fix test case

---------

Co-authored-by: Hugh Nimmo-Smith <hughns@element.io>
This commit is contained in:
Robin
2024-11-06 04:36:48 -05:00
committed by GitHub
parent 22cca2874a
commit d3f069e763
22 changed files with 1178 additions and 339 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
src/grid/CallLayout.ts
View File

@@ -8,8 +8,8 @@ Please see LICENSE in the repository root for full details.
import { BehaviorSubject, Observable } from "rxjs";
import { ComponentType } from "react";
import { MediaViewModel, UserMediaViewModel } from "../state/MediaViewModel";
import { LayoutProps } from "./Grid";
import { TileViewModel } from "../state/TileViewModel";
export interface Bounds {
width: number;
@@ -42,19 +42,6 @@ export interface CallLayoutInputs {
pipAlignment: BehaviorSubject<Alignment>;
}
export interface GridTileModel {
type: "grid";
vm: UserMediaViewModel;
}
export interface SpotlightTileModel {
type: "spotlight";
vms: MediaViewModel[];
maximised: boolean;
}
export type TileModel = GridTileModel | SpotlightTileModel;
export interface CallLayoutOutputs<Model> {
/**
* Whether the scrolling layer of the layout should appear on top.
@@ -63,11 +50,11 @@ export interface CallLayoutOutputs<Model> {
/**
* The visually fixed (non-scrolling) layer of the layout.
*/
fixed: ComponentType<LayoutProps<Model, TileModel, HTMLDivElement>>;
fixed: ComponentType<LayoutProps<Model, TileViewModel, HTMLDivElement>>;
/**
* The layer of the layout that can overflow and be scrolled.
*/
scrolling: ComponentType<LayoutProps<Model, TileModel, HTMLDivElement>>;
scrolling: ComponentType<LayoutProps<Model, TileViewModel, HTMLDivElement>>;
}
/**

50
src/grid/Grid.tsx
View File

@@ -24,6 +24,7 @@ import {
createContext,
forwardRef,
memo,
useCallback,
useContext,
useEffect,
useMemo,
@@ -33,6 +34,8 @@ import {
import useMeasure from "react-use-measure";
import classNames from "classnames";
import { logger } from "matrix-js-sdk/src/logger";
import { useObservableEagerState } from "observable-hooks";
import { fromEvent, map, startWith } from "rxjs";
import styles from "./Grid.module.css";
import { useMergedRefs } from "../useMergedRefs";
@@ -51,6 +54,7 @@ interface Tile<Model> {
id: string;
model: Model;
onDrag: DragCallback | undefined;
setVisible: (visible: boolean) => void;
}
type PlacedTile<Model> = Tile<Model> & Rect;
@@ -84,6 +88,7 @@ interface SlotProps<Model> extends Omit<ComponentProps<"div">, "onDrag"> {
id: string;
model: Model;
onDrag?: DragCallback;
onVisibilityChange?: (visible: boolean) => void;
style?: CSSProperties;
className?: string;
}
@@ -131,6 +136,11 @@ export function useUpdateLayout(): void {
);
}
const windowHeightObservable = fromEvent(window, "resize").pipe(
startWith(null),
map(() => window.innerHeight),
);
export interface LayoutProps<LayoutModel, TileModel, R extends HTMLElement> {
ref: LegacyRef<R>;
model: LayoutModel;
@@ -232,6 +242,7 @@ export function Grid<
const [gridRoot, gridRef2] = useState<HTMLElement | null>(null);
const gridRef = useMergedRefs<HTMLElement>(gridRef1, gridRef2);
const windowHeight = useObservableEagerState(windowHeightObservable);
const [layoutRoot, setLayoutRoot] = useState<HTMLElement | null>(null);
const [generation, setGeneration] = useState<number | null>(null);
const tiles = useInitial(() => new Map<string, Tile<TileModel>>());
@@ -239,12 +250,34 @@ export function Grid<
const Slot: FC<SlotProps<TileModel>> = useMemo(
() =>
function Slot({ id, model, onDrag, style, className, ...props }) {
function Slot({
id,
model,
onDrag,
onVisibilityChange,
style,
className,
...props
}) {
const ref = useRef<HTMLDivElement | null>(null);
const prevVisible = useRef<boolean | null>(null);
const setVisible = useCallback(
(visible: boolean) => {
if (
onVisibilityChange !== undefined &&
visible !== prevVisible.current
) {
onVisibilityChange(visible);
prevVisible.current = visible;
}
},
[onVisibilityChange],
);
useEffect(() => {
tiles.set(id, { id, model, onDrag });
tiles.set(id, { id, model, onDrag, setVisible });
return (): void => void tiles.delete(id);
}, [id, model, onDrag]);
}, [id, model, onDrag, setVisible]);
return (
<div
@@ -302,6 +335,17 @@ export function Grid<
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [gridRoot, layoutRoot, tiles, gridBounds, generation]);
// The height of the portion of the grid visible at any given time
const visibleHeight = useMemo(
() => Math.min(gridBounds.bottom, windowHeight) - gridBounds.top,
[gridBounds, windowHeight],
);
useEffect(() => {
for (const tile of placedTiles)
tile.setVisible(tile.y + tile.height <= visibleHeight);
}, [placedTiles, visibleHeight]);
// Drag state is stored in a ref rather than component state, because we use
// react-spring's imperative API during gestures to improve responsiveness
const dragState = useRef<DragState | null>(null);

35
src/grid/GridLayout.tsx
View File

@@ -12,12 +12,7 @@ import { useObservableEagerState } from "observable-hooks";
import { GridLayout as GridLayoutModel } from "../state/CallViewModel";
import styles from "./GridLayout.module.css";
import { useInitial } from "../useInitial";
import {
CallLayout,
GridTileModel,
TileModel,
arrangeTiles,
} from "./CallLayout";
import { CallLayout, arrangeTiles } from "./CallLayout";
import { DragCallback, useUpdateLayout } from "./Grid";
interface GridCSSProperties extends CSSProperties {
@@ -49,15 +44,6 @@ export const makeGridLayout: CallLayout<GridLayoutModel> = ({
),
),
);
const tileModel: TileModel | undefined = useMemo(
() =>
model.spotlight && {
type: "spotlight",
vms: model.spotlight,
maximised: false,
},
[model.spotlight],
);
const onDragSpotlight: DragCallback = useCallback(
({ xRatio, yRatio }) =>
@@ -70,11 +56,11 @@ export const makeGridLayout: CallLayout<GridLayoutModel> = ({
return (
<div ref={ref} className={styles.fixed}>
{tileModel && (
{model.spotlight && (
<Slot
className={styles.slot}
id="spotlight"
model={tileModel}
model={model.spotlight}
onDrag={onDragSpotlight}
data-block-alignment={alignment.block}
data-inline-alignment={alignment.inline}
@@ -93,11 +79,6 @@ export const makeGridLayout: CallLayout<GridLayoutModel> = ({
[width, minHeight, model.grid.length],
);
const tileModels: GridTileModel[] = useMemo(
() => model.grid.map((vm) => ({ type: "grid", vm })),
[model.grid],
);
return (
<div
ref={ref}
@@ -111,8 +92,14 @@ export const makeGridLayout: CallLayout<GridLayoutModel> = ({
} as GridCSSProperties
}
>
{tileModels.map((m) => (
<Slot key={m.vm.id} className={styles.slot} id={m.vm.id} model={m} />
{model.grid.map((m) => (
<Slot
key={m.id}
className={styles.slot}
id={m.id}
model={m}
onVisibilityChange={m.setVisible}
/>
))}
</div>
);

21
src/grid/OneOnOneLayout.tsx
View File

@@ -10,7 +10,7 @@ import { useObservableEagerState } from "observable-hooks";
import classNames from "classnames";
import { OneOnOneLayout as OneOnOneLayoutModel } from "../state/CallViewModel";
import { CallLayout, GridTileModel, arrangeTiles } from "./CallLayout";
import { CallLayout, arrangeTiles } from "./CallLayout";
import styles from "./OneOnOneLayout.module.css";
import { DragCallback, useUpdateLayout } from "./Grid";
@@ -38,15 +38,6 @@ export const makeOneOnOneLayout: CallLayout<OneOnOneLayoutModel> = ({
[width, height],
);
const remoteTileModel: GridTileModel = useMemo(
() => ({ type: "grid", vm: model.remote }),
[model.remote],
);
const localTileModel: GridTileModel = useMemo(
() => ({ type: "grid", vm: model.local }),
[model.local],
);
const onDragLocalTile: DragCallback = useCallback(
({ xRatio, yRatio }) =>
pipAlignment.next({
@@ -59,16 +50,18 @@ export const makeOneOnOneLayout: CallLayout<OneOnOneLayoutModel> = ({
return (
<div ref={ref} className={styles.layer}>
<Slot
id={remoteTileModel.vm.id}
model={remoteTileModel}
id={model.remote.id}
model={model.remote}
onVisibilityChange={model.remote.setVisible}
className={styles.container}
style={{ width: tileWidth, height: tileHeight }}
>
<Slot
className={classNames(styles.slot, styles.local)}
id={localTileModel.vm.id}
model={localTileModel}
id={model.local.id}
model={model.local}
onDrag={onDragLocalTile}
onVisibilityChange={model.local.setVisible}
data-block-alignment={pipAlignmentValue.block}
data-inline-alignment={pipAlignmentValue.inline}
/>

22
src/grid/SpotlightExpandedLayout.tsx
View File

@@ -5,11 +5,11 @@ SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { forwardRef, useCallback, useMemo } from "react";
import { forwardRef, useCallback } from "react";
import { useObservableEagerState } from "observable-hooks";
import { SpotlightExpandedLayout as SpotlightExpandedLayoutModel } from "../state/CallViewModel";
import { CallLayout, GridTileModel, SpotlightTileModel } from "./CallLayout";
import { CallLayout } from "./CallLayout";
import { DragCallback, useUpdateLayout } from "./Grid";
import styles from "./SpotlightExpandedLayout.module.css";
@@ -27,17 +27,13 @@ export const makeSpotlightExpandedLayout: CallLayout<
ref,
) {
useUpdateLayout();
const spotlightTileModel: SpotlightTileModel = useMemo(
() => ({ type: "spotlight", vms: model.spotlight, maximised: true }),
[model.spotlight],
);
return (
<div ref={ref} className={styles.layer}>
<Slot
className={styles.spotlight}
id="spotlight"
model={spotlightTileModel}
model={model.spotlight}
/>
</div>
);
@@ -50,11 +46,6 @@ export const makeSpotlightExpandedLayout: CallLayout<
useUpdateLayout();
const pipAlignmentValue = useObservableEagerState(pipAlignment);
const pipTileModel: GridTileModel | undefined = useMemo(
() => model.pip && { type: "grid", vm: model.pip },
[model.pip],
);
const onDragPip: DragCallback = useCallback(
({ xRatio, yRatio }) =>
pipAlignment.next({
@@ -66,12 +57,13 @@ export const makeSpotlightExpandedLayout: CallLayout<
return (
<div ref={ref} className={styles.layer}>
{pipTileModel && (
{model.pip && (
<Slot
className={styles.pip}
id="pip"
model={pipTileModel}
id={model.pip.id}
model={model.pip}
onDrag={onDragPip}
onVisibilityChange={model.pip.setVisible}
data-block-alignment={pipAlignmentValue.block}
data-inline-alignment={pipAlignmentValue.inline}
/>

33
src/grid/SpotlightLandscapeLayout.tsx
View File

@@ -5,11 +5,11 @@ SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { forwardRef, useMemo } from "react";
import { forwardRef } from "react";
import { useObservableEagerState } from "observable-hooks";
import classNames from "classnames";
import { CallLayout, GridTileModel, TileModel } from "./CallLayout";
import { CallLayout } from "./CallLayout";
import { SpotlightLandscapeLayout as SpotlightLandscapeLayoutModel } from "../state/CallViewModel";
import styles from "./SpotlightLandscapeLayout.module.css";
import { useUpdateLayout } from "./Grid";
@@ -30,19 +30,15 @@ export const makeSpotlightLandscapeLayout: CallLayout<
) {
useUpdateLayout();
useObservableEagerState(minBounds);
const tileModel: TileModel = useMemo(
() => ({
type: "spotlight",
vms: model.spotlight,
maximised: false,
}),
[model.spotlight],
);
return (
<div ref={ref} className={styles.layer}>
<div className={styles.spotlight}>
<Slot className={styles.slot} id="spotlight" model={tileModel} />
<Slot
className={styles.slot}
id="spotlight"
model={model.spotlight}
/>
</div>
<div className={styles.grid} />
</div>
@@ -55,25 +51,24 @@ export const makeSpotlightLandscapeLayout: CallLayout<
) {
useUpdateLayout();
useObservableEagerState(minBounds);
const tileModels: GridTileModel[] = useMemo(
() => model.grid.map((vm) => ({ type: "grid", vm })),
[model.grid],
);
const withIndicators =
useObservableEagerState(model.spotlight.media).length > 1;
return (
<div ref={ref} className={styles.layer}>
<div
className={classNames(styles.spotlight, {
[styles.withIndicators]: model.spotlight.length > 1,
[styles.withIndicators]: withIndicators,
})}
/>
<div className={styles.grid}>
{tileModels.map((m) => (
{model.grid.map((m) => (
<Slot
key={m.vm.id}
key={m.id}
className={styles.slot}
id={m.vm.id}
id={m.id}
model={m}
onVisibilityChange={m.setVisible}
/>
))}
</div>

38
src/grid/SpotlightPortraitLayout.tsx
View File

@@ -5,16 +5,11 @@ SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { CSSProperties, forwardRef, useMemo } from "react";
import { CSSProperties, forwardRef } from "react";
import { useObservableEagerState } from "observable-hooks";
import classNames from "classnames";
import {
CallLayout,
GridTileModel,
TileModel,
arrangeTiles,
} from "./CallLayout";
import { CallLayout, arrangeTiles } from "./CallLayout";
import { SpotlightPortraitLayout as SpotlightPortraitLayoutModel } from "../state/CallViewModel";
import styles from "./SpotlightPortraitLayout.module.css";
import { useUpdateLayout } from "./Grid";
@@ -40,19 +35,15 @@ export const makeSpotlightPortraitLayout: CallLayout<
ref,
) {
useUpdateLayout();
const tileModel: TileModel = useMemo(
() => ({
type: "spotlight",
vms: model.spotlight,
maximised: true,
}),
[model.spotlight],
);
return (
<div ref={ref} className={styles.layer}>
<div className={styles.spotlight}>
<Slot className={styles.slot} id="spotlight" model={tileModel} />
<Slot
className={styles.slot}
id="spotlight"
model={model.spotlight}
/>
</div>
</div>
);
@@ -71,10 +62,8 @@ export const makeSpotlightPortraitLayout: CallLayout<
width,
model.grid.length,
);
const tileModels: GridTileModel[] = useMemo(
() => model.grid.map((vm) => ({ type: "grid", vm })),
[model.grid],
);
const withIndicators =
useObservableEagerState(model.spotlight.media).length > 1;
return (
<div
@@ -90,16 +79,17 @@ export const makeSpotlightPortraitLayout: CallLayout<
>
<div
className={classNames(styles.spotlight, {
[styles.withIndicators]: model.spotlight.length > 1,
[styles.withIndicators]: withIndicators,
})}
/>
<div className={styles.grid}>
{tileModels.map((m) => (
{model.grid.map((m) => (
<Slot
key={m.vm.id}
key={m.id}
className={styles.slot}
id={m.vm.id}
id={m.id}
model={m}
onVisibilityChange={m.setVisible}
/>
))}
</div>

24
src/room/InCallView.tsx
View File

@@ -28,7 +28,7 @@ import {
import useMeasure from "react-use-measure";
import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession";
import classNames from "classnames";
import { BehaviorSubject, of } from "rxjs";
import { BehaviorSubject } from "rxjs";
import { useObservableEagerState } from "observable-hooks";
import { logger } from "matrix-js-sdk/src/logger";
@@ -73,7 +73,6 @@ import { E2eeType } from "../e2ee/e2eeType";
import { makeGridLayout } from "../grid/GridLayout";
import {
CallLayoutOutputs,
TileModel,
defaultPipAlignment,
defaultSpotlightAlignment,
} from "../grid/CallLayout";
@@ -81,6 +80,7 @@ import { makeOneOnOneLayout } from "../grid/OneOnOneLayout";
import { makeSpotlightExpandedLayout } from "../grid/SpotlightExpandedLayout";
import { makeSpotlightLandscapeLayout } from "../grid/SpotlightLandscapeLayout";
import { makeSpotlightPortraitLayout } from "../grid/SpotlightPortraitLayout";
import { GridTileViewModel, TileViewModel } from "../state/TileViewModel";
import { ReactionsProvider, useReactions } from "../useReactions";
import handSoundOgg from "../sound/raise_hand.ogg?url";
import handSoundMp3 from "../sound/raise_hand.mp3?url";
@@ -379,7 +379,7 @@ export const InCallView: FC<InCallViewProps> = ({
() =>
forwardRef<
HTMLDivElement,
PropsWithoutRef<TileProps<TileModel, HTMLDivElement>>
PropsWithoutRef<TileProps<TileViewModel, HTMLDivElement>>
>(function Tile(
{ className, style, targetWidth, targetHeight, model },
ref,
@@ -388,13 +388,6 @@ export const InCallView: FC<InCallViewProps> = ({
const onToggleExpanded = useObservableEagerState(
vm.toggleSpotlightExpanded,
);
const showVideo = useObservableEagerState(
useMemo(
() =>
model.type === "grid" ? vm.showGridVideo(model.vm) : of(true),
[model],
),
);
const showSpeakingIndicatorsValue = useObservableEagerState(
vm.showSpeakingIndicators,
);
@@ -402,23 +395,21 @@ export const InCallView: FC<InCallViewProps> = ({
vm.showSpotlightIndicators,
);
return model.type === "grid" ? (
return model instanceof GridTileViewModel ? (
<GridTile
ref={ref}
vm={model.vm}
vm={model}
onOpenProfile={openProfile}
targetWidth={targetWidth}
targetHeight={targetHeight}
className={classNames(className, styles.tile)}
style={style}
showVideo={showVideo}
showSpeakingIndicators={showSpeakingIndicatorsValue}
/>
) : (
<SpotlightTile
ref={ref}
vms={model.vms}
maximised={model.maximised}
vm={model}
expanded={spotlightExpanded}
onToggleExpanded={onToggleExpanded}
targetWidth={targetWidth}
@@ -452,8 +443,7 @@ export const InCallView: FC<InCallViewProps> = ({
return (
<SpotlightTile
className={classNames(styles.tile, styles.maximised)}
vms={layout.spotlight!}
maximised
vm={layout.spotlight}
expanded
onToggleExpanded={null}
targetWidth={gridBounds.height}

454
src/state/CallViewModel.test.ts
View File

@@ -6,14 +6,24 @@ Please see LICENSE in the repository root for full details.
*/
import { test, vi, onTestFinished } from "vitest";
import { map, Observable } from "rxjs";
import {
combineLatest,
debounceTime,
distinctUntilChanged,
map,
Observable,
of,
switchMap,
} from "rxjs";
import { MatrixClient } from "matrix-js-sdk/src/matrix";
import {
ConnectionState,
LocalParticipant,
Participant,
RemoteParticipant,
} from "livekit-client";
import * as ComponentsCore from "@livekit/components-core";
import { isEqual } from "lodash";
import { CallViewModel, Layout } from "./CallViewModel";
import {
@@ -22,7 +32,6 @@ import {
mockMatrixRoom,
mockMember,
mockRemoteParticipant,
OurRunHelpers,
withTestScheduler,
} from "../utils/test";
import {
@@ -33,12 +42,14 @@ import { E2eeType } from "../e2ee/e2eeType";
vi.mock("@livekit/components-core");
const aliceId = "@alice:example.org:AAAA";
const bobId = "@bob:example.org:BBBB";
const alice = mockMember({ userId: "@alice:example.org" });
const bob = mockMember({ userId: "@bob:example.org" });
const carol = mockMember({ userId: "@carol:example.org" });
const dave = mockMember({ userId: "@dave:example.org" });
const aliceId = `${alice.userId}:AAAA`;
const bobId = `${bob.userId}:BBBB`;
const daveId = `${dave.userId}:DDDD`;
const localParticipant = mockLocalParticipant({ identity: "" });
const aliceParticipant = mockRemoteParticipant({ identity: aliceId });
@@ -51,12 +62,9 @@ const bobSharingScreen = mockRemoteParticipant({
identity: bobId,
isScreenShareEnabled: true,
});
const daveParticipant = mockRemoteParticipant({ identity: daveId });
const members = new Map([
[alice.userId, alice],
[bob.userId, bob],
[carol.userId, carol],
]);
const members = new Map([alice, bob, carol, dave].map((p) => [p.userId, p]));
export interface GridLayoutSummary {
type: "grid";
@@ -101,38 +109,71 @@ export type LayoutSummary =
| OneOnOneLayoutSummary
| PipLayoutSummary;
function summarizeLayout(l: Layout): LayoutSummary {
switch (l.type) {
case "grid":
return {
type: l.type,
spotlight: l.spotlight?.map((vm) => vm.id),
grid: l.grid.map((vm) => vm.id),
};
case "spotlight-landscape":
case "spotlight-portrait":
return {
type: l.type,
spotlight: l.spotlight.map((vm) => vm.id),
grid: l.grid.map((vm) => vm.id),
};
case "spotlight-expanded":
return {
type: l.type,
spotlight: l.spotlight.map((vm) => vm.id),
pip: l.pip?.id,
};
case "one-on-one":
return { type: l.type, local: l.local.id, remote: l.remote.id };
case "pip":
return { type: l.type, spotlight: l.spotlight.map((vm) => vm.id) };
}
function summarizeLayout(l: Observable<Layout>): Observable<LayoutSummary> {
return l.pipe(
switchMap((l) => {
switch (l.type) {
case "grid":
return combineLatest(
[
l.spotlight?.media ?? of(undefined),
...l.grid.map((vm) => vm.media),
],
(spotlight, ...grid) => ({
type: l.type,
spotlight: spotlight?.map((vm) => vm.id),
grid: grid.map((vm) => vm.id),
}),
);
case "spotlight-landscape":
case "spotlight-portrait":
return combineLatest(
[l.spotlight.media, ...l.grid.map((vm) => vm.media)],
(spotlight, ...grid) => ({
type: l.type,
spotlight: spotlight.map((vm) => vm.id),
grid: grid.map((vm) => vm.id),
}),
);
case "spotlight-expanded":
return combineLatest(
[l.spotlight.media, l.pip?.media ?? of(undefined)],
(spotlight, pip) => ({
type: l.type,
spotlight: spotlight.map((vm) => vm.id),
pip: pip?.id,
}),
);
case "one-on-one":
return combineLatest(
[l.local.media, l.remote.media],
(local, remote) => ({
type: l.type,
local: local.id,
remote: remote.id,
}),
);
case "pip":
return l.spotlight.media.pipe(
map((spotlight) => ({
type: l.type,
spotlight: spotlight.map((vm) => vm.id),
})),
);
}
}),
// Sometimes there can be multiple (synchronous) updates per frame. We only
// care about the most recent value for each time step, so discard these
// extra values.
debounceTime(0),
distinctUntilChanged(isEqual),
);
}
function withCallViewModel(
{ cold }: OurRunHelpers,
remoteParticipants: Observable<RemoteParticipant[]>,
connectionState: Observable<ECConnectionState>,
speaking: Map<Participant, Observable<boolean>>,
continuation: (vm: CallViewModel) => void,
): void {
const participantsSpy = vi
@@ -141,15 +182,17 @@ function withCallViewModel(
const mediaSpy = vi
.spyOn(ComponentsCore, "observeParticipantMedia")
.mockImplementation((p) =>
cold("a", {
a: { participant: p } as Partial<
ComponentsCore.ParticipantMedia<LocalParticipant>
> as ComponentsCore.ParticipantMedia<LocalParticipant>,
}),
of({ participant: p } as Partial<
ComponentsCore.ParticipantMedia<LocalParticipant>
> as ComponentsCore.ParticipantMedia<LocalParticipant>),
);
const eventsSpy = vi
.spyOn(ComponentsCore, "observeParticipantEvents")
.mockImplementation((p) => cold("a", { a: p }));
.mockImplementation((p) =>
(speaking.get(p) ?? of(false)).pipe(
map((s) => ({ ...p, isSpeaking: s }) as Participant),
),
);
const vm = new CallViewModel(
mockMatrixRoom({
@@ -176,107 +219,103 @@ function withCallViewModel(
}
test("participants are retained during a focus switch", () => {
withTestScheduler((helpers) => {
const { hot, expectObservable } = helpers;
withTestScheduler(({ cold, expectObservable }) => {
// Participants disappear on frame 2 and come back on frame 3
const partMarbles = "a-ba";
const participantMarbles = "a-ba";
// Start switching focus on frame 1 and reconnect on frame 3
const connMarbles = "ab-a";
const connectionMarbles = " cs-c";
// The visible participants should remain the same throughout the switch
const laytMarbles = "aaaa 2997ms a 56998ms a";
const layoutMarbles = " a";
withCallViewModel(
helpers,
hot(partMarbles, {
cold(participantMarbles, {
a: [aliceParticipant, bobParticipant],
b: [],
}),
hot(connMarbles, {
a: ConnectionState.Connected,
b: ECAddonConnectionState.ECSwitchingFocus,
cold(connectionMarbles, {
c: ConnectionState.Connected,
s: ECAddonConnectionState.ECSwitchingFocus,
}),
new Map(),
(vm) => {
expectObservable(vm.layout.pipe(map(summarizeLayout))).toBe(
laytMarbles,
{
a: {
type: "grid",
spotlight: undefined,
grid: ["local:0", `${aliceId}:0`, `${bobId}:0`],
},
expectObservable(summarizeLayout(vm.layout)).toBe(layoutMarbles, {
a: {
type: "grid",
spotlight: undefined,
grid: ["local:0", `${aliceId}:0`, `${bobId}:0`],
},
);
});
},
);
});
});
test("screen sharing activates spotlight layout", () => {
withTestScheduler((helpers) => {
const { hot, schedule, expectObservable } = helpers;
withTestScheduler(({ cold, schedule, expectObservable }) => {
// Start with no screen shares, then have Alice and Bob share their screens,
// then return to no screen shares, then have just Alice share for a bit
const partMarbles = "abc---d---a-b---a";
const participantMarbles = " abcda-ba";
// While there are no screen shares, switch to spotlight manually, and then
// switch back to grid at the end
const modeMarbles = "-----------a--------b";
const modeMarbles = " -----s--g";
// We should automatically enter spotlight for the first round of screen
// sharing, then return to grid, then manually go into spotlight, and
// remain in spotlight until we manually go back to grid
const laytMarbles = "ab(cc)(dd)ae(bb)(ee)a 59979ms a";
// Speaking indicators should always be shown except for when the active
// speaker is present in the spotlight
const showMarbles = "y----------ny---n---y";
const layoutMarbles = " abcdaefeg";
const showSpeakingMarbles = "y----nyny";
withCallViewModel(
helpers,
hot(partMarbles, {
cold(participantMarbles, {
a: [aliceParticipant, bobParticipant],
b: [aliceSharingScreen, bobParticipant],
c: [aliceSharingScreen, bobSharingScreen],
d: [aliceParticipant, bobSharingScreen],
}),
hot("a", { a: ConnectionState.Connected }),
of(ConnectionState.Connected),
new Map(),
(vm) => {
schedule(modeMarbles, {
a: () => vm.setGridMode("spotlight"),
b: () => vm.setGridMode("grid"),
s: () => vm.setGridMode("spotlight"),
g: () => vm.setGridMode("grid"),
});
expectObservable(vm.layout.pipe(map(summarizeLayout))).toBe(
laytMarbles,
{
a: {
type: "grid",
spotlight: undefined,
grid: ["local:0", `${aliceId}:0`, `${bobId}:0`],
},
b: {
type: "spotlight-landscape",
spotlight: [`${aliceId}:0:screen-share`],
grid: ["local:0", `${aliceId}:0`, `${bobId}:0`],
},
c: {
type: "spotlight-landscape",
spotlight: [
`${aliceId}:0:screen-share`,
`${bobId}:0:screen-share`,
],
grid: ["local:0", `${aliceId}:0`, `${bobId}:0`],
},
d: {
type: "spotlight-landscape",
spotlight: [`${bobId}:0:screen-share`],
grid: ["local:0", `${aliceId}:0`, `${bobId}:0`],
},
e: {
type: "spotlight-landscape",
spotlight: [`${aliceId}:0`],
grid: ["local:0", `${aliceId}:0`, `${bobId}:0`],
},
expectObservable(summarizeLayout(vm.layout)).toBe(layoutMarbles, {
a: {
type: "grid",
spotlight: undefined,
grid: ["local:0", `${aliceId}:0`, `${bobId}:0`],
},
);
expectObservable(vm.showSpeakingIndicators).toBe(showMarbles, {
b: {
type: "spotlight-landscape",
spotlight: [`${aliceId}:0:screen-share`],
grid: ["local:0", `${aliceId}:0`, `${bobId}:0`],
},
c: {
type: "spotlight-landscape",
spotlight: [`${aliceId}:0:screen-share`, `${bobId}:0:screen-share`],
grid: ["local:0", `${aliceId}:0`, `${bobId}:0`],
},
d: {
type: "spotlight-landscape",
spotlight: [`${bobId}:0:screen-share`],
grid: ["local:0", `${aliceId}:0`, `${bobId}:0`],
},
e: {
type: "spotlight-landscape",
spotlight: [`${aliceId}:0`],
grid: ["local:0", `${bobId}:0`],
},
f: {
type: "spotlight-landscape",
spotlight: [`${aliceId}:0:screen-share`],
grid: ["local:0", `${bobId}:0`, `${aliceId}:0`],
},
g: {
type: "grid",
spotlight: undefined,
grid: ["local:0", `${bobId}:0`, `${aliceId}:0`],
},
});
expectObservable(vm.showSpeakingIndicators).toBe(showSpeakingMarbles, {
y: true,
n: false,
});
@@ -284,3 +323,200 @@ test("screen sharing activates spotlight layout", () => {
);
});
});
test("participants stay in the same order unless to appear/disappear", () => {
withTestScheduler(({ cold, schedule, expectObservable }) => {
const modeMarbles = "a";
// First Bob speaks, then Dave, then Alice
const aSpeakingMarbles = "n- 1998ms - 1999ms y";
const bSpeakingMarbles = "ny 1998ms n 1999ms ";
const dSpeakingMarbles = "n- 1998ms y 1999ms n";
// Nothing should change when Bob speaks, because Bob is already on screen.
// When Dave speaks he should switch with Alice because she's the one who
// hasn't spoken at all. Then when Alice speaks, she should return to her
// place at the top.
const layoutMarbles = " a 1999ms b 1999ms a 57999ms c 1999ms a";
withCallViewModel(
of([aliceParticipant, bobParticipant, daveParticipant]),
of(ConnectionState.Connected),
new Map([
[aliceParticipant, cold(aSpeakingMarbles, { y: true, n: false })],
[bobParticipant, cold(bSpeakingMarbles, { y: true, n: false })],
[daveParticipant, cold(dSpeakingMarbles, { y: true, n: false })],
]),
(vm) => {
schedule(modeMarbles, {
a: () => {
// We imagine that only three tiles (the first three) will be visible
// on screen at a time
vm.layout.subscribe((layout) => {
if (layout.type === "grid") {
for (let i = 0; i < layout.grid.length; i++)
layout.grid[i].setVisible(i < 3);
}
});
},
});
expectObservable(summarizeLayout(vm.layout)).toBe(layoutMarbles, {
a: {
type: "grid",
spotlight: undefined,
grid: ["local:0", `${aliceId}:0`, `${bobId}:0`, `${daveId}:0`],
},
b: {
type: "grid",
spotlight: undefined,
grid: ["local:0", `${daveId}:0`, `${bobId}:0`, `${aliceId}:0`],
},
c: {
type: "grid",
spotlight: undefined,
grid: ["local:0", `${aliceId}:0`, `${daveId}:0`, `${bobId}:0`],
},
});
},
);
});
});
test("spotlight speakers swap places", () => {
withTestScheduler(({ cold, schedule, expectObservable }) => {
// Go immediately into spotlight mode for the test
const modeMarbles = " s";
// First Bob speaks, then Dave, then Alice
const aSpeakingMarbles = "n--y";
const bSpeakingMarbles = "nyn";
const dSpeakingMarbles = "n-yn";
// Alice should start in the spotlight, then Bob, then Dave, then Alice
// again. However, the positions of Dave and Bob in the grid should be
// reversed by the end because they've been swapped in and out of the
// spotlight.
const layoutMarbles = " abcd";
withCallViewModel(
of([aliceParticipant, bobParticipant, daveParticipant]),
of(ConnectionState.Connected),
new Map([
[aliceParticipant, cold(aSpeakingMarbles, { y: true, n: false })],
[bobParticipant, cold(bSpeakingMarbles, { y: true, n: false })],
[daveParticipant, cold(dSpeakingMarbles, { y: true, n: false })],
]),
(vm) => {
schedule(modeMarbles, { s: () => vm.setGridMode("spotlight") });
expectObservable(summarizeLayout(vm.layout)).toBe(layoutMarbles, {
a: {
type: "spotlight-landscape",
spotlight: [`${aliceId}:0`],
grid: ["local:0", `${bobId}:0`, `${daveId}:0`],
},
b: {
type: "spotlight-landscape",
spotlight: [`${bobId}:0`],
grid: ["local:0", `${aliceId}:0`, `${daveId}:0`],
},
c: {
type: "spotlight-landscape",
spotlight: [`${daveId}:0`],
grid: ["local:0", `${aliceId}:0`, `${bobId}:0`],
},
d: {
type: "spotlight-landscape",
spotlight: [`${aliceId}:0`],
grid: ["local:0", `${daveId}:0`, `${bobId}:0`],
},
});
},
);
});
});
test("layout enters picture-in-picture mode when requested", () => {
withTestScheduler(({ schedule, expectObservable }) => {
// Enable then disable picture-in-picture
const pipControlMarbles = "-ed";
// Should go into picture-in-picture layout then back to grid
const layoutMarbles = " aba";
withCallViewModel(
of([aliceParticipant, bobParticipant]),
of(ConnectionState.Connected),
new Map(),
(vm) => {
schedule(pipControlMarbles, {
e: () => window.controls.enablePip(),
d: () => window.controls.disablePip(),
});
expectObservable(summarizeLayout(vm.layout)).toBe(layoutMarbles, {
a: {
type: "grid",
spotlight: undefined,
grid: ["local:0", `${aliceId}:0`, `${bobId}:0`],
},
b: {
type: "pip",
spotlight: [`${aliceId}:0`],
},
});
},
);
});
});
test("spotlight remembers whether it's expanded", () => {
withTestScheduler(({ schedule, expectObservable }) => {
// Start in spotlight mode, then switch to grid and back to spotlight a
// couple times
const modeMarbles = " s-gs-gs";
// Expand and collapse the spotlight
const expandMarbles = "-a--a";
// Spotlight should stay expanded during the first mode switch, and stay
// collapsed during the second mode switch
const layoutMarbles = "abcbada";
withCallViewModel(
of([aliceParticipant, bobParticipant]),
of(ConnectionState.Connected),
new Map(),
(vm) => {
schedule(modeMarbles, {
s: () => vm.setGridMode("spotlight"),
g: () => vm.setGridMode("grid"),
});
schedule(expandMarbles, {
a: () => {
let toggle: () => void;
vm.toggleSpotlightExpanded.subscribe((val) => (toggle = val!));
toggle!();
},
});
expectObservable(summarizeLayout(vm.layout)).toBe(layoutMarbles, {
a: {
type: "spotlight-landscape",
spotlight: [`${aliceId}:0`],
grid: ["local:0", `${bobId}:0`],
},
b: {
type: "spotlight-expanded",
spotlight: [`${aliceId}:0`],
pip: "local:0",
},
c: {
type: "grid",
spotlight: undefined,
grid: ["local:0", `${aliceId}:0`, `${bobId}:0`],
},
d: {
type: "grid",
spotlight: undefined,
grid: ["local:0", `${bobId}:0`, `${aliceId}:0`],
},
});
},
);
});
});

230
src/state/CallViewModel.ts
View File

@@ -46,7 +46,6 @@ import {
switchMap,
switchScan,
take,
throttleTime,
timer,
withLatestFrom,
} from "rxjs";
@@ -70,6 +69,12 @@ import { ObservableScope } from "./ObservableScope";
import { duplicateTiles } from "../settings/settings";
import { isFirefox } from "../Platform";
import { setPipEnabled } from "../controls";
import { GridTileViewModel, SpotlightTileViewModel } from "./TileViewModel";
import { TileStore } from "./TileStore";
import { gridLikeLayout } from "./GridLikeLayout";
import { spotlightExpandedLayout } from "./SpotlightExpandedLayout";
import { oneOnOneLayout } from "./OneOnOneLayout";
import { pipLayout } from "./PipLayout";
import { EncryptionSystem } from "../e2ee/sharedKeyManagement";
// How long we wait after a focus switch before showing the real participant
@@ -80,39 +85,82 @@ const POST_FOCUS_PARTICIPANT_UPDATE_DELAY_MS = 3000;
// on mobile. No spotlight tile should be shown below this threshold.
const smallMobileCallThreshold = 3;
export interface GridLayout {
export interface GridLayoutMedia {
type: "grid";
spotlight?: MediaViewModel[];
grid: UserMediaViewModel[];
}
export interface SpotlightLandscapeLayout {
export interface SpotlightLandscapeLayoutMedia {
type: "spotlight-landscape";
spotlight: MediaViewModel[];
grid: UserMediaViewModel[];
}
export interface SpotlightPortraitLayout {
export interface SpotlightPortraitLayoutMedia {
type: "spotlight-portrait";
spotlight: MediaViewModel[];
grid: UserMediaViewModel[];
}
export interface SpotlightExpandedLayout {
export interface SpotlightExpandedLayoutMedia {
type: "spotlight-expanded";
spotlight: MediaViewModel[];
pip?: UserMediaViewModel;
}
export interface OneOnOneLayoutMedia {
type: "one-on-one";
local: UserMediaViewModel;
remote: UserMediaViewModel;
}
export interface PipLayoutMedia {
type: "pip";
spotlight: MediaViewModel[];
}
export type LayoutMedia =
| GridLayoutMedia
| SpotlightLandscapeLayoutMedia
| SpotlightPortraitLayoutMedia
| SpotlightExpandedLayoutMedia
| OneOnOneLayoutMedia
| PipLayoutMedia;
export interface GridLayout {
type: "grid";
spotlight?: SpotlightTileViewModel;
grid: GridTileViewModel[];
}
export interface SpotlightLandscapeLayout {
type: "spotlight-landscape";
spotlight: SpotlightTileViewModel;
grid: GridTileViewModel[];
}
export interface SpotlightPortraitLayout {
type: "spotlight-portrait";
spotlight: SpotlightTileViewModel;
grid: GridTileViewModel[];
}
export interface SpotlightExpandedLayout {
type: "spotlight-expanded";
spotlight: SpotlightTileViewModel;
pip?: GridTileViewModel;
}
export interface OneOnOneLayout {
type: "one-on-one";
local: LocalUserMediaViewModel;
remote: RemoteUserMediaViewModel;
local: GridTileViewModel;
remote: GridTileViewModel;
}
export interface PipLayout {
type: "pip";
spotlight: MediaViewModel[];
spotlight: SpotlightTileViewModel;
}
/**
@@ -161,6 +209,12 @@ enum SortingBin {
SelfNotAlwaysShown,
}
interface LayoutScanState {
layout: Layout | null;
tiles: TileStore;
visibleTiles: Set<GridTileViewModel>;
}
class UserMedia {
private readonly scope = new ObservableScope();
public readonly vm: UserMediaViewModel;
@@ -426,12 +480,6 @@ export class CallViewModel extends ViewModel {
this.scope.state(),
);
private readonly hasRemoteScreenShares: Observable<boolean> =
this.screenShares.pipe(
map((ms) => ms.some((m) => !m.vm.local)),
distinctUntilChanged(),
);
private readonly spotlightSpeaker: Observable<UserMediaViewModel> =
this.userMedia.pipe(
switchMap((mediaItems) =>
@@ -466,7 +514,6 @@ export class CallViewModel extends ViewModel {
),
map((speaker) => speaker.vm),
this.scope.state(),
throttleTime(1600, undefined, { leading: true, trailing: true }),
);
private readonly grid: Observable<UserMediaViewModel[]> = this.userMedia.pipe(
@@ -536,6 +583,14 @@ export class CallViewModel extends ViewModel {
this.scope.state(),
);
private readonly hasRemoteScreenShares: Observable<boolean> =
this.spotlight.pipe(
map((spotlight) =>
spotlight.some((vm) => !vm.local && vm instanceof ScreenShareViewModel),
),
distinctUntilChanged(),
);
private readonly pip: Observable<UserMediaViewModel | null> =
this.spotlightAndPip.pipe(switchMap(([, pip]) => pip));
@@ -616,7 +671,7 @@ export class CallViewModel extends ViewModel {
screenShares.length === 0,
);
private readonly gridLayout: Observable<Layout> = combineLatest(
private readonly gridLayout: Observable<LayoutMedia> = combineLatest(
[this.grid, this.spotlight],
(grid, spotlight) => ({
type: "grid",
@@ -627,38 +682,44 @@ export class CallViewModel extends ViewModel {
}),
);
private readonly spotlightLandscapeLayout: Observable<Layout> = combineLatest(
[this.grid, this.spotlight],
(grid, spotlight) => ({ type: "spotlight-landscape", spotlight, grid }),
);
private readonly spotlightLandscapeLayout: Observable<LayoutMedia> =
combineLatest([this.grid, this.spotlight], (grid, spotlight) => ({
type: "spotlight-landscape",
spotlight,
grid,
}));
private readonly spotlightPortraitLayout: Observable<Layout> = combineLatest(
[this.grid, this.spotlight],
(grid, spotlight) => ({ type: "spotlight-portrait", spotlight, grid }),
);
private readonly spotlightPortraitLayout: Observable<LayoutMedia> =
combineLatest([this.grid, this.spotlight], (grid, spotlight) => ({
type: "spotlight-portrait",
spotlight,
grid,
}));
private readonly spotlightExpandedLayout: Observable<Layout> = combineLatest(
[this.spotlight, this.pip],
(spotlight, pip) => ({
private readonly spotlightExpandedLayout: Observable<LayoutMedia> =
combineLatest([this.spotlight, this.pip], (spotlight, pip) => ({
type: "spotlight-expanded",
spotlight,
pip: pip ?? undefined,
}),
);
}));
private readonly oneOnOneLayout: Observable<Layout> = this.grid.pipe(
map((grid) => ({
type: "one-on-one",
local: grid.find((vm) => vm.local) as LocalUserMediaViewModel,
remote: grid.find((vm) => !vm.local) as RemoteUserMediaViewModel,
})),
);
private readonly oneOnOneLayout: Observable<LayoutMedia> =
this.mediaItems.pipe(
map((grid) => ({
type: "one-on-one",
local: grid.find((vm) => vm.vm.local)!.vm as LocalUserMediaViewModel,
remote: grid.find((vm) => !vm.vm.local)!.vm as RemoteUserMediaViewModel,
})),
);
private readonly pipLayout: Observable<Layout> = this.spotlight.pipe(
private readonly pipLayout: Observable<LayoutMedia> = this.spotlight.pipe(
map((spotlight) => ({ type: "pip", spotlight })),
);
public readonly layout: Observable<Layout> = this.windowMode.pipe(
/**
* The media to be used to produce a layout.
*/
private readonly layoutMedia: Observable<LayoutMedia> = this.windowMode.pipe(
switchMap((windowMode) => {
switch (windowMode) {
case "normal":
@@ -719,48 +780,95 @@ export class CallViewModel extends ViewModel {
this.scope.state(),
);
/**
* The layout of tiles in the call interface.
*/
public readonly layout: Observable<Layout> = this.layoutMedia.pipe(
// Each layout will produce a set of tiles, and these tiles have an
// observable indicating whether they're visible. We loop this information
// back into the layout process by using switchScan.
switchScan<
LayoutMedia,
LayoutScanState,
Observable<LayoutScanState & { layout: Layout }>
>(
({ tiles: prevTiles, visibleTiles }, media) => {
let layout: Layout;
let newTiles: TileStore;
switch (media.type) {
case "grid":
case "spotlight-landscape":
case "spotlight-portrait":
[layout, newTiles] = gridLikeLayout(media, visibleTiles, prevTiles);
break;
case "spotlight-expanded":
[layout, newTiles] = spotlightExpandedLayout(
media,
visibleTiles,
prevTiles,
);
break;
case "one-on-one":
[layout, newTiles] = oneOnOneLayout(media, visibleTiles, prevTiles);
break;
case "pip":
[layout, newTiles] = pipLayout(media, visibleTiles, prevTiles);
break;
}
// Take all of the 'visible' observables and combine them into one big
// observable array
const visibilities =
newTiles.gridTiles.length === 0
? of([])
: combineLatest(newTiles.gridTiles.map((tile) => tile.visible));
return visibilities.pipe(
map((visibilities) => ({
layout: layout,
tiles: newTiles,
visibleTiles: new Set(
newTiles.gridTiles.filter((_tile, i) => visibilities[i]),
),
})),
);
},
{
layout: null,
tiles: TileStore.empty(),
visibleTiles: new Set(),
},
),
map(({ layout }) => layout),
this.scope.state(),
);
public showSpotlightIndicators: Observable<boolean> = this.layout.pipe(
map((l) => l.type !== "grid"),
this.scope.state(),
);
/**
* Determines whether video should be shown for a certain piece of media
* appearing in the grid.
*/
public showGridVideo(vm: MediaViewModel): Observable<boolean> {
return this.layout.pipe(
map(
(l) =>
!(
(l.type === "spotlight-landscape" ||
l.type === "spotlight-portrait") &&
// This media is already visible in the spotlight; avoid duplication
l.spotlight.some((spotlightVm) => spotlightVm === vm)
),
),
distinctUntilChanged(),
);
}
public showSpeakingIndicators: Observable<boolean> = this.layout.pipe(
map((l) => {
switchMap((l) => {
switch (l.type) {
case "spotlight-landscape":
case "spotlight-portrait":
// If the spotlight is showing the active speaker, we can do without
// speaking indicators as they're a redundant visual cue. But if
// screen sharing feeds are in the spotlight we still need them.
return l.spotlight[0] instanceof ScreenShareViewModel;
return l.spotlight.media.pipe(
map((models: MediaViewModel[]) =>
models.some((m) => m instanceof ScreenShareViewModel),
),
);
// In expanded spotlight layout, the active speaker is always shown in
// the picture-in-picture tile so there is no need for speaking
// indicators. And in one-on-one layout there's no question as to who is
// speaking.
case "spotlight-expanded":
case "one-on-one":
return false;
return of(false);
default:
return true;
return of(true);
}
}),
this.scope.state(),

43
src/state/GridLikeLayout.ts Normal file
View File

@@ -0,0 +1,43 @@
/*
Copyright 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { Layout, LayoutMedia } from "./CallViewModel";
import { TileStore } from "./TileStore";
import { GridTileViewModel } from "./TileViewModel";
export type GridLikeLayoutType =
| "grid"
| "spotlight-landscape"
| "spotlight-portrait";
/**
* Produces a grid-like layout (any layout with a grid and possibly a spotlight)
* with the given media.
*/
export function gridLikeLayout(
media: LayoutMedia & { type: GridLikeLayoutType },
visibleTiles: Set<GridTileViewModel>,
prevTiles: TileStore,
): [Layout & { type: GridLikeLayoutType }, TileStore] {
const update = prevTiles.from(visibleTiles);
if (media.spotlight !== undefined)
update.registerSpotlight(
media.spotlight,
media.type === "spotlight-portrait",
);
for (const mediaVm of media.grid) update.registerGridTile(mediaVm);
const tiles = update.build();
return [
{
type: media.type,
spotlight: tiles.spotlightTile,
grid: tiles.gridTiles,
} as Layout & { type: GridLikeLayoutType },
tiles,
];
}

32
src/state/OneOnOneLayout.ts Normal file
View File

@@ -0,0 +1,32 @@
/*
Copyright 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { OneOnOneLayout, OneOnOneLayoutMedia } from "./CallViewModel";
import { TileStore } from "./TileStore";
import { GridTileViewModel } from "./TileViewModel";
/**
* Produces a one-on-one layout with the given media.
*/
export function oneOnOneLayout(
media: OneOnOneLayoutMedia,
visibleTiles: Set<GridTileViewModel>,
prevTiles: TileStore,
): [OneOnOneLayout, TileStore] {
const update = prevTiles.from(visibleTiles);
update.registerGridTile(media.local);
update.registerGridTile(media.remote);
const tiles = update.build();
return [
{
type: media.type,
local: tiles.gridTilesByMedia.get(media.local)!,
remote: tiles.gridTilesByMedia.get(media.remote)!,
},
tiles,
];
}

30
src/state/PipLayout.ts Normal file
View File

@@ -0,0 +1,30 @@
/*
Copyright 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { PipLayout, PipLayoutMedia } from "./CallViewModel";
import { TileStore } from "./TileStore";
import { GridTileViewModel } from "./TileViewModel";
/**
* Produces a picture-in-picture layout with the given media.
*/
export function pipLayout(
media: PipLayoutMedia,
visibleTiles: Set<GridTileViewModel>,
prevTiles: TileStore,
): [PipLayout, TileStore] {
const update = prevTiles.from(visibleTiles);
update.registerSpotlight(media.spotlight, true);
const tiles = update.build();
return [
{
type: media.type,
spotlight: tiles.spotlightTile!,
},
tiles,
];
}

36
src/state/SpotlightExpandedLayout.ts Normal file
View File

@@ -0,0 +1,36 @@
/*
Copyright 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import {
SpotlightExpandedLayout,
SpotlightExpandedLayoutMedia,
} from "./CallViewModel";
import { TileStore } from "./TileStore";
import { GridTileViewModel } from "./TileViewModel";
/**
* Produces an expanded spotlight layout with the given media.
*/
export function spotlightExpandedLayout(
media: SpotlightExpandedLayoutMedia,
visibleTiles: Set<GridTileViewModel>,
prevTiles: TileStore,
): [SpotlightExpandedLayout, TileStore] {
const update = prevTiles.from(visibleTiles);
update.registerSpotlight(media.spotlight, true);
if (media.pip !== undefined) update.registerGridTile(media.pip);
const tiles = update.build();
return [
{
type: media.type,
spotlight: tiles.spotlightTile!,
pip: tiles.gridTiles[0],
},
tiles,
];
}

259
src/state/TileStore.ts Normal file
View File

@@ -0,0 +1,259 @@
/*
Copyright 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { BehaviorSubject } from "rxjs";
import { MediaViewModel, UserMediaViewModel } from "./MediaViewModel";
import { GridTileViewModel, SpotlightTileViewModel } from "./TileViewModel";
import { fillGaps } from "../utils/iter";
class SpotlightTileData {
private readonly media_: BehaviorSubject<MediaViewModel[]>;
public get media(): MediaViewModel[] {
return this.media_.value;
}
public set media(value: MediaViewModel[]) {
this.media_.next(value);
}
private readonly maximised_: BehaviorSubject<boolean>;
public get maximised(): boolean {
return this.maximised_.value;
}
public set maximised(value: boolean) {
this.maximised_.next(value);
}
public readonly vm: SpotlightTileViewModel;
public constructor(media: MediaViewModel[], maximised: boolean) {
this.media_ = new BehaviorSubject(media);
this.maximised_ = new BehaviorSubject(maximised);
this.vm = new SpotlightTileViewModel(this.media_, this.maximised_);
}
public destroy(): void {
this.vm.destroy();
}
}
class GridTileData {
private readonly media_: BehaviorSubject<UserMediaViewModel>;
public get media(): UserMediaViewModel {
return this.media_.value;
}
public set media(value: UserMediaViewModel) {
this.media_.next(value);
}
public readonly vm: GridTileViewModel;
public constructor(media: UserMediaViewModel) {
this.media_ = new BehaviorSubject(media);
this.vm = new GridTileViewModel(this.media_);
}
public destroy(): void {
this.vm.destroy();
}
}
/**
* A collection of tiles to be mapped to a layout.
*/
export class TileStore {
private constructor(
private readonly spotlight: SpotlightTileData | null,
private readonly grid: GridTileData[],
) {}
public readonly spotlightTile = this.spotlight?.vm;
public readonly gridTiles = this.grid.map(({ vm }) => vm);
public readonly gridTilesByMedia = new Map(
this.grid.map(({ vm, media }) => [media, vm]),
);
/**
* Creates an an empty collection of tiles.
*/
public static empty(): TileStore {
return new TileStore(null, []);
}
/**
* Creates a builder which can be used to update the collection, passing
* ownership of the tiles to the updated collection.
*/
public from(visibleTiles: Set<GridTileViewModel>): TileStoreBuilder {
return new TileStoreBuilder(
this.spotlight,
this.grid,
(spotlight, grid) => new TileStore(spotlight, grid),
visibleTiles,
);
}
}
/**
* A builder for a new collection of tiles. Will reuse tiles and destroy unused
* tiles from a previous collection where appropriate.
*/
export class TileStoreBuilder {
private spotlight: SpotlightTileData | null = null;
private readonly prevSpotlightSpeaker =
this.prevSpotlight?.media.length === 1 &&
"speaking" in this.prevSpotlight.media[0] &&
this.prevSpotlight.media[0];
private readonly prevGridByMedia = new Map(
this.prevGrid.map((entry, i) => [entry.media, [entry, i]] as const),
);
// The total number of grid entries that we have so far
private numGridEntries = 0;
// A sparse array of grid entries which should be kept in the same spots as
// which they appeared in the previous grid
private readonly stationaryGridEntries: GridTileData[] = new Array(
this.prevGrid.length,
);
// Grid entries which should now enter the visible section of the grid
private readonly visibleGridEntries: GridTileData[] = [];
// Grid entries which should now enter the invisible section of the grid
private readonly invisibleGridEntries: GridTileData[] = [];
public constructor(
private readonly prevSpotlight: SpotlightTileData | null,
private readonly prevGrid: GridTileData[],
private readonly construct: (
spotlight: SpotlightTileData | null,
grid: GridTileData[],
) => TileStore,
private readonly visibleTiles: Set<GridTileViewModel>,
) {}
/**
* Sets the contents of the spotlight tile. If this is never called, there
* will be no spotlight tile.
*/
public registerSpotlight(media: MediaViewModel[], maximised: boolean): void {
if (this.spotlight !== null) throw new Error("Spotlight already set");
if (this.numGridEntries > 0)
throw new Error("Spotlight must be registered before grid tiles");
// Reuse the previous spotlight tile if it exists
if (this.prevSpotlight === null) {
this.spotlight = new SpotlightTileData(media, maximised);
} else {
this.spotlight = this.prevSpotlight;
this.spotlight.media = media;
this.spotlight.maximised = maximised;
}
}
/**
* Sets up a grid tile for the given media. If this is never called for some
* media, then that media will have no grid tile.
*/
public registerGridTile(media: UserMediaViewModel): void {
if (this.spotlight !== null) {
// We actually *don't* want spotlight speakers to appear in both the
// spotlight and the grid, so they're filtered out here
if (!media.local && this.spotlight.media.includes(media)) return;
// When the spotlight speaker changes, we would see one grid tile appear
// and another grid tile disappear. This would be an undesirable layout
// shift, so instead what we do is take the speaker's grid tile and swap
// the media out, so it can remain where it is in the layout.
if (
media === this.prevSpotlightSpeaker &&
this.spotlight.media.length === 1 &&
"speaking" in this.spotlight.media[0] &&
this.prevSpotlightSpeaker !== this.spotlight.media[0]
) {
const prev = this.prevGridByMedia.get(this.spotlight.media[0]);
if (prev !== undefined) {
const [entry, prevIndex] = prev;
const previouslyVisible = this.visibleTiles.has(entry.vm);
const nowVisible = this.visibleTiles.has(
this.prevGrid[this.numGridEntries]?.vm,
);
// If it doesn't need to move between the visible/invisible sections of
// the grid, then we can keep it where it was and swap the media
if (previouslyVisible === nowVisible) {
this.stationaryGridEntries[prevIndex] = entry;
// Do the media swap
entry.media = media;
this.prevGridByMedia.delete(this.spotlight.media[0]);
this.prevGridByMedia.set(media, prev);
} else {
// Create a new tile; this will cause a layout shift but I'm not
// sure there's any other straightforward option in this case
(nowVisible
? this.visibleGridEntries
: this.invisibleGridEntries
).push(new GridTileData(media));
}
this.numGridEntries++;
return;
}
}
}
// Was there previously a tile with this same media?
const prev = this.prevGridByMedia.get(media);
if (prev === undefined) {
// Create a new tile
(this.visibleTiles.has(this.prevGrid[this.numGridEntries]?.vm)
? this.visibleGridEntries
: this.invisibleGridEntries
).push(new GridTileData(media));
} else {
// Reuse the existing tile
const [entry, prevIndex] = prev;
const previouslyVisible = this.visibleTiles.has(entry.vm);
const nowVisible = this.visibleTiles.has(
this.prevGrid[this.numGridEntries]?.vm,
);
// If it doesn't need to move between the visible/invisible sections of
// the grid, then we can keep it exactly where it was previously
if (previouslyVisible === nowVisible)
this.stationaryGridEntries[prevIndex] = entry;
// Otherwise, queue this tile to be moved
else
(nowVisible ? this.visibleGridEntries : this.invisibleGridEntries).push(
entry,
);
}
this.numGridEntries++;
}
/**
* Constructs a new collection of all registered tiles, transferring ownership
* of the tiles to the new collection. Any tiles present in the previous
* collection but not the new collection will be destroyed.
*/
public build(): TileStore {
// Piece together the grid
const grid = [
...fillGaps(this.stationaryGridEntries, [
...this.visibleGridEntries,
...this.invisibleGridEntries,
]),
];
// Destroy unused tiles
if (this.spotlight === null && this.prevSpotlight !== null)
this.prevSpotlight.destroy();
const gridEntries = new Set(grid);
for (const entry of this.prevGrid)
if (!gridEntries.has(entry)) entry.destroy();
return this.construct(this.spotlight, grid);
}
}

43
src/state/TileViewModel.ts Normal file
View File

@@ -0,0 +1,43 @@
/*
Copyright 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { BehaviorSubject, Observable } from "rxjs";
import { ViewModel } from "./ViewModel";
import { MediaViewModel, UserMediaViewModel } from "./MediaViewModel";
let nextId = 0;
function createId(): string {
return (nextId++).toString();
}
export class GridTileViewModel extends ViewModel {
public readonly id = createId();
private readonly visible_ = new BehaviorSubject(false);
/**
* Whether the tile is visible within the current viewport.
*/
public readonly visible: Observable<boolean> = this.visible_;
public setVisible = (value: boolean): void => this.visible_.next(value);
public constructor(public readonly media: Observable<UserMediaViewModel>) {
super();
}
}
export class SpotlightTileViewModel extends ViewModel {
public constructor(
public readonly media: Observable<MediaViewModel[]>,
public readonly maximised: Observable<boolean>,
) {
super();
}
}
export type TileViewModel = GridTileViewModel | SpotlightTileViewModel;

11
src/tile/GridTile.test.tsx
View File

@@ -9,12 +9,20 @@ import { RemoteTrackPublication } from "livekit-client";
import { test, expect } from "vitest";
import { render, screen } from "@testing-library/react";
import { axe } from "vitest-axe";
import { of } from "rxjs";
import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession";
import { GridTile } from "./GridTile";
import { withRemoteMedia } from "../utils/test";
import { GridTileViewModel } from "../state/TileViewModel";
import { ReactionsProvider } from "../useReactions";
global.IntersectionObserver = class MockIntersectionObserver {
public observe(): void {}
public unobserve(): void {}
public disconnect(): void {}
} as unknown as typeof IntersectionObserver;
test("GridTile is accessible", async () => {
await withRemoteMedia(
{
@@ -42,11 +50,10 @@ test("GridTile is accessible", async () => {
const { container } = render(
<ReactionsProvider rtcSession={fakeRtcSession}>
<GridTile
vm={vm}
vm={new GridTileViewModel(of(vm))}
onOpenProfile={() => {}}
targetWidth={300}
targetHeight={200}
showVideo
showSpeakingIndicators
/>
</ReactionsProvider>,

23
src/tile/GridTile.tsx
View File

@@ -10,6 +10,7 @@ import {
ReactNode,
forwardRef,
useCallback,
useRef,
useState,
} from "react";
import { animated } from "@react-spring/web";
@@ -44,6 +45,8 @@ import {
import { Slider } from "../Slider";
import { MediaView } from "./MediaView";
import { useLatest } from "../useLatest";
import { GridTileViewModel } from "../state/TileViewModel";
import { useMergedRefs } from "../useMergedRefs";
import { useReactions } from "../useReactions";
interface TileProps {
@@ -52,7 +55,6 @@ interface TileProps {
targetWidth: number;
targetHeight: number;
displayName: string;
showVideo: boolean;
showSpeakingIndicators: boolean;
}
@@ -67,7 +69,6 @@ const UserMediaTile = forwardRef<HTMLDivElement, UserMediaTileProps>(
(
{
vm,
showVideo,
showSpeakingIndicators,
menuStart,
menuEnd,
@@ -119,7 +120,7 @@ const UserMediaTile = forwardRef<HTMLDivElement, UserMediaTileProps>(
video={video}
member={vm.member}
unencryptedWarning={unencryptedWarning}
videoEnabled={videoEnabled && showVideo}
videoEnabled={videoEnabled}
videoFit={cropVideo ? "cover" : "contain"}
className={classNames(className, styles.tile, {
[styles.speaking]: showSpeaking,
@@ -277,25 +278,27 @@ const RemoteUserMediaTile = forwardRef<
RemoteUserMediaTile.displayName = "RemoteUserMediaTile";
interface GridTileProps {
vm: UserMediaViewModel;
vm: GridTileViewModel;
onOpenProfile: (() => void) | null;
targetWidth: number;
targetHeight: number;
className?: string;
style?: ComponentProps<typeof animated.div>["style"];
showVideo: boolean;
showSpeakingIndicators: boolean;
}
export const GridTile = forwardRef<HTMLDivElement, GridTileProps>(
({ vm, onOpenProfile, ...props }, ref) => {
const displayName = useDisplayName(vm);
({ vm, onOpenProfile, ...props }, theirRef) => {
const ourRef = useRef<HTMLDivElement | null>(null);
const ref = useMergedRefs(ourRef, theirRef);
const media = useObservableEagerState(vm.media);
const displayName = useDisplayName(media);
if (vm instanceof LocalUserMediaViewModel) {
if (media instanceof LocalUserMediaViewModel) {
return (
<LocalUserMediaTile
ref={ref}
vm={vm}
vm={media}
onOpenProfile={onOpenProfile}
displayName={displayName}
{...props}
@@ -305,7 +308,7 @@ export const GridTile = forwardRef<HTMLDivElement, GridTileProps>(
return (
<RemoteUserMediaTile
ref={ref}
vm={vm}
vm={media}
displayName={displayName}
{...props}
/>

5
src/tile/SpotlightTile.test.tsx
View File

@@ -9,9 +9,11 @@ import { test, expect, vi } from "vitest";
import { isInaccessible, render, screen } from "@testing-library/react";
import { axe } from "vitest-axe";
import userEvent from "@testing-library/user-event";
import { of } from "rxjs";
import { SpotlightTile } from "./SpotlightTile";
import { withLocalMedia, withRemoteMedia } from "../utils/test";
import { SpotlightTileViewModel } from "../state/TileViewModel";
global.IntersectionObserver = class MockIntersectionObserver {
public observe(): void {}
@@ -36,10 +38,9 @@ test("SpotlightTile is accessible", async () => {
const toggleExpanded = vi.fn();
const { container } = render(
<SpotlightTile
vms={[vm1, vm2]}
vm={new SpotlightTileViewModel(of([vm1, vm2]), of(false))}
targetWidth={300}
targetHeight={200}
maximised={false}
expanded={false}
onToggleExpanded={toggleExpanded}
showIndicators

51
src/tile/SpotlightTile.tsx
View File

@@ -42,6 +42,7 @@ import { useMergedRefs } from "../useMergedRefs";
import { useObservableRef } from "../state/useObservable";
import { useReactiveState } from "../useReactiveState";
import { useLatest } from "../useLatest";
import { SpotlightTileViewModel } from "../state/TileViewModel";
interface SpotlightItemBaseProps {
className?: string;
@@ -86,14 +87,16 @@ const SpotlightUserMediaItem = forwardRef<
const videoEnabled = useObservableEagerState(vm.videoEnabled);
const cropVideo = useObservableEagerState(vm.cropVideo);
const baseProps: SpotlightUserMediaItemBaseProps = {
const baseProps: SpotlightUserMediaItemBaseProps &
RefAttributes<HTMLDivElement> = {
ref,
videoEnabled,
videoFit: cropVideo ? "cover" : "contain",
...props,
};
return vm instanceof LocalUserMediaViewModel ? (
<SpotlightLocalUserMediaItem ref={ref} vm={vm} {...baseProps} />
<SpotlightLocalUserMediaItem vm={vm} {...baseProps} />
) : (
<MediaView mirror={false} {...baseProps} />
);
@@ -175,8 +178,7 @@ const SpotlightItem = forwardRef<HTMLDivElement, SpotlightItemProps>(
SpotlightItem.displayName = "SpotlightItem";
interface Props {
vms: MediaViewModel[];
maximised: boolean;
vm: SpotlightTileViewModel;
expanded: boolean;
onToggleExpanded: (() => void) | null;
targetWidth: number;
@@ -189,8 +191,7 @@ interface Props {
export const SpotlightTile = forwardRef<HTMLDivElement, Props>(
(
{
vms,
maximised,
vm,
expanded,
onToggleExpanded,
targetWidth,
@@ -204,12 +205,14 @@ export const SpotlightTile = forwardRef<HTMLDivElement, Props>(
const { t } = useTranslation();
const [root, ourRef] = useObservableRef<HTMLDivElement | null>(null);
const ref = useMergedRefs(ourRef, theirRef);
const [visibleId, setVisibleId] = useState(vms[0].id);
const latestVms = useLatest(vms);
const maximised = useObservableEagerState(vm.maximised);
const media = useObservableEagerState(vm.media);
const [visibleId, setVisibleId] = useState(media[0].id);
const latestMedia = useLatest(media);
const latestVisibleId = useLatest(visibleId);
const visibleIndex = vms.findIndex((vm) => vm.id === visibleId);
const visibleIndex = media.findIndex((vm) => vm.id === visibleId);
const canGoBack = visibleIndex > 0;
const canGoToNext = visibleIndex !== -1 && visibleIndex < vms.length - 1;
const canGoToNext = visibleIndex !== -1 && visibleIndex < media.length - 1;
// To keep track of which item is visible, we need an intersection observer
// hooked up to the root element and the items. Because the items will run
@@ -234,28 +237,30 @@ export const SpotlightTile = forwardRef<HTMLDivElement, Props>(
const [scrollToId, setScrollToId] = useReactiveState<string | null>(
(prev) =>
prev == null || prev === visibleId || vms.every((vm) => vm.id !== prev)
prev == null ||
prev === visibleId ||
media.every((vm) => vm.id !== prev)
? null
: prev,
[visibleId],
);
const onBackClick = useCallback(() => {
const vms = latestVms.current;
const visibleIndex = vms.findIndex(
const media = latestMedia.current;
const visibleIndex = media.findIndex(
(vm) => vm.id === latestVisibleId.current,
);
if (visibleIndex > 0) setScrollToId(vms[visibleIndex - 1].id);
}, [latestVisibleId, latestVms, setScrollToId]);
if (visibleIndex > 0) setScrollToId(media[visibleIndex - 1].id);
}, [latestVisibleId, latestMedia, setScrollToId]);
const onNextClick = useCallback(() => {
const vms = latestVms.current;
const visibleIndex = vms.findIndex(
const media = latestMedia.current;
const visibleIndex = media.findIndex(
(vm) => vm.id === latestVisibleId.current,
);
if (visibleIndex !== -1 && visibleIndex !== vms.length - 1)
setScrollToId(vms[visibleIndex + 1].id);
}, [latestVisibleId, latestVms, setScrollToId]);
if (visibleIndex !== -1 && visibleIndex !== media.length - 1)
setScrollToId(media[visibleIndex + 1].id);
}, [latestVisibleId, latestMedia, setScrollToId]);
const ToggleExpandIcon = expanded ? CollapseIcon : ExpandIcon;
@@ -277,7 +282,7 @@ export const SpotlightTile = forwardRef<HTMLDivElement, Props>(
</button>
)}
<div className={styles.contents}>
{vms.map((vm) => (
{media.map((vm) => (
<SpotlightItem
key={vm.id}
vm={vm}
@@ -316,10 +321,10 @@ export const SpotlightTile = forwardRef<HTMLDivElement, Props>(
{!expanded && (
<div
className={classNames(styles.indicators, {
[styles.show]: showIndicators && vms.length > 1,
[styles.show]: showIndicators && media.length > 1,
})}
>
{vms.map((vm) => (
{media.map((vm) => (
<div
key={vm.id}
className={styles.item}

22
src/utils/iter.test.ts Normal file
View File

@@ -0,0 +1,22 @@
/*
Copyright 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { test, expect } from "vitest";
import { fillGaps } from "./iter";
test("fillGaps filters out gaps", () => {
expect([
...fillGaps([1, undefined, undefined, undefined, 3], [2]),
]).toStrictEqual([1, 2, 3]);
});
test("fillGaps adds extra filler elements to the end", () => {
expect([
...fillGaps([1, undefined, 3, undefined], [2, 4, 5, 6]),
]).toStrictEqual([1, 2, 3, 4, 5, 6]);
});

36
src/utils/iter.ts Normal file
View File

@@ -0,0 +1,36 @@
/*
Copyright 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
/**
* Fills in the 'undefined' gaps in a collection by drawing items from a second
* collection, or simply filtering out the gap if no items are left. If filler
* items remain at the end, they will be appended to the resulting collection.
*/
export function fillGaps<A>(
gappy: Iterable<A | undefined>,
filler: Iterable<A>,
): Iterable<A> {
return {
[Symbol.iterator](): Iterator<A> {
const gappyIter = gappy[Symbol.iterator]();
const fillerIter = filler[Symbol.iterator]();
return {
next(): IteratorResult<A> {
let gappyItem: IteratorResult<A | undefined>;
do {
gappyItem = gappyIter.next();
if (!gappyItem.done && gappyItem.value !== undefined)
return gappyItem as IteratorYieldResult<A>;
const fillerItem = fillerIter.next();
if (!fillerItem.done) return fillerItem;
} while (!gappyItem.done);
return gappyItem;
},
};
},
};
}