Matrix/element-call-Github
1
0
Fork 0
mirror of https://github.com/vector-im/element-call.git synced 2026-03-04 05:37:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2894 from robintown/stable-visibility

Browse Source 
Determine which tiles are on screen in a more stable manner
This commit is contained in:
Robin
2024-12-13 09:16:49 -05:00
committed by GitHub
parent 92813bd3ca 53565ddb76
commit 0c66ec5d71
14 changed files with 152 additions and 141 deletions
Download Patch File Download Diff File Expand all files Collapse all files

80
src/grid/Grid.tsx
View File

@@ -24,7 +24,6 @@ import {
createContext,
forwardRef,
memo,
useCallback,
useContext,
useEffect,
useMemo,
@@ -54,7 +53,6 @@ interface Tile<Model> {
id: string;
model: Model;
onDrag: DragCallback | undefined;
setVisible: (visible: boolean) => void;
}
type PlacedTile<Model> = Tile<Model> & Rect;
@@ -88,7 +86,6 @@ interface SlotProps<Model> extends Omit<ComponentProps<"div">, "onDrag"> {
id: string;
model: Model;
onDrag?: DragCallback;
onVisibilityChange?: (visible: boolean) => void;
style?: CSSProperties;
className?: string;
}
@@ -115,24 +112,47 @@ function offset(element: HTMLElement, relativeTo: Element): Offset {
}
}
export type VisibleTilesCallback = (visibleTiles: number) => void;
interface LayoutContext {
setGeneration: Dispatch<SetStateAction<number | null>>;
setVisibleTilesCallback: Dispatch<
SetStateAction<VisibleTilesCallback | null>
>;
}
const LayoutContext = createContext<LayoutContext | null>(null);
function useLayoutContext(): LayoutContext {
const context = useContext(LayoutContext);
if (context === null)
throw new Error("useUpdateLayout called outside a Grid layout context");
return context;
}
/**
* Enables Grid to react to layout changes. You must call this in your Layout
* component or else Grid will not be reactive.
*/
export function useUpdateLayout(): void {
const context = useContext(LayoutContext);
if (context === null)
throw new Error("useUpdateLayout called outside a Grid layout context");
const { setGeneration } = useLayoutContext();
// On every render, tell Grid that the layout may have changed
useEffect(() =>
context.setGeneration((prev) => (prev === null ? 0 : prev + 1)),
useEffect(() => setGeneration((prev) => (prev === null ? 0 : prev + 1)));
}
/**
* Asks Grid to call a callback whenever the number of visible tiles may have
* changed.
*/
export function useVisibleTiles(callback: VisibleTilesCallback): void {
const { setVisibleTilesCallback } = useLayoutContext();
useEffect(
() => setVisibleTilesCallback(() => callback),
[callback, setVisibleTilesCallback],
);
useEffect(
() => (): void => setVisibleTilesCallback(null),
[setVisibleTilesCallback],
);
}
@@ -245,39 +265,20 @@ export function Grid<
const windowHeight = useObservableEagerState(windowHeightObservable);
const [layoutRoot, setLayoutRoot] = useState<HTMLElement | null>(null);
const [generation, setGeneration] = useState<number | null>(null);
const [visibleTilesCallback, setVisibleTilesCallback] =
useState<VisibleTilesCallback | null>(null);
const tiles = useInitial(() => new Map<string, Tile<TileModel>>());
const prefersReducedMotion = usePrefersReducedMotion();
const Slot: FC<SlotProps<TileModel>> = useMemo(
() =>
function Slot({
id,
model,
onDrag,
onVisibilityChange,
style,
className,
...props
}) {
function Slot({ id, model, onDrag, style, className, ...props }) {
const ref = useRef<HTMLDivElement | null>(null);
const prevVisible = useRef<boolean | null>(null);
const setVisible = useCallback(
(visible: boolean) => {
if (
onVisibilityChange !== undefined &&
visible !== prevVisible.current
) {
onVisibilityChange(visible);
prevVisible.current = visible;
}
},
[onVisibilityChange],
);
useEffect(() => {
tiles.set(id, { id, model, onDrag, setVisible });
tiles.set(id, { id, model, onDrag });
return (): void => void tiles.delete(id);
}, [id, model, onDrag, setVisible]);
}, [id, model, onDrag]);
return (
<div
@@ -307,7 +308,10 @@ export function Grid<
[],
);
const context: LayoutContext = useMemo(() => ({ setGeneration }), []);
const context: LayoutContext = useMemo(
() => ({ setGeneration, setVisibleTilesCallback }),
[setVisibleTilesCallback],
);
// Combine the tile definitions and slots together to create placed tiles
const placedTiles = useMemo(() => {
@@ -342,9 +346,11 @@ export function Grid<
);
useEffect(() => {
for (const tile of placedTiles)
tile.setVisible(tile.y + tile.height <= visibleHeight);
}, [placedTiles, visibleHeight]);
visibleTilesCallback?.(
placedTiles.filter((tile) => tile.y + tile.height <= visibleHeight)
.length,
);
}, [placedTiles, visibleTilesCallback, visibleHeight]);
// Drag state is stored in a ref rather than component state, because we use
// react-spring's imperative API during gestures to improve responsiveness

11
src/grid/GridLayout.tsx
View File

@@ -13,7 +13,7 @@ import { type GridLayout as GridLayoutModel } from "../state/CallViewModel";
import styles from "./GridLayout.module.css";
import { useInitial } from "../useInitial";
import { type CallLayout, arrangeTiles } from "./CallLayout";
import { type DragCallback, useUpdateLayout } from "./Grid";
import { type DragCallback, useUpdateLayout, useVisibleTiles } from "./Grid";
interface GridCSSProperties extends CSSProperties {
"--gap": string;
@@ -73,6 +73,7 @@ export const makeGridLayout: CallLayout<GridLayoutModel> = ({
// The scrolling part of the layout is where all the grid tiles live
scrolling: forwardRef(function GridLayout({ model, Slot }, ref) {
useUpdateLayout();
useVisibleTiles(model.setVisibleTiles);
const { width, height: minHeight } = useObservableEagerState(minBounds);
const { gap, tileWidth, tileHeight } = useMemo(
() => arrangeTiles(width, minHeight, model.grid.length),
@@ -93,13 +94,7 @@ export const makeGridLayout: CallLayout<GridLayoutModel> = ({
}
>
{model.grid.map((m) => (
<Slot
key={m.id}
className={styles.slot}
id={m.id}
model={m}
onVisibilityChange={m.setVisible}
/>
<Slot key={m.id} className={styles.slot} id={m.id} model={m} />
))}
</div>
);

2
src/grid/OneOnOneLayout.tsx
View File

@@ -52,7 +52,6 @@ export const makeOneOnOneLayout: CallLayout<OneOnOneLayoutModel> = ({
<Slot
id={model.remote.id}
model={model.remote}
onVisibilityChange={model.remote.setVisible}
className={styles.container}
style={{ width: tileWidth, height: tileHeight }}
>
@@ -61,7 +60,6 @@ export const makeOneOnOneLayout: CallLayout<OneOnOneLayoutModel> = ({
id={model.local.id}
model={model.local}
onDrag={onDragLocalTile}
onVisibilityChange={model.local.setVisible}
data-block-alignment={pipAlignmentValue.block}
data-inline-alignment={pipAlignmentValue.inline}
/>

1
src/grid/SpotlightExpandedLayout.tsx
View File

@@ -63,7 +63,6 @@ export const makeSpotlightExpandedLayout: CallLayout<
id={model.pip.id}
model={model.pip}
onDrag={onDragPip}
onVisibilityChange={model.pip.setVisible}
data-block-alignment={pipAlignmentValue.block}
data-inline-alignment={pipAlignmentValue.inline}
/>

11
src/grid/SpotlightLandscapeLayout.tsx
View File

@@ -12,7 +12,7 @@ import classNames from "classnames";
import { type CallLayout } from "./CallLayout";
import { type SpotlightLandscapeLayout as SpotlightLandscapeLayoutModel } from "../state/CallViewModel";
import styles from "./SpotlightLandscapeLayout.module.css";
import { useUpdateLayout } from "./Grid";
import { useUpdateLayout, useVisibleTiles } from "./Grid";
/**
* An implementation of the "spotlight landscape" layout, in which the spotlight
@@ -50,6 +50,7 @@ export const makeSpotlightLandscapeLayout: CallLayout<
ref,
) {
useUpdateLayout();
useVisibleTiles(model.setVisibleTiles);
useObservableEagerState(minBounds);
const withIndicators =
useObservableEagerState(model.spotlight.media).length > 1;
@@ -63,13 +64,7 @@ export const makeSpotlightLandscapeLayout: CallLayout<
/>
<div className={styles.grid}>
{model.grid.map((m) => (
<Slot
key={m.id}
className={styles.slot}
id={m.id}
model={m}
onVisibilityChange={m.setVisible}
/>
<Slot key={m.id} className={styles.slot} id={m.id} model={m} />
))}
</div>
</div>

11
src/grid/SpotlightPortraitLayout.tsx
View File

@@ -12,7 +12,7 @@ import classNames from "classnames";
import { type CallLayout, arrangeTiles } from "./CallLayout";
import { type SpotlightPortraitLayout as SpotlightPortraitLayoutModel } from "../state/CallViewModel";
import styles from "./SpotlightPortraitLayout.module.css";
import { useUpdateLayout } from "./Grid";
import { useUpdateLayout, useVisibleTiles } from "./Grid";
interface GridCSSProperties extends CSSProperties {
"--grid-gap": string;
@@ -54,6 +54,7 @@ export const makeSpotlightPortraitLayout: CallLayout<
ref,
) {
useUpdateLayout();
useVisibleTiles(model.setVisibleTiles);
const { width } = useObservableEagerState(minBounds);
const { gap, tileWidth, tileHeight } = arrangeTiles(
width,
@@ -84,13 +85,7 @@ export const makeSpotlightPortraitLayout: CallLayout<
/>
<div className={styles.grid}>
{model.grid.map((m) => (
<Slot
key={m.id}
className={styles.slot}
id={m.id}
model={m}
onVisibilityChange={m.setVisible}
/>
<Slot key={m.id} className={styles.slot} id={m.id} model={m} />
))}
</div>
</div>

67
src/state/CallViewModel.test.ts
View File

@@ -376,16 +376,16 @@ test("screen sharing activates spotlight layout", () => {
test("participants stay in the same order unless to appear/disappear", () => {
withTestScheduler(({ hot, schedule, expectObservable }) => {
const modeInputMarbles = " a";
const visibilityInputMarbles = "a";
// First Bob speaks, then Dave, then Alice
const aSpeakingInputMarbles = "n- 1998ms - 1999ms y";
const bSpeakingInputMarbles = "ny 1998ms n 1999ms -";
const dSpeakingInputMarbles = "n- 1998ms y 1999ms n";
const aSpeakingInputMarbles = " n- 1998ms - 1999ms y";
const bSpeakingInputMarbles = " ny 1998ms n 1999ms -";
const dSpeakingInputMarbles = " n- 1998ms y 1999ms n";
// Nothing should change when Bob speaks, because Bob is already on screen.
// When Dave speaks he should switch with Alice because she's the one who
// hasn't spoken at all. Then when Alice speaks, she should return to her
// place at the top.
const expectedLayoutMarbles = "a 1999ms b 1999ms a 57999ms c 1999ms a";
const expectedLayoutMarbles = " a 1999ms b 1999ms a 57999ms c 1999ms a";
withCallViewModel(
of([aliceParticipant, bobParticipant, daveParticipant]),
@@ -397,15 +397,12 @@ test("participants stay in the same order unless to appear/disappear", () => {
[daveParticipant, hot(dSpeakingInputMarbles, { y: true, n: false })],
]),
(vm) => {
schedule(modeInputMarbles, {
schedule(visibilityInputMarbles, {
a: () => {
// We imagine that only three tiles (the first three) will be visible
// on screen at a time
vm.layout.subscribe((layout) => {
if (layout.type === "grid") {
for (let i = 0; i < layout.grid.length; i++)
layout.grid[i].setVisible(i < 3);
}
if (layout.type === "grid") layout.setVisibleTiles(3);
});
},
});
@@ -435,6 +432,56 @@ test("participants stay in the same order unless to appear/disappear", () => {
});
});
test("participants adjust order when space becomes constrained", () => {
withTestScheduler(({ hot, schedule, expectObservable }) => {
// Start with all tiles on screen then shrink to 3
const visibilityInputMarbles = "a-b";
// Bob and Dave speak
const bSpeakingInputMarbles = " ny";
const dSpeakingInputMarbles = " ny";
// Nothing should change when Bob or Dave initially speak, because they are
// on screen. When the screen becomes smaller Alice should move off screen
// to make way for the speakers (specifically, she should swap with Dave).
const expectedLayoutMarbles = " a-b";
withCallViewModel(
of([aliceParticipant, bobParticipant, daveParticipant]),
of([aliceRtcMember, bobRtcMember, daveRtcMember]),
of(ConnectionState.Connected),
new Map([
[bobParticipant, hot(bSpeakingInputMarbles, { y: true, n: false })],
[daveParticipant, hot(dSpeakingInputMarbles, { y: true, n: false })],
]),
(vm) => {
let setVisibleTiles: ((value: number) => void) | null = null;
vm.layout.subscribe((layout) => {
if (layout.type === "grid") setVisibleTiles = layout.setVisibleTiles;
});
schedule(visibilityInputMarbles, {
a: () => setVisibleTiles!(Infinity),
b: () => setVisibleTiles!(3),
});
expectObservable(summarizeLayout(vm.layout)).toBe(
expectedLayoutMarbles,
{
a: {
type: "grid",
spotlight: undefined,
grid: ["local:0", `${aliceId}:0`, `${bobId}:0`, `${daveId}:0`],
},
b: {
type: "grid",
spotlight: undefined,
grid: ["local:0", `${daveId}:0`, `${bobId}:0`, `${aliceId}:0`],
},
},
);
},
);
});
});
test("spotlight speakers swap places", () => {
withTestScheduler(({ hot, schedule, expectObservable }) => {
// Go immediately into spotlight mode for the test

65
src/state/CallViewModel.ts
View File

@@ -143,18 +143,21 @@ export interface GridLayout {
type: "grid";
spotlight?: SpotlightTileViewModel;
grid: GridTileViewModel[];
setVisibleTiles: (value: number) => void;
}
export interface SpotlightLandscapeLayout {
type: "spotlight-landscape";
spotlight: SpotlightTileViewModel;
grid: GridTileViewModel[];
setVisibleTiles: (value: number) => void;
}
export interface SpotlightPortraitLayout {
type: "spotlight-portrait";
spotlight: SpotlightTileViewModel;
grid: GridTileViewModel[];
setVisibleTiles: (value: number) => void;
}
export interface SpotlightExpandedLayout {
@@ -223,7 +226,6 @@ enum SortingBin {
interface LayoutScanState {
layout: Layout | null;
tiles: TileStore;
visibleTiles: Set<GridTileViewModel>;
}
class UserMedia {
@@ -891,62 +893,53 @@ export class CallViewModel extends ViewModel {
this.scope.state(),
);
// There is a cyclical dependency here: the layout algorithms want to know
// which tiles are on screen, but to know which tiles are on screen we have to
// first render a layout. To deal with this we assume initially that no tiles
// are visible, and loop the data back into the layouts with a Subject.
private readonly visibleTiles = new Subject<number>();
private readonly setVisibleTiles = (value: number): void =>
this.visibleTiles.next(value);
public readonly layoutInternals: Observable<
LayoutScanState & { layout: Layout }
> = this.layoutMedia.pipe(
// Each layout will produce a set of tiles, and these tiles have an
// observable indicating whether they're visible. We loop this information
// back into the layout process by using switchScan.
switchScan<
LayoutMedia,
LayoutScanState,
Observable<LayoutScanState & { layout: Layout }>
> = combineLatest([
this.layoutMedia,
this.visibleTiles.pipe(startWith(0), distinctUntilChanged()),
]).pipe(
scan<
[LayoutMedia, number],
LayoutScanState & { layout: Layout },
LayoutScanState
>(
({ tiles: prevTiles, visibleTiles }, media) => {
({ tiles: prevTiles }, [media, visibleTiles]) => {
let layout: Layout;
let newTiles: TileStore;
switch (media.type) {
case "grid":
case "spotlight-landscape":
case "spotlight-portrait":
[layout, newTiles] = gridLikeLayout(media, visibleTiles, prevTiles);
break;
case "spotlight-expanded":
[layout, newTiles] = spotlightExpandedLayout(
[layout, newTiles] = gridLikeLayout(
media,
visibleTiles,
this.setVisibleTiles,
prevTiles,
);
break;
case "spotlight-expanded":
[layout, newTiles] = spotlightExpandedLayout(media, prevTiles);
break;
case "one-on-one":
[layout, newTiles] = oneOnOneLayout(media, visibleTiles, prevTiles);
[layout, newTiles] = oneOnOneLayout(media, prevTiles);
break;
case "pip":
[layout, newTiles] = pipLayout(media, visibleTiles, prevTiles);
[layout, newTiles] = pipLayout(media, prevTiles);
break;
}
// Take all of the 'visible' observables and combine them into one big
// observable array
const visibilities =
newTiles.gridTiles.length === 0
? of([])
: combineLatest(newTiles.gridTiles.map((tile) => tile.visible));
return visibilities.pipe(
map((visibilities) => ({
layout: layout,
tiles: newTiles,
visibleTiles: new Set(
newTiles.gridTiles.filter((_tile, i) => visibilities[i]),
),
})),
);
},
{
layout: null,
tiles: TileStore.empty(),
visibleTiles: new Set(),
return { layout, tiles: newTiles };
},
{ layout: null, tiles: TileStore.empty() },
),
this.scope.state(),
);

5
src/state/GridLikeLayout.ts
View File

@@ -7,7 +7,6 @@ Please see LICENSE in the repository root for full details.
import { type Layout, type LayoutMedia } from "./CallViewModel";
import { type TileStore } from "./TileStore";
import { type GridTileViewModel } from "./TileViewModel";
export type GridLikeLayoutType =
| "grid"
@@ -20,7 +19,8 @@ export type GridLikeLayoutType =
*/
export function gridLikeLayout(
media: LayoutMedia & { type: GridLikeLayoutType },
visibleTiles: Set<GridTileViewModel>,
visibleTiles: number,
setVisibleTiles: (value: number) => void,
prevTiles: TileStore,
): [Layout & { type: GridLikeLayoutType }, TileStore] {
const update = prevTiles.from(visibleTiles);
@@ -37,6 +37,7 @@ export function gridLikeLayout(
type: media.type,
spotlight: tiles.spotlightTile,
grid: tiles.gridTiles,
setVisibleTiles,
} as Layout & { type: GridLikeLayoutType },
tiles,
];

4
src/state/OneOnOneLayout.ts
View File

@@ -7,17 +7,15 @@ Please see LICENSE in the repository root for full details.
import { type OneOnOneLayout, type OneOnOneLayoutMedia } from "./CallViewModel";
import { type TileStore } from "./TileStore";
import { type GridTileViewModel } from "./TileViewModel";
/**
* Produces a one-on-one layout with the given media.
*/
export function oneOnOneLayout(
media: OneOnOneLayoutMedia,
visibleTiles: Set<GridTileViewModel>,
prevTiles: TileStore,
): [OneOnOneLayout, TileStore] {
const update = prevTiles.from(visibleTiles);
const update = prevTiles.from(2);
update.registerGridTile(media.local);
update.registerGridTile(media.remote);
const tiles = update.build();

4
src/state/PipLayout.ts
View File

@@ -7,17 +7,15 @@ Please see LICENSE in the repository root for full details.
import { type PipLayout, type PipLayoutMedia } from "./CallViewModel";
import { type TileStore } from "./TileStore";
import { type GridTileViewModel } from "./TileViewModel";
/**
* Produces a picture-in-picture layout with the given media.
*/
export function pipLayout(
media: PipLayoutMedia,
visibleTiles: Set<GridTileViewModel>,
prevTiles: TileStore,
): [PipLayout, TileStore] {
const update = prevTiles.from(visibleTiles);
const update = prevTiles.from(0);
update.registerSpotlight(media.spotlight, true);
const tiles = update.build();
return [

4
src/state/SpotlightExpandedLayout.ts
View File

@@ -10,17 +10,15 @@ import {
type SpotlightExpandedLayoutMedia,
} from "./CallViewModel";
import { type TileStore } from "./TileStore";
import { type GridTileViewModel } from "./TileViewModel";
/**
* Produces an expanded spotlight layout with the given media.
*/
export function spotlightExpandedLayout(
media: SpotlightExpandedLayoutMedia,
visibleTiles: Set<GridTileViewModel>,
prevTiles: TileStore,
): [SpotlightExpandedLayout, TileStore] {
const update = prevTiles.from(visibleTiles);
const update = prevTiles.from(1);
update.registerSpotlight(media.spotlight, true);
if (media.pip !== undefined) update.registerGridTile(media.pip);
const tiles = update.build();

18
src/state/TileStore.ts
View File

@@ -101,7 +101,7 @@ export class TileStore {
* Creates a builder which can be used to update the collection, passing
* ownership of the tiles to the updated collection.
*/
public from(visibleTiles: Set<GridTileViewModel>): TileStoreBuilder {
public from(visibleTiles: number): TileStoreBuilder {
return new TileStoreBuilder(
this.spotlight,
this.grid,
@@ -146,7 +146,7 @@ export class TileStoreBuilder {
spotlight: SpotlightTileData | null,
grid: GridTileData[],
) => TileStore,
private readonly visibleTiles: Set<GridTileViewModel>,
private readonly visibleTiles: number,
/**
* A number incremented on each update, just for debugging purposes.
*/
@@ -204,10 +204,8 @@ export class TileStoreBuilder {
const prev = this.prevGridByMedia.get(this.spotlight.media[0]);
if (prev !== undefined) {
const [entry, prevIndex] = prev;
const previouslyVisible = this.visibleTiles.has(entry.vm);
const nowVisible = this.visibleTiles.has(
this.prevGrid[this.numGridEntries]?.vm,
);
const previouslyVisible = prevIndex < this.visibleTiles;
const nowVisible = this.numGridEntries < this.visibleTiles;
// If it doesn't need to move between the visible/invisible sections of
// the grid, then we can keep it where it was and swap the media
@@ -236,17 +234,15 @@ export class TileStoreBuilder {
const prev = this.prevGridByMedia.get(media);
if (prev === undefined) {
// Create a new tile
(this.visibleTiles.has(this.prevGrid[this.numGridEntries]?.vm)
(this.numGridEntries < this.visibleTiles
? this.visibleGridEntries
: this.invisibleGridEntries
).push(new GridTileData(media));
} else {
// Reuse the existing tile
const [entry, prevIndex] = prev;
const previouslyVisible = this.visibleTiles.has(entry.vm);
const nowVisible = this.visibleTiles.has(
this.prevGrid[this.numGridEntries]?.vm,
);
const previouslyVisible = prevIndex < this.visibleTiles;
const nowVisible = this.numGridEntries < this.visibleTiles;
// If it doesn't need to move between the visible/invisible sections of
// the grid, then we can keep it exactly where it was previously
if (previouslyVisible === nowVisible)

10
src/state/TileViewModel.ts
View File

@@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { BehaviorSubject, type Observable } from "rxjs";
import { type Observable } from "rxjs";
import { ViewModel } from "./ViewModel";
import { type MediaViewModel, type UserMediaViewModel } from "./MediaViewModel";
@@ -18,14 +18,6 @@ function createId(): string {
export class GridTileViewModel extends ViewModel {
public readonly id = createId();
private readonly visible_ = new BehaviorSubject(false);
/**
* Whether the tile is visible within the current viewport.
*/
public readonly visible: Observable<boolean> = this.visible_;
public setVisible = (value: boolean): void => this.visible_.next(value);
public constructor(public readonly media: Observable<UserMediaViewModel>) {
super();
}