Matrix/element-call-Github
1
0
Fork 0
mirror of https://github.com/vector-im/element-call.git synced 2026-03-31 07:00:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'livekit' into toger5/tiles_based_on_rtc_member

Browse Source
This commit is contained in:
Hugh Nimmo-Smith
2024-11-06 13:42:45 +00:00
parent fecd789d02 c45f724279
commit 733b685e96
57 changed files with 2159 additions and 1101 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
src/button/RaisedHandToggleButton.tsx
View File

@@ -62,7 +62,7 @@ export function RaiseHandToggleButton({
client,
rtcSession,
}: RaisedHandToggleButtonProps): ReactNode {
const { raisedHands, myReactionId } = useReactions();
const { raisedHands, lowerHand } = useReactions();
const [busy, setBusy] = useState(false);
const userId = client.getUserId()!;
const isHandRaised = !!raisedHands[userId];
@@ -71,16 +71,9 @@ export function RaiseHandToggleButton({
const toggleRaisedHand = useCallback(() => {
const raiseHand = async (): Promise<void> => {
if (isHandRaised) {
if (!myReactionId) {
logger.warn(`Hand raised but no reaction event to redact!`);
return;
}
try {
setBusy(true);
await client.redactEvent(rtcSession.room.roomId, myReactionId);
logger.debug("Redacted raise hand event");
} catch (ex) {
logger.error("Failed to redact reaction event", myReactionId, ex);
await lowerHand();
} finally {
setBusy(false);
}
@@ -118,9 +111,9 @@ export function RaiseHandToggleButton({
client,
isHandRaised,
memberships,
myReactionId,
rtcSession.room.roomId,
userId,
lowerHand,
]);
return (

3
src/config/ConfigOptions.ts
View File

@@ -122,6 +122,9 @@ export const DEFAULT_CONFIG: ResolvedConfigOptions = {
server_name: "localhost",
},
},
features: {
feature_use_device_session_member_events: true,
},
eula: "https://static.element.io/legal/online-EULA.pdf",
media_devices: {
enable_audio: true,

19
src/grid/CallLayout.ts
View File

@@ -8,8 +8,8 @@ Please see LICENSE in the repository root for full details.
import { BehaviorSubject, Observable } from "rxjs";
import { ComponentType } from "react";
import { MediaViewModel, UserMediaViewModel } from "../state/MediaViewModel";
import { LayoutProps } from "./Grid";
import { TileViewModel } from "../state/TileViewModel";
export interface Bounds {
width: number;
@@ -42,19 +42,6 @@ export interface CallLayoutInputs {
pipAlignment: BehaviorSubject<Alignment>;
}
export interface GridTileModel {
type: "grid";
vm: UserMediaViewModel;
}
export interface SpotlightTileModel {
type: "spotlight";
vms: MediaViewModel[];
maximised: boolean;
}
export type TileModel = GridTileModel | SpotlightTileModel;
export interface CallLayoutOutputs<Model> {
/**
* Whether the scrolling layer of the layout should appear on top.
@@ -63,11 +50,11 @@ export interface CallLayoutOutputs<Model> {
/**
* The visually fixed (non-scrolling) layer of the layout.
*/
fixed: ComponentType<LayoutProps<Model, TileModel, HTMLDivElement>>;
fixed: ComponentType<LayoutProps<Model, TileViewModel, HTMLDivElement>>;
/**
* The layer of the layout that can overflow and be scrolled.
*/
scrolling: ComponentType<LayoutProps<Model, TileModel, HTMLDivElement>>;
scrolling: ComponentType<LayoutProps<Model, TileViewModel, HTMLDivElement>>;
}
/**

50
src/grid/Grid.tsx
View File

@@ -24,6 +24,7 @@ import {
createContext,
forwardRef,
memo,
useCallback,
useContext,
useEffect,
useMemo,
@@ -33,6 +34,8 @@ import {
import useMeasure from "react-use-measure";
import classNames from "classnames";
import { logger } from "matrix-js-sdk/src/logger";
import { useObservableEagerState } from "observable-hooks";
import { fromEvent, map, startWith } from "rxjs";
import styles from "./Grid.module.css";
import { useMergedRefs } from "../useMergedRefs";
@@ -51,6 +54,7 @@ interface Tile<Model> {
id: string;
model: Model;
onDrag: DragCallback | undefined;
setVisible: (visible: boolean) => void;
}
type PlacedTile<Model> = Tile<Model> & Rect;
@@ -84,6 +88,7 @@ interface SlotProps<Model> extends Omit<ComponentProps<"div">, "onDrag"> {
id: string;
model: Model;
onDrag?: DragCallback;
onVisibilityChange?: (visible: boolean) => void;
style?: CSSProperties;
className?: string;
}
@@ -131,6 +136,11 @@ export function useUpdateLayout(): void {
);
}
const windowHeightObservable = fromEvent(window, "resize").pipe(
startWith(null),
map(() => window.innerHeight),
);
export interface LayoutProps<LayoutModel, TileModel, R extends HTMLElement> {
ref: LegacyRef<R>;
model: LayoutModel;
@@ -232,6 +242,7 @@ export function Grid<
const [gridRoot, gridRef2] = useState<HTMLElement | null>(null);
const gridRef = useMergedRefs<HTMLElement>(gridRef1, gridRef2);
const windowHeight = useObservableEagerState(windowHeightObservable);
const [layoutRoot, setLayoutRoot] = useState<HTMLElement | null>(null);
const [generation, setGeneration] = useState<number | null>(null);
const tiles = useInitial(() => new Map<string, Tile<TileModel>>());
@@ -239,12 +250,34 @@ export function Grid<
const Slot: FC<SlotProps<TileModel>> = useMemo(
() =>
function Slot({ id, model, onDrag, style, className, ...props }) {
function Slot({
id,
model,
onDrag,
onVisibilityChange,
style,
className,
...props
}) {
const ref = useRef<HTMLDivElement | null>(null);
const prevVisible = useRef<boolean | null>(null);
const setVisible = useCallback(
(visible: boolean) => {
if (
onVisibilityChange !== undefined &&
visible !== prevVisible.current
) {
onVisibilityChange(visible);
prevVisible.current = visible;
}
},
[onVisibilityChange],
);
useEffect(() => {
tiles.set(id, { id, model, onDrag });
tiles.set(id, { id, model, onDrag, setVisible });
return (): void => void tiles.delete(id);
}, [id, model, onDrag]);
}, [id, model, onDrag, setVisible]);
return (
<div
@@ -302,6 +335,17 @@ export function Grid<
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [gridRoot, layoutRoot, tiles, gridBounds, generation]);
// The height of the portion of the grid visible at any given time
const visibleHeight = useMemo(
() => Math.min(gridBounds.bottom, windowHeight) - gridBounds.top,
[gridBounds, windowHeight],
);
useEffect(() => {
for (const tile of placedTiles)
tile.setVisible(tile.y + tile.height <= visibleHeight);
}, [placedTiles, visibleHeight]);
// Drag state is stored in a ref rather than component state, because we use
// react-spring's imperative API during gestures to improve responsiveness
const dragState = useRef<DragState | null>(null);

35
src/grid/GridLayout.tsx
View File

@@ -12,12 +12,7 @@ import { useObservableEagerState } from "observable-hooks";
import { GridLayout as GridLayoutModel } from "../state/CallViewModel";
import styles from "./GridLayout.module.css";
import { useInitial } from "../useInitial";
import {
CallLayout,
GridTileModel,
TileModel,
arrangeTiles,
} from "./CallLayout";
import { CallLayout, arrangeTiles } from "./CallLayout";
import { DragCallback, useUpdateLayout } from "./Grid";
interface GridCSSProperties extends CSSProperties {
@@ -49,15 +44,6 @@ export const makeGridLayout: CallLayout<GridLayoutModel> = ({
),
),
);
const tileModel: TileModel | undefined = useMemo(
() =>
model.spotlight && {
type: "spotlight",
vms: model.spotlight,
maximised: false,
},
[model.spotlight],
);
const onDragSpotlight: DragCallback = useCallback(
({ xRatio, yRatio }) =>
@@ -70,11 +56,11 @@ export const makeGridLayout: CallLayout<GridLayoutModel> = ({
return (
<div ref={ref} className={styles.fixed}>
{tileModel && (
{model.spotlight && (
<Slot
className={styles.slot}
id="spotlight"
model={tileModel}
model={model.spotlight}
onDrag={onDragSpotlight}
data-block-alignment={alignment.block}
data-inline-alignment={alignment.inline}
@@ -93,11 +79,6 @@ export const makeGridLayout: CallLayout<GridLayoutModel> = ({
[width, minHeight, model.grid.length],
);
const tileModels: GridTileModel[] = useMemo(
() => model.grid.map((vm) => ({ type: "grid", vm })),
[model.grid],
);
return (
<div
ref={ref}
@@ -111,8 +92,14 @@ export const makeGridLayout: CallLayout<GridLayoutModel> = ({
} as GridCSSProperties
}
>
{tileModels.map((m) => (
<Slot key={m.vm.id} className={styles.slot} id={m.vm.id} model={m} />
{model.grid.map((m) => (
<Slot
key={m.id}
className={styles.slot}
id={m.id}
model={m}
onVisibilityChange={m.setVisible}
/>
))}
</div>
);

21
src/grid/OneOnOneLayout.tsx
View File

@@ -10,7 +10,7 @@ import { useObservableEagerState } from "observable-hooks";
import classNames from "classnames";
import { OneOnOneLayout as OneOnOneLayoutModel } from "../state/CallViewModel";
import { CallLayout, GridTileModel, arrangeTiles } from "./CallLayout";
import { CallLayout, arrangeTiles } from "./CallLayout";
import styles from "./OneOnOneLayout.module.css";
import { DragCallback, useUpdateLayout } from "./Grid";
@@ -38,15 +38,6 @@ export const makeOneOnOneLayout: CallLayout<OneOnOneLayoutModel> = ({
[width, height],
);
const remoteTileModel: GridTileModel = useMemo(
() => ({ type: "grid", vm: model.remote }),
[model.remote],
);
const localTileModel: GridTileModel = useMemo(
() => ({ type: "grid", vm: model.local }),
[model.local],
);
const onDragLocalTile: DragCallback = useCallback(
({ xRatio, yRatio }) =>
pipAlignment.next({
@@ -59,16 +50,18 @@ export const makeOneOnOneLayout: CallLayout<OneOnOneLayoutModel> = ({
return (
<div ref={ref} className={styles.layer}>
<Slot
id={remoteTileModel.vm.id}
model={remoteTileModel}
id={model.remote.id}
model={model.remote}
onVisibilityChange={model.remote.setVisible}
className={styles.container}
style={{ width: tileWidth, height: tileHeight }}
>
<Slot
className={classNames(styles.slot, styles.local)}
id={localTileModel.vm.id}
model={localTileModel}
id={model.local.id}
model={model.local}
onDrag={onDragLocalTile}
onVisibilityChange={model.local.setVisible}
data-block-alignment={pipAlignmentValue.block}
data-inline-alignment={pipAlignmentValue.inline}
/>

22
src/grid/SpotlightExpandedLayout.tsx
View File

@@ -5,11 +5,11 @@ SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { forwardRef, useCallback, useMemo } from "react";
import { forwardRef, useCallback } from "react";
import { useObservableEagerState } from "observable-hooks";
import { SpotlightExpandedLayout as SpotlightExpandedLayoutModel } from "../state/CallViewModel";
import { CallLayout, GridTileModel, SpotlightTileModel } from "./CallLayout";
import { CallLayout } from "./CallLayout";
import { DragCallback, useUpdateLayout } from "./Grid";
import styles from "./SpotlightExpandedLayout.module.css";
@@ -27,17 +27,13 @@ export const makeSpotlightExpandedLayout: CallLayout<
ref,
) {
useUpdateLayout();
const spotlightTileModel: SpotlightTileModel = useMemo(
() => ({ type: "spotlight", vms: model.spotlight, maximised: true }),
[model.spotlight],
);
return (
<div ref={ref} className={styles.layer}>
<Slot
className={styles.spotlight}
id="spotlight"
model={spotlightTileModel}
model={model.spotlight}
/>
</div>
);
@@ -50,11 +46,6 @@ export const makeSpotlightExpandedLayout: CallLayout<
useUpdateLayout();
const pipAlignmentValue = useObservableEagerState(pipAlignment);
const pipTileModel: GridTileModel | undefined = useMemo(
() => model.pip && { type: "grid", vm: model.pip },
[model.pip],
);
const onDragPip: DragCallback = useCallback(
({ xRatio, yRatio }) =>
pipAlignment.next({
@@ -66,12 +57,13 @@ export const makeSpotlightExpandedLayout: CallLayout<
return (
<div ref={ref} className={styles.layer}>
{pipTileModel && (
{model.pip && (
<Slot
className={styles.pip}
id="pip"
model={pipTileModel}
id={model.pip.id}
model={model.pip}
onDrag={onDragPip}
onVisibilityChange={model.pip.setVisible}
data-block-alignment={pipAlignmentValue.block}
data-inline-alignment={pipAlignmentValue.inline}
/>

33
src/grid/SpotlightLandscapeLayout.tsx
View File

@@ -5,11 +5,11 @@ SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { forwardRef, useMemo } from "react";
import { forwardRef } from "react";
import { useObservableEagerState } from "observable-hooks";
import classNames from "classnames";
import { CallLayout, GridTileModel, TileModel } from "./CallLayout";
import { CallLayout } from "./CallLayout";
import { SpotlightLandscapeLayout as SpotlightLandscapeLayoutModel } from "../state/CallViewModel";
import styles from "./SpotlightLandscapeLayout.module.css";
import { useUpdateLayout } from "./Grid";
@@ -30,19 +30,15 @@ export const makeSpotlightLandscapeLayout: CallLayout<
) {
useUpdateLayout();
useObservableEagerState(minBounds);
const tileModel: TileModel = useMemo(
() => ({
type: "spotlight",
vms: model.spotlight,
maximised: false,
}),
[model.spotlight],
);
return (
<div ref={ref} className={styles.layer}>
<div className={styles.spotlight}>
<Slot className={styles.slot} id="spotlight" model={tileModel} />
<Slot
className={styles.slot}
id="spotlight"
model={model.spotlight}
/>
</div>
<div className={styles.grid} />
</div>
@@ -55,25 +51,24 @@ export const makeSpotlightLandscapeLayout: CallLayout<
) {
useUpdateLayout();
useObservableEagerState(minBounds);
const tileModels: GridTileModel[] = useMemo(
() => model.grid.map((vm) => ({ type: "grid", vm })),
[model.grid],
);
const withIndicators =
useObservableEagerState(model.spotlight.media).length > 1;
return (
<div ref={ref} className={styles.layer}>
<div
className={classNames(styles.spotlight, {
[styles.withIndicators]: model.spotlight.length > 1,
[styles.withIndicators]: withIndicators,
})}
/>
<div className={styles.grid}>
{tileModels.map((m) => (
{model.grid.map((m) => (
<Slot
key={m.vm.id}
key={m.id}
className={styles.slot}
id={m.vm.id}
id={m.id}
model={m}
onVisibilityChange={m.setVisible}
/>
))}
</div>

38
src/grid/SpotlightPortraitLayout.tsx
View File

@@ -5,16 +5,11 @@ SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { CSSProperties, forwardRef, useMemo } from "react";
import { CSSProperties, forwardRef } from "react";
import { useObservableEagerState } from "observable-hooks";
import classNames from "classnames";
import {
CallLayout,
GridTileModel,
TileModel,
arrangeTiles,
} from "./CallLayout";
import { CallLayout, arrangeTiles } from "./CallLayout";
import { SpotlightPortraitLayout as SpotlightPortraitLayoutModel } from "../state/CallViewModel";
import styles from "./SpotlightPortraitLayout.module.css";
import { useUpdateLayout } from "./Grid";
@@ -40,19 +35,15 @@ export const makeSpotlightPortraitLayout: CallLayout<
ref,
) {
useUpdateLayout();
const tileModel: TileModel = useMemo(
() => ({
type: "spotlight",
vms: model.spotlight,
maximised: true,
}),
[model.spotlight],
);
return (
<div ref={ref} className={styles.layer}>
<div className={styles.spotlight}>
<Slot className={styles.slot} id="spotlight" model={tileModel} />
<Slot
className={styles.slot}
id="spotlight"
model={model.spotlight}
/>
</div>
</div>
);
@@ -71,10 +62,8 @@ export const makeSpotlightPortraitLayout: CallLayout<
width,
model.grid.length,
);
const tileModels: GridTileModel[] = useMemo(
() => model.grid.map((vm) => ({ type: "grid", vm })),
[model.grid],
);
const withIndicators =
useObservableEagerState(model.spotlight.media).length > 1;
return (
<div
@@ -90,16 +79,17 @@ export const makeSpotlightPortraitLayout: CallLayout<
>
<div
className={classNames(styles.spotlight, {
[styles.withIndicators]: model.spotlight.length > 1,
[styles.withIndicators]: withIndicators,
})}
/>
<div className={styles.grid}>
{tileModels.map((m) => (
{model.grid.map((m) => (
<Slot
key={m.vm.id}
key={m.id}
className={styles.slot}
id={m.vm.id}
id={m.id}
model={m}
onVisibilityChange={m.setVisible}
/>
))}
</div>

9
src/reactions/RaisedHandIndicator.module.css
View File

@@ -5,6 +5,11 @@
color: var(--cpd-color-icon-secondary);
}
.button {
display: contents;
background: none;
}
.raisedHandWidget > p {
padding: none;
margin-top: auto;
@@ -42,11 +47,11 @@
height: var(--cpd-space-6x);
display: inline-block;
text-align: center;
font-size: 16px;
font-size: 1.3em;
}
.raisedHandLarge > span {
width: var(--cpd-space-8x);
height: var(--cpd-space-8x);
font-size: 22px;
font-size: 1.9em;
}

12
src/reactions/RaisedHandIndicator.test.tsx
View File

@@ -40,4 +40,16 @@ describe("RaisedHandIndicator", () => {
);
expect(container.firstChild).toMatchSnapshot();
});
test("can be clicked", () => {
const dateTime = new Date();
let wasClicked = false;
const { getByRole } = render(
<RaisedHandIndicator
raisedHandTime={dateTime}
onClick={() => (wasClicked = true)}
/>,
);
getByRole("button").click();
expect(wasClicked).toBe(true);
});
});

65
src/reactions/RaisedHandIndicator.tsx
View File

@@ -5,7 +5,13 @@ SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { ReactNode, useEffect, useState } from "react";
import {
MouseEventHandler,
ReactNode,
useCallback,
useEffect,
useState,
} from "react";
import classNames from "classnames";
import "@formatjs/intl-durationformat/polyfill";
import { DurationFormat } from "@formatjs/intl-durationformat";
@@ -23,13 +29,26 @@ export function RaisedHandIndicator({
raisedHandTime,
minature,
showTimer,
onClick,
}: {
raisedHandTime?: Date;
minature?: boolean;
showTimer?: boolean;
onClick?: () => void;
}): ReactNode {
const [raisedHandDuration, setRaisedHandDuration] = useState("");
const clickCallback = useCallback<MouseEventHandler<HTMLButtonElement>>(
(event) => {
if (!onClick) {
return;
}
event.preventDefault();
onClick();
},
[onClick],
);
// This effect creates a simple timer effect.
useEffect(() => {
if (!raisedHandTime || !showTimer) {
@@ -52,26 +71,40 @@ export function RaisedHandIndicator({
return (): void => clearInterval(to);
}, [setRaisedHandDuration, raisedHandTime, showTimer]);
if (raisedHandTime) {
return (
if (!raisedHandTime) {
return;
}
const content = (
<div
className={classNames(styles.raisedHandWidget, {
[styles.raisedHandWidgetLarge]: !minature,
})}
>
<div
className={classNames(styles.raisedHandWidget, {
[styles.raisedHandWidgetLarge]: !minature,
className={classNames(styles.raisedHand, {
[styles.raisedHandLarge]: !minature,
})}
>
<div
className={classNames(styles.raisedHand, {
[styles.raisedHandLarge]: !minature,
})}
>
<span role="img" aria-label="raised hand">
✋
</span>
</div>
{showTimer && <p>{raisedHandDuration}</p>}
<span role="img" aria-label="raised hand">
✋
</span>
</div>
{showTimer && <p>{raisedHandDuration}</p>}
</div>
);
if (onClick) {
return (
<button
aria-label="lower raised hand"
className={styles.button}
onClick={clickCallback}
>
{content}
</button>
);
}
return null;
return content;
}

37
src/room/InCallView.tsx
View File

@@ -28,8 +28,8 @@ import {
import useMeasure from "react-use-measure";
import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession";
import classNames from "classnames";
import { BehaviorSubject, of } from "rxjs";
import { useObservableEagerState } from "observable-hooks";
import { BehaviorSubject, map } from "rxjs";
import { useObservable, useObservableEagerState } from "observable-hooks";
import { logger } from "matrix-js-sdk/src/logger";
import LogoMark from "../icons/LogoMark.svg?react";
@@ -65,7 +65,6 @@ import { ECConnectionState } from "../livekit/useECConnectionState";
import { useOpenIDSFU } from "../livekit/openIDSFU";
import { CallViewModel, GridMode, Layout } from "../state/CallViewModel";
import { Grid, TileProps } from "../grid/Grid";
import { useObservable } from "../state/useObservable";
import { useInitial } from "../useInitial";
import { SpotlightTile } from "../tile/SpotlightTile";
import { EncryptionSystem } from "../e2ee/sharedKeyManagement";
@@ -73,7 +72,6 @@ import { E2eeType } from "../e2ee/e2eeType";
import { makeGridLayout } from "../grid/GridLayout";
import {
CallLayoutOutputs,
TileModel,
defaultPipAlignment,
defaultSpotlightAlignment,
} from "../grid/CallLayout";
@@ -81,6 +79,7 @@ import { makeOneOnOneLayout } from "../grid/OneOnOneLayout";
import { makeSpotlightExpandedLayout } from "../grid/SpotlightExpandedLayout";
import { makeSpotlightLandscapeLayout } from "../grid/SpotlightLandscapeLayout";
import { makeSpotlightPortraitLayout } from "../grid/SpotlightPortraitLayout";
import { GridTileViewModel, TileViewModel } from "../state/TileViewModel";
import { ReactionsProvider, useReactions } from "../useReactions";
import handSoundOgg from "../sound/raise_hand.ogg?url";
import handSoundMp3 from "../sound/raise_hand.mp3?url";
@@ -103,7 +102,10 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
sfuConfig,
props.e2eeSystem,
);
const connStateObservable = useObservable(connState);
const connStateObservable = useObservable(
(inputs) => inputs.pipe(map(([connState]) => connState)),
[connState],
);
const [vm, setVm] = useState<CallViewModel | null>(null);
useEffect(() => {
@@ -304,7 +306,10 @@ export const InCallView: FC<InCallViewProps> = ({
windowMode,
],
);
const gridBoundsObservable = useObservable(gridBounds);
const gridBoundsObservable = useObservable(
(inputs) => inputs.pipe(map(([gridBounds]) => gridBounds)),
[gridBounds],
);
const spotlightAlignment = useInitial(
() => new BehaviorSubject(defaultSpotlightAlignment),
@@ -375,7 +380,7 @@ export const InCallView: FC<InCallViewProps> = ({
() =>
forwardRef<
HTMLDivElement,
PropsWithoutRef<TileProps<TileModel, HTMLDivElement>>
PropsWithoutRef<TileProps<TileViewModel, HTMLDivElement>>
>(function Tile(
{ className, style, targetWidth, targetHeight, model },
ref,
@@ -384,13 +389,6 @@ export const InCallView: FC<InCallViewProps> = ({
const onToggleExpanded = useObservableEagerState(
vm.toggleSpotlightExpanded,
);
const showVideo = useObservableEagerState(
useMemo(
() =>
model.type === "grid" ? vm.showGridVideo(model.vm) : of(true),
[model],
),
);
const showSpeakingIndicatorsValue = useObservableEagerState(
vm.showSpeakingIndicators,
);
@@ -398,23 +396,21 @@ export const InCallView: FC<InCallViewProps> = ({
vm.showSpotlightIndicators,
);
return model.type === "grid" ? (
return model instanceof GridTileViewModel ? (
<GridTile
ref={ref}
vm={model.vm}
vm={model}
onOpenProfile={openProfile}
targetWidth={targetWidth}
targetHeight={targetHeight}
className={classNames(className, styles.tile)}
style={style}
showVideo={showVideo}
showSpeakingIndicators={showSpeakingIndicatorsValue}
/>
) : (
<SpotlightTile
ref={ref}
vms={model.vms}
maximised={model.maximised}
vm={model}
expanded={spotlightExpanded}
onToggleExpanded={onToggleExpanded}
targetWidth={targetWidth}
@@ -448,8 +444,7 @@ export const InCallView: FC<InCallViewProps> = ({
return (
<SpotlightTile
className={classNames(styles.tile, styles.maximised)}
vms={layout.spotlight!}
maximised
vm={layout.spotlight}
expanded
onToggleExpanded={null}
targetWidth={gridBounds.height}

5
src/rtcSessionHelper.test.ts
View File

@@ -81,6 +81,9 @@ test("It joins the correct Session", async () => {
focus_selection: "oldest_membership",
type: "livekit",
},
{ manageMediaKeys: false },
{
manageMediaKeys: false,
useLegacyMemberEvents: false,
},
);
});

513
src/state/CallViewModel.test.ts
View File

@@ -6,19 +6,25 @@ Please see LICENSE in the repository root for full details.
*/
import { test, vi, onTestFinished } from "vitest";
import { map, Observable, of } from "rxjs";
import {
combineLatest,
debounceTime,
distinctUntilChanged,
map,
Observable,
of,
switchMap,
} from "rxjs";
import { MatrixClient } from "matrix-js-sdk/src/matrix";
import {
ConnectionState,
LocalParticipant,
Participant,
RemoteParticipant,
} from "livekit-client";
import * as ComponentsCore from "@livekit/components-core";
import {
CallMembership,
MatrixRTCSession,
MatrixRTCSessionEvent,
} from "matrix-js-sdk/src/matrixrtc";
import { isEqual } from "lodash";
import { CallMembership, MatrixRTCSession, MatrixRTCSessionEvent } from "matrix-js-sdk/src/matrixrtc";
import { CallViewModel, Layout } from "./CallViewModel";
import {
@@ -27,7 +33,6 @@ import {
mockMatrixRoom,
mockRoomMember,
mockRemoteParticipant,
OurRunHelpers,
withTestScheduler,
mockMembership,
} from "../utils/test";
@@ -40,41 +45,40 @@ import { MockRoom, MockRTCSession } from "../useReactions.test";
vi.mock("@livekit/components-core");
const carolId = "@carol:example.org";
const carolDev = "CCCC";
const aliceId = "@alice:example.org";
const alice = mockRoomMember({ userId: "@alice:example.org" });
const bob = mockRoomMember({ userId: "@bob:example.org" });
const carol = mockRoomMember({ userId: "@carol:example.org" });
const dave = mockRoomMember({ userId: "@dave:example.org" });
const aliceDev = "AAAA";
const aliceRTCId = aliceId + ":" + aliceDev;
const bobId = "@bob:example.org";
const bobDev = "BBBB";
const bobRTCId = bobId + ":" + bobDev;
const alice = mockRoomMember({ userId: aliceId });
const bob = mockRoomMember({ userId: bobId });
const carol = mockRoomMember({ userId: carolId });
const carolDev = "CCCC";
const daveDev = "DDDD";
const aliceId = `${alice.userId}:${aliceDev}`;
const bobId = `${bob.userId}:${bobDev}`;
const carolId = `${carol.userId}:${carolDev}`;
const daveId = `${dave.userId}:${daveDev}`;
const localParticipant = mockLocalParticipant({ identity: "" });
const aliceParticipant = mockRemoteParticipant({ identity: aliceRTCId });
const aliceParticipant = mockRemoteParticipant({ identity: aliceId });
const aliceSharingScreen = mockRemoteParticipant({
identity: aliceRTCId,
identity: aliceId,
isScreenShareEnabled: true,
});
const bobParticipant = mockRemoteParticipant({ identity: bobRTCId });
const bobParticipant = mockRemoteParticipant({ identity: bobId });
const bobSharingScreen = mockRemoteParticipant({
identity: bobRTCId,
identity: bobId,
isScreenShareEnabled: true,
});
const daveParticipant = mockRemoteParticipant({ identity: daveId });
const members = new Map([
[alice.userId, alice],
[bob.userId, bob],
[carol.userId, carol],
]);
const members = new Map([alice, bob, carol, dave].map((p) => [p.userId, p]));
const rtcMemberAlice = mockMembership(aliceId, aliceDev);
const rtcMemberBob = mockMembership(bobId, bobDev);
const rtcMemberCarol = mockMembership(carolId, carolDev);
const aliceRtcMember = mockMembership(aliceId, aliceDev);
const bobRtcMember = mockMembership(bobId, bobDev);
const carolRtcMember = mockMembership(carolId, carolDev);
const daveRtcMember = mockMembership(daveId, daveDev);
export interface GridLayoutSummary {
type: "grid";
@@ -119,39 +123,72 @@ export type LayoutSummary =
| OneOnOneLayoutSummary
| PipLayoutSummary;
function summarizeLayout(l: Layout): LayoutSummary {
switch (l.type) {
case "grid":
return {
type: l.type,
spotlight: l.spotlight?.map((vm) => vm.id),
grid: l.grid.map((vm) => vm.id),
};
case "spotlight-landscape":
case "spotlight-portrait":
return {
type: l.type,
spotlight: l.spotlight.map((vm) => vm.id),
grid: l.grid.map((vm) => vm.id),
};
case "spotlight-expanded":
return {
type: l.type,
spotlight: l.spotlight.map((vm) => vm.id),
pip: l.pip?.id,
};
case "one-on-one":
return { type: l.type, local: l.local.id, remote: l.remote.id };
case "pip":
return { type: l.type, spotlight: l.spotlight.map((vm) => vm.id) };
}
function summarizeLayout(l: Observable<Layout>): Observable<LayoutSummary> {
return l.pipe(
switchMap((l) => {
switch (l.type) {
case "grid":
return combineLatest(
[
l.spotlight?.media ?? of(undefined),
...l.grid.map((vm) => vm.media),
],
(spotlight, ...grid) => ({
type: l.type,
spotlight: spotlight?.map((vm) => vm.id),
grid: grid.map((vm) => vm.id),
}),
);
case "spotlight-landscape":
case "spotlight-portrait":
return combineLatest(
[l.spotlight.media, ...l.grid.map((vm) => vm.media)],
(spotlight, ...grid) => ({
type: l.type,
spotlight: spotlight.map((vm) => vm.id),
grid: grid.map((vm) => vm.id),
}),
);
case "spotlight-expanded":
return combineLatest(
[l.spotlight.media, l.pip?.media ?? of(undefined)],
(spotlight, pip) => ({
type: l.type,
spotlight: spotlight.map((vm) => vm.id),
pip: pip?.id,
}),
);
case "one-on-one":
return combineLatest(
[l.local.media, l.remote.media],
(local, remote) => ({
type: l.type,
local: local.id,
remote: remote.id,
}),
);
case "pip":
return l.spotlight.media.pipe(
map((spotlight) => ({
type: l.type,
spotlight: spotlight.map((vm) => vm.id),
})),
);
}
}),
// Sometimes there can be multiple (synchronous) updates per frame. We only
// care about the most recent value for each time step, so discard these
// extra values.
debounceTime(0),
distinctUntilChanged(isEqual),
);
}
function withCallViewModel(
{ cold }: OurRunHelpers,
remoteParticipants: Observable<RemoteParticipant[]>,
rtcMembers: Observable<Partial<CallMembership>[]>,
connectionState: Observable<ECConnectionState>,
speaking: Map<Participant, Observable<boolean>>,
continuation: (vm: CallViewModel) => void,
): void {
const room = mockMatrixRoom({
@@ -172,19 +209,30 @@ function withCallViewModel(
const mediaSpy = vi
.spyOn(ComponentsCore, "observeParticipantMedia")
.mockImplementation((p) =>
cold("a", {
a: { participant: p } as Partial<
ComponentsCore.ParticipantMedia<LocalParticipant>
> as ComponentsCore.ParticipantMedia<LocalParticipant>,
}),
of({ participant: p } as Partial<
ComponentsCore.ParticipantMedia<LocalParticipant>
> as ComponentsCore.ParticipantMedia<LocalParticipant>),
);
const eventsSpy = vi
.spyOn(ComponentsCore, "observeParticipantEvents")
.mockImplementation((p) => cold("a", { a: p }));
.mockImplementation((p) =>
(speaking.get(p) ?? of(false)).pipe(
map((s) => ({ ...p, isSpeaking: s }) as Participant),
),
);
const roomEventSelectorSpy = vi
.spyOn(ComponentsCore, "roomEventSelector")
.mockImplementation((room, eventType) => of());
const liveKitRoom = mockLivekitRoom(
{ localParticipant },
{ remoteParticipants },
);
const vm = new CallViewModel(
rtcSession as unknown as MatrixRTCSession,
mockLivekitRoom({ localParticipant }),
liveKitRoom,
{
kind: E2eeType.PER_PARTICIPANT,
},
@@ -196,115 +244,316 @@ function withCallViewModel(
participantsSpy!.mockRestore();
mediaSpy!.mockRestore();
eventsSpy!.mockRestore();
roomEventSelectorSpy!.mockRestore();
});
continuation(vm);
}
test("participants are retained during a focus switch", () => {
withTestScheduler((helpers) => {
const { hot, expectObservable } = helpers;
withTestScheduler(({ cold, expectObservable }) => {
// Participants disappear on frame 2 and come back on frame 3
const partMarbles = "a-ba";
// The RTC members never disappear
const rtcMemberMarbles = "a---";
const participantMarbles = "a-ba";
// Start switching focus on frame 1 and reconnect on frame 3
const connMarbles = "ab-a";
const connectionMarbles = " cs-c";
// The visible participants should remain the same throughout the switch
const laytMarbles = "aaaa 2997ms a 56998ms a";
const layoutMarbles = " a";
withCallViewModel(
helpers,
hot(partMarbles, {
cold(participantMarbles, {
a: [aliceParticipant, bobParticipant],
b: [],
}),
hot(rtcMemberMarbles, {
a: [rtcMemberAlice, rtcMemberBob, rtcMemberCarol],
}),
hot(connMarbles, {
a: ConnectionState.Connected,
b: ECAddonConnectionState.ECSwitchingFocus,
of([aliceRtcMember, bobRtcMember]),
cold(connectionMarbles, {
c: ConnectionState.Connected,
s: ECAddonConnectionState.ECSwitchingFocus,
}),
new Map(),
(vm) => {
expectObservable(vm.layout.pipe(map(summarizeLayout))).toBe(
laytMarbles,
{
a: {
type: "grid",
spotlight: undefined,
grid: ["local:0", `${aliceRTCId}:0`, `${bobRTCId}:0`],
},
expectObservable(summarizeLayout(vm.layout)).toBe(layoutMarbles, {
a: {
type: "grid",
spotlight: undefined,
grid: ["local:0", `${aliceId}:0`, `${bobId}:0`],
},
);
});
},
);
});
});
test("screen sharing activates spotlight layout", () => {
withTestScheduler((helpers) => {
const { hot, schedule, expectObservable } = helpers;
withTestScheduler(({ cold, schedule, expectObservable }) => {
// Start with no screen shares, then have Alice and Bob share their screens,
// then return to no screen shares, then have just Alice share for a bit
const partMarbles = "abc---d---a-b---a";
const participantMarbles = " abcda-ba";
// While there are no screen shares, switch to spotlight manually, and then
// switch back to grid at the end
const modeMarbles = "-----------a--------b";
const modeMarbles = " -----s--g";
// We should automatically enter spotlight for the first round of screen
// sharing, then return to grid, then manually go into spotlight, and
// remain in spotlight until we manually go back to grid
const laytMarbles = "ab(cc)(dd)ae(bb)(ee)a 59979ms a";
const layoutMarbles = " abcdaefeg";
const showSpeakingMarbles = "y----nyny";
withCallViewModel(
helpers,
hot(partMarbles, {
cold(participantMarbles, {
a: [aliceParticipant, bobParticipant],
b: [aliceSharingScreen, bobParticipant],
c: [aliceSharingScreen, bobSharingScreen],
d: [aliceParticipant, bobSharingScreen],
}),
of([rtcMemberAlice, rtcMemberAlice]),
hot("a", { a: ConnectionState.Connected }),
of([aliceRtcMember, bobRtcMember]),
of(ConnectionState.Connected),
new Map(),
(vm) => {
schedule(modeMarbles, {
a: () => vm.setGridMode("spotlight"),
b: () => vm.setGridMode("grid"),
s: () => vm.setGridMode("spotlight"),
g: () => vm.setGridMode("grid"),
});
expectObservable(vm.layout.pipe(map(summarizeLayout))).toBe(
laytMarbles,
{
a: {
type: "grid",
spotlight: undefined,
grid: ["local:0", `${aliceId}:0`, `${bobId}:0`],
},
b: {
type: "spotlight-landscape",
spotlight: [`${aliceId}:0:screen-share`],
grid: ["local:0", `${aliceId}:0`, `${bobId}:0`],
},
c: {
type: "spotlight-landscape",
spotlight: [
`${aliceId}:0:screen-share`,
`${bobId}:0:screen-share`,
],
grid: ["local:0", `${aliceId}:0`, `${bobId}:0`],
},
d: {
type: "spotlight-landscape",
spotlight: [`${bobId}:0:screen-share`],
grid: ["local:0", `${aliceId}:0`, `${bobId}:0`],
},
e: {
type: "spotlight-landscape",
spotlight: [`${aliceId}:0`],
grid: ["local:0", `${aliceId}:0`, `${bobId}:0`],
},
expectObservable(summarizeLayout(vm.layout)).toBe(layoutMarbles, {
a: {
type: "grid",
spotlight: undefined,
grid: ["local:0", `${aliceId}:0`, `${bobId}:0`],
},
);
b: {
type: "spotlight-landscape",
spotlight: [`${aliceId}:0:screen-share`],
grid: ["local:0", `${aliceId}:0`, `${bobId}:0`],
},
c: {
type: "spotlight-landscape",
spotlight: [`${aliceId}:0:screen-share`, `${bobId}:0:screen-share`],
grid: ["local:0", `${aliceId}:0`, `${bobId}:0`],
},
d: {
type: "spotlight-landscape",
spotlight: [`${bobId}:0:screen-share`],
grid: ["local:0", `${aliceId}:0`, `${bobId}:0`],
},
e: {
type: "spotlight-landscape",
spotlight: [`${aliceId}:0`],
grid: ["local:0", `${bobId}:0`],
},
f: {
type: "spotlight-landscape",
spotlight: [`${aliceId}:0:screen-share`],
grid: ["local:0", `${bobId}:0`, `${aliceId}:0`],
},
g: {
type: "grid",
spotlight: undefined,
grid: ["local:0", `${bobId}:0`, `${aliceId}:0`],
},
});
expectObservable(vm.showSpeakingIndicators).toBe(showSpeakingMarbles, {
y: true,
n: false,
});
},
);
});
});
test("participants stay in the same order unless to appear/disappear", () => {
withTestScheduler(({ cold, schedule, expectObservable }) => {
const modeMarbles = "a";
// First Bob speaks, then Dave, then Alice
const aSpeakingMarbles = "n- 1998ms - 1999ms y";
const bSpeakingMarbles = "ny 1998ms n 1999ms ";
const dSpeakingMarbles = "n- 1998ms y 1999ms n";
// Nothing should change when Bob speaks, because Bob is already on screen.
// When Dave speaks he should switch with Alice because she's the one who
// hasn't spoken at all. Then when Alice speaks, she should return to her
// place at the top.
const layoutMarbles = " a 1999ms b 1999ms a 57999ms c 1999ms a";
withCallViewModel(
of([aliceParticipant, bobParticipant, daveParticipant]),
of([aliceRtcMember, bobRtcMember, daveRtcMember]),
of(ConnectionState.Connected),
new Map([
[aliceParticipant, cold(aSpeakingMarbles, { y: true, n: false })],
[bobParticipant, cold(bSpeakingMarbles, { y: true, n: false })],
[daveParticipant, cold(dSpeakingMarbles, { y: true, n: false })],
]),
(vm) => {
schedule(modeMarbles, {
a: () => {
// We imagine that only three tiles (the first three) will be visible
// on screen at a time
vm.layout.subscribe((layout) => {
if (layout.type === "grid") {
for (let i = 0; i < layout.grid.length; i++)
layout.grid[i].setVisible(i < 3);
}
});
},
});
expectObservable(summarizeLayout(vm.layout)).toBe(layoutMarbles, {
a: {
type: "grid",
spotlight: undefined,
grid: ["local:0", `${aliceId}:0`, `${bobId}:0`, `${daveId}:0`],
},
b: {
type: "grid",
spotlight: undefined,
grid: ["local:0", `${daveId}:0`, `${bobId}:0`, `${aliceId}:0`],
},
c: {
type: "grid",
spotlight: undefined,
grid: ["local:0", `${aliceId}:0`, `${daveId}:0`, `${bobId}:0`],
},
});
},
);
});
});
test("spotlight speakers swap places", () => {
withTestScheduler(({ cold, schedule, expectObservable }) => {
// Go immediately into spotlight mode for the test
const modeMarbles = " s";
// First Bob speaks, then Dave, then Alice
const aSpeakingMarbles = "n--y";
const bSpeakingMarbles = "nyn";
const dSpeakingMarbles = "n-yn";
// Alice should start in the spotlight, then Bob, then Dave, then Alice
// again. However, the positions of Dave and Bob in the grid should be
// reversed by the end because they've been swapped in and out of the
// spotlight.
const layoutMarbles = " abcd";
withCallViewModel(
of([aliceParticipant, bobParticipant, daveParticipant]),
of([aliceRtcMember, bobRtcMember, daveRtcMember]),
of(ConnectionState.Connected),
new Map([
[aliceParticipant, cold(aSpeakingMarbles, { y: true, n: false })],
[bobParticipant, cold(bSpeakingMarbles, { y: true, n: false })],
[daveParticipant, cold(dSpeakingMarbles, { y: true, n: false })],
]),
(vm) => {
schedule(modeMarbles, { s: () => vm.setGridMode("spotlight") });
expectObservable(summarizeLayout(vm.layout)).toBe(layoutMarbles, {
a: {
type: "spotlight-landscape",
spotlight: [`${aliceId}:0`],
grid: ["local:0", `${bobId}:0`, `${daveId}:0`],
},
b: {
type: "spotlight-landscape",
spotlight: [`${bobId}:0`],
grid: ["local:0", `${aliceId}:0`, `${daveId}:0`],
},
c: {
type: "spotlight-landscape",
spotlight: [`${daveId}:0`],
grid: ["local:0", `${aliceId}:0`, `${bobId}:0`],
},
d: {
type: "spotlight-landscape",
spotlight: [`${aliceId}:0`],
grid: ["local:0", `${daveId}:0`, `${bobId}:0`],
},
});
},
);
});
});
test("layout enters picture-in-picture mode when requested", () => {
withTestScheduler(({ schedule, expectObservable }) => {
// Enable then disable picture-in-picture
const pipControlMarbles = "-ed";
// Should go into picture-in-picture layout then back to grid
const layoutMarbles = " aba";
withCallViewModel(
of([aliceParticipant, bobParticipant]),
of([aliceRtcMember, bobRtcMember]),
of(ConnectionState.Connected),
new Map(),
(vm) => {
schedule(pipControlMarbles, {
e: () => window.controls.enablePip(),
d: () => window.controls.disablePip(),
});
expectObservable(summarizeLayout(vm.layout)).toBe(layoutMarbles, {
a: {
type: "grid",
spotlight: undefined,
grid: ["local:0", `${aliceId}:0`, `${bobId}:0`],
},
b: {
type: "pip",
spotlight: [`${aliceId}:0`],
},
});
},
);
});
});
test("spotlight remembers whether it's expanded", () => {
withTestScheduler(({ schedule, expectObservable }) => {
// Start in spotlight mode, then switch to grid and back to spotlight a
// couple times
const modeMarbles = " s-gs-gs";
// Expand and collapse the spotlight
const expandMarbles = "-a--a";
// Spotlight should stay expanded during the first mode switch, and stay
// collapsed during the second mode switch
const layoutMarbles = "abcbada";
withCallViewModel(
of([aliceParticipant, bobParticipant]),
of([aliceRtcMember, bobRtcMember]),
of(ConnectionState.Connected),
new Map(),
(vm) => {
schedule(modeMarbles, {
s: () => vm.setGridMode("spotlight"),
g: () => vm.setGridMode("grid"),
});
schedule(expandMarbles, {
a: () => {
let toggle: () => void;
vm.toggleSpotlightExpanded.subscribe((val) => (toggle = val!));
toggle!();
},
});
expectObservable(summarizeLayout(vm.layout)).toBe(layoutMarbles, {
a: {
type: "spotlight-landscape",
spotlight: [`${aliceId}:0`],
grid: ["local:0", `${bobId}:0`],
},
b: {
type: "spotlight-expanded",
spotlight: [`${aliceId}:0`],
pip: "local:0",
},
c: {
type: "grid",
spotlight: undefined,
grid: ["local:0", `${aliceId}:0`, `${bobId}:0`],
},
d: {
type: "grid",
spotlight: undefined,
grid: ["local:0", `${bobId}:0`, `${aliceId}:0`],
},
});
},
);
});

241
src/state/CallViewModel.ts
View File

@@ -71,6 +71,12 @@ import { ObservableScope } from "./ObservableScope";
import { duplicateTiles, nonMemberTiles } from "../settings/settings";
import { isFirefox } from "../Platform";
import { setPipEnabled } from "../controls";
import { GridTileViewModel, SpotlightTileViewModel } from "./TileViewModel";
import { TileStore } from "./TileStore";
import { gridLikeLayout } from "./GridLikeLayout";
import { spotlightExpandedLayout } from "./SpotlightExpandedLayout";
import { oneOnOneLayout } from "./OneOnOneLayout";
import { pipLayout } from "./PipLayout";
import { EncryptionSystem } from "../e2ee/sharedKeyManagement";
// How long we wait after a focus switch before showing the real participant
@@ -81,39 +87,82 @@ const POST_FOCUS_PARTICIPANT_UPDATE_DELAY_MS = 3000;
// on mobile. No spotlight tile should be shown below this threshold.
const smallMobileCallThreshold = 3;
export interface GridLayout {
export interface GridLayoutMedia {
type: "grid";
spotlight?: MediaViewModel[];
grid: UserMediaViewModel[];
}
export interface SpotlightLandscapeLayout {
export interface SpotlightLandscapeLayoutMedia {
type: "spotlight-landscape";
spotlight: MediaViewModel[];
grid: UserMediaViewModel[];
}
export interface SpotlightPortraitLayout {
export interface SpotlightPortraitLayoutMedia {
type: "spotlight-portrait";
spotlight: MediaViewModel[];
grid: UserMediaViewModel[];
}
export interface SpotlightExpandedLayout {
export interface SpotlightExpandedLayoutMedia {
type: "spotlight-expanded";
spotlight: MediaViewModel[];
pip?: UserMediaViewModel;
}
export interface OneOnOneLayoutMedia {
type: "one-on-one";
local: UserMediaViewModel;
remote: UserMediaViewModel;
}
export interface PipLayoutMedia {
type: "pip";
spotlight: MediaViewModel[];
}
export type LayoutMedia =
| GridLayoutMedia
| SpotlightLandscapeLayoutMedia
| SpotlightPortraitLayoutMedia
| SpotlightExpandedLayoutMedia
| OneOnOneLayoutMedia
| PipLayoutMedia;
export interface GridLayout {
type: "grid";
spotlight?: SpotlightTileViewModel;
grid: GridTileViewModel[];
}
export interface SpotlightLandscapeLayout {
type: "spotlight-landscape";
spotlight: SpotlightTileViewModel;
grid: GridTileViewModel[];
}
export interface SpotlightPortraitLayout {
type: "spotlight-portrait";
spotlight: SpotlightTileViewModel;
grid: GridTileViewModel[];
}
export interface SpotlightExpandedLayout {
type: "spotlight-expanded";
spotlight: SpotlightTileViewModel;
pip?: GridTileViewModel;
}
export interface OneOnOneLayout {
type: "one-on-one";
local: LocalUserMediaViewModel;
remote: RemoteUserMediaViewModel;
local: GridTileViewModel;
remote: GridTileViewModel;
}
export interface PipLayout {
type: "pip";
spotlight: MediaViewModel[];
spotlight: SpotlightTileViewModel;
}
/**
@@ -162,6 +211,12 @@ enum SortingBin {
SelfNotAlwaysShown,
}
interface LayoutScanState {
layout: Layout | null;
tiles: TileStore;
visibleTiles: Set<GridTileViewModel>;
}
class UserMedia {
private readonly scope = new ObservableScope();
public readonly vm: UserMediaViewModel;
@@ -176,6 +231,7 @@ class UserMedia {
member: RoomMember | undefined,
participant: LocalParticipant | RemoteParticipant | undefined,
encryptionSystem: EncryptionSystem,
livekitRoom: LivekitRoom,
) {
this.participant = new BehaviorSubject(participant);
@@ -185,6 +241,7 @@ class UserMedia {
member,
this.participant.asObservable() as Observable<LocalParticipant>,
encryptionSystem,
livekitRoom,
);
} else {
this.vm = new RemoteUserMediaViewModel(
@@ -194,6 +251,7 @@ class UserMedia {
RemoteParticipant | undefined
>,
encryptionSystem,
livekitRoom,
);
}
@@ -247,6 +305,7 @@ class ScreenShare {
member: RoomMember | undefined,
participant: LocalParticipant | RemoteParticipant,
encryptionSystem: EncryptionSystem,
liveKitRoom: LivekitRoom,
) {
this.participant = new BehaviorSubject(participant);
@@ -255,6 +314,7 @@ class ScreenShare {
member,
this.participant.asObservable(),
encryptionSystem,
liveKitRoom,
);
}
@@ -437,6 +497,7 @@ export class CallViewModel extends ViewModel {
member,
participant,
this.encryptionSystem,
this.livekitRoom
),
];
}
@@ -450,6 +511,7 @@ export class CallViewModel extends ViewModel {
member,
participant,
this.encryptionSystem,
this.livekitRoom,
),
];
}
@@ -662,6 +724,14 @@ export class CallViewModel extends ViewModel {
this.scope.state(),
);
private readonly hasRemoteScreenShares: Observable<boolean> =
this.spotlight.pipe(
map((spotlight) =>
spotlight.some((vm) => !vm.local && vm instanceof ScreenShareViewModel),
),
distinctUntilChanged(),
);
private readonly pip: Observable<UserMediaViewModel | null> =
this.spotlightAndPip.pipe(switchMap(([, pip]) => pip));
@@ -742,7 +812,7 @@ export class CallViewModel extends ViewModel {
screenShares.length === 0,
);
private readonly gridLayout: Observable<Layout> = combineLatest(
private readonly gridLayout: Observable<LayoutMedia> = combineLatest(
[this.grid, this.spotlight],
(grid, spotlight) => ({
type: "grid",
@@ -753,38 +823,44 @@ export class CallViewModel extends ViewModel {
}),
);
private readonly spotlightLandscapeLayout: Observable<Layout> = combineLatest(
[this.grid, this.spotlight],
(grid, spotlight) => ({ type: "spotlight-landscape", spotlight, grid }),
);
private readonly spotlightLandscapeLayout: Observable<LayoutMedia> =
combineLatest([this.grid, this.spotlight], (grid, spotlight) => ({
type: "spotlight-landscape",
spotlight,
grid,
}));
private readonly spotlightPortraitLayout: Observable<Layout> = combineLatest(
[this.grid, this.spotlight],
(grid, spotlight) => ({ type: "spotlight-portrait", spotlight, grid }),
);
private readonly spotlightPortraitLayout: Observable<LayoutMedia> =
combineLatest([this.grid, this.spotlight], (grid, spotlight) => ({
type: "spotlight-portrait",
spotlight,
grid,
}));
private readonly spotlightExpandedLayout: Observable<Layout> = combineLatest(
[this.spotlight, this.pip],
(spotlight, pip) => ({
private readonly spotlightExpandedLayout: Observable<LayoutMedia> =
combineLatest([this.spotlight, this.pip], (spotlight, pip) => ({
type: "spotlight-expanded",
spotlight,
pip: pip ?? undefined,
}),
);
}));
private readonly oneOnOneLayout: Observable<Layout> = this.grid.pipe(
map((grid) => ({
type: "one-on-one",
local: grid.find((vm) => vm.local) as LocalUserMediaViewModel,
remote: grid.find((vm) => !vm.local) as RemoteUserMediaViewModel,
})),
);
private readonly oneOnOneLayout: Observable<LayoutMedia> =
this.mediaItems.pipe(
map((grid) => ({
type: "one-on-one",
local: grid.find((vm) => vm.vm.local)!.vm as LocalUserMediaViewModel,
remote: grid.find((vm) => !vm.vm.local)!.vm as RemoteUserMediaViewModel,
})),
);
private readonly pipLayout: Observable<Layout> = this.spotlight.pipe(
private readonly pipLayout: Observable<LayoutMedia> = this.spotlight.pipe(
map((spotlight) => ({ type: "pip", spotlight })),
);
public readonly layout: Observable<Layout> = this.windowMode.pipe(
/**
* The media to be used to produce a layout.
*/
private readonly layoutMedia: Observable<LayoutMedia> = this.windowMode.pipe(
switchMap((windowMode) => {
switch (windowMode) {
case "normal":
@@ -845,32 +921,97 @@ export class CallViewModel extends ViewModel {
this.scope.state(),
);
/**
* The layout of tiles in the call interface.
*/
public readonly layout: Observable<Layout> = this.layoutMedia.pipe(
// Each layout will produce a set of tiles, and these tiles have an
// observable indicating whether they're visible. We loop this information
// back into the layout process by using switchScan.
switchScan<
LayoutMedia,
LayoutScanState,
Observable<LayoutScanState & { layout: Layout }>
>(
({ tiles: prevTiles, visibleTiles }, media) => {
let layout: Layout;
let newTiles: TileStore;
switch (media.type) {
case "grid":
case "spotlight-landscape":
case "spotlight-portrait":
[layout, newTiles] = gridLikeLayout(media, visibleTiles, prevTiles);
break;
case "spotlight-expanded":
[layout, newTiles] = spotlightExpandedLayout(
media,
visibleTiles,
prevTiles,
);
break;
case "one-on-one":
[layout, newTiles] = oneOnOneLayout(media, visibleTiles, prevTiles);
break;
case "pip":
[layout, newTiles] = pipLayout(media, visibleTiles, prevTiles);
break;
}
// Take all of the 'visible' observables and combine them into one big
// observable array
const visibilities =
newTiles.gridTiles.length === 0
? of([])
: combineLatest(newTiles.gridTiles.map((tile) => tile.visible));
return visibilities.pipe(
map((visibilities) => ({
layout: layout,
tiles: newTiles,
visibleTiles: new Set(
newTiles.gridTiles.filter((_tile, i) => visibilities[i]),
),
})),
);
},
{
layout: null,
tiles: TileStore.empty(),
visibleTiles: new Set(),
},
),
map(({ layout }) => layout),
this.scope.state(),
);
public showSpotlightIndicators: Observable<boolean> = this.layout.pipe(
map((l) => l.type !== "grid"),
this.scope.state(),
);
/**
* Determines whether video should be shown for a certain piece of media
* appearing in the grid.
*/
public showGridVideo(vm: MediaViewModel): Observable<boolean> {
return this.layout.pipe(
map(
(l) =>
!(
(l.type === "spotlight-landscape" ||
l.type === "spotlight-portrait") &&
// This media is already visible in the spotlight; avoid duplication
l.spotlight.some((spotlightVm) => spotlightVm === vm)
),
),
distinctUntilChanged(),
);
}
public showSpeakingIndicators: Observable<boolean> = this.layout.pipe(
map((l) => l.type !== "one-on-one" && !l.type.startsWith("spotlight-")),
switchMap((l) => {
switch (l.type) {
case "spotlight-landscape":
case "spotlight-portrait":
// If the spotlight is showing the active speaker, we can do without
// speaking indicators as they're a redundant visual cue. But if
// screen sharing feeds are in the spotlight we still need them.
return l.spotlight.media.pipe(
map((models: MediaViewModel[]) =>
models.some((m) => m instanceof ScreenShareViewModel),
),
);
// In expanded spotlight layout, the active speaker is always shown in
// the picture-in-picture tile so there is no need for speaking
// indicators. And in one-on-one layout there's no question as to who is
// speaking.
case "spotlight-expanded":
case "one-on-one":
return of(false);
default:
return of(true);
}
}),
this.scope.state(),
);

43
src/state/GridLikeLayout.ts Normal file
View File

@@ -0,0 +1,43 @@
/*
Copyright 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { Layout, LayoutMedia } from "./CallViewModel";
import { TileStore } from "./TileStore";
import { GridTileViewModel } from "./TileViewModel";
export type GridLikeLayoutType =
| "grid"
| "spotlight-landscape"
| "spotlight-portrait";
/**
* Produces a grid-like layout (any layout with a grid and possibly a spotlight)
* with the given media.
*/
export function gridLikeLayout(
media: LayoutMedia & { type: GridLikeLayoutType },
visibleTiles: Set<GridTileViewModel>,
prevTiles: TileStore,
): [Layout & { type: GridLikeLayoutType }, TileStore] {
const update = prevTiles.from(visibleTiles);
if (media.spotlight !== undefined)
update.registerSpotlight(
media.spotlight,
media.type === "spotlight-portrait",
);
for (const mediaVm of media.grid) update.registerGridTile(mediaVm);
const tiles = update.build();
return [
{
type: media.type,
spotlight: tiles.spotlightTile,
grid: tiles.gridTiles,
} as Layout & { type: GridLikeLayoutType },
tiles,
];
}

190
src/state/MediaViewModel.ts
View File

@@ -11,6 +11,7 @@ import {
VideoSource,
observeParticipantEvents,
observeParticipantMedia,
roomEventSelector,
} from "@livekit/components-core";
import {
LocalParticipant,
@@ -21,6 +22,9 @@ import {
Track,
TrackEvent,
facingModeFromLocalTrack,
Room as LivekitRoom,
RoomEvent as LivekitRoomEvent,
RemoteTrack,
} from "livekit-client";
import { RoomMember, RoomMemberEvent } from "matrix-js-sdk/src/matrix";
import {
@@ -28,13 +32,18 @@ import {
Observable,
Subject,
combineLatest,
distinctUntilChanged,
distinctUntilKeyChanged,
filter,
fromEvent,
interval,
map,
merge,
of,
shareReplay,
startWith,
switchMap,
throttleTime,
} from "rxjs";
import { useEffect } from "react";
@@ -90,6 +99,115 @@ export function observeTrackReference(
return obs;
}
function observeRemoteTrackReceivingOkay(
participant: Participant,
source: Track.Source,
): Observable<boolean | undefined> {
let lastStats: {
framesDecoded: number | undefined;
framesDropped: number | undefined;
framesReceived: number | undefined;
} = {
framesDecoded: undefined,
framesDropped: undefined,
framesReceived: undefined,
};
return combineLatest([
observeTrackReference(participant, source),
interval(1000).pipe(startWith(0)),
]).pipe(
switchMap(async ([trackReference]) => {
const track = trackReference.publication?.track;
if (!track || !(track instanceof RemoteTrack)) {
return undefined;
}
const report = await track.getRTCStatsReport();
if (!report) {
return undefined;
}
for (const v of report.values()) {
if (v.type === "inbound-rtp") {
const { framesDecoded, framesDropped, framesReceived } =
v as RTCInboundRtpStreamStats;
return {
framesDecoded,
framesDropped,
framesReceived,
};
}
}
return undefined;
}),
filter((newStats) => !!newStats),
map((newStats): boolean | undefined => {
const oldStats = lastStats;
lastStats = newStats;
if (
typeof newStats.framesReceived === "number" &&
typeof oldStats.framesReceived === "number" &&
typeof newStats.framesDecoded === "number" &&
typeof oldStats.framesDecoded === "number"
) {
const framesReceivedDelta =
newStats.framesReceived - oldStats.framesReceived;
const framesDecodedDelta =
newStats.framesDecoded - oldStats.framesDecoded;
// if we received >0 frames and managed to decode >0 frames then we treat that as success
if (framesReceivedDelta > 0) {
return framesDecodedDelta > 0;
}
}
// no change
return undefined;
}),
filter((x) => typeof x === "boolean"),
startWith(undefined),
);
}
function encryptionErrorObservable(
room: LivekitRoom,
participant: Participant,
encryptionSystem: EncryptionSystem,
criteria: string,
): Observable<boolean> {
return roomEventSelector(room, LivekitRoomEvent.EncryptionError).pipe(
map((e) => {
const [err] = e;
if (encryptionSystem.kind === E2eeType.PER_PARTICIPANT) {
return (
// Ideally we would pull the participant identity from the field on the error.
// However, it gets lost in the serialization process between workers.
// So, instead we do a string match
(err?.message.includes(participant.identity) &&
err?.message.includes(criteria)) ??
false
);
} else if (encryptionSystem.kind === E2eeType.SHARED_KEY) {
return !!err?.message.includes(criteria);
}
return false;
}),
throttleTime(1000), // Throttle to avoid spamming the UI
startWith(false),
);
}
export enum EncryptionStatus {
Connecting,
Okay,
KeyMissing,
KeyInvalid,
PasswordInvalid,
}
abstract class BaseMediaViewModel extends ViewModel {
/**
* Whether the media belongs to the local user.
@@ -112,6 +230,8 @@ abstract class BaseMediaViewModel extends ViewModel {
map((p) => !!p),
);
public readonly encryptionStatus: Observable<EncryptionStatus>;
public constructor(
/**
* An opaque identifier for this media.
@@ -132,6 +252,7 @@ abstract class BaseMediaViewModel extends ViewModel {
encryptionSystem: EncryptionSystem,
audioSource: AudioSource,
videoSource: VideoSource,
livekitRoom: LivekitRoom,
) {
super();
const audio = observeTrackReference(participant, audioSource).pipe(
@@ -146,7 +267,64 @@ abstract class BaseMediaViewModel extends ViewModel {
encryptionSystem.kind !== E2eeType.NONE &&
(a?.publication?.isEncrypted === false ||
v?.publication?.isEncrypted === false),
).pipe(this.scope.state());
).pipe(this.scope.state());
if (participant.isLocal || encryptionSystem.kind === E2eeType.NONE) {
this.encryptionStatus = of(EncryptionStatus.Okay).pipe(
this.scope.state(),
);
} else if (encryptionSystem.kind === E2eeType.PER_PARTICIPANT) {
this.encryptionStatus = combineLatest([
encryptionErrorObservable(
livekitRoom,
participant,
encryptionSystem,
"MissingKey",
),
encryptionErrorObservable(
livekitRoom,
participant,
encryptionSystem,
"InvalidKey",
),
observeRemoteTrackReceivingOkay(participant, audioSource),
observeRemoteTrackReceivingOkay(participant, videoSource),
]).pipe(
map(([keyMissing, keyInvalid, audioOkay, videoOkay]) => {
if (keyMissing) return EncryptionStatus.KeyMissing;
if (keyInvalid) return EncryptionStatus.KeyInvalid;
if (audioOkay || videoOkay) return EncryptionStatus.Okay;
return undefined; // no change
}),
filter((x) => !!x),
startWith(EncryptionStatus.Connecting),
this.scope.state(),
);
} else {
this.encryptionStatus = combineLatest([
encryptionErrorObservable(
livekitRoom,
participant,
encryptionSystem,
"InvalidKey",
),
observeRemoteTrackReceivingOkay(participant, audioSource),
observeRemoteTrackReceivingOkay(participant, videoSource),
]).pipe(
map(
([keyInvalid, audioOkay, videoOkay]):
| EncryptionStatus
| undefined => {
if (keyInvalid) return EncryptionStatus.PasswordInvalid;
if (audioOkay || videoOkay) return EncryptionStatus.Okay;
return undefined; // no change
},
),
filter((x) => !!x),
startWith(EncryptionStatus.Connecting),
this.scope.state(),
);
}
}
}
@@ -201,6 +379,7 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel {
member: RoomMember | undefined,
participant: Observable<LocalParticipant | RemoteParticipant | undefined>,
encryptionSystem: EncryptionSystem,
livekitRoom: LivekitRoom,
) {
super(
id,
@@ -209,6 +388,7 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel {
encryptionSystem,
Track.Source.Microphone,
Track.Source.Camera,
livekitRoom,
);
const media = participant.pipe(
@@ -261,8 +441,9 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel {
member: RoomMember | undefined,
participant: Observable<LocalParticipant | undefined>,
encryptionSystem: EncryptionSystem,
livekitRoom: LivekitRoom,
) {
super(id, member, participant, encryptionSystem);
super(id, member, participant, encryptionSystem, livekitRoom);
}
}
@@ -321,8 +502,9 @@ export class RemoteUserMediaViewModel extends BaseUserMediaViewModel {
member: RoomMember | undefined,
participant: Observable<RemoteParticipant | undefined>,
encryptionSystem: EncryptionSystem,
livekitRoom: LivekitRoom,
) {
super(id, member, participant, encryptionSystem);
super(id, member, participant, encryptionSystem, livekitRoom);
// Sync the local volume with LiveKit
combineLatest([
@@ -353,6 +535,7 @@ export class ScreenShareViewModel extends BaseMediaViewModel {
member: RoomMember | undefined,
participant: Observable<LocalParticipant | RemoteParticipant>,
encryptionSystem: EncryptionSystem,
livekitRoom: LivekitRoom,
) {
super(
id,
@@ -361,6 +544,7 @@ export class ScreenShareViewModel extends BaseMediaViewModel {
encryptionSystem,
Track.Source.ScreenShareAudio,
Track.Source.ScreenShare,
livekitRoom,
);
}
}

32
src/state/OneOnOneLayout.ts Normal file
View File

@@ -0,0 +1,32 @@
/*
Copyright 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { OneOnOneLayout, OneOnOneLayoutMedia } from "./CallViewModel";
import { TileStore } from "./TileStore";
import { GridTileViewModel } from "./TileViewModel";
/**
* Produces a one-on-one layout with the given media.
*/
export function oneOnOneLayout(
media: OneOnOneLayoutMedia,
visibleTiles: Set<GridTileViewModel>,
prevTiles: TileStore,
): [OneOnOneLayout, TileStore] {
const update = prevTiles.from(visibleTiles);
update.registerGridTile(media.local);
update.registerGridTile(media.remote);
const tiles = update.build();
return [
{
type: media.type,
local: tiles.gridTilesByMedia.get(media.local)!,
remote: tiles.gridTilesByMedia.get(media.remote)!,
},
tiles,
];
}

30
src/state/PipLayout.ts Normal file
View File

@@ -0,0 +1,30 @@
/*
Copyright 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { PipLayout, PipLayoutMedia } from "./CallViewModel";
import { TileStore } from "./TileStore";
import { GridTileViewModel } from "./TileViewModel";
/**
* Produces a picture-in-picture layout with the given media.
*/
export function pipLayout(
media: PipLayoutMedia,
visibleTiles: Set<GridTileViewModel>,
prevTiles: TileStore,
): [PipLayout, TileStore] {
const update = prevTiles.from(visibleTiles);
update.registerSpotlight(media.spotlight, true);
const tiles = update.build();
return [
{
type: media.type,
spotlight: tiles.spotlightTile!,
},
tiles,
];
}

36
src/state/SpotlightExpandedLayout.ts Normal file
View File

@@ -0,0 +1,36 @@
/*
Copyright 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import {
SpotlightExpandedLayout,
SpotlightExpandedLayoutMedia,
} from "./CallViewModel";
import { TileStore } from "./TileStore";
import { GridTileViewModel } from "./TileViewModel";
/**
* Produces an expanded spotlight layout with the given media.
*/
export function spotlightExpandedLayout(
media: SpotlightExpandedLayoutMedia,
visibleTiles: Set<GridTileViewModel>,
prevTiles: TileStore,
): [SpotlightExpandedLayout, TileStore] {
const update = prevTiles.from(visibleTiles);
update.registerSpotlight(media.spotlight, true);
if (media.pip !== undefined) update.registerGridTile(media.pip);
const tiles = update.build();
return [
{
type: media.type,
spotlight: tiles.spotlightTile!,
pip: tiles.gridTiles[0],
},
tiles,
];
}

259
src/state/TileStore.ts Normal file
View File

@@ -0,0 +1,259 @@
/*
Copyright 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { BehaviorSubject } from "rxjs";
import { MediaViewModel, UserMediaViewModel } from "./MediaViewModel";
import { GridTileViewModel, SpotlightTileViewModel } from "./TileViewModel";
import { fillGaps } from "../utils/iter";
class SpotlightTileData {
private readonly media_: BehaviorSubject<MediaViewModel[]>;
public get media(): MediaViewModel[] {
return this.media_.value;
}
public set media(value: MediaViewModel[]) {
this.media_.next(value);
}
private readonly maximised_: BehaviorSubject<boolean>;
public get maximised(): boolean {
return this.maximised_.value;
}
public set maximised(value: boolean) {
this.maximised_.next(value);
}
public readonly vm: SpotlightTileViewModel;
public constructor(media: MediaViewModel[], maximised: boolean) {
this.media_ = new BehaviorSubject(media);
this.maximised_ = new BehaviorSubject(maximised);
this.vm = new SpotlightTileViewModel(this.media_, this.maximised_);
}
public destroy(): void {
this.vm.destroy();
}
}
class GridTileData {
private readonly media_: BehaviorSubject<UserMediaViewModel>;
public get media(): UserMediaViewModel {
return this.media_.value;
}
public set media(value: UserMediaViewModel) {
this.media_.next(value);
}
public readonly vm: GridTileViewModel;
public constructor(media: UserMediaViewModel) {
this.media_ = new BehaviorSubject(media);
this.vm = new GridTileViewModel(this.media_);
}
public destroy(): void {
this.vm.destroy();
}
}
/**
* A collection of tiles to be mapped to a layout.
*/
export class TileStore {
private constructor(
private readonly spotlight: SpotlightTileData | null,
private readonly grid: GridTileData[],
) {}
public readonly spotlightTile = this.spotlight?.vm;
public readonly gridTiles = this.grid.map(({ vm }) => vm);
public readonly gridTilesByMedia = new Map(
this.grid.map(({ vm, media }) => [media, vm]),
);
/**
* Creates an an empty collection of tiles.
*/
public static empty(): TileStore {
return new TileStore(null, []);
}
/**
* Creates a builder which can be used to update the collection, passing
* ownership of the tiles to the updated collection.
*/
public from(visibleTiles: Set<GridTileViewModel>): TileStoreBuilder {
return new TileStoreBuilder(
this.spotlight,
this.grid,
(spotlight, grid) => new TileStore(spotlight, grid),
visibleTiles,
);
}
}
/**
* A builder for a new collection of tiles. Will reuse tiles and destroy unused
* tiles from a previous collection where appropriate.
*/
export class TileStoreBuilder {
private spotlight: SpotlightTileData | null = null;
private readonly prevSpotlightSpeaker =
this.prevSpotlight?.media.length === 1 &&
"speaking" in this.prevSpotlight.media[0] &&
this.prevSpotlight.media[0];
private readonly prevGridByMedia = new Map(
this.prevGrid.map((entry, i) => [entry.media, [entry, i]] as const),
);
// The total number of grid entries that we have so far
private numGridEntries = 0;
// A sparse array of grid entries which should be kept in the same spots as
// which they appeared in the previous grid
private readonly stationaryGridEntries: GridTileData[] = new Array(
this.prevGrid.length,
);
// Grid entries which should now enter the visible section of the grid
private readonly visibleGridEntries: GridTileData[] = [];
// Grid entries which should now enter the invisible section of the grid
private readonly invisibleGridEntries: GridTileData[] = [];
public constructor(
private readonly prevSpotlight: SpotlightTileData | null,
private readonly prevGrid: GridTileData[],
private readonly construct: (
spotlight: SpotlightTileData | null,
grid: GridTileData[],
) => TileStore,
private readonly visibleTiles: Set<GridTileViewModel>,
) {}
/**
* Sets the contents of the spotlight tile. If this is never called, there
* will be no spotlight tile.
*/
public registerSpotlight(media: MediaViewModel[], maximised: boolean): void {
if (this.spotlight !== null) throw new Error("Spotlight already set");
if (this.numGridEntries > 0)
throw new Error("Spotlight must be registered before grid tiles");
// Reuse the previous spotlight tile if it exists
if (this.prevSpotlight === null) {
this.spotlight = new SpotlightTileData(media, maximised);
} else {
this.spotlight = this.prevSpotlight;
this.spotlight.media = media;
this.spotlight.maximised = maximised;
}
}
/**
* Sets up a grid tile for the given media. If this is never called for some
* media, then that media will have no grid tile.
*/
public registerGridTile(media: UserMediaViewModel): void {
if (this.spotlight !== null) {
// We actually *don't* want spotlight speakers to appear in both the
// spotlight and the grid, so they're filtered out here
if (!media.local && this.spotlight.media.includes(media)) return;
// When the spotlight speaker changes, we would see one grid tile appear
// and another grid tile disappear. This would be an undesirable layout
// shift, so instead what we do is take the speaker's grid tile and swap
// the media out, so it can remain where it is in the layout.
if (
media === this.prevSpotlightSpeaker &&
this.spotlight.media.length === 1 &&
"speaking" in this.spotlight.media[0] &&
this.prevSpotlightSpeaker !== this.spotlight.media[0]
) {
const prev = this.prevGridByMedia.get(this.spotlight.media[0]);
if (prev !== undefined) {
const [entry, prevIndex] = prev;
const previouslyVisible = this.visibleTiles.has(entry.vm);
const nowVisible = this.visibleTiles.has(
this.prevGrid[this.numGridEntries]?.vm,
);
// If it doesn't need to move between the visible/invisible sections of
// the grid, then we can keep it where it was and swap the media
if (previouslyVisible === nowVisible) {
this.stationaryGridEntries[prevIndex] = entry;
// Do the media swap
entry.media = media;
this.prevGridByMedia.delete(this.spotlight.media[0]);
this.prevGridByMedia.set(media, prev);
} else {
// Create a new tile; this will cause a layout shift but I'm not
// sure there's any other straightforward option in this case
(nowVisible
? this.visibleGridEntries
: this.invisibleGridEntries
).push(new GridTileData(media));
}
this.numGridEntries++;
return;
}
}
}
// Was there previously a tile with this same media?
const prev = this.prevGridByMedia.get(media);
if (prev === undefined) {
// Create a new tile
(this.visibleTiles.has(this.prevGrid[this.numGridEntries]?.vm)
? this.visibleGridEntries
: this.invisibleGridEntries
).push(new GridTileData(media));
} else {
// Reuse the existing tile
const [entry, prevIndex] = prev;
const previouslyVisible = this.visibleTiles.has(entry.vm);
const nowVisible = this.visibleTiles.has(
this.prevGrid[this.numGridEntries]?.vm,
);
// If it doesn't need to move between the visible/invisible sections of
// the grid, then we can keep it exactly where it was previously
if (previouslyVisible === nowVisible)
this.stationaryGridEntries[prevIndex] = entry;
// Otherwise, queue this tile to be moved
else
(nowVisible ? this.visibleGridEntries : this.invisibleGridEntries).push(
entry,
);
}
this.numGridEntries++;
}
/**
* Constructs a new collection of all registered tiles, transferring ownership
* of the tiles to the new collection. Any tiles present in the previous
* collection but not the new collection will be destroyed.
*/
public build(): TileStore {
// Piece together the grid
const grid = [
...fillGaps(this.stationaryGridEntries, [
...this.visibleGridEntries,
...this.invisibleGridEntries,
]),
];
// Destroy unused tiles
if (this.spotlight === null && this.prevSpotlight !== null)
this.prevSpotlight.destroy();
const gridEntries = new Set(grid);
for (const entry of this.prevGrid)
if (!gridEntries.has(entry)) entry.destroy();
return this.construct(this.spotlight, grid);
}
}

43
src/state/TileViewModel.ts Normal file
View File

@@ -0,0 +1,43 @@
/*
Copyright 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { BehaviorSubject, Observable } from "rxjs";
import { ViewModel } from "./ViewModel";
import { MediaViewModel, UserMediaViewModel } from "./MediaViewModel";
let nextId = 0;
function createId(): string {
return (nextId++).toString();
}
export class GridTileViewModel extends ViewModel {
public readonly id = createId();
private readonly visible_ = new BehaviorSubject(false);
/**
* Whether the tile is visible within the current viewport.
*/
public readonly visible: Observable<boolean> = this.visible_;
public setVisible = (value: boolean): void => this.visible_.next(value);
public constructor(public readonly media: Observable<UserMediaViewModel>) {
super();
}
}
export class SpotlightTileViewModel extends ViewModel {
public constructor(
public readonly media: Observable<MediaViewModel[]>,
public readonly maximised: Observable<boolean>,
) {
super();
}
}
export type TileViewModel = GridTileViewModel | SpotlightTileViewModel;

34
src/state/useObservable.ts
View File

@@ -1,34 +0,0 @@
/*
Copyright 2023, 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { Ref, useCallback, useRef } from "react";
import { BehaviorSubject, Observable } from "rxjs";
import { useInitial } from "../useInitial";
/**
* React hook that creates an Observable from a changing value. The Observable
* replays its current value upon subscription and emits whenever the value
* changes.
*/
export function useObservable<T>(value: T): Observable<T> {
const subject = useRef<BehaviorSubject<T>>();
subject.current ??= new BehaviorSubject(value);
if (value !== subject.current.value) subject.current.next(value);
return subject.current;
}
/**
* React hook that creates a ref and an Observable that emits any values
* stored in the ref. The Observable replays the value currently stored in the
* ref upon subscription.
*/
export function useObservableRef<T>(initialValue: T): [Observable<T>, Ref<T>] {
const subject = useInitial(() => new BehaviorSubject(initialValue));
const ref = useCallback((value: T) => subject.next(value), [subject]);
return [subject, ref];
}

11
src/tile/GridTile.test.tsx
View File

@@ -9,12 +9,20 @@ import { RemoteTrackPublication } from "livekit-client";
import { test, expect } from "vitest";
import { render, screen } from "@testing-library/react";
import { axe } from "vitest-axe";
import { of } from "rxjs";
import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession";
import { GridTile } from "./GridTile";
import { withRemoteMedia } from "../utils/test";
import { GridTileViewModel } from "../state/TileViewModel";
import { ReactionsProvider } from "../useReactions";
global.IntersectionObserver = class MockIntersectionObserver {
public observe(): void {}
public unobserve(): void {}
public disconnect(): void {}
} as unknown as typeof IntersectionObserver;
test("GridTile is accessible", async () => {
await withRemoteMedia(
{
@@ -42,11 +50,10 @@ test("GridTile is accessible", async () => {
const { container } = render(
<ReactionsProvider rtcSession={fakeRtcSession}>
<GridTile
vm={vm}
vm={new GridTileViewModel(of(vm))}
onOpenProfile={() => {}}
targetWidth={300}
targetHeight={200}
showVideo
showSpeakingIndicators
/>
</ReactionsProvider>,

30
src/tile/GridTile.tsx
View File

@@ -10,6 +10,7 @@ import {
ReactNode,
forwardRef,
useCallback,
useRef,
useState,
} from "react";
import { animated } from "@react-spring/web";
@@ -44,6 +45,8 @@ import {
import { Slider } from "../Slider";
import { MediaView } from "./MediaView";
import { useLatest } from "../useLatest";
import { GridTileViewModel } from "../state/TileViewModel";
import { useMergedRefs } from "../useMergedRefs";
import { useReactions } from "../useReactions";
interface TileProps {
@@ -52,7 +55,6 @@ interface TileProps {
targetWidth: number;
targetHeight: number;
displayName: string;
showVideo: boolean;
showSpeakingIndicators: boolean;
}
@@ -67,7 +69,6 @@ const UserMediaTile = forwardRef<HTMLDivElement, UserMediaTileProps>(
(
{
vm,
showVideo,
showSpeakingIndicators,
menuStart,
menuEnd,
@@ -80,6 +81,7 @@ const UserMediaTile = forwardRef<HTMLDivElement, UserMediaTileProps>(
const { t } = useTranslation();
const video = useObservableEagerState(vm.video);
const unencryptedWarning = useObservableEagerState(vm.unencryptedWarning);
const encryptionStatus = useObservableEagerState(vm.encryptionStatus);
const audioEnabled = useObservableEagerState(vm.audioEnabled);
const videoEnabled = useObservableEagerState(vm.videoEnabled);
const speaking = useObservableEagerState(vm.speaking);
@@ -94,7 +96,7 @@ const UserMediaTile = forwardRef<HTMLDivElement, UserMediaTileProps>(
},
[vm],
);
const { raisedHands } = useReactions();
const { raisedHands, lowerHand } = useReactions();
const MicIcon = audioEnabled ? MicOnSolidIcon : MicOffSolidIcon;
@@ -113,6 +115,8 @@ const UserMediaTile = forwardRef<HTMLDivElement, UserMediaTileProps>(
);
const handRaised: Date | undefined = raisedHands[vm.member?.userId ?? ""];
const raisedHandOnClick =
vm.local && handRaised ? (): void => void lowerHand() : undefined;
const showSpeaking = showSpeakingIndicators && speaking;
@@ -122,7 +126,8 @@ const UserMediaTile = forwardRef<HTMLDivElement, UserMediaTileProps>(
video={video}
member={vm.member}
unencryptedWarning={unencryptedWarning}
videoEnabled={videoEnabled && showVideo}
encryptionStatus={encryptionStatus}
videoEnabled={videoEnabled}
videoFit={cropVideo ? "cover" : "contain"}
className={classNames(className, styles.tile, {
[styles.speaking]: showSpeaking,
@@ -158,6 +163,7 @@ const UserMediaTile = forwardRef<HTMLDivElement, UserMediaTileProps>(
</Menu>
}
raisedHandTime={handRaised}
raisedHandOnClick={raisedHandOnClick}
{...props}
/>
);
@@ -287,25 +293,27 @@ const RemoteUserMediaTile = forwardRef<
RemoteUserMediaTile.displayName = "RemoteUserMediaTile";
interface GridTileProps {
vm: UserMediaViewModel;
vm: GridTileViewModel;
onOpenProfile: (() => void) | null;
targetWidth: number;
targetHeight: number;
className?: string;
style?: ComponentProps<typeof animated.div>["style"];
showVideo: boolean;
showSpeakingIndicators: boolean;
}
export const GridTile = forwardRef<HTMLDivElement, GridTileProps>(
({ vm, onOpenProfile, ...props }, ref) => {
const displayName = useDisplayName(vm);
({ vm, onOpenProfile, ...props }, theirRef) => {
const ourRef = useRef<HTMLDivElement | null>(null);
const ref = useMergedRefs(ourRef, theirRef);
const media = useObservableEagerState(vm.media);
const displayName = useDisplayName(media);
if (vm instanceof LocalUserMediaViewModel) {
if (media instanceof LocalUserMediaViewModel) {
return (
<LocalUserMediaTile
ref={ref}
vm={vm}
vm={media}
onOpenProfile={onOpenProfile}
displayName={displayName}
{...props}
@@ -315,7 +323,7 @@ export const GridTile = forwardRef<HTMLDivElement, GridTileProps>(
return (
<RemoteUserMediaTile
ref={ref}
vm={vm}
vm={media}
displayName={displayName}
{...props}
/>

21
src/tile/MediaView.module.css
View File

@@ -85,7 +85,7 @@ unconditionally select the container so we can use cqmin units */
display: grid;
grid-template-columns: 1fr auto;
grid-template-rows: 1fr auto;
grid-template-areas: ". ." "nameTag button";
grid-template-areas: "status status" "nameTag button";
gap: var(--cpd-space-1x);
place-items: start;
}
@@ -106,6 +106,25 @@ unconditionally select the container so we can use cqmin units */
max-inline-size: 100%;
}
.status {
grid-area: status;
justify-self: center;
align-self: start;
padding: var(--cpd-space-1x);
padding-block: var(--cpd-space-1x);
color: var(--cpd-color-text-primary);
background-color: var(--cpd-color-bg-canvas-default);
display: flex;
align-items: center;
border-radius: var(--cpd-radius-pill-effect);
user-select: none;
overflow: hidden;
box-shadow: var(--small-drop-shadow);
box-sizing: border-box;
max-inline-size: 100%;
text-align: center;
}
.nameTag > svg,
.nameTag > span {
flex-shrink: 0;

26
src/tile/MediaView.tsx
View File

@@ -17,6 +17,7 @@ import { ErrorIcon } from "@vector-im/compound-design-tokens/assets/web/icons";
import styles from "./MediaView.module.css";
import { Avatar } from "../Avatar";
import { EncryptionStatus } from "../state/MediaViewModel";
import { RaisedHandIndicator } from "../reactions/RaisedHandIndicator";
import { showHandRaisedTimer, useSetting } from "../settings/settings";
@@ -31,10 +32,12 @@ interface Props extends ComponentProps<typeof animated.div> {
member: RoomMember | undefined;
videoEnabled: boolean;
unencryptedWarning: boolean;
encryptionStatus: EncryptionStatus;
nameTagLeadingIcon?: ReactNode;
displayName: string;
primaryButton?: ReactNode;
raisedHandTime?: Date;
raisedHandOnClick?: () => void;
}
export const MediaView = forwardRef<HTMLDivElement, Props>(
@@ -53,7 +56,9 @@ export const MediaView = forwardRef<HTMLDivElement, Props>(
nameTagLeadingIcon,
displayName,
primaryButton,
encryptionStatus,
raisedHandTime,
raisedHandOnClick,
...props
},
ref,
@@ -67,7 +72,11 @@ export const MediaView = forwardRef<HTMLDivElement, Props>(
<animated.div
className={classNames(styles.media, className, {
[styles.mirror]: mirror,
[styles.videoMuted]: !videoEnabled,
[styles.videoMuted]:
!videoEnabled ||
![EncryptionStatus.Connecting, EncryptionStatus.Okay].includes(
encryptionStatus,
),
})}
style={style}
ref={ref}
@@ -93,10 +102,25 @@ export const MediaView = forwardRef<HTMLDivElement, Props>(
)}
</div>
<div className={styles.fg}>
{encryptionStatus !== EncryptionStatus.Okay && (
<div className={styles.status}>
<Text as="span" size="sm" weight="medium" className={styles.name}>
{encryptionStatus === EncryptionStatus.Connecting &&
t("e2ee_encryption_status.connecting")}
{encryptionStatus === EncryptionStatus.KeyMissing &&
t("e2ee_encryption_status.key_missing")}
{encryptionStatus === EncryptionStatus.KeyInvalid &&
t("e2ee_encryption_status.key_invalid")}
{encryptionStatus === EncryptionStatus.PasswordInvalid &&
t("e2ee_encryption_status.password_invalid")}
</Text>
</div>
)}
<RaisedHandIndicator
raisedHandTime={raisedHandTime}
minature={avatarSize < 96}
showTimer={handRaiseTimerVisible}
onClick={raisedHandOnClick}
/>
{/* {keys &&
keys.map(({ index, key }) => (

5
src/tile/SpotlightTile.test.tsx
View File

@@ -9,9 +9,11 @@ import { test, expect, vi } from "vitest";
import { isInaccessible, render, screen } from "@testing-library/react";
import { axe } from "vitest-axe";
import userEvent from "@testing-library/user-event";
import { of } from "rxjs";
import { SpotlightTile } from "./SpotlightTile";
import { withLocalMedia, withRemoteMedia } from "../utils/test";
import { SpotlightTileViewModel } from "../state/TileViewModel";
global.IntersectionObserver = class MockIntersectionObserver {
public observe(): void {}
@@ -36,10 +38,9 @@ test("SpotlightTile is accessible", async () => {
const toggleExpanded = vi.fn();
const { container } = render(
<SpotlightTile
vms={[vm1, vm2]}
vm={new SpotlightTileViewModel(of([vm1, vm2]), of(false))}
targetWidth={300}
targetHeight={200}
maximised={false}
expanded={false}
onToggleExpanded={toggleExpanded}
showIndicators

60
src/tile/SpotlightTile.tsx
View File

@@ -22,7 +22,7 @@ import {
} from "@vector-im/compound-design-tokens/assets/web/icons";
import { animated } from "@react-spring/web";
import { Observable, map } from "rxjs";
import { useObservableEagerState } from "observable-hooks";
import { useObservableEagerState, useObservableRef } from "observable-hooks";
import { useTranslation } from "react-i18next";
import classNames from "classnames";
import { TrackReferenceOrPlaceholder } from "@livekit/components-core";
@@ -31,6 +31,7 @@ import { RoomMember } from "matrix-js-sdk/src/matrix";
import { MediaView } from "./MediaView";
import styles from "./SpotlightTile.module.css";
import {
EncryptionStatus,
LocalUserMediaViewModel,
MediaViewModel,
ScreenShareViewModel,
@@ -39,9 +40,9 @@ import {
} from "../state/MediaViewModel";
import { useInitial } from "../useInitial";
import { useMergedRefs } from "../useMergedRefs";
import { useObservableRef } from "../state/useObservable";
import { useReactiveState } from "../useReactiveState";
import { useLatest } from "../useLatest";
import { SpotlightTileViewModel } from "../state/TileViewModel";
interface SpotlightItemBaseProps {
className?: string;
@@ -51,6 +52,7 @@ interface SpotlightItemBaseProps {
video: TrackReferenceOrPlaceholder | undefined;
member: RoomMember | undefined;
unencryptedWarning: boolean;
encryptionStatus: EncryptionStatus;
displayName: string;
"aria-hidden"?: boolean;
}
@@ -86,14 +88,16 @@ const SpotlightUserMediaItem = forwardRef<
const videoEnabled = useObservableEagerState(vm.videoEnabled);
const cropVideo = useObservableEagerState(vm.cropVideo);
const baseProps: SpotlightUserMediaItemBaseProps = {
const baseProps: SpotlightUserMediaItemBaseProps &
RefAttributes<HTMLDivElement> = {
ref,
videoEnabled,
videoFit: cropVideo ? "cover" : "contain",
...props,
};
return vm instanceof LocalUserMediaViewModel ? (
<SpotlightLocalUserMediaItem ref={ref} vm={vm} {...baseProps} />
<SpotlightLocalUserMediaItem vm={vm} {...baseProps} />
) : (
<MediaView mirror={false} {...baseProps} />
);
@@ -130,6 +134,7 @@ const SpotlightItem = forwardRef<HTMLDivElement, SpotlightItemProps>(
const displayName = useDisplayName(vm);
const video = useObservableEagerState(vm.video);
const unencryptedWarning = useObservableEagerState(vm.unencryptedWarning);
const encryptionStatus = useObservableEagerState(vm.encryptionStatus);
// Hook this item up to the intersection observer
useEffect(() => {
@@ -156,6 +161,7 @@ const SpotlightItem = forwardRef<HTMLDivElement, SpotlightItemProps>(
member: vm.member,
unencryptedWarning,
displayName,
encryptionStatus,
"aria-hidden": ariaHidden,
};
@@ -175,8 +181,7 @@ const SpotlightItem = forwardRef<HTMLDivElement, SpotlightItemProps>(
SpotlightItem.displayName = "SpotlightItem";
interface Props {
vms: MediaViewModel[];
maximised: boolean;
vm: SpotlightTileViewModel;
expanded: boolean;
onToggleExpanded: (() => void) | null;
targetWidth: number;
@@ -189,8 +194,7 @@ interface Props {
export const SpotlightTile = forwardRef<HTMLDivElement, Props>(
(
{
vms,
maximised,
vm,
expanded,
onToggleExpanded,
targetWidth,
@@ -202,14 +206,16 @@ export const SpotlightTile = forwardRef<HTMLDivElement, Props>(
theirRef,
) => {
const { t } = useTranslation();
const [root, ourRef] = useObservableRef<HTMLDivElement | null>(null);
const [ourRef, root] = useObservableRef<HTMLDivElement | null>(null);
const ref = useMergedRefs(ourRef, theirRef);
const [visibleId, setVisibleId] = useState(vms[0].id);
const latestVms = useLatest(vms);
const maximised = useObservableEagerState(vm.maximised);
const media = useObservableEagerState(vm.media);
const [visibleId, setVisibleId] = useState(media[0].id);
const latestMedia = useLatest(media);
const latestVisibleId = useLatest(visibleId);
const visibleIndex = vms.findIndex((vm) => vm.id === visibleId);
const visibleIndex = media.findIndex((vm) => vm.id === visibleId);
const canGoBack = visibleIndex > 0;
const canGoToNext = visibleIndex !== -1 && visibleIndex < vms.length - 1;
const canGoToNext = visibleIndex !== -1 && visibleIndex < media.length - 1;
// To keep track of which item is visible, we need an intersection observer
// hooked up to the root element and the items. Because the items will run
@@ -234,28 +240,30 @@ export const SpotlightTile = forwardRef<HTMLDivElement, Props>(
const [scrollToId, setScrollToId] = useReactiveState<string | null>(
(prev) =>
prev == null || prev === visibleId || vms.every((vm) => vm.id !== prev)
prev == null ||
prev === visibleId ||
media.every((vm) => vm.id !== prev)
? null
: prev,
[visibleId],
);
const onBackClick = useCallback(() => {
const vms = latestVms.current;
const visibleIndex = vms.findIndex(
const media = latestMedia.current;
const visibleIndex = media.findIndex(
(vm) => vm.id === latestVisibleId.current,
);
if (visibleIndex > 0) setScrollToId(vms[visibleIndex - 1].id);
}, [latestVisibleId, latestVms, setScrollToId]);
if (visibleIndex > 0) setScrollToId(media[visibleIndex - 1].id);
}, [latestVisibleId, latestMedia, setScrollToId]);
const onNextClick = useCallback(() => {
const vms = latestVms.current;
const visibleIndex = vms.findIndex(
const media = latestMedia.current;
const visibleIndex = media.findIndex(
(vm) => vm.id === latestVisibleId.current,
);
if (visibleIndex !== -1 && visibleIndex !== vms.length - 1)
setScrollToId(vms[visibleIndex + 1].id);
}, [latestVisibleId, latestVms, setScrollToId]);
if (visibleIndex !== -1 && visibleIndex !== media.length - 1)
setScrollToId(media[visibleIndex + 1].id);
}, [latestVisibleId, latestMedia, setScrollToId]);
const ToggleExpandIcon = expanded ? CollapseIcon : ExpandIcon;
@@ -277,7 +285,7 @@ export const SpotlightTile = forwardRef<HTMLDivElement, Props>(
</button>
)}
<div className={styles.contents}>
{vms.map((vm) => (
{media.map((vm) => (
<SpotlightItem
key={vm.id}
vm={vm}
@@ -316,10 +324,10 @@ export const SpotlightTile = forwardRef<HTMLDivElement, Props>(
{!expanded && (
<div
className={classNames(styles.indicators, {
[styles.show]: showIndicators && vms.length > 1,
[styles.show]: showIndicators && media.length > 1,
})}
>
{vms.map((vm) => (
{media.map((vm) => (
<div
key={vm.id}
className={styles.item}

12
src/useReactions.test.tsx
View File

@@ -46,7 +46,7 @@ const membership: Record<string, string> = {
};
const TestComponent: FC = () => {
const { raisedHands, myReactionId } = useReactions();
const { raisedHands } = useReactions();
return (
<div>
<ul>
@@ -57,7 +57,6 @@ const TestComponent: FC = () => {
</li>
))}
</ul>
<p>{myReactionId ? "Local reaction" : "No local reaction"}</p>
</div>
);
};
@@ -175,15 +174,6 @@ describe("useReactions", () => {
);
expect(queryByRole("list")?.children).to.have.lengthOf(0);
});
test("handles own raised hand", async () => {
const room = new MockRoom();
const rtcSession = new MockRTCSession(room);
const { queryByText } = render(
<TestComponentWrapper rtcSession={rtcSession} />,
);
await act(() => room.testSendReaction(memberEventAlice));
expect(queryByText("Local reaction")).toBeTruthy();
});
test("handles incoming raised hand", async () => {
const room = new MockRoom();
const rtcSession = new MockRTCSession(room);

36
src/useReactions.tsx
View File

@@ -30,7 +30,7 @@ import { useClientState } from "./ClientContext";
interface ReactionsContextType {
raisedHands: Record<string, Date>;
supportsReactions: boolean;
myReactionId: string | null;
lowerHand: () => Promise<void>;
}
const ReactionsContext = createContext<ReactionsContextType | undefined>(
@@ -80,13 +80,6 @@ export const ReactionsProvider = ({
const room = rtcSession.room;
const myUserId = room.client.getUserId();
// Calculate our own reaction event.
const myReactionId = useMemo(
(): string | null =>
(myUserId && raisedHands[myUserId]?.reactionEventId) ?? null,
[raisedHands, myUserId],
);
// Reduce the data down for the consumers.
const resultRaisedHands = useMemo(
() =>
@@ -235,12 +228,37 @@ export const ReactionsProvider = ({
};
}, [room, addRaisedHand, removeRaisedHand, memberships, raisedHands]);
const lowerHand = useCallback(async () => {
if (
!myUserId ||
clientState?.state !== "valid" ||
!clientState.authenticated ||
!raisedHands[myUserId]
) {
return;
}
const myReactionId = raisedHands[myUserId].reactionEventId;
if (!myReactionId) {
logger.warn(`Hand raised but no reaction event to redact!`);
return;
}
try {
await clientState.authenticated.client.redactEvent(
rtcSession.room.roomId,
myReactionId,
);
logger.debug("Redacted raise hand event");
} catch (ex) {
logger.error("Failed to redact reaction event", myReactionId, ex);
}
}, [myUserId, raisedHands, clientState, rtcSession]);
return (
<ReactionsContext.Provider
value={{
raisedHands: resultRaisedHands,
supportsReactions,
myReactionId,
lowerHand,
}}
>
{children}

22
src/utils/iter.test.ts Normal file
View File

@@ -0,0 +1,22 @@
/*
Copyright 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { test, expect } from "vitest";
import { fillGaps } from "./iter";
test("fillGaps filters out gaps", () => {
expect([
...fillGaps([1, undefined, undefined, undefined, 3], [2]),
]).toStrictEqual([1, 2, 3]);
});
test("fillGaps adds extra filler elements to the end", () => {
expect([
...fillGaps([1, undefined, 3, undefined], [2, 4, 5, 6]),
]).toStrictEqual([1, 2, 3, 4, 5, 6]);
});

36
src/utils/iter.ts Normal file
View File

@@ -0,0 +1,36 @@
/*
Copyright 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
/**
* Fills in the 'undefined' gaps in a collection by drawing items from a second
* collection, or simply filtering out the gap if no items are left. If filler
* items remain at the end, they will be appended to the resulting collection.
*/
export function fillGaps<A>(
gappy: Iterable<A | undefined>,
filler: Iterable<A>,
): Iterable<A> {
return {
[Symbol.iterator](): Iterator<A> {
const gappyIter = gappy[Symbol.iterator]();
const fillerIter = filler[Symbol.iterator]();
return {
next(): IteratorResult<A> {
let gappyItem: IteratorResult<A | undefined>;
do {
gappyItem = gappyIter.next();
if (!gappyItem.done && gappyItem.value !== undefined)
return gappyItem as IteratorYieldResult<A>;
const fillerItem = fillerIter.next();
if (!fillerItem.done) return fillerItem;
} while (!gappyItem.done);
return gappyItem;
},
};
},
};
}

33
src/utils/test.ts
View File

@@ -4,7 +4,7 @@ Copyright 2023, 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { map, of } from "rxjs";
import { map, Observable, of } from "rxjs";
import { RunHelpers, TestScheduler } from "rxjs/testing";
import { expect, vi } from "vitest";
import {
@@ -130,8 +130,27 @@ export function mockMatrixRoom(room: Partial<MatrixRoom>): MatrixRoom {
return { ...mockEmitter(), ...room } as Partial<MatrixRoom> as MatrixRoom;
}
export function mockLivekitRoom(room: Partial<LivekitRoom>): LivekitRoom {
return { ...mockEmitter(), ...room } as Partial<LivekitRoom> as LivekitRoom;
export function mockLivekitRoom(
room: Partial<LivekitRoom>,
{
remoteParticipants,
}: { remoteParticipants?: Observable<RemoteParticipant[]> } = {},
): LivekitRoom {
const livekitRoom = {
...mockEmitter(),
...room,
} as Partial<LivekitRoom> as LivekitRoom;
if (remoteParticipants) {
livekitRoom.remoteParticipants = new Map();
remoteParticipants.subscribe((newRemoteParticipants) => {
livekitRoom.remoteParticipants.clear();
newRemoteParticipants.forEach((p) => {
livekitRoom.remoteParticipants.set(p.identity, p);
});
});
}
return livekitRoom;
}
export function mockLocalParticipant(
@@ -150,13 +169,15 @@ export async function withLocalMedia(
member: Partial<RoomMember>,
continuation: (vm: LocalUserMediaViewModel) => void | Promise<void>,
): Promise<void> {
const localParticipant = mockLocalParticipant({});
const vm = new LocalUserMediaViewModel(
"local",
mockRoomMember(member),
of(mockLocalParticipant({})),
of(localParticipant),
{
kind: E2eeType.PER_PARTICIPANT,
},
mockLivekitRoom({ localParticipant }),
);
try {
await continuation(vm);
@@ -183,13 +204,15 @@ export async function withRemoteMedia(
participant: Partial<RemoteParticipant>,
continuation: (vm: RemoteUserMediaViewModel) => void | Promise<void>,
): Promise<void> {
const remoteParticipant = mockRemoteParticipant(participant);
const vm = new RemoteUserMediaViewModel(
"remote",
mockRoomMember(member),
of(mockRemoteParticipant(participant)),
of(remoteParticipant),
{
kind: E2eeType.PER_PARTICIPANT,
},
mockLivekitRoom({}, { remoteParticipants: of([remoteParticipant]) }),
);
try {
await continuation(vm);