Matrix/element-call-Github
1
0
Fork 0
mirror of https://github.com/vector-im/element-call.git synced 2026-03-31 07:00:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/livekit' into hs/reactions-viewcallmodel

Browse Source
This commit is contained in:
Half-Shot
2024-12-16 15:46:09 +00:00
parent 5c580bed14 e4bd9d7cf9
commit ada70007cc
49 changed files with 833 additions and 570 deletions
Download Patch File Download Diff File Expand all files Collapse all files

80
src/grid/Grid.tsx
View File

@@ -24,7 +24,6 @@ import {
createContext,
forwardRef,
memo,
useCallback,
useContext,
useEffect,
useMemo,
@@ -54,7 +53,6 @@ interface Tile<Model> {
id: string;
model: Model;
onDrag: DragCallback | undefined;
setVisible: (visible: boolean) => void;
}
type PlacedTile<Model> = Tile<Model> & Rect;
@@ -88,7 +86,6 @@ interface SlotProps<Model> extends Omit<ComponentProps<"div">, "onDrag"> {
id: string;
model: Model;
onDrag?: DragCallback;
onVisibilityChange?: (visible: boolean) => void;
style?: CSSProperties;
className?: string;
}
@@ -115,24 +112,47 @@ function offset(element: HTMLElement, relativeTo: Element): Offset {
}
}
export type VisibleTilesCallback = (visibleTiles: number) => void;
interface LayoutContext {
setGeneration: Dispatch<SetStateAction<number | null>>;
setVisibleTilesCallback: Dispatch<
SetStateAction<VisibleTilesCallback | null>
>;
}
const LayoutContext = createContext<LayoutContext | null>(null);
function useLayoutContext(): LayoutContext {
const context = useContext(LayoutContext);
if (context === null)
throw new Error("useUpdateLayout called outside a Grid layout context");
return context;
}
/**
* Enables Grid to react to layout changes. You must call this in your Layout
* component or else Grid will not be reactive.
*/
export function useUpdateLayout(): void {
const context = useContext(LayoutContext);
if (context === null)
throw new Error("useUpdateLayout called outside a Grid layout context");
const { setGeneration } = useLayoutContext();
// On every render, tell Grid that the layout may have changed
useEffect(() =>
context.setGeneration((prev) => (prev === null ? 0 : prev + 1)),
useEffect(() => setGeneration((prev) => (prev === null ? 0 : prev + 1)));
}
/**
* Asks Grid to call a callback whenever the number of visible tiles may have
* changed.
*/
export function useVisibleTiles(callback: VisibleTilesCallback): void {
const { setVisibleTilesCallback } = useLayoutContext();
useEffect(
() => setVisibleTilesCallback(() => callback),
[callback, setVisibleTilesCallback],
);
useEffect(
() => (): void => setVisibleTilesCallback(null),
[setVisibleTilesCallback],
);
}
@@ -245,39 +265,20 @@ export function Grid<
const windowHeight = useObservableEagerState(windowHeightObservable);
const [layoutRoot, setLayoutRoot] = useState<HTMLElement | null>(null);
const [generation, setGeneration] = useState<number | null>(null);
const [visibleTilesCallback, setVisibleTilesCallback] =
useState<VisibleTilesCallback | null>(null);
const tiles = useInitial(() => new Map<string, Tile<TileModel>>());
const prefersReducedMotion = usePrefersReducedMotion();
const Slot: FC<SlotProps<TileModel>> = useMemo(
() =>
function Slot({
id,
model,
onDrag,
onVisibilityChange,
style,
className,
...props
}) {
function Slot({ id, model, onDrag, style, className, ...props }) {
const ref = useRef<HTMLDivElement | null>(null);
const prevVisible = useRef<boolean | null>(null);
const setVisible = useCallback(
(visible: boolean) => {
if (
onVisibilityChange !== undefined &&
visible !== prevVisible.current
) {
onVisibilityChange(visible);
prevVisible.current = visible;
}
},
[onVisibilityChange],
);
useEffect(() => {
tiles.set(id, { id, model, onDrag, setVisible });
tiles.set(id, { id, model, onDrag });
return (): void => void tiles.delete(id);
}, [id, model, onDrag, setVisible]);
}, [id, model, onDrag]);
return (
<div
@@ -307,7 +308,10 @@ export function Grid<
[],
);
const context: LayoutContext = useMemo(() => ({ setGeneration }), []);
const context: LayoutContext = useMemo(
() => ({ setGeneration, setVisibleTilesCallback }),
[setVisibleTilesCallback],
);
// Combine the tile definitions and slots together to create placed tiles
const placedTiles = useMemo(() => {
@@ -342,9 +346,11 @@ export function Grid<
);
useEffect(() => {
for (const tile of placedTiles)
tile.setVisible(tile.y + tile.height <= visibleHeight);
}, [placedTiles, visibleHeight]);
visibleTilesCallback?.(
placedTiles.filter((tile) => tile.y + tile.height <= visibleHeight)
.length,
);
}, [placedTiles, visibleTilesCallback, visibleHeight]);
// Drag state is stored in a ref rather than component state, because we use
// react-spring's imperative API during gestures to improve responsiveness

11
src/grid/GridLayout.tsx
View File

@@ -13,7 +13,7 @@ import { type GridLayout as GridLayoutModel } from "../state/CallViewModel";
import styles from "./GridLayout.module.css";
import { useInitial } from "../useInitial";
import { type CallLayout, arrangeTiles } from "./CallLayout";
import { type DragCallback, useUpdateLayout } from "./Grid";
import { type DragCallback, useUpdateLayout, useVisibleTiles } from "./Grid";
interface GridCSSProperties extends CSSProperties {
"--gap": string;
@@ -73,6 +73,7 @@ export const makeGridLayout: CallLayout<GridLayoutModel> = ({
// The scrolling part of the layout is where all the grid tiles live
scrolling: forwardRef(function GridLayout({ model, Slot }, ref) {
useUpdateLayout();
useVisibleTiles(model.setVisibleTiles);
const { width, height: minHeight } = useObservableEagerState(minBounds);
const { gap, tileWidth, tileHeight } = useMemo(
() => arrangeTiles(width, minHeight, model.grid.length),
@@ -93,13 +94,7 @@ export const makeGridLayout: CallLayout<GridLayoutModel> = ({
}
>
{model.grid.map((m) => (
<Slot
key={m.id}
className={styles.slot}
id={m.id}
model={m}
onVisibilityChange={m.setVisible}
/>
<Slot key={m.id} className={styles.slot} id={m.id} model={m} />
))}
</div>
);

2
src/grid/OneOnOneLayout.tsx
View File

@@ -52,7 +52,6 @@ export const makeOneOnOneLayout: CallLayout<OneOnOneLayoutModel> = ({
<Slot
id={model.remote.id}
model={model.remote}
onVisibilityChange={model.remote.setVisible}
className={styles.container}
style={{ width: tileWidth, height: tileHeight }}
>
@@ -61,7 +60,6 @@ export const makeOneOnOneLayout: CallLayout<OneOnOneLayoutModel> = ({
id={model.local.id}
model={model.local}
onDrag={onDragLocalTile}
onVisibilityChange={model.local.setVisible}
data-block-alignment={pipAlignmentValue.block}
data-inline-alignment={pipAlignmentValue.inline}
/>

1
src/grid/SpotlightExpandedLayout.tsx
View File

@@ -63,7 +63,6 @@ export const makeSpotlightExpandedLayout: CallLayout<
id={model.pip.id}
model={model.pip}
onDrag={onDragPip}
onVisibilityChange={model.pip.setVisible}
data-block-alignment={pipAlignmentValue.block}
data-inline-alignment={pipAlignmentValue.inline}
/>

11
src/grid/SpotlightLandscapeLayout.tsx
View File

@@ -12,7 +12,7 @@ import classNames from "classnames";
import { type CallLayout } from "./CallLayout";
import { type SpotlightLandscapeLayout as SpotlightLandscapeLayoutModel } from "../state/CallViewModel";
import styles from "./SpotlightLandscapeLayout.module.css";
import { useUpdateLayout } from "./Grid";
import { useUpdateLayout, useVisibleTiles } from "./Grid";
/**
* An implementation of the "spotlight landscape" layout, in which the spotlight
@@ -50,6 +50,7 @@ export const makeSpotlightLandscapeLayout: CallLayout<
ref,
) {
useUpdateLayout();
useVisibleTiles(model.setVisibleTiles);
useObservableEagerState(minBounds);
const withIndicators =
useObservableEagerState(model.spotlight.media).length > 1;
@@ -63,13 +64,7 @@ export const makeSpotlightLandscapeLayout: CallLayout<
/>
<div className={styles.grid}>
{model.grid.map((m) => (
<Slot
key={m.id}
className={styles.slot}
id={m.id}
model={m}
onVisibilityChange={m.setVisible}
/>
<Slot key={m.id} className={styles.slot} id={m.id} model={m} />
))}
</div>
</div>

11
src/grid/SpotlightPortraitLayout.tsx
View File

@@ -12,7 +12,7 @@ import classNames from "classnames";
import { type CallLayout, arrangeTiles } from "./CallLayout";
import { type SpotlightPortraitLayout as SpotlightPortraitLayoutModel } from "../state/CallViewModel";
import styles from "./SpotlightPortraitLayout.module.css";
import { useUpdateLayout } from "./Grid";
import { useUpdateLayout, useVisibleTiles } from "./Grid";
interface GridCSSProperties extends CSSProperties {
"--grid-gap": string;
@@ -54,6 +54,7 @@ export const makeSpotlightPortraitLayout: CallLayout<
ref,
) {
useUpdateLayout();
useVisibleTiles(model.setVisibleTiles);
const { width } = useObservableEagerState(minBounds);
const { gap, tileWidth, tileHeight } = arrangeTiles(
width,
@@ -84,13 +85,7 @@ export const makeSpotlightPortraitLayout: CallLayout<
/>
<div className={styles.grid}>
{model.grid.map((m) => (
<Slot
key={m.id}
className={styles.slot}
id={m.id}
model={m}
onVisibilityChange={m.setVisible}
/>
<Slot key={m.id} className={styles.slot} id={m.id} model={m} />
))}
</div>
</div>

137
src/livekit/MediaDevicesContext.tsx
View File

@@ -16,7 +16,8 @@ import {
useState,
} from "react";
import { createMediaDeviceObserver } from "@livekit/components-core";
import { type Observable } from "rxjs";
import { map, startWith } from "rxjs";
import { useObservableEagerState } from "observable-hooks";
import { logger } from "matrix-js-sdk/src/logger";
import {
@@ -26,11 +27,25 @@ import {
videoInput as videoInputSetting,
type Setting,
} from "../settings/settings";
import { isFirefox } from "../Platform";
export type DeviceLabel =
| { type: "name"; name: string }
| { type: "number"; number: number }
| { type: "default"; name: string | null };
export interface MediaDevice {
available: MediaDeviceInfo[];
/**
* A map from available device IDs to labels.
*/
available: Map<string, DeviceLabel>;
selectedId: string | undefined;
/**
* The group ID of the selected device.
*/
// This is exposed sort of ad-hoc because it's only needed for knowing when to
// restart the tracks of default input devices, and ideally this behavior
// would be encapsulated somehow…
selectedGroupId: string | undefined;
select: (deviceId: string) => void;
}
@@ -42,26 +57,10 @@ export interface MediaDevices {
stopUsingDeviceNames: () => void;
}
// Cargo-culted from @livekit/components-react
function useObservableState<T>(
observable: Observable<T> | undefined,
startWith: T,
): T {
const [state, setState] = useState<T>(startWith);
useEffect(() => {
// observable state doesn't run in SSR
if (typeof window === "undefined" || !observable) return;
const subscription = observable.subscribe(setState);
return (): void => subscription.unsubscribe();
}, [observable]);
return state;
}
function useMediaDevice(
kind: MediaDeviceKind,
setting: Setting<string | undefined>,
usingNames: boolean,
alwaysDefault: boolean = false,
): MediaDevice {
// Make sure we don't needlessly reset to a device observer without names,
// once permissions are already given
@@ -81,43 +80,91 @@ function useMediaDevice(
kind,
() => logger.error("Error creating MediaDeviceObserver"),
requestPermissions,
),
).pipe(startWith([])),
[kind, requestPermissions],
);
const available = useObservableState(deviceObserver, []);
const [preferredId, select] = useSetting(setting);
const available = useObservableEagerState(
useMemo(
() =>
deviceObserver.pipe(
map((availableRaw) => {
// Sometimes browsers (particularly Firefox) can return multiple device
// entries for the exact same device ID; using a map deduplicates them
let available = new Map<string, DeviceLabel>(
availableRaw.map((d, i) => [
d.deviceId,
d.label
? { type: "name", name: d.label }
: { type: "number", number: i + 1 },
]),
);
// Create a virtual default audio output for browsers that don't have one.
// Its device ID must be the empty string because that's what setSinkId
// recognizes.
if (
kind === "audiooutput" &&
available.size &&
!available.has("") &&
!available.has("default")
)
available = new Map([
["", { type: "default", name: availableRaw[0]?.label || null }],
...available,
]);
// Note: creating virtual default input devices would be another problem
// entirely, because requesting a media stream from deviceId "" won't
// automatically track the default device.
return available;
}),
),
[kind, deviceObserver],
),
);
return useMemo(() => {
let selectedId: string | undefined = undefined;
if (!alwaysDefault && available) {
const [preferredId, select] = useSetting(setting);
const selectedId = useMemo(() => {
if (available.size) {
// If the preferred device is available, use it. Or if every available
// device ID is falsy, the browser is probably just being paranoid about
// fingerprinting and we should still try using the preferred device.
// Worst case it is not available and the browser will gracefully fall
// back to some other device for us when requesting the media stream.
// Otherwise, select the first available device.
selectedId =
available.some((d) => d.deviceId === preferredId) ||
available.every((d) => d.deviceId === "")
? preferredId
: available.at(0)?.deviceId;
return (preferredId !== undefined && available.has(preferredId)) ||
(available.size === 1 && available.has(""))
? preferredId
: available.keys().next().value;
}
return undefined;
}, [available, preferredId]);
const selectedGroupId = useObservableEagerState(
useMemo(
() =>
deviceObserver.pipe(
map(
(availableRaw) =>
availableRaw.find((d) => d.deviceId === selectedId)?.groupId,
),
),
[deviceObserver, selectedId],
),
);
return {
available: available
? // Sometimes browsers (particularly Firefox) can return multiple
// device entries for the exact same device ID; deduplicate them
[...new Map(available.map((d) => [d.deviceId, d])).values()]
: [],
return useMemo(
() => ({
available,
selectedId,
selectedGroupId,
select,
};
}, [available, preferredId, select, alwaysDefault]);
}),
[available, selectedId, selectedGroupId, select],
);
}
export const deviceStub: MediaDevice = {
available: [],
available: new Map(),
selectedId: undefined,
selectedGroupId: undefined,
select: () => {},
};
export const devicesStub: MediaDevices = {
@@ -139,15 +186,6 @@ export const MediaDevicesProvider: FC<Props> = ({ children }) => {
const [numCallersUsingNames, setNumCallersUsingNames] = useState(0);
const usingNames = numCallersUsingNames > 0;
// Setting the audio device to something other than 'undefined' breaks echo-cancellation
// and even can introduce multiple different output devices for one call.
const alwaysUseDefaultAudio = isFirefox();
// On FF we dont need to query the names
// (call enumerateDevices + create meadia stream to trigger permissions)
// for ouput devices because the selector wont be shown on FF.
const useOutputNames = usingNames && !isFirefox();
const audioInput = useMediaDevice(
"audioinput",
audioInputSetting,
@@ -156,8 +194,7 @@ export const MediaDevicesProvider: FC<Props> = ({ children }) => {
const audioOutput = useMediaDevice(
"audiooutput",
audioOutputSetting,
useOutputNames,
alwaysUseDefaultAudio,
usingNames,
);
const videoInput = useMediaDevice(
"videoinput",

10
src/livekit/useLiveKit.ts
View File

@@ -7,7 +7,7 @@ Please see LICENSE in the repository root for full details.
import {
ConnectionState,
type E2EEOptions,
type E2EEManagerOptions,
ExternalE2EEKeyProvider,
Room,
type RoomOptions,
@@ -45,7 +45,7 @@ export function useLiveKit(
sfuConfig: SFUConfig | undefined,
e2eeSystem: EncryptionSystem,
): UseLivekitResult {
const e2eeOptions = useMemo((): E2EEOptions | undefined => {
const e2eeOptions = useMemo((): E2EEManagerOptions | undefined => {
if (e2eeSystem.kind === E2eeType.NONE) return undefined;
if (e2eeSystem.kind === E2eeType.PER_PARTICIPANT) {
@@ -290,18 +290,14 @@ export function useLiveKit(
room.localParticipant.audioTrackPublications.values(),
).find((d) => d.source === Track.Source.Microphone)?.track;
const defaultDevice = device.available.find(
(d) => d.deviceId === "default",
);
if (
defaultDevice &&
activeMicTrack &&
// only restart if the stream is still running: LiveKit will detect
// when a track stops & restart appropriately, so this is not our job.
// Plus, we need to avoid restarting again if the track is already in
// the process of being restarted.
activeMicTrack.mediaStreamTrack.readyState !== "ended" &&
defaultDevice.groupId !==
device.selectedGroupId !==
activeMicTrack.mediaStreamTrack.getSettings().groupId
) {
// It's different, so restart the track, ie. cause Livekit to do another

1
src/room/InCallView.tsx
View File

@@ -134,7 +134,6 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
useEffect(() => {
if (livekitRoom !== undefined && reader !== undefined) {
console.log("Creating new VM");
const vm = new CallViewModel(
props.rtcSession,
livekitRoom,

29
src/room/MuteStates.test.tsx
View File

@@ -6,13 +6,13 @@ Please see LICENSE in the repository root for full details.
*/
import { afterAll, afterEach, describe, expect, it, vi } from "vitest";
import React, { type ReactNode } from "react";
import { beforeEach } from "vitest";
import { type ReactNode } from "react";
import { render, screen } from "@testing-library/react";
import { MemoryRouter } from "react-router-dom";
import { useMuteStates } from "./MuteStates";
import {
type DeviceLabel,
type MediaDevice,
type MediaDevices,
MediaDevicesContext,
@@ -63,10 +63,11 @@ const mockCamera: MediaDeviceInfo = {
},
};
function mockDevices(available: MediaDeviceInfo[]): MediaDevice {
function mockDevices(available: Map<string, DeviceLabel>): MediaDevice {
return {
available,
selectedId: "",
selectedGroupId: "",
select: (): void => {},
};
}
@@ -83,25 +84,29 @@ function mockMediaDevices(
} = { microphone: true, speaker: true, camera: true },
): MediaDevices {
return {
audioInput: mockDevices(microphone ? [mockMicrophone] : []),
audioOutput: mockDevices(speaker ? [mockSpeaker] : []),
videoInput: mockDevices(camera ? [mockCamera] : []),
audioInput: mockDevices(
microphone
? new Map([[mockMicrophone.deviceId, mockMicrophone]])
: new Map(),
),
audioOutput: mockDevices(
speaker ? new Map([[mockSpeaker.deviceId, mockSpeaker]]) : new Map(),
),
videoInput: mockDevices(
camera ? new Map([[mockCamera.deviceId, mockCamera]]) : new Map(),
),
startUsingDeviceNames: (): void => {},
stopUsingDeviceNames: (): void => {},
};
}
describe("useMuteStates", () => {
beforeEach(() => {
vi.spyOn(React, "useContext").mockReturnValue({});
});
afterEach(() => {
vi.restoreAllMocks();
vi.clearAllMocks();
});
afterAll(() => {
vi.clearAllMocks();
vi.resetAllMocks();
});
it("disabled when no input devices", () => {

4
src/room/MuteStates.ts
View File

@@ -58,12 +58,12 @@ function useMuteState(
): MuteState {
const [enabled, setEnabled] = useReactiveState<boolean | undefined>(
(prev) =>
device.available.length > 0 ? (prev ?? enabledByDefault()) : undefined,
device.available.size > 0 ? (prev ?? enabledByDefault()) : undefined,
[device],
);
return useMemo(
() =>
device.available.length === 0
device.available.size === 0
? deviceUnavailable
: {
enabled: enabled ?? false,

18
src/settings/DeveloperSettingsTab.tsx
View File

@@ -13,6 +13,7 @@ import {
useSetting,
duplicateTiles as duplicateTilesSetting,
debugTileLayout as debugTileLayoutSetting,
showNonMemberTiles as showNonMemberTilesSetting,
} from "./settings";
import type { MatrixClient } from "matrix-js-sdk/src/client";
@@ -26,6 +27,9 @@ export const DeveloperSettingsTab: FC<Props> = ({ client }) => {
const [debugTileLayout, setDebugTileLayout] = useSetting(
debugTileLayoutSetting,
);
const [showNonMemberTiles, setShowNonMemberTiles] = useSetting(
showNonMemberTilesSetting,
);
return (
<>
@@ -85,6 +89,20 @@ export const DeveloperSettingsTab: FC<Props> = ({ client }) => {
}
/>
</FieldRow>
<FieldRow>
<InputField
id="showNonMemberTiles"
type="checkbox"
label={t("developer_mode.show_non_member_tiles")}
checked={!!showNonMemberTiles}
onChange={useCallback(
(event: ChangeEvent<HTMLInputElement>): void => {
setShowNonMemberTiles(event.target.checked);
},
[setShowNonMemberTiles],
)}
/>
</FieldRow>
</>
);
};

4
src/settings/DeviceSelection.module.css
View File

@@ -16,3 +16,7 @@
flex-direction: column;
gap: var(--cpd-space-4x);
}
.secondary {
color: var(--cpd-color-text-secondary);
}

86
src/settings/DeviceSelection.tsx
View File

@@ -5,7 +5,14 @@ SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { type ChangeEvent, type FC, useCallback, useId } from "react";
import {
type ChangeEvent,
type FC,
type ReactElement,
type ReactNode,
useCallback,
useId,
} from "react";
import {
Heading,
InlineField,
@@ -13,16 +20,23 @@ import {
RadioControl,
Separator,
} from "@vector-im/compound-web";
import { Trans, useTranslation } from "react-i18next";
import { type MediaDevice } from "../livekit/MediaDevicesContext";
import styles from "./DeviceSelection.module.css";
interface Props {
devices: MediaDevice;
caption: string;
title: string;
numberedLabel: (number: number) => string;
}
export const DeviceSelection: FC<Props> = ({ devices, caption }) => {
export const DeviceSelection: FC<Props> = ({
devices,
title,
numberedLabel,
}) => {
const { t } = useTranslation();
const groupId = useId();
const onChange = useCallback(
(e: ChangeEvent<HTMLInputElement>) => {
@@ -31,7 +45,7 @@ export const DeviceSelection: FC<Props> = ({ devices, caption }) => {
[devices],
);
if (devices.available.length == 0) return null;
if (devices.available.size == 0) return null;
return (
<div className={styles.selection}>
@@ -42,29 +56,53 @@ export const DeviceSelection: FC<Props> = ({ devices, caption }) => {
as="h4"
className={styles.title}
>
{caption}
{title}
</Heading>
<Separator className={styles.separator} />
<div className={styles.options}>
{devices.available.map(({ deviceId, label }, index) => (
<InlineField
key={deviceId}
name={groupId}
control={
<RadioControl
checked={deviceId === devices.selectedId}
onChange={onChange}
value={deviceId}
/>
}
>
<Label>
{!!label && label.trim().length > 0
? label
: `${caption} ${index + 1}`}
</Label>
</InlineField>
))}
{[...devices.available].map(([id, label]) => {
let labelText: ReactNode;
switch (label.type) {
case "name":
labelText = label.name;
break;
case "number":
labelText = numberedLabel(label.number);
break;
case "default":
labelText =
label.name === null ? (
t("settings.devices.default")
) : (
<Trans
i18nKey="settings.devices.default_named"
name={label.name}
>
Default{" "}
<span className={styles.secondary}>
({{ name: label.name } as unknown as ReactElement})
</span>
</Trans>
);
break;
}
return (
<InlineField
key={id}
name={groupId}
control={
<RadioControl
checked={id === devices.selectedId}
onChange={onChange}
value={id}
/>
}
>
<Label>{labelText}</Label>
</InlineField>
);
})}
</div>
</div>
);

6
src/settings/PreferencesSettingsTab.tsx
View File

@@ -41,13 +41,13 @@ export const PreferencesSettingsTab: FC = () => {
return (
<div>
<Text>{t("settings.preferences_tab_body")}</Text>
<Text>{t("settings.preferences_tab.introduction")}</Text>
<FieldRow>
<InputField
id="showHandRaisedTimer"
label={t("settings.preferences_tab_show_hand_raised_timer_label")}
label={t("settings.preferences_tab.show_hand_raised_timer_label")}
description={t(
"settings.preferences_tab_show_hand_raised_timer_description",
"settings.preferences_tab.show_hand_raised_timer_description",
)}
type="checkbox"
checked={showHandRaisedTimer}

20
src/settings/SettingsModal.tsx
View File

@@ -25,7 +25,6 @@ import {
soundEffectVolumeSetting,
developerMode,
} from "./settings";
import { isFirefox } from "../Platform";
import { PreferencesSettingsTab } from "./PreferencesSettingsTab";
import { Slider } from "../Slider";
import { DeviceSelection } from "./DeviceSelection";
@@ -76,14 +75,16 @@ export const SettingsModal: FC<Props> = ({
<Form>
<DeviceSelection
devices={devices.audioInput}
caption={t("common.microphone")}
title={t("settings.devices.microphone")}
numberedLabel={(n) =>
t("settings.devices.microphone_numbered", { n })
}
/>
<DeviceSelection
devices={devices.audioOutput}
title={t("settings.devices.speaker")}
numberedLabel={(n) => t("settings.devices.speaker_numbered", { n })}
/>
{!isFirefox() && (
<DeviceSelection
devices={devices.audioOutput}
caption={t("settings.speaker_device_selection_label")}
/>
)}
<div className={styles.volumeSlider}>
<label>{t("settings.audio_tab.effect_volume_label")}</label>
<p>{t("settings.audio_tab.effect_volume_description")}</p>
@@ -109,7 +110,8 @@ export const SettingsModal: FC<Props> = ({
<Form>
<DeviceSelection
devices={devices.videoInput}
caption={t("common.camera")}
title={t("settings.devices.camera")}
numberedLabel={(n) => t("settings.devices.camera_numbered", { n })}
/>
</Form>
),

4
src/settings/settings.ts
View File

@@ -72,6 +72,10 @@ export const developerMode = new Setting("developer-settings-tab", false);
export const duplicateTiles = new Setting("duplicate-tiles", 0);
export const showNonMemberTiles = new Setting<boolean>(
"show-non-member-tiles",
false,
);
export const debugTileLayout = new Setting("debug-tile-layout", false);
export const audioInput = new Setting<string | undefined>(

118
src/state/CallViewModel.test.ts
View File

@@ -48,6 +48,7 @@ import {
} from "../livekit/useECConnectionState";
import { E2eeType } from "../e2ee/e2eeType";
import type { RaisedHandInfo } from "../reactions";
import { showNonMemberTiles } from "../settings/settings";
vi.mock("@livekit/components-core");
@@ -385,16 +386,16 @@ test("screen sharing activates spotlight layout", () => {
test("participants stay in the same order unless to appear/disappear", () => {
withTestScheduler(({ hot, schedule, expectObservable }) => {
const modeInputMarbles = " a";
const visibilityInputMarbles = "a";
// First Bob speaks, then Dave, then Alice
const aSpeakingInputMarbles = "n- 1998ms - 1999ms y";
const bSpeakingInputMarbles = "ny 1998ms n 1999ms -";
const dSpeakingInputMarbles = "n- 1998ms y 1999ms n";
const aSpeakingInputMarbles = " n- 1998ms - 1999ms y";
const bSpeakingInputMarbles = " ny 1998ms n 1999ms -";
const dSpeakingInputMarbles = " n- 1998ms y 1999ms n";
// Nothing should change when Bob speaks, because Bob is already on screen.
// When Dave speaks he should switch with Alice because she's the one who
// hasn't spoken at all. Then when Alice speaks, she should return to her
// place at the top.
const expectedLayoutMarbles = "a 1999ms b 1999ms a 57999ms c 1999ms a";
const expectedLayoutMarbles = " a 1999ms b 1999ms a 57999ms c 1999ms a";
withCallViewModel(
of([aliceParticipant, bobParticipant, daveParticipant]),
@@ -406,15 +407,12 @@ test("participants stay in the same order unless to appear/disappear", () => {
[daveParticipant, hot(dSpeakingInputMarbles, { y: true, n: false })],
]),
(vm) => {
schedule(modeInputMarbles, {
schedule(visibilityInputMarbles, {
a: () => {
// We imagine that only three tiles (the first three) will be visible
// on screen at a time
vm.layout.subscribe((layout) => {
if (layout.type === "grid") {
for (let i = 0; i < layout.grid.length; i++)
layout.grid[i].setVisible(i < 3);
}
if (layout.type === "grid") layout.setVisibleTiles(3);
});
},
});
@@ -444,6 +442,56 @@ test("participants stay in the same order unless to appear/disappear", () => {
});
});
test("participants adjust order when space becomes constrained", () => {
withTestScheduler(({ hot, schedule, expectObservable }) => {
// Start with all tiles on screen then shrink to 3
const visibilityInputMarbles = "a-b";
// Bob and Dave speak
const bSpeakingInputMarbles = " ny";
const dSpeakingInputMarbles = " ny";
// Nothing should change when Bob or Dave initially speak, because they are
// on screen. When the screen becomes smaller Alice should move off screen
// to make way for the speakers (specifically, she should swap with Dave).
const expectedLayoutMarbles = " a-b";
withCallViewModel(
of([aliceParticipant, bobParticipant, daveParticipant]),
of([aliceRtcMember, bobRtcMember, daveRtcMember]),
of(ConnectionState.Connected),
new Map([
[bobParticipant, hot(bSpeakingInputMarbles, { y: true, n: false })],
[daveParticipant, hot(dSpeakingInputMarbles, { y: true, n: false })],
]),
(vm) => {
let setVisibleTiles: ((value: number) => void) | null = null;
vm.layout.subscribe((layout) => {
if (layout.type === "grid") setVisibleTiles = layout.setVisibleTiles;
});
schedule(visibilityInputMarbles, {
a: () => setVisibleTiles!(Infinity),
b: () => setVisibleTiles!(3),
});
expectObservable(summarizeLayout(vm.layout)).toBe(
expectedLayoutMarbles,
{
a: {
type: "grid",
spotlight: undefined,
grid: ["local:0", `${aliceId}:0`, `${bobId}:0`, `${daveId}:0`],
},
b: {
type: "grid",
spotlight: undefined,
grid: ["local:0", `${daveId}:0`, `${bobId}:0`, `${aliceId}:0`],
},
},
);
},
);
});
});
test("spotlight speakers swap places", () => {
withTestScheduler(({ hot, schedule, expectObservable }) => {
// Go immediately into spotlight mode for the test
@@ -648,6 +696,53 @@ test("participants must have a MatrixRTCSession to be visible", () => {
});
});
test("shows participants without MatrixRTCSession when enabled in settings", () => {
try {
// enable the setting:
showNonMemberTiles.setValue(true);
withTestScheduler(({ hot, expectObservable }) => {
const scenarioInputMarbles = " abc";
const expectedLayoutMarbles = "abc";
withCallViewModel(
hot(scenarioInputMarbles, {
a: [],
b: [aliceParticipant],
c: [aliceParticipant, bobParticipant],
}),
of([]), // No one joins the MatrixRTC session
of(ConnectionState.Connected),
new Map(),
(vm) => {
vm.setGridMode("grid");
expectObservable(summarizeLayout(vm.layout)).toBe(
expectedLayoutMarbles,
{
a: {
type: "grid",
spotlight: undefined,
grid: ["local:0"],
},
b: {
type: "one-on-one",
local: "local:0",
remote: `${aliceId}:0`,
},
c: {
type: "grid",
spotlight: undefined,
grid: ["local:0", `${aliceId}:0`, `${bobId}:0`],
},
},
);
},
);
});
} finally {
showNonMemberTiles.setValue(showNonMemberTiles.defaultValue);
}
});
it("should show at least one tile per MatrixRTCSession", () => {
withTestScheduler(({ hot, expectObservable }) => {
// iterate through some combinations of MatrixRTC memberships
@@ -714,8 +809,7 @@ it("should rank raised hands above video feeds and below speakers and presenters
// on screen at a time
vm.layout.subscribe((layout) => {
if (layout.type === "grid") {
for (let i = 0; i < layout.grid.length; i++)
layout.grid[i].setVisible(i <= 1);
layout.setVisibleTiles(2);
}
});
},

134
src/state/CallViewModel.ts
View File

@@ -73,6 +73,7 @@ import {
duplicateTiles,
playReactionsSound,
showReactions,
showNonMemberTiles,
} from "../settings/settings";
import { isFirefox } from "../Platform";
import { setPipEnabled } from "../controls";
@@ -92,6 +93,7 @@ import {
type ReactionInfo,
type ReactionOption,
} from "../reactions";
import { shallowEquals } from "../utils/array";
// How long we wait after a focus switch before showing the real participant
// list again
@@ -152,18 +154,21 @@ export interface GridLayout {
type: "grid";
spotlight?: SpotlightTileViewModel;
grid: GridTileViewModel[];
setVisibleTiles: (value: number) => void;
}
export interface SpotlightLandscapeLayout {
type: "spotlight-landscape";
spotlight: SpotlightTileViewModel;
grid: GridTileViewModel[];
setVisibleTiles: (value: number) => void;
}
export interface SpotlightPortraitLayout {
type: "spotlight-portrait";
spotlight: SpotlightTileViewModel;
grid: GridTileViewModel[];
setVisibleTiles: (value: number) => void;
}
export interface SpotlightExpandedLayout {
@@ -236,7 +241,6 @@ enum SortingBin {
interface LayoutScanState {
layout: Layout | null;
tiles: TileStore;
visibleTiles: Set<GridTileViewModel>;
}
class UserMedia {
@@ -466,6 +470,7 @@ export class CallViewModel extends ViewModel {
this.matrixRTCSession,
MatrixRTCSessionEvent.MembershipsChanged,
).pipe(startWith(null)),
showNonMemberTiles.value,
]).pipe(
scan(
(
@@ -475,6 +480,7 @@ export class CallViewModel extends ViewModel {
{ participant: localParticipant },
duplicateTiles,
_membershipsChanged,
showNonMemberTiles,
],
) => {
const newItems = new Map(
@@ -514,9 +520,17 @@ export class CallViewModel extends ViewModel {
}
for (let i = 0; i < 1 + duplicateTiles; i++) {
const indexedMediaId = `${livekitParticipantId}:${i}`;
const prevMedia = prevItems.get(indexedMediaId);
let prevMedia = prevItems.get(indexedMediaId);
if (prevMedia && prevMedia instanceof UserMedia) {
prevMedia.updateParticipant(participant);
if (prevMedia.vm.member === undefined) {
// We have a previous media created because of the `debugShowNonMember` flag.
// In this case we actually replace the media item.
// This "hack" never occurs if we do not use the `debugShowNonMember` debugging
// option and if we always find a room member for each rtc member (which also
// only fails if we have a fundamental problem)
prevMedia = undefined;
}
}
yield [
indexedMediaId,
@@ -558,7 +572,57 @@ export class CallViewModel extends ViewModel {
}.bind(this)(),
);
return newItems;
// Generate non member items (items without a corresponding MatrixRTC member)
// Those items should not be rendered, they are participants in LiveKit that do not have a corresponding
// MatrixRTC members. This cannot be any good:
// - A malicious user impersonates someone
// - Someone injects abusive content
// - The user cannot have encryption keys so it makes no sense to participate
// We can only trust users that have a MatrixRTC member event.
//
// This is still available as a debug option. This can be useful
// - If one wants to test scalability using the LiveKit CLI.
// - If an experimental project does not yet do the MatrixRTC bits.
// - If someone wants to debug if the LiveKit connection works but MatrixRTC room state failed to arrive.
const newNonMemberItems = showNonMemberTiles
? new Map(
function* (this: CallViewModel): Iterable<[string, MediaItem]> {
for (const participant of remoteParticipants) {
for (let i = 0; i < 1 + duplicateTiles; i++) {
const maybeNonMemberParticipantId =
participant.identity + ":" + i;
if (!newItems.has(maybeNonMemberParticipantId)) {
const nonMemberId = maybeNonMemberParticipantId;
yield [
nonMemberId,
prevItems.get(nonMemberId) ??
new UserMedia(
nonMemberId,
undefined,
participant,
this.encryptionSystem,
this.livekitRoom,
of(null),
of(null),
),
];
}
}
}
}.bind(this)(),
)
: new Map();
if (newNonMemberItems.size > 0) {
logger.debug("Added NonMember items: ", newNonMemberItems);
}
const combinedNew = new Map([
...newNonMemberItems.entries(),
...newItems.entries(),
]);
for (const [id, t] of prevItems) if (!combinedNew.has(id)) t.destroy();
return combinedNew;
},
new Map<string, MediaItem>(),
),
@@ -674,6 +738,8 @@ export class CallViewModel extends ViewModel {
bins.sort(([, bin1], [, bin2]) => bin1 - bin2).map(([m]) => m.vm),
);
}),
distinctUntilChanged(shallowEquals),
this.scope.state(),
);
private readonly spotlight: Observable<MediaViewModel[]> =
@@ -687,6 +753,7 @@ export class CallViewModel extends ViewModel {
map((speaker) => (speaker ? [speaker] : [])),
);
}),
distinctUntilChanged(shallowEquals),
this.scope.state(),
);
@@ -920,62 +987,53 @@ export class CallViewModel extends ViewModel {
this.scope.state(),
);
// There is a cyclical dependency here: the layout algorithms want to know
// which tiles are on screen, but to know which tiles are on screen we have to
// first render a layout. To deal with this we assume initially that no tiles
// are visible, and loop the data back into the layouts with a Subject.
private readonly visibleTiles = new Subject<number>();
private readonly setVisibleTiles = (value: number): void =>
this.visibleTiles.next(value);
public readonly layoutInternals: Observable<
LayoutScanState & { layout: Layout }
> = this.layoutMedia.pipe(
// Each layout will produce a set of tiles, and these tiles have an
// observable indicating whether they're visible. We loop this information
// back into the layout process by using switchScan.
switchScan<
LayoutMedia,
LayoutScanState,
Observable<LayoutScanState & { layout: Layout }>
> = combineLatest([
this.layoutMedia,
this.visibleTiles.pipe(startWith(0), distinctUntilChanged()),
]).pipe(
scan<
[LayoutMedia, number],
LayoutScanState & { layout: Layout },
LayoutScanState
>(
({ tiles: prevTiles, visibleTiles }, media) => {
({ tiles: prevTiles }, [media, visibleTiles]) => {
let layout: Layout;
let newTiles: TileStore;
switch (media.type) {
case "grid":
case "spotlight-landscape":
case "spotlight-portrait":
[layout, newTiles] = gridLikeLayout(media, visibleTiles, prevTiles);
break;
case "spotlight-expanded":
[layout, newTiles] = spotlightExpandedLayout(
[layout, newTiles] = gridLikeLayout(
media,
visibleTiles,
this.setVisibleTiles,
prevTiles,
);
break;
case "spotlight-expanded":
[layout, newTiles] = spotlightExpandedLayout(media, prevTiles);
break;
case "one-on-one":
[layout, newTiles] = oneOnOneLayout(media, visibleTiles, prevTiles);
[layout, newTiles] = oneOnOneLayout(media, prevTiles);
break;
case "pip":
[layout, newTiles] = pipLayout(media, visibleTiles, prevTiles);
[layout, newTiles] = pipLayout(media, prevTiles);
break;
}
// Take all of the 'visible' observables and combine them into one big
// observable array
const visibilities =
newTiles.gridTiles.length === 0
? of([])
: combineLatest(newTiles.gridTiles.map((tile) => tile.visible));
return visibilities.pipe(
map((visibilities) => ({
layout: layout,
tiles: newTiles,
visibleTiles: new Set(
newTiles.gridTiles.filter((_tile, i) => visibilities[i]),
),
})),
);
},
{
layout: null,
tiles: TileStore.empty(),
visibleTiles: new Set(),
return { layout, tiles: newTiles };
},
{ layout: null, tiles: TileStore.empty() },
),
this.scope.state(),
);

5
src/state/GridLikeLayout.ts
View File

@@ -7,7 +7,6 @@ Please see LICENSE in the repository root for full details.
import { type Layout, type LayoutMedia } from "./CallViewModel";
import { type TileStore } from "./TileStore";
import { type GridTileViewModel } from "./TileViewModel";
export type GridLikeLayoutType =
| "grid"
@@ -20,7 +19,8 @@ export type GridLikeLayoutType =
*/
export function gridLikeLayout(
media: LayoutMedia & { type: GridLikeLayoutType },
visibleTiles: Set<GridTileViewModel>,
visibleTiles: number,
setVisibleTiles: (value: number) => void,
prevTiles: TileStore,
): [Layout & { type: GridLikeLayoutType }, TileStore] {
const update = prevTiles.from(visibleTiles);
@@ -37,6 +37,7 @@ export function gridLikeLayout(
type: media.type,
spotlight: tiles.spotlightTile,
grid: tiles.gridTiles,
setVisibleTiles,
} as Layout & { type: GridLikeLayoutType },
tiles,
];

4
src/state/OneOnOneLayout.ts
View File

@@ -7,17 +7,15 @@ Please see LICENSE in the repository root for full details.
import { type OneOnOneLayout, type OneOnOneLayoutMedia } from "./CallViewModel";
import { type TileStore } from "./TileStore";
import { type GridTileViewModel } from "./TileViewModel";
/**
* Produces a one-on-one layout with the given media.
*/
export function oneOnOneLayout(
media: OneOnOneLayoutMedia,
visibleTiles: Set<GridTileViewModel>,
prevTiles: TileStore,
): [OneOnOneLayout, TileStore] {
const update = prevTiles.from(visibleTiles);
const update = prevTiles.from(2);
update.registerGridTile(media.local);
update.registerGridTile(media.remote);
const tiles = update.build();

4
src/state/PipLayout.ts
View File

@@ -7,17 +7,15 @@ Please see LICENSE in the repository root for full details.
import { type PipLayout, type PipLayoutMedia } from "./CallViewModel";
import { type TileStore } from "./TileStore";
import { type GridTileViewModel } from "./TileViewModel";
/**
* Produces a picture-in-picture layout with the given media.
*/
export function pipLayout(
media: PipLayoutMedia,
visibleTiles: Set<GridTileViewModel>,
prevTiles: TileStore,
): [PipLayout, TileStore] {
const update = prevTiles.from(visibleTiles);
const update = prevTiles.from(0);
update.registerSpotlight(media.spotlight, true);
const tiles = update.build();
return [

4
src/state/SpotlightExpandedLayout.ts
View File

@@ -10,17 +10,15 @@ import {
type SpotlightExpandedLayoutMedia,
} from "./CallViewModel";
import { type TileStore } from "./TileStore";
import { type GridTileViewModel } from "./TileViewModel";
/**
* Produces an expanded spotlight layout with the given media.
*/
export function spotlightExpandedLayout(
media: SpotlightExpandedLayoutMedia,
visibleTiles: Set<GridTileViewModel>,
prevTiles: TileStore,
): [SpotlightExpandedLayout, TileStore] {
const update = prevTiles.from(visibleTiles);
const update = prevTiles.from(1);
update.registerSpotlight(media.spotlight, true);
if (media.pip !== undefined) update.registerGridTile(media.pip);
const tiles = update.build();

18
src/state/TileStore.ts
View File

@@ -101,7 +101,7 @@ export class TileStore {
* Creates a builder which can be used to update the collection, passing
* ownership of the tiles to the updated collection.
*/
public from(visibleTiles: Set<GridTileViewModel>): TileStoreBuilder {
public from(visibleTiles: number): TileStoreBuilder {
return new TileStoreBuilder(
this.spotlight,
this.grid,
@@ -146,7 +146,7 @@ export class TileStoreBuilder {
spotlight: SpotlightTileData | null,
grid: GridTileData[],
) => TileStore,
private readonly visibleTiles: Set<GridTileViewModel>,
private readonly visibleTiles: number,
/**
* A number incremented on each update, just for debugging purposes.
*/
@@ -204,10 +204,8 @@ export class TileStoreBuilder {
const prev = this.prevGridByMedia.get(this.spotlight.media[0]);
if (prev !== undefined) {
const [entry, prevIndex] = prev;
const previouslyVisible = this.visibleTiles.has(entry.vm);
const nowVisible = this.visibleTiles.has(
this.prevGrid[this.numGridEntries]?.vm,
);
const previouslyVisible = prevIndex < this.visibleTiles;
const nowVisible = this.numGridEntries < this.visibleTiles;
// If it doesn't need to move between the visible/invisible sections of
// the grid, then we can keep it where it was and swap the media
@@ -236,17 +234,15 @@ export class TileStoreBuilder {
const prev = this.prevGridByMedia.get(media);
if (prev === undefined) {
// Create a new tile
(this.visibleTiles.has(this.prevGrid[this.numGridEntries]?.vm)
(this.numGridEntries < this.visibleTiles
? this.visibleGridEntries
: this.invisibleGridEntries
).push(new GridTileData(media));
} else {
// Reuse the existing tile
const [entry, prevIndex] = prev;
const previouslyVisible = this.visibleTiles.has(entry.vm);
const nowVisible = this.visibleTiles.has(
this.prevGrid[this.numGridEntries]?.vm,
);
const previouslyVisible = prevIndex < this.visibleTiles;
const nowVisible = this.numGridEntries < this.visibleTiles;
// If it doesn't need to move between the visible/invisible sections of
// the grid, then we can keep it exactly where it was previously
if (previouslyVisible === nowVisible)

10
src/state/TileViewModel.ts
View File

@@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { BehaviorSubject, type Observable } from "rxjs";
import { type Observable } from "rxjs";
import { ViewModel } from "./ViewModel";
import { type MediaViewModel, type UserMediaViewModel } from "./MediaViewModel";
@@ -18,14 +18,6 @@ function createId(): string {
export class GridTileViewModel extends ViewModel {
public readonly id = createId();
private readonly visible_ = new BehaviorSubject(false);
/**
* Whether the tile is visible within the current viewport.
*/
public readonly visible: Observable<boolean> = this.visible_;
public setVisible = (value: boolean): void => this.visible_.next(value);
public constructor(public readonly media: Observable<UserMediaViewModel>) {
super();
}

3
src/useAudioContext.test.tsx
View File

@@ -101,7 +101,8 @@ test("will use the correct device", () => {
audioInput: deviceStub,
audioOutput: {
selectedId: "chosen-device",
available: [],
selectedGroupId: "",
available: new Map(),
select: () => {},
},
videoInput: deviceStub,

16
src/utils/array.ts Normal file
View File

@@ -0,0 +1,16 @@
/*
Copyright 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
/**
* Determine whether two arrays are equal by shallow comparison.
*/
export function shallowEquals<A>(first: A[], second: A[]): boolean {
if (first.length !== second.length) return false;
for (let i = 0; i < first.length; i++)
if (first[i] !== second[i]) return false;
return true;
}