Merge branch 'livekit' into more-more-performance

This commit is contained in:
Robin
2026-08-17 11:12:17 +02:00
42 changed files with 2025 additions and 813 deletions

View File

@@ -31,6 +31,11 @@ interface Props {
max: number;
step: number;
disabled?: boolean;
/**
* Custom formatter for the tooltip label. If not provided, the value is
* displayed as a percentage.
*/
tooltipFormatter?: (value: number) => string;
}
/**
@@ -46,6 +51,7 @@ export const Slider: FC<Props> = ({
max,
step,
disabled,
tooltipFormatter,
}) => {
const onValueChange = useCallback(
([v]: number[]) => onValueChangeProp(v),
@@ -71,7 +77,14 @@ export const Slider: FC<Props> = ({
<Range className={styles.highlight} />
</Track>
{/* Note: This is expected not to be visible on mobile.*/}
<Tooltip placement="top" label={Math.round(value * 100).toString() + "%"}>
<Tooltip
placement="top"
label={
tooltipFormatter
? tooltipFormatter(value)
: Math.round(value * 100).toString() + "%"
}
>
<Thumb className={styles.handle} aria-label={label} />
</Tooltip>
</Root>

View File

@@ -74,10 +74,7 @@ export interface ConfigOptions {
livekit?: {
// The link to the service that returns a livekit url and token to use it.
// This is a fallback link in case the homeserver in use does not advertise
// a livekit service url in the client well-known.
// The well known needs to be formatted like so:
// {"type":"livekit", "livekit_service_url":"https://livekit.example.com"}
// and stored under the key: "org.matrix.msc4143.rtc_foci"
// a livekit service url over the transports endpoint.
livekit_service_url: string;
};
@@ -105,6 +102,61 @@ export interface ConfigOptions {
*/
ssla?: string;
/**
* Media quality settings for video and screen sharing.
* These override the hardcoded LiveKit defaults.
*/
media_quality?: {
/**
* Video codec preference. The server must also have the codec enabled.
* @default "vp8"
*/
video_codec?: "vp8" | "vp9" | "h264" | "av1";
/**
* Camera video settings.
*/
video?: {
/** Max resolution height in pixels (e.g. 720, 1080, 1440). @default 720 */
max_resolution?: number;
/** Max bitrate in bits per second. @default 1700000 */
max_bitrate?: number;
/** Max framerate. @default 30 */
max_framerate?: number;
/**
* Simulcast layers as an array of {height, bitrate} objects,
* ordered from lowest to highest quality.
* @default [{height: 180, bitrate: 160000}, {height: 360, bitrate: 450000}]
*/
simulcast_layers?: Array<{
height: number;
bitrate: number;
}>;
};
/**
* Screen share settings.
*/
screen_share?: {
/** Max resolution height in pixels. @default 1080 */
max_resolution?: number;
/** Max bitrate in bits per second. @default 5000000 */
max_bitrate?: number;
/** Max framerate. @default 30 */
max_framerate?: number;
/**
* Simulcast layers for screen sharing as an array of {height, bitrate, framerate} objects,
* ordered from lowest to highest quality. If omitted, LiveKit SDK defaults apply (1 extra
* layer at half resolution).
*/
simulcast_layers?: Array<{
height: number;
bitrate: number;
framerate?: number;
}>;
};
};
media_devices?: {
/**
* Defines whether participants should start with audio enabled by default.
@@ -185,6 +237,24 @@ export interface ConfigOptions {
export interface ResolvedConfigOptions extends ConfigOptions {
sync_disconnect_grace_period_ms: number;
ssla: string;
media_quality: Required<
Pick<NonNullable<ConfigOptions["media_quality"]>, "video_codec">
> & {
video: Required<
Pick<
NonNullable<NonNullable<ConfigOptions["media_quality"]>["video"]>,
"max_resolution" | "max_bitrate" | "max_framerate"
>
>;
screen_share: Required<
Pick<
NonNullable<
NonNullable<ConfigOptions["media_quality"]>["screen_share"]
>,
"max_resolution" | "max_bitrate" | "max_framerate"
>
>;
};
matrix_rtc_session: {
wait_for_key_rotation_ms?: number;
delayed_leave_event_delay_ms: number;
@@ -201,6 +271,19 @@ export const DEFAULT_CONFIG: ResolvedConfigOptions = {
},
sync_disconnect_grace_period_ms: 10000,
ssla: "https://static.element.io/legal/element-software-and-services-license-agreement-uk-1.pdf",
media_quality: {
video_codec: "vp8",
video: {
max_resolution: 720,
max_bitrate: 1_700_000,
max_framerate: 30,
},
screen_share: {
max_resolution: 1080,
max_bitrate: 5_000_000,
max_framerate: 30,
},
},
matrix_rtc_session: {
delayed_leave_event_delay_ms: 10000,
network_error_retry_ms: 1000,

View File

@@ -41,6 +41,8 @@ import { TileWrapper } from "./TileWrapper";
import { usePrefersReducedMotion } from "../usePrefersReducedMotion";
import { useInitial } from "../useInitial";
const MAX_ANIMATED_TILES = 50; // Capped for performance reasons
interface Rect {
x: number;
y: number;
@@ -285,7 +287,6 @@ export function Grid<
const [visibleTilesCallback, setVisibleTilesCallback] =
useState<VisibleTilesCallback | null>(null);
const tiles = useInitial(() => new Map<string, Tile<TileModel>>());
const prefersReducedMotion = usePrefersReducedMotion();
const Slot: FC<SlotProps<TileModel>> = useMemo(
() =>
@@ -372,6 +373,10 @@ export function Grid<
// react-spring's imperative API during gestures to improve responsiveness
const dragState = useRef<DragState | null>(null);
// If true, disables animations
const immediate =
usePrefersReducedMotion() || placedTiles.length > MAX_ANIMATED_TILES;
const [tileTransitions, springRef] = useTransition(
placedTiles,
() => ({
@@ -389,9 +394,9 @@ export function Grid<
y,
width,
height,
immediate: prefersReducedMotion,
immediate,
}),
enter: { opacity: 1, scale: 1, immediate: prefersReducedMotion },
enter: { opacity: 1, scale: 1, immediate },
update: ({
id,
x,
@@ -406,9 +411,9 @@ export function Grid<
y,
width,
height,
immediate: prefersReducedMotion,
immediate,
},
leave: { opacity: 0, scale: 0, immediate: prefersReducedMotion },
leave: { opacity: 0, scale: 0, immediate },
config: { mass: 0.7, tension: 252, friction: 25 },
}),
// react-spring's types are bugged and can't infer the spring type
@@ -441,8 +446,7 @@ export function Grid<
y: tile.y,
width: tile.width,
height: tile.height,
immediate:
prefersReducedMotion || ((key): boolean => key === "zIndex"),
immediate: immediate || ((key): boolean => key === "zIndex"),
// Allow the tile's position to settle before pushing its
// z-index back down
delay: (key): number => (key === "zIndex" ? 500 : 0),
@@ -453,7 +457,7 @@ export function Grid<
x: tileX,
y: tileY,
immediate:
prefersReducedMotion ||
immediate ||
((key): boolean =>
key === "zIndex" || key === "x" || key === "y"),
},

View File

@@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type ReactNode } from "react";
import { type FC, type ReactNode } from "react";
import { useObservableEagerState } from "observable-hooks";
import classNames from "classnames";
@@ -13,6 +13,9 @@ import { type CallLayout } from "./CallLayout";
import { type SpotlightLandscapeLayout as SpotlightLandscapeLayoutModel } from "../state/layout-types.ts";
import styles from "./SpotlightLandscapeLayout.module.css";
import { useUpdateLayout, useVisibleTiles } from "./Grid";
import { type MediaViewModel } from "../state/media/MediaViewModel.ts";
import { type Behavior } from "../state/Behavior.ts";
import { useBehavior } from "../useBehavior.ts";
/**
* An implementation of the "spotlight landscape" layout, in which the spotlight
@@ -54,16 +57,10 @@ export const makeSpotlightLandscapeLayout: CallLayout<
useUpdateLayout();
useVisibleTiles(model.setVisibleTiles);
useObservableEagerState(minBounds$);
const withIndicators =
useObservableEagerState(model.spotlight.media$).length > 1;
return (
<div ref={ref} className={styles.layer}>
<div
className={classNames(styles.spotlight, {
[styles.withIndicators]: withIndicators,
})}
/>
<SpotlightSlot media$={model.spotlight.media$} />
<div className={styles.grid}>
{model.grid.map((m) => (
<Slot key={m.id} className={styles.slot} id={m.id} model={m} />
@@ -73,3 +70,20 @@ export const makeSpotlightLandscapeLayout: CallLayout<
);
},
});
interface SpotlightSlotProps {
media$: Behavior<MediaViewModel[]>;
}
// This component isolates the subscription to the spotlight media so that it
// can change without causing the whole layout to re-render
const SpotlightSlot: FC<SpotlightSlotProps> = ({ media$ }) => {
const withIndicators = useBehavior(media$).length > 1;
return (
<div
className={classNames(styles.spotlight, {
[styles.withIndicators]: withIndicators,
})}
/>
);
};

View File

@@ -30,6 +30,7 @@ import {
import { getUrlParams } from "./UrlParams";
import { Config } from "./config/Config";
import { seedSettingsFromConfig } from "./settings/settings";
import { platform } from "./Platform";
import { isFailure } from "./utils/fetch";
import { initializeWidget } from "./widget";
@@ -237,6 +238,7 @@ export class Initializer {
this.loadStates.config = LoadState.Loading;
Config.init().then(
() => {
seedSettingsFromConfig(Config.get().media_quality);
this.loadStates.config = LoadState.Loaded;
this.initStep(resolve);
},

213
src/livekit/options.test.ts Normal file
View File

@@ -0,0 +1,213 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { describe, expect, it, vi } from "vitest";
import { VideoPresets, type VideoPreset } from "livekit-client";
import { buildLiveKitOptions, getLiveKitOptions } from "./options";
import { Config } from "../config/Config";
vi.mock("../config/Config", () => ({
Config: {
get: vi.fn(),
},
}));
describe("buildLiveKitOptions", () => {
it("returns sensible defaults with no config", () => {
const opts = buildLiveKitOptions();
expect(opts.adaptiveStream).toBe(true);
expect(opts.dynacast).toBe(true);
expect(opts.videoCaptureDefaults?.resolution).toEqual(
VideoPresets.h720.resolution,
);
expect(opts.publishDefaults?.videoCodec).toBe("vp8");
expect(opts.publishDefaults?.videoEncoding).toEqual({
maxBitrate: 1_700_000,
maxFramerate: 30,
});
expect(opts.publishDefaults?.screenShareEncoding).toEqual({
maxBitrate: 5_000_000,
maxFramerate: 30,
});
expect(opts.publishDefaults?.videoSimulcastLayers).toEqual([
VideoPresets.h180,
VideoPresets.h360,
]);
});
it("applies video codec from config", () => {
const opts = buildLiveKitOptions({ video_codec: "vp9" });
expect(opts.publishDefaults?.videoCodec).toBe("vp9");
});
it("applies video resolution and encoding from config", () => {
const baseVideoConfig = {
max_resolution: 1080,
max_bitrate: 3_000_000,
max_framerate: 60,
};
const opts1080 = buildLiveKitOptions({
video: baseVideoConfig,
});
const opts1440 = buildLiveKitOptions({
video: { ...baseVideoConfig, max_resolution: 1440 },
});
const opts2160 = buildLiveKitOptions({
video: { ...baseVideoConfig, max_resolution: 2160 },
});
expect(opts1080.videoCaptureDefaults?.resolution).toEqual(
VideoPresets.h1080.resolution,
);
expect(opts1440.videoCaptureDefaults?.resolution).toEqual(
VideoPresets.h1440.resolution,
);
expect(opts2160.videoCaptureDefaults?.resolution).toEqual(
VideoPresets.h2160.resolution,
);
expect(opts1080.publishDefaults?.videoEncoding).toEqual({
maxBitrate: 3_000_000,
maxFramerate: 60,
});
});
it("applies screen share encoding from config", () => {
const opts = buildLiveKitOptions({
screen_share: {
max_bitrate: 8_000_000,
max_framerate: 15,
},
});
expect(opts.publishDefaults?.screenShareEncoding).toEqual({
maxBitrate: 8_000_000,
maxFramerate: 15,
});
});
it("uses DEFAULT_CONFIG defaults when only resolution is set", () => {
const opts = buildLiveKitOptions({
screen_share: {
max_resolution: 720,
},
});
// Bitrate and framerate fall back to DEFAULT_CONFIG, not the preset
expect(opts.publishDefaults?.screenShareEncoding).toEqual({
maxBitrate: 5_000_000,
maxFramerate: 30,
});
});
it("maps low resolutions to the closest preset, rounding up", () => {
const expectations: [number, VideoPreset][] = [
[180, VideoPresets.h180],
[360, VideoPresets.h360],
[480, VideoPresets.h540],
[540, VideoPresets.h540],
[720, VideoPresets.h720],
];
for (const [height, preset] of expectations) {
const opts = buildLiveKitOptions({ video: { max_resolution: height } });
expect(opts.videoCaptureDefaults?.resolution).toEqual(preset.resolution);
}
});
it("screen share layers fall back to max_framerate, then 30", () => {
const fromMax = buildLiveKitOptions({
screen_share: {
max_framerate: 15,
simulcast_layers: [{ height: 540, bitrate: 1_000_000 }],
},
});
expect(
fromMax.publishDefaults?.screenShareSimulcastLayers?.[0],
).toMatchObject({ encoding: { maxFramerate: 15 } });
const fromDefault = buildLiveKitOptions({
screen_share: {
simulcast_layers: [{ height: 540, bitrate: 1_000_000 }],
},
});
expect(
fromDefault.publishDefaults?.screenShareSimulcastLayers?.[0],
).toMatchObject({ encoding: { maxFramerate: 30 } });
});
it("applies custom video simulcast layers", () => {
const opts = buildLiveKitOptions({
video: {
simulcast_layers: [
{ height: 180, bitrate: 100_000 },
{ height: 360, bitrate: 300_000 },
{ height: 540, bitrate: 600_000 },
],
max_framerate: 24,
},
});
const layers = opts.publishDefaults?.videoSimulcastLayers;
expect(layers).toHaveLength(3);
expect(layers?.[0]).toMatchObject({
width: 320,
height: 180,
encoding: { maxBitrate: 100_000, maxFramerate: 24 },
});
expect(layers?.[2]).toMatchObject({
width: 960,
height: 540,
encoding: { maxBitrate: 600_000, maxFramerate: 24 },
});
});
it("applies custom screen share simulcast layers", () => {
const opts = buildLiveKitOptions({
screen_share: {
simulcast_layers: [{ height: 540, bitrate: 1_000_000, framerate: 5 }],
},
});
const layers = opts.publishDefaults?.screenShareSimulcastLayers;
expect(layers).toHaveLength(1);
expect(layers?.[0]).toMatchObject({
width: 960,
height: 540,
encoding: { maxBitrate: 1_000_000, maxFramerate: 5 },
});
});
it("does not include screenShareSimulcastLayers when not configured", () => {
const opts = buildLiveKitOptions();
expect(opts.publishDefaults?.screenShareSimulcastLayers).toBeUndefined();
});
it("backupCodec always uses stock VP8 720p encoding", () => {
const opts = buildLiveKitOptions({
video_codec: "av1",
video: { max_bitrate: 10_000_000, max_framerate: 60 },
});
const backup = opts.publishDefaults?.backupCodec as {
codec: string;
encoding: { maxBitrate: number; maxFramerate: number };
};
expect(backup.codec).toBe("vp8");
expect(backup.encoding).toEqual(VideoPresets.h720.encoding);
});
});
describe("getLiveKitOptions", () => {
it("reads from Config singleton", () => {
vi.mocked(Config.get).mockReturnValue({
media_quality: { video_codec: "h264" },
} as ReturnType<typeof Config.get>);
const opts = getLiveKitOptions();
expect(opts.publishDefaults?.videoCodec).toBe("h264");
});
it("throws when Config is not initialized", () => {
vi.mocked(Config.get).mockImplementation(() => {
throw new Error("Config not initialized");
});
expect(() => getLiveKitOptions()).toThrow("Config not initialized");
});
});

View File

@@ -9,46 +9,144 @@ import {
AudioPresets,
DefaultReconnectPolicy,
type RoomOptions,
ScreenSharePresets,
type TrackPublishDefaults,
type VideoPreset,
VideoPresets,
VideoPreset as VideoPresetClass,
} from "livekit-client";
const defaultLiveKitPublishOptions: TrackPublishDefaults = {
audioPreset: AudioPresets.music,
dtx: true,
// disable red because the livekit server strips out red packets for clients
// that don't support it (firefox) but of course that doesn't work with e2ee.
red: false,
forceStereo: false,
simulcast: true,
videoSimulcastLayers: [VideoPresets.h180, VideoPresets.h360] as VideoPreset[],
screenShareEncoding: ScreenSharePresets.h1080fps30.encoding,
stopMicTrackOnMute: false,
videoCodec: "vp8",
videoEncoding: VideoPresets.h720.encoding,
backupCodec: { codec: "vp8", encoding: VideoPresets.h720.encoding },
} as const;
import { Config } from "../config/Config";
import { DEFAULT_CONFIG, type ConfigOptions } from "../config/ConfigOptions";
export const defaultLiveKitOptions: RoomOptions = {
// automatically manage subscribed video quality
adaptiveStream: true,
/**
* Find the closest matching VideoPreset for a given height.
*/
function videoPresetForHeight(height: number): VideoPreset {
if (height <= 180) return VideoPresets.h180;
if (height <= 360) return VideoPresets.h360;
if (height <= 540) return VideoPresets.h540;
if (height <= 720) return VideoPresets.h720;
if (height <= 1080) return VideoPresets.h1080;
if (height <= 1440) return VideoPresets.h1440;
return VideoPresets.h2160;
}
// optimize publishing bandwidth and CPU for published tracks
dynacast: true,
/**
* Build LiveKit publish options from config, falling back to sensible defaults.
*/
function buildPublishOptions(
mediaQuality: ConfigOptions["media_quality"],
): TrackPublishDefaults {
const defaults = DEFAULT_CONFIG.media_quality;
const videoConf = mediaQuality?.video;
const screenConf = mediaQuality?.screen_share;
const codec = mediaQuality?.video_codec ?? defaults.video_codec;
// capture settings
videoCaptureDefaults: {
resolution: VideoPresets.h720.resolution,
},
// Camera video encoding
const videoEncoding = {
maxBitrate: videoConf?.max_bitrate ?? defaults.video.max_bitrate,
maxFramerate: videoConf?.max_framerate ?? defaults.video.max_framerate,
};
// publish settings
publishDefaults: defaultLiveKitPublishOptions,
// Camera simulcast layers
let videoSimulcastLayers: VideoPreset[];
if (videoConf?.simulcast_layers) {
videoSimulcastLayers = videoConf.simulcast_layers.map(
(layer) =>
new VideoPresetClass(
Math.round((layer.height * 16) / 9),
layer.height,
layer.bitrate,
videoConf?.max_framerate ?? defaults.video.max_framerate,
),
);
} else {
videoSimulcastLayers = [VideoPresets.h180, VideoPresets.h360];
}
// default LiveKit options that seem to be sane
stopLocalTrackOnUnpublish: true,
reconnectPolicy: new DefaultReconnectPolicy(),
disconnectOnPageLeave: true,
webAudioMix: false,
};
// Screen share encoding
const screenShareEncoding = {
maxBitrate: screenConf?.max_bitrate ?? defaults.screen_share.max_bitrate,
maxFramerate:
screenConf?.max_framerate ?? defaults.screen_share.max_framerate,
};
// Screen share simulcast layers
let screenShareSimulcastLayers: VideoPreset[] | undefined;
if (screenConf?.simulcast_layers) {
screenShareSimulcastLayers = screenConf.simulcast_layers.map(
(layer) =>
new VideoPresetClass(
Math.round((layer.height * 16) / 9),
layer.height,
layer.bitrate,
layer.framerate ?? screenConf?.max_framerate ?? 30,
),
);
}
return {
audioPreset: AudioPresets.music,
dtx: true,
// disable red because the livekit server strips out red packets for clients
// that don't support it (firefox) but of course that doesn't work with e2ee.
red: false,
forceStereo: false,
simulcast: true,
videoSimulcastLayers: videoSimulcastLayers as VideoPreset[],
screenShareEncoding,
...(screenShareSimulcastLayers && {
screenShareSimulcastLayers: screenShareSimulcastLayers as VideoPreset[],
}),
stopMicTrackOnMute: false,
videoCodec: codec,
videoEncoding,
backupCodec: {
codec: "vp8",
encoding: VideoPresets.h720.encoding,
},
} as TrackPublishDefaults;
}
/**
* Build LiveKit RoomOptions from config.
* Call this after Config.init() has resolved.
*/
export function buildLiveKitOptions(
mediaQuality?: ConfigOptions["media_quality"],
): RoomOptions {
const videoHeight =
mediaQuality?.video?.max_resolution ??
DEFAULT_CONFIG.media_quality.video.max_resolution;
const basePreset = videoPresetForHeight(videoHeight);
return {
// automatically manage subscribed video quality
adaptiveStream: true,
// optimize publishing bandwidth and CPU for published tracks
dynacast: true,
// capture settings
videoCaptureDefaults: {
resolution: basePreset.resolution,
},
// publish settings
publishDefaults: buildPublishOptions(mediaQuality),
// default LiveKit options that seem to be sane
stopLocalTrackOnUnpublish: true,
reconnectPolicy: new DefaultReconnectPolicy(),
disconnectOnPageLeave: true,
webAudioMix: false,
};
}
/**
* Get LiveKit options, reading from the loaded Config singleton.
* Requires Config.init() to have resolved first.
*/
export function getLiveKitOptions(): RoomOptions {
return buildLiveKitOptions(Config.get().media_quality);
}

View File

@@ -8,6 +8,7 @@ Please see LICENSE in the repository root for full details.
import {
type ChangeEvent,
type FC,
type ReactNode,
useCallback,
useEffect,
useMemo,
@@ -29,12 +30,14 @@ import {
InlineField,
Label,
RadioControl,
Separator,
} from "@vector-im/compound-web";
import { type Room as LivekitRoom } from "livekit-client";
import { FieldRow, InputField } from "../input/Input";
import { Config } from "../config/Config";
import {
type Setting,
useSetting,
duplicateTiles as duplicateTilesSetting,
debugTileLayout as debugTileLayoutSetting,
@@ -43,10 +46,26 @@ import {
alwaysShowIphoneEarpiece as alwaysShowIphoneEarpieceSetting,
matrixRTCMode as matrixRTCModeSetting,
customLivekitUrl as customLivekitUrlSetting,
advancedScreenShare as advancedScreenShareSetting,
screenShareResolution as screenShareResolutionSetting,
screenShareFramerate as screenShareFramerateSetting,
screenShareBitrate as screenShareBitrateSetting,
screenShareCodec as screenShareCodecSetting,
advancedCamera as advancedCameraSetting,
cameraResolution as cameraResolutionSetting,
cameraFramerate as cameraFramerateSetting,
cameraBitrate as cameraBitrateSetting,
cameraCodec as cameraCodecSetting,
echoCancellationSetting,
noiseSuppressionSetting,
autoGainControlSetting,
type VideoCodec,
enableExtendedLivekitLogs as enableExtendedLivekitLogsSetting,
} from "./settings";
import { MatrixRTCMode } from "../config/ConfigOptions";
import styles from "./DeveloperSettingsTab.module.css";
import settingsStyles from "./SettingsModal.module.css";
import { Slider } from "../Slider";
import { useUrlParams } from "../UrlParams";
import { getSFUConfigWithOpenID } from "../livekit/openIDSFU";
@@ -138,6 +157,185 @@ export const DeveloperSettingsTab: FC<Props> = ({
return null;
}, [livekitRooms]);
const MediaQualitySettings: React.FC<{
id: string;
header: string;
toggleLabel: string;
description: string;
toggleSetting: Setting<boolean>;
resolutionSetting: Setting<string>;
framerateSetting: Setting<number>;
bitrateSetting: Setting<number>;
codecSetting: Setting<VideoCodec>;
resolutionOptions: { value: string; label: string }[];
bitrateRange: { min: number; max: number; step: number };
}> = ({
id,
header,
toggleLabel,
description,
toggleSetting,
resolutionSetting,
framerateSetting,
bitrateSetting,
codecSetting,
resolutionOptions,
bitrateRange,
}): ReactNode => {
const [advancedEnabled, setAdvancedEnabled] = useSetting(toggleSetting);
const [resolution, setResolution] = useSetting(resolutionSetting);
const [framerate, setFramerate] = useSetting(framerateSetting);
const [framerateRaw, setFramerateRaw] = useState(framerate);
const [bitrate, setBitrate] = useSetting(bitrateSetting);
const [bitrateRaw, setBitrateRaw] = useState(bitrate);
const [codec, setCodec] = useSetting(codecSetting);
return (
<>
<h4>{header}</h4>
<FieldRow>
<InputField
id={`${id}Toggle`}
label={toggleLabel}
description={description}
type="checkbox"
checked={advancedEnabled}
onChange={(e): void => setAdvancedEnabled(e.target.checked)}
/>
</FieldRow>
{advancedEnabled && (
<>
<div className={settingsStyles.volumeSlider}>
<label htmlFor={`${id}Resolution`}>
{t("settings.resolution_label", "Resolution")}
</label>
<select
id={`${id}Resolution`}
value={resolution}
onChange={(e): void => setResolution(e.target.value)}
>
{resolutionOptions.map((opt) => (
<option key={opt.value} value={opt.value}>
{opt.label}
</option>
))}
</select>
</div>
<div className={settingsStyles.volumeSlider}>
<label>
{t("settings.framerate_label", "Framerate")}
{": "}
<span className={settingsStyles.settingValue}>
{framerateRaw} fps
</span>
</label>
<Slider
label={t("settings.framerate_label", "Framerate")}
value={framerateRaw}
onValueChange={setFramerateRaw}
onValueCommit={setFramerate}
min={5}
max={60}
step={5}
tooltipFormatter={(v): string => `${v} fps`}
/>
</div>
<div className={settingsStyles.volumeSlider}>
<label>
{t("settings.bitrate_label", "Bitrate")}
{": "}
<span className={settingsStyles.settingValue}>
{(bitrateRaw / 1_000_000).toFixed(1)} Mbps
</span>
</label>
<Slider
label={t("settings.bitrate_label", "Bitrate")}
value={bitrateRaw}
onValueChange={setBitrateRaw}
onValueCommit={setBitrate}
min={bitrateRange.min}
max={bitrateRange.max}
step={bitrateRange.step}
tooltipFormatter={(v): string =>
`${(v / 1_000_000).toFixed(1)} Mbps`
}
/>
</div>
<div className={settingsStyles.volumeSlider}>
<label htmlFor={`${id}Codec`}>
{t("settings.codec_label", "Codec")}
</label>
<select
id={`${id}Codec`}
value={codec}
onChange={(e): void => setCodec(e.target.value as VideoCodec)}
>
<option value="vp8">VP8</option>
<option value="vp9">VP9</option>
<option value="h264">H.264</option>
<option value="av1">AV1</option>
</select>
</div>
</>
)}
</>
);
};
const AudioProcessingSettings: React.FC = (): ReactNode => {
const [echoCancellation, setEchoCancellation] = useSetting(
echoCancellationSetting,
);
const [noiseSuppression, setNoiseSuppression] = useSetting(
noiseSuppressionSetting,
);
const [autoGainControl, setAutoGainControl] = useSetting(
autoGainControlSetting,
);
return (
<>
<h4>{t("settings.audio_processing_header", "Audio processing")}</h4>
<p>
{t(
"settings.audio_processing_description",
"Changes apply on next call join.",
)}
</p>
<FieldRow>
<InputField
id="echoCancellation"
label={t("settings.echo_cancellation_label", "Echo cancellation")}
type="checkbox"
checked={echoCancellation}
onChange={(e): void => setEchoCancellation(e.target.checked)}
/>
</FieldRow>
<FieldRow>
<InputField
id="noiseSuppression"
label={t("settings.noise_suppression_label", "Noise suppression")}
type="checkbox"
checked={noiseSuppression}
onChange={(e): void => setNoiseSuppression(e.target.checked)}
/>
</FieldRow>
<FieldRow>
<InputField
id="autoGainControl"
label={t(
"settings.auto_gain_control_label",
"Automatic gain control",
)}
type="checkbox"
checked={autoGainControl}
onChange={(e): void => setAutoGainControl(e.target.checked)}
/>
</FieldRow>
</>
);
};
return (
<>
<p>
@@ -166,6 +364,7 @@ export const DeveloperSettingsTab: FC<Props> = ({
id: client.getDeviceId() || "unknown",
})}
</p>
<Separator />
<FieldRow>
<InputField
id="duplicateTiles"
@@ -315,6 +514,7 @@ export const DeveloperSettingsTab: FC<Props> = ({
<ErrorMessage>{customLivekitUrlUpdateError}</ErrorMessage>
)}
</EditInPlace>
<Separator />
<Heading as="h3" type="body" weight="semibold" size="lg">
{t("developer_mode.matrixRTCMode.title")}
</Heading>
@@ -403,6 +603,61 @@ export const DeveloperSettingsTab: FC<Props> = ({
</ul>
</div>
))}
<Separator />
<MediaQualitySettings
id="camera"
header={t("settings.camera_header", "Camera quality")}
toggleLabel={t(
"settings.advanced_camera_label",
"Advanced camera settings",
)}
description={t(
"settings.advanced_camera_description",
"Configure resolution, framerate, bitrate, and codec for camera video. Changes apply on next call join.",
)}
toggleSetting={advancedCameraSetting}
resolutionSetting={cameraResolutionSetting}
framerateSetting={cameraFramerateSetting}
bitrateSetting={cameraBitrateSetting}
codecSetting={cameraCodecSetting}
resolutionOptions={[
{ value: "640x360", label: "360p" },
{ value: "960x540", label: "540p" },
{ value: "1280x720", label: "720p" },
{ value: "1920x1080", label: "1080p" },
{ value: "2560x1440", label: "1440p" },
]}
bitrateRange={{ min: 200_000, max: 8_000_000, step: 100_000 }}
/>
<Separator />
<MediaQualitySettings
id="screenShare"
header={t("settings.screen_share_header", "Screen sharing")}
toggleLabel={t(
"settings.advanced_screen_share_label",
"Advanced screen share settings",
)}
description={t(
"settings.advanced_screen_share_description",
"Configure resolution, framerate, bitrate, and codec for screen sharing",
)}
toggleSetting={advancedScreenShareSetting}
resolutionSetting={screenShareResolutionSetting}
framerateSetting={screenShareFramerateSetting}
bitrateSetting={screenShareBitrateSetting}
codecSetting={screenShareCodecSetting}
resolutionOptions={[
{ value: "1024x576", label: "576p" },
{ value: "1280x720", label: "720p" },
{ value: "1920x1080", label: "1080p" },
{ value: "2560x1440", label: "1440p" },
{ value: "3840x2160", label: "4K" },
]}
bitrateRange={{ min: 500_000, max: 15_000_000, step: 500_000 }}
/>
<Separator />
<AudioProcessingSettings />
<Separator />
<p>{t("developer_mode.environment_variables")}</p>
<pre>{JSON.stringify(env, null, 2)}</pre>
<p>{t("developer_mode.url_params")}</p>

View File

@@ -33,3 +33,27 @@ Please see LICENSE in the repository root for full details.
.volumeSlider > p {
color: var(--cpd-color-text-secondary);
}
.volumeSlider > select {
display: block;
width: 100%;
padding: 8px 12px;
margin-top: var(--cpd-space-1x);
border: 1px solid var(--cpd-color-border-interactive-primary);
border-radius: 4px;
background-color: var(--cpd-color-bg-canvas-default);
color: var(--cpd-color-text-primary);
font-size: var(--font-size-body);
font-family: inherit;
cursor: pointer;
}
.volumeSlider > select:focus {
outline: none;
border-color: var(--cpd-color-text-link-external);
}
.settingValue {
font-weight: normal;
color: var(--cpd-color-text-secondary);
}

View File

@@ -152,7 +152,13 @@ export const SettingsModal: FC<Props> = ({
/>
<div className={styles.volumeSlider}>
<label>{t("settings.audio_tab.effect_volume_label")}</label>
<label>
{t("settings.audio_tab.effect_volume_label")}
{": "}
<span className={styles.settingValue}>
{Math.round(soundVolumeRaw * 100)}%
</span>
</label>
<p>{t("settings.audio_tab.effect_volume_description")}</p>
<Slider
label={t("video_tile.volume")}

View File

@@ -17,6 +17,12 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
<p>
Device ID: DEVICE123
</p>
<div
class="_separator_13qwf_8"
data-kind="primary"
data-orientation="horizontal"
role="separator"
/>
<div
class="_fieldRow_1bd8c0"
>
@@ -251,10 +257,16 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
class="_message_1o4d9_86 _help-message_1o4d9_92"
id="radix-_r_8_"
>
Currently, no overwrite is set. Url from well-known or config is used.
Currently, no overwrite is set. Url from config is used.
</span>
</div>
</form>
<div
class="_separator_13qwf_8"
data-kind="primary"
data-orientation="horizontal"
role="separator"
/>
<h3
class="_typography_6v6n8_153 _font-body-lg-semibold_6v6n8_74"
>
@@ -463,6 +475,242 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
</p>
<ul />
</div>
<div
class="_separator_13qwf_8"
data-kind="primary"
data-orientation="horizontal"
role="separator"
/>
<h4>
Camera quality
</h4>
<div
class="_fieldRow_1bd8c0"
>
<div
class="_field_1bd8c0 _checkboxField_1bd8c0"
>
<input
aria-describedby="_r_k_"
id="cameraToggle"
type="checkbox"
/>
<label
for="cameraToggle"
>
<div
class="_checkbox_1bd8c0"
>
<svg
fill="none"
height="24"
stroke="#000"
stroke-linecap="round"
stroke-linejoin="round"
stroke-width="2"
viewBox="0 0 24 24"
width="24"
xmlns="http://www.w3.org/2000/svg"
>
<path
d="m20 6-11 11-5-5"
/>
</svg>
</div>
Advanced camera settings
</label>
<p
class="_description_1bd8c0"
id="_r_k_"
>
Configure resolution, framerate, bitrate, and codec for camera video. Changes apply on next call join.
</p>
</div>
</div>
<div
class="_separator_13qwf_8"
data-kind="primary"
data-orientation="horizontal"
role="separator"
/>
<h4>
Screen sharing
</h4>
<div
class="_fieldRow_1bd8c0"
>
<div
class="_field_1bd8c0 _checkboxField_1bd8c0"
>
<input
aria-describedby="_r_l_"
id="screenShareToggle"
type="checkbox"
/>
<label
for="screenShareToggle"
>
<div
class="_checkbox_1bd8c0"
>
<svg
fill="none"
height="24"
stroke="#000"
stroke-linecap="round"
stroke-linejoin="round"
stroke-width="2"
viewBox="0 0 24 24"
width="24"
xmlns="http://www.w3.org/2000/svg"
>
<path
d="m20 6-11 11-5-5"
/>
</svg>
</div>
Advanced screen share settings
</label>
<p
class="_description_1bd8c0"
id="_r_l_"
>
Configure resolution, framerate, bitrate, and codec for screen sharing
</p>
</div>
</div>
<div
class="_separator_13qwf_8"
data-kind="primary"
data-orientation="horizontal"
role="separator"
/>
<h4>
Audio processing
</h4>
<p>
Changes apply on next call join.
</p>
<div
class="_fieldRow_1bd8c0"
>
<div
class="_field_1bd8c0 _checkboxField_1bd8c0"
>
<input
aria-describedby="_r_m_"
checked=""
id="echoCancellation"
type="checkbox"
/>
<label
for="echoCancellation"
>
<div
class="_checkbox_1bd8c0"
>
<svg
fill="none"
height="24"
stroke="#000"
stroke-linecap="round"
stroke-linejoin="round"
stroke-width="2"
viewBox="0 0 24 24"
width="24"
xmlns="http://www.w3.org/2000/svg"
>
<path
d="m20 6-11 11-5-5"
/>
</svg>
</div>
Echo cancellation
</label>
</div>
</div>
<div
class="_fieldRow_1bd8c0"
>
<div
class="_field_1bd8c0 _checkboxField_1bd8c0"
>
<input
aria-describedby="_r_n_"
checked=""
id="noiseSuppression"
type="checkbox"
/>
<label
for="noiseSuppression"
>
<div
class="_checkbox_1bd8c0"
>
<svg
fill="none"
height="24"
stroke="#000"
stroke-linecap="round"
stroke-linejoin="round"
stroke-width="2"
viewBox="0 0 24 24"
width="24"
xmlns="http://www.w3.org/2000/svg"
>
<path
d="m20 6-11 11-5-5"
/>
</svg>
</div>
Noise suppression
</label>
</div>
</div>
<div
class="_fieldRow_1bd8c0"
>
<div
class="_field_1bd8c0 _checkboxField_1bd8c0"
>
<input
aria-describedby="_r_o_"
checked=""
id="autoGainControl"
type="checkbox"
/>
<label
for="autoGainControl"
>
<div
class="_checkbox_1bd8c0"
>
<svg
fill="none"
height="24"
stroke="#000"
stroke-linecap="round"
stroke-linejoin="round"
stroke-width="2"
viewBox="0 0 24 24"
width="24"
xmlns="http://www.w3.org/2000/svg"
>
<path
d="m20 6-11 11-5-5"
/>
</svg>
</div>
Automatic gain control
</label>
</div>
</div>
<div
class="_separator_13qwf_8"
data-kind="primary"
data-orientation="horizontal"
role="separator"
/>
<p>
Environment variables
</p>

View File

@@ -0,0 +1,111 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { describe, expect, it, beforeEach } from "vitest";
import {
Setting,
parseResolution,
seedSettingsFromConfig,
screenShareCodec,
screenShareResolution,
screenShareFramerate,
screenShareBitrate,
cameraCodec,
cameraResolution,
cameraFramerate,
cameraBitrate,
} from "./settings";
beforeEach(() => {
localStorage.clear();
});
describe("parseResolution", () => {
it("parses a WIDTHxHEIGHT string", () => {
expect(parseResolution("1920x1080")).toEqual({
width: 1920,
height: 1080,
});
});
it("parses non-standard resolutions", () => {
expect(parseResolution("640x360")).toEqual({ width: 640, height: 360 });
});
});
describe("Setting", () => {
describe("seedFromConfig", () => {
it("updates value when no localStorage value exists", () => {
const setting = new Setting<string>("test-seed", "default");
setting.seedFromConfig("from-config");
expect(setting.getValue()).toBe("from-config");
});
it("does not override an existing localStorage value", () => {
localStorage.setItem(
"matrix-setting-test-seed-existing",
JSON.stringify("user-set"),
);
const setting = new Setting<string>("test-seed-existing", "default");
expect(setting.getValue()).toBe("user-set");
setting.seedFromConfig("from-config");
expect(setting.getValue()).toBe("user-set");
});
});
});
describe("seedSettingsFromConfig", () => {
it("does nothing when config is undefined", () => {
const codecBefore = screenShareCodec.getValue();
seedSettingsFromConfig(undefined);
expect(screenShareCodec.getValue()).toBe(codecBefore);
});
it("seeds video codec to both camera and screen share", () => {
seedSettingsFromConfig({ video_codec: "av1" });
expect(screenShareCodec.getValue()).toBe("av1");
expect(cameraCodec.getValue()).toBe("av1");
});
it("seeds screen share settings from config", () => {
seedSettingsFromConfig({
screen_share: {
max_resolution: 720,
max_framerate: 15,
max_bitrate: 2_000_000,
},
});
expect(screenShareResolution.getValue()).toBe("1280x720");
expect(screenShareFramerate.getValue()).toBe(15);
expect(screenShareBitrate.getValue()).toBe(2_000_000);
});
it("seeds camera/video settings from config", () => {
seedSettingsFromConfig({
video: {
max_resolution: 1080,
max_framerate: 60,
max_bitrate: 4_000_000,
},
});
expect(cameraResolution.getValue()).toBe("1920x1080");
expect(cameraFramerate.getValue()).toBe(60);
expect(cameraBitrate.getValue()).toBe(4_000_000);
});
it("only seeds provided fields, leaves others at defaults", () => {
const defaultFramerate = screenShareFramerate.getValue();
seedSettingsFromConfig({
screen_share: {
max_bitrate: 3_000_000,
},
});
expect(screenShareBitrate.getValue()).toBe(3_000_000);
expect(screenShareFramerate.getValue()).toBe(defaultFramerate);
});
});

View File

@@ -21,6 +21,7 @@ export class Setting<T> {
this.key = `matrix-setting-${key}`;
const storedValue = localStorage.getItem(this.key);
this.hasStoredValue = storedValue !== null;
let initialValue = defaultValue;
if (storedValue !== null) {
try {
@@ -40,6 +41,7 @@ export class Setting<T> {
}
private readonly key: string;
private readonly hasStoredValue: boolean;
private readonly _value$: BehaviorSubject<T>;
private readonly _lastUpdateReason$: BehaviorSubject<string | null>;
@@ -54,6 +56,17 @@ export class Setting<T> {
public readonly getValue = (): T => {
return this._value$.getValue();
};
/**
* Update the setting's value from a config source, but only if the user
* hasn't explicitly set a value in localStorage. This lets admins set
* org-wide defaults in config.json that users can override.
*/
public seedFromConfig(value: T): void {
if (!this.hasStoredValue) {
this._value$.next(value);
}
}
}
/**
@@ -144,3 +157,130 @@ export const customLivekitUrl = new Setting<string | null>(
"custom-livekit-url",
null,
);
export type VideoCodec = "vp8" | "vp9" | "h264" | "av1";
/**
* Parse a "WIDTHxHEIGHT" resolution string into numeric width and height.
*/
export function parseResolution(res: string): {
width: number;
height: number;
} {
const [w, h] = res.split("x").map(Number);
return { width: w, height: h };
}
export const advancedScreenShare = new Setting<boolean>(
"advanced-screen-share",
false,
);
export const screenShareResolution = new Setting<string>(
"screen-share-resolution",
"1920x1080",
);
export const screenShareFramerate = new Setting<number>(
"screen-share-framerate",
30,
);
export const screenShareBitrate = new Setting<number>(
"screen-share-bitrate",
5_000_000,
);
export const screenShareCodec = new Setting<VideoCodec>(
"screen-share-codec",
"vp9",
);
// Camera video quality settings
export const advancedCamera = new Setting<boolean>("advanced-camera", false);
export const cameraResolution = new Setting<string>(
"camera-resolution",
"1280x720",
);
export const cameraFramerate = new Setting<number>("camera-framerate", 30);
export const cameraBitrate = new Setting<number>("camera-bitrate", 1_700_000);
export const cameraCodec = new Setting<VideoCodec>("camera-codec", "vp8");
// Audio processing settings
export const echoCancellationSetting = new Setting<boolean>(
"echo-cancellation",
true,
);
export const noiseSuppressionSetting = new Setting<boolean>(
"noise-suppression",
true,
);
export const autoGainControlSetting = new Setting<boolean>(
"auto-gain-control",
true,
);
/**
* Seed setting defaults from config.json's media_quality section.
* Call this after Config.init() has resolved.
* Only updates settings that the user hasn't explicitly set in localStorage.
*/
export function seedSettingsFromConfig(
mediaQuality:
| {
video_codec?: VideoCodec;
video?: {
max_resolution?: number;
max_bitrate?: number;
max_framerate?: number;
};
screen_share?: {
max_resolution?: number;
max_bitrate?: number;
max_framerate?: number;
};
}
| undefined,
): void {
if (!mediaQuality) return;
const codec = mediaQuality.video_codec;
if (codec) {
screenShareCodec.seedFromConfig(codec);
cameraCodec.seedFromConfig(codec);
}
const screen = mediaQuality.screen_share;
if (screen) {
if (screen.max_resolution) {
const width = Math.round((screen.max_resolution * 16) / 9);
screenShareResolution.seedFromConfig(`${width}x${screen.max_resolution}`);
}
if (screen.max_framerate) {
screenShareFramerate.seedFromConfig(screen.max_framerate);
}
if (screen.max_bitrate) {
screenShareBitrate.seedFromConfig(screen.max_bitrate);
}
}
const video = mediaQuality.video;
if (video) {
if (video.max_resolution) {
const width = Math.round((video.max_resolution * 16) / 9);
cameraResolution.seedFromConfig(`${width}x${video.max_resolution}`);
}
if (video.max_framerate) {
cameraFramerate.seedFromConfig(video.max_framerate);
}
if (video.max_bitrate) {
cameraBitrate.seedFromConfig(video.max_bitrate);
}
}
}

View File

@@ -19,7 +19,10 @@ import { useClient } from "../ClientContext";
import { Config } from "../config/Config";
import { type RageshakeRequestModal } from "../room/RageshakeRequestModal";
import { getUrlParams } from "../UrlParams";
import { deepCompare } from "matrix-js-sdk/lib/utils";
import { advancedCamera as advancedCameraSetting } from "./settings";
import { advancedScreenShare as advancedScreenShareSetting } from "./settings";
import { DEFAULT_CONFIG } from "../config/ConfigOptions";
const gzip = async (text: string): Promise<Blob> => {
// pako is relatively large (200KB), so we only import it when needed
const { gzip: pakoGzip } = await import("pako");
@@ -245,6 +248,20 @@ export function useSubmitRageshake(
}
}
// Add custom media related information to the rageshake issue description.
// Used to quickly identify issues due to untested configurations.
if (
!deepCompare(Config.get().media_quality, DEFAULT_CONFIG.media_quality)
) {
body.append("custom_media_quality_in_config", "true");
}
if (advancedCameraSetting.getValue()) {
body.append("devTools_advancedCameraSettings", "true");
}
if (advancedScreenShareSetting.getValue()) {
body.append("devTools_advancedScreenShareSetting", "true");
}
if (navigator.storage && navigator.storage.estimate) {
try {
const estimate: {

View File

@@ -22,6 +22,8 @@ import { useSubmitRageshake, getRageshakeSubmitUrl } from "./submit-rageshake";
import { ClientContextProvider } from "../ClientContext";
import { getUrlParams } from "../UrlParams";
import { mockConfig } from "../utils/test";
import { DEFAULT_CONFIG } from "../config/ConfigOptions";
import { advancedCamera, advancedScreenShare } from "./settings";
vi.mock("../UrlParams", () => ({ getUrlParams: vi.fn() }));
@@ -201,6 +203,60 @@ describe("useSubmitRageshake", () => {
});
});
describe("media quality metadata", () => {
const submitAndGetBody = async (): Promise<FormData> => {
const fetchFn = vi.fn().mockResolvedValue({
status: 200,
});
vi.stubGlobal("fetch", fetchFn);
renderWithMockClient(() => "https://rageshake.localhost/foo", false);
screen.getByTestId("submit").click();
await waitFor(() => {
expect(screen.getByTestId("sent").textContent).toBe("true");
});
return fetchFn.mock.calls[0][1].body as FormData;
};
beforeEach(() => {
vi.unstubAllGlobals();
});
afterEach(() => {
advancedCamera.setValue(advancedCamera.defaultValue);
advancedScreenShare.setValue(advancedScreenShare.defaultValue);
vi.clearAllMocks();
});
it("omits media quality fields when config and settings are default", async () => {
mockConfig({});
const body = await submitAndGetBody();
expect(body.get("custom_media_quality_in_config")).toBeNull();
expect(body.get("devTools_advancedCameraSettings")).toBeNull();
expect(body.get("devTools_advancedScreenShareSetting")).toBeNull();
});
it("includes custom_media_quality_in_config when media_quality differs from default", async () => {
mockConfig({
media_quality: {
...DEFAULT_CONFIG.media_quality,
video_codec: "h264",
},
});
const body = await submitAndGetBody();
expect(body.get("custom_media_quality_in_config")).toBe("true");
});
it("includes devTools flags when advanced media settings are enabled", async () => {
mockConfig({});
advancedCamera.setValue(true);
advancedScreenShare.setValue(true);
const body = await submitAndGetBody();
expect(body.get("devTools_advancedCameraSettings")).toBe("true");
expect(body.get("devTools_advancedScreenShareSetting")).toBe("true");
});
});
describe("when rageshake is not available", () => {
it("starts unsent", () => {
renderWithMockClient(() => undefined, false);

View File

@@ -513,8 +513,6 @@ export function createCallViewModel$(
livekitKeyProvider,
getUrlParams().controlledAudioDevices,
options.livekitRoomFactory,
getUrlParams().echoCancellation,
getUrlParams().noiseSuppression,
);
const connectionManager = createConnectionManager$({

View File

@@ -138,9 +138,6 @@ export function withCallViewModel(mode: MatrixRTCMode) {
public getSyncState(): SyncState {
return syncState;
}
public getAccessToken(): string | null {
return "a-token";
}
})() as Partial<MatrixClient> as MatrixClient,
getMembers: () => roomMembers,
getMembersWithMembership: () => roomMembers,

View File

@@ -20,7 +20,6 @@ import {
afterAll,
beforeEach,
} from "vitest";
import { AutoDiscovery } from "matrix-js-sdk/lib/autodiscovery";
import { BehaviorSubject, map, of } from "rxjs";
import { logger } from "matrix-js-sdk/lib/logger";
import { type LocalParticipant, type LocalTrack } from "livekit-client";
@@ -78,34 +77,10 @@ describe("LocalMembership", () => {
livekit_alias: "my-oldest-member-service-alias",
};
const focusConfigFromWellKnown = {
type: "livekit",
livekit_service_url: "http://my-well-known-service-url.com",
};
const focusConfigFromWellKnown2 = {
type: "livekit",
livekit_service_url: "http://my-well-known-service-url2.com",
};
const clientWellKnown = {
"org.matrix.msc4143.rtc_foci": [
focusConfigFromWellKnown,
focusConfigFromWellKnown2,
],
};
mockConfig({
livekit: { livekit_service_url: "http://my-default-service-url.com" },
});
vi.spyOn(AutoDiscovery, "getRawClientConfig").mockImplementation(
async (domain) => {
if (domain === "example.org") {
return Promise.resolve(clientWellKnown);
}
return Promise.resolve({});
},
);
const mockedSession = vi.mocked({
room: {
roomId: "roomId",
@@ -132,7 +107,7 @@ describe("LocalMembership", () => {
ownMemberMock,
{
livekit_alias: "roomId",
livekit_service_url: "http://my-well-known-service-url.com",
livekit_service_url: "http://my-livekit-service-url.com",
type: "livekit",
},
{
@@ -150,7 +125,7 @@ describe("LocalMembership", () => {
[
{
livekit_alias: "roomId",
livekit_service_url: "http://my-well-known-service-url.com",
livekit_service_url: "http://my-livekit-service-url.com",
type: "livekit",
},
],
@@ -161,50 +136,6 @@ describe("LocalMembership", () => {
}),
);
});
it("It should not fail with configuration error if homeserver config has livekit url but not fallback", () => {
mockConfig({});
vi.spyOn(AutoDiscovery, "getRawClientConfig").mockResolvedValue({
"org.matrix.msc4143.rtc_foci": [
{
type: "livekit",
livekit_service_url: "http://my-well-known-service-url.com",
},
],
});
const mockedSession = vi.mocked({
room: {
roomId: "roomId",
client: {
getDomain: vi.fn().mockReturnValue("example.org"),
getOpenIdToken: vi.fn().mockResolvedValue({
access_token: "ACCCESS_TOKEN",
token_type: "Bearer",
matrix_server_name: "localhost",
expires_in: 10000,
}),
},
},
memberships: [],
getFocusInUse: vi.fn(),
joinRTCSession: vi.fn(),
}) as unknown as MatrixRTCSession;
enterRTCSession(
mockedSession,
ownMemberMock,
{
livekit_alias: "roomId",
livekit_service_url: "http://my-well-known-service-url.com",
type: "livekit",
},
{
encryptMedia: true,
matrixRTCMode: MATRIX_RTC_MODE,
},
);
});
});
const defaultCreateLocalMemberValues = {

View File

@@ -10,6 +10,7 @@ import {
ParticipantEvent,
type LocalParticipant,
type ScreenShareCaptureOptions,
type TrackPublishOptions,
RoomEvent,
MediaDeviceFailure,
} from "livekit-client";
@@ -53,6 +54,14 @@ import {
import { ElementWidgetActions, widget } from "../../../widget.ts";
import { getUrlParams } from "../../../UrlParams.ts";
import { PosthogAnalytics } from "../../../analytics/PosthogAnalytics.ts";
import {
advancedScreenShare,
screenShareResolution,
screenShareFramerate,
screenShareBitrate,
screenShareCodec,
parseResolution,
} from "../../../settings/settings.ts";
import { MatrixRTCMode } from "../../../config/ConfigOptions.ts";
import { Config } from "../../../config/Config.ts";
import {
@@ -108,7 +117,6 @@ export type LocalMemberState =
};
/*
* - get well known
* - get oldest membership
* - get transport to use
* - get openId + jwt token
@@ -719,6 +727,43 @@ export const createLocalMembership$ = ({
surfaceSwitching: "include",
systemAudio: "include",
};
let publishOptions: TrackPublishOptions | undefined;
if (advancedScreenShare.getValue()) {
// User has advanced screen share settings enabled
const { width, height } = parseResolution(
screenShareResolution.getValue(),
);
const fps = screenShareFramerate.getValue();
const bps = screenShareBitrate.getValue();
const codec = screenShareCodec.getValue();
screenshareSettings.resolution = {
width,
height,
frameRate: fps,
};
publishOptions = {
screenShareEncoding: {
maxBitrate: bps,
maxFramerate: fps,
},
videoCodec: codec,
};
} else {
// Fall back to config.json settings if available
const screenConf = Config.get().media_quality?.screen_share;
if (screenConf?.max_resolution) {
screenshareSettings.resolution = {
width: Math.round((screenConf.max_resolution * 16) / 9),
height: screenConf.max_resolution,
frameRate: screenConf.max_framerate ?? 30,
};
}
}
const targetScreenshareState = !sharingScreen$.value;
logger.info(
`toggleScreenSharing called. Switching ${
@@ -734,7 +779,11 @@ export const createLocalMembership$ = ({
// is still initializing or publishing tracks, because there's no
// technical reason to disallow this. LiveKit will publish if it can.
participant$.value
?.setScreenShareEnabled(targetScreenshareState, screenshareSettings)
?.setScreenShareEnabled(
targetScreenshareState,
screenshareSettings,
publishOptions,
)
.catch(logger.error);
};
}

View File

@@ -63,8 +63,7 @@ describe("LocalTransport", () => {
client: {
// eslint-disable-next-line @typescript-eslint/naming-convention
_unstable_getRTCTransports: async () => Promise.resolve([]),
getAccessToken: vi.fn().mockReturnValue("access_token"),
getDomain: () => "",
getDomain: () => "example.org",
baseUrl: "example.org",
// These won't be called in this error path but satisfy the type
getOpenIdToken: vi.fn(),
@@ -77,9 +76,11 @@ describe("LocalTransport", () => {
await flushPromises();
expect(() => advertised$.value).toThrow(
new MatrixRTCTransportMissingError(""),
new MatrixRTCTransportMissingError("example.org"),
);
expect(() => active$.value).toThrow(
new MatrixRTCTransportMissingError("example.org"),
);
expect(() => active$.value).toThrow(new MatrixRTCTransportMissingError(""));
});
it("throws FailToGetOpenIdToken when OpenID fetch fails", async () => {
@@ -103,10 +104,8 @@ describe("LocalTransport", () => {
useOldestMember: false,
memberships$: constant(new Epoch<CallMembership[]>([])),
client: {
baseUrl: "https://lk.example.org",
// Use empty domain to skip .well-known and use config directly
getDomain: () => "",
getAccessToken: vi.fn().mockReturnValue("access_token"),
baseUrl: "https://example.org",
getDomain: () => "example.org",
// eslint-disable-next-line @typescript-eslint/naming-convention
_unstable_getRTCTransports: async () => Promise.resolve([]),
getOpenIdToken: vi.fn(),
@@ -150,11 +149,10 @@ describe("LocalTransport", () => {
client: {
// eslint-disable-next-line @typescript-eslint/naming-convention
_unstable_getRTCTransports: async () => Promise.resolve([]),
getDomain: () => "",
getDomain: () => "example.org",
getOpenIdToken: vi.fn(),
getDeviceId: vi.fn(),
baseUrl: "https://lk.example.org",
getAccessToken: vi.fn().mockReturnValue("access_token"),
baseUrl: "https://example.org",
},
ownMembershipIdentity: ownMemberMock,
forceJwtEndpoint: JwtEndpointVersion.Legacy,
@@ -221,13 +219,12 @@ describe("LocalTransport", () => {
useOldestMember: true,
memberships$: scope.behavior(memberships$.pipe(trackEpoch())),
client: {
getDomain: () => "",
getDomain: () => "example.org",
// eslint-disable-next-line @typescript-eslint/naming-convention
_unstable_getRTCTransports: async () => Promise.resolve([]),
getAccessToken: vi.fn().mockReturnValue("access_token"),
getOpenIdToken: vi.fn(),
getDeviceId: vi.fn(),
baseUrl: "https://lk.example.org",
baseUrl: "https://example.org",
},
ownMembershipIdentity: ownMemberMock,
forceJwtEndpoint: JwtEndpointVersion.Legacy,
@@ -278,14 +275,13 @@ describe("LocalTransport", () => {
useOldestMember: true,
memberships$: scope.behavior(memberships$.pipe(trackEpoch())),
client: {
getDomain: () => "",
getDomain: () => "example.org",
// eslint-disable-next-line @typescript-eslint/naming-convention
_unstable_getRTCTransports: async () =>
Promise.resolve([aliceTransport]),
getAccessToken: vi.fn().mockReturnValue("access_token"),
getOpenIdToken: vi.fn(),
getDeviceId: vi.fn(),
baseUrl: "https://lk.example.org",
baseUrl: "https://example.org",
},
ownMembershipIdentity: ownMemberMock,
forceJwtEndpoint: JwtEndpointVersion.Legacy,
@@ -330,10 +326,9 @@ describe("LocalTransport", () => {
memberships$: constant(new Epoch<CallMembership[]>([])),
client: {
baseUrl: "https://example.org",
getDomain: vi.fn().mockReturnValue(""),
getDomain: vi.fn().mockReturnValue("example.org"),
// eslint-disable-next-line @typescript-eslint/naming-convention
_unstable_getRTCTransports: vi.fn().mockResolvedValue([]),
getAccessToken: vi.fn().mockReturnValue("access_token"),
getOpenIdToken: vi.fn(),
getDeviceId: vi.fn(),
},
@@ -421,42 +416,6 @@ describe("LocalTransport", () => {
});
});
it("Should not call _unstable_getRTCTransports in widget mode but use well-known", async () => {
mockConfig({
livekit: { livekit_service_url: "https://do-not-use.lk.example.org" },
});
localTransportOpts.client.getDomain.mockReturnValue("example.org");
fetchMock.getOnce("https://example.org/.well-known/matrix/client", {
"org.matrix.msc4143.rtc_foci": [
{
type: "livekit",
livekit_service_url: "https://use-me.jwt.call.example.org",
},
],
});
localTransportOpts.client.getAccessToken.mockReturnValue(null);
const { advertised$, active$ } =
createLocalTransport$(localTransportOpts);
openIdResolver.resolve?.(openIdResponse);
expect(advertised$.value).toBe(null);
expect(active$.value).toBe(null);
await flushPromises();
expect(
localTransportOpts.client._unstable_getRTCTransports,
).not.toHaveBeenCalled();
const expectedTransport = {
type: "livekit",
livekit_service_url: "https://use-me.jwt.call.example.org",
};
expect(advertised$.value).toStrictEqual(expectedTransport);
});
it("fails fast if the openID request fails for backend config", async () => {
localTransportOpts.client._unstable_getRTCTransports.mockResolvedValue([
{ type: "livekit", livekit_service_url: "https://lk.example.org" },
@@ -469,51 +428,6 @@ describe("LocalTransport", () => {
).rejects.toThrow(expect.any(FailToGetOpenIdToken));
});
it("supports getting transport via well-known", async () => {
localTransportOpts.client.getDomain.mockReturnValue("example.org");
fetchMock.getOnce("https://example.org/.well-known/matrix/client", {
"org.matrix.msc4143.rtc_foci": [
{ type: "livekit", livekit_service_url: "https://lk.example.org" },
],
});
const { advertised$, active$ } =
createLocalTransport$(localTransportOpts);
openIdResolver.resolve?.(openIdResponse);
expect(advertised$.value).toBe(null);
expect(active$.value).toBe(null);
await flushPromises();
const expectedTransport = {
livekit_service_url: "https://lk.example.org",
type: "livekit",
};
expect(advertised$.value).toStrictEqual(expectedTransport);
expect(active$.value).toStrictEqual({
transport: expectedTransport,
sfuConfig: {
jwt: "e30=.eyJzdWIiOiJAbWU6ZXhhbXBsZS5vcmc6QUJDREVGIiwidmlkZW8iOnsicm9vbSI6IiFleGFtcGxlX3Jvb21faWQifX0=.e30=",
livekitAlias: "Akph4alDMhen",
livekitIdentity: "@lk_user:ABCDEF",
url: "https://lk.example.org",
},
});
expect(fetchMock.done()).toEqual(true);
});
it("fails fast if the openId request fails for the well-known config", async () => {
localTransportOpts.client.getDomain.mockReturnValue("example.org");
fetchMock.getOnce("https://example.org/.well-known/matrix/client", {
"org.matrix.msc4143.rtc_foci": [
{ type: "livekit", livekit_service_url: "https://lk.example.org" },
],
});
openIdResolver.reject(
new FailToGetOpenIdToken(new Error("Test driven error")),
);
await expect(async () =>
lastValueFrom(createLocalTransport$(localTransportOpts).active$),
).rejects.toThrow(expect.any(FailToGetOpenIdToken));
});
it("throws if no options are available", async () => {
const { advertised$, active$ } = createLocalTransport$({
scope: testScope(),
@@ -524,11 +438,10 @@ describe("LocalTransport", () => {
delayId$: constant(null),
memberships$: constant(new Epoch<CallMembership[]>([])),
client: {
getDomain: () => "",
getDomain: () => "example.org",
baseUrl: "https://example.org",
// eslint-disable-next-line @typescript-eslint/naming-convention
_unstable_getRTCTransports: async () => Promise.resolve([]),
getAccessToken: vi.fn().mockReturnValue("access_token"),
// These won't be called in this error path but satisfy the type
getOpenIdToken: vi.fn(),
getDeviceId: vi.fn(),
@@ -537,10 +450,10 @@ describe("LocalTransport", () => {
await flushPromises();
expect(() => advertised$.value).toThrow(
new MatrixRTCTransportMissingError(""),
new MatrixRTCTransportMissingError("example.org"),
);
expect(() => active$.value).toThrow(
new MatrixRTCTransportMissingError(""),
new MatrixRTCTransportMissingError("example.org"),
);
});
});
@@ -565,11 +478,10 @@ describe("LocalTransport", () => {
delayId$: delayId$,
memberships$: constant(new Epoch<CallMembership[]>([])),
client: {
getDomain: () => "",
getDomain: () => "example.org",
baseUrl: "https://example.org",
// eslint-disable-next-line @typescript-eslint/naming-convention
_unstable_getRTCTransports: async () => Promise.resolve([]),
getAccessToken: vi.fn().mockReturnValue("access_token"),
// These won't be called in this error path but satisfy the type
getOpenIdToken: vi.fn(),
getDeviceId: vi.fn(),

View File

@@ -26,7 +26,6 @@ import {
tap,
} from "rxjs";
import { logger as rootLogger, type Logger } from "matrix-js-sdk/lib/logger";
import { AutoDiscovery } from "matrix-js-sdk/lib/autodiscovery";
import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
import { type Behavior } from "../../Behavior.ts";
@@ -57,7 +56,7 @@ interface Props {
memberships$: Behavior<Epoch<CallMembership[]>>;
client: Pick<
MatrixClient,
"getDomain" | "baseUrl" | "_unstable_getRTCTransports" | "getAccessToken"
"getDomain" | "baseUrl" | "_unstable_getRTCTransports"
> &
OpenIDClientParts;
// Used by the jwt service to create the livekit room and compute the livekit alias.
@@ -150,7 +149,6 @@ export const createLocalTransport$ = ({
const transportDiscovery = new RtcTransportAutoDiscovery({
client: client,
resolvedConfig: Config.get(),
wellKnownFetcher: AutoDiscovery.getRawClientConfig.bind(AutoDiscovery),
logger: logger,
});
@@ -309,7 +307,7 @@ async function doOpenIdAndJWTFromUrl(
roomId: string,
client: Pick<
MatrixClient,
"getDomain" | "baseUrl" | "_unstable_getRTCTransports" | "getAccessToken"
"getDomain" | "baseUrl" | "_unstable_getRTCTransports"
> &
OpenIDClientParts,
delayId?: string,
@@ -339,7 +337,7 @@ function observeLocalTransportForOldestMembership(
preferredTransport$: Observable<LocalTransportWithSFUConfig>,
client: Pick<
MatrixClient,
"getDomain" | "baseUrl" | "_unstable_getRTCTransports" | "getAccessToken"
"getDomain" | "baseUrl" | "_unstable_getRTCTransports"
> &
OpenIDClientParts,
ownMembershipIdentity: CallMembershipIdentityParts,

View File

@@ -13,7 +13,7 @@ import {
type MockedObject,
vi,
} from "vitest";
import { type IClientWellKnown, MatrixError } from "matrix-js-sdk";
import { MatrixError } from "matrix-js-sdk";
import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
import {
type LivekitTransportConfig,
@@ -33,9 +33,9 @@ const backendTransport: LivekitTransportConfig = {
livekit_service_url: "https://backend.example.org",
};
const wellKnownTransport: LivekitTransportConfig = {
const configTransport: LivekitTransportConfig = {
type: "livekit",
livekit_service_url: "https://well-known.example.org",
livekit_service_url: "https://config.example.org",
};
function makeClient(): MockedObject<DiscoveryClient> {
@@ -43,7 +43,6 @@ function makeClient(): MockedObject<DiscoveryClient> {
getDomain: vi.fn().mockReturnValue("example.org"),
baseUrl: "https://matrix.example.org",
_unstable_getRTCTransports: vi.fn().mockResolvedValue([]),
getAccessToken: vi.fn().mockReturnValue("access_token"),
getOpenIdToken: vi.fn(),
getDeviceId: vi.fn(),
} as unknown as MockedObject<DiscoveryClient>;
@@ -59,12 +58,6 @@ function makeResolvedConfig(livekitServiceUrl?: string): ResolvedConfigOptions {
} as ResolvedConfigOptions;
}
function makeWellKnown(rtcFoci?: Transport[]): IClientWellKnown {
return {
"org.matrix.msc4143.rtc_foci": rtcFoci,
} as unknown as IClientWellKnown;
}
describe("RtcTransportAutoDiscovery", () => {
beforeEach(() => {
vi.clearAllMocks();
@@ -75,33 +68,27 @@ describe("RtcTransportAutoDiscovery", () => {
{ transports: [{ type: "not_livekit" }, backendTransport] },
];
it.each(VALID_TEST_CASES)(
"prefers backend transport over well-known and app config $transports",
"prefers backend transport other app config $transports",
async ({ transports }) => {
// it("prefers backend transport over well-known and app config", async () => {
const client = makeClient();
client._unstable_getRTCTransports.mockResolvedValue(transports);
const wellKnownFetcher = vi
.fn<(domain: string) => Promise<IClientWellKnown>>()
.mockResolvedValue(makeWellKnown([wellKnownTransport]));
const discovery = new RtcTransportAutoDiscovery({
client,
resolvedConfig: makeResolvedConfig("https://config.example.org"),
wellKnownFetcher,
resolvedConfig: makeResolvedConfig(configTransport.livekit_service_url),
logger: rootLogger,
});
await expect(
discovery.discoverPreferredTransport(),
).resolves.toStrictEqual(backendTransport);
const discoveredTransport = await discovery.discoverPreferredTransport();
expect(discoveredTransport).toStrictEqual(backendTransport);
expect(discoveredTransport).not.toStrictEqual(configTransport);
expect(client._unstable_getRTCTransports).toHaveBeenCalledTimes(1);
expect(wellKnownFetcher).not.toHaveBeenCalled();
},
);
it("Retries limit_exceeded backend transport over well-known", async () => {
it("Retries limit_exceeded backend transport", async () => {
const client = makeClient();
client._unstable_getRTCTransports
.mockRejectedValueOnce(
@@ -116,14 +103,9 @@ describe("RtcTransportAutoDiscovery", () => {
)
.mockResolvedValue([backendTransport]);
const wellKnownFetcher = vi
.fn<(domain: string) => Promise<IClientWellKnown>>()
.mockResolvedValue(makeWellKnown([wellKnownTransport]));
const discovery = new RtcTransportAutoDiscovery({
client,
resolvedConfig: makeResolvedConfig("https://config.example.org"),
wellKnownFetcher,
logger: rootLogger,
});
@@ -132,7 +114,6 @@ describe("RtcTransportAutoDiscovery", () => {
);
expect(client._unstable_getRTCTransports).toHaveBeenCalledTimes(2);
expect(wellKnownFetcher).not.toHaveBeenCalled();
});
const INVALID_TEST_CASES: Array<{ transports: Transport[] }> = [
@@ -140,91 +121,30 @@ describe("RtcTransportAutoDiscovery", () => {
{ transports: [{ type: "not_livekit" }] },
];
it.each(INVALID_TEST_CASES)(
"falls back to well-known when backend has no (valid) livekit transports $transports",
"falls back to config when backend has no (valid) livekit transports $transports",
async ({ transports }) => {
const client = makeClient();
client._unstable_getRTCTransports.mockResolvedValue(transports);
const wellKnownFetcher = vi
.fn<(domain: string) => Promise<IClientWellKnown>>()
.mockResolvedValue(makeWellKnown([wellKnownTransport]));
const discovery = new RtcTransportAutoDiscovery({
client,
resolvedConfig: makeResolvedConfig("https://config.example.org"),
wellKnownFetcher,
resolvedConfig: makeResolvedConfig(configTransport.livekit_service_url),
logger: rootLogger,
});
await expect(
discovery.discoverPreferredTransport(),
).resolves.toStrictEqual(wellKnownTransport);
expect(wellKnownFetcher).toHaveBeenCalledWith("example.org");
const discoveredTransport = await discovery.discoverPreferredTransport();
expect(discoveredTransport).not.toStrictEqual(backendTransport);
expect(discoveredTransport).toStrictEqual(configTransport);
},
);
it("skips backend discovery in widget mode and uses well-known", async () => {
const client = makeClient();
// widget mode is detected by the absence of an access token
client.getAccessToken.mockReturnValue(null);
const wellKnownFetcher = vi
.fn<(domain: string) => Promise<IClientWellKnown>>()
.mockResolvedValue(makeWellKnown([wellKnownTransport]));
const discovery = new RtcTransportAutoDiscovery({
client,
resolvedConfig: makeResolvedConfig("https://config.example.org"),
wellKnownFetcher,
logger: rootLogger,
});
await expect(discovery.discoverPreferredTransport()).resolves.toStrictEqual(
wellKnownTransport,
);
expect(client._unstable_getRTCTransports).not.toHaveBeenCalled();
expect(wellKnownFetcher).toHaveBeenCalledWith("example.org");
});
it("falls back to app config when backend fails and well-known has no rtc_foci", async () => {
const client = makeClient();
client._unstable_getRTCTransports.mockRejectedValue(
new MatrixError({ errcode: "M_UNKNOWN" }, 404),
);
const wellKnownFetcher = vi
.fn<(domain: string) => Promise<IClientWellKnown>>()
.mockResolvedValue({} as IClientWellKnown);
const discovery = new RtcTransportAutoDiscovery({
client,
resolvedConfig: makeResolvedConfig("https://config.example.org"),
wellKnownFetcher,
logger: rootLogger,
});
await expect(discovery.discoverPreferredTransport()).resolves.toStrictEqual(
{
type: "livekit",
livekit_service_url: "https://config.example.org",
},
);
});
it("returns null when backend, well-known and config are all unavailable", async () => {
it("returns null when backend and config are all unavailable", async () => {
const client = makeClient();
client._unstable_getRTCTransports.mockResolvedValue([]);
const wellKnownFetcher = vi
.fn<(domain: string) => Promise<IClientWellKnown>>()
.mockResolvedValue({} as IClientWellKnown);
const discovery = new RtcTransportAutoDiscovery({
client,
resolvedConfig: makeResolvedConfig(undefined),
wellKnownFetcher,
logger: rootLogger,
});

View File

@@ -8,7 +8,7 @@ import {
isLivekitTransportConfig,
type LivekitTransportConfig,
} from "matrix-js-sdk/lib/matrixrtc";
import { type IClientWellKnown, type MatrixClient } from "matrix-js-sdk";
import { type MatrixClient } from "matrix-js-sdk";
import { type Logger } from "matrix-js-sdk/lib/logger";
import type { ResolvedConfigOptions } from "../../../config/ConfigOptions.ts";
@@ -16,33 +16,27 @@ import { doNetworkOperationWithRetry } from "../../../utils/matrix.ts";
type TransportDiscoveryClient = Pick<
MatrixClient,
"getDomain" | "_unstable_getRTCTransports" | "getAccessToken"
"getDomain" | "_unstable_getRTCTransports"
>;
export interface RtcTransportAutoDiscoveryProps {
client: TransportDiscoveryClient;
resolvedConfig: ResolvedConfigOptions;
wellKnownFetcher: (domain: string) => Promise<IClientWellKnown>;
logger: Logger;
}
export class RtcTransportAutoDiscovery {
private readonly client: TransportDiscoveryClient;
private readonly resolvedConfig: ResolvedConfigOptions;
private readonly wellKnownFetcher: (
domain: string,
) => Promise<IClientWellKnown>;
private readonly logger: Logger;
public constructor({
client,
resolvedConfig,
wellKnownFetcher,
logger,
}: RtcTransportAutoDiscoveryProps) {
this.client = client;
this.resolvedConfig = resolvedConfig;
this.wellKnownFetcher = wellKnownFetcher;
this.logger = logger.getChild("[RtcTransportAutoDiscovery]");
}
@@ -56,21 +50,7 @@ export class RtcTransportAutoDiscovery {
return backendTransport;
}
this.logger.info("No backend transport found, falling back to well-known");
// 2) .well-known transports
const wellKnownTransport = await this.tryWellKnownTransports();
if (wellKnownTransport) {
this.logger.info(
`Found .well-known transport: ${wellKnownTransport.livekit_service_url}`,
);
return wellKnownTransport;
}
this.logger.info(
"No .well-known transport found, falling back to app config",
);
// 3) app config URL
// 2) app config URL
const configTransport = this.tryConfigTransport();
if (configTransport) {
this.logger.info(
@@ -90,72 +70,23 @@ export class RtcTransportAutoDiscovery {
private async tryBackendTransports(): Promise<LivekitTransportConfig | null> {
const client = this.client;
// MSC4143: Attempt to fetch transports from backend.
// TODO: Workaround for an issue in the js-sdk RoomWidgetClient that
// is not yet implementing _unstable_getRTCTransports properly (via widget API new action).
// For now we just skip this call if we are in a widget.
// In widget mode the client is a `RoomWidgetClient` which has no access token (it is using the widget API).
// Could be removed once the js-sdk is fixed (https://github.com/matrix-org/matrix-js-sdk/issues/5245)
const isSPA = !!client.getAccessToken();
if (isSPA && "_unstable_getRTCTransports" in client) {
this.logger.info("First try to use getRTCTransports end point ...");
try {
const transportList = await doNetworkOperationWithRetry(async () =>
client._unstable_getRTCTransports(),
);
const first = transportList.find(isLivekitTransportConfig);
if (first) {
return first;
} else {
this.logger.info(
`No livekit transport found in getRTCTransports end point`,
transportList,
);
}
} catch (ex) {
this.logger.info(`Failed to use getRTCTransports end point: ${ex}`);
}
} else {
this.logger.debug(`getRTCTransports end point not available`);
}
return null;
}
/**
* Fetches the first rtc_foci from the .well-known/matrix/client.
* This will not throw errors, but instead just log them and return null if the expected config is not found or malformed.
* @private
*/
private async tryWellKnownTransports(): Promise<LivekitTransportConfig | null> {
// Legacy MSC4143 (to be removed) WELL_KNOWN: Prioritize the .well-known/matrix/client, if available.
const client = this.client;
const domain = client.getDomain();
if (domain) {
// we use AutoDiscovery instead of relying on the MatrixClient having already
// been fully configured and started
const wellKnownFoci = await this.wellKnownFetcher(domain);
const fociConfig = wellKnownFoci["org.matrix.msc4143.rtc_foci"];
if (fociConfig) {
if (!Array.isArray(fociConfig)) {
this.logger.warn(
`org.matrix.msc4143.rtc_foci is not an array in .well-known`,
);
} else {
return fociConfig[0];
}
this.logger.info("First try to use getRTCTransports end point ...");
try {
const transportList = await doNetworkOperationWithRetry(async () =>
client._unstable_getRTCTransports(),
);
const first = transportList.find(isLivekitTransportConfig);
if (first) {
return first;
} else {
this.logger.info(
`No .well-known "org.matrix.msc4143.rtc_foci" found for ${domain}`,
wellKnownFoci,
`No livekit transport found in getRTCTransports end point`,
transportList,
);
}
} else {
// Should never happen, but just in case
this.logger.warn(`No domain configured for client`);
} catch (ex) {
this.logger.info(`Failed to use getRTCTransports end point: ${ex}`);
}
return null;
}

View File

@@ -12,7 +12,7 @@ import {
type E2EEManagerOptions,
type BaseE2EEManager,
} from "livekit-client";
import { type Logger } from "matrix-js-sdk/lib/logger";
import { logger, type Logger } from "matrix-js-sdk/lib/logger";
// imported as inline to support worker when loaded from a cdn (cross domain)
import E2EEWorker from "livekit-client/e2ee-worker?worker&inline";
import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
@@ -27,7 +27,18 @@ import type {
import type { MediaDevices } from "../../MediaDevices.ts";
import type { Behavior } from "../../Behavior.ts";
import type { ProcessorState } from "../../../livekit/TrackProcessorContext.tsx";
import { defaultLiveKitOptions } from "../../../livekit/options.ts";
import { getLiveKitOptions } from "../../../livekit/options.ts";
import {
advancedCamera,
cameraResolution,
cameraFramerate,
cameraBitrate,
cameraCodec,
parseResolution,
echoCancellationSetting,
noiseSuppressionSetting,
autoGainControlSetting,
} from "../../../settings/settings.ts";
// TODO evaluate if this should be done like the Publisher Factory
export interface ConnectionFactory {
@@ -53,8 +64,6 @@ export class ECConnectionFactory implements ConnectionFactory {
* @param livekitKeyProvider - Optional key provider for end-to-end encryption.
* @param controlledAudioDevices - Option to indicate whether audio output device is controlled externally (native mobile app).
* @param livekitRoomFactory - Optional factory function (for testing) to create LivekitRoom instances. If not provided, a default factory is used.
* @param echoCancellation - Whether to enable echo cancellation for audio capture.
* @param noiseSuppression - Whether to enable noise suppression for audio capture.
*/
public constructor(
private client: OpenIDClientParts,
@@ -64,25 +73,22 @@ export class ECConnectionFactory implements ConnectionFactory {
livekitKeyProvider: BaseKeyProvider | undefined,
private controlledAudioDevices: boolean,
livekitRoomFactory?: () => LivekitRoom,
echoCancellation: boolean = true,
noiseSuppression: boolean = true,
) {
const defaultFactory = (): LivekitRoom =>
new LivekitRoom(
generateRoomOption({
devices: this.devices,
processorState: this.processorState$.value,
e2eeLivekitOptions: livekitKeyProvider && {
keyProvider: livekitKeyProvider,
// It's important that every room use a separate E2EE worker.
// They get confused if given streams from multiple rooms.
worker: new E2EEWorker(),
},
controlledAudioDevices: this.controlledAudioDevices,
echoCancellation,
noiseSuppression,
}),
);
const defaultFactory = (): LivekitRoom => {
const roomOptions = generateRoomOption({
devices: this.devices,
processorState: this.processorState$.value,
e2eeLivekitOptions: livekitKeyProvider && {
keyProvider: livekitKeyProvider,
// It's important that every room use a separate E2EE worker.
// They get confused if given streams from multiple rooms.
worker: new E2EEWorker(),
},
controlledAudioDevices: this.controlledAudioDevices,
});
logger.info("[ECConnectionFactory] livekit room options: ", roomOptions);
return new LivekitRoom(roomOptions);
};
this.livekitRoomFactory = livekitRoomFactory ?? defaultFactory;
}
@@ -119,14 +125,13 @@ export class ECConnectionFactory implements ConnectionFactory {
/**
* Generate the initial LiveKit RoomOptions based on the current media devices and processor state.
* Reads audio processing and camera quality settings directly from Settings.
*/
function generateRoomOption({
devices,
processorState,
e2eeLivekitOptions,
controlledAudioDevices,
echoCancellation,
noiseSuppression,
}: {
devices: MediaDevices;
processorState: ProcessorState;
@@ -135,21 +140,44 @@ function generateRoomOption({
| { e2eeManager: BaseE2EEManager }
| undefined;
controlledAudioDevices: boolean;
echoCancellation: boolean;
noiseSuppression: boolean;
}): RoomOptions {
const liveKitOptions = getLiveKitOptions();
// Apply advanced camera settings if enabled
let videoCaptureDefaults = {
...liveKitOptions.videoCaptureDefaults,
deviceId: devices.videoInput.selected$.value?.id,
processor: processorState.processor,
};
let publishDefaults = liveKitOptions.publishDefaults;
if (advancedCamera.getValue()) {
const { width, height } = parseResolution(cameraResolution.getValue());
const fps = cameraFramerate.getValue();
const bps = cameraBitrate.getValue();
const codec = cameraCodec.getValue();
videoCaptureDefaults = {
...videoCaptureDefaults,
resolution: { width, height, frameRate: fps },
};
publishDefaults = {
...publishDefaults,
videoEncoding: { maxBitrate: bps, maxFramerate: fps },
videoCodec: codec,
};
}
return {
...defaultLiveKitOptions,
videoCaptureDefaults: {
...defaultLiveKitOptions.videoCaptureDefaults,
deviceId: devices.videoInput.selected$.value?.id,
processor: processorState.processor,
},
...liveKitOptions,
videoCaptureDefaults,
publishDefaults,
audioCaptureDefaults: {
...defaultLiveKitOptions.audioCaptureDefaults,
...liveKitOptions.audioCaptureDefaults,
deviceId: devices.audioInput.selected$.value?.id,
echoCancellation,
noiseSuppression,
echoCancellation: echoCancellationSetting.getValue(),
noiseSuppression: noiseSuppressionSetting.getValue(),
autoGainControl: autoGainControlSetting.getValue(),
},
audioOutput: {
// When using controlled audio devices, we don't want to set the

View File

@@ -22,6 +22,16 @@ import {
} from "../../../utils/test.ts";
import type { ProcessorState } from "../../../livekit/TrackProcessorContext.tsx";
import { constant } from "../../Behavior";
import {
echoCancellationSetting,
noiseSuppressionSetting,
autoGainControlSetting,
advancedCamera,
cameraResolution,
cameraFramerate,
cameraBitrate,
cameraCodec,
} from "../../../settings/settings.ts";
// At the top of your test file, after imports
vi.mock("livekit-client", async (importOriginal) => {
@@ -58,11 +68,14 @@ describe("ECConnectionFactory - Audio inputs options", () => {
{ echo: false, noise: true },
{ echo: false, noise: false },
])(
"it sets echoCancellation=$echo and noiseSuppression=$noise based on constructor parameters",
"it sets echoCancellation=$echo and noiseSuppression=$noise based on settings",
({ echo, noise }) => {
// test("it sets echoCancellation and noiseSuppression based on constructor parameters", () => {
const RoomConstructor = vi.mocked(LivekitRoom);
// Set audio processing settings
echoCancellationSetting.setValue(echo);
noiseSuppressionSetting.setValue(noise);
const ecConnectionFactory = new ECConnectionFactory(
mockClient,
"!roomid:example.org",
@@ -73,9 +86,6 @@ describe("ECConnectionFactory - Audio inputs options", () => {
}),
undefined,
false,
undefined,
echo,
noise,
);
ecConnectionFactory.createConnection(
testScope,
@@ -101,9 +111,13 @@ describe("ECConnectionFactory - ControlledAudioDevice", () => {
test.each([{ controlled: true }, { controlled: false }])(
"it sets controlledAudioDevice=$controlled then uses deviceId accordingly",
({ controlled }) => {
// test("it sets echoCancellation and noiseSuppression based on constructor parameters", () => {
const RoomConstructor = vi.mocked(LivekitRoom);
// Explicitly set audio settings so the test doesn't depend on defaults
echoCancellationSetting.setValue(true);
noiseSuppressionSetting.setValue(true);
autoGainControlSetting.setValue(true);
const ecConnectionFactory = new ECConnectionFactory(
mockClient,
"!roomid:example.org",
@@ -120,9 +134,6 @@ describe("ECConnectionFactory - ControlledAudioDevice", () => {
}),
undefined,
controlled,
undefined,
false,
false,
);
ecConnectionFactory.createConnection(
testScope,
@@ -143,6 +154,114 @@ describe("ECConnectionFactory - ControlledAudioDevice", () => {
);
});
describe("ECConnectionFactory - Camera quality settings", () => {
test("it uses default video options when advancedCamera is disabled", () => {
const RoomConstructor = vi.mocked(LivekitRoom);
advancedCamera.setValue(false);
const ecConnectionFactory = new ECConnectionFactory(
mockClient,
"!roomid:example.org",
mockMediaDevices({}),
new BehaviorSubject<ProcessorState>({
supported: true,
processor: undefined,
}),
undefined,
false,
);
ecConnectionFactory.createConnection(
testScope,
exampleTransport,
ownMemberMock,
logger,
);
// publishDefaults should use config defaults (vp8), not custom settings
expect(RoomConstructor).toHaveBeenCalledWith(
expect.objectContaining({
publishDefaults: expect.objectContaining({
videoCodec: "vp8",
}),
}),
);
});
test("it applies custom camera resolution, encoding, and codec when advancedCamera is enabled", () => {
const RoomConstructor = vi.mocked(LivekitRoom);
advancedCamera.setValue(true);
cameraResolution.setValue("1920x1080");
cameraFramerate.setValue(60);
cameraBitrate.setValue(4_000_000);
cameraCodec.setValue("vp9");
const ecConnectionFactory = new ECConnectionFactory(
mockClient,
"!roomid:example.org",
mockMediaDevices({}),
new BehaviorSubject<ProcessorState>({
supported: true,
processor: undefined,
}),
undefined,
false,
);
ecConnectionFactory.createConnection(
testScope,
exampleTransport,
ownMemberMock,
logger,
);
expect(RoomConstructor).toHaveBeenCalledWith(
expect.objectContaining({
videoCaptureDefaults: expect.objectContaining({
resolution: { width: 1920, height: 1080, frameRate: 60 },
}),
publishDefaults: expect.objectContaining({
videoEncoding: { maxBitrate: 4_000_000, maxFramerate: 60 },
videoCodec: "vp9",
}),
}),
);
});
test("it applies autoGainControl from settings", () => {
const RoomConstructor = vi.mocked(LivekitRoom);
autoGainControlSetting.setValue(false);
echoCancellationSetting.setValue(true);
noiseSuppressionSetting.setValue(true);
const ecConnectionFactory = new ECConnectionFactory(
mockClient,
"!roomid:example.org",
mockMediaDevices({}),
new BehaviorSubject<ProcessorState>({
supported: true,
processor: undefined,
}),
undefined,
false,
);
ecConnectionFactory.createConnection(
testScope,
exampleTransport,
ownMemberMock,
logger,
);
expect(RoomConstructor).toHaveBeenCalledWith(
expect.objectContaining({
audioCaptureDefaults: expect.objectContaining({
autoGainControl: false,
}),
}),
);
});
});
afterEach(() => {
testScope.end();
fetchMock.reset();

View File

@@ -5,7 +5,10 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type RemoteTrackPublication } from "livekit-client";
import {
type LocalTrackPublication,
type RemoteTrackPublication,
} from "livekit-client";
import { test, expect } from "vitest";
import { act, render, screen } from "@testing-library/react";
import { axe } from "vitest-axe";
@@ -17,6 +20,9 @@ import {
mockRtcMembership,
mockRemoteMedia,
mockRemoteParticipant,
mockLocalMedia,
mockLocalParticipant,
mockMediaDevices,
} from "../utils/test";
import { GridTileViewModel } from "../state/TileViewModel";
import { ReactionsSenderProvider } from "../reactions/useReactionsSender";
@@ -54,7 +60,7 @@ const callVm = {
handsRaised$: constant({}),
} as Partial<CallViewModel> as CallViewModel;
test("GridTile is accessible", async () => {
test("GridTile displays remote media", async () => {
const vm = mockRemoteMedia(
mockRtcMembership("@alice:example.org", "AAAA"),
{
@@ -88,6 +94,40 @@ test("GridTile is accessible", async () => {
screen.getByText("Alice");
});
test("GridTile displays local media", async () => {
const vm = mockLocalMedia(
mockRtcMembership("@alice:example.org", "AAAA"),
{
rawDisplayName: "Alice",
getMxcAvatarUrl: () => "mxc://adfsg",
},
mockLocalParticipant({
getTrackPublication: () =>
({}) as Partial<LocalTrackPublication> as LocalTrackPublication,
}),
mockMediaDevices({}),
);
const { container } = render(
<ReactionsSenderProvider vm={callVm} rtcSession={fakeRtcSession}>
<GridTile
vm={new GridTileViewModel(constant(vm))}
onOpenProfile={() => {}}
targetWidth={300}
targetHeight={200}
showSpeakingIndicators
showNameTags
showRingingStatus
showOutline
focusable
/>
</ReactionsSenderProvider>,
);
expect(await axe(container)).toHaveNoViolations();
// Name should be visible
screen.getByText("Alice");
});
test("GridTile displays ringing media", async () => {
const pickupState$ = new BehaviorSubject<
RingingMediaViewModel["pickupState$"]["value"]

View File

@@ -13,6 +13,7 @@ import {
useCallback,
useRef,
useState,
useMemo,
} from "react";
import { type animated } from "@react-spring/web";
import classNames from "classnames";
@@ -103,20 +104,22 @@ interface UserMediaTileProps extends TileProps {
playbackMuted: boolean;
waitingForMedia?: boolean;
primaryButton?: ReactNode;
menuStart?: ReactNode;
menuEnd?: ReactNode;
focusUrl: string | undefined;
}
const UserMediaTile: FC<UserMediaTileProps> = ({
/**
* A user media tile without a context menu.
*/
// The context menu is kept separate from this component for performance
// reasons (c.f. UserMediaTile)
const UserMediaTileInner: FC<UserMediaTileProps & { menu: ReactNode }> = ({
ref,
vm,
showSpeakingIndicators,
playbackMuted,
waitingForMedia,
primaryButton,
menuStart,
menuEnd,
menu,
className,
focusUrl,
displayName,
@@ -156,24 +159,26 @@ const UserMediaTile: FC<UserMediaTileProps> = ({
: t("microphone_off");
const [menuOpen, setMenuOpen] = useState(false);
const menu = (
<>
{menuStart}
{/*
No additional menu item (used to be the manual fit to frame.
Placeholder for future menu items that should be placed here.
*/}
{menuEnd}
</>
const menuTrigger = useMemo(
() => (
<button
aria-label={t("common.options")}
tabIndex={focusable ? undefined : -1}
>
<OverflowHorizontalIcon aria-hidden width={20} height={20} />
</button>
),
[t, focusable],
);
const raisedHandOnClick = vm.local
? (): void => void toggleRaisedHand()
: undefined;
const raisedHandOnClick = useMemo(
() => (vm.local ? (): void => void toggleRaisedHand() : undefined),
[vm.local, toggleRaisedHand],
);
const showSpeaking = showSpeakingIndicators && speaking;
const tile = (
return (
<MediaView
ref={ref}
video={video}
@@ -202,14 +207,7 @@ const UserMediaTile: FC<UserMediaTileProps> = ({
open={menuOpen}
onOpenChange={setMenuOpen}
title={displayName}
trigger={
<button
aria-label={t("common.options")}
tabIndex={focusable ? undefined : -1}
>
<OverflowHorizontalIcon aria-hidden width={20} height={20} />
</button>
}
trigger={menuTrigger}
side="left"
align="start"
>
@@ -231,9 +229,37 @@ const UserMediaTile: FC<UserMediaTileProps> = ({
{...props}
/>
);
};
/**
* A user media tile enhanced with a context menu.
*/
const UserMediaTile: FC<
UserMediaTileProps & { menuStart?: ReactNode; menuEnd?: ReactNode }
> = ({ menuStart, menuEnd, ...props }) => {
const menu = useMemo(
() => (
<>
{menuStart}
{/*
No additional menu item (used to be the manual fit to frame.
Placeholder for future menu items that should be placed here.
*/}
{menuEnd}
</>
),
[menuStart, menuEnd],
);
// ContextMenu is expensive to render, so we avoid subscribing to any
// frequently-changing behaviors here and instead keep them isolated in the
// UserMediaTileInner component
return (
<ContextMenu title={displayName} trigger={tile} hasAccessibleAlternative>
<ContextMenu
title={props.displayName}
trigger={<UserMediaTileInner {...props} menu={menu} />}
hasAccessibleAlternative
>
{menu}
</ContextMenu>
);
@@ -269,6 +295,29 @@ const LocalUserMediaTile: FC<LocalUserMediaTileProps> = ({
[vm, latestAlwaysShow],
);
const menuStart = useMemo(
() => (
<ToggleMenuItem
Icon={VisibilityOnIcon}
label={t("video_tile.always_show")}
checked={alwaysShow}
onSelect={onSelectAlwaysShow}
/>
),
[t, alwaysShow, onSelectAlwaysShow],
);
const menuEnd = useMemo(
() =>
onOpenProfile && (
<MenuItem
Icon={UserProfileIcon}
label={t("common.profile")}
onSelect={onOpenProfile}
/>
),
[t, onOpenProfile],
);
return (
<UserMediaTile
ref={ref}
@@ -287,23 +336,8 @@ const LocalUserMediaTile: FC<LocalUserMediaTileProps> = ({
</button>
)
}
menuStart={
<ToggleMenuItem
Icon={VisibilityOnIcon}
label={t("video_tile.always_show")}
checked={alwaysShow}
onSelect={onSelectAlwaysShow}
/>
}
menuEnd={
onOpenProfile && (
<MenuItem
Icon={UserProfileIcon}
label={t("common.profile")}
onSelect={onOpenProfile}
/>
)
}
menuStart={menuStart}
menuEnd={menuEnd}
focusable={focusable}
focusUrl={focusUrl}
{...props}

View File

@@ -12,7 +12,7 @@ import { i18nKey } from "./i18n";
export enum ErrorCode {
/**
* Configuration problem due to no MatrixRTC backend/SFU is exposed via .well-known and no fallback configured.
* Configuration problem due to no MatrixRTC transport provided by homeserver and no fallback configured.
*/
MISSING_MATRIX_RTC_TRANSPORT = "MISSING_MATRIX_RTC_TRANSPORT",
CONNECTION_LOST_ERROR = "CONNECTION_LOST_ERROR",
@@ -67,7 +67,7 @@ export class ElementCallError extends Error {
}
/**
* Configuration problem due to no MatrixRTC backend/SFU is exposed via .well-known and no fallback configured.
* Configuration problem due to no MatrixRTC transport provided by homeserver and no fallback configured.
*/
export class MatrixRTCTransportMissingError extends ElementCallError {
public domain: string;

View File

@@ -19,6 +19,7 @@ const createRoomWidgetClientSpy = vi.mocked(createRoomWidgetClient);
vi.mock("./config/Config", () => ({
Config: {
init: vi.fn().mockImplementation(async () => Promise.resolve()),
get: vi.fn().mockReturnValue({}),
},
}));
const configInitSpy = vi.mocked(Config.init);
@@ -109,6 +110,7 @@ describe("widget", () => {
sendToDevice: sendRecvToDevice,
receiveToDevice: sendRecvToDevice,
turnServers: false,
rtcTransports: true,
sendDelayedEvents: true,
updateDelayedEvents: true,
sendSticky: true,

View File

@@ -21,6 +21,7 @@ import type { IWidgetApiRequest } from "matrix-widget-api";
import { LazyEventEmitter } from "./LazyEventEmitter";
import { getUrlParams } from "./UrlParams";
import { Config } from "./config/Config";
import { seedSettingsFromConfig } from "./settings/settings";
import { ElementCallReactionEventType } from "./reactions";
// Subset of the actions in element-web
@@ -173,6 +174,7 @@ export const initializeWidget = (
sendToDevice: sendRecvToDevice,
receiveToDevice: sendRecvToDevice,
turnServers: false,
rtcTransports: true,
sendDelayedEvents: true,
updateDelayedEvents: true,
sendSticky: true,
@@ -195,7 +197,8 @@ export const initializeWidget = (
// Wait for the config file to be ready (we load very early on so it might not
// be otherwise)
await Config.init();
await client.startClient({ clientWellKnownPollPeriod: 60 * 10 });
seedSettingsFromConfig(Config.get().media_quality);
await client.startClient();
return client;
};