use the matrixRtcSdk

This commit is contained in:
Timo K.
2026-10-14 10:23:29 +02:00
committed by Timo
parent 10b34b0218
commit 5f3db07b76
94 changed files with 1664 additions and 12281 deletions
-1
View File
@@ -41,7 +41,6 @@ import {
import { MAX_PARTICIPANT_COUNT_FOR_SOUND } from "../state/CallViewModel/CallViewModel";
import { initializeWidget } from "../widget";
initializeWidget();
vitest.mock("livekit-client/e2ee-worker?worker");
vitest.mock("../useAudioContext");
vitest.mock("../soundUtils");
vitest.mock("../rtcSessionHelpers", async (importOriginal) => ({
+1 -1
View File
@@ -61,7 +61,7 @@ import {
} from "../HostBridge";
import { type JoinCallData } from "../widget";
import { MatrixRTCTransportMissingError } from "../utils/errors";
import { ProcessorProvider } from "../livekit/TrackProcessorContext";
import { ProcessorProvider } from "../tracks/TrackProcessorContext";
import { MediaDevicesContext } from "../MediaDevicesContext";
import { constant } from "../state/Behavior";
+56 -67
View File
@@ -15,22 +15,20 @@ import {
vi,
} from "vitest";
import { act, render, type RenderResult } from "@testing-library/react";
import { type LocalParticipant } from "livekit-client";
import { BehaviorSubject, of } from "rxjs";
import { BehaviorSubject } from "rxjs";
import { BrowserRouter } from "react-router-dom";
import { TooltipProvider } from "@vector-im/compound-web";
import { RoomContext, useLocalParticipant } from "@livekit/components-react";
import userEvent from "@testing-library/user-event";
import { ActiveCall, InCallView } from "./InCallView";
import {
mockLivekitRoom,
mockLocalParticipant,
mockMediaDevices,
mockMuteStates,
mockRemoteParticipant,
mockRtcMembership,
type MockRTCSession,
mockMatrixRTCClient,
mockMemberMedia,
mockRTCMember,
} from "../utils/test";
import { E2eeType } from "../e2ee/e2eeType";
import {
@@ -41,15 +39,17 @@ import {
type CallViewModel,
type CallViewModelOptions,
} from "../state/CallViewModel/CallViewModel";
import { alice, local } from "../utils/test-fixtures";
import { alice, local, aliceRtcMember } from "../utils/test-fixtures";
import { ReactionsSenderProvider } from "../reactions/useReactionsSender";
import { useRoomEncryptionSystem } from "../e2ee/sharedKeyManagement";
import { LivekitRoomAudioRenderer } from "../livekit/MatrixAudioRenderer";
import { MemberAudioRenderer } from "../tracks/MemberAudioRenderer";
import type * as MatrixRTCSdk from "@element-hq/matrixrtc-sdk";
import { constant, type ObservableScope } from "@element-hq/matrixrtc-sdk";
import { MediaDevicesContext } from "../MediaDevicesContext";
import { type MediaDevices as ECMediaDevices } from "../state/MediaDevices";
import { AppBar } from "../AppBar";
import { type MatrixInfo } from "./VideoPreview";
import { ProcessorProvider } from "../livekit/TrackProcessorContext";
import { ProcessorProvider } from "../tracks/TrackProcessorContext";
import { initializeWidget } from "../widget";
import { RootElementProvider } from "../RootElementContext";
@@ -68,21 +68,19 @@ vi.mock("../soundUtils");
vi.mock("../useAudioContext");
vi.mock("../tile/GridTile");
vi.mock("../tile/SpotlightTile");
vi.mock("@livekit/components-react");
vi.mock("livekit-client/e2ee-worker?worker");
vi.mock("../e2ee/sharedKeyManagement");
vi.mock("../livekit/MatrixAudioRenderer");
vi.mock("../tracks/MemberAudioRenderer");
// ActiveCall builds the client itself; the view is what these tests are about
const createMatrixRTCClientMock = vi.hoisted(() => vi.fn());
vi.mock("@element-hq/matrixrtc-sdk", async (importOriginal) => ({
...(await importOriginal<typeof MatrixRTCSdk>()),
createMatrixRTCClient: createMatrixRTCClientMock,
}));
vi.mock("react-use-measure", () => ({
default: (): [() => void, object] => [(): void => {}, {}],
}));
const localRtcMember = mockRtcMembership("@carol:example.org", "CCCC");
const localParticipant = mockLocalParticipant({
identity: "@local:example.org:AAAAAA",
});
const remoteParticipant = mockRemoteParticipant({
identity: "@alice:example.org:AAAAAA",
});
const matrixInfo = {
userId: "",
@@ -100,21 +98,26 @@ let useRoomEncryptionSystemMock: MockedFunction<typeof useRoomEncryptionSystem>;
beforeEach(() => {
vi.clearAllMocks();
// MatrixAudioRenderer is tested separately.
(
LivekitRoomAudioRenderer as MockedFunction<typeof LivekitRoomAudioRenderer>
).mockImplementation((_props) => {
return <div>mocked: MatrixAudioRenderer</div>;
});
(
useLocalParticipant as MockedFunction<typeof useLocalParticipant>
).mockImplementation(
() =>
({
isScreenShareEnabled: false,
localParticipant: localRtcMember as unknown as LocalParticipant,
}) as unknown as ReturnType<typeof useLocalParticipant>,
createMatrixRTCClientMock.mockImplementation((scope: ObservableScope) =>
mockMatrixRTCClient(scope, {
localMember$: constant(
mockRTCMember(true, {
membership: localRtcMember,
media$: constant(mockMemberMedia({ local: true })),
}),
),
remoteMembers$: constant([
mockRTCMember(false, {
membership: aliceRtcMember,
media$: constant(mockMemberMedia()),
}),
]),
}),
);
// MemberAudioRenderer is tested separately.
(
MemberAudioRenderer as MockedFunction<typeof MemberAudioRenderer>
).mockImplementation(() => <div>mocked: MemberAudioRenderer</div>);
useRoomEncryptionSystemMock =
useRoomEncryptionSystem as typeof useRoomEncryptionSystemMock;
useRoomEncryptionSystemMock.mockReturnValue({ kind: E2eeType.NONE });
@@ -131,14 +134,6 @@ function createInCallView(args: CreateInCallViewArgs = {}): RenderResult & {
} {
const mediaDevices = args.mediaDevices ?? mockMediaDevices({});
const muteState = mockMuteStates();
const livekitRoom = mockLivekitRoom(
{
localParticipant,
},
{
remoteParticipants$: of([remoteParticipant]),
},
);
const { vm, footerVm, developerSettingsVm, rtcSession } =
getBasicCallViewModelEnvironment(
[local, alice],
@@ -174,9 +169,7 @@ function createInCallView(args: CreateInCallViewArgs = {}): RenderResult & {
vm={vm}
rtcSession={rtcSession.asMockedSession()}
>
<TooltipProvider>
<RoomContext value={livekitRoom}>{content}</RoomContext>
</TooltipProvider>
<TooltipProvider>{content}</TooltipProvider>
</ReactionsSenderProvider>
</MediaDevicesContext>
</BrowserRouter>,
@@ -244,18 +237,16 @@ describe("ActiveCall", () => {
<MediaDevicesContext value={mediaDevices}>
<ProcessorProvider>
<TooltipProvider>
<RoomContext value={mockLivekitRoom({ localParticipant })}>
<ActiveCall
client={matrixRoom.client}
rtcSession={rtcSession.asMockedSession()}
matrixRoom={matrixRoom}
muteStates={mockMuteStates()}
matrixInfo={matrixInfo}
onShareClick={null}
e2eeSystem={{ kind: E2eeType.NONE }}
onLeft={(): void => {}}
/>
</RoomContext>
<ActiveCall
client={matrixRoom.client}
rtcSession={rtcSession.asMockedSession()}
matrixRoom={matrixRoom}
muteStates={mockMuteStates()}
matrixInfo={matrixInfo}
onShareClick={null}
e2eeSystem={{ kind: E2eeType.NONE }}
onLeft={(): void => {}}
/>
</TooltipProvider>
</ProcessorProvider>
</MediaDevicesContext>
@@ -294,18 +285,16 @@ describe("ActiveCall", () => {
<MediaDevicesContext value={mediaDevices}>
<ProcessorProvider>
<TooltipProvider>
<RoomContext value={mockLivekitRoom({ localParticipant })}>
<ActiveCall
client={matrixRoom.client}
rtcSession={rtcSession.asMockedSession()}
matrixRoom={matrixRoom}
muteStates={mockMuteStates()}
matrixInfo={matrixInfo}
onShareClick={null}
e2eeSystem={{ kind: E2eeType.NONE }}
onLeft={(): void => {}}
/>
</RoomContext>
<ActiveCall
client={matrixRoom.client}
rtcSession={rtcSession.asMockedSession()}
matrixRoom={matrixRoom}
muteStates={mockMuteStates()}
matrixInfo={matrixInfo}
onShareClick={null}
e2eeSystem={{ kind: E2eeType.NONE }}
onLeft={(): void => {}}
/>
</TooltipProvider>
</ProcessorProvider>
</MediaDevicesContext>
+156 -45
View File
@@ -46,7 +46,10 @@ import {
} from "../state/CallViewModel/CallViewModel.ts";
import { Grid, type TileProps } from "../grid/Grid";
import { SpotlightTile } from "../tile/SpotlightTile";
import { type EncryptionSystem } from "../e2ee/sharedKeyManagement";
import {
type EncryptionSystem,
type MatrixRTCMode,
} from "@element-hq/matrixrtc-sdk";
import { E2eeType } from "../e2ee/e2eeType";
import { makeGridLayout } from "../grid/GridLayout";
import { type CallLayoutOutputs } from "../grid/CallLayout";
@@ -63,9 +66,36 @@ import {
import { ReactionsAudioRenderer } from "./ReactionAudioRenderer";
import { ReactionsOverlay } from "./ReactionsOverlay";
import { CallEventAudioRenderer } from "./CallEventAudioRenderer";
import { matrixRTCMode as matrixRTCModeSetting } from "../settings/settings";
import {
createMatrixRTCClient,
type MatrixRTCClientOptions,
constant,
type Behavior,
} from "@element-hq/matrixrtc-sdk";
import { map } from "rxjs";
import {
advancedCamera,
advancedScreenShare,
autoGainControlSetting,
cameraBitrate,
cameraCodec,
cameraFramerate,
cameraResolution,
customLivekitUrl,
echoCancellationSetting,
matrixRTCMode as matrixRTCModeSetting,
noiseSuppressionSetting,
parseResolution,
screenShareBitrate,
screenShareCodec,
screenShareFramerate,
screenShareResolution,
} from "../settings/settings";
import { Config } from "../config/Config";
import { type MediaDevice } from "../state/MediaDevices";
import { createSentCallNotification$ } from "../state/CallViewModel/CallNotificationLifecycle.ts";
import { ReactionsReader } from "../reactions/ReactionsReader";
import { LivekitRoomAudioRenderer } from "../livekit/MatrixAudioRenderer.tsx";
import { MemberAudioRenderer } from "../tracks/MemberAudioRenderer.tsx";
import { muteAllAudio$ } from "../state/MuteAllAudioModel.ts";
import { useMediaDevices } from "../MediaDevicesContext.ts";
import { EarpieceOverlay } from "./EarpieceOverlay.tsx";
@@ -76,10 +106,9 @@ import {
} from "../AppBar.tsx";
import { useBehavior } from "../useBehavior.ts";
import { useValueBehavior } from "../useValueBehavior.ts";
import { constant } from "../state/Behavior.ts";
import { Toast } from "../Toast.tsx";
import overlayStyles from "../Overlay.module.css";
import { useTrackProcessorState$ } from "../livekit/TrackProcessorContext.tsx";
import { useTrackProcessorState$ } from "../tracks/TrackProcessorContext.tsx";
import { type Layout } from "../state/layout-types.ts";
import { ObservableScope } from "../state/ObservableScope.ts";
import { CallFooter, type FooterSnapshot } from "../components/CallFooter.tsx";
@@ -100,7 +129,7 @@ declare module "react" {
export interface ActiveCallProps extends Omit<
InCallViewProps,
"vm" | "livekitRoom" | "connState" | "footerVm" | "developerSettingsVm"
"vm" | "footerVm" | "developerSettingsVm"
> {
e2eeSystem: EncryptionSystem;
// TODO refactor those reasons into an enum
@@ -131,9 +160,28 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
const { autoLeaveWhenOthersLeft, waitForCallPickup, sendNotificationType } =
urlParams;
const rtcClient = createMatrixRTCClient(
scope,
props.client,
props.matrixRoom,
{
microphoneEnabled: props.muteStates.audio.enabled$.value,
cameraEnabled: props.muteStates.video.enabled$.value,
audioInputDeviceId$: selectedDeviceId$(scope, mediaDevices.audioInput),
videoInputDeviceId$: selectedDeviceId$(scope, mediaDevices.videoInput),
// A host that routes audio itself leaves the browser's output alone
audioOutputDeviceId$: urlParams.controlledAudioDevices
? constant(undefined)
: selectedDeviceId$(scope, mediaDevices.audioOutput),
videoProcessor$: scope.behavior(
trackProcessorState$.pipe(map((state) => state.processor)),
),
},
matrixRTCClientOptions(urlParams, props.e2eeSystem),
);
const vm = createCallViewModel$(
scope,
props.rtcSession,
rtcClient,
props.matrixRoom,
mediaDevices,
props.muteStates,
@@ -143,14 +191,14 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
hostBridge,
autoLeaveWhenOthersLeft,
waitForCallPickup: waitForCallPickup && sendNotificationType === "ring",
// We merely sample the current mode here, so the user would need to
// manually rejoin to switch to a different one.
matrixRTCMode: matrixRTCModeSetting.value$.value,
sentCallNotification$: createSentCallNotification$(
scope,
props.rtcSession,
),
windowSize$: scope.behavior(observeElementSize$(rootElement)),
},
reactionsReader.raisedHands$,
reactionsReader.reactions$,
trackProcessorState$,
);
// TODO move this somewhere else once we use the callViewModel in the lobby as well!
vm.join();
@@ -250,16 +298,6 @@ export const InCallView: FC<InCallViewProps> = ({
const { sendReaction, toggleRaisedHand } = useReactionsSender();
useWakeLock();
// TODO-MULTI-SFU This is unused now??
// const connectionState = useBehavior(vm.livekitConnectionState$);
// annoyingly we don't get the disconnection reason this way,
// only by listening for the emitted event
// This needs to be done differential. with the vm connection state we start with Disconnected.
// TODO-MULTI-SFU decide how to handle this properly
// @BillCarsonFr
// if (connectionState === ConnectionState.Disconnected)
// throw new ConnectionLostError();
const containerRef1 = useRef<HTMLDivElement | null>(null);
const [containerRef2, bounds] = useMeasure();
@@ -296,7 +334,7 @@ export const InCallView: FC<InCallViewProps> = ({
);
const ringingVm = useBehavior(vm.ringingVm$);
const audioParticipants = useBehavior(vm.livekitRoomItems$);
const remoteMembers = useBehavior(vm.remoteMembers$);
const participantCount = useBehavior(vm.participantCount$);
const reconnecting = useBehavior(vm.reconnecting$);
const screenShareError = useBehavior(vm.screenShareError$);
@@ -646,7 +684,11 @@ export const InCallView: FC<InCallViewProps> = ({
const footer = footerVm !== null && (
<CallFooter className={styles.footer} ref={footerRef} vm={footerVm} />
);
const allConnections = useBehavior(vm.allConnections$);
const connectedTransports = useBehavior(vm.connectedTransports$);
const localMember = useBehavior(vm.localMember$);
const localTransport = useBehavior(
localMember?.transport$ ?? constant(undefined),
);
return (
// The pointer handler here exists to control the visibility of the footer,
@@ -665,15 +707,7 @@ export const InCallView: FC<InCallViewProps> = ({
onPointerOut={onPointerOut}
>
{header}
{audioParticipants.map(({ livekitRoom, transport, participants }) => (
<LivekitRoomAudioRenderer
key={JSON.stringify(transport)}
transport={transport}
livekitRoom={livekitRoom}
validIdentities={participants}
muted={muteAllAudio}
/>
))}
<MemberAudioRenderer members={remoteMembers} muted={muteAllAudio} />
{renderContent()}
<CallEventAudioRenderer vm={vm} muted={muteAllAudio} />
<ReactionsAudioRenderer vm={vm} muted={muteAllAudio} />
@@ -694,22 +728,99 @@ export const InCallView: FC<InCallViewProps> = ({
tab={settingsTab}
onTabChange={setSettingsTab}
developerSettingsVm={developerSettingsVm}
livekitRooms={allConnections.map(
({ livekitRoom: room, livekitAlias, transport, role }) => ({
room,
livekitAlias,
role,
url:
// It should be relatively clear to the user which kind of URL
// this is, based on the scheme (ws:// vs. http://)
"url" in transport
? transport.url
: transport.livekit_service_url,
}),
)}
transports={connectedTransports.map((transport) => ({
transport,
local: transport === localTransport,
}))}
/>
</>
)}
</div>
);
};
/** The id of the device the user picked, for the client to capture or play on. */
export function selectedDeviceId$(
scope: ObservableScope,
device: MediaDevice<unknown, { id: string }>,
): Behavior<string | undefined> {
return scope.behavior(device.selected$.pipe(map((d) => d?.id)));
}
/**
* What the client needs from Element Call's configuration, settings and URL:
* the SDK reads none of them itself. Shared with the legacy SDK bundle so that
* a widget built from it behaves like the app.
*/
export function matrixRTCClientOptions(
urlParams: ReturnType<typeof useUrlParams>,
encryptionSystem: EncryptionSystem,
): MatrixRTCClientOptions {
const config = Config.get();
const session = config.matrix_rtc_session;
const cameraSettings = advancedCamera.getValue()
? {
resolution: {
...parseResolution(cameraResolution.getValue()),
frameRate: cameraFramerate.getValue(),
},
maxBitrate: cameraBitrate.getValue(),
maxFramerate: cameraFramerate.getValue(),
codec: cameraCodec.getValue(),
}
: undefined;
const screenConfig = config.media_quality?.screen_share;
const screenShareSettings = advancedScreenShare.getValue()
? {
resolution: {
...parseResolution(screenShareResolution.getValue()),
frameRate: screenShareFramerate.getValue(),
},
maxBitrate: screenShareBitrate.getValue(),
maxFramerate: screenShareFramerate.getValue(),
codec: screenShareCodec.getValue(),
}
: screenConfig?.max_resolution
? {
resolution: {
width: Math.round((screenConfig.max_resolution * 16) / 9),
height: screenConfig.max_resolution,
frameRate: screenConfig.max_framerate ?? 30,
},
}
: undefined;
return {
encryptionSystem,
// matrix_rtc_mode in config.json overrides the user's Developer Settings
// choice. We merely sample the current mode here, so the user would need
// to manually rejoin to switch to a different one.
matrixRTCMode:
(config.matrix_rtc_mode as MatrixRTCMode | undefined) ??
matrixRTCModeSetting.value$.value,
sendNotificationType: urlParams.sendNotificationType,
applicationData: urlParams.callIntent
? { "m.call.intent": urlParams.callIntent }
: undefined,
timings: {
syncDisconnectGracePeriodMs: config.sync_disconnect_grace_period_ms,
networkErrorRetryMs: session.network_error_retry_ms,
waitForKeyRotationMs: session.wait_for_key_rotation_ms,
membershipEventExpiryMs: session.membership_event_expiry_ms,
keyRotationParticipantLimit: session.key_rotation_participant_limit,
delayedLeave: session.delayed_leave,
delegatedDelayedLeave: session.delegated_delayed_leave,
},
mediaQuality: config.media_quality,
capture: {
audio: {
echoCancellation: echoCancellationSetting.getValue(),
noiseSuppression: noiseSuppressionSetting.getValue(),
autoGainControl: autoGainControlSetting.getValue(),
},
camera: cameraSettings,
screenShare: screenShareSettings,
},
transportUrl: customLivekitUrl.value$.value ?? undefined,
fallbackTransportUrl: config.livekit?.livekit_service_url,
};
}
+2 -2
View File
@@ -14,14 +14,14 @@ import { type MatrixClient, type RoomSummary } from "matrix-js-sdk";
import { KnockLobbyView } from "./KnockLobbyView";
import { LeaveToHomeProvider } from "../LeaveToHomeContext";
import { MediaDevicesContext } from "../MediaDevicesContext";
import { type ProcessorState } from "../livekit/TrackProcessorContext";
import { type ProcessorState } from "../tracks/TrackProcessorContext";
import { mockMediaDevices } from "../utils/test";
vi.mock("@livekit/components-react", () => ({
usePreviewTracks: (): unknown[] => [],
}));
vi.mock("../livekit/TrackProcessorContext", () => ({
vi.mock("../tracks/TrackProcessorContext", () => ({
useTrackProcessor: (): ProcessorState => ({
supported: false,
processor: undefined,
+2 -2
View File
@@ -28,7 +28,7 @@ import {
} from "../utils/test";
import { type MediaDevices } from "../state/MediaDevices";
import { MediaDevicesContext } from "../MediaDevicesContext";
import { type ProcessorState } from "../livekit/TrackProcessorContext";
import { type ProcessorState } from "../tracks/TrackProcessorContext";
import { type EncryptionSystem } from "../e2ee/sharedKeyManagement";
import lobbyStyles from "./LobbyView.module.css";
import headerStyles from "../Header.module.css";
@@ -41,7 +41,7 @@ vi.mock("@livekit/components-react", () => ({
usePreviewTracks: (): unknown[] => [],
}));
vi.mock("../livekit/TrackProcessorContext", () => ({
vi.mock("../tracks/TrackProcessorContext", () => ({
useTrackProcessor: (): ProcessorState => ({
supported: false,
processor: undefined,
+1 -1
View File
@@ -42,7 +42,7 @@ import { useInitial } from "../useInitial";
import {
useTrackProcessor,
useTrackProcessorSync,
} from "../livekit/TrackProcessorContext";
} from "../tracks/TrackProcessorContext";
import { getValue } from "../utils/observable";
import { useBehavior } from "../useBehavior";
import { CallFooter, type FooterSnapshot } from "../components/CallFooter";
-1
View File
@@ -48,7 +48,6 @@ function TestComponent({ vm }: { vm: CallViewModel }): ReactNode {
);
}
vitest.mock("livekit-client/e2ee-worker?worker");
vitest.mock("../useAudioContext");
vitest.mock("../soundUtils");
+1 -3
View File
@@ -6,7 +6,7 @@ Please see LICENSE in the repository root for full details.
*/
import { render } from "@testing-library/react";
import { expect, test, afterEach, vi } from "vitest";
import { expect, test, afterEach } from "vitest";
import { act } from "react";
import { showReactions } from "../settings/settings";
@@ -22,8 +22,6 @@ import { getBasicCallViewModelEnvironment } from "../utils/test-viewmodel";
import { initializeWidget } from "../widget";
initializeWidget();
vi.mock("livekit-client/e2ee-worker?worker");
afterEach(() => {
showReactions.setValue(showReactions.defaultValue);
});
@@ -84,6 +84,9 @@ exports[`InCallView > rendering > renders 1`] = `
class="_nav_e4b327 _rightNav_e4b327"
/>
</header>
<div>
mocked: MemberAudioRenderer
</div>
<div
class="_scrollingGrid_4e7ff8 _grid_b0d1cd"
>