Matrix/element-call-Github
1
0
Fork 0
mirror of https://github.com/vector-im/element-call.git synced 2026-01-30 03:15:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3521 from element-hq/valere/multi-sfu/connection_states

Browse Source 
Connection State: Error Handling + Cleaning / refactoring /
This commit is contained in:
Valere Fedronic
2025-10-13 10:15:55 +02:00
committed by GitHub
parent de5f5198dd fc2384e330
commit c846ea6447
31 changed files with 1765 additions and 719 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
package.json
View File

@@ -99,6 +99,7 @@
"eslint-plugin-react-hooks": "^5.0.0",
"eslint-plugin-rxjs": "^5.0.3",
"eslint-plugin-unicorn": "^56.0.0",
"fetch-mock": "11.1.5",
"global-jsdom": "^26.0.0",
"i18next": "^24.0.0",
"i18next-browser-languagedetector": "^8.0.0",

8
src/MediaDevicesContext.ts
View File

@@ -23,14 +23,6 @@ export function useMediaDevices(): MediaDevices {
return mediaDevices;
}
export const useIsEarpiece = (): boolean => {
const devices = useMediaDevices();
const audioOutput = useObservableEagerState(devices.audioOutput.selected$);
const available = useObservableEagerState(devices.audioOutput.available$);
if (!audioOutput?.id) return false;
return available.get(audioOutput.id)?.type === "earpiece";
};
/**
* A convenience hook to get the audio node configuration for the earpiece.
* It will check the `useAsEarpiece` of the `audioOutput` device and return

3
src/button/ReactionToggleButton.test.tsx
View File

@@ -10,7 +10,6 @@ import { expect, test } from "vitest";
import { TooltipProvider } from "@vector-im/compound-web";
import { userEvent } from "@testing-library/user-event";
import { type ReactNode } from "react";
import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc";
import { ReactionToggleButton } from "./ReactionToggleButton";
import { ElementCallReactionEventType } from "../reactions";
@@ -33,7 +32,7 @@ function TestComponent({
<TooltipProvider>
<ReactionsSenderProvider
vm={vm}
rtcSession={rtcSession as unknown as MatrixRTCSession}
rtcSession={rtcSession.asMockedSession()}
>
<ReactionToggleButton vm={vm} identifier={localIdent} />
</ReactionsSenderProvider>

224
src/livekit/MatrixAudioRenderer.test.tsx
View File

@@ -6,20 +6,28 @@ Please see LICENSE in the repository root for full details.
*/
import { afterEach, beforeEach, expect, it, vi } from "vitest";
import { render } from "@testing-library/react";
import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc";
import { render, type RenderResult } from "@testing-library/react";
import {
getTrackReferenceId,
type TrackReference,
} from "@livekit/components-core";
import { type RemoteAudioTrack } from "livekit-client";
import {
type Participant,
type RemoteAudioTrack,
type Room,
Track,
} from "livekit-client";
import { type ReactNode } from "react";
import { useTracks } from "@livekit/components-react";
import { testAudioContext } from "../useAudioContext.test";
import * as MediaDevicesContext from "../MediaDevicesContext";
import { LivekitRoomAudioRenderer } from "./MatrixAudioRenderer";
import { mockMediaDevices, mockTrack } from "../utils/test";
import {
mockMediaDevices,
mockRemoteParticipant,
mockTrack,
} from "../utils/test";
export const TestAudioContextConstructor = vi.fn(() => testAudioContext);
@@ -48,42 +56,203 @@ vi.mock("@livekit/components-react", async (importOriginal) => {
};
});
const tracks = [mockTrack("test:123")];
vi.mocked(useTracks).mockReturnValue(tracks);
let tracks: TrackReference[] = [];
it("should render for member", () => {
const { container, queryAllByTestId } = render(
/**
* Render the test component with given rtc members and livekit participant identities.
*
* It is possible to have rtc members that are not in livekit (e.g. not yet joined) and vice versa.
*
* @param rtcMembers - Array of active rtc members with userId and deviceId.
* @param livekitParticipantIdentities - Array of livekit participant (that are publishing).
* @param explicitTracks - Array of tracks available in livekit, if not provided, one audio track per livekitParticipantIdentities will be created.
* */
function renderTestComponent(
rtcMembers: { userId: string; deviceId: string }[],
livekitParticipantIdentities: string[],
explicitTracks?: {
participantId: string;
kind: Track.Kind;
source: Track.Source;
}[],
): RenderResult {
const liveKitParticipants = livekitParticipantIdentities.map((identity) =>
mockRemoteParticipant({ identity }),
);
const participants = rtcMembers.flatMap(({ userId, deviceId }) => {
const p = liveKitParticipants.find(
(p) => p.identity === `${userId}:${deviceId}`,
);
return p === undefined ? [] : [p];
});
const livekitRoom = {
remoteParticipants: new Map<string, Participant>(
liveKitParticipants.map((p) => [p.identity, p]),
),
} as unknown as Room;
if (explicitTracks?.length ?? 0 > 0) {
tracks = explicitTracks!.map(({ participantId, source, kind }) => {
const participant =
liveKitParticipants.find((p) => p.identity === participantId) ??
mockRemoteParticipant({ identity: participantId });
return mockTrack(participant, kind, source);
});
} else {
tracks = participants.map((p) => mockTrack(p));
}
vi.mocked(useTracks).mockReturnValue(tracks);
return render(
<MediaDevicesProvider value={mockMediaDevices({})}>
<LivekitRoomAudioRenderer
members={[{ sender: "test", deviceId: "123" }] as CallMembership[]}
validIdentities={participants.map((p) => p.identity)}
livekitRoom={livekitRoom}
url={""}
/>
</MediaDevicesProvider>,
);
}
it("should render for member", () => {
const { container, queryAllByTestId } = renderTestComponent(
[{ userId: "@alice", deviceId: "DEV0" }],
["@alice:DEV0"],
);
expect(container).toBeTruthy();
expect(queryAllByTestId("audio")).toHaveLength(1);
});
it("should not render without member", () => {
const memberships = [
{ sender: "othermember", deviceId: "123" },
] as CallMembership[];
const { container, queryAllByTestId } = render(
<MediaDevicesProvider value={mockMediaDevices({})}>
<LivekitRoomAudioRenderer members={memberships} />
</MediaDevicesProvider>,
const { container, queryAllByTestId } = renderTestComponent(
[{ userId: "@bob", deviceId: "DEV0" }],
["@alice:DEV0"],
);
expect(container).toBeTruthy();
expect(queryAllByTestId("audio")).toHaveLength(0);
});
const TEST_CASES: {
name: string;
rtcUsers: { userId: string; deviceId: string }[];
livekitParticipantIdentities: string[];
explicitTracks?: {
participantId: string;
kind: Track.Kind;
source: Track.Source;
}[];
expectedAudioTracks: number;
}[] = [
{
name: "single user single device",
rtcUsers: [
{ userId: "@alice", deviceId: "DEV0" },
{ userId: "@alice", deviceId: "DEV1" },
{ userId: "@bob", deviceId: "DEV0" },
],
livekitParticipantIdentities: ["@alice:DEV0", "@bob:DEV0", "@alice:DEV1"],
expectedAudioTracks: 3,
},
// Charlie is a rtc member but not in livekit
{
name: "Charlie is rtc member but not in livekit",
rtcUsers: [
{ userId: "@alice", deviceId: "DEV0" },
{ userId: "@bob", deviceId: "DEV0" },
{ userId: "@charlie", deviceId: "DEV0" },
],
livekitParticipantIdentities: ["@alice:DEV0", "@bob:DEV0"],
expectedAudioTracks: 2,
},
// Charlie is in livekit but not rtc member
{
name: "Charlie is in livekit but not rtc member",
rtcUsers: [
{ userId: "@alice", deviceId: "DEV0" },
{ userId: "@bob", deviceId: "DEV0" },
],
livekitParticipantIdentities: ["@alice:DEV0", "@bob:DEV0", "@charlie:DEV0"],
expectedAudioTracks: 2,
},
{
name: "no audio track, only video track",
rtcUsers: [{ userId: "@alice", deviceId: "DEV0" }],
livekitParticipantIdentities: ["@alice:DEV0"],
explicitTracks: [
{
participantId: "@alice:DEV0",
kind: Track.Kind.Video,
source: Track.Source.Camera,
},
],
expectedAudioTracks: 0,
},
{
name: "Audio track from unknown source",
rtcUsers: [{ userId: "@alice", deviceId: "DEV0" }],
livekitParticipantIdentities: ["@alice:DEV0"],
explicitTracks: [
{
participantId: "@alice:DEV0",
kind: Track.Kind.Audio,
source: Track.Source.Unknown,
},
],
expectedAudioTracks: 1,
},
{
name: "Audio track from other device",
rtcUsers: [{ userId: "@alice", deviceId: "DEV0" }],
livekitParticipantIdentities: ["@alice:DEV0"],
explicitTracks: [
{
participantId: "@alice:DEV1",
kind: Track.Kind.Audio,
source: Track.Source.Microphone,
},
],
expectedAudioTracks: 0,
},
{
name: "two audio tracks, microphone and screenshare",
rtcUsers: [{ userId: "@alice", deviceId: "DEV0" }],
livekitParticipantIdentities: ["@alice:DEV0"],
explicitTracks: [
{
participantId: "@alice:DEV0",
kind: Track.Kind.Audio,
source: Track.Source.Microphone,
},
{
participantId: "@alice:DEV0",
kind: Track.Kind.Audio,
source: Track.Source.ScreenShareAudio,
},
],
expectedAudioTracks: 2,
},
];
it.each(TEST_CASES)(
`should render sound test cases $name`,
({
rtcUsers,
livekitParticipantIdentities,
explicitTracks,
expectedAudioTracks,
}) => {
const { queryAllByTestId } = renderTestComponent(
rtcUsers,
livekitParticipantIdentities,
explicitTracks,
);
expect(queryAllByTestId("audio")).toHaveLength(expectedAudioTracks);
},
);
it("should not setup audioContext gain and pan if there is no need to.", () => {
render(
<MediaDevicesProvider value={mockMediaDevices({})}>
<LivekitRoomAudioRenderer
members={[{ sender: "test", deviceId: "123" }] as CallMembership[]}
/>
</MediaDevicesProvider>,
);
renderTestComponent([{ userId: "@bob", deviceId: "DEV0" }], ["@bob:DEV0"]);
const audioTrack = tracks[0].publication.track! as RemoteAudioTrack;
expect(audioTrack.setAudioContext).toHaveBeenCalledTimes(1);
@@ -100,13 +269,8 @@ it("should setup audioContext gain and pan", () => {
pan: 1,
volume: 0.1,
});
render(
<MediaDevicesProvider value={mockMediaDevices({})}>
<LivekitRoomAudioRenderer
members={[{ sender: "test", deviceId: "123" }] as CallMembership[]}
/>
</MediaDevicesProvider>,
);
renderTestComponent([{ userId: "@bob", deviceId: "DEV0" }], ["@bob:DEV0"]);
const audioTrack = tracks[0].publication.track! as RemoteAudioTrack;
expect(audioTrack.setAudioContext).toHaveBeenCalled();

86
src/livekit/MatrixAudioRenderer.tsx
View File

@@ -6,21 +6,21 @@ Please see LICENSE in the repository root for full details.
*/
import { getTrackReferenceId } from "@livekit/components-core";
import { type Room as LivekitRoom, type Participant } from "livekit-client";
import { type Room as LivekitRoom } from "livekit-client";
import { type RemoteAudioTrack, Track } from "livekit-client";
import { useEffect, useMemo, useRef, useState, type ReactNode } from "react";
import { useEffect, useMemo, useState, type ReactNode } from "react";
import {
useTracks,
AudioTrack,
type AudioTrackProps,
} from "@livekit/components-react";
import { type RoomMember } from "matrix-js-sdk";
import { logger } from "matrix-js-sdk/lib/logger";
import { type ParticipantId } from "matrix-js-sdk/lib/matrixrtc";
import { useEarpieceAudioConfig } from "../MediaDevicesContext";
import { useReactiveState } from "../useReactiveState";
import * as controls from "../controls";
import {} from "@livekit/components-core";
export interface MatrixAudioRendererProps {
/**
* The service URL of the LiveKit room.
@@ -28,14 +28,11 @@ export interface MatrixAudioRendererProps {
url: string;
livekitRoom: LivekitRoom;
/**
* The list of participants to render audio for.
* The list of participant identities to render audio for.
* This list needs to be composed based on the matrixRTC members so that we do not play audio from users
* that are not expected to be in the rtc session.
* that are not expected to be in the rtc session (local user is excluded).
*/
participants: {
participant: Participant;
member: RoomMember;
}[];
validIdentities: ParticipantId[];
/**
* If set to `true`, mutes all audio tracks rendered by the component.
* @remarks
@@ -44,9 +41,9 @@ export interface MatrixAudioRendererProps {
muted?: boolean;
}
const prefixedLogger = logger.getChild("[MatrixAudioRenderer]");
/**
* The `MatrixAudioRenderer` component is a drop-in solution for adding audio to your LiveKit app.
* It takes care of handling remote participants’ audio tracks and makes sure that microphones and screen share are audible.
* Takes care of handling remote participants’ audio tracks and makes sure that microphones and screen share are audible.
*
* It also takes care of the earpiece audio configuration for iOS devices.
* This is done by using the WebAudio API to create a stereo pan effect that mimics the earpiece audio.
@@ -61,33 +58,9 @@ export interface MatrixAudioRendererProps {
export function LivekitRoomAudioRenderer({
url,
livekitRoom,
participants,
validIdentities,
muted,
}: MatrixAudioRendererProps): ReactNode {
const participantSet = useMemo(
() => new Set(participants.map(({ participant }) => participant)),
[participants],
);
const loggedInvalidIdentities = useRef(new Set<string>());
/**
* Log an invalid livekit track identity.
* A invalid identity is one that does not match any of the matrix rtc members.
*
* @param identity The identity of the track that is invalid
* @param validIdentities The list of valid identities
*/
const logInvalid = (identity: string): void => {
if (loggedInvalidIdentities.current.has(identity)) return;
logger.warn(
`[MatrixAudioRenderer] Audio track ${identity} from ${url} has no matching matrix call member`,
`current members: ${participants.map((p) => p.participant.identity)}`,
`track will not get rendered`,
);
loggedInvalidIdentities.current.add(identity);
};
const tracks = useTracks(
[
Track.Source.Microphone,
@@ -99,28 +72,23 @@ export function LivekitRoomAudioRenderer({
onlySubscribed: true,
room: livekitRoom,
},
).filter((ref) => {
const isValid = participantSet?.has(ref.participant);
if (!isValid && !ref.participant.isLocal)
logInvalid(ref.participant.identity);
return (
!ref.participant.isLocal &&
ref.publication.kind === Track.Kind.Audio &&
isValid
);
});
useEffect(() => {
if (
loggedInvalidIdentities.current.size &&
tracks.every((t) => participantSet.has(t.participant))
) {
logger.debug(
`[MatrixAudioRenderer] All audio tracks from ${url} have a matching matrix call member identity.`,
);
loggedInvalidIdentities.current.clear();
}
}, [tracks, participantSet, url]);
)
// Only keep audio tracks
.filter((ref) => ref.publication.kind === Track.Kind.Audio)
// Only keep tracks from participants that are in the validIdentities list
.filter((ref) => {
const isValid = validIdentities.includes(ref.participant.identity);
if (!isValid) {
// Log that there is an invalid identity, that means that someone is publishing audio that is not expected to be in the call.
prefixedLogger.warn(
`Audio track ${ref.participant.identity} from ${url} has no matching matrix call member`,
`current members: ${validIdentities.join()}`,
`track will not get rendered`,
);
return false;
}
return true;
});
// This component is also (in addition to the "only play audio for connected members" logic above)
// responsible for mimicking earpiece audio on iPhones.

6
src/main.tsx
View File

@@ -60,9 +60,9 @@ if (fatalError !== null) {
Initializer.initBeforeReact()
.then(() => {
root.render(
// <StrictMode>
<App vm={new AppViewModel()} />,
// </StrictMode>,
<StrictMode>
<App vm={new AppViewModel()} />,
</StrictMode>,
);
})
.catch((e) => {

21
src/reactions/ReactionsReader.test.tsx
View File

@@ -7,7 +7,6 @@ Please see LICENSE in the repository root for full details.
import { renderHook } from "@testing-library/react";
import { afterEach, test, vitest } from "vitest";
import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc";
import {
RoomEvent as MatrixRoomEvent,
MatrixEvent,
@@ -38,7 +37,7 @@ test("handles a hand raised reaction", () => {
withTestScheduler(({ schedule, expectObservable }) => {
renderHook(() => {
const { raisedHands$ } = new ReactionsReader(
rtcSession as unknown as MatrixRTCSession,
rtcSession.asMockedSession(),
);
schedule("ab", {
a: () => {},
@@ -86,7 +85,7 @@ test("handles a redaction", () => {
withTestScheduler(({ schedule, expectObservable }) => {
renderHook(() => {
const { raisedHands$ } = new ReactionsReader(
rtcSession as unknown as MatrixRTCSession,
rtcSession.asMockedSession(),
);
schedule("abc", {
a: () => {},
@@ -149,7 +148,7 @@ test("handles waiting for event decryption", () => {
withTestScheduler(({ schedule, expectObservable }) => {
renderHook(() => {
const { raisedHands$ } = new ReactionsReader(
rtcSession as unknown as MatrixRTCSession,
rtcSession.asMockedSession(),
);
schedule("abc", {
a: () => {},
@@ -218,7 +217,7 @@ test("hands rejecting events without a proper membership", () => {
withTestScheduler(({ schedule, expectObservable }) => {
renderHook(() => {
const { raisedHands$ } = new ReactionsReader(
rtcSession as unknown as MatrixRTCSession,
rtcSession.asMockedSession(),
);
schedule("ab", {
a: () => {},
@@ -262,9 +261,7 @@ test("handles a reaction", () => {
withTestScheduler(({ schedule, time, expectObservable }) => {
renderHook(() => {
const { reactions$ } = new ReactionsReader(
rtcSession as unknown as MatrixRTCSession,
);
const { reactions$ } = new ReactionsReader(rtcSession.asMockedSession());
schedule(`abc`, {
a: () => {},
b: () => {
@@ -320,9 +317,7 @@ test("ignores bad reaction events", () => {
withTestScheduler(({ schedule, expectObservable }) => {
renderHook(() => {
const { reactions$ } = new ReactionsReader(
rtcSession as unknown as MatrixRTCSession,
);
const { reactions$ } = new ReactionsReader(rtcSession.asMockedSession());
schedule("ab", {
a: () => {},
b: () => {
@@ -444,9 +439,7 @@ test("that reactions cannot be spammed", () => {
withTestScheduler(({ schedule, expectObservable }) => {
renderHook(() => {
const { reactions$ } = new ReactionsReader(
rtcSession as unknown as MatrixRTCSession,
);
const { reactions$ } = new ReactionsReader(rtcSession.asMockedSession());
schedule("abcd", {
a: () => {},
b: () => {

3
src/room/CallEventAudioRenderer.test.tsx
View File

@@ -155,7 +155,8 @@ test("plays one sound when a hand is raised", () => {
act(() => {
handRaisedSubject$.next({
[bobRtcMember.callId]: {
// TODO: What is this string supposed to be?
[`${bobRtcMember.sender}:${bobRtcMember.deviceId}`]: {
time: new Date(),
membershipEventId: "",
reactionEventId: "",

2
src/room/GroupCallErrorBoundary.test.tsx
View File

@@ -106,7 +106,7 @@ test("should render the error page with link back to home", async () => {
await screen.findByText("Call is not supported");
expect(screen.getByText(/Domain: example\.com/i)).toBeInTheDocument();
expect(
screen.getByText(/Error Code: MISSING_MATRIX_RTC_FOCUS/i),
screen.getByText(/Error Code: MISSING_MATRIX_RTC_TRANSPORT/i),
).toBeInTheDocument();
await screen.findByRole("button", { name: "Return to home screen" });

15
src/room/GroupCallView.test.tsx
View File

@@ -26,7 +26,6 @@ import { type RelationsContainer } from "matrix-js-sdk/lib/models/relations-cont
import { useState } from "react";
import { TooltipProvider } from "@vector-im/compound-web";
import { type MuteStates } from "./MuteStates";
import { prefetchSounds } from "../soundUtils";
import { useAudioContext } from "../useAudioContext";
import { ActiveCall } from "./InCallView";
@@ -47,6 +46,7 @@ import { ProcessorProvider } from "../livekit/TrackProcessorContext";
import { MediaDevicesContext } from "../MediaDevicesContext";
import { HeaderStyle } from "../UrlParams";
import { constant } from "../state/Behavior";
import { type MuteStates } from "../state/MuteStates.ts";
vi.mock("../soundUtils");
vi.mock("../useAudioContext");
@@ -117,7 +117,7 @@ function createGroupCallView(
widget: WidgetHelpers | null,
joined = true,
): {
rtcSession: MockRTCSession;
rtcSession: MatrixRTCSession;
getByText: ReturnType<typeof render>["getByText"];
} {
const client = {
@@ -150,7 +150,8 @@ function createGroupCallView(
const muteState = {
audio: { enabled: false },
video: { enabled: false },
} as MuteStates;
// TODO-MULTI-SFU: This cast isn't valid, it's likely the cause of some current test failures
} as unknown as MuteStates;
const { getByText } = render(
<BrowserRouter>
<TooltipProvider>
@@ -163,10 +164,12 @@ function createGroupCallView(
preload={false}
skipLobby={false}
header={HeaderStyle.Standard}
rtcSession={rtcSession as unknown as MatrixRTCSession}
isJoined={joined}
rtcSession={rtcSession.asMockedSession()}
muteStates={muteState}
widget={widget}
// TODO-MULTI-SFU: Make joined and setJoined work
joined={true}
setJoined={function (value: boolean): void {}}
/>
</ProcessorProvider>
</MediaDevicesContext>
@@ -175,7 +178,7 @@ function createGroupCallView(
);
return {
getByText,
rtcSession,
rtcSession: rtcSession.asMockedSession(),
};
}

17
src/room/InCallView.test.tsx
View File

@@ -15,7 +15,6 @@ import {
} from "vitest";
import { act, render, type RenderResult } from "@testing-library/react";
import { type MatrixClient, JoinRule, type RoomState } from "matrix-js-sdk";
import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc";
import { type RelationsContainer } from "matrix-js-sdk/lib/models/relations-container";
import { type LocalParticipant } from "livekit-client";
import { of } from "rxjs";
@@ -24,7 +23,6 @@ import { TooltipProvider } from "@vector-im/compound-web";
import { RoomContext, useLocalParticipant } from "@livekit/components-react";
import { RoomAndToDeviceEvents } from "matrix-js-sdk/lib/matrixrtc/RoomAndToDeviceKeyTransport";
import { type MuteStates } from "./MuteStates";
import { InCallView } from "./InCallView";
import {
mockLivekitRoom,
@@ -32,6 +30,7 @@ import {
mockMatrixRoom,
mockMatrixRoomMember,
mockMediaDevices,
mockMuteStates,
mockRemoteParticipant,
mockRtcMembership,
type MockRTCSession,
@@ -133,10 +132,7 @@ function createInCallView(): RenderResult & {
} as Partial<RoomState> as RoomState,
});
const muteState = {
audio: { enabled: false },
video: { enabled: false },
} as MuteStates;
const muteState = mockMuteStates();
const livekitRoom = mockLivekitRoom(
{
localParticipant,
@@ -153,14 +149,14 @@ function createInCallView(): RenderResult & {
<MediaDevicesContext value={mockMediaDevices({})}>
<ReactionsSenderProvider
vm={vm}
rtcSession={rtcSession as unknown as MatrixRTCSession}
rtcSession={rtcSession.asMockedSession()}
>
<TooltipProvider>
<RoomContext value={livekitRoom}>
<InCallView
client={client}
header={HeaderStyle.Standard}
rtcSession={rtcSession as unknown as MatrixRTCSession}
rtcSession={rtcSession.asMockedSession()}
muteStates={muteState}
vm={vm}
matrixInfo={{
@@ -176,11 +172,6 @@ function createInCallView(): RenderResult & {
},
}}
matrixRoom={room}
livekitRoom={livekitRoom}
participantCount={0}
onLeft={function (): void {
throw new Error("Function not implemented.");
}}
onShareClick={null}
/>
</RoomContext>

13
src/room/InCallView.tsx
View File

@@ -23,7 +23,7 @@ import useMeasure from "react-use-measure";
import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc";
import classNames from "classnames";
import { BehaviorSubject, map } from "rxjs";
import { useObservable, useObservableEagerState } from "observable-hooks";
import { useObservable } from "observable-hooks";
import { logger } from "matrix-js-sdk/lib/logger";
import { RoomAndToDeviceEvents } from "matrix-js-sdk/lib/matrixrtc/RoomAndToDeviceKeyTransport";
import {
@@ -31,7 +31,6 @@ import {
VolumeOnSolidIcon,
} from "@vector-im/compound-design-tokens/assets/web/icons";
import { useTranslation } from "react-i18next";
import { ConnectionState } from "livekit-client";
import LogoMark from "../icons/LogoMark.svg?react";
import LogoType from "../icons/LogoType.svg?react";
@@ -113,7 +112,6 @@ import { prefetchSounds } from "../soundUtils";
import { useAudioContext } from "../useAudioContext";
import ringtoneMp3 from "../sound/ringtone.mp3?url";
import ringtoneOgg from "../sound/ringtone.ogg?url";
import { ConnectionLostError } from "../utils/errors.ts";
import { useTrackProcessorObservable$ } from "../livekit/TrackProcessorContext.tsx";
const maxTapDurationMs = 400;
@@ -207,7 +205,8 @@ export const InCallView: FC<InCallViewProps> = ({
useReactionsSender();
useWakeLock();
const connectionState = useObservableEagerState(vm.livekitConnectionState$);
// TODO-MULTI-SFU This is unused now??
// const connectionState = useObservableEagerState(vm.livekitConnectionState$);
// annoyingly we don't get the disconnection reason this way,
// only by listening for the emitted event
@@ -287,7 +286,7 @@ export const InCallView: FC<InCallViewProps> = ({
);
const allLivekitRooms = useBehavior(vm.allLivekitRooms$);
const participantsByRoom = useBehavior(vm.participantsByRoom$);
const audioParticipants = useBehavior(vm.audioParticipants$);
const participantCount = useBehavior(vm.participantCount$);
const reconnecting = useBehavior(vm.reconnecting$);
const windowMode = useBehavior(vm.windowMode$);
@@ -861,12 +860,12 @@ export const InCallView: FC<InCallViewProps> = ({
</Text>
)
}
{participantsByRoom.map(({ livekitRoom, url, participants }) => (
{audioParticipants.map(({ livekitRoom, url, participants }) => (
<LivekitRoomAudioRenderer
key={url}
url={url}
livekitRoom={livekitRoom}
participants={participants}
validIdentities={participants.map((p) => p.identity)}
muted={muteAllAudio}
/>
))}

4
src/room/MuteStates.test.tsx
View File

@@ -5,6 +5,9 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
// TODO-MULTI-SFU: These tests need to be ported to the new MuteStates class.
/*
import {
afterAll,
afterEach,
@@ -321,3 +324,4 @@ describe("useMuteStates in VITE_PACKAGE='embedded' (widget) mode", () => {
expect(screen.getByTestId("video-enabled").textContent).toBe("true");
});
});
*/

178
src/room/MuteStates.ts
View File

@@ -1,178 +0,0 @@
/*
Copyright 2023, 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
type Dispatch,
type SetStateAction,
useCallback,
useEffect,
useMemo,
} from "react";
import { type IWidgetApiRequest } from "matrix-widget-api";
import { logger } from "matrix-js-sdk/lib/logger";
import { useObservableEagerState } from "observable-hooks";
import {
type DeviceLabel,
type SelectedDevice,
type MediaDevice,
} from "../state/MediaDevices";
import { useIsEarpiece, useMediaDevices } from "../MediaDevicesContext";
import { useReactiveState } from "../useReactiveState";
import { ElementWidgetActions, widget } from "../widget";
import { Config } from "../config/Config";
import { useUrlParams } from "../UrlParams";
/**
* If there already are this many participants in the call, we automatically mute
* the user.
*/
export const MUTE_PARTICIPANT_COUNT = 8;
interface DeviceAvailable {
enabled: boolean;
setEnabled: Dispatch<SetStateAction<boolean>>;
}
interface DeviceUnavailable {
enabled: false;
setEnabled: null;
}
const deviceUnavailable: DeviceUnavailable = {
enabled: false,
setEnabled: null,
};
type MuteState = DeviceAvailable | DeviceUnavailable;
export interface MuteStates {
audio: MuteState;
video: MuteState;
}
function useMuteState(
device: MediaDevice<DeviceLabel, SelectedDevice>,
enabledByDefault: () => boolean,
forceUnavailable: boolean = false,
): MuteState {
const available = useObservableEagerState(device.available$);
const [enabled, setEnabled] = useReactiveState<boolean | undefined>(
// Determine the default value once devices are actually connected
(prev) => prev ?? (available.size > 0 ? enabledByDefault() : undefined),
[available.size],
);
return useMemo(
() =>
available.size === 0 || forceUnavailable
? deviceUnavailable
: {
enabled: enabled ?? false,
setEnabled: setEnabled as Dispatch<SetStateAction<boolean>>,
},
[available.size, enabled, forceUnavailable, setEnabled],
);
}
export function useMuteStates(isJoined: boolean): MuteStates {
const devices = useMediaDevices();
const { skipLobby, defaultAudioEnabled, defaultVideoEnabled } =
useUrlParams();
const audio = useMuteState(
devices.audioInput,
() =>
(defaultAudioEnabled ?? Config.get().media_devices.enable_audio) &&
allowJoinUnmuted(skipLobby, isJoined),
);
useEffect(() => {
// If audio is enabled, we need to request the device names again,
// because iOS will not be able to switch to the correct device after un-muting.
// This is one of the main changes that makes iOS work with bluetooth audio devices.
if (audio.enabled) {
devices.requestDeviceNames();
}
}, [audio.enabled, devices]);
const isEarpiece = useIsEarpiece();
const video = useMuteState(
devices.videoInput,
() =>
(defaultVideoEnabled ?? Config.get().media_devices.enable_video) &&
allowJoinUnmuted(skipLobby, isJoined),
isEarpiece, // Force video to be unavailable if using earpiece
);
useEffect(() => {
widget?.api.transport
.send(ElementWidgetActions.DeviceMute, {
audio_enabled: audio.enabled,
video_enabled: video.enabled,
})
.catch((e) =>
logger.warn("Could not send DeviceMute action to widget", e),
);
}, [audio, video]);
const onMuteStateChangeRequest = useCallback(
(ev: CustomEvent<IWidgetApiRequest>) => {
// First copy the current state into our new state.
const newState = {
audio_enabled: audio.enabled,
video_enabled: video.enabled,
};
// Update new state if there are any requested changes from the widget action
// in `ev.detail.data`.
if (
ev.detail.data.audio_enabled != null &&
typeof ev.detail.data.audio_enabled === "boolean"
) {
audio.setEnabled?.(ev.detail.data.audio_enabled);
newState.audio_enabled = ev.detail.data.audio_enabled;
}
if (
ev.detail.data.video_enabled != null &&
typeof ev.detail.data.video_enabled === "boolean"
) {
video.setEnabled?.(ev.detail.data.video_enabled);
newState.video_enabled = ev.detail.data.video_enabled;
}
// Always reply with the new (now "current") state.
// This allows to also use this action to just get the unaltered current state
// by using a fromWidget request with: `ev.detail.data = {}`
widget!.api.transport.reply(ev.detail, newState);
},
[audio, video],
);
useEffect(() => {
// We setup a event listener for the widget action ElementWidgetActions.DeviceMute.
if (widget) {
// only setup the listener in widget mode
widget.lazyActions.on(
ElementWidgetActions.DeviceMute,
onMuteStateChangeRequest,
);
return (): void => {
// return a call to `off` so that we always clean up our listener.
widget?.lazyActions.off(
ElementWidgetActions.DeviceMute,
onMuteStateChangeRequest,
);
};
}
}, [onMuteStateChangeRequest]);
return useMemo(() => ({ audio, video }), [audio, video]);
}
function allowJoinUnmuted(skipLobby: boolean, isJoined: boolean): boolean {
return (
(!skipLobby && !isJoined) || import.meta.env.VITE_PACKAGE === "embedded"
);
}

7
src/room/VideoPreview.test.tsx
View File

@@ -5,12 +5,11 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { expect, describe, it, vi, beforeAll } from "vitest";
import { expect, describe, it, beforeAll } from "vitest";
import { render } from "@testing-library/react";
import { type MatrixInfo, VideoPreview } from "./VideoPreview";
import { E2eeType } from "../e2ee/e2eeType";
import { mockMuteStates } from "../utils/test";
describe("VideoPreview", () => {
const matrixInfo: MatrixInfo = {
@@ -42,7 +41,7 @@ describe("VideoPreview", () => {
const { queryByRole } = render(
<VideoPreview
matrixInfo={matrixInfo}
muteStates={mockMuteStates()}
videoEnabled={false}
videoTrack={null}
children={<></>}
/>,
@@ -54,7 +53,7 @@ describe("VideoPreview", () => {
const { queryByRole } = render(
<VideoPreview
matrixInfo={matrixInfo}
muteStates={mockMuteStates()}
videoEnabled
videoTrack={null}
children={<></>}
/>,

6
src/room/__snapshots__/GroupCallErrorBoundary.test.tsx.snap
View File

@@ -292,7 +292,7 @@ exports[`should have a close button in widget mode 1`] = `
Call is not supported
</h1>
<p>
The server is not configured to work with Element Call. Please contact your server admin (Domain: example.com, Error Code: MISSING_MATRIX_RTC_FOCUS).
The server is not configured to work with Element Call. Please contact your server admin (Domain: example.com, Error Code: MISSING_MATRIX_RTC_TRANSPORT).
</p>
<button
class="_button_vczzf_8"
@@ -445,7 +445,7 @@ exports[`should render the error page with link back to home 1`] = `
Call is not supported
</h1>
<p>
The server is not configured to work with Element Call. Please contact your server admin (Domain: example.com, Error Code: MISSING_MATRIX_RTC_FOCUS).
The server is not configured to work with Element Call. Please contact your server admin (Domain: example.com, Error Code: MISSING_MATRIX_RTC_TRANSPORT).
</p>
<button
class="_button_vczzf_8 homeLink"
@@ -598,7 +598,7 @@ exports[`should report correct error for 'Call is not supported' 1`] = `
Call is not supported
</h1>
<p>
The server is not configured to work with Element Call. Please contact your server admin (Domain: example.com, Error Code: MISSING_MATRIX_RTC_FOCUS).
The server is not configured to work with Element Call. Please contact your server admin (Domain: example.com, Error Code: MISSING_MATRIX_RTC_TRANSPORT).
</p>
<button
class="_button_vczzf_8 homeLink"

35
src/rtcSessionHelpers.test.ts
View File

@@ -85,7 +85,16 @@ test("It joins the correct Session", async () => {
}),
joinRoomSession: vi.fn(),
}) as unknown as MatrixRTCSession;
await enterRTCSession(mockedSession, false);
await enterRTCSession(
mockedSession,
{
livekit_alias: "roomId",
livekit_service_url: "http://my-well-known-service-url.com",
type: "livekit",
},
true,
);
expect(mockedSession.joinRoomSession).toHaveBeenLastCalledWith(
[
@@ -180,8 +189,18 @@ test("It fails with configuration error if no live kit url config is set in fall
joinRoomSession: vi.fn(),
}) as unknown as MatrixRTCSession;
await expect(enterRTCSession(mockedSession, false)).rejects.toThrowError(
expect.objectContaining({ code: ErrorCode.MISSING_MATRIX_RTC_FOCUS }),
await expect(
enterRTCSession(
mockedSession,
{
livekit_alias: "roomId",
livekit_service_url: "http://my-well-known-service-url.com",
type: "livekit",
},
true,
),
).rejects.toThrowError(
expect.objectContaining({ code: ErrorCode.MISSING_MATRIX_RTC_TRANSPORT }),
);
});
@@ -214,5 +233,13 @@ test("It should not fail with configuration error if homeserver config has livek
joinRoomSession: vi.fn(),
}) as unknown as MatrixRTCSession;
await enterRTCSession(mockedSession, false);
await enterRTCSession(
mockedSession,
{
livekit_alias: "roomId",
livekit_service_url: "http://my-well-known-service-url.com",
type: "livekit",
},
true,
);
});

31
src/state/Async.ts
View File

@@ -5,16 +5,14 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
catchError,
from,
map,
Observable,
of,
startWith,
switchMap,
} from "rxjs";
import { catchError, from, map, type Observable, of, startWith } from "rxjs";
/**
* Data that may need to be loaded asynchronously.
*
* This type is for when you need to represent the current state of an operation
* involving Promises as **immutable data**. See the async$ function below.
*/
export type Async<A> =
| { state: "loading" }
| { state: "error"; value: Error }
@@ -24,18 +22,29 @@ export const loading: Async<never> = { state: "loading" };
export function error(value: Error): Async<never> {
return { state: "error", value };
}
export function ready<A>(value: A): Async<A> {
return { state: "ready", value };
}
export function async<A>(promise: Promise<A>): Observable<Async<A>> {
/**
* Turn a Promise into an Observable async value. The Observable will have the
* value "loading" while the Promise is pending, "ready" when the Promise
* resolves, and "error" when the Promise rejects.
*/
export function async$<A>(promise: Promise<A>): Observable<Async<A>> {
return from(promise).pipe(
map(ready),
startWith(loading),
catchError((e) => of(error(e))),
catchError((e: unknown) =>
of(error((e as Error) ?? new Error("Unknown error"))),
),
);
}
/**
* If the async value is ready, apply the given function to the inner value.
*/
export function mapAsync<A, B>(
async: Async<A>,
project: (value: A) => B,

13
src/state/CallViewModel.test.ts
View File

@@ -40,7 +40,6 @@ import * as ComponentsCore from "@livekit/components-core";
import {
Status,
type CallMembership,
type MatrixRTCSession,
type IRTCNotificationContent,
type ICallNotifyContent,
MatrixRTCSessionEvent,
@@ -68,7 +67,7 @@ import {
type ECConnectionState,
} from "../livekit/useECConnectionState";
import { E2eeType } from "../e2ee/e2eeType";
import type { RaisedHandInfo } from "../reactions";
import type { RaisedHandInfo, ReactionInfo } from "../reactions";
import {
alice,
aliceDoppelganger,
@@ -95,6 +94,7 @@ import { ObservableScope } from "./ObservableScope";
import { MediaDevices } from "./MediaDevices";
import { getValue } from "../utils/observable";
import { type Behavior, constant } from "./Behavior";
import type { ProcessorState } from "../livekit/TrackProcessorContext.tsx";
const getUrlParams = vi.hoisted(() => vi.fn(() => ({})));
vi.mock("../UrlParams", () => ({ getUrlParams }));
@@ -341,15 +341,20 @@ function withCallViewModel(
.mockImplementation((_room, _eventType) => of());
const muteStates = mockMuteStates();
const raisedHands$ = new BehaviorSubject<Record<string, RaisedHandInfo>>({});
const reactions$ = new BehaviorSubject<Record<string, ReactionInfo>>({});
const vm = new CallViewModel(
rtcSession as unknown as MatrixRTCSession,
rtcSession.asMockedSession(),
room,
mediaDevices,
muteStates,
options,
raisedHands$,
new BehaviorSubject({}),
reactions$,
new BehaviorSubject<ProcessorState>({
processor: undefined,
supported: undefined,
}),
);
onTestFinished(() => {

167
src/state/CallViewModel.ts
View File

@@ -14,7 +14,7 @@ import {
type Room as LivekitRoom,
type LocalParticipant,
ParticipantEvent,
type RemoteParticipant,
RemoteParticipant,
type Participant,
} from "livekit-client";
import E2EEWorker from "livekit-client/e2ee-worker?worker";
@@ -126,12 +126,17 @@ import {
} from "../rtcSessionHelpers";
import { E2eeType } from "../e2ee/e2eeType";
import { MatrixKeyProvider } from "../e2ee/matrixKeyProvider";
import { Connection, PublishConnection } from "./Connection";
import {
type Connection,
type ConnectionOpts,
RemoteConnection,
} from "./Connection";
import { type MuteStates } from "./MuteStates";
import { getUrlParams } from "../UrlParams";
import { type ProcessorState } from "../livekit/TrackProcessorContext";
import { ElementWidgetActions, widget } from "../widget";
import { type Async, async, mapAsync, ready } from "./Async";
import { PublishConnection } from "./PublishConnection.ts";
import { type Async, async$, mapAsync, ready } from "./Async";
export interface CallViewModelOptions {
encryptionSystem: EncryptionSystem;
@@ -518,7 +523,7 @@ export class CallViewModel extends ViewModel {
joined
? combineLatest(
[
async(this.preferredTransport),
async$(this.preferredTransport),
this.memberships$,
multiSfu.value$,
],
@@ -538,7 +543,9 @@ export class CallViewModel extends ViewModel {
const oldest = this.matrixRTCSession.getOldestMembership();
if (oldest !== undefined) {
const selection = oldest.getTransport(oldest);
if (isLivekitTransport(selection)) local = ready(selection);
// TODO selection can be null if no transport is configured should we report an error?
if (selection && isLivekitTransport(selection))
local = ready(selection);
}
}
return { local, remote };
@@ -559,49 +566,58 @@ export class CallViewModel extends ViewModel {
/**
* The transport over which we should be actively publishing our media.
* null when not joined.
*/
private readonly localTransport$: Behavior<Async<LivekitTransport> | null> =
this.scope.behavior(
this.transports$.pipe(
map((transports) => transports?.local ?? null),
distinctUntilChanged(deepCompare),
distinctUntilChanged<Async<LivekitTransport> | null>(deepCompare),
),
);
private readonly localConnectionAndTransport$ = this.scope.behavior(
this.localTransport$.pipe(
map(
(transport) =>
transport &&
mapAsync(transport, (transport) => ({
connection: new PublishConnection(
transport,
this.livekitAlias,
this.matrixRTCSession.room.client,
this.scope,
this.remoteTransports$,
this.mediaDevices,
this.muteStates,
this.e2eeLivekitOptions(),
this.scope.behavior(this.trackProcessorState$),
),
transport,
})),
/**
* The local connection over which we will publish our media. It could
* possibly also have some remote users' media available on it.
* null when not joined.
*/
private readonly localConnection$: Behavior<Async<PublishConnection> | null> =
this.scope.behavior(
this.localTransport$.pipe(
map(
(transport) =>
transport &&
mapAsync(transport, (transport) => {
const opts: ConnectionOpts = {
transport,
client: this.matrixRTCSession.room.client,
scope: this.scope,
remoteTransports$: this.remoteTransports$,
};
return new PublishConnection(
opts,
this.mediaDevices,
this.muteStates,
this.e2eeLivekitOptions(),
this.scope.behavior(this.trackProcessorState$),
);
}),
),
),
),
);
private readonly localConnection$ = this.scope.behavior(
this.localConnectionAndTransport$.pipe(
map((value) => value && mapAsync(value, ({ connection }) => connection)),
),
);
);
public readonly livekitConnectionState$ = this.scope.behavior(
this.localConnection$.pipe(
switchMap((c) =>
c?.state === "ready"
? c.value.connectionState$
? // TODO mapping to ConnectionState for compatibility, but we should use the full state?
c.value.focusConnectionState$.pipe(
map((s) => {
if (s.state === "ConnectedToLkRoom") return s.connectionState;
return ConnectionState.Disconnected;
}),
distinctUntilChanged(),
)
: of(ConnectionState.Disconnected),
),
),
@@ -639,16 +655,19 @@ export class CallViewModel extends ViewModel {
"SFU remoteConnections$ construct new connection: ",
remoteServiceUrl,
);
nextConnection = new Connection(
{
const args: ConnectionOpts = {
transport: {
type: "livekit",
livekit_service_url: remoteServiceUrl,
livekit_alias: this.livekitAlias,
type: "livekit",
},
this.livekitAlias,
this.matrixRTCSession.room.client,
this.scope,
this.remoteTransports$,
client: this.matrixRTCSession.room.client,
scope: this.scope,
remoteTransports$: this.remoteTransports$,
};
nextConnection = new RemoteConnection(
args,
this.e2eeLivekitOptions(),
);
} else {
@@ -700,15 +719,15 @@ export class CallViewModel extends ViewModel {
map((connections) =>
[...connections.values()].map((c) => ({
room: c.livekitRoom,
url: c.transport.livekit_service_url,
url: c.localTransport.livekit_service_url,
isLocal: c instanceof PublishConnection,
})),
),
),
);
private readonly userId = this.matrixRoom.client.getUserId();
private readonly deviceId = this.matrixRoom.client.getDeviceId();
private readonly userId = this.matrixRoom.client.getUserId()!;
private readonly deviceId = this.matrixRoom.client.getDeviceId()!;
private readonly matrixConnected$ = this.scope.behavior(
// To consider ourselves connected to MatrixRTC, we check the following:
@@ -785,7 +804,7 @@ export class CallViewModel extends ViewModel {
* Lists, for each LiveKit room, the LiveKit participants whose media should
* be presented.
*/
public readonly participantsByRoom$ = this.scope.behavior<
private readonly participantsByRoom$ = this.scope.behavior<
{
livekitRoom: LivekitRoom;
url: string;
@@ -797,17 +816,16 @@ export class CallViewModel extends ViewModel {
}[]
>(
// TODO: Move this logic into Connection/PublishConnection if possible
this.localConnectionAndTransport$
this.localConnection$
.pipe(
switchMap((values) => {
if (values?.state !== "ready") return [];
const localConnection = values.value.connection;
switchMap((localConnection) => {
if (localConnection?.state !== "ready") return [];
const memberError = (): never => {
throw new Error("No room member for call membership");
};
const localParticipant = {
id: "local",
participant: localConnection.livekitRoom.localParticipant,
participant: localConnection.value.livekitRoom.localParticipant,
member:
this.matrixRoom.getMember(this.userId ?? "") ?? memberError(),
};
@@ -815,7 +833,7 @@ export class CallViewModel extends ViewModel {
return this.remoteConnections$.pipe(
switchMap((remoteConnections) =>
combineLatest(
[localConnection, ...remoteConnections].map((c) =>
[localConnection.value, ...remoteConnections].map((c) =>
c.publishingParticipants$.pipe(
map((ps) => {
const participants: {
@@ -834,12 +852,12 @@ export class CallViewModel extends ViewModel {
this.matrixRoom,
)?.member ?? memberError(),
}));
if (c === localConnection)
if (c === localConnection.value)
participants.push(localParticipant);
return {
livekitRoom: c.livekitRoom,
url: c.transport.livekit_service_url,
url: c.localTransport.livekit_service_url,
participants,
};
}),
@@ -853,6 +871,25 @@ export class CallViewModel extends ViewModel {
.pipe(startWith([]), pauseWhen(this.pretendToBeDisconnected$)),
);
/**
* Lists, for each LiveKit room, the LiveKit participants whose audio should
* be rendered.
*/
// (This is effectively just participantsByRoom$ with a stricter type)
public readonly audioParticipants$ = this.scope.behavior(
this.participantsByRoom$.pipe(
map((data) =>
data.map(({ livekitRoom, url, participants }) => ({
livekitRoom,
url,
participants: participants.flatMap(({ participant }) =>
participant instanceof RemoteParticipant ? [participant] : [],
),
})),
),
),
);
/**
* Displaynames for each member of the call. This will disambiguate
* any displaynames that clashes with another member. Only members
@@ -874,7 +911,11 @@ export class CallViewModel extends ViewModel {
],
(memberships, _displaynames) => {
const displaynameMap = new Map<string, string>([
["local", this.matrixRoom.getMember(this.userId!)!.rawDisplayName],
[
"local",
this.matrixRoom.getMember(this.userId)?.rawDisplayName ??
this.userId,
],
]);
const room = this.matrixRoom;
@@ -1942,16 +1983,26 @@ export class CallViewModel extends ViewModel {
.pipe(this.scope.bind())
.subscribe(({ start, stop }) => {
for (const c of stop) {
logger.info(`Disconnecting from ${c.transport.livekit_service_url}`);
c.stop();
logger.info(
`Disconnecting from ${c.localTransport.livekit_service_url}`,
);
c.stop().catch((err) => {
// TODO: better error handling
logger.error(
`Fail to stop connection to ${c.localTransport.livekit_service_url}`,
err,
);
});
}
for (const c of start) {
c.start().then(
() =>
logger.info(`Connected to ${c.transport.livekit_service_url}`),
logger.info(
`Connected to ${c.localTransport.livekit_service_url}`,
),
(e) =>
logger.error(
`Failed to start connection to ${c.transport.livekit_service_url}`,
`Failed to start connection to ${c.localTransport.livekit_service_url}`,
e,
),
);

750
src/state/Connection.test.ts Normal file
View File

@@ -0,0 +1,750 @@
/*
Copyright 2025 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
afterEach,
describe,
expect,
it,
type Mock,
type MockedObject,
onTestFinished,
vi,
} from "vitest";
import { BehaviorSubject, of } from "rxjs";
import {
ConnectionState,
type LocalParticipant,
type RemoteParticipant,
type Room as LivekitRoom,
RoomEvent,
type RoomOptions,
} from "livekit-client";
import fetchMock from "fetch-mock";
import EventEmitter from "events";
import { type IOpenIDToken } from "matrix-js-sdk";
import type {
CallMembership,
LivekitTransport,
} from "matrix-js-sdk/lib/matrixrtc";
import {
type ConnectionOpts,
type FocusConnectionState,
RemoteConnection,
} from "./Connection.ts";
import { ObservableScope } from "./ObservableScope.ts";
import { type OpenIDClientParts } from "../livekit/openIDSFU.ts";
import { FailToGetOpenIdToken } from "../utils/errors.ts";
import { PublishConnection } from "./PublishConnection.ts";
import { mockMediaDevices, mockMuteStates } from "../utils/test.ts";
import type { ProcessorState } from "../livekit/TrackProcessorContext.tsx";
import { type MuteStates } from "./MuteStates.ts";
let testScope: ObservableScope;
let client: MockedObject<OpenIDClientParts>;
let fakeLivekitRoom: MockedObject<LivekitRoom>;
let localParticipantEventEmiter: EventEmitter;
let fakeLocalParticipant: MockedObject<LocalParticipant>;
let fakeRoomEventEmiter: EventEmitter;
let fakeMembershipsFocusMap$: BehaviorSubject<
{ membership: CallMembership; transport: LivekitTransport }[]
>;
const livekitFocus: LivekitTransport = {
livekit_alias: "!roomID:example.org",
livekit_service_url: "https://matrix-rtc.example.org/livekit/jwt",
type: "livekit",
};
function setupTest(): void {
testScope = new ObservableScope();
client = vi.mocked<OpenIDClientParts>({
getOpenIdToken: vi.fn().mockResolvedValue({
access_token: "rYsmGUEwNjKgJYyeNUkZseJN",
token_type: "Bearer",
matrix_server_name: "example.org",
expires_in: 3600,
}),
getDeviceId: vi.fn().mockReturnValue("ABCDEF"),
} as unknown as OpenIDClientParts);
fakeMembershipsFocusMap$ = new BehaviorSubject<
{ membership: CallMembership; transport: LivekitTransport }[]
>([]);
localParticipantEventEmiter = new EventEmitter();
fakeLocalParticipant = vi.mocked<LocalParticipant>({
identity: "@me:example.org",
isMicrophoneEnabled: vi.fn().mockReturnValue(true),
getTrackPublication: vi.fn().mockReturnValue(undefined),
on: localParticipantEventEmiter.on.bind(localParticipantEventEmiter),
off: localParticipantEventEmiter.off.bind(localParticipantEventEmiter),
addListener: localParticipantEventEmiter.addListener.bind(
localParticipantEventEmiter,
),
removeListener: localParticipantEventEmiter.removeListener.bind(
localParticipantEventEmiter,
),
removeAllListeners: localParticipantEventEmiter.removeAllListeners.bind(
localParticipantEventEmiter,
),
} as unknown as LocalParticipant);
fakeRoomEventEmiter = new EventEmitter();
fakeLivekitRoom = vi.mocked<LivekitRoom>({
connect: vi.fn(),
disconnect: vi.fn(),
remoteParticipants: new Map(),
localParticipant: fakeLocalParticipant,
state: ConnectionState.Disconnected,
on: fakeRoomEventEmiter.on.bind(fakeRoomEventEmiter),
off: fakeRoomEventEmiter.off.bind(fakeRoomEventEmiter),
addListener: fakeRoomEventEmiter.addListener.bind(fakeRoomEventEmiter),
removeListener:
fakeRoomEventEmiter.removeListener.bind(fakeRoomEventEmiter),
removeAllListeners:
fakeRoomEventEmiter.removeAllListeners.bind(fakeRoomEventEmiter),
setE2EEEnabled: vi.fn().mockResolvedValue(undefined),
} as unknown as LivekitRoom);
}
function setupRemoteConnection(): RemoteConnection {
const opts: ConnectionOpts = {
client: client,
transport: livekitFocus,
remoteTransports$: fakeMembershipsFocusMap$,
scope: testScope,
livekitRoomFactory: () => fakeLivekitRoom,
};
fetchMock.post(`${livekitFocus.livekit_service_url}/sfu/get`, () => {
return {
status: 200,
body: {
url: "wss://matrix-rtc.m.localhost/livekit/sfu",
jwt: "ATOKEN",
},
};
});
fakeLivekitRoom.connect.mockResolvedValue(undefined);
return new RemoteConnection(opts, undefined);
}
afterEach(() => {
vi.useRealTimers();
vi.clearAllMocks();
fetchMock.reset();
});
describe("Start connection states", () => {
it("start in initialized state", () => {
setupTest();
const opts: ConnectionOpts = {
client: client,
transport: livekitFocus,
remoteTransports$: fakeMembershipsFocusMap$,
scope: testScope,
livekitRoomFactory: () => fakeLivekitRoom,
};
const connection = new RemoteConnection(opts, undefined);
expect(connection.focusConnectionState$.getValue().state).toEqual(
"Initialized",
);
});
it("fail to getOpenId token then error state", async () => {
setupTest();
vi.useFakeTimers();
const opts: ConnectionOpts = {
client: client,
transport: livekitFocus,
remoteTransports$: fakeMembershipsFocusMap$,
scope: testScope,
livekitRoomFactory: () => fakeLivekitRoom,
};
const connection = new RemoteConnection(opts, undefined);
const capturedStates: FocusConnectionState[] = [];
const s = connection.focusConnectionState$.subscribe((value) => {
capturedStates.push(value);
});
onTestFinished(() => s.unsubscribe());
const deferred = Promise.withResolvers<IOpenIDToken>();
client.getOpenIdToken.mockImplementation(
async (): Promise<IOpenIDToken> => {
return await deferred.promise;
},
);
connection.start().catch(() => {
// expected to throw
});
let capturedState = capturedStates.pop();
expect(capturedState).toBeDefined();
expect(capturedState!.state).toEqual("FetchingConfig");
deferred.reject(new FailToGetOpenIdToken(new Error("Failed to get token")));
await vi.runAllTimersAsync();
capturedState = capturedStates.pop();
if (capturedState!.state === "FailedToStart") {
expect(capturedState!.error.message).toEqual("Something went wrong");
expect(capturedState!.focus.livekit_alias).toEqual(
livekitFocus.livekit_alias,
);
} else {
expect.fail(
"Expected FailedToStart state but got " + capturedState?.state,
);
}
});
it("fail to get JWT token and error state", async () => {
setupTest();
vi.useFakeTimers();
const opts: ConnectionOpts = {
client: client,
transport: livekitFocus,
remoteTransports$: fakeMembershipsFocusMap$,
scope: testScope,
livekitRoomFactory: () => fakeLivekitRoom,
};
const connection = new RemoteConnection(opts, undefined);
const capturedStates: FocusConnectionState[] = [];
const s = connection.focusConnectionState$.subscribe((value) => {
capturedStates.push(value);
});
onTestFinished(() => s.unsubscribe());
const deferredSFU = Promise.withResolvers<void>();
// mock the /sfu/get call
fetchMock.post(`${livekitFocus.livekit_service_url}/sfu/get`, async () => {
await deferredSFU.promise;
return {
status: 500,
body: "Internal Server Error",
};
});
connection.start().catch(() => {
// expected to throw
});
let capturedState = capturedStates.pop();
expect(capturedState).toBeDefined();
expect(capturedState?.state).toEqual("FetchingConfig");
deferredSFU.resolve();
await vi.runAllTimersAsync();
capturedState = capturedStates.pop();
if (capturedState?.state === "FailedToStart") {
expect(capturedState?.error.message).toContain(
"SFU Config fetch failed with exception Error",
);
expect(capturedState?.focus.livekit_alias).toEqual(
livekitFocus.livekit_alias,
);
} else {
expect.fail(
"Expected FailedToStart state but got " + capturedState?.state,
);
}
});
it("fail to connect to livekit error state", async () => {
setupTest();
vi.useFakeTimers();
const opts: ConnectionOpts = {
client: client,
transport: livekitFocus,
remoteTransports$: fakeMembershipsFocusMap$,
scope: testScope,
livekitRoomFactory: () => fakeLivekitRoom,
};
const connection = new RemoteConnection(opts, undefined);
const capturedStates: FocusConnectionState[] = [];
const s = connection.focusConnectionState$.subscribe((value) => {
capturedStates.push(value);
});
onTestFinished(() => s.unsubscribe());
const deferredSFU = Promise.withResolvers<void>();
// mock the /sfu/get call
fetchMock.post(`${livekitFocus.livekit_service_url}/sfu/get`, () => {
return {
status: 200,
body: {
url: "wss://matrix-rtc.m.localhost/livekit/sfu",
jwt: "ATOKEN",
},
};
});
fakeLivekitRoom.connect.mockImplementation(async () => {
await deferredSFU.promise;
throw new Error("Failed to connect to livekit");
});
connection.start().catch(() => {
// expected to throw
});
let capturedState = capturedStates.pop();
expect(capturedState).toBeDefined();
expect(capturedState?.state).toEqual("FetchingConfig");
deferredSFU.resolve();
await vi.runAllTimersAsync();
capturedState = capturedStates.pop();
if (capturedState && capturedState?.state === "FailedToStart") {
expect(capturedState.error.message).toContain(
"Failed to connect to livekit",
);
expect(capturedState.focus.livekit_alias).toEqual(
livekitFocus.livekit_alias,
);
} else {
expect.fail(
"Expected FailedToStart state but got " + JSON.stringify(capturedState),
);
}
});
it("connection states happy path", async () => {
vi.useFakeTimers();
setupTest();
const connection = setupRemoteConnection();
const capturedStates: FocusConnectionState[] = [];
const s = connection.focusConnectionState$.subscribe((value) => {
capturedStates.push(value);
});
onTestFinished(() => s.unsubscribe());
await connection.start();
await vi.runAllTimersAsync();
const initialState = capturedStates.shift();
expect(initialState?.state).toEqual("Initialized");
const fetchingState = capturedStates.shift();
expect(fetchingState?.state).toEqual("FetchingConfig");
const connectingState = capturedStates.shift();
expect(connectingState?.state).toEqual("ConnectingToLkRoom");
const connectedState = capturedStates.shift();
expect(connectedState?.state).toEqual("ConnectedToLkRoom");
});
it("should relay livekit events once connected", async () => {
setupTest();
const connection = setupRemoteConnection();
await connection.start();
let capturedStates: FocusConnectionState[] = [];
const s = connection.focusConnectionState$.subscribe((value) => {
capturedStates.push(value);
});
onTestFinished(() => s.unsubscribe());
const states = [
ConnectionState.Disconnected,
ConnectionState.Connecting,
ConnectionState.Connected,
ConnectionState.SignalReconnecting,
ConnectionState.Connecting,
ConnectionState.Connected,
ConnectionState.Reconnecting,
];
for (const state of states) {
fakeRoomEventEmiter.emit(RoomEvent.ConnectionStateChanged, state);
}
for (const state of states) {
const s = capturedStates.shift();
expect(s?.state).toEqual("ConnectedToLkRoom");
const connectedState = s as FocusConnectionState & {
state: "ConnectedToLkRoom";
};
expect(connectedState.connectionState).toEqual(state);
// should always have the focus info
expect(connectedState.focus.livekit_alias).toEqual(
livekitFocus.livekit_alias,
);
expect(connectedState.focus.livekit_service_url).toEqual(
livekitFocus.livekit_service_url,
);
}
// If the state is not ConnectedToLkRoom, no events should be relayed anymore
await connection.stop();
capturedStates = [];
for (const state of states) {
fakeRoomEventEmiter.emit(RoomEvent.ConnectionStateChanged, state);
}
expect(capturedStates.length).toEqual(0);
});
it("shutting down the scope should stop the connection", async () => {
setupTest();
vi.useFakeTimers();
const connection = setupRemoteConnection();
await connection.start();
const stopSpy = vi.spyOn(connection, "stop");
testScope.end();
expect(stopSpy).toHaveBeenCalled();
expect(fakeLivekitRoom.disconnect).toHaveBeenCalled();
});
});
function fakeRemoteLivekitParticipant(id: string): RemoteParticipant {
return vi.mocked<RemoteParticipant>({
identity: id,
} as unknown as RemoteParticipant);
}
function fakeRtcMemberShip(userId: string, deviceId: string): CallMembership {
return vi.mocked<CallMembership>({
sender: userId,
deviceId: deviceId,
} as unknown as CallMembership);
}
describe("Publishing participants observations", () => {
it("should emit the list of publishing participants", async () => {
setupTest();
const connection = setupRemoteConnection();
const bobIsAPublisher = Promise.withResolvers<void>();
const danIsAPublisher = Promise.withResolvers<void>();
const observedPublishers: {
participant: RemoteParticipant;
membership: CallMembership;
}[][] = [];
const s = connection.publishingParticipants$.subscribe((publishers) => {
observedPublishers.push(publishers);
if (
publishers.some(
(p) => p.participant.identity === "@bob:example.org:DEV111",
)
) {
bobIsAPublisher.resolve();
}
if (
publishers.some(
(p) => p.participant.identity === "@dan:example.org:DEV333",
)
) {
danIsAPublisher.resolve();
}
});
onTestFinished(() => s.unsubscribe());
// The publishingParticipants$ observable is derived from the current members of the
// livekitRoom and the rtc membership in order to publish the members that are publishing
// on this connection.
let participants: RemoteParticipant[] = [
fakeRemoteLivekitParticipant("@alice:example.org:DEV000"),
fakeRemoteLivekitParticipant("@bob:example.org:DEV111"),
fakeRemoteLivekitParticipant("@carol:example.org:DEV222"),
fakeRemoteLivekitParticipant("@dan:example.org:DEV333"),
];
// Let's simulate 3 members on the livekitRoom
vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get").mockReturnValue(
new Map(participants.map((p) => [p.identity, p])),
);
for (const participant of participants) {
fakeRoomEventEmiter.emit(RoomEvent.ParticipantConnected, participant);
}
// At this point there should be no publishers
expect(observedPublishers.pop()!.length).toEqual(0);
const otherFocus: LivekitTransport = {
livekit_alias: "!roomID:example.org",
livekit_service_url: "https://other-matrix-rtc.example.org/livekit/jwt",
type: "livekit",
};
const rtcMemberships = [
// Say bob is on the same focus
{
membership: fakeRtcMemberShip("@bob:example.org", "DEV111"),
transport: livekitFocus,
},
// Alice and carol is on a different focus
{
membership: fakeRtcMemberShip("@alice:example.org", "DEV000"),
transport: otherFocus,
},
{
membership: fakeRtcMemberShip("@carol:example.org", "DEV222"),
transport: otherFocus,
},
// NO DAVE YET
];
// signal this change in rtc memberships
fakeMembershipsFocusMap$.next(rtcMemberships);
// We should have bob has a publisher now
await bobIsAPublisher.promise;
const publishers = observedPublishers.pop();
expect(publishers?.length).toEqual(1);
expect(publishers?.[0].participant.identity).toEqual(
"@bob:example.org:DEV111",
);
// Now let's make dan join the rtc memberships
rtcMemberships.push({
membership: fakeRtcMemberShip("@dan:example.org", "DEV333"),
transport: livekitFocus,
});
fakeMembershipsFocusMap$.next(rtcMemberships);
// We should have bob and dan has publishers now
await danIsAPublisher.promise;
const twoPublishers = observedPublishers.pop();
expect(twoPublishers?.length).toEqual(2);
expect(
twoPublishers?.some(
(p) => p.participant.identity === "@bob:example.org:DEV111",
),
).toBeTruthy();
expect(
twoPublishers?.some(
(p) => p.participant.identity === "@dan:example.org:DEV333",
),
).toBeTruthy();
// Now let's make bob leave the livekit room
participants = participants.filter(
(p) => p.identity !== "@bob:example.org:DEV111",
);
vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get").mockReturnValue(
new Map(participants.map((p) => [p.identity, p])),
);
fakeRoomEventEmiter.emit(
RoomEvent.ParticipantDisconnected,
fakeRemoteLivekitParticipant("@bob:example.org:DEV111"),
);
const updatedPublishers = observedPublishers.pop();
expect(updatedPublishers?.length).toEqual(1);
expect(
updatedPublishers?.some(
(p) => p.participant.identity === "@dan:example.org:DEV333",
),
).toBeTruthy();
});
it("should be scoped to parent scope", (): void => {
setupTest();
const connection = setupRemoteConnection();
let observedPublishers: {
participant: RemoteParticipant;
membership: CallMembership;
}[][] = [];
const s = connection.publishingParticipants$.subscribe((publishers) => {
observedPublishers.push(publishers);
});
onTestFinished(() => s.unsubscribe());
let participants: RemoteParticipant[] = [
fakeRemoteLivekitParticipant("@bob:example.org:DEV111"),
];
// Let's simulate 3 members on the livekitRoom
vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get").mockReturnValue(
new Map(participants.map((p) => [p.identity, p])),
);
for (const participant of participants) {
fakeRoomEventEmiter.emit(RoomEvent.ParticipantConnected, participant);
}
// At this point there should be no publishers
expect(observedPublishers.pop()!.length).toEqual(0);
const rtcMemberships = [
// Say bob is on the same focus
{
membership: fakeRtcMemberShip("@bob:example.org", "DEV111"),
transport: livekitFocus,
},
];
// signal this change in rtc memberships
fakeMembershipsFocusMap$.next(rtcMemberships);
// We should have bob has a publisher now
const publishers = observedPublishers.pop();
expect(publishers?.length).toEqual(1);
expect(publishers?.[0].participant.identity).toEqual(
"@bob:example.org:DEV111",
);
// end the parent scope
testScope.end();
observedPublishers = [];
// SHOULD NOT emit any more publishers as the scope is ended
participants = participants.filter(
(p) => p.identity !== "@bob:example.org:DEV111",
);
vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get").mockReturnValue(
new Map(participants.map((p) => [p.identity, p])),
);
fakeRoomEventEmiter.emit(
RoomEvent.ParticipantDisconnected,
fakeRemoteLivekitParticipant("@bob:example.org:DEV111"),
);
expect(observedPublishers.length).toEqual(0);
});
});
describe("PublishConnection", () => {
// let fakeBlurProcessor: ProcessorWrapper<BackgroundOptions>;
let roomFactoryMock: Mock<() => LivekitRoom>;
let muteStates: MockedObject<MuteStates>;
function setUpPublishConnection(): void {
setupTest();
roomFactoryMock = vi.fn().mockReturnValue(fakeLivekitRoom);
muteStates = mockMuteStates();
// fakeBlurProcessor = vi.mocked<ProcessorWrapper<BackgroundOptions>>({
// name: "BackgroundBlur",
// restart: vi.fn().mockResolvedValue(undefined),
// setOptions: vi.fn().mockResolvedValue(undefined),
// getOptions: vi.fn().mockReturnValue({ strength: 0.5 }),
// isRunning: vi.fn().mockReturnValue(false)
// });
}
describe("Livekit room creation", () => {
function createSetup(): void {
setUpPublishConnection();
const fakeTrackProcessorSubject$ = new BehaviorSubject<ProcessorState>({
supported: true,
processor: undefined,
});
const opts: ConnectionOpts = {
client: client,
transport: livekitFocus,
remoteTransports$: fakeMembershipsFocusMap$,
scope: testScope,
livekitRoomFactory: roomFactoryMock,
};
const audioInput = {
available$: of(new Map([["mic1", { id: "mic1" }]])),
selected$: new BehaviorSubject({ id: "mic1" }),
select(): void {},
};
const videoInput = {
available$: of(new Map([["cam1", { id: "cam1" }]])),
selected$: new BehaviorSubject({ id: "cam1" }),
select(): void {},
};
const audioOutput = {
available$: of(new Map([["speaker", { id: "speaker" }]])),
selected$: new BehaviorSubject({ id: "speaker" }),
select(): void {},
};
// TODO understand what is wrong with our mocking that requires ts-expect-error
const fakeDevices = mockMediaDevices({
// @ts-expect-error Mocking only
audioInput,
// @ts-expect-error Mocking only
videoInput,
// @ts-expect-error Mocking only
audioOutput,
});
new PublishConnection(
opts,
fakeDevices,
muteStates,
undefined,
fakeTrackProcessorSubject$,
);
}
it("should create room with proper initial audio and video settings", () => {
createSetup();
expect(roomFactoryMock).toHaveBeenCalled();
const lastCallArgs =
roomFactoryMock.mock.calls[roomFactoryMock.mock.calls.length - 1];
const roomOptions = lastCallArgs.pop() as unknown as RoomOptions;
expect(roomOptions).toBeDefined();
expect(roomOptions!.videoCaptureDefaults?.deviceId).toEqual("cam1");
expect(roomOptions!.audioCaptureDefaults?.deviceId).toEqual("mic1");
expect(roomOptions!.audioOutput?.deviceId).toEqual("speaker");
});
it("respect controlledAudioDevices", () => {
// TODO: Refactor the code to make it testable.
// The UrlParams module is a singleton has a cache and is very hard to test.
// This breaks other tests as well if not handled properly.
// vi.mock(import("./../UrlParams"), () => {
// return {
// getUrlParams: vi.fn().mockReturnValue({
// controlledAudioDevices: true
// })
// };
// });
});
});
});

420
src/state/Connection.ts
View File

@@ -10,293 +10,241 @@ import {
connectionStateObserver,
} from "@livekit/components-core";
import {
ConnectionState,
Room as LivekitRoom,
type ConnectionState,
type E2EEOptions,
Track,
LocalVideoTrack,
Room as LivekitRoom,
type RoomOptions,
} from "livekit-client";
import { type MatrixClient } from "matrix-js-sdk";
import {
type LivekitTransport,
type CallMembership,
type LivekitTransport,
} from "matrix-js-sdk/lib/matrixrtc";
import {
combineLatest,
map,
NEVER,
type Observable,
type Subscription,
switchMap,
} from "rxjs";
import { logger } from "matrix-js-sdk/lib/logger";
import { BehaviorSubject, combineLatest } from "rxjs";
import { type SelectedDevice, type MediaDevices } from "./MediaDevices";
import { getSFUConfigWithOpenID } from "../livekit/openIDSFU";
import {
getSFUConfigWithOpenID,
type OpenIDClientParts,
type SFUConfig,
} from "../livekit/openIDSFU";
import { type Behavior } from "./Behavior";
import { type ObservableScope } from "./ObservableScope";
import { defaultLiveKitOptions } from "../livekit/options";
import { getValue } from "../utils/observable";
import { getUrlParams } from "../UrlParams";
import { type MuteStates } from "./MuteStates";
import {
type ProcessorState,
trackProcessorSync,
} from "../livekit/TrackProcessorContext";
import { observeTrackReference$ } from "./MediaViewModel";
export interface ConnectionOpts {
/** The focus server to connect to. */
transport: LivekitTransport;
/** The Matrix client to use for OpenID and SFU config requests. */
client: OpenIDClientParts;
/** The observable scope to use for this connection. */
scope: ObservableScope;
/** An observable of the current RTC call memberships and their associated focus. */
remoteTransports$: Behavior<
{ membership: CallMembership; transport: LivekitTransport }[]
>;
/** Optional factory to create the Livekit room, mainly for testing purposes. */
livekitRoomFactory?: (options?: RoomOptions) => LivekitRoom;
}
export type FocusConnectionState =
| { state: "Initialized" }
| { state: "FetchingConfig"; focus: LivekitTransport }
| { state: "ConnectingToLkRoom"; focus: LivekitTransport }
| { state: "PublishingTracks"; focus: LivekitTransport }
| { state: "FailedToStart"; error: Error; focus: LivekitTransport }
| {
state: "ConnectedToLkRoom";
connectionState: ConnectionState;
focus: LivekitTransport;
}
| { state: "Stopped"; focus: LivekitTransport };
/**
* A connection to a Matrix RTC LiveKit backend.
*
* Expose observables for participants and connection state.
*/
export class Connection {
// Private Behavior
private readonly _focusConnectionState$ =
new BehaviorSubject<FocusConnectionState>({ state: "Initialized" });
/**
* The current state of the connection to the focus server.
*/
public readonly focusConnectionState$: Behavior<FocusConnectionState> =
this._focusConnectionState$;
/**
* Whether the connection has been stopped.
* @see Connection.stop
* */
protected stopped = false;
/**
* Starts the connection.
*
* This will:
* 1. Request an OpenId token `request_token` (allows matrix users to verify their identity with a third-party service.)
* 2. Use this token to request the SFU config to the MatrixRtc authentication service.
* 3. Connect to the configured LiveKit room.
*/
public async start(): Promise<void> {
this.stopped = false;
const { url, jwt } = await this.sfuConfig;
if (!this.stopped) await this.livekitRoom.connect(url, jwt);
try {
this._focusConnectionState$.next({
state: "FetchingConfig",
focus: this.localTransport,
});
// TODO could this be loaded earlier to save time?
const { url, jwt } = await this.getSFUConfigWithOpenID();
// If we were stopped while fetching the config, don't proceed to connect
if (this.stopped) return;
this._focusConnectionState$.next({
state: "ConnectingToLkRoom",
focus: this.localTransport,
});
await this.livekitRoom.connect(url, jwt);
// If we were stopped while connecting, don't proceed to update state.
if (this.stopped) return;
this._focusConnectionState$.next({
state: "ConnectedToLkRoom",
focus: this.localTransport,
connectionState: this.livekitRoom.state,
});
} catch (error) {
this._focusConnectionState$.next({
state: "FailedToStart",
error: error instanceof Error ? error : new Error(`${error}`),
focus: this.localTransport,
});
throw error;
}
}
public stop(): void {
protected async getSFUConfigWithOpenID(): Promise<SFUConfig> {
return await getSFUConfigWithOpenID(
this.client,
this.localTransport.livekit_service_url,
this.localTransport.livekit_alias,
);
}
/**
* Stops the connection.
*
* This will disconnect from the LiveKit room.
* If the connection is already stopped, this is a no-op.
*/
public async stop(): Promise<void> {
if (this.stopped) return;
void this.livekitRoom.disconnect();
await this.livekitRoom.disconnect();
this._focusConnectionState$.next({
state: "Stopped",
focus: this.localTransport,
});
this.stopped = true;
}
protected readonly sfuConfig = getSFUConfigWithOpenID(
this.client,
this.transport.livekit_service_url,
this.livekitAlias,
);
private readonly participantsIncludingSubscribers$;
/**
* An observable of the participants that are publishing on this connection.
* This is derived from `participantsIncludingSubscribers$` and `remoteTransports$`.
* It filters the participants to only those that are associated with a membership that claims to publish on this connection.
*/
public readonly publishingParticipants$;
public readonly livekitRoom: LivekitRoom;
public connectionState$: Behavior<ConnectionState>;
public constructor(
public readonly transport: LivekitTransport,
protected readonly livekitAlias: string,
protected readonly client: MatrixClient,
protected readonly scope: ObservableScope,
protected readonly remoteTransports$: Behavior<
{ membership: CallMembership; transport: LivekitTransport }[]
>,
e2eeLivekitOptions: E2EEOptions | undefined,
livekitRoom: LivekitRoom | undefined = undefined,
/**
* The focus server to connect to.
*/
public readonly localTransport: LivekitTransport;
private readonly client: OpenIDClientParts;
/**
* Creates a new connection to a matrix RTC LiveKit backend.
*
* @param livekitRoom - LiveKit room instance to use.
* @param opts - Connection options {@link ConnectionOpts}.
*
*/
protected constructor(
public readonly livekitRoom: LivekitRoom,
opts: ConnectionOpts,
) {
this.livekitRoom =
livekitRoom ??
new LivekitRoom({
...defaultLiveKitOptions,
e2ee: e2eeLivekitOptions,
});
this.participantsIncludingSubscribers$ = this.scope.behavior(
const { transport, client, scope, remoteTransports$ } = opts;
this.localTransport = transport;
this.client = client;
const participantsIncludingSubscribers$ = scope.behavior(
connectedParticipantsObserver(this.livekitRoom),
[],
);
this.publishingParticipants$ = this.scope.behavior(
this.publishingParticipants$ = scope.behavior(
combineLatest(
[this.participantsIncludingSubscribers$, this.remoteTransports$],
[participantsIncludingSubscribers$, remoteTransports$],
(participants, remoteTransports) =>
remoteTransports
// Find all members that claim to publish on this connection
.flatMap(({ membership, transport }) =>
transport.livekit_service_url ===
this.transport.livekit_service_url
this.localTransport.livekit_service_url
? [membership]
: [],
)
// Pair with their associated LiveKit participant (if any)
.map((membership) => {
// Uses flatMap to filter out memberships with no associated rtc participant ([])
.flatMap((membership) => {
const id = `${membership.sender}:${membership.deviceId}`;
const participant = participants.find((p) => p.identity === id);
return { participant, membership };
return participant ? [{ participant, membership }] : [];
}),
),
[],
);
this.connectionState$ = this.scope.behavior<ConnectionState>(
connectionStateObserver(this.livekitRoom),
);
this.scope.onEnd(() => this.stop());
scope
.behavior<ConnectionState>(connectionStateObserver(this.livekitRoom))
.subscribe((connectionState) => {
const current = this._focusConnectionState$.value;
// Only update the state if we are already connected to the LiveKit room.
if (current.state === "ConnectedToLkRoom") {
this._focusConnectionState$.next({
state: "ConnectedToLkRoom",
connectionState,
focus: current.focus,
});
}
});
scope.onEnd(() => void this.stop());
}
}
export class PublishConnection extends Connection {
public async start(): Promise<void> {
this.stopped = false;
this.muteStates.audio.setHandler(async (desired) => {
try {
await this.livekitRoom.localParticipant.setMicrophoneEnabled(desired);
} catch (e) {
logger.error("Failed to update LiveKit audio input mute state", e);
}
return this.livekitRoom.localParticipant.isMicrophoneEnabled;
});
this.muteStates.video.setHandler(async (desired) => {
try {
await this.livekitRoom.localParticipant.setCameraEnabled(desired);
} catch (e) {
logger.error("Failed to update LiveKit video input mute state", e);
}
return this.livekitRoom.localParticipant.isCameraEnabled;
});
const { url, jwt } = await this.sfuConfig;
if (!this.stopped) await this.livekitRoom.connect(url, jwt);
if (!this.stopped) {
// TODO-MULTI-SFU: Prepublish a microphone track
const audio = this.muteStates.audio.enabled$.value;
const video = this.muteStates.video.enabled$.value;
// createTracks throws if called with audio=false and video=false
if (audio || video) {
const tracks = await this.livekitRoom.localParticipant.createTracks({
audio,
video,
});
for (const track of tracks) {
await this.livekitRoom.localParticipant.publishTrack(track);
}
}
}
}
public stop(): void {
this.muteStates.audio.unsetHandler();
this.muteStates.video.unsetHandler();
super.stop();
}
/**
* A remote connection to the Matrix RTC LiveKit backend.
*
* This connection is used for subscribing to remote participants.
* It does not publish any local tracks.
*/
export class RemoteConnection extends Connection {
/**
* Creates a new remote connection to a matrix RTC LiveKit backend.
* @param opts
* @param sharedE2eeOption - The shared E2EE options to use for the connection.
*/
public constructor(
transport: LivekitTransport,
livekitAlias: string,
client: MatrixClient,
scope: ObservableScope,
remoteTransports$: Behavior<
{ membership: CallMembership; transport: LivekitTransport }[]
>,
devices: MediaDevices,
private readonly muteStates: MuteStates,
e2eeLivekitOptions: E2EEOptions | undefined,
trackerProcessorState$: Behavior<ProcessorState>,
opts: ConnectionOpts,
sharedE2eeOption: E2EEOptions | undefined,
) {
logger.info("[LivekitRoom] Create LiveKit room");
const { controlledAudioDevices } = getUrlParams();
const room = new LivekitRoom({
const factory =
opts.livekitRoomFactory ??
((options: RoomOptions): LivekitRoom => new LivekitRoom(options));
const livekitRoom = factory({
...defaultLiveKitOptions,
videoCaptureDefaults: {
...defaultLiveKitOptions.videoCaptureDefaults,
deviceId: devices.videoInput.selected$.value?.id,
processor: trackerProcessorState$.value.processor,
},
audioCaptureDefaults: {
...defaultLiveKitOptions.audioCaptureDefaults,
deviceId: devices.audioInput.selected$.value?.id,
},
audioOutput: {
// When using controlled audio devices, we don't want to set the
// deviceId here, because it will be set by the native app.
// (also the id does not need to match a browser device id)
deviceId: controlledAudioDevices
? undefined
: getValue(devices.audioOutput.selected$)?.id,
},
e2ee: e2eeLivekitOptions,
e2ee: sharedE2eeOption,
});
room.setE2EEEnabled(e2eeLivekitOptions !== undefined).catch((e) => {
logger.error("Failed to set E2EE enabled on room", e);
});
super(
transport,
livekitAlias,
client,
scope,
remoteTransports$,
e2eeLivekitOptions,
room,
);
// Setup track processor syncing (blur)
const track$ = this.scope.behavior(
observeTrackReference$(room.localParticipant, Track.Source.Camera).pipe(
map((trackRef) => {
const track = trackRef?.publication?.track;
return track instanceof LocalVideoTrack ? track : null;
}),
),
);
trackProcessorSync(track$, trackerProcessorState$);
const syncDevice = (
kind: MediaDeviceKind,
selected$: Observable<SelectedDevice | undefined>,
): Subscription =>
selected$.pipe(this.scope.bind()).subscribe((device) => {
if (this.connectionState$.value !== ConnectionState.Connected) return;
logger.info(
"[LivekitRoom] syncDevice room.getActiveDevice(kind) !== d.id :",
this.livekitRoom.getActiveDevice(kind),
" !== ",
device?.id,
);
if (
device !== undefined &&
this.livekitRoom.getActiveDevice(kind) !== device.id
) {
this.livekitRoom
.switchActiveDevice(kind, device.id)
.catch((e) =>
logger.error(`Failed to sync ${kind} device with LiveKit`, e),
);
}
});
syncDevice("audioinput", devices.audioInput.selected$);
if (!controlledAudioDevices)
syncDevice("audiooutput", devices.audioOutput.selected$);
syncDevice("videoinput", devices.videoInput.selected$);
// Restart the audio input track whenever we detect that the active media
// device has changed to refer to a different hardware device. We do this
// for the sake of Chrome, which provides a "default" device that is meant
// to match the system's default audio input, whatever that may be.
// This is special-cased for only audio inputs because we need to dig around
// in the LocalParticipant object for the track object and there's not a nice
// way to do that generically. There is usually no OS-level default video capture
// device anyway, and audio outputs work differently.
devices.audioInput.selected$
.pipe(
switchMap((device) => device?.hardwareDeviceChange$ ?? NEVER),
this.scope.bind(),
)
.subscribe(() => {
if (this.connectionState$.value !== ConnectionState.Connected) return;
const activeMicTrack = Array.from(
this.livekitRoom.localParticipant.audioTrackPublications.values(),
).find((d) => d.source === Track.Source.Microphone)?.track;
if (
activeMicTrack &&
// only restart if the stream is still running: LiveKit will detect
// when a track stops & restart appropriately, so this is not our job.
// Plus, we need to avoid restarting again if the track is already in
// the process of being restarted.
activeMicTrack.mediaStreamTrack.readyState !== "ended"
) {
// Restart the track, which will cause Livekit to do another
// getUserMedia() call with deviceId: default to get the *new* default device.
// Note that room.switchActiveDevice() won't work: Livekit will ignore it because
// the deviceId hasn't changed (was & still is default).
this.livekitRoom.localParticipant
.getTrackPublication(Track.Source.Microphone)
?.audioTrack?.restartTrack()
.catch((e) => {
logger.error(`Failed to restart audio device track`, e);
});
}
});
super(livekitRoom, opts);
}
}

10
src/state/MuteStates.ts
View File

@@ -88,7 +88,10 @@ class MuteState<Label, Selected> {
} else {
subscriber.next(enabled);
syncing = true;
sync();
sync().catch((err) => {
// TODO: better error handling
logger.error("MuteState: handler error", err);
});
}
}
};
@@ -97,7 +100,10 @@ class MuteState<Label, Selected> {
latestDesired = desired;
if (syncing === false) {
syncing = true;
sync();
sync().catch((err) => {
// TODO: better error handling
logger.error("MuteState: handler error", err);
});
}
});
return (): void => s.unsubscribe();

277
src/state/PublishConnection.ts Normal file
View File

@@ -0,0 +1,277 @@
/*
Copyright 2025 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
ConnectionState,
type E2EEOptions,
LocalVideoTrack,
Room as LivekitRoom,
type RoomOptions,
Track,
} from "livekit-client";
import {
map,
NEVER,
type Observable,
type Subscription,
switchMap,
} from "rxjs";
import { logger } from "matrix-js-sdk/lib/logger";
import type { Behavior } from "./Behavior.ts";
import type { MediaDevices, SelectedDevice } from "./MediaDevices.ts";
import type { MuteStates } from "./MuteStates.ts";
import {
type ProcessorState,
trackProcessorSync,
} from "../livekit/TrackProcessorContext.tsx";
import { getUrlParams } from "../UrlParams.ts";
import { defaultLiveKitOptions } from "../livekit/options.ts";
import { getValue } from "../utils/observable.ts";
import { observeTrackReference$ } from "./MediaViewModel.ts";
import { Connection, type ConnectionOpts } from "./Connection.ts";
import { type ObservableScope } from "./ObservableScope.ts";
/**
* A connection to the local LiveKit room, the one the user is publishing to.
* This connection will publish the local user's audio and video tracks.
*/
export class PublishConnection extends Connection {
/**
* Creates a new PublishConnection.
* @param args - The connection options. {@link ConnectionOpts}
* @param devices - The media devices to use for audio and video input.
* @param muteStates - The mute states for audio and video.
* @param e2eeLivekitOptions - The E2EE options to use for the LiveKit room. Use to share the same key provider across connections!.
* @param trackerProcessorState$ - The processor state for the video track processor (e.g. background blur).
*/
public constructor(
args: ConnectionOpts,
devices: MediaDevices,
private readonly muteStates: MuteStates,
e2eeLivekitOptions: E2EEOptions | undefined,
trackerProcessorState$: Behavior<ProcessorState>,
) {
const { scope } = args;
logger.info("[LivekitRoom] Create LiveKit room");
const { controlledAudioDevices } = getUrlParams();
const factory =
args.livekitRoomFactory ??
((options: RoomOptions): LivekitRoom => new LivekitRoom(options));
const room = factory(
generateRoomOption(
devices,
trackerProcessorState$.value,
controlledAudioDevices,
e2eeLivekitOptions,
),
);
room.setE2EEEnabled(e2eeLivekitOptions !== undefined).catch((e) => {
logger.error("Failed to set E2EE enabled on room", e);
});
super(room, args);
// Setup track processor syncing (blur)
this.observeTrackProcessors(scope, room, trackerProcessorState$);
// Observe mute state changes and update LiveKit microphone/camera states accordingly
this.observeMuteStates(scope);
// Observe media device changes and update LiveKit active devices accordingly
this.observeMediaDevices(scope, devices, controlledAudioDevices);
this.workaroundRestartAudioInputTrackChrome(devices, scope);
}
/**
* Start the connection to LiveKit and publish local tracks.
*
* This will:
* 1. Request an OpenId token `request_token` (allows matrix users to verify their identity with a third-party service.)
* 2. Use this token to request the SFU config to the MatrixRtc authentication service.
* 3. Connect to the configured LiveKit room.
* 4. Create local audio and video tracks based on the current mute states and publish them to the room.
*/
public async start(): Promise<void> {
this.stopped = false;
await super.start();
if (this.stopped) return;
// TODO this can throw errors? It will also prompt for permissions if not already granted
const tracks = await this.livekitRoom.localParticipant.createTracks({
audio: this.muteStates.audio.enabled$.value,
video: this.muteStates.video.enabled$.value,
});
if (this.stopped) return;
for (const track of tracks) {
// TODO: handle errors? Needs the signaling connection to be up, but it has some retries internally
// with a timeout.
await this.livekitRoom.localParticipant.publishTrack(track);
if (this.stopped) return;
// TODO: check if the connection is still active? and break the loop if not?
}
}
/// Private methods
// Restart the audio input track whenever we detect that the active media
// device has changed to refer to a different hardware device. We do this
// for the sake of Chrome, which provides a "default" device that is meant
// to match the system's default audio input, whatever that may be.
// This is special-cased for only audio inputs because we need to dig around
// in the LocalParticipant object for the track object and there's not a nice
// way to do that generically. There is usually no OS-level default video capture
// device anyway, and audio outputs work differently.
private workaroundRestartAudioInputTrackChrome(
devices: MediaDevices,
scope: ObservableScope,
): void {
devices.audioInput.selected$
.pipe(
switchMap((device) => device?.hardwareDeviceChange$ ?? NEVER),
scope.bind(),
)
.subscribe(() => {
if (this.livekitRoom.state != ConnectionState.Connected) return;
const activeMicTrack = Array.from(
this.livekitRoom.localParticipant.audioTrackPublications.values(),
).find((d) => d.source === Track.Source.Microphone)?.track;
if (
activeMicTrack &&
// only restart if the stream is still running: LiveKit will detect
// when a track stops & restart appropriately, so this is not our job.
// Plus, we need to avoid restarting again if the track is already in
// the process of being restarted.
activeMicTrack.mediaStreamTrack.readyState !== "ended"
) {
// Restart the track, which will cause Livekit to do another
// getUserMedia() call with deviceId: default to get the *new* default device.
// Note that room.switchActiveDevice() won't work: Livekit will ignore it because
// the deviceId hasn't changed (was & still is default).
this.livekitRoom.localParticipant
.getTrackPublication(Track.Source.Microphone)
?.audioTrack?.restartTrack()
.catch((e) => {
logger.error(`Failed to restart audio device track`, e);
});
}
});
}
// Observe changes in the selected media devices and update the LiveKit room accordingly.
private observeMediaDevices(
scope: ObservableScope,
devices: MediaDevices,
controlledAudioDevices: boolean,
): void {
const syncDevice = (
kind: MediaDeviceKind,
selected$: Observable<SelectedDevice | undefined>,
): Subscription =>
selected$.pipe(scope.bind()).subscribe((device) => {
if (this.livekitRoom.state != ConnectionState.Connected) return;
// if (this.connectionState$.value !== ConnectionState.Connected) return;
logger.info(
"[LivekitRoom] syncDevice room.getActiveDevice(kind) !== d.id :",
this.livekitRoom.getActiveDevice(kind),
" !== ",
device?.id,
);
if (
device !== undefined &&
this.livekitRoom.getActiveDevice(kind) !== device.id
) {
this.livekitRoom
.switchActiveDevice(kind, device.id)
.catch((e) =>
logger.error(`Failed to sync ${kind} device with LiveKit`, e),
);
}
});
syncDevice("audioinput", devices.audioInput.selected$);
if (!controlledAudioDevices)
syncDevice("audiooutput", devices.audioOutput.selected$);
syncDevice("videoinput", devices.videoInput.selected$);
}
/**
* Observe changes in the mute states and update the LiveKit room accordingly.
* @param scope
* @private
*/
private observeMuteStates(scope: ObservableScope): void {
this.muteStates.audio.setHandler(async (desired) => {
try {
await this.livekitRoom.localParticipant.setMicrophoneEnabled(desired);
} catch (e) {
logger.error("Failed to update LiveKit audio input mute state", e);
}
return this.livekitRoom.localParticipant.isMicrophoneEnabled;
});
this.muteStates.video.setHandler(async (desired) => {
try {
await this.livekitRoom.localParticipant.setCameraEnabled(desired);
} catch (e) {
logger.error("Failed to update LiveKit video input mute state", e);
}
return this.livekitRoom.localParticipant.isCameraEnabled;
});
scope.onEnd(() => {
this.muteStates.audio.unsetHandler();
this.muteStates.video.unsetHandler();
});
}
private observeTrackProcessors(
scope: ObservableScope,
room: LivekitRoom,
trackerProcessorState$: Behavior<ProcessorState>,
): void {
const track$ = scope.behavior(
observeTrackReference$(room.localParticipant, Track.Source.Camera).pipe(
map((trackRef) => {
const track = trackRef?.publication?.track;
return track instanceof LocalVideoTrack ? track : null;
}),
),
);
trackProcessorSync(track$, trackerProcessorState$);
}
}
// Generate the initial LiveKit RoomOptions based on the current media devices and processor state.
function generateRoomOption(
devices: MediaDevices,
processorState: ProcessorState,
controlledAudioDevices: boolean,
e2eeLivekitOptions: E2EEOptions | undefined,
): RoomOptions {
return {
...defaultLiveKitOptions,
videoCaptureDefaults: {
...defaultLiveKitOptions.videoCaptureDefaults,
deviceId: devices.videoInput.selected$.value?.id,
processor: processorState.processor,
},
audioCaptureDefaults: {
...defaultLiveKitOptions.audioCaptureDefaults,
deviceId: devices.audioInput.selected$.value?.id,
},
audioOutput: {
// When using controlled audio devices, we don't want to set the
// deviceId here, because it will be set by the native app.
// (also the id does not need to match a browser device id)
deviceId: controlledAudioDevices
? undefined
: getValue(devices.audioOutput.selected$)?.id,
},
e2ee: e2eeLivekitOptions,
};
}

26
src/tile/MediaView.test.tsx
View File

@@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { describe, expect, it, test } from "vitest";
import { describe, expect, it, test, vi } from "vitest";
import { render, screen } from "@testing-library/react";
import { axe } from "vitest-axe";
import { TooltipProvider } from "@vector-im/compound-web";
@@ -16,6 +16,7 @@ import {
import { LocalTrackPublication, Track } from "livekit-client";
import { TrackInfo } from "@livekit/protocol";
import { type ComponentProps } from "react";
import { type RoomMember } from "matrix-js-sdk";
import { MediaView } from "./MediaView";
import { EncryptionStatus } from "../state/MediaViewModel";
@@ -45,7 +46,10 @@ describe("MediaView", () => {
mirror: false,
unencryptedWarning: false,
video: trackReference,
member: undefined,
member: vi.mocked<RoomMember>({
userId: "@alice:example.com",
getMxcAvatarUrl: vi.fn().mockReturnValue(undefined),
} as unknown as RoomMember),
localParticipant: false,
focusable: true,
};
@@ -59,9 +63,9 @@ describe("MediaView", () => {
test("neither video nor avatar are shown", () => {
render(<MediaView {...baseProps} video={trackReferencePlaceholder} />);
expect(screen.queryByTestId("video")).toBeNull();
expect(screen.queryAllByRole("img", { name: "some name" }).length).toBe(
0,
);
expect(
screen.queryAllByRole("img", { name: "@alice:example.com" }).length,
).toBe(0);
});
});
@@ -70,14 +74,18 @@ describe("MediaView", () => {
render(
<MediaView {...baseProps} video={undefined} localParticipant={true} />,
);
expect(screen.getByRole("img", { name: "some name" })).toBeVisible();
expect(
screen.getByRole("img", { name: "@alice:example.com" }),
).toBeVisible();
expect(screen.queryAllByText("Waiting for media...").length).toBe(0);
});
it("shows avatar and label for remote user", () => {
render(
<MediaView {...baseProps} video={undefined} localParticipant={false} />,
);
expect(screen.getByRole("img", { name: "some name" })).toBeVisible();
expect(
screen.getByRole("img", { name: "@alice:example.com" }),
).toBeVisible();
expect(screen.getByText("Waiting for media...")).toBeVisible();
});
});
@@ -131,7 +139,9 @@ describe("MediaView", () => {
<MediaView {...baseProps} videoEnabled={false} />
</TooltipProvider>,
);
expect(screen.getByRole("img", { name: "some name" })).toBeVisible();
expect(
screen.getByRole("img", { name: "@alice:example.com" }),
).toBeVisible();
expect(screen.getByTestId("video")).not.toBeVisible();
});
});

19
src/useCallViewKeyboardShortcuts.test.tsx
View File

@@ -23,14 +23,14 @@ import {
// The TestComponent just wraps a button around that hook.
interface TestComponentProps {
setMicrophoneMuted?: (muted: boolean) => void;
setAudioEnabled?: (enabled: boolean) => void;
onButtonClick?: () => void;
sendReaction?: () => void;
toggleHandRaised?: () => void;
}
const TestComponent: FC<TestComponentProps> = ({
setMicrophoneMuted = (): void => {},
setAudioEnabled = (): void => {},
onButtonClick = (): void => {},
sendReaction = (reaction: ReactionOption): void => {},
toggleHandRaised = (): void => {},
@@ -40,7 +40,7 @@ const TestComponent: FC<TestComponentProps> = ({
ref,
() => {},
() => {},
setMicrophoneMuted,
setAudioEnabled,
sendReaction,
toggleHandRaised,
);
@@ -57,12 +57,13 @@ test("spacebar unmutes", async () => {
render(
<TestComponent
onButtonClick={() => (muted = false)}
setMicrophoneMuted={(m) => {
muted = m;
setAudioEnabled={(m) => {
muted = !m;
}}
/>,
);
expect(muted).toBe(true);
await user.keyboard("[Space>]");
expect(muted).toBe(false);
await user.keyboard("[/Space]");
@@ -73,15 +74,15 @@ test("spacebar unmutes", async () => {
test("spacebar prioritizes pressing a button", async () => {
const user = userEvent.setup();
const setMuted = vi.fn();
const setAudioEnabled = vi.fn();
const onClick = vi.fn();
render(
<TestComponent setMicrophoneMuted={setMuted} onButtonClick={onClick} />,
<TestComponent setAudioEnabled={setAudioEnabled} onButtonClick={onClick} />,
);
await user.tab(); // Focus the button
await user.keyboard("[Space]");
expect(setMuted).not.toBeCalled();
expect(setAudioEnabled).not.toBeCalled();
expect(onClick).toBeCalled();
});
@@ -129,7 +130,7 @@ test("unmuting happens in place of the default action", async () => {
tabIndex={0}
onKeyDown={(e) => defaultPrevented(e.isDefaultPrevented())}
>
<TestComponent setMicrophoneMuted={() => {}} />
<TestComponent setAudioEnabled={() => {}} />
</video>,
);

40
src/useMatrixRTCSessionJoinState.ts
View File

@@ -1,40 +0,0 @@
/*
Copyright 2023, 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { logger } from "matrix-js-sdk/lib/logger";
import {
type MatrixRTCSession,
MatrixRTCSessionEvent,
} from "matrix-js-sdk/lib/matrixrtc";
import { TypedEventEmitter } from "matrix-js-sdk";
import { useCallback, useEffect } from "react";
import { useTypedEventEmitterState } from "./useEvents";
const dummySession = new TypedEventEmitter();
export function useMatrixRTCSessionJoinState(
rtcSession: MatrixRTCSession | undefined,
): boolean {
// React doesn't allow you to run a hook conditionally, so we have to plug in
// a dummy event emitter in case there is no rtcSession yet
const isJoined = useTypedEventEmitterState(
rtcSession ?? dummySession,
MatrixRTCSessionEvent.JoinStateChanged,
useCallback(() => rtcSession?.isJoined() ?? false, [rtcSession]),
);
useEffect(() => {
logger.info(
`Session in room ${rtcSession?.room.roomId} changed to ${
isJoined ? "joined" : "left"
}`,
);
}, [rtcSession, isJoined]);
return isJoined;
}

3
src/utils/test-fixtures.ts
View File

@@ -9,7 +9,6 @@ import {
mockRtcMembership,
mockMatrixRoomMember,
mockRemoteParticipant,
mockLocalParticipant,
} from "./test";
export const localRtcMember = mockRtcMembership("@carol:example.org", "1111");
@@ -18,7 +17,7 @@ export const localRtcMemberDevice2 = mockRtcMembership(
"2222",
);
export const local = mockMatrixRoomMember(localRtcMember);
export const localParticipant = mockLocalParticipant({ identity: "" });
// export const localParticipant = mockLocalParticipant({ identity: "" });
export const localId = `${local.userId}:${localRtcMember.deviceId}`;
export const aliceRtcMember = mockRtcMembership("@alice:example.org", "AAAA");

11
src/utils/test-viewmodel.ts
View File

@@ -5,10 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
type CallMembership,
type MatrixRTCSession,
} from "matrix-js-sdk/lib/matrixrtc";
import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc";
import { BehaviorSubject, of } from "rxjs";
import { vitest } from "vitest";
import { type RelationsContainer } from "matrix-js-sdk/lib/models/relations-container";
@@ -99,12 +96,12 @@ export function getBasicRTCSession(
initialRtcMemberships,
);
const rtcSession = new MockRTCSession(matrixRoom).withMemberships(
const fakeRtcSession = new MockRTCSession(matrixRoom).withMemberships(
rtcMemberships$,
);
return {
rtcSession,
rtcSession: fakeRtcSession,
matrixRoom,
rtcMemberships$,
};
@@ -137,7 +134,7 @@ export function getBasicCallViewModelEnvironment(
// const remoteParticipants$ = of([aliceParticipant]);
const vm = new CallViewModel(
rtcSession as unknown as MatrixRTCSession,
rtcSession.asMockedSession(),
matrixRoom,
mockMediaDevices({}),
mockMuteStates(),

43
src/utils/test.ts
View File

@@ -6,7 +6,7 @@ Please see LICENSE in the repository root for full details.
*/
import { map, type Observable, of, type SchedulerLike } from "rxjs";
import { type RunHelpers, TestScheduler } from "rxjs/testing";
import { expect, vi, vitest } from "vitest";
import { expect, type MockedObject, vi, vitest } from "vitest";
import {
type RoomMember,
type Room as MatrixRoom,
@@ -23,11 +23,13 @@ import {
type SessionMembershipData,
Status,
type LivekitFocusSelection,
type MatrixRTCSession,
} from "matrix-js-sdk/lib/matrixrtc";
import { type MembershipManagerEventHandlerMap } from "matrix-js-sdk/lib/matrixrtc/IMembershipManager";
import {
type LocalParticipant,
type LocalTrackPublication,
type Participant,
type RemoteParticipant,
type RemoteTrackPublication,
type Room as LivekitRoom,
@@ -191,8 +193,12 @@ export function mockRtcMembership(
const event = new MatrixEvent({
sender: typeof user === "string" ? user : user.userId,
event_id: `$-ev-${randomUUID()}:example.org`,
content: data,
});
return new CallMembership(event, data);
const cms = new CallMembership(event);
vi.mocked(cms).getTransport = vi.fn().mockReturnValue(fociPreferred[0]);
return cms;
}
// Maybe it'd be good to move this to matrix-js-sdk? Our testing needs are
@@ -205,6 +211,10 @@ export function mockMatrixRoomMember(
return {
...mockEmitter(),
userId: rtcMembership.sender,
getMxcAvatarUrl(): string | undefined {
return undefined;
},
rawDisplayName: rtcMembership.sender,
...member,
} as RoomMember;
}
@@ -331,6 +341,19 @@ export class MockRTCSession extends TypedEventEmitter<
RoomAndToDeviceEventsHandlerMap &
MembershipManagerEventHandlerMap
> {
public asMockedSession(): MockedObject<MatrixRTCSession> {
const session = this as unknown as MockedObject<MatrixRTCSession>;
vi.mocked(session).reemitEncryptionKeys = vi
.fn<() => void>()
.mockReturnValue(undefined);
vi.mocked(session).getOldestMembership = vi
.fn<() => CallMembership | undefined>()
.mockReturnValue(this.memberships[0]);
return session;
}
public readonly statistics = {
counters: {},
};
@@ -389,15 +412,17 @@ export class MockRTCSession extends TypedEventEmitter<
}
}
export const mockTrack = (identity: string): TrackReference =>
export const mockTrack = (
participant: Participant,
kind?: Track.Kind,
source?: Track.Source,
): TrackReference =>
({
participant: {
identity,
},
participant,
publication: {
kind: Track.Kind.Audio,
source: "mic",
trackSid: "123",
kind: kind ?? Track.Kind.Audio,
source: source ?? Track.Source.Microphone,
trackSid: `123##${participant.identity}`,
track: {
attach: vi.fn(),
detach: vi.fn(),

45
yarn.lock
View File

@@ -5176,6 +5176,13 @@ __metadata:
languageName: node
linkType: hard
"@types/glob-to-regexp@npm:^0.4.4":
version: 0.4.4
resolution: "@types/glob-to-regexp@npm:0.4.4"
checksum: 10c0/7288ff853850d8302a8770a3698b187fc3970ad12ee6427f0b3758a3e7a0ebb0bd993abc6ebaaa979d09695b4194157d2bfaa7601b0fb9ed72c688b4c1298b88
languageName: node
linkType: hard
"@types/grecaptcha@npm:^3.0.9":
version: 3.0.9
resolution: "@types/grecaptcha@npm:3.0.9"
@@ -7528,6 +7535,7 @@ __metadata:
eslint-plugin-react-hooks: "npm:^5.0.0"
eslint-plugin-rxjs: "npm:^5.0.3"
eslint-plugin-unicorn: "npm:^56.0.0"
fetch-mock: "npm:11.1.5"
global-jsdom: "npm:^26.0.0"
i18next: "npm:^24.0.0"
i18next-browser-languagedetector: "npm:^8.0.0"
@@ -8495,6 +8503,22 @@ __metadata:
languageName: node
linkType: hard
"fetch-mock@npm:11.1.5":
version: 11.1.5
resolution: "fetch-mock@npm:11.1.5"
dependencies:
"@types/glob-to-regexp": "npm:^0.4.4"
dequal: "npm:^2.0.3"
glob-to-regexp: "npm:^0.4.1"
is-subset: "npm:^0.1.1"
regexparam: "npm:^3.0.0"
peerDependenciesMeta:
node-fetch:
optional: true
checksum: 10c0/f32f1d7879b654a3fab7c3576901193ddd4c63cb9aeae2ed66ff42062400c0937d4696b1a5171e739d5f62470e6554e190f14816789f5e3b2bf1ad90208222e6
languageName: node
linkType: hard
"fflate@npm:^0.4.8":
version: 0.4.8
resolution: "fflate@npm:0.4.8"
@@ -8876,6 +8900,13 @@ __metadata:
languageName: node
linkType: hard
"glob-to-regexp@npm:^0.4.1":
version: 0.4.1
resolution: "glob-to-regexp@npm:0.4.1"
checksum: 10c0/0486925072d7a916f052842772b61c3e86247f0a80cc0deb9b5a3e8a1a9faad5b04fb6f58986a09f34d3e96cd2a22a24b7e9882fb1cf904c31e9a310de96c429
languageName: node
linkType: hard
"glob@npm:^10.2.2, glob@npm:^10.3.10, glob@npm:^10.3.7, glob@npm:^10.4.1":
version: 10.4.5
resolution: "glob@npm:10.4.5"
@@ -9611,6 +9642,13 @@ __metadata:
languageName: node
linkType: hard
"is-subset@npm:^0.1.1":
version: 0.1.1
resolution: "is-subset@npm:0.1.1"
checksum: 10c0/d8125598ab9077a76684e18726fb915f5cea7a7358ed0c6ff723f4484d71a0a9981ee5aae06c44de99cfdef0fefce37438c6257ab129e53c82045ea0c2acdebf
languageName: node
linkType: hard
"is-symbol@npm:^1.0.4, is-symbol@npm:^1.1.1":
version: 1.1.1
resolution: "is-symbol@npm:1.1.1"
@@ -12043,6 +12081,13 @@ __metadata:
languageName: node
linkType: hard
"regexparam@npm:^3.0.0":
version: 3.0.0
resolution: "regexparam@npm:3.0.0"
checksum: 10c0/a6430d7b97d5a7d5518f37a850b6b73aab479029d02f46af4fa0e8e4a1d7aad05b7a0d2d10c86ded21a14d5f0fa4c68525f873a5fca2efeefcccd93c36627459
languageName: node
linkType: hard
"regexpu-core@npm:^6.2.0":
version: 6.2.0
resolution: "regexpu-core@npm:6.2.0"