Matrix/element-call-Github
1
0
Fork 0
mirror of https://github.com/vector-im/element-call.git synced 2026-03-16 06:17:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'voip-team/rebased-multiSFU' of github.com:element-hq/element-call into voip-team/rebased-multiSFU

Browse Source
This commit is contained in:
Robin
2025-10-14 09:22:08 -04:00
parent a18700cbcd 93d763f58f
commit 392c51fbf6
51 changed files with 2340 additions and 1067 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
package.json
View File

@@ -99,6 +99,7 @@
"eslint-plugin-react-hooks": "^5.0.0",
"eslint-plugin-rxjs": "^5.0.3",
"eslint-plugin-unicorn": "^56.0.0",
"fetch-mock": "11.1.5",
"global-jsdom": "^26.0.0",
"i18next": "^24.0.0",
"i18next-browser-languagedetector": "^8.0.0",
@@ -108,7 +109,7 @@
"livekit-client": "^2.13.0",
"lodash-es": "^4.17.21",
"loglevel": "^1.9.1",
"matrix-js-sdk": "github:matrix-org/matrix-js-sdk#head=toger5/use-relation-based-CallMembership-create-ts",
"matrix-js-sdk": "github:matrix-org/matrix-js-sdk#head=develop",
"matrix-widget-api": "^1.13.0",
"normalize.css": "^8.0.1",
"observable-hooks": "^4.2.3",

8
src/MediaDevicesContext.ts
View File

@@ -23,14 +23,6 @@ export function useMediaDevices(): MediaDevices {
return mediaDevices;
}
export const useIsEarpiece = (): boolean => {
const devices = useMediaDevices();
const audioOutput = useObservableEagerState(devices.audioOutput.selected$);
const available = useObservableEagerState(devices.audioOutput.available$);
if (!audioOutput?.id) return false;
return available.get(audioOutput.id)?.type === "earpiece";
};
/**
* A convenience hook to get the audio node configuration for the earpiece.
* It will check the `useAsEarpiece` of the `audioOutput` device and return

25
src/RTCConnectionStats.tsx
View File

@@ -19,10 +19,26 @@ import mediaViewStyles from "../src/tile/MediaView.module.css";
interface Props {
audio?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats;
video?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats;
focusUrl?: string;
}
const extractDomain = (url: string): string => {
try {
const parsedUrl = new URL(url);
return parsedUrl.hostname; // Returns "kdk.cpm"
} catch (error) {
console.error("Invalid URL:", error);
return url;
}
};
// This is only used in developer mode for debugging purposes, so we don't need full localization
export const RTCConnectionStats: FC<Props> = ({ audio, video, ...rest }) => {
export const RTCConnectionStats: FC<Props> = ({
audio,
video,
focusUrl,
...rest
}) => {
const [showModal, setShowModal] = useState(false);
const [modalContents, setModalContents] = useState<
"video" | "audio" | "none"
@@ -55,6 +71,13 @@ export const RTCConnectionStats: FC<Props> = ({ audio, video, ...rest }) => {
</pre>
</div>
</Modal>
{focusUrl && (
<div>
<Text as="span" size="xs" title="focusURL">
&nbsp;{extractDomain(focusUrl)}
</Text>
</div>
)}
{audio && (
<div>
<Button

3
src/button/ReactionToggleButton.test.tsx
View File

@@ -10,7 +10,6 @@ import { expect, test } from "vitest";
import { TooltipProvider } from "@vector-im/compound-web";
import { userEvent } from "@testing-library/user-event";
import { type ReactNode } from "react";
import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc";
import { ReactionToggleButton } from "./ReactionToggleButton";
import { ElementCallReactionEventType } from "../reactions";
@@ -33,7 +32,7 @@ function TestComponent({
<TooltipProvider>
<ReactionsSenderProvider
vm={vm}
rtcSession={rtcSession as unknown as MatrixRTCSession}
rtcSession={rtcSession.asMockedSession()}
>
<ReactionToggleButton vm={vm} identifier={localIdent} />
</ReactionsSenderProvider>

2
src/grid/GridLayout.tsx
View File

@@ -14,7 +14,7 @@ import {
import { distinctUntilChanged } from "rxjs";
import { useObservableEagerState } from "observable-hooks";
import { type GridLayout as GridLayoutModel } from "../state/CallViewModel";
import { type GridLayout as GridLayoutModel } from "../state/layout-types.ts";
import styles from "./GridLayout.module.css";
import { useInitial } from "../useInitial";
import { type CallLayout, arrangeTiles } from "./CallLayout";

2
src/grid/OneOnOneLayout.tsx
View File

@@ -9,7 +9,7 @@ import { type ReactNode, useCallback, useMemo } from "react";
import { useObservableEagerState } from "observable-hooks";
import classNames from "classnames";
import { type OneOnOneLayout as OneOnOneLayoutModel } from "../state/CallViewModel";
import { type OneOnOneLayout as OneOnOneLayoutModel } from "../state/layout-types.ts";
import { type CallLayout, arrangeTiles } from "./CallLayout";
import styles from "./OneOnOneLayout.module.css";
import { type DragCallback, useUpdateLayout } from "./Grid";

2
src/grid/SpotlightExpandedLayout.tsx
View File

@@ -7,7 +7,7 @@ Please see LICENSE in the repository root for full details.
import { type ReactNode, useCallback } from "react";
import { type SpotlightExpandedLayout as SpotlightExpandedLayoutModel } from "../state/CallViewModel";
import { type SpotlightExpandedLayout as SpotlightExpandedLayoutModel } from "../state/layout-types.ts";
import { type CallLayout } from "./CallLayout";
import { type DragCallback, useUpdateLayout } from "./Grid";
import styles from "./SpotlightExpandedLayout.module.css";

2
src/grid/SpotlightLandscapeLayout.tsx
View File

@@ -10,7 +10,7 @@ import { useObservableEagerState } from "observable-hooks";
import classNames from "classnames";
import { type CallLayout } from "./CallLayout";
import { type SpotlightLandscapeLayout as SpotlightLandscapeLayoutModel } from "../state/CallViewModel";
import { type SpotlightLandscapeLayout as SpotlightLandscapeLayoutModel } from "../state/layout-types.ts";
import styles from "./SpotlightLandscapeLayout.module.css";
import { useUpdateLayout, useVisibleTiles } from "./Grid";

2
src/grid/SpotlightPortraitLayout.tsx
View File

@@ -10,7 +10,7 @@ import { useObservableEagerState } from "observable-hooks";
import classNames from "classnames";
import { type CallLayout, arrangeTiles } from "./CallLayout";
import { type SpotlightPortraitLayout as SpotlightPortraitLayoutModel } from "../state/CallViewModel";
import { type SpotlightPortraitLayout as SpotlightPortraitLayoutModel } from "../state/layout-types.ts";
import styles from "./SpotlightPortraitLayout.module.css";
import { useUpdateLayout, useVisibleTiles } from "./Grid";
import { useBehavior } from "../useBehavior";

4
src/home/useGroupCallRooms.ts
View File

@@ -141,8 +141,8 @@ export function useGroupCallRooms(client: MatrixClient): GroupCallRoom[] {
.filter(roomIsJoinable);
const sortedRooms = sortRooms(client, rooms);
Promise.all(
sortedRooms.map(async (room) => {
const session = await client.matrixRTC.getRoomSession(room);
sortedRooms.map((room) => {
const session = client.matrixRTC.getRoomSession(room);
return {
roomAlias: room.getCanonicalAlias() ?? undefined,
roomName: room.name,

224
src/livekit/MatrixAudioRenderer.test.tsx
View File

@@ -6,20 +6,28 @@ Please see LICENSE in the repository root for full details.
*/
import { afterEach, beforeEach, expect, it, vi } from "vitest";
import { render } from "@testing-library/react";
import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc";
import { render, type RenderResult } from "@testing-library/react";
import {
getTrackReferenceId,
type TrackReference,
} from "@livekit/components-core";
import { type RemoteAudioTrack } from "livekit-client";
import {
type Participant,
type RemoteAudioTrack,
type Room,
Track,
} from "livekit-client";
import { type ReactNode } from "react";
import { useTracks } from "@livekit/components-react";
import { testAudioContext } from "../useAudioContext.test";
import * as MediaDevicesContext from "../MediaDevicesContext";
import { LivekitRoomAudioRenderer } from "./MatrixAudioRenderer";
import { mockMediaDevices, mockTrack } from "../utils/test";
import {
mockMediaDevices,
mockRemoteParticipant,
mockTrack,
} from "../utils/test";
export const TestAudioContextConstructor = vi.fn(() => testAudioContext);
@@ -48,42 +56,203 @@ vi.mock("@livekit/components-react", async (importOriginal) => {
};
});
const tracks = [mockTrack("test:123")];
vi.mocked(useTracks).mockReturnValue(tracks);
let tracks: TrackReference[] = [];
it("should render for member", () => {
const { container, queryAllByTestId } = render(
/**
* Render the test component with given rtc members and livekit participant identities.
*
* It is possible to have rtc members that are not in livekit (e.g. not yet joined) and vice versa.
*
* @param rtcMembers - Array of active rtc members with userId and deviceId.
* @param livekitParticipantIdentities - Array of livekit participant (that are publishing).
* @param explicitTracks - Array of tracks available in livekit, if not provided, one audio track per livekitParticipantIdentities will be created.
* */
function renderTestComponent(
rtcMembers: { userId: string; deviceId: string }[],
livekitParticipantIdentities: string[],
explicitTracks?: {
participantId: string;
kind: Track.Kind;
source: Track.Source;
}[],
): RenderResult {
const liveKitParticipants = livekitParticipantIdentities.map((identity) =>
mockRemoteParticipant({ identity }),
);
const participants = rtcMembers.flatMap(({ userId, deviceId }) => {
const p = liveKitParticipants.find(
(p) => p.identity === `${userId}:${deviceId}`,
);
return p === undefined ? [] : [p];
});
const livekitRoom = {
remoteParticipants: new Map<string, Participant>(
liveKitParticipants.map((p) => [p.identity, p]),
),
} as unknown as Room;
if (explicitTracks?.length ?? 0 > 0) {
tracks = explicitTracks!.map(({ participantId, source, kind }) => {
const participant =
liveKitParticipants.find((p) => p.identity === participantId) ??
mockRemoteParticipant({ identity: participantId });
return mockTrack(participant, kind, source);
});
} else {
tracks = participants.map((p) => mockTrack(p));
}
vi.mocked(useTracks).mockReturnValue(tracks);
return render(
<MediaDevicesProvider value={mockMediaDevices({})}>
<LivekitRoomAudioRenderer
members={[{ sender: "test", deviceId: "123" }] as CallMembership[]}
validIdentities={participants.map((p) => p.identity)}
livekitRoom={livekitRoom}
url={""}
/>
</MediaDevicesProvider>,
);
}
it("should render for member", () => {
const { container, queryAllByTestId } = renderTestComponent(
[{ userId: "@alice", deviceId: "DEV0" }],
["@alice:DEV0"],
);
expect(container).toBeTruthy();
expect(queryAllByTestId("audio")).toHaveLength(1);
});
it("should not render without member", () => {
const memberships = [
{ sender: "othermember", deviceId: "123" },
] as CallMembership[];
const { container, queryAllByTestId } = render(
<MediaDevicesProvider value={mockMediaDevices({})}>
<LivekitRoomAudioRenderer members={memberships} />
</MediaDevicesProvider>,
const { container, queryAllByTestId } = renderTestComponent(
[{ userId: "@bob", deviceId: "DEV0" }],
["@alice:DEV0"],
);
expect(container).toBeTruthy();
expect(queryAllByTestId("audio")).toHaveLength(0);
});
const TEST_CASES: {
name: string;
rtcUsers: { userId: string; deviceId: string }[];
livekitParticipantIdentities: string[];
explicitTracks?: {
participantId: string;
kind: Track.Kind;
source: Track.Source;
}[];
expectedAudioTracks: number;
}[] = [
{
name: "single user single device",
rtcUsers: [
{ userId: "@alice", deviceId: "DEV0" },
{ userId: "@alice", deviceId: "DEV1" },
{ userId: "@bob", deviceId: "DEV0" },
],
livekitParticipantIdentities: ["@alice:DEV0", "@bob:DEV0", "@alice:DEV1"],
expectedAudioTracks: 3,
},
// Charlie is a rtc member but not in livekit
{
name: "Charlie is rtc member but not in livekit",
rtcUsers: [
{ userId: "@alice", deviceId: "DEV0" },
{ userId: "@bob", deviceId: "DEV0" },
{ userId: "@charlie", deviceId: "DEV0" },
],
livekitParticipantIdentities: ["@alice:DEV0", "@bob:DEV0"],
expectedAudioTracks: 2,
},
// Charlie is in livekit but not rtc member
{
name: "Charlie is in livekit but not rtc member",
rtcUsers: [
{ userId: "@alice", deviceId: "DEV0" },
{ userId: "@bob", deviceId: "DEV0" },
],
livekitParticipantIdentities: ["@alice:DEV0", "@bob:DEV0", "@charlie:DEV0"],
expectedAudioTracks: 2,
},
{
name: "no audio track, only video track",
rtcUsers: [{ userId: "@alice", deviceId: "DEV0" }],
livekitParticipantIdentities: ["@alice:DEV0"],
explicitTracks: [
{
participantId: "@alice:DEV0",
kind: Track.Kind.Video,
source: Track.Source.Camera,
},
],
expectedAudioTracks: 0,
},
{
name: "Audio track from unknown source",
rtcUsers: [{ userId: "@alice", deviceId: "DEV0" }],
livekitParticipantIdentities: ["@alice:DEV0"],
explicitTracks: [
{
participantId: "@alice:DEV0",
kind: Track.Kind.Audio,
source: Track.Source.Unknown,
},
],
expectedAudioTracks: 1,
},
{
name: "Audio track from other device",
rtcUsers: [{ userId: "@alice", deviceId: "DEV0" }],
livekitParticipantIdentities: ["@alice:DEV0"],
explicitTracks: [
{
participantId: "@alice:DEV1",
kind: Track.Kind.Audio,
source: Track.Source.Microphone,
},
],
expectedAudioTracks: 0,
},
{
name: "two audio tracks, microphone and screenshare",
rtcUsers: [{ userId: "@alice", deviceId: "DEV0" }],
livekitParticipantIdentities: ["@alice:DEV0"],
explicitTracks: [
{
participantId: "@alice:DEV0",
kind: Track.Kind.Audio,
source: Track.Source.Microphone,
},
{
participantId: "@alice:DEV0",
kind: Track.Kind.Audio,
source: Track.Source.ScreenShareAudio,
},
],
expectedAudioTracks: 2,
},
];
it.each(TEST_CASES)(
`should render sound test cases $name`,
({
rtcUsers,
livekitParticipantIdentities,
explicitTracks,
expectedAudioTracks,
}) => {
const { queryAllByTestId } = renderTestComponent(
rtcUsers,
livekitParticipantIdentities,
explicitTracks,
);
expect(queryAllByTestId("audio")).toHaveLength(expectedAudioTracks);
},
);
it("should not setup audioContext gain and pan if there is no need to.", () => {
render(
<MediaDevicesProvider value={mockMediaDevices({})}>
<LivekitRoomAudioRenderer
members={[{ sender: "test", deviceId: "123" }] as CallMembership[]}
/>
</MediaDevicesProvider>,
);
renderTestComponent([{ userId: "@bob", deviceId: "DEV0" }], ["@bob:DEV0"]);
const audioTrack = tracks[0].publication.track! as RemoteAudioTrack;
expect(audioTrack.setAudioContext).toHaveBeenCalledTimes(1);
@@ -100,13 +269,8 @@ it("should setup audioContext gain and pan", () => {
pan: 1,
volume: 0.1,
});
render(
<MediaDevicesProvider value={mockMediaDevices({})}>
<LivekitRoomAudioRenderer
members={[{ sender: "test", deviceId: "123" }] as CallMembership[]}
/>
</MediaDevicesProvider>,
);
renderTestComponent([{ userId: "@bob", deviceId: "DEV0" }], ["@bob:DEV0"]);
const audioTrack = tracks[0].publication.track! as RemoteAudioTrack;
expect(audioTrack.setAudioContext).toHaveBeenCalled();

86
src/livekit/MatrixAudioRenderer.tsx
View File

@@ -6,21 +6,21 @@ Please see LICENSE in the repository root for full details.
*/
import { getTrackReferenceId } from "@livekit/components-core";
import { type Room as LivekitRoom, type Participant } from "livekit-client";
import { type Room as LivekitRoom } from "livekit-client";
import { type RemoteAudioTrack, Track } from "livekit-client";
import { useEffect, useMemo, useRef, useState, type ReactNode } from "react";
import { useEffect, useMemo, useState, type ReactNode } from "react";
import {
useTracks,
AudioTrack,
type AudioTrackProps,
} from "@livekit/components-react";
import { type RoomMember } from "matrix-js-sdk";
import { logger } from "matrix-js-sdk/lib/logger";
import { type ParticipantId } from "matrix-js-sdk/lib/matrixrtc";
import { useEarpieceAudioConfig } from "../MediaDevicesContext";
import { useReactiveState } from "../useReactiveState";
import * as controls from "../controls";
import {} from "@livekit/components-core";
export interface MatrixAudioRendererProps {
/**
* The service URL of the LiveKit room.
@@ -28,14 +28,11 @@ export interface MatrixAudioRendererProps {
url: string;
livekitRoom: LivekitRoom;
/**
* The list of participants to render audio for.
* The list of participant identities to render audio for.
* This list needs to be composed based on the matrixRTC members so that we do not play audio from users
* that are not expected to be in the rtc session.
* that are not expected to be in the rtc session (local user is excluded).
*/
participants: {
participant: Participant;
member: RoomMember;
}[];
validIdentities: ParticipantId[];
/**
* If set to `true`, mutes all audio tracks rendered by the component.
* @remarks
@@ -44,9 +41,9 @@ export interface MatrixAudioRendererProps {
muted?: boolean;
}
const prefixedLogger = logger.getChild("[MatrixAudioRenderer]");
/**
* The `MatrixAudioRenderer` component is a drop-in solution for adding audio to your LiveKit app.
* It takes care of handling remote participants’ audio tracks and makes sure that microphones and screen share are audible.
* Takes care of handling remote participants’ audio tracks and makes sure that microphones and screen share are audible.
*
* It also takes care of the earpiece audio configuration for iOS devices.
* This is done by using the WebAudio API to create a stereo pan effect that mimics the earpiece audio.
@@ -61,33 +58,9 @@ export interface MatrixAudioRendererProps {
export function LivekitRoomAudioRenderer({
url,
livekitRoom,
participants,
validIdentities,
muted,
}: MatrixAudioRendererProps): ReactNode {
const participantSet = useMemo(
() => new Set(participants.map(({ participant }) => participant)),
[participants],
);
const loggedInvalidIdentities = useRef(new Set<string>());
/**
* Log an invalid livekit track identity.
* A invalid identity is one that does not match any of the matrix rtc members.
*
* @param identity The identity of the track that is invalid
* @param validIdentities The list of valid identities
*/
const logInvalid = (identity: string): void => {
if (loggedInvalidIdentities.current.has(identity)) return;
logger.warn(
`[MatrixAudioRenderer] Audio track ${identity} from ${url} has no matching matrix call member`,
`current members: ${participants.map((p) => p.participant.identity)}`,
`track will not get rendered`,
);
loggedInvalidIdentities.current.add(identity);
};
const tracks = useTracks(
[
Track.Source.Microphone,
@@ -99,28 +72,23 @@ export function LivekitRoomAudioRenderer({
onlySubscribed: true,
room: livekitRoom,
},
).filter((ref) => {
const isValid = participantSet?.has(ref.participant);
if (!isValid && !ref.participant.isLocal)
logInvalid(ref.participant.identity);
return (
!ref.participant.isLocal &&
ref.publication.kind === Track.Kind.Audio &&
isValid
);
});
useEffect(() => {
if (
loggedInvalidIdentities.current.size &&
tracks.every((t) => participantSet.has(t.participant))
) {
logger.debug(
`[MatrixAudioRenderer] All audio tracks from ${url} have a matching matrix call member identity.`,
);
loggedInvalidIdentities.current.clear();
}
}, [tracks, participantSet, url]);
)
// Only keep audio tracks
.filter((ref) => ref.publication.kind === Track.Kind.Audio)
// Only keep tracks from participants that are in the validIdentities list
.filter((ref) => {
const isValid = validIdentities.includes(ref.participant.identity);
if (!isValid) {
// Log that there is an invalid identity, that means that someone is publishing audio that is not expected to be in the call.
prefixedLogger.warn(
`Audio track ${ref.participant.identity} from ${url} has no matching matrix call member`,
`current members: ${validIdentities.join()}`,
`track will not get rendered`,
);
return false;
}
return true;
});
// This component is also (in addition to the "only play audio for connected members" logic above)
// responsible for mimicking earpiece audio on iPhones.

6
src/main.tsx
View File

@@ -60,9 +60,9 @@ if (fatalError !== null) {
Initializer.initBeforeReact()
.then(() => {
root.render(
// <StrictMode>
<App vm={new AppViewModel()} />,
// </StrictMode>,
<StrictMode>
<App vm={new AppViewModel()} />,
</StrictMode>,
);
})
.catch((e) => {

21
src/reactions/ReactionsReader.test.tsx
View File

@@ -7,7 +7,6 @@ Please see LICENSE in the repository root for full details.
import { renderHook } from "@testing-library/react";
import { afterEach, test, vitest } from "vitest";
import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc";
import {
RoomEvent as MatrixRoomEvent,
MatrixEvent,
@@ -38,7 +37,7 @@ test("handles a hand raised reaction", () => {
withTestScheduler(({ schedule, expectObservable }) => {
renderHook(() => {
const { raisedHands$ } = new ReactionsReader(
rtcSession as unknown as MatrixRTCSession,
rtcSession.asMockedSession(),
);
schedule("ab", {
a: () => {},
@@ -86,7 +85,7 @@ test("handles a redaction", () => {
withTestScheduler(({ schedule, expectObservable }) => {
renderHook(() => {
const { raisedHands$ } = new ReactionsReader(
rtcSession as unknown as MatrixRTCSession,
rtcSession.asMockedSession(),
);
schedule("abc", {
a: () => {},
@@ -149,7 +148,7 @@ test("handles waiting for event decryption", () => {
withTestScheduler(({ schedule, expectObservable }) => {
renderHook(() => {
const { raisedHands$ } = new ReactionsReader(
rtcSession as unknown as MatrixRTCSession,
rtcSession.asMockedSession(),
);
schedule("abc", {
a: () => {},
@@ -218,7 +217,7 @@ test("hands rejecting events without a proper membership", () => {
withTestScheduler(({ schedule, expectObservable }) => {
renderHook(() => {
const { raisedHands$ } = new ReactionsReader(
rtcSession as unknown as MatrixRTCSession,
rtcSession.asMockedSession(),
);
schedule("ab", {
a: () => {},
@@ -262,9 +261,7 @@ test("handles a reaction", () => {
withTestScheduler(({ schedule, time, expectObservable }) => {
renderHook(() => {
const { reactions$ } = new ReactionsReader(
rtcSession as unknown as MatrixRTCSession,
);
const { reactions$ } = new ReactionsReader(rtcSession.asMockedSession());
schedule(`abc`, {
a: () => {},
b: () => {
@@ -320,9 +317,7 @@ test("ignores bad reaction events", () => {
withTestScheduler(({ schedule, expectObservable }) => {
renderHook(() => {
const { reactions$ } = new ReactionsReader(
rtcSession as unknown as MatrixRTCSession,
);
const { reactions$ } = new ReactionsReader(rtcSession.asMockedSession());
schedule("ab", {
a: () => {},
b: () => {
@@ -444,9 +439,7 @@ test("that reactions cannot be spammed", () => {
withTestScheduler(({ schedule, expectObservable }) => {
renderHook(() => {
const { reactions$ } = new ReactionsReader(
rtcSession as unknown as MatrixRTCSession,
);
const { reactions$ } = new ReactionsReader(rtcSession.asMockedSession());
schedule("abcd", {
a: () => {},
b: () => {

3
src/room/CallEventAudioRenderer.test.tsx
View File

@@ -155,7 +155,8 @@ test("plays one sound when a hand is raised", () => {
act(() => {
handRaisedSubject$.next({
[bobRtcMember.callId]: {
// TODO: What is this string supposed to be?
[`${bobRtcMember.sender}:${bobRtcMember.deviceId}`]: {
time: new Date(),
membershipEventId: "",
reactionEventId: "",

2
src/room/GroupCallErrorBoundary.test.tsx
View File

@@ -106,7 +106,7 @@ test("should render the error page with link back to home", async () => {
await screen.findByText("Call is not supported");
expect(screen.getByText(/Domain: example\.com/i)).toBeInTheDocument();
expect(
screen.getByText(/Error Code: MISSING_MATRIX_RTC_FOCUS/i),
screen.getByText(/Error Code: MISSING_MATRIX_RTC_TRANSPORT/i),
).toBeInTheDocument();
await screen.findByRole("button", { name: "Return to home screen" });

15
src/room/GroupCallView.test.tsx
View File

@@ -26,7 +26,6 @@ import { type RelationsContainer } from "matrix-js-sdk/lib/models/relations-cont
import { useState } from "react";
import { TooltipProvider } from "@vector-im/compound-web";
import { type MuteStates } from "./MuteStates";
import { prefetchSounds } from "../soundUtils";
import { useAudioContext } from "../useAudioContext";
import { ActiveCall } from "./InCallView";
@@ -47,6 +46,7 @@ import { ProcessorProvider } from "../livekit/TrackProcessorContext";
import { MediaDevicesContext } from "../MediaDevicesContext";
import { HeaderStyle } from "../UrlParams";
import { constant } from "../state/Behavior";
import { type MuteStates } from "../state/MuteStates.ts";
vi.mock("../soundUtils");
vi.mock("../useAudioContext");
@@ -117,7 +117,7 @@ function createGroupCallView(
widget: WidgetHelpers | null,
joined = true,
): {
rtcSession: MockRTCSession;
rtcSession: MatrixRTCSession;
getByText: ReturnType<typeof render>["getByText"];
} {
const client = {
@@ -150,7 +150,8 @@ function createGroupCallView(
const muteState = {
audio: { enabled: false },
video: { enabled: false },
} as MuteStates;
// TODO-MULTI-SFU: This cast isn't valid, it's likely the cause of some current test failures
} as unknown as MuteStates;
const { getByText } = render(
<BrowserRouter>
<TooltipProvider>
@@ -163,10 +164,12 @@ function createGroupCallView(
preload={false}
skipLobby={false}
header={HeaderStyle.Standard}
rtcSession={rtcSession as unknown as MatrixRTCSession}
isJoined={joined}
rtcSession={rtcSession.asMockedSession()}
muteStates={muteState}
widget={widget}
// TODO-MULTI-SFU: Make joined and setJoined work
joined={true}
setJoined={function (value: boolean): void {}}
/>
</ProcessorProvider>
</MediaDevicesContext>
@@ -175,7 +178,7 @@ function createGroupCallView(
);
return {
getByText,
rtcSession,
rtcSession: rtcSession.asMockedSession(),
};
}

17
src/room/InCallView.test.tsx
View File

@@ -15,7 +15,6 @@ import {
} from "vitest";
import { act, render, type RenderResult } from "@testing-library/react";
import { type MatrixClient, JoinRule, type RoomState } from "matrix-js-sdk";
import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc";
import { type RelationsContainer } from "matrix-js-sdk/lib/models/relations-container";
import { type LocalParticipant } from "livekit-client";
import { of } from "rxjs";
@@ -24,7 +23,6 @@ import { TooltipProvider } from "@vector-im/compound-web";
import { RoomContext, useLocalParticipant } from "@livekit/components-react";
import { RoomAndToDeviceEvents } from "matrix-js-sdk/lib/matrixrtc/RoomAndToDeviceKeyTransport";
import { type MuteStates } from "./MuteStates";
import { InCallView } from "./InCallView";
import {
mockLivekitRoom,
@@ -32,6 +30,7 @@ import {
mockMatrixRoom,
mockMatrixRoomMember,
mockMediaDevices,
mockMuteStates,
mockRemoteParticipant,
mockRtcMembership,
type MockRTCSession,
@@ -133,10 +132,7 @@ function createInCallView(): RenderResult & {
} as Partial<RoomState> as RoomState,
});
const muteState = {
audio: { enabled: false },
video: { enabled: false },
} as MuteStates;
const muteState = mockMuteStates();
const livekitRoom = mockLivekitRoom(
{
localParticipant,
@@ -153,14 +149,14 @@ function createInCallView(): RenderResult & {
<MediaDevicesContext value={mockMediaDevices({})}>
<ReactionsSenderProvider
vm={vm}
rtcSession={rtcSession as unknown as MatrixRTCSession}
rtcSession={rtcSession.asMockedSession()}
>
<TooltipProvider>
<RoomContext value={livekitRoom}>
<InCallView
client={client}
header={HeaderStyle.Standard}
rtcSession={rtcSession as unknown as MatrixRTCSession}
rtcSession={rtcSession.asMockedSession()}
muteStates={muteState}
vm={vm}
matrixInfo={{
@@ -176,11 +172,6 @@ function createInCallView(): RenderResult & {
},
}}
matrixRoom={room}
livekitRoom={livekitRoom}
participantCount={0}
onLeft={function (): void {
throw new Error("Function not implemented.");
}}
onShareClick={null}
/>
</RoomContext>

24
src/room/InCallView.tsx
View File

@@ -23,7 +23,7 @@ import useMeasure from "react-use-measure";
import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc";
import classNames from "classnames";
import { BehaviorSubject, map } from "rxjs";
import { useObservable, useObservableEagerState } from "observable-hooks";
import { useObservable } from "observable-hooks";
import { logger } from "matrix-js-sdk/lib/logger";
import { RoomAndToDeviceEvents } from "matrix-js-sdk/lib/matrixrtc/RoomAndToDeviceKeyTransport";
import {
@@ -31,7 +31,6 @@ import {
VolumeOnSolidIcon,
} from "@vector-im/compound-design-tokens/assets/web/icons";
import { useTranslation } from "react-i18next";
import { ConnectionState } from "livekit-client";
import LogoMark from "../icons/LogoMark.svg?react";
import LogoType from "../icons/LogoType.svg?react";
@@ -60,11 +59,7 @@ import { type MuteStates } from "../state/MuteStates";
import { type MatrixInfo } from "./VideoPreview";
import { InviteButton } from "../button/InviteButton";
import { LayoutToggle } from "./LayoutToggle";
import {
CallViewModel,
type GridMode,
type Layout,
} from "../state/CallViewModel";
import { CallViewModel, type GridMode } from "../state/CallViewModel";
import { Grid, type TileProps } from "../grid/Grid";
import { useInitial } from "../useInitial";
import { SpotlightTile } from "../tile/SpotlightTile";
@@ -113,8 +108,8 @@ import { prefetchSounds } from "../soundUtils";
import { useAudioContext } from "../useAudioContext";
import ringtoneMp3 from "../sound/ringtone.mp3?url";
import ringtoneOgg from "../sound/ringtone.ogg?url";
import { ConnectionLostError } from "../utils/errors.ts";
import { useTrackProcessorObservable$ } from "../livekit/TrackProcessorContext.tsx";
import { type Layout } from "../state/layout-types.ts";
const maxTapDurationMs = 400;
@@ -207,7 +202,8 @@ export const InCallView: FC<InCallViewProps> = ({
useReactionsSender();
useWakeLock();
const connectionState = useObservableEagerState(vm.livekitConnectionState$);
// TODO-MULTI-SFU This is unused now??
// const connectionState = useObservableEagerState(vm.livekitConnectionState$);
// annoyingly we don't get the disconnection reason this way,
// only by listening for the emitted event
@@ -287,7 +283,7 @@ export const InCallView: FC<InCallViewProps> = ({
);
const allLivekitRooms = useBehavior(vm.allLivekitRooms$);
const participantsByRoom = useBehavior(vm.participantsByRoom$);
const audioParticipants = useBehavior(vm.audioParticipants$);
const participantCount = useBehavior(vm.participantCount$);
const reconnecting = useBehavior(vm.reconnecting$);
const windowMode = useBehavior(vm.windowMode$);
@@ -301,6 +297,10 @@ export const InCallView: FC<InCallViewProps> = ({
const audioOutputSwitcher = useBehavior(vm.audioOutputSwitcher$);
const sharingScreen = useBehavior(vm.sharingScreen$);
const fatalCallError = useBehavior(vm.configError$);
// Stop the rendering and throw for the error boundary
if (fatalCallError) throw fatalCallError;
// We need to set the proper timings on the animation based upon the sound length.
const ringDuration = pickupPhaseAudio?.soundDuration["waiting"] ?? 1;
useEffect((): (() => void) => {
@@ -861,12 +861,12 @@ export const InCallView: FC<InCallViewProps> = ({
</Text>
)
}
{participantsByRoom.map(({ livekitRoom, url, participants }) => (
{audioParticipants.map(({ livekitRoom, url, participants }) => (
<LivekitRoomAudioRenderer
key={url}
url={url}
livekitRoom={livekitRoom}
participants={participants}
validIdentities={participants.map((p) => p.identity)}
muted={muteAllAudio}
/>
))}

4
src/room/MuteStates.test.tsx
View File

@@ -5,6 +5,9 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
// TODO-MULTI-SFU: These tests need to be ported to the new MuteStates class.
/*
import {
afterAll,
afterEach,
@@ -321,3 +324,4 @@ describe("useMuteStates in VITE_PACKAGE='embedded' (widget) mode", () => {
expect(screen.getByTestId("video-enabled").textContent).toBe("true");
});
});
*/

178
src/room/MuteStates.ts
View File

@@ -1,178 +0,0 @@
/*
Copyright 2023, 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
type Dispatch,
type SetStateAction,
useCallback,
useEffect,
useMemo,
} from "react";
import { type IWidgetApiRequest } from "matrix-widget-api";
import { logger } from "matrix-js-sdk/lib/logger";
import { useObservableEagerState } from "observable-hooks";
import {
type DeviceLabel,
type SelectedDevice,
type MediaDevice,
} from "../state/MediaDevices";
import { useIsEarpiece, useMediaDevices } from "../MediaDevicesContext";
import { useReactiveState } from "../useReactiveState";
import { ElementWidgetActions, widget } from "../widget";
import { Config } from "../config/Config";
import { useUrlParams } from "../UrlParams";
/**
* If there already are this many participants in the call, we automatically mute
* the user.
*/
export const MUTE_PARTICIPANT_COUNT = 8;
interface DeviceAvailable {
enabled: boolean;
setEnabled: Dispatch<SetStateAction<boolean>>;
}
interface DeviceUnavailable {
enabled: false;
setEnabled: null;
}
const deviceUnavailable: DeviceUnavailable = {
enabled: false,
setEnabled: null,
};
type MuteState = DeviceAvailable | DeviceUnavailable;
export interface MuteStates {
audio: MuteState;
video: MuteState;
}
function useMuteState(
device: MediaDevice<DeviceLabel, SelectedDevice>,
enabledByDefault: () => boolean,
forceUnavailable: boolean = false,
): MuteState {
const available = useObservableEagerState(device.available$);
const [enabled, setEnabled] = useReactiveState<boolean | undefined>(
// Determine the default value once devices are actually connected
(prev) => prev ?? (available.size > 0 ? enabledByDefault() : undefined),
[available.size],
);
return useMemo(
() =>
available.size === 0 || forceUnavailable
? deviceUnavailable
: {
enabled: enabled ?? false,
setEnabled: setEnabled as Dispatch<SetStateAction<boolean>>,
},
[available.size, enabled, forceUnavailable, setEnabled],
);
}
export function useMuteStates(isJoined: boolean): MuteStates {
const devices = useMediaDevices();
const { skipLobby, defaultAudioEnabled, defaultVideoEnabled } =
useUrlParams();
const audio = useMuteState(
devices.audioInput,
() =>
(defaultAudioEnabled ?? Config.get().media_devices.enable_audio) &&
allowJoinUnmuted(skipLobby, isJoined),
);
useEffect(() => {
// If audio is enabled, we need to request the device names again,
// because iOS will not be able to switch to the correct device after un-muting.
// This is one of the main changes that makes iOS work with bluetooth audio devices.
if (audio.enabled) {
devices.requestDeviceNames();
}
}, [audio.enabled, devices]);
const isEarpiece = useIsEarpiece();
const video = useMuteState(
devices.videoInput,
() =>
(defaultVideoEnabled ?? Config.get().media_devices.enable_video) &&
allowJoinUnmuted(skipLobby, isJoined),
isEarpiece, // Force video to be unavailable if using earpiece
);
useEffect(() => {
widget?.api.transport
.send(ElementWidgetActions.DeviceMute, {
audio_enabled: audio.enabled,
video_enabled: video.enabled,
})
.catch((e) =>
logger.warn("Could not send DeviceMute action to widget", e),
);
}, [audio, video]);
const onMuteStateChangeRequest = useCallback(
(ev: CustomEvent<IWidgetApiRequest>) => {
// First copy the current state into our new state.
const newState = {
audio_enabled: audio.enabled,
video_enabled: video.enabled,
};
// Update new state if there are any requested changes from the widget action
// in `ev.detail.data`.
if (
ev.detail.data.audio_enabled != null &&
typeof ev.detail.data.audio_enabled === "boolean"
) {
audio.setEnabled?.(ev.detail.data.audio_enabled);
newState.audio_enabled = ev.detail.data.audio_enabled;
}
if (
ev.detail.data.video_enabled != null &&
typeof ev.detail.data.video_enabled === "boolean"
) {
video.setEnabled?.(ev.detail.data.video_enabled);
newState.video_enabled = ev.detail.data.video_enabled;
}
// Always reply with the new (now "current") state.
// This allows to also use this action to just get the unaltered current state
// by using a fromWidget request with: `ev.detail.data = {}`
widget!.api.transport.reply(ev.detail, newState);
},
[audio, video],
);
useEffect(() => {
// We setup a event listener for the widget action ElementWidgetActions.DeviceMute.
if (widget) {
// only setup the listener in widget mode
widget.lazyActions.on(
ElementWidgetActions.DeviceMute,
onMuteStateChangeRequest,
);
return (): void => {
// return a call to `off` so that we always clean up our listener.
widget?.lazyActions.off(
ElementWidgetActions.DeviceMute,
onMuteStateChangeRequest,
);
};
}
}, [onMuteStateChangeRequest]);
return useMemo(() => ({ audio, video }), [audio, video]);
}
function allowJoinUnmuted(skipLobby: boolean, isJoined: boolean): boolean {
return (
(!skipLobby && !isJoined) || import.meta.env.VITE_PACKAGE === "embedded"
);
}

7
src/room/VideoPreview.test.tsx
View File

@@ -5,12 +5,11 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { expect, describe, it, vi, beforeAll } from "vitest";
import { expect, describe, it, beforeAll } from "vitest";
import { render } from "@testing-library/react";
import { type MatrixInfo, VideoPreview } from "./VideoPreview";
import { E2eeType } from "../e2ee/e2eeType";
import { mockMuteStates } from "../utils/test";
describe("VideoPreview", () => {
const matrixInfo: MatrixInfo = {
@@ -42,7 +41,7 @@ describe("VideoPreview", () => {
const { queryByRole } = render(
<VideoPreview
matrixInfo={matrixInfo}
muteStates={mockMuteStates()}
videoEnabled={false}
videoTrack={null}
children={<></>}
/>,
@@ -54,7 +53,7 @@ describe("VideoPreview", () => {
const { queryByRole } = render(
<VideoPreview
matrixInfo={matrixInfo}
muteStates={mockMuteStates()}
videoEnabled
videoTrack={null}
children={<></>}
/>,

6
src/room/__snapshots__/GroupCallErrorBoundary.test.tsx.snap
View File

@@ -292,7 +292,7 @@ exports[`should have a close button in widget mode 1`] = `
Call is not supported
</h1>
<p>
The server is not configured to work with Element Call. Please contact your server admin (Domain: example.com, Error Code: MISSING_MATRIX_RTC_FOCUS).
The server is not configured to work with Element Call. Please contact your server admin (Domain: example.com, Error Code: MISSING_MATRIX_RTC_TRANSPORT).
</p>
<button
class="_button_vczzf_8"
@@ -445,7 +445,7 @@ exports[`should render the error page with link back to home 1`] = `
Call is not supported
</h1>
<p>
The server is not configured to work with Element Call. Please contact your server admin (Domain: example.com, Error Code: MISSING_MATRIX_RTC_FOCUS).
The server is not configured to work with Element Call. Please contact your server admin (Domain: example.com, Error Code: MISSING_MATRIX_RTC_TRANSPORT).
</p>
<button
class="_button_vczzf_8 homeLink"
@@ -598,7 +598,7 @@ exports[`should report correct error for 'Call is not supported' 1`] = `
Call is not supported
</h1>
<p>
The server is not configured to work with Element Call. Please contact your server admin (Domain: example.com, Error Code: MISSING_MATRIX_RTC_FOCUS).
The server is not configured to work with Element Call. Please contact your server admin (Domain: example.com, Error Code: MISSING_MATRIX_RTC_TRANSPORT).
</p>
<button
class="_button_vczzf_8 homeLink"

76
src/rtcSessionHelpers.test.ts
View File

@@ -13,8 +13,8 @@ import EventEmitter from "events";
import { enterRTCSession, leaveRTCSession } from "../src/rtcSessionHelpers";
import { mockConfig } from "./utils/test";
import { ElementWidgetActions, widget } from "./widget";
import { ErrorCode } from "./utils/errors.ts";
const USE_MUTI_SFU = false;
const getUrlParams = vi.hoisted(() => vi.fn(() => ({})));
vi.mock("./UrlParams", () => ({ getUrlParams }));
@@ -85,41 +85,35 @@ test("It joins the correct Session", async () => {
}),
joinRoomSession: vi.fn(),
}) as unknown as MatrixRTCSession;
await enterRTCSession(mockedSession, false);
await enterRTCSession(
mockedSession,
{
livekit_alias: "roomId",
livekit_service_url: "http://my-well-known-service-url.com",
type: "livekit",
},
{
encryptMedia: true,
useMultiSfu: USE_MUTI_SFU,
},
);
expect(mockedSession.joinRoomSession).toHaveBeenLastCalledWith(
[
{
livekit_alias: "my-oldest-member-service-alias",
livekit_service_url: "http://my-oldest-member-service-url.com",
type: "livekit",
},
{
livekit_alias: "roomId",
livekit_service_url: "http://my-well-known-service-url.com",
type: "livekit",
},
{
livekit_alias: "roomId",
livekit_service_url: "http://my-well-known-service-url2.com",
type: "livekit",
},
{
livekit_alias: "roomId",
livekit_service_url: "http://my-default-service-url.com",
type: "livekit",
},
],
{
focus_selection: "oldest_membership",
type: "livekit",
},
{
manageMediaKeys: false,
undefined,
expect.objectContaining({
manageMediaKeys: true,
useLegacyMemberEvents: false,
useNewMembershipManager: true,
useExperimentalToDeviceTransport: false,
},
}),
);
});
@@ -164,27 +158,6 @@ test("leaveRTCSession doesn't close the widget when returning to lobby", async (
await testLeaveRTCSession("user", false);
});
test("It fails with configuration error if no live kit url config is set in fallback", async () => {
mockConfig({});
vi.spyOn(AutoDiscovery, "getRawClientConfig").mockResolvedValue({});
const mockedSession = vi.mocked({
room: {
roomId: "roomId",
client: {
getDomain: vi.fn().mockReturnValue("example.org"),
},
},
memberships: [],
getFocusInUse: vi.fn(),
joinRoomSession: vi.fn(),
}) as unknown as MatrixRTCSession;
await expect(enterRTCSession(mockedSession, false)).rejects.toThrowError(
expect.objectContaining({ code: ErrorCode.MISSING_MATRIX_RTC_FOCUS }),
);
});
test("It should not fail with configuration error if homeserver config has livekit url but not fallback", async () => {
mockConfig({});
vi.spyOn(AutoDiscovery, "getRawClientConfig").mockResolvedValue({
@@ -214,5 +187,16 @@ test("It should not fail with configuration error if homeserver config has livek
joinRoomSession: vi.fn(),
}) as unknown as MatrixRTCSession;
await enterRTCSession(mockedSession, false);
await enterRTCSession(
mockedSession,
{
livekit_alias: "roomId",
livekit_service_url: "http://my-well-known-service-url.com",
type: "livekit",
},
{
encryptMedia: true,
useMultiSfu: USE_MUTI_SFU,
},
);
});

33
src/rtcSessionHelpers.ts
View File

@@ -97,14 +97,39 @@ export async function makeTransport(
return transport;
}
export interface EnterRTCSessionOptions {
encryptMedia: boolean;
// TODO: remove this flag, the new membership manager is stable enough
useNewMembershipManager?: boolean;
// TODO: remove this flag, to-device transport is stable enough now
useExperimentalToDeviceTransport?: boolean;
/** EXPERIMENTAL: If true, will use the multi-sfu codepath where each member connects to its SFU instead of everyone connecting to an elected on. */
useMultiSfu?: boolean;
}
/**
* TODO! document this function properly
* @param rtcSession
* @param transport
* @param options
*/
export async function enterRTCSession(
rtcSession: MatrixRTCSession,
transport: LivekitTransport,
encryptMedia: boolean,
useNewMembershipManager = true,
useExperimentalToDeviceTransport = false,
useMultiSfu = true,
options: EnterRTCSessionOptions = {
encryptMedia: true,
useNewMembershipManager: true,
useExperimentalToDeviceTransport: false,
useMultiSfu: true,
},
): Promise<void> {
const {
encryptMedia,
useNewMembershipManager = true,
useExperimentalToDeviceTransport = false,
useMultiSfu = true,
} = options;
PosthogAnalytics.instance.eventCallEnded.cacheStartCall(new Date());
PosthogAnalytics.instance.eventCallStarted.track(rtcSession.room.roomId);

31
src/state/Async.ts
View File

@@ -5,16 +5,14 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
catchError,
from,
map,
Observable,
of,
startWith,
switchMap,
} from "rxjs";
import { catchError, from, map, type Observable, of, startWith } from "rxjs";
/**
* Data that may need to be loaded asynchronously.
*
* This type is for when you need to represent the current state of an operation
* involving Promises as **immutable data**. See the async$ function below.
*/
export type Async<A> =
| { state: "loading" }
| { state: "error"; value: Error }
@@ -24,18 +22,29 @@ export const loading: Async<never> = { state: "loading" };
export function error(value: Error): Async<never> {
return { state: "error", value };
}
export function ready<A>(value: A): Async<A> {
return { state: "ready", value };
}
export function async<A>(promise: Promise<A>): Observable<Async<A>> {
/**
* Turn a Promise into an Observable async value. The Observable will have the
* value "loading" while the Promise is pending, "ready" when the Promise
* resolves, and "error" when the Promise rejects.
*/
export function async$<A>(promise: Promise<A>): Observable<Async<A>> {
return from(promise).pipe(
map(ready),
startWith(loading),
catchError((e) => of(error(e))),
catchError((e: unknown) =>
of(error((e as Error) ?? new Error("Unknown error"))),
),
);
}
/**
* If the async value is ready, apply the given function to the inner value.
*/
export function mapAsync<A, B>(
async: Async<A>,
project: (value: A) => B,

83
src/state/CallViewModel.test.ts
View File

@@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { test, vi, onTestFinished, it, describe } from "vitest";
import { test, vi, onTestFinished, it, describe, expect } from "vitest";
import EventEmitter from "events";
import {
BehaviorSubject,
@@ -40,18 +40,15 @@ import * as ComponentsCore from "@livekit/components-core";
import {
Status,
type CallMembership,
type MatrixRTCSession,
type IRTCNotificationContent,
type ICallNotifyContent,
MatrixRTCSessionEvent,
} from "matrix-js-sdk/lib/matrixrtc";
import { deepCompare } from "matrix-js-sdk/lib/utils";
import { AutoDiscovery } from "matrix-js-sdk/lib/autodiscovery";
import {
CallViewModel,
type CallViewModelOptions,
type Layout,
} from "./CallViewModel";
import { CallViewModel, type CallViewModelOptions } from "./CallViewModel";
import { type Layout } from "./layout-types";
import {
mockLocalParticipant,
mockMatrixRoom,
@@ -62,13 +59,14 @@ import {
MockRTCSession,
mockMediaDevices,
mockMuteStates,
mockConfig,
} from "../utils/test";
import {
ECAddonConnectionState,
type ECConnectionState,
} from "../livekit/useECConnectionState";
import { E2eeType } from "../e2ee/e2eeType";
import type { RaisedHandInfo } from "../reactions";
import type { RaisedHandInfo, ReactionInfo } from "../reactions";
import {
alice,
aliceDoppelganger,
@@ -95,6 +93,11 @@ import { ObservableScope } from "./ObservableScope";
import { MediaDevices } from "./MediaDevices";
import { getValue } from "../utils/observable";
import { type Behavior, constant } from "./Behavior";
import type { ProcessorState } from "../livekit/TrackProcessorContext.tsx";
import {
type ElementCallError,
MatrixRTCTransportMissingError,
} from "../utils/errors.ts";
const getUrlParams = vi.hoisted(() => vi.fn(() => ({})));
vi.mock("../UrlParams", () => ({ getUrlParams }));
@@ -341,15 +344,20 @@ function withCallViewModel(
.mockImplementation((_room, _eventType) => of());
const muteStates = mockMuteStates();
const raisedHands$ = new BehaviorSubject<Record<string, RaisedHandInfo>>({});
const reactions$ = new BehaviorSubject<Record<string, ReactionInfo>>({});
const vm = new CallViewModel(
rtcSession as unknown as MatrixRTCSession,
rtcSession.asMockedSession(),
room,
mediaDevices,
muteStates,
options,
raisedHands$,
new BehaviorSubject({}),
reactions$,
new BehaviorSubject<ProcessorState>({
processor: undefined,
supported: undefined,
}),
);
onTestFinished(() => {
@@ -363,6 +371,61 @@ function withCallViewModel(
continuation(vm, rtcSession, { raisedHands$: raisedHands$ }, setSyncState);
}
test("test missing RTC config error", async () => {
const rtcMemberships$ = new BehaviorSubject<CallMembership[]>([]);
const emitter = new EventEmitter();
const client = vi.mocked<MatrixClient>({
on: emitter.on.bind(emitter),
off: emitter.off.bind(emitter),
getSyncState: vi.fn().mockReturnValue(SyncState.Syncing),
getUserId: vi.fn().mockReturnValue("@user:localhost"),
getUser: vi.fn().mockReturnValue(null),
getDeviceId: vi.fn().mockReturnValue("DEVICE"),
credentials: {
userId: "@user:localhost",
},
getCrypto: vi.fn().mockReturnValue(undefined),
getDomain: vi.fn().mockReturnValue("example.org"),
} as unknown as MatrixClient);
const matrixRoom = mockMatrixRoom({
roomId: "!myRoomId:example.com",
client,
getMember: vi.fn().mockReturnValue(undefined),
});
const fakeRtcSession = new MockRTCSession(matrixRoom).withMemberships(
rtcMemberships$,
);
mockConfig({});
vi.spyOn(AutoDiscovery, "getRawClientConfig").mockResolvedValue({});
const callVM = new CallViewModel(
fakeRtcSession.asMockedSession(),
matrixRoom,
mockMediaDevices({}),
mockMuteStates(),
{
encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
autoLeaveWhenOthersLeft: false,
},
new BehaviorSubject({} as Record<string, RaisedHandInfo>),
new BehaviorSubject({} as Record<string, ReactionInfo>),
of({ processor: undefined, supported: false }),
);
const failPromise = Promise.withResolvers<ElementCallError>();
callVM.configError$.subscribe((error) => {
if (error) {
failPromise.resolve(error);
}
});
const error = await failPromise.promise;
expect(error).toBeInstanceOf(MatrixRTCTransportMissingError);
});
test("participants are retained during a focus switch", () => {
withTestScheduler(({ behavior, expectObservable }) => {
// Participants disappear on frame 2 and come back on frame 3

539
src/state/CallViewModel.ts
View File

@@ -5,39 +5,33 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { observeParticipantEvents } from "@livekit/components-core";
import {
ConnectionState,
type BaseKeyProvider,
ConnectionState,
type E2EEOptions,
ExternalE2EEKeyProvider,
type Room as LivekitRoom,
type LocalParticipant,
ParticipantEvent,
type RemoteParticipant,
type Participant,
RemoteParticipant,
type Room as LivekitRoom,
} from "livekit-client";
import E2EEWorker from "livekit-client/e2ee-worker?worker";
import {
ClientEvent,
type EventTimelineSetHandlerMap,
EventType,
type Room as MatrixRoom,
RoomEvent,
type RoomMember,
RoomStateEvent,
SyncState,
type Room as MatrixRoom,
type EventTimelineSetHandlerMap,
EventType,
RoomEvent,
} from "matrix-js-sdk";
import { deepCompare } from "matrix-js-sdk/lib/utils";
import {
BehaviorSubject,
EMPTY,
NEVER,
type Observable,
Subject,
combineLatest,
concat,
distinctUntilChanged,
EMPTY,
endWith,
filter,
from,
@@ -45,6 +39,8 @@ import {
ignoreElements,
map,
merge,
NEVER,
type Observable,
of,
pairwise,
race,
@@ -53,6 +49,7 @@ import {
skip,
skipWhile,
startWith,
Subject,
switchAll,
switchMap,
switchScan,
@@ -80,7 +77,7 @@ import { ViewModel } from "./ViewModel";
import {
LocalUserMediaViewModel,
type MediaViewModel,
RemoteUserMediaViewModel,
type RemoteUserMediaViewModel,
ScreenShareViewModel,
type UserMediaViewModel,
} from "./MediaViewModel";
@@ -90,7 +87,6 @@ import {
finalizeValue,
pauseWhen,
} from "../utils/observable";
import { ObservableScope } from "./ObservableScope";
import {
duplicateTiles,
multiSfu,
@@ -99,10 +95,6 @@ import {
} from "../settings/settings";
import { isFirefox } from "../Platform";
import { setPipEnabled$ } from "../controls";
import {
type GridTileViewModel,
type SpotlightTileViewModel,
} from "./TileViewModel";
import { TileStore } from "./TileStore";
import { gridLikeLayout } from "./GridLikeLayout";
import { spotlightExpandedLayout } from "./SpotlightExpandedLayout";
@@ -114,11 +106,10 @@ import {
type ReactionInfo,
type ReactionOption,
} from "../reactions";
import { observeSpeaker$ } from "./observeSpeaker";
import { shallowEquals } from "../utils/array";
import { calculateDisplayName, shouldDisambiguate } from "../utils/displayname";
import { type MediaDevices } from "./MediaDevices";
import { constant, type Behavior } from "./Behavior";
import { type Behavior, constant } from "./Behavior";
import {
enterRTCSession,
getLivekitAlias,
@@ -126,12 +117,29 @@ import {
} from "../rtcSessionHelpers";
import { E2eeType } from "../e2ee/e2eeType";
import { MatrixKeyProvider } from "../e2ee/matrixKeyProvider";
import { Connection, PublishConnection } from "./Connection";
import {
type Connection,
type ConnectionOpts,
RemoteConnection,
} from "./Connection";
import { type MuteStates } from "./MuteStates";
import { getUrlParams } from "../UrlParams";
import { type ProcessorState } from "../livekit/TrackProcessorContext";
import { ElementWidgetActions, widget } from "../widget";
import { type Async, async, mapAsync, ready } from "./Async";
import { PublishConnection } from "./PublishConnection.ts";
import { type Async, async$, mapAsync, ready } from "./Async";
import { sharingScreen$, UserMedia } from "./UserMedia.ts";
import { ScreenShare } from "./ScreenShare.ts";
import {
type GridLayoutMedia,
type Layout,
type LayoutMedia,
type OneOnOneLayoutMedia,
type SpotlightExpandedLayoutMedia,
type SpotlightLandscapeLayoutMedia,
type SpotlightPortraitLayoutMedia,
} from "./layout-types.ts";
import { ElementCallError, UnknownCallError } from "../utils/errors.ts";
export interface CallViewModelOptions {
encryptionSystem: EncryptionSystem;
@@ -156,99 +164,6 @@ const smallMobileCallThreshold = 3;
// with the interface
const showFooterMs = 4000;
export interface GridLayoutMedia {
type: "grid";
spotlight?: MediaViewModel[];
grid: UserMediaViewModel[];
}
export interface SpotlightLandscapeLayoutMedia {
type: "spotlight-landscape";
spotlight: MediaViewModel[];
grid: UserMediaViewModel[];
}
export interface SpotlightPortraitLayoutMedia {
type: "spotlight-portrait";
spotlight: MediaViewModel[];
grid: UserMediaViewModel[];
}
export interface SpotlightExpandedLayoutMedia {
type: "spotlight-expanded";
spotlight: MediaViewModel[];
pip?: UserMediaViewModel;
}
export interface OneOnOneLayoutMedia {
type: "one-on-one";
local: UserMediaViewModel;
remote: UserMediaViewModel;
}
export interface PipLayoutMedia {
type: "pip";
spotlight: MediaViewModel[];
}
export type LayoutMedia =
| GridLayoutMedia
| SpotlightLandscapeLayoutMedia
| SpotlightPortraitLayoutMedia
| SpotlightExpandedLayoutMedia
| OneOnOneLayoutMedia
| PipLayoutMedia;
export interface GridLayout {
type: "grid";
spotlight?: SpotlightTileViewModel;
grid: GridTileViewModel[];
setVisibleTiles: (value: number) => void;
}
export interface SpotlightLandscapeLayout {
type: "spotlight-landscape";
spotlight: SpotlightTileViewModel;
grid: GridTileViewModel[];
setVisibleTiles: (value: number) => void;
}
export interface SpotlightPortraitLayout {
type: "spotlight-portrait";
spotlight: SpotlightTileViewModel;
grid: GridTileViewModel[];
setVisibleTiles: (value: number) => void;
}
export interface SpotlightExpandedLayout {
type: "spotlight-expanded";
spotlight: SpotlightTileViewModel;
pip?: GridTileViewModel;
}
export interface OneOnOneLayout {
type: "one-on-one";
local: GridTileViewModel;
remote: GridTileViewModel;
}
export interface PipLayout {
type: "pip";
spotlight: SpotlightTileViewModel;
}
/**
* A layout defining the media tiles present on screen and their visual
* arrangement.
*/
export type Layout =
| GridLayout
| SpotlightLandscapeLayout
| SpotlightPortraitLayout
| SpotlightExpandedLayout
| OneOnOneLayout
| PipLayout;
export type GridMode = "grid" | "spotlight";
export type WindowMode = "normal" | "narrow" | "flat" | "pip";
@@ -292,151 +207,8 @@ interface LayoutScanState {
tiles: TileStore;
}
class UserMedia {
private readonly scope = new ObservableScope();
public readonly vm: UserMediaViewModel;
private readonly participant$: BehaviorSubject<
LocalParticipant | RemoteParticipant | undefined
>;
public readonly speaker$: Behavior<boolean>;
public readonly presenter$: Behavior<boolean>;
public constructor(
public readonly id: string,
member: RoomMember,
participant: LocalParticipant | RemoteParticipant | undefined,
encryptionSystem: EncryptionSystem,
livekitRoom: LivekitRoom,
mediaDevices: MediaDevices,
pretendToBeDisconnected$: Behavior<boolean>,
displayname$: Observable<string>,
handRaised$: Observable<Date | null>,
reaction$: Observable<ReactionOption | null>,
) {
this.participant$ = new BehaviorSubject(participant);
if (participant?.isLocal) {
this.vm = new LocalUserMediaViewModel(
this.id,
member,
this.participant$ as Behavior<LocalParticipant>,
encryptionSystem,
livekitRoom,
mediaDevices,
this.scope.behavior(displayname$),
this.scope.behavior(handRaised$),
this.scope.behavior(reaction$),
);
} else {
this.vm = new RemoteUserMediaViewModel(
id,
member,
this.participant$.asObservable() as Observable<
RemoteParticipant | undefined
>,
encryptionSystem,
livekitRoom,
pretendToBeDisconnected$,
this.scope.behavior(displayname$),
this.scope.behavior(handRaised$),
this.scope.behavior(reaction$),
);
}
this.speaker$ = this.scope.behavior(observeSpeaker$(this.vm.speaking$));
this.presenter$ = this.scope.behavior(
this.participant$.pipe(
switchMap((p) => (p === undefined ? of(false) : sharingScreen$(p))),
),
);
}
public updateParticipant(
newParticipant: LocalParticipant | RemoteParticipant | undefined,
): void {
if (this.participant$.value !== newParticipant) {
// Update the BehaviourSubject in the UserMedia.
this.participant$.next(newParticipant);
}
}
public destroy(): void {
this.scope.end();
this.vm.destroy();
}
}
class ScreenShare {
private readonly scope = new ObservableScope();
public readonly vm: ScreenShareViewModel;
private readonly participant$: BehaviorSubject<
LocalParticipant | RemoteParticipant
>;
public constructor(
id: string,
member: RoomMember,
participant: LocalParticipant | RemoteParticipant,
encryptionSystem: EncryptionSystem,
livekitRoom: LivekitRoom,
pretendToBeDisconnected$: Behavior<boolean>,
displayName$: Observable<string>,
) {
this.participant$ = new BehaviorSubject(participant);
this.vm = new ScreenShareViewModel(
id,
member,
this.participant$.asObservable(),
encryptionSystem,
livekitRoom,
pretendToBeDisconnected$,
this.scope.behavior(displayName$),
participant.isLocal,
);
}
public destroy(): void {
this.scope.end();
this.vm.destroy();
}
}
type MediaItem = UserMedia | ScreenShare;
function getRoomMemberFromRtcMember(
rtcMember: CallMembership,
room: MatrixRoom,
): { id: string; member: RoomMember | undefined } {
// WARN! This is not exactly the sender but the user defined in the state key.
// This will be available once we change to the new "member as object" format in the MatrixRTC object.
let id = rtcMember.sender + ":" + rtcMember.deviceId;
if (!rtcMember.sender) {
return { id, member: undefined };
}
if (
rtcMember.sender === room.client.getUserId() &&
rtcMember.deviceId === room.client.getDeviceId()
) {
id = "local";
}
const member = room.getMember(rtcMember.sender) ?? undefined;
return { id, member };
}
function sharingScreen$(p: Participant): Observable<boolean> {
return observeParticipantEvents(
p,
ParticipantEvent.TrackPublished,
ParticipantEvent.TrackUnpublished,
ParticipantEvent.LocalTrackPublished,
ParticipantEvent.LocalTrackUnpublished,
).pipe(map((p) => p.isScreenShareEnabled));
}
export class CallViewModel extends ViewModel {
private readonly urlParams = getUrlParams();
@@ -454,6 +226,19 @@ export class CallViewModel extends ViewModel {
}
: undefined;
private readonly _configError$ = new BehaviorSubject<ElementCallError | null>(
null,
);
/**
* If there is a configuration error with the call (e.g. misconfigured E2EE).
* This is a fatal error that prevents the call from being created/joined.
* Should render a blocking error screen.
*/
public get configError$(): Behavior<ElementCallError | null> {
return this._configError$;
}
private readonly join$ = new Subject<void>();
public join(): void {
@@ -503,7 +288,7 @@ export class CallViewModel extends ViewModel {
* The transport that we would personally prefer to publish on (if not for the
* transport preferences of others, perhaps).
*/
private readonly preferredTransport = makeTransport(this.matrixRTCSession);
private readonly preferredTransport$: Observable<Async<LivekitTransport>>;
/**
* Lists the transports used by ourselves, plus all other MatrixRTC session
@@ -523,11 +308,7 @@ export class CallViewModel extends ViewModel {
switchMap((joined) =>
joined
? combineLatest(
[
async(this.preferredTransport),
this.memberships$,
multiSfu.value$,
],
[this.preferredTransport$, this.memberships$, multiSfu.value$],
(preferred, memberships, multiSfu) => {
const oldestMembership =
this.matrixRTCSession.getOldestMembership();
@@ -544,9 +325,18 @@ export class CallViewModel extends ViewModel {
const oldest = this.matrixRTCSession.getOldestMembership();
if (oldest !== undefined) {
const selection = oldest.getTransport(oldest);
if (isLivekitTransport(selection)) local = ready(selection);
// TODO selection can be null if no transport is configured should we report an error?
if (selection && isLivekitTransport(selection))
local = ready(selection);
}
}
if (local.state === "error") {
this._configError$.next(
local.value instanceof ElementCallError
? local.value
: new UnknownCallError(local.value),
);
}
return { local, remote, preferred, multiSfu };
},
)
@@ -565,12 +355,13 @@ export class CallViewModel extends ViewModel {
/**
* The transport over which we should be actively publishing our media.
* null when not joined.
*/
private readonly localTransport$: Behavior<Async<LivekitTransport> | null> =
this.scope.behavior(
this.transports$.pipe(
map((transports) => transports?.local ?? null),
distinctUntilChanged(deepCompare),
distinctUntilChanged<Async<LivekitTransport> | null>(deepCompare),
),
);
@@ -603,40 +394,48 @@ export class CallViewModel extends ViewModel {
),
);
private readonly localConnectionAndTransport$ = this.scope.behavior(
this.localTransport$.pipe(
map(
(transport) =>
transport &&
mapAsync(transport, (transport) => ({
connection: new PublishConnection(
transport,
this.livekitAlias,
this.matrixRTCSession.room.client,
this.scope,
this.remoteTransports$,
this.mediaDevices,
this.muteStates,
this.e2eeLivekitOptions(),
this.scope.behavior(this.trackProcessorState$),
),
transport,
})),
/**
* The local connection over which we will publish our media. It could
* possibly also have some remote users' media available on it.
* null when not joined.
*/
private readonly localConnection$: Behavior<Async<PublishConnection> | null> =
this.scope.behavior(
this.localTransport$.pipe(
map(
(transport) =>
transport &&
mapAsync(transport, (transport) => {
const opts: ConnectionOpts = {
transport,
client: this.matrixRTCSession.room.client,
scope: this.scope,
remoteTransports$: this.remoteTransports$,
};
return new PublishConnection(
opts,
this.mediaDevices,
this.muteStates,
this.e2eeLivekitOptions(),
this.scope.behavior(this.trackProcessorState$),
);
}),
),
),
),
);
private readonly localConnection$ = this.scope.behavior(
this.localConnectionAndTransport$.pipe(
map((value) => value && mapAsync(value, ({ connection }) => connection)),
),
);
);
public readonly livekitConnectionState$ = this.scope.behavior(
this.localConnection$.pipe(
switchMap((c) =>
c?.state === "ready"
? c.value.connectionState$
? // TODO mapping to ConnectionState for compatibility, but we should use the full state?
c.value.focusConnectionState$.pipe(
map((s) => {
if (s.state === "ConnectedToLkRoom") return s.connectionState;
return ConnectionState.Disconnected;
}),
distinctUntilChanged(),
)
: of(ConnectionState.Disconnected),
),
),
@@ -674,16 +473,19 @@ export class CallViewModel extends ViewModel {
"SFU remoteConnections$ construct new connection: ",
remoteServiceUrl,
);
nextConnection = new Connection(
{
const args: ConnectionOpts = {
transport: {
type: "livekit",
livekit_service_url: remoteServiceUrl,
livekit_alias: this.livekitAlias,
type: "livekit",
},
this.livekitAlias,
this.matrixRTCSession.room.client,
this.scope,
this.remoteTransports$,
client: this.matrixRTCSession.room.client,
scope: this.scope,
remoteTransports$: this.remoteTransports$,
};
nextConnection = new RemoteConnection(
args,
this.e2eeLivekitOptions(),
);
} else {
@@ -735,15 +537,15 @@ export class CallViewModel extends ViewModel {
map((connections) =>
[...connections.values()].map((c) => ({
room: c.livekitRoom,
url: c.transport.livekit_service_url,
url: c.localTransport.livekit_service_url,
isLocal: c instanceof PublishConnection,
})),
),
),
);
private readonly userId = this.matrixRoom.client.getUserId();
private readonly deviceId = this.matrixRoom.client.getDeviceId();
private readonly userId = this.matrixRoom.client.getUserId()!;
private readonly deviceId = this.matrixRoom.client.getDeviceId()!;
private readonly matrixConnected$ = this.scope.behavior(
// To consider ourselves connected to MatrixRTC, we check the following:
@@ -820,7 +622,7 @@ export class CallViewModel extends ViewModel {
* Lists, for each LiveKit room, the LiveKit participants whose media should
* be presented.
*/
public readonly participantsByRoom$ = this.scope.behavior<
private readonly participantsByRoom$ = this.scope.behavior<
{
livekitRoom: LivekitRoom;
url: string;
@@ -832,17 +634,16 @@ export class CallViewModel extends ViewModel {
}[]
>(
// TODO: Move this logic into Connection/PublishConnection if possible
this.localConnectionAndTransport$
this.localConnection$
.pipe(
switchMap((values) => {
if (values?.state !== "ready") return [];
const localConnection = values.value.connection;
switchMap((localConnection) => {
if (localConnection?.state !== "ready") return [];
const memberError = (): never => {
throw new Error("No room member for call membership");
};
const localParticipant = {
id: "local",
participant: localConnection.livekitRoom.localParticipant,
participant: localConnection.value.livekitRoom.localParticipant,
member:
this.matrixRoom.getMember(this.userId ?? "") ?? memberError(),
};
@@ -850,7 +651,7 @@ export class CallViewModel extends ViewModel {
return this.remoteConnections$.pipe(
switchMap((remoteConnections) =>
combineLatest(
[localConnection, ...remoteConnections].map((c) =>
[localConnection.value, ...remoteConnections].map((c) =>
c.publishingParticipants$.pipe(
map((ps) => {
const participants: {
@@ -869,12 +670,12 @@ export class CallViewModel extends ViewModel {
this.matrixRoom,
)?.member ?? memberError(),
}));
if (c === localConnection)
if (c === localConnection.value)
participants.push(localParticipant);
return {
livekitRoom: c.livekitRoom,
url: c.transport.livekit_service_url,
url: c.localTransport.livekit_service_url,
participants,
};
}),
@@ -888,6 +689,25 @@ export class CallViewModel extends ViewModel {
.pipe(startWith([]), pauseWhen(this.pretendToBeDisconnected$)),
);
/**
* Lists, for each LiveKit room, the LiveKit participants whose audio should
* be rendered.
*/
// (This is effectively just participantsByRoom$ with a stricter type)
public readonly audioParticipants$ = this.scope.behavior(
this.participantsByRoom$.pipe(
map((data) =>
data.map(({ livekitRoom, url, participants }) => ({
livekitRoom,
url,
participants: participants.flatMap(({ participant }) =>
participant instanceof RemoteParticipant ? [participant] : [],
),
})),
),
),
);
/**
* Displaynames for each member of the call. This will disambiguate
* any displaynames that clashes with another member. Only members
@@ -909,7 +729,11 @@ export class CallViewModel extends ViewModel {
],
(memberships, _displaynames) => {
const displaynameMap = new Map<string, string>([
["local", this.matrixRoom.getMember(this.userId!)!.rawDisplayName],
[
"local",
this.matrixRoom.getMember(this.userId)?.rawDisplayName ??
this.userId,
],
]);
const room = this.matrixRoom;
@@ -961,7 +785,11 @@ export class CallViewModel extends ViewModel {
scan((prevItems, [participantsByRoom, duplicateTiles]) => {
const newItems: Map<string, UserMedia | ScreenShare> = new Map(
function* (this: CallViewModel): Iterable<[string, MediaItem]> {
for (const { livekitRoom, participants } of participantsByRoom) {
for (const {
livekitRoom,
participants,
url,
} of participantsByRoom) {
for (const { id, participant, member } of participants) {
for (let i = 0; i < 1 + duplicateTiles; i++) {
const mediaId = `${id}:${i}`;
@@ -981,6 +809,7 @@ export class CallViewModel extends ViewModel {
participant,
this.options.encryptionSystem,
livekitRoom,
url,
this.mediaDevices,
this.pretendToBeDisconnected$,
this.memberDisplaynames$.pipe(
@@ -1002,6 +831,7 @@ export class CallViewModel extends ViewModel {
participant,
this.options.encryptionSystem,
livekitRoom,
url,
this.pretendToBeDisconnected$,
this.memberDisplaynames$.pipe(
map((m) => m.get(id) ?? "[👻]"),
@@ -1972,23 +1802,47 @@ export class CallViewModel extends ViewModel {
) {
super();
this.preferredTransport$ = async$(
makeTransport(this.matrixRTCSession),
).pipe(this.scope.bind());
// Start and stop local and remote connections as needed
this.connectionInstructions$
.pipe(this.scope.bind())
.subscribe(({ start, stop }) => {
for (const c of stop) {
logger.info(`Disconnecting from ${c.transport.livekit_service_url}`);
c.stop();
logger.info(
`Disconnecting from ${c.localTransport.livekit_service_url}`,
);
c.stop().catch((err) => {
// TODO: better error handling
logger.error(
`Fail to stop connection to ${c.localTransport.livekit_service_url}`,
err,
);
});
}
for (const c of start) {
c.start().then(
() =>
logger.info(`Connected to ${c.transport.livekit_service_url}`),
(e) =>
logger.error(
`Failed to start connection to ${c.transport.livekit_service_url}`,
e,
logger.info(
`Connected to ${c.localTransport.livekit_service_url}`,
),
(e) => {
// We only want to report fatal errors `_configError$` for the publish connection.
// If there is an error with another connection, it will not terminate the call and will be displayed
// on eacn tile.
if (
c instanceof PublishConnection &&
e instanceof ElementCallError
) {
this._configError$.next(e);
}
logger.error(
`Failed to start connection to ${c.localTransport.livekit_service_url}`,
e,
);
},
);
}
});
@@ -1997,14 +1851,13 @@ export class CallViewModel extends ViewModel {
this.scope.reconcile(this.advertisedTransport$, async (advertised) => {
if (advertised !== null) {
try {
await enterRTCSession(
this.matrixRTCSession,
advertised.transport,
this.options.encryptionSystem.kind !== E2eeType.NONE,
true,
true,
advertised.multiSfu,
);
this._configError$.next(null);
await enterRTCSession(this.matrixRTCSession, advertised.transport, {
encryptMedia: this.options.encryptionSystem.kind !== E2eeType.NONE,
useExperimentalToDeviceTransport: true,
useNewMembershipManager: true,
useMultiSfu: advertised.multiSfu,
});
} catch (e) {
logger.error("Error entering RTC session", e);
}
@@ -2115,3 +1968,25 @@ function getE2eeKeyProvider(
return keyProvider;
}
}
function getRoomMemberFromRtcMember(
rtcMember: CallMembership,
room: MatrixRoom,
): { id: string; member: RoomMember | undefined } {
// WARN! This is not exactly the sender but the user defined in the state key.
// This will be available once we change to the new "member as object" format in the MatrixRTC object.
let id = rtcMember.sender + ":" + rtcMember.deviceId;
if (!rtcMember.sender) {
return { id, member: undefined };
}
if (
rtcMember.sender === room.client.getUserId() &&
rtcMember.deviceId === room.client.getDeviceId()
) {
id = "local";
}
const member = room.getMember(rtcMember.sender) ?? undefined;
return { id, member };
}

750
src/state/Connection.test.ts Normal file
View File

@@ -0,0 +1,750 @@
/*
Copyright 2025 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
afterEach,
describe,
expect,
it,
type Mock,
type MockedObject,
onTestFinished,
vi,
} from "vitest";
import { BehaviorSubject, of } from "rxjs";
import {
ConnectionState,
type LocalParticipant,
type RemoteParticipant,
type Room as LivekitRoom,
RoomEvent,
type RoomOptions,
} from "livekit-client";
import fetchMock from "fetch-mock";
import EventEmitter from "events";
import { type IOpenIDToken } from "matrix-js-sdk";
import type {
CallMembership,
LivekitTransport,
} from "matrix-js-sdk/lib/matrixrtc";
import {
type ConnectionOpts,
type FocusConnectionState,
RemoteConnection,
} from "./Connection.ts";
import { ObservableScope } from "./ObservableScope.ts";
import { type OpenIDClientParts } from "../livekit/openIDSFU.ts";
import { FailToGetOpenIdToken } from "../utils/errors.ts";
import { PublishConnection } from "./PublishConnection.ts";
import { mockMediaDevices, mockMuteStates } from "../utils/test.ts";
import type { ProcessorState } from "../livekit/TrackProcessorContext.tsx";
import { type MuteStates } from "./MuteStates.ts";
let testScope: ObservableScope;
let client: MockedObject<OpenIDClientParts>;
let fakeLivekitRoom: MockedObject<LivekitRoom>;
let localParticipantEventEmiter: EventEmitter;
let fakeLocalParticipant: MockedObject<LocalParticipant>;
let fakeRoomEventEmiter: EventEmitter;
let fakeMembershipsFocusMap$: BehaviorSubject<
{ membership: CallMembership; transport: LivekitTransport }[]
>;
const livekitFocus: LivekitTransport = {
livekit_alias: "!roomID:example.org",
livekit_service_url: "https://matrix-rtc.example.org/livekit/jwt",
type: "livekit",
};
function setupTest(): void {
testScope = new ObservableScope();
client = vi.mocked<OpenIDClientParts>({
getOpenIdToken: vi.fn().mockResolvedValue({
access_token: "rYsmGUEwNjKgJYyeNUkZseJN",
token_type: "Bearer",
matrix_server_name: "example.org",
expires_in: 3600,
}),
getDeviceId: vi.fn().mockReturnValue("ABCDEF"),
} as unknown as OpenIDClientParts);
fakeMembershipsFocusMap$ = new BehaviorSubject<
{ membership: CallMembership; transport: LivekitTransport }[]
>([]);
localParticipantEventEmiter = new EventEmitter();
fakeLocalParticipant = vi.mocked<LocalParticipant>({
identity: "@me:example.org",
isMicrophoneEnabled: vi.fn().mockReturnValue(true),
getTrackPublication: vi.fn().mockReturnValue(undefined),
on: localParticipantEventEmiter.on.bind(localParticipantEventEmiter),
off: localParticipantEventEmiter.off.bind(localParticipantEventEmiter),
addListener: localParticipantEventEmiter.addListener.bind(
localParticipantEventEmiter,
),
removeListener: localParticipantEventEmiter.removeListener.bind(
localParticipantEventEmiter,
),
removeAllListeners: localParticipantEventEmiter.removeAllListeners.bind(
localParticipantEventEmiter,
),
} as unknown as LocalParticipant);
fakeRoomEventEmiter = new EventEmitter();
fakeLivekitRoom = vi.mocked<LivekitRoom>({
connect: vi.fn(),
disconnect: vi.fn(),
remoteParticipants: new Map(),
localParticipant: fakeLocalParticipant,
state: ConnectionState.Disconnected,
on: fakeRoomEventEmiter.on.bind(fakeRoomEventEmiter),
off: fakeRoomEventEmiter.off.bind(fakeRoomEventEmiter),
addListener: fakeRoomEventEmiter.addListener.bind(fakeRoomEventEmiter),
removeListener:
fakeRoomEventEmiter.removeListener.bind(fakeRoomEventEmiter),
removeAllListeners:
fakeRoomEventEmiter.removeAllListeners.bind(fakeRoomEventEmiter),
setE2EEEnabled: vi.fn().mockResolvedValue(undefined),
} as unknown as LivekitRoom);
}
function setupRemoteConnection(): RemoteConnection {
const opts: ConnectionOpts = {
client: client,
transport: livekitFocus,
remoteTransports$: fakeMembershipsFocusMap$,
scope: testScope,
livekitRoomFactory: () => fakeLivekitRoom,
};
fetchMock.post(`${livekitFocus.livekit_service_url}/sfu/get`, () => {
return {
status: 200,
body: {
url: "wss://matrix-rtc.m.localhost/livekit/sfu",
jwt: "ATOKEN",
},
};
});
fakeLivekitRoom.connect.mockResolvedValue(undefined);
return new RemoteConnection(opts, undefined);
}
afterEach(() => {
vi.useRealTimers();
vi.clearAllMocks();
fetchMock.reset();
});
describe("Start connection states", () => {
it("start in initialized state", () => {
setupTest();
const opts: ConnectionOpts = {
client: client,
transport: livekitFocus,
remoteTransports$: fakeMembershipsFocusMap$,
scope: testScope,
livekitRoomFactory: () => fakeLivekitRoom,
};
const connection = new RemoteConnection(opts, undefined);
expect(connection.focusConnectionState$.getValue().state).toEqual(
"Initialized",
);
});
it("fail to getOpenId token then error state", async () => {
setupTest();
vi.useFakeTimers();
const opts: ConnectionOpts = {
client: client,
transport: livekitFocus,
remoteTransports$: fakeMembershipsFocusMap$,
scope: testScope,
livekitRoomFactory: () => fakeLivekitRoom,
};
const connection = new RemoteConnection(opts, undefined);
const capturedStates: FocusConnectionState[] = [];
const s = connection.focusConnectionState$.subscribe((value) => {
capturedStates.push(value);
});
onTestFinished(() => s.unsubscribe());
const deferred = Promise.withResolvers<IOpenIDToken>();
client.getOpenIdToken.mockImplementation(
async (): Promise<IOpenIDToken> => {
return await deferred.promise;
},
);
connection.start().catch(() => {
// expected to throw
});
let capturedState = capturedStates.pop();
expect(capturedState).toBeDefined();
expect(capturedState!.state).toEqual("FetchingConfig");
deferred.reject(new FailToGetOpenIdToken(new Error("Failed to get token")));
await vi.runAllTimersAsync();
capturedState = capturedStates.pop();
if (capturedState!.state === "FailedToStart") {
expect(capturedState!.error.message).toEqual("Something went wrong");
expect(capturedState!.focus.livekit_alias).toEqual(
livekitFocus.livekit_alias,
);
} else {
expect.fail(
"Expected FailedToStart state but got " + capturedState?.state,
);
}
});
it("fail to get JWT token and error state", async () => {
setupTest();
vi.useFakeTimers();
const opts: ConnectionOpts = {
client: client,
transport: livekitFocus,
remoteTransports$: fakeMembershipsFocusMap$,
scope: testScope,
livekitRoomFactory: () => fakeLivekitRoom,
};
const connection = new RemoteConnection(opts, undefined);
const capturedStates: FocusConnectionState[] = [];
const s = connection.focusConnectionState$.subscribe((value) => {
capturedStates.push(value);
});
onTestFinished(() => s.unsubscribe());
const deferredSFU = Promise.withResolvers<void>();
// mock the /sfu/get call
fetchMock.post(`${livekitFocus.livekit_service_url}/sfu/get`, async () => {
await deferredSFU.promise;
return {
status: 500,
body: "Internal Server Error",
};
});
connection.start().catch(() => {
// expected to throw
});
let capturedState = capturedStates.pop();
expect(capturedState).toBeDefined();
expect(capturedState?.state).toEqual("FetchingConfig");
deferredSFU.resolve();
await vi.runAllTimersAsync();
capturedState = capturedStates.pop();
if (capturedState?.state === "FailedToStart") {
expect(capturedState?.error.message).toContain(
"SFU Config fetch failed with exception Error",
);
expect(capturedState?.focus.livekit_alias).toEqual(
livekitFocus.livekit_alias,
);
} else {
expect.fail(
"Expected FailedToStart state but got " + capturedState?.state,
);
}
});
it("fail to connect to livekit error state", async () => {
setupTest();
vi.useFakeTimers();
const opts: ConnectionOpts = {
client: client,
transport: livekitFocus,
remoteTransports$: fakeMembershipsFocusMap$,
scope: testScope,
livekitRoomFactory: () => fakeLivekitRoom,
};
const connection = new RemoteConnection(opts, undefined);
const capturedStates: FocusConnectionState[] = [];
const s = connection.focusConnectionState$.subscribe((value) => {
capturedStates.push(value);
});
onTestFinished(() => s.unsubscribe());
const deferredSFU = Promise.withResolvers<void>();
// mock the /sfu/get call
fetchMock.post(`${livekitFocus.livekit_service_url}/sfu/get`, () => {
return {
status: 200,
body: {
url: "wss://matrix-rtc.m.localhost/livekit/sfu",
jwt: "ATOKEN",
},
};
});
fakeLivekitRoom.connect.mockImplementation(async () => {
await deferredSFU.promise;
throw new Error("Failed to connect to livekit");
});
connection.start().catch(() => {
// expected to throw
});
let capturedState = capturedStates.pop();
expect(capturedState).toBeDefined();
expect(capturedState?.state).toEqual("FetchingConfig");
deferredSFU.resolve();
await vi.runAllTimersAsync();
capturedState = capturedStates.pop();
if (capturedState && capturedState?.state === "FailedToStart") {
expect(capturedState.error.message).toContain(
"Failed to connect to livekit",
);
expect(capturedState.focus.livekit_alias).toEqual(
livekitFocus.livekit_alias,
);
} else {
expect.fail(
"Expected FailedToStart state but got " + JSON.stringify(capturedState),
);
}
});
it("connection states happy path", async () => {
vi.useFakeTimers();
setupTest();
const connection = setupRemoteConnection();
const capturedStates: FocusConnectionState[] = [];
const s = connection.focusConnectionState$.subscribe((value) => {
capturedStates.push(value);
});
onTestFinished(() => s.unsubscribe());
await connection.start();
await vi.runAllTimersAsync();
const initialState = capturedStates.shift();
expect(initialState?.state).toEqual("Initialized");
const fetchingState = capturedStates.shift();
expect(fetchingState?.state).toEqual("FetchingConfig");
const connectingState = capturedStates.shift();
expect(connectingState?.state).toEqual("ConnectingToLkRoom");
const connectedState = capturedStates.shift();
expect(connectedState?.state).toEqual("ConnectedToLkRoom");
});
it("should relay livekit events once connected", async () => {
setupTest();
const connection = setupRemoteConnection();
await connection.start();
let capturedStates: FocusConnectionState[] = [];
const s = connection.focusConnectionState$.subscribe((value) => {
capturedStates.push(value);
});
onTestFinished(() => s.unsubscribe());
const states = [
ConnectionState.Disconnected,
ConnectionState.Connecting,
ConnectionState.Connected,
ConnectionState.SignalReconnecting,
ConnectionState.Connecting,
ConnectionState.Connected,
ConnectionState.Reconnecting,
];
for (const state of states) {
fakeRoomEventEmiter.emit(RoomEvent.ConnectionStateChanged, state);
}
for (const state of states) {
const s = capturedStates.shift();
expect(s?.state).toEqual("ConnectedToLkRoom");
const connectedState = s as FocusConnectionState & {
state: "ConnectedToLkRoom";
};
expect(connectedState.connectionState).toEqual(state);
// should always have the focus info
expect(connectedState.focus.livekit_alias).toEqual(
livekitFocus.livekit_alias,
);
expect(connectedState.focus.livekit_service_url).toEqual(
livekitFocus.livekit_service_url,
);
}
// If the state is not ConnectedToLkRoom, no events should be relayed anymore
await connection.stop();
capturedStates = [];
for (const state of states) {
fakeRoomEventEmiter.emit(RoomEvent.ConnectionStateChanged, state);
}
expect(capturedStates.length).toEqual(0);
});
it("shutting down the scope should stop the connection", async () => {
setupTest();
vi.useFakeTimers();
const connection = setupRemoteConnection();
await connection.start();
const stopSpy = vi.spyOn(connection, "stop");
testScope.end();
expect(stopSpy).toHaveBeenCalled();
expect(fakeLivekitRoom.disconnect).toHaveBeenCalled();
});
});
function fakeRemoteLivekitParticipant(id: string): RemoteParticipant {
return vi.mocked<RemoteParticipant>({
identity: id,
} as unknown as RemoteParticipant);
}
function fakeRtcMemberShip(userId: string, deviceId: string): CallMembership {
return vi.mocked<CallMembership>({
sender: userId,
deviceId: deviceId,
} as unknown as CallMembership);
}
describe("Publishing participants observations", () => {
it("should emit the list of publishing participants", async () => {
setupTest();
const connection = setupRemoteConnection();
const bobIsAPublisher = Promise.withResolvers<void>();
const danIsAPublisher = Promise.withResolvers<void>();
const observedPublishers: {
participant: RemoteParticipant;
membership: CallMembership;
}[][] = [];
const s = connection.publishingParticipants$.subscribe((publishers) => {
observedPublishers.push(publishers);
if (
publishers.some(
(p) => p.participant.identity === "@bob:example.org:DEV111",
)
) {
bobIsAPublisher.resolve();
}
if (
publishers.some(
(p) => p.participant.identity === "@dan:example.org:DEV333",
)
) {
danIsAPublisher.resolve();
}
});
onTestFinished(() => s.unsubscribe());
// The publishingParticipants$ observable is derived from the current members of the
// livekitRoom and the rtc membership in order to publish the members that are publishing
// on this connection.
let participants: RemoteParticipant[] = [
fakeRemoteLivekitParticipant("@alice:example.org:DEV000"),
fakeRemoteLivekitParticipant("@bob:example.org:DEV111"),
fakeRemoteLivekitParticipant("@carol:example.org:DEV222"),
fakeRemoteLivekitParticipant("@dan:example.org:DEV333"),
];
// Let's simulate 3 members on the livekitRoom
vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get").mockReturnValue(
new Map(participants.map((p) => [p.identity, p])),
);
for (const participant of participants) {
fakeRoomEventEmiter.emit(RoomEvent.ParticipantConnected, participant);
}
// At this point there should be no publishers
expect(observedPublishers.pop()!.length).toEqual(0);
const otherFocus: LivekitTransport = {
livekit_alias: "!roomID:example.org",
livekit_service_url: "https://other-matrix-rtc.example.org/livekit/jwt",
type: "livekit",
};
const rtcMemberships = [
// Say bob is on the same focus
{
membership: fakeRtcMemberShip("@bob:example.org", "DEV111"),
transport: livekitFocus,
},
// Alice and carol is on a different focus
{
membership: fakeRtcMemberShip("@alice:example.org", "DEV000"),
transport: otherFocus,
},
{
membership: fakeRtcMemberShip("@carol:example.org", "DEV222"),
transport: otherFocus,
},
// NO DAVE YET
];
// signal this change in rtc memberships
fakeMembershipsFocusMap$.next(rtcMemberships);
// We should have bob has a publisher now
await bobIsAPublisher.promise;
const publishers = observedPublishers.pop();
expect(publishers?.length).toEqual(1);
expect(publishers?.[0].participant.identity).toEqual(
"@bob:example.org:DEV111",
);
// Now let's make dan join the rtc memberships
rtcMemberships.push({
membership: fakeRtcMemberShip("@dan:example.org", "DEV333"),
transport: livekitFocus,
});
fakeMembershipsFocusMap$.next(rtcMemberships);
// We should have bob and dan has publishers now
await danIsAPublisher.promise;
const twoPublishers = observedPublishers.pop();
expect(twoPublishers?.length).toEqual(2);
expect(
twoPublishers?.some(
(p) => p.participant.identity === "@bob:example.org:DEV111",
),
).toBeTruthy();
expect(
twoPublishers?.some(
(p) => p.participant.identity === "@dan:example.org:DEV333",
),
).toBeTruthy();
// Now let's make bob leave the livekit room
participants = participants.filter(
(p) => p.identity !== "@bob:example.org:DEV111",
);
vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get").mockReturnValue(
new Map(participants.map((p) => [p.identity, p])),
);
fakeRoomEventEmiter.emit(
RoomEvent.ParticipantDisconnected,
fakeRemoteLivekitParticipant("@bob:example.org:DEV111"),
);
const updatedPublishers = observedPublishers.pop();
expect(updatedPublishers?.length).toEqual(1);
expect(
updatedPublishers?.some(
(p) => p.participant.identity === "@dan:example.org:DEV333",
),
).toBeTruthy();
});
it("should be scoped to parent scope", (): void => {
setupTest();
const connection = setupRemoteConnection();
let observedPublishers: {
participant: RemoteParticipant;
membership: CallMembership;
}[][] = [];
const s = connection.publishingParticipants$.subscribe((publishers) => {
observedPublishers.push(publishers);
});
onTestFinished(() => s.unsubscribe());
let participants: RemoteParticipant[] = [
fakeRemoteLivekitParticipant("@bob:example.org:DEV111"),
];
// Let's simulate 3 members on the livekitRoom
vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get").mockReturnValue(
new Map(participants.map((p) => [p.identity, p])),
);
for (const participant of participants) {
fakeRoomEventEmiter.emit(RoomEvent.ParticipantConnected, participant);
}
// At this point there should be no publishers
expect(observedPublishers.pop()!.length).toEqual(0);
const rtcMemberships = [
// Say bob is on the same focus
{
membership: fakeRtcMemberShip("@bob:example.org", "DEV111"),
transport: livekitFocus,
},
];
// signal this change in rtc memberships
fakeMembershipsFocusMap$.next(rtcMemberships);
// We should have bob has a publisher now
const publishers = observedPublishers.pop();
expect(publishers?.length).toEqual(1);
expect(publishers?.[0].participant.identity).toEqual(
"@bob:example.org:DEV111",
);
// end the parent scope
testScope.end();
observedPublishers = [];
// SHOULD NOT emit any more publishers as the scope is ended
participants = participants.filter(
(p) => p.identity !== "@bob:example.org:DEV111",
);
vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get").mockReturnValue(
new Map(participants.map((p) => [p.identity, p])),
);
fakeRoomEventEmiter.emit(
RoomEvent.ParticipantDisconnected,
fakeRemoteLivekitParticipant("@bob:example.org:DEV111"),
);
expect(observedPublishers.length).toEqual(0);
});
});
describe("PublishConnection", () => {
// let fakeBlurProcessor: ProcessorWrapper<BackgroundOptions>;
let roomFactoryMock: Mock<() => LivekitRoom>;
let muteStates: MockedObject<MuteStates>;
function setUpPublishConnection(): void {
setupTest();
roomFactoryMock = vi.fn().mockReturnValue(fakeLivekitRoom);
muteStates = mockMuteStates();
// fakeBlurProcessor = vi.mocked<ProcessorWrapper<BackgroundOptions>>({
// name: "BackgroundBlur",
// restart: vi.fn().mockResolvedValue(undefined),
// setOptions: vi.fn().mockResolvedValue(undefined),
// getOptions: vi.fn().mockReturnValue({ strength: 0.5 }),
// isRunning: vi.fn().mockReturnValue(false)
// });
}
describe("Livekit room creation", () => {
function createSetup(): void {
setUpPublishConnection();
const fakeTrackProcessorSubject$ = new BehaviorSubject<ProcessorState>({
supported: true,
processor: undefined,
});
const opts: ConnectionOpts = {
client: client,
transport: livekitFocus,
remoteTransports$: fakeMembershipsFocusMap$,
scope: testScope,
livekitRoomFactory: roomFactoryMock,
};
const audioInput = {
available$: of(new Map([["mic1", { id: "mic1" }]])),
selected$: new BehaviorSubject({ id: "mic1" }),
select(): void {},
};
const videoInput = {
available$: of(new Map([["cam1", { id: "cam1" }]])),
selected$: new BehaviorSubject({ id: "cam1" }),
select(): void {},
};
const audioOutput = {
available$: of(new Map([["speaker", { id: "speaker" }]])),
selected$: new BehaviorSubject({ id: "speaker" }),
select(): void {},
};
// TODO understand what is wrong with our mocking that requires ts-expect-error
const fakeDevices = mockMediaDevices({
// @ts-expect-error Mocking only
audioInput,
// @ts-expect-error Mocking only
videoInput,
// @ts-expect-error Mocking only
audioOutput,
});
new PublishConnection(
opts,
fakeDevices,
muteStates,
undefined,
fakeTrackProcessorSubject$,
);
}
it("should create room with proper initial audio and video settings", () => {
createSetup();
expect(roomFactoryMock).toHaveBeenCalled();
const lastCallArgs =
roomFactoryMock.mock.calls[roomFactoryMock.mock.calls.length - 1];
const roomOptions = lastCallArgs.pop() as unknown as RoomOptions;
expect(roomOptions).toBeDefined();
expect(roomOptions!.videoCaptureDefaults?.deviceId).toEqual("cam1");
expect(roomOptions!.audioCaptureDefaults?.deviceId).toEqual("mic1");
expect(roomOptions!.audioOutput?.deviceId).toEqual("speaker");
});
it("respect controlledAudioDevices", () => {
// TODO: Refactor the code to make it testable.
// The UrlParams module is a singleton has a cache and is very hard to test.
// This breaks other tests as well if not handled properly.
// vi.mock(import("./../UrlParams"), () => {
// return {
// getUrlParams: vi.fn().mockReturnValue({
// controlledAudioDevices: true
// })
// };
// });
});
});
});

449
src/state/Connection.ts
View File

@@ -10,293 +10,272 @@ import {
connectionStateObserver,
} from "@livekit/components-core";
import {
ConnectionState,
Room as LivekitRoom,
ConnectionError,
type ConnectionState,
type E2EEOptions,
Track,
LocalVideoTrack,
Room as LivekitRoom,
type RoomOptions,
} from "livekit-client";
import { type MatrixClient } from "matrix-js-sdk";
import {
type LivekitTransport,
type CallMembership,
type LivekitTransport,
} from "matrix-js-sdk/lib/matrixrtc";
import {
combineLatest,
map,
NEVER,
type Observable,
type Subscription,
switchMap,
} from "rxjs";
import { logger } from "matrix-js-sdk/lib/logger";
import { BehaviorSubject, combineLatest } from "rxjs";
import { type SelectedDevice, type MediaDevices } from "./MediaDevices";
import { getSFUConfigWithOpenID } from "../livekit/openIDSFU";
import {
getSFUConfigWithOpenID,
type OpenIDClientParts,
type SFUConfig,
} from "../livekit/openIDSFU";
import { type Behavior } from "./Behavior";
import { type ObservableScope } from "./ObservableScope";
import { defaultLiveKitOptions } from "../livekit/options";
import { getValue } from "../utils/observable";
import { getUrlParams } from "../UrlParams";
import { type MuteStates } from "./MuteStates";
import {
type ProcessorState,
trackProcessorSync,
} from "../livekit/TrackProcessorContext";
import { observeTrackReference$ } from "./MediaViewModel";
InsufficientCapacityError,
SFURoomCreationRestrictedError,
} from "../utils/errors.ts";
export interface ConnectionOpts {
/** The focus server to connect to. */
transport: LivekitTransport;
/** The Matrix client to use for OpenID and SFU config requests. */
client: OpenIDClientParts;
/** The observable scope to use for this connection. */
scope: ObservableScope;
/** An observable of the current RTC call memberships and their associated focus. */
remoteTransports$: Behavior<
{ membership: CallMembership; transport: LivekitTransport }[]
>;
/** Optional factory to create the Livekit room, mainly for testing purposes. */
livekitRoomFactory?: (options?: RoomOptions) => LivekitRoom;
}
export type FocusConnectionState =
| { state: "Initialized" }
| { state: "FetchingConfig"; focus: LivekitTransport }
| { state: "ConnectingToLkRoom"; focus: LivekitTransport }
| { state: "PublishingTracks"; focus: LivekitTransport }
| { state: "FailedToStart"; error: Error; focus: LivekitTransport }
| {
state: "ConnectedToLkRoom";
connectionState: ConnectionState;
focus: LivekitTransport;
}
| { state: "Stopped"; focus: LivekitTransport };
/**
* A connection to a Matrix RTC LiveKit backend.
*
* Expose observables for participants and connection state.
*/
export class Connection {
// Private Behavior
private readonly _focusConnectionState$ =
new BehaviorSubject<FocusConnectionState>({ state: "Initialized" });
/**
* The current state of the connection to the focus server.
*/
public readonly focusConnectionState$: Behavior<FocusConnectionState> =
this._focusConnectionState$;
/**
* Whether the connection has been stopped.
* @see Connection.stop
* */
protected stopped = false;
/**
* Starts the connection.
*
* This will:
* 1. Request an OpenId token `request_token` (allows matrix users to verify their identity with a third-party service.)
* 2. Use this token to request the SFU config to the MatrixRtc authentication service.
* 3. Connect to the configured LiveKit room.
*
* @throws {InsufficientCapacityError} if the LiveKit server indicates that it has insufficient capacity to accept the connection.
* @throws {SFURoomCreationRestrictedError} if the LiveKit server indicates that the room does not exist and cannot be created.
*/
public async start(): Promise<void> {
this.stopped = false;
const { url, jwt } = await this.sfuConfig;
if (!this.stopped) await this.livekitRoom.connect(url, jwt);
try {
this._focusConnectionState$.next({
state: "FetchingConfig",
focus: this.localTransport,
});
// TODO could this be loaded earlier to save time?
const { url, jwt } = await this.getSFUConfigWithOpenID();
// If we were stopped while fetching the config, don't proceed to connect
if (this.stopped) return;
this._focusConnectionState$.next({
state: "ConnectingToLkRoom",
focus: this.localTransport,
});
try {
await this.livekitRoom.connect(url, jwt);
} catch (e) {
// LiveKit uses 503 to indicate that the server has hit its track limits.
// https://github.com/livekit/livekit/blob/fcb05e97c5a31812ecf0ca6f7efa57c485cea9fb/pkg/service/rtcservice.go#L171
// It also errors with a status code of 200 (yes, really) for room
// participant limits.
// LiveKit Cloud uses 429 for connection limits.
// Either way, all these errors can be explained as "insufficient capacity".
if (e instanceof ConnectionError) {
if (e.status === 503 || e.status === 200 || e.status === 429) {
throw new InsufficientCapacityError();
}
if (e.status === 404) {
// error msg is "Could not establish signal connection: requested room does not exist"
// The room does not exist. There are two different modes of operation for the SFU:
// - the room is created on the fly when connecting (livekit `auto_create` option)
// - Only authorized users can create rooms, so the room must exist before connecting (done by the auth jwt service)
// In the first case there will not be a 404, so we are in the second case.
throw new SFURoomCreationRestrictedError();
}
}
throw e;
}
// If we were stopped while connecting, don't proceed to update state.
if (this.stopped) return;
this._focusConnectionState$.next({
state: "ConnectedToLkRoom",
focus: this.localTransport,
connectionState: this.livekitRoom.state,
});
} catch (error) {
this._focusConnectionState$.next({
state: "FailedToStart",
error: error instanceof Error ? error : new Error(`${error}`),
focus: this.localTransport,
});
throw error;
}
}
public stop(): void {
protected async getSFUConfigWithOpenID(): Promise<SFUConfig> {
return await getSFUConfigWithOpenID(
this.client,
this.localTransport.livekit_service_url,
this.localTransport.livekit_alias,
);
}
/**
* Stops the connection.
*
* This will disconnect from the LiveKit room.
* If the connection is already stopped, this is a no-op.
*/
public async stop(): Promise<void> {
if (this.stopped) return;
void this.livekitRoom.disconnect();
await this.livekitRoom.disconnect();
this._focusConnectionState$.next({
state: "Stopped",
focus: this.localTransport,
});
this.stopped = true;
}
protected readonly sfuConfig = getSFUConfigWithOpenID(
this.client,
this.transport.livekit_service_url,
this.livekitAlias,
);
private readonly participantsIncludingSubscribers$;
/**
* An observable of the participants that are publishing on this connection.
* This is derived from `participantsIncludingSubscribers$` and `remoteTransports$`.
* It filters the participants to only those that are associated with a membership that claims to publish on this connection.
*/
public readonly publishingParticipants$;
public readonly livekitRoom: LivekitRoom;
public connectionState$: Behavior<ConnectionState>;
public constructor(
public readonly transport: LivekitTransport,
protected readonly livekitAlias: string,
protected readonly client: MatrixClient,
protected readonly scope: ObservableScope,
protected readonly remoteTransports$: Behavior<
{ membership: CallMembership; transport: LivekitTransport }[]
>,
e2eeLivekitOptions: E2EEOptions | undefined,
livekitRoom: LivekitRoom | undefined = undefined,
/**
* The focus server to connect to.
*/
public readonly localTransport: LivekitTransport;
private readonly client: OpenIDClientParts;
/**
* Creates a new connection to a matrix RTC LiveKit backend.
*
* @param livekitRoom - LiveKit room instance to use.
* @param opts - Connection options {@link ConnectionOpts}.
*
*/
protected constructor(
public readonly livekitRoom: LivekitRoom,
opts: ConnectionOpts,
) {
this.livekitRoom =
livekitRoom ??
new LivekitRoom({
...defaultLiveKitOptions,
e2ee: e2eeLivekitOptions,
});
this.participantsIncludingSubscribers$ = this.scope.behavior(
const { transport, client, scope, remoteTransports$ } = opts;
this.localTransport = transport;
this.client = client;
const participantsIncludingSubscribers$ = scope.behavior(
connectedParticipantsObserver(this.livekitRoom),
[],
);
this.publishingParticipants$ = this.scope.behavior(
this.publishingParticipants$ = scope.behavior(
combineLatest(
[this.participantsIncludingSubscribers$, this.remoteTransports$],
[participantsIncludingSubscribers$, remoteTransports$],
(participants, remoteTransports) =>
remoteTransports
// Find all members that claim to publish on this connection
.flatMap(({ membership, transport }) =>
transport.livekit_service_url ===
this.transport.livekit_service_url
this.localTransport.livekit_service_url
? [membership]
: [],
)
// Pair with their associated LiveKit participant (if any)
.map((membership) => {
// Uses flatMap to filter out memberships with no associated rtc participant ([])
.flatMap((membership) => {
const id = `${membership.sender}:${membership.deviceId}`;
const participant = participants.find((p) => p.identity === id);
return { participant, membership };
return participant ? [{ participant, membership }] : [];
}),
),
[],
);
this.connectionState$ = this.scope.behavior<ConnectionState>(
connectionStateObserver(this.livekitRoom),
);
this.scope.onEnd(() => this.stop());
scope
.behavior<ConnectionState>(connectionStateObserver(this.livekitRoom))
.subscribe((connectionState) => {
const current = this._focusConnectionState$.value;
// Only update the state if we are already connected to the LiveKit room.
if (current.state === "ConnectedToLkRoom") {
this._focusConnectionState$.next({
state: "ConnectedToLkRoom",
connectionState,
focus: current.focus,
});
}
});
scope.onEnd(() => void this.stop());
}
}
export class PublishConnection extends Connection {
public async start(): Promise<void> {
this.stopped = false;
this.muteStates.audio.setHandler(async (desired) => {
try {
await this.livekitRoom.localParticipant.setMicrophoneEnabled(desired);
} catch (e) {
logger.error("Failed to update LiveKit audio input mute state", e);
}
return this.livekitRoom.localParticipant.isMicrophoneEnabled;
});
this.muteStates.video.setHandler(async (desired) => {
try {
await this.livekitRoom.localParticipant.setCameraEnabled(desired);
} catch (e) {
logger.error("Failed to update LiveKit video input mute state", e);
}
return this.livekitRoom.localParticipant.isCameraEnabled;
});
const { url, jwt } = await this.sfuConfig;
if (!this.stopped) await this.livekitRoom.connect(url, jwt);
if (!this.stopped) {
// TODO-MULTI-SFU: Prepublish a microphone track
const audio = this.muteStates.audio.enabled$.value;
const video = this.muteStates.video.enabled$.value;
// createTracks throws if called with audio=false and video=false
if (audio || video) {
const tracks = await this.livekitRoom.localParticipant.createTracks({
audio,
video,
});
for (const track of tracks) {
await this.livekitRoom.localParticipant.publishTrack(track);
}
}
}
}
public stop(): void {
this.muteStates.audio.unsetHandler();
this.muteStates.video.unsetHandler();
super.stop();
}
/**
* A remote connection to the Matrix RTC LiveKit backend.
*
* This connection is used for subscribing to remote participants.
* It does not publish any local tracks.
*/
export class RemoteConnection extends Connection {
/**
* Creates a new remote connection to a matrix RTC LiveKit backend.
* @param opts
* @param sharedE2eeOption - The shared E2EE options to use for the connection.
*/
public constructor(
transport: LivekitTransport,
livekitAlias: string,
client: MatrixClient,
scope: ObservableScope,
remoteTransports$: Behavior<
{ membership: CallMembership; transport: LivekitTransport }[]
>,
devices: MediaDevices,
private readonly muteStates: MuteStates,
e2eeLivekitOptions: E2EEOptions | undefined,
trackerProcessorState$: Behavior<ProcessorState>,
opts: ConnectionOpts,
sharedE2eeOption: E2EEOptions | undefined,
) {
logger.info("[LivekitRoom] Create LiveKit room");
const { controlledAudioDevices } = getUrlParams();
const room = new LivekitRoom({
const factory =
opts.livekitRoomFactory ??
((options: RoomOptions): LivekitRoom => new LivekitRoom(options));
const livekitRoom = factory({
...defaultLiveKitOptions,
videoCaptureDefaults: {
...defaultLiveKitOptions.videoCaptureDefaults,
deviceId: devices.videoInput.selected$.value?.id,
processor: trackerProcessorState$.value.processor,
},
audioCaptureDefaults: {
...defaultLiveKitOptions.audioCaptureDefaults,
deviceId: devices.audioInput.selected$.value?.id,
},
audioOutput: {
// When using controlled audio devices, we don't want to set the
// deviceId here, because it will be set by the native app.
// (also the id does not need to match a browser device id)
deviceId: controlledAudioDevices
? undefined
: getValue(devices.audioOutput.selected$)?.id,
},
e2ee: e2eeLivekitOptions,
e2ee: sharedE2eeOption,
});
room.setE2EEEnabled(e2eeLivekitOptions !== undefined).catch((e) => {
logger.error("Failed to set E2EE enabled on room", e);
});
super(
transport,
livekitAlias,
client,
scope,
remoteTransports$,
e2eeLivekitOptions,
room,
);
// Setup track processor syncing (blur)
const track$ = this.scope.behavior(
observeTrackReference$(room.localParticipant, Track.Source.Camera).pipe(
map((trackRef) => {
const track = trackRef?.publication?.track;
return track instanceof LocalVideoTrack ? track : null;
}),
),
);
trackProcessorSync(track$, trackerProcessorState$);
const syncDevice = (
kind: MediaDeviceKind,
selected$: Observable<SelectedDevice | undefined>,
): Subscription =>
selected$.pipe(this.scope.bind()).subscribe((device) => {
if (this.connectionState$.value !== ConnectionState.Connected) return;
logger.info(
"[LivekitRoom] syncDevice room.getActiveDevice(kind) !== d.id :",
this.livekitRoom.getActiveDevice(kind),
" !== ",
device?.id,
);
if (
device !== undefined &&
this.livekitRoom.getActiveDevice(kind) !== device.id
) {
this.livekitRoom
.switchActiveDevice(kind, device.id)
.catch((e) =>
logger.error(`Failed to sync ${kind} device with LiveKit`, e),
);
}
});
syncDevice("audioinput", devices.audioInput.selected$);
if (!controlledAudioDevices)
syncDevice("audiooutput", devices.audioOutput.selected$);
syncDevice("videoinput", devices.videoInput.selected$);
// Restart the audio input track whenever we detect that the active media
// device has changed to refer to a different hardware device. We do this
// for the sake of Chrome, which provides a "default" device that is meant
// to match the system's default audio input, whatever that may be.
// This is special-cased for only audio inputs because we need to dig around
// in the LocalParticipant object for the track object and there's not a nice
// way to do that generically. There is usually no OS-level default video capture
// device anyway, and audio outputs work differently.
devices.audioInput.selected$
.pipe(
switchMap((device) => device?.hardwareDeviceChange$ ?? NEVER),
this.scope.bind(),
)
.subscribe(() => {
if (this.connectionState$.value !== ConnectionState.Connected) return;
const activeMicTrack = Array.from(
this.livekitRoom.localParticipant.audioTrackPublications.values(),
).find((d) => d.source === Track.Source.Microphone)?.track;
if (
activeMicTrack &&
// only restart if the stream is still running: LiveKit will detect
// when a track stops & restart appropriately, so this is not our job.
// Plus, we need to avoid restarting again if the track is already in
// the process of being restarted.
activeMicTrack.mediaStreamTrack.readyState !== "ended"
) {
// Restart the track, which will cause Livekit to do another
// getUserMedia() call with deviceId: default to get the *new* default device.
// Note that room.switchActiveDevice() won't work: Livekit will ignore it because
// the deviceId hasn't changed (was & still is default).
this.livekitRoom.localParticipant
.getTrackPublication(Track.Source.Microphone)
?.audioTrack?.restartTrack()
.catch((e) => {
logger.error(`Failed to restart audio device track`, e);
});
}
});
super(livekitRoom, opts);
}
}

2
src/state/GridLikeLayout.ts
View File

@@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type Layout, type LayoutMedia } from "./CallViewModel";
import { type Layout, type LayoutMedia } from "./layout-types.ts";
import { type TileStore } from "./TileStore";
export type GridLikeLayoutType =

9
src/state/MediaViewModel.ts
View File

@@ -266,6 +266,7 @@ abstract class BaseMediaViewModel extends ViewModel {
audioSource: AudioSource,
videoSource: VideoSource,
livekitRoom: LivekitRoom,
public readonly focusURL: string,
public readonly displayName$: Behavior<string>,
) {
super();
@@ -407,6 +408,7 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel {
participant$: Observable<LocalParticipant | RemoteParticipant | undefined>,
encryptionSystem: EncryptionSystem,
livekitRoom: LivekitRoom,
focusUrl: string,
displayName$: Behavior<string>,
public readonly handRaised$: Behavior<Date | null>,
public readonly reaction$: Behavior<ReactionOption | null>,
@@ -419,6 +421,7 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel {
Track.Source.Microphone,
Track.Source.Camera,
livekitRoom,
focusUrl,
displayName$,
);
@@ -539,6 +542,7 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel {
participant$: Behavior<LocalParticipant | undefined>,
encryptionSystem: EncryptionSystem,
livekitRoom: LivekitRoom,
focusURL: string,
private readonly mediaDevices: MediaDevices,
displayName$: Behavior<string>,
handRaised$: Behavior<Date | null>,
@@ -550,6 +554,7 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel {
participant$,
encryptionSystem,
livekitRoom,
focusURL,
displayName$,
handRaised$,
reaction$,
@@ -645,6 +650,7 @@ export class RemoteUserMediaViewModel extends BaseUserMediaViewModel {
participant$: Observable<RemoteParticipant | undefined>,
encryptionSystem: EncryptionSystem,
livekitRoom: LivekitRoom,
focusUrl: string,
private readonly pretendToBeDisconnected$: Behavior<boolean>,
displayname$: Behavior<string>,
handRaised$: Behavior<Date | null>,
@@ -656,6 +662,7 @@ export class RemoteUserMediaViewModel extends BaseUserMediaViewModel {
participant$,
encryptionSystem,
livekitRoom,
focusUrl,
displayname$,
handRaised$,
reaction$,
@@ -740,6 +747,7 @@ export class ScreenShareViewModel extends BaseMediaViewModel {
participant$: Observable<LocalParticipant | RemoteParticipant>,
encryptionSystem: EncryptionSystem,
livekitRoom: LivekitRoom,
focusUrl: string,
private readonly pretendToBeDisconnected$: Behavior<boolean>,
displayname$: Behavior<string>,
public readonly local: boolean,
@@ -752,6 +760,7 @@ export class ScreenShareViewModel extends BaseMediaViewModel {
Track.Source.ScreenShareAudio,
Track.Source.ScreenShare,
livekitRoom,
focusUrl,
displayname$,
);
}

10
src/state/MuteStates.ts
View File

@@ -88,7 +88,10 @@ class MuteState<Label, Selected> {
} else {
subscriber.next(enabled);
syncing = true;
sync();
sync().catch((err) => {
// TODO: better error handling
logger.error("MuteState: handler error", err);
});
}
}
};
@@ -97,7 +100,10 @@ class MuteState<Label, Selected> {
latestDesired = desired;
if (syncing === false) {
syncing = true;
sync();
sync().catch((err) => {
// TODO: better error handling
logger.error("MuteState: handler error", err);
});
}
});
return (): void => s.unsubscribe();

2
src/state/OneOnOneLayout.ts
View File

@@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type OneOnOneLayout, type OneOnOneLayoutMedia } from "./CallViewModel";
import { type OneOnOneLayout, type OneOnOneLayoutMedia } from "./layout-types";
import { type TileStore } from "./TileStore";
/**

2
src/state/PipLayout.ts
View File

@@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type PipLayout, type PipLayoutMedia } from "./CallViewModel";
import { type PipLayout, type PipLayoutMedia } from "./layout-types.ts";
import { type TileStore } from "./TileStore";
/**

280
src/state/PublishConnection.ts Normal file
View File

@@ -0,0 +1,280 @@
/*
Copyright 2025 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
ConnectionState,
type E2EEOptions,
LocalVideoTrack,
Room as LivekitRoom,
type RoomOptions,
Track,
} from "livekit-client";
import {
map,
NEVER,
type Observable,
type Subscription,
switchMap,
} from "rxjs";
import { logger } from "matrix-js-sdk/lib/logger";
import type { Behavior } from "./Behavior.ts";
import type { MediaDevices, SelectedDevice } from "./MediaDevices.ts";
import type { MuteStates } from "./MuteStates.ts";
import {
type ProcessorState,
trackProcessorSync,
} from "../livekit/TrackProcessorContext.tsx";
import { getUrlParams } from "../UrlParams.ts";
import { defaultLiveKitOptions } from "../livekit/options.ts";
import { getValue } from "../utils/observable.ts";
import { observeTrackReference$ } from "./MediaViewModel.ts";
import { Connection, type ConnectionOpts } from "./Connection.ts";
import { type ObservableScope } from "./ObservableScope.ts";
/**
* A connection to the local LiveKit room, the one the user is publishing to.
* This connection will publish the local user's audio and video tracks.
*/
export class PublishConnection extends Connection {
/**
* Creates a new PublishConnection.
* @param args - The connection options. {@link ConnectionOpts}
* @param devices - The media devices to use for audio and video input.
* @param muteStates - The mute states for audio and video.
* @param e2eeLivekitOptions - The E2EE options to use for the LiveKit room. Use to share the same key provider across connections!.
* @param trackerProcessorState$ - The processor state for the video track processor (e.g. background blur).
*/
public constructor(
args: ConnectionOpts,
devices: MediaDevices,
private readonly muteStates: MuteStates,
e2eeLivekitOptions: E2EEOptions | undefined,
trackerProcessorState$: Behavior<ProcessorState>,
) {
const { scope } = args;
logger.info("[LivekitRoom] Create LiveKit room");
const { controlledAudioDevices } = getUrlParams();
const factory =
args.livekitRoomFactory ??
((options: RoomOptions): LivekitRoom => new LivekitRoom(options));
const room = factory(
generateRoomOption(
devices,
trackerProcessorState$.value,
controlledAudioDevices,
e2eeLivekitOptions,
),
);
room.setE2EEEnabled(e2eeLivekitOptions !== undefined).catch((e) => {
logger.error("Failed to set E2EE enabled on room", e);
});
super(room, args);
// Setup track processor syncing (blur)
this.observeTrackProcessors(scope, room, trackerProcessorState$);
// Observe mute state changes and update LiveKit microphone/camera states accordingly
this.observeMuteStates(scope);
// Observe media device changes and update LiveKit active devices accordingly
this.observeMediaDevices(scope, devices, controlledAudioDevices);
this.workaroundRestartAudioInputTrackChrome(devices, scope);
}
/**
* Start the connection to LiveKit and publish local tracks.
*
* This will:
* 1. Request an OpenId token `request_token` (allows matrix users to verify their identity with a third-party service.)
* 2. Use this token to request the SFU config to the MatrixRtc authentication service.
* 3. Connect to the configured LiveKit room.
* 4. Create local audio and video tracks based on the current mute states and publish them to the room.
*
* @throws {InsufficientCapacityError} if the LiveKit server indicates that it has insufficient capacity to accept the connection.
* @throws {SFURoomCreationRestrictedError} if the LiveKit server indicates that the room does not exist and cannot be created.
*/
public async start(): Promise<void> {
this.stopped = false;
await super.start();
if (this.stopped) return;
// TODO this can throw errors? It will also prompt for permissions if not already granted
const tracks = await this.livekitRoom.localParticipant.createTracks({
audio: this.muteStates.audio.enabled$.value,
video: this.muteStates.video.enabled$.value,
});
if (this.stopped) return;
for (const track of tracks) {
// TODO: handle errors? Needs the signaling connection to be up, but it has some retries internally
// with a timeout.
await this.livekitRoom.localParticipant.publishTrack(track);
if (this.stopped) return;
// TODO: check if the connection is still active? and break the loop if not?
}
}
/// Private methods
// Restart the audio input track whenever we detect that the active media
// device has changed to refer to a different hardware device. We do this
// for the sake of Chrome, which provides a "default" device that is meant
// to match the system's default audio input, whatever that may be.
// This is special-cased for only audio inputs because we need to dig around
// in the LocalParticipant object for the track object and there's not a nice
// way to do that generically. There is usually no OS-level default video capture
// device anyway, and audio outputs work differently.
private workaroundRestartAudioInputTrackChrome(
devices: MediaDevices,
scope: ObservableScope,
): void {
devices.audioInput.selected$
.pipe(
switchMap((device) => device?.hardwareDeviceChange$ ?? NEVER),
scope.bind(),
)
.subscribe(() => {
if (this.livekitRoom.state != ConnectionState.Connected) return;
const activeMicTrack = Array.from(
this.livekitRoom.localParticipant.audioTrackPublications.values(),
).find((d) => d.source === Track.Source.Microphone)?.track;
if (
activeMicTrack &&
// only restart if the stream is still running: LiveKit will detect
// when a track stops & restart appropriately, so this is not our job.
// Plus, we need to avoid restarting again if the track is already in
// the process of being restarted.
activeMicTrack.mediaStreamTrack.readyState !== "ended"
) {
// Restart the track, which will cause Livekit to do another
// getUserMedia() call with deviceId: default to get the *new* default device.
// Note that room.switchActiveDevice() won't work: Livekit will ignore it because
// the deviceId hasn't changed (was & still is default).
this.livekitRoom.localParticipant
.getTrackPublication(Track.Source.Microphone)
?.audioTrack?.restartTrack()
.catch((e) => {
logger.error(`Failed to restart audio device track`, e);
});
}
});
}
// Observe changes in the selected media devices and update the LiveKit room accordingly.
private observeMediaDevices(
scope: ObservableScope,
devices: MediaDevices,
controlledAudioDevices: boolean,
): void {
const syncDevice = (
kind: MediaDeviceKind,
selected$: Observable<SelectedDevice | undefined>,
): Subscription =>
selected$.pipe(scope.bind()).subscribe((device) => {
if (this.livekitRoom.state != ConnectionState.Connected) return;
// if (this.connectionState$.value !== ConnectionState.Connected) return;
logger.info(
"[LivekitRoom] syncDevice room.getActiveDevice(kind) !== d.id :",
this.livekitRoom.getActiveDevice(kind),
" !== ",
device?.id,
);
if (
device !== undefined &&
this.livekitRoom.getActiveDevice(kind) !== device.id
) {
this.livekitRoom
.switchActiveDevice(kind, device.id)
.catch((e) =>
logger.error(`Failed to sync ${kind} device with LiveKit`, e),
);
}
});
syncDevice("audioinput", devices.audioInput.selected$);
if (!controlledAudioDevices)
syncDevice("audiooutput", devices.audioOutput.selected$);
syncDevice("videoinput", devices.videoInput.selected$);
}
/**
* Observe changes in the mute states and update the LiveKit room accordingly.
* @param scope
* @private
*/
private observeMuteStates(scope: ObservableScope): void {
this.muteStates.audio.setHandler(async (desired) => {
try {
await this.livekitRoom.localParticipant.setMicrophoneEnabled(desired);
} catch (e) {
logger.error("Failed to update LiveKit audio input mute state", e);
}
return this.livekitRoom.localParticipant.isMicrophoneEnabled;
});
this.muteStates.video.setHandler(async (desired) => {
try {
await this.livekitRoom.localParticipant.setCameraEnabled(desired);
} catch (e) {
logger.error("Failed to update LiveKit video input mute state", e);
}
return this.livekitRoom.localParticipant.isCameraEnabled;
});
scope.onEnd(() => {
this.muteStates.audio.unsetHandler();
this.muteStates.video.unsetHandler();
});
}
private observeTrackProcessors(
scope: ObservableScope,
room: LivekitRoom,
trackerProcessorState$: Behavior<ProcessorState>,
): void {
const track$ = scope.behavior(
observeTrackReference$(room.localParticipant, Track.Source.Camera).pipe(
map((trackRef) => {
const track = trackRef?.publication?.track;
return track instanceof LocalVideoTrack ? track : null;
}),
),
);
trackProcessorSync(track$, trackerProcessorState$);
}
}
// Generate the initial LiveKit RoomOptions based on the current media devices and processor state.
function generateRoomOption(
devices: MediaDevices,
processorState: ProcessorState,
controlledAudioDevices: boolean,
e2eeLivekitOptions: E2EEOptions | undefined,
): RoomOptions {
return {
...defaultLiveKitOptions,
videoCaptureDefaults: {
...defaultLiveKitOptions.videoCaptureDefaults,
deviceId: devices.videoInput.selected$.value?.id,
processor: processorState.processor,
},
audioCaptureDefaults: {
...defaultLiveKitOptions.audioCaptureDefaults,
deviceId: devices.audioInput.selected$.value?.id,
},
audioOutput: {
// When using controlled audio devices, we don't want to set the
// deviceId here, because it will be set by the native app.
// (also the id does not need to match a browser device id)
deviceId: controlledAudioDevices
? undefined
: getValue(devices.audioOutput.selected$)?.id,
},
e2ee: e2eeLivekitOptions,
};
}

57
src/state/ScreenShare.ts Normal file
View File

@@ -0,0 +1,57 @@
/*
Copyright 2025 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { BehaviorSubject, type Observable } from "rxjs";
import {
type LocalParticipant,
type RemoteParticipant,
type Room as LivekitRoom,
} from "livekit-client";
import { ObservableScope } from "./ObservableScope.ts";
import { ScreenShareViewModel } from "./MediaViewModel.ts";
import type { RoomMember } from "matrix-js-sdk";
import type { EncryptionSystem } from "../e2ee/sharedKeyManagement.ts";
import type { Behavior } from "./Behavior.ts";
// TODO Document this
export class ScreenShare {
private readonly scope = new ObservableScope();
public readonly vm: ScreenShareViewModel;
private readonly participant$: BehaviorSubject<
LocalParticipant | RemoteParticipant
>;
public constructor(
id: string,
member: RoomMember,
participant: LocalParticipant | RemoteParticipant,
encryptionSystem: EncryptionSystem,
livekitRoom: LivekitRoom,
focusUrl: string,
pretendToBeDisconnected$: Behavior<boolean>,
displayName$: Observable<string>,
) {
this.participant$ = new BehaviorSubject(participant);
this.vm = new ScreenShareViewModel(
id,
member,
this.participant$.asObservable(),
encryptionSystem,
livekitRoom,
focusUrl,
pretendToBeDisconnected$,
this.scope.behavior(displayName$),
participant.isLocal,
);
}
public destroy(): void {
this.scope.end();
this.vm.destroy();
}
}

2
src/state/SpotlightExpandedLayout.ts
View File

@@ -8,7 +8,7 @@ Please see LICENSE in the repository root for full details.
import {
type SpotlightExpandedLayout,
type SpotlightExpandedLayoutMedia,
} from "./CallViewModel";
} from "./layout-types";
import { type TileStore } from "./TileStore";
/**

121
src/state/UserMedia.ts Normal file
View File

@@ -0,0 +1,121 @@
/*
Copyright 2025 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { BehaviorSubject, map, type Observable, of, switchMap } from "rxjs";
import {
type LocalParticipant,
type Participant,
ParticipantEvent,
type RemoteParticipant,
type Room as LivekitRoom,
} from "livekit-client";
import { observeParticipantEvents } from "@livekit/components-core";
import { ObservableScope } from "./ObservableScope.ts";
import {
LocalUserMediaViewModel,
RemoteUserMediaViewModel,
type UserMediaViewModel,
} from "./MediaViewModel.ts";
import type { Behavior } from "./Behavior.ts";
import type { RoomMember } from "matrix-js-sdk";
import type { EncryptionSystem } from "../e2ee/sharedKeyManagement.ts";
import type { MediaDevices } from "./MediaDevices.ts";
import type { ReactionOption } from "../reactions";
import { observeSpeaker$ } from "./observeSpeaker.ts";
/**
* TODO Document this
*/
export class UserMedia {
private readonly scope = new ObservableScope();
public readonly vm: UserMediaViewModel;
private readonly participant$: BehaviorSubject<
LocalParticipant | RemoteParticipant | undefined
>;
public readonly speaker$: Behavior<boolean>;
public readonly presenter$: Behavior<boolean>;
public constructor(
public readonly id: string,
member: RoomMember,
participant: LocalParticipant | RemoteParticipant | undefined,
encryptionSystem: EncryptionSystem,
livekitRoom: LivekitRoom,
focusURL: string,
mediaDevices: MediaDevices,
pretendToBeDisconnected$: Behavior<boolean>,
displayname$: Observable<string>,
handRaised$: Observable<Date | null>,
reaction$: Observable<ReactionOption | null>,
) {
this.participant$ = new BehaviorSubject(participant);
if (participant?.isLocal) {
this.vm = new LocalUserMediaViewModel(
this.id,
member,
this.participant$ as Behavior<LocalParticipant>,
encryptionSystem,
livekitRoom,
focusURL,
mediaDevices,
this.scope.behavior(displayname$),
this.scope.behavior(handRaised$),
this.scope.behavior(reaction$),
);
} else {
this.vm = new RemoteUserMediaViewModel(
id,
member,
this.participant$.asObservable() as Observable<
RemoteParticipant | undefined
>,
encryptionSystem,
livekitRoom,
focusURL,
pretendToBeDisconnected$,
this.scope.behavior(displayname$),
this.scope.behavior(handRaised$),
this.scope.behavior(reaction$),
);
}
this.speaker$ = this.scope.behavior(observeSpeaker$(this.vm.speaking$));
this.presenter$ = this.scope.behavior(
this.participant$.pipe(
switchMap((p) => (p === undefined ? of(false) : sharingScreen$(p))),
),
);
}
public updateParticipant(
newParticipant: LocalParticipant | RemoteParticipant | undefined,
): void {
if (this.participant$.value !== newParticipant) {
// Update the BehaviourSubject in the UserMedia.
this.participant$.next(newParticipant);
}
}
public destroy(): void {
this.scope.end();
this.vm.destroy();
}
}
export function sharingScreen$(p: Participant): Observable<boolean> {
return observeParticipantEvents(
p,
ParticipantEvent.TrackPublished,
ParticipantEvent.TrackUnpublished,
ParticipantEvent.LocalTrackPublished,
ParticipantEvent.LocalTrackUnpublished,
).pipe(map((p) => p.isScreenShareEnabled));
}

108
src/state/layout-types.ts Normal file
View File

@@ -0,0 +1,108 @@
/*
Copyright 2025 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
type GridTileViewModel,
type SpotlightTileViewModel,
} from "./TileViewModel.ts";
import {
type MediaViewModel,
type UserMediaViewModel,
} from "./MediaViewModel.ts";
export interface GridLayoutMedia {
type: "grid";
spotlight?: MediaViewModel[];
grid: UserMediaViewModel[];
}
export interface SpotlightLandscapeLayoutMedia {
type: "spotlight-landscape";
spotlight: MediaViewModel[];
grid: UserMediaViewModel[];
}
export interface SpotlightPortraitLayoutMedia {
type: "spotlight-portrait";
spotlight: MediaViewModel[];
grid: UserMediaViewModel[];
}
export interface SpotlightExpandedLayoutMedia {
type: "spotlight-expanded";
spotlight: MediaViewModel[];
pip?: UserMediaViewModel;
}
export interface OneOnOneLayoutMedia {
type: "one-on-one";
local: UserMediaViewModel;
remote: UserMediaViewModel;
}
export interface PipLayoutMedia {
type: "pip";
spotlight: MediaViewModel[];
}
export type LayoutMedia =
| GridLayoutMedia
| SpotlightLandscapeLayoutMedia
| SpotlightPortraitLayoutMedia
| SpotlightExpandedLayoutMedia
| OneOnOneLayoutMedia
| PipLayoutMedia;
export interface GridLayout {
type: "grid";
spotlight?: SpotlightTileViewModel;
grid: GridTileViewModel[];
setVisibleTiles: (value: number) => void;
}
export interface SpotlightLandscapeLayout {
type: "spotlight-landscape";
spotlight: SpotlightTileViewModel;
grid: GridTileViewModel[];
setVisibleTiles: (value: number) => void;
}
export interface SpotlightPortraitLayout {
type: "spotlight-portrait";
spotlight: SpotlightTileViewModel;
grid: GridTileViewModel[];
setVisibleTiles: (value: number) => void;
}
export interface SpotlightExpandedLayout {
type: "spotlight-expanded";
spotlight: SpotlightTileViewModel;
pip?: GridTileViewModel;
}
export interface OneOnOneLayout {
type: "one-on-one";
local: GridTileViewModel;
remote: GridTileViewModel;
}
export interface PipLayout {
type: "pip";
spotlight: SpotlightTileViewModel;
}
/**
* A layout defining the media tiles present on screen and their visual
* arrangement.
*/
export type Layout =
| GridLayout
| SpotlightLandscapeLayout
| SpotlightPortraitLayout
| SpotlightExpandedLayout
| OneOnOneLayout
| PipLayout;

1
src/tile/GridTile.tsx
View File

@@ -190,6 +190,7 @@ const UserMediaTile: FC<UserMediaTileProps> = ({
currentReaction={reaction ?? undefined}
raisedHandOnClick={raisedHandOnClick}
localParticipant={vm.local}
focusUrl={vm.focusURL}
audioStreamStats={audioStreamStats}
videoStreamStats={videoStreamStats}
{...props}

26
src/tile/MediaView.test.tsx
View File

@@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { describe, expect, it, test } from "vitest";
import { describe, expect, it, test, vi } from "vitest";
import { render, screen } from "@testing-library/react";
import { axe } from "vitest-axe";
import { TooltipProvider } from "@vector-im/compound-web";
@@ -16,6 +16,7 @@ import {
import { LocalTrackPublication, Track } from "livekit-client";
import { TrackInfo } from "@livekit/protocol";
import { type ComponentProps } from "react";
import { type RoomMember } from "matrix-js-sdk";
import { MediaView } from "./MediaView";
import { EncryptionStatus } from "../state/MediaViewModel";
@@ -45,7 +46,10 @@ describe("MediaView", () => {
mirror: false,
unencryptedWarning: false,
video: trackReference,
member: undefined,
member: vi.mocked<RoomMember>({
userId: "@alice:example.com",
getMxcAvatarUrl: vi.fn().mockReturnValue(undefined),
} as unknown as RoomMember),
localParticipant: false,
focusable: true,
};
@@ -59,9 +63,9 @@ describe("MediaView", () => {
test("neither video nor avatar are shown", () => {
render(<MediaView {...baseProps} video={trackReferencePlaceholder} />);
expect(screen.queryByTestId("video")).toBeNull();
expect(screen.queryAllByRole("img", { name: "some name" }).length).toBe(
0,
);
expect(
screen.queryAllByRole("img", { name: "@alice:example.com" }).length,
).toBe(0);
});
});
@@ -70,14 +74,18 @@ describe("MediaView", () => {
render(
<MediaView {...baseProps} video={undefined} localParticipant={true} />,
);
expect(screen.getByRole("img", { name: "some name" })).toBeVisible();
expect(
screen.getByRole("img", { name: "@alice:example.com" }),
).toBeVisible();
expect(screen.queryAllByText("Waiting for media...").length).toBe(0);
});
it("shows avatar and label for remote user", () => {
render(
<MediaView {...baseProps} video={undefined} localParticipant={false} />,
);
expect(screen.getByRole("img", { name: "some name" })).toBeVisible();
expect(
screen.getByRole("img", { name: "@alice:example.com" }),
).toBeVisible();
expect(screen.getByText("Waiting for media...")).toBeVisible();
});
});
@@ -131,7 +139,9 @@ describe("MediaView", () => {
<MediaView {...baseProps} videoEnabled={false} />
</TooltipProvider>,
);
expect(screen.getByRole("img", { name: "some name" })).toBeVisible();
expect(
screen.getByRole("img", { name: "@alice:example.com" }),
).toBeVisible();
expect(screen.getByTestId("video")).not.toBeVisible();
});
});

4
src/tile/MediaView.tsx
View File

@@ -46,6 +46,8 @@ interface Props extends ComponentProps<typeof animated.div> {
localParticipant: boolean;
audioStreamStats?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats;
videoStreamStats?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats;
// The focus url, mainly for debugging purposes
focusUrl?: string;
}
export const MediaView: FC<Props> = ({
@@ -71,6 +73,7 @@ export const MediaView: FC<Props> = ({
localParticipant,
audioStreamStats,
videoStreamStats,
focusUrl,
...props
}) => {
const { t } = useTranslation();
@@ -134,6 +137,7 @@ export const MediaView: FC<Props> = ({
<RTCConnectionStats
audio={audioStreamStats}
video={videoStreamStats}
focusUrl={focusUrl}
/>
)}
{/* TODO: Bring this back once encryption status is less broken */}

2
src/tile/SpotlightTile.tsx
View File

@@ -78,7 +78,7 @@ const SpotlightLocalUserMediaItem: FC<SpotlightLocalUserMediaItemProps> = ({
...props
}) => {
const mirror = useBehavior(vm.mirror$);
return <MediaView mirror={mirror} {...props} />;
return <MediaView mirror={mirror} focusUrl={vm.focusURL} {...props} />;
};
SpotlightLocalUserMediaItem.displayName = "SpotlightLocalUserMediaItem";

19
src/useCallViewKeyboardShortcuts.test.tsx
View File

@@ -23,14 +23,14 @@ import {
// The TestComponent just wraps a button around that hook.
interface TestComponentProps {
setMicrophoneMuted?: (muted: boolean) => void;
setAudioEnabled?: (enabled: boolean) => void;
onButtonClick?: () => void;
sendReaction?: () => void;
toggleHandRaised?: () => void;
}
const TestComponent: FC<TestComponentProps> = ({
setMicrophoneMuted = (): void => {},
setAudioEnabled = (): void => {},
onButtonClick = (): void => {},
sendReaction = (reaction: ReactionOption): void => {},
toggleHandRaised = (): void => {},
@@ -40,7 +40,7 @@ const TestComponent: FC<TestComponentProps> = ({
ref,
() => {},
() => {},
setMicrophoneMuted,
setAudioEnabled,
sendReaction,
toggleHandRaised,
);
@@ -57,12 +57,13 @@ test("spacebar unmutes", async () => {
render(
<TestComponent
onButtonClick={() => (muted = false)}
setMicrophoneMuted={(m) => {
muted = m;
setAudioEnabled={(m) => {
muted = !m;
}}
/>,
);
expect(muted).toBe(true);
await user.keyboard("[Space>]");
expect(muted).toBe(false);
await user.keyboard("[/Space]");
@@ -73,15 +74,15 @@ test("spacebar unmutes", async () => {
test("spacebar prioritizes pressing a button", async () => {
const user = userEvent.setup();
const setMuted = vi.fn();
const setAudioEnabled = vi.fn();
const onClick = vi.fn();
render(
<TestComponent setMicrophoneMuted={setMuted} onButtonClick={onClick} />,
<TestComponent setAudioEnabled={setAudioEnabled} onButtonClick={onClick} />,
);
await user.tab(); // Focus the button
await user.keyboard("[Space]");
expect(setMuted).not.toBeCalled();
expect(setAudioEnabled).not.toBeCalled();
expect(onClick).toBeCalled();
});
@@ -129,7 +130,7 @@ test("unmuting happens in place of the default action", async () => {
tabIndex={0}
onKeyDown={(e) => defaultPrevented(e.isDefaultPrevented())}
>
<TestComponent setMicrophoneMuted={() => {}} />
<TestComponent setAudioEnabled={() => {}} />
</video>,
);

40
src/useMatrixRTCSessionJoinState.ts
View File

@@ -1,40 +0,0 @@
/*
Copyright 2023, 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { logger } from "matrix-js-sdk/lib/logger";
import {
type MatrixRTCSession,
MatrixRTCSessionEvent,
} from "matrix-js-sdk/lib/matrixrtc";
import { TypedEventEmitter } from "matrix-js-sdk";
import { useCallback, useEffect } from "react";
import { useTypedEventEmitterState } from "./useEvents";
const dummySession = new TypedEventEmitter();
export function useMatrixRTCSessionJoinState(
rtcSession: MatrixRTCSession | undefined,
): boolean {
// React doesn't allow you to run a hook conditionally, so we have to plug in
// a dummy event emitter in case there is no rtcSession yet
const isJoined = useTypedEventEmitterState(
rtcSession ?? dummySession,
MatrixRTCSessionEvent.JoinStateChanged,
useCallback(() => rtcSession?.isJoined() ?? false, [rtcSession]),
);
useEffect(() => {
logger.info(
`Session in room ${rtcSession?.room.roomId} changed to ${
isJoined ? "joined" : "left"
}`,
);
}, [rtcSession, isJoined]);
return isJoined;
}

3
src/utils/test-fixtures.ts
View File

@@ -9,7 +9,6 @@ import {
mockRtcMembership,
mockMatrixRoomMember,
mockRemoteParticipant,
mockLocalParticipant,
} from "./test";
export const localRtcMember = mockRtcMembership("@carol:example.org", "1111");
@@ -18,7 +17,7 @@ export const localRtcMemberDevice2 = mockRtcMembership(
"2222",
);
export const local = mockMatrixRoomMember(localRtcMember);
export const localParticipant = mockLocalParticipant({ identity: "" });
// export const localParticipant = mockLocalParticipant({ identity: "" });
export const localId = `${local.userId}:${localRtcMember.deviceId}`;
export const aliceRtcMember = mockRtcMembership("@alice:example.org", "AAAA");

11
src/utils/test-viewmodel.ts
View File

@@ -5,10 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
type CallMembership,
type MatrixRTCSession,
} from "matrix-js-sdk/lib/matrixrtc";
import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc";
import { BehaviorSubject, of } from "rxjs";
import { vitest } from "vitest";
import { type RelationsContainer } from "matrix-js-sdk/lib/models/relations-container";
@@ -99,12 +96,12 @@ export function getBasicRTCSession(
initialRtcMemberships,
);
const rtcSession = new MockRTCSession(matrixRoom).withMemberships(
const fakeRtcSession = new MockRTCSession(matrixRoom).withMemberships(
rtcMemberships$,
);
return {
rtcSession,
rtcSession: fakeRtcSession,
matrixRoom,
rtcMemberships$,
};
@@ -137,7 +134,7 @@ export function getBasicCallViewModelEnvironment(
// const remoteParticipants$ = of([aliceParticipant]);
const vm = new CallViewModel(
rtcSession as unknown as MatrixRTCSession,
rtcSession.asMockedSession(),
matrixRoom,
mockMediaDevices({}),
mockMuteStates(),

45
src/utils/test.ts
View File

@@ -6,7 +6,7 @@ Please see LICENSE in the repository root for full details.
*/
import { map, type Observable, of, type SchedulerLike } from "rxjs";
import { type RunHelpers, TestScheduler } from "rxjs/testing";
import { expect, vi, vitest } from "vitest";
import { expect, type MockedObject, vi, vitest } from "vitest";
import {
type RoomMember,
type Room as MatrixRoom,
@@ -23,11 +23,13 @@ import {
type SessionMembershipData,
Status,
type LivekitFocusSelection,
type MatrixRTCSession,
} from "matrix-js-sdk/lib/matrixrtc";
import { type MembershipManagerEventHandlerMap } from "matrix-js-sdk/lib/matrixrtc/IMembershipManager";
import {
type LocalParticipant,
type LocalTrackPublication,
type Participant,
type RemoteParticipant,
type RemoteTrackPublication,
type Room as LivekitRoom,
@@ -191,8 +193,12 @@ export function mockRtcMembership(
const event = new MatrixEvent({
sender: typeof user === "string" ? user : user.userId,
event_id: `$-ev-${randomUUID()}:example.org`,
content: data,
});
return new CallMembership(event, data);
const cms = new CallMembership(event, data);
vi.mocked(cms).getTransport = vi.fn().mockReturnValue(fociPreferred[0]);
return cms;
}
// Maybe it'd be good to move this to matrix-js-sdk? Our testing needs are
@@ -205,6 +211,10 @@ export function mockMatrixRoomMember(
return {
...mockEmitter(),
userId: rtcMembership.sender,
getMxcAvatarUrl(): string | undefined {
return undefined;
},
rawDisplayName: rtcMembership.sender,
...member,
} as RoomMember;
}
@@ -264,6 +274,7 @@ export async function withLocalMedia(
kind: E2eeType.PER_PARTICIPANT,
},
mockLivekitRoom({ localParticipant }),
"https://rtc-example.org",
mediaDevices,
constant(roomMember.rawDisplayName ?? "nodisplayname"),
constant(null),
@@ -304,6 +315,7 @@ export async function withRemoteMedia(
kind: E2eeType.PER_PARTICIPANT,
},
mockLivekitRoom({}, { remoteParticipants$: of([remoteParticipant]) }),
"https://rtc-example.org",
constant(false),
constant(roomMember.rawDisplayName ?? "nodisplayname"),
constant(null),
@@ -331,6 +343,19 @@ export class MockRTCSession extends TypedEventEmitter<
RoomAndToDeviceEventsHandlerMap &
MembershipManagerEventHandlerMap
> {
public asMockedSession(): MockedObject<MatrixRTCSession> {
const session = this as unknown as MockedObject<MatrixRTCSession>;
vi.mocked(session).reemitEncryptionKeys = vi
.fn<() => void>()
.mockReturnValue(undefined);
vi.mocked(session).getOldestMembership = vi
.fn<() => CallMembership | undefined>()
.mockReturnValue(this.memberships[0]);
return session;
}
public readonly statistics = {
counters: {},
};
@@ -389,15 +414,17 @@ export class MockRTCSession extends TypedEventEmitter<
}
}
export const mockTrack = (identity: string): TrackReference =>
export const mockTrack = (
participant: Participant,
kind?: Track.Kind,
source?: Track.Source,
): TrackReference =>
({
participant: {
identity,
},
participant,
publication: {
kind: Track.Kind.Audio,
source: "mic",
trackSid: "123",
kind: kind ?? Track.Kind.Audio,
source: source ?? Track.Source.Microphone,
trackSid: `123##${participant.identity}`,
track: {
attach: vi.fn(),
detach: vi.fn(),

3
vite.config.ts
View File

@@ -97,6 +97,9 @@ export default ({
cert: fs.readFileSync("./backend/dev_tls_m.localhost.crt"),
},
},
worker: {
format: "es",
},
build: {
minify: mode === "production" ? true : false,
sourcemap: true,

53
yarn.lock
View File

@@ -5176,6 +5176,13 @@ __metadata:
languageName: node
linkType: hard
"@types/glob-to-regexp@npm:^0.4.4":
version: 0.4.4
resolution: "@types/glob-to-regexp@npm:0.4.4"
checksum: 10c0/7288ff853850d8302a8770a3698b187fc3970ad12ee6427f0b3758a3e7a0ebb0bd993abc6ebaaa979d09695b4194157d2bfaa7601b0fb9ed72c688b4c1298b88
languageName: node
linkType: hard
"@types/grecaptcha@npm:^3.0.9":
version: 3.0.9
resolution: "@types/grecaptcha@npm:3.0.9"
@@ -7528,6 +7535,7 @@ __metadata:
eslint-plugin-react-hooks: "npm:^5.0.0"
eslint-plugin-rxjs: "npm:^5.0.3"
eslint-plugin-unicorn: "npm:^56.0.0"
fetch-mock: "npm:11.1.5"
global-jsdom: "npm:^26.0.0"
i18next: "npm:^24.0.0"
i18next-browser-languagedetector: "npm:^8.0.0"
@@ -7537,7 +7545,7 @@ __metadata:
livekit-client: "npm:^2.13.0"
lodash-es: "npm:^4.17.21"
loglevel: "npm:^1.9.1"
matrix-js-sdk: "github:matrix-org/matrix-js-sdk#head=toger5/use-relation-based-CallMembership-create-ts"
matrix-js-sdk: "github:matrix-org/matrix-js-sdk#head=develop"
matrix-widget-api: "npm:^1.13.0"
normalize.css: "npm:^8.0.1"
observable-hooks: "npm:^4.2.3"
@@ -8495,6 +8503,22 @@ __metadata:
languageName: node
linkType: hard
"fetch-mock@npm:11.1.5":
version: 11.1.5
resolution: "fetch-mock@npm:11.1.5"
dependencies:
"@types/glob-to-regexp": "npm:^0.4.4"
dequal: "npm:^2.0.3"
glob-to-regexp: "npm:^0.4.1"
is-subset: "npm:^0.1.1"
regexparam: "npm:^3.0.0"
peerDependenciesMeta:
node-fetch:
optional: true
checksum: 10c0/f32f1d7879b654a3fab7c3576901193ddd4c63cb9aeae2ed66ff42062400c0937d4696b1a5171e739d5f62470e6554e190f14816789f5e3b2bf1ad90208222e6
languageName: node
linkType: hard
"fflate@npm:^0.4.8":
version: 0.4.8
resolution: "fflate@npm:0.4.8"
@@ -8876,6 +8900,13 @@ __metadata:
languageName: node
linkType: hard
"glob-to-regexp@npm:^0.4.1":
version: 0.4.1
resolution: "glob-to-regexp@npm:0.4.1"
checksum: 10c0/0486925072d7a916f052842772b61c3e86247f0a80cc0deb9b5a3e8a1a9faad5b04fb6f58986a09f34d3e96cd2a22a24b7e9882fb1cf904c31e9a310de96c429
languageName: node
linkType: hard
"glob@npm:^10.2.2, glob@npm:^10.3.10, glob@npm:^10.3.7, glob@npm:^10.4.1":
version: 10.4.5
resolution: "glob@npm:10.4.5"
@@ -9611,6 +9642,13 @@ __metadata:
languageName: node
linkType: hard
"is-subset@npm:^0.1.1":
version: 0.1.1
resolution: "is-subset@npm:0.1.1"
checksum: 10c0/d8125598ab9077a76684e18726fb915f5cea7a7358ed0c6ff723f4484d71a0a9981ee5aae06c44de99cfdef0fefce37438c6257ab129e53c82045ea0c2acdebf
languageName: node
linkType: hard
"is-symbol@npm:^1.0.4, is-symbol@npm:^1.1.1":
version: 1.1.1
resolution: "is-symbol@npm:1.1.1"
@@ -10297,9 +10335,9 @@ __metadata:
languageName: node
linkType: hard
"matrix-js-sdk@github:matrix-org/matrix-js-sdk#head=toger5/use-relation-based-CallMembership-create-ts":
"matrix-js-sdk@github:matrix-org/matrix-js-sdk#head=develop":
version: 38.4.0
resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=4608506288c6beaa252982d224e996e23e51f681"
resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=fd949fe486038099ee111a72b57ce711e85bc352"
dependencies:
"@babel/runtime": "npm:^7.12.5"
"@matrix-org/matrix-sdk-crypto-wasm": "npm:^15.3.0"
@@ -10315,7 +10353,7 @@ __metadata:
sdp-transform: "npm:^2.14.1"
unhomoglyph: "npm:^1.0.6"
uuid: "npm:13"
checksum: 10c0/2e896d6a92cb3bbb47c120a39dd1a0030b4bf02289cb914f6c848b564208f421ada605e8efb68f6d9d55a0d2e3f86698b6076cb029e9bab2bac0f70f7250dd17
checksum: 10c0/d334811074726482b58089fef6c9e98a462fbc1d91c63798307648bdc1349d2e154aa31f391690e2c9cd90eee61a3be9fe8872873e7f828b529d9268d2a25b78
languageName: node
linkType: hard
@@ -12043,6 +12081,13 @@ __metadata:
languageName: node
linkType: hard
"regexparam@npm:^3.0.0":
version: 3.0.0
resolution: "regexparam@npm:3.0.0"
checksum: 10c0/a6430d7b97d5a7d5518f37a850b6b73aab479029d02f46af4fa0e8e4a1d7aad05b7a0d2d10c86ded21a14d5f0fa4c68525f873a5fca2efeefcccd93c36627459
languageName: node
linkType: hard
"regexpu-core@npm:^6.2.0":
version: 6.2.0
resolution: "regexpu-core@npm:6.2.0"