Merge remote-tracking branch 'origin/voip-team/rebased-multiSFU' into toger5/sticky-events-version

This commit is contained in:
Will Hunt
2025-10-15 15:27:34 +01:00
39 changed files with 902 additions and 666 deletions

View File

@@ -19,10 +19,26 @@ import mediaViewStyles from "../src/tile/MediaView.module.css";
interface Props {
audio?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats;
video?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats;
focusUrl?: string;
}
const extractDomain = (url: string): string => {
try {
const parsedUrl = new URL(url);
return parsedUrl.hostname; // Returns "kdk.cpm"
} catch (error) {
console.error("Invalid URL:", error);
return url;
}
};
// This is only used in developer mode for debugging purposes, so we don't need full localization
export const RTCConnectionStats: FC<Props> = ({ audio, video, ...rest }) => {
export const RTCConnectionStats: FC<Props> = ({
audio,
video,
focusUrl,
...rest
}) => {
const [showModal, setShowModal] = useState(false);
const [modalContents, setModalContents] = useState<
"video" | "audio" | "none"
@@ -55,6 +71,13 @@ export const RTCConnectionStats: FC<Props> = ({ audio, video, ...rest }) => {
</pre>
</div>
</Modal>
{focusUrl && (
<div>
<Text as="span" size="xs" title="focusURL">
&nbsp;{extractDomain(focusUrl)}
</Text>
</div>
)}
{audio && (
<div>
<Button

View File

@@ -19,7 +19,7 @@ import { alice, local, localRtcMember } from "../utils/test-fixtures";
import { type MockRTCSession } from "../utils/test";
import { ReactionsSenderProvider } from "../reactions/useReactionsSender";
const localIdent = `${localRtcMember.sender}:${localRtcMember.deviceId}`;
const localIdent = `${localRtcMember.userId}:${localRtcMember.deviceId}`;
function TestComponent({
rtcSession,

View File

@@ -14,7 +14,7 @@ import {
import { distinctUntilChanged } from "rxjs";
import { useObservableEagerState } from "observable-hooks";
import { type GridLayout as GridLayoutModel } from "../state/CallViewModel";
import { type GridLayout as GridLayoutModel } from "../state/layout-types.ts";
import styles from "./GridLayout.module.css";
import { useInitial } from "../useInitial";
import { type CallLayout, arrangeTiles } from "./CallLayout";

View File

@@ -9,7 +9,7 @@ import { type ReactNode, useCallback, useMemo } from "react";
import { useObservableEagerState } from "observable-hooks";
import classNames from "classnames";
import { type OneOnOneLayout as OneOnOneLayoutModel } from "../state/CallViewModel";
import { type OneOnOneLayout as OneOnOneLayoutModel } from "../state/layout-types.ts";
import { type CallLayout, arrangeTiles } from "./CallLayout";
import styles from "./OneOnOneLayout.module.css";
import { type DragCallback, useUpdateLayout } from "./Grid";

View File

@@ -7,7 +7,7 @@ Please see LICENSE in the repository root for full details.
import { type ReactNode, useCallback } from "react";
import { type SpotlightExpandedLayout as SpotlightExpandedLayoutModel } from "../state/CallViewModel";
import { type SpotlightExpandedLayout as SpotlightExpandedLayoutModel } from "../state/layout-types.ts";
import { type CallLayout } from "./CallLayout";
import { type DragCallback, useUpdateLayout } from "./Grid";
import styles from "./SpotlightExpandedLayout.module.css";

View File

@@ -10,7 +10,7 @@ import { useObservableEagerState } from "observable-hooks";
import classNames from "classnames";
import { type CallLayout } from "./CallLayout";
import { type SpotlightLandscapeLayout as SpotlightLandscapeLayoutModel } from "../state/CallViewModel";
import { type SpotlightLandscapeLayout as SpotlightLandscapeLayoutModel } from "../state/layout-types.ts";
import styles from "./SpotlightLandscapeLayout.module.css";
import { useUpdateLayout, useVisibleTiles } from "./Grid";

View File

@@ -10,7 +10,7 @@ import { useObservableEagerState } from "observable-hooks";
import classNames from "classnames";
import { type CallLayout, arrangeTiles } from "./CallLayout";
import { type SpotlightPortraitLayout as SpotlightPortraitLayoutModel } from "../state/CallViewModel";
import { type SpotlightPortraitLayout as SpotlightPortraitLayoutModel } from "../state/layout-types.ts";
import styles from "./SpotlightPortraitLayout.module.css";
import { useUpdateLayout, useVisibleTiles } from "./Grid";
import { useBehavior } from "../useBehavior";

View File

@@ -47,7 +47,7 @@ test("handles a hand raised reaction", () => {
new MatrixEvent({
room_id: rtcSession.room.roomId,
event_id: reactionEventId,
sender: localRtcMember.sender,
sender: localRtcMember.userId,
type: EventType.Reaction,
origin_server_ts: localTimestamp.getTime(),
content: {
@@ -67,7 +67,7 @@ test("handles a hand raised reaction", () => {
expectObservable(raisedHands$).toBe("ab", {
a: {},
b: {
[`${localRtcMember.sender}:${localRtcMember.deviceId}`]: {
[`${localRtcMember.userId}:${localRtcMember.deviceId}`]: {
reactionEventId,
membershipEventId: localRtcMember.eventId,
time: localTimestamp,
@@ -95,7 +95,7 @@ test("handles a redaction", () => {
new MatrixEvent({
room_id: rtcSession.room.roomId,
event_id: reactionEventId,
sender: localRtcMember.sender,
sender: localRtcMember.userId,
type: EventType.Reaction,
origin_server_ts: localTimestamp.getTime(),
content: {
@@ -117,7 +117,7 @@ test("handles a redaction", () => {
new MatrixEvent({
room_id: rtcSession.room.roomId,
event_id: reactionEventId,
sender: localRtcMember.sender,
sender: localRtcMember.userId,
type: EventType.RoomRedaction,
redacts: reactionEventId,
}),
@@ -129,7 +129,7 @@ test("handles a redaction", () => {
expectObservable(raisedHands$).toBe("abc", {
a: {},
b: {
[`${localRtcMember.sender}:${localRtcMember.deviceId}`]: {
[`${localRtcMember.userId}:${localRtcMember.deviceId}`]: {
reactionEventId,
membershipEventId: localRtcMember.eventId,
time: localTimestamp,
@@ -156,7 +156,7 @@ test("handles waiting for event decryption", () => {
const encryptedEvent = new MatrixEvent({
room_id: rtcSession.room.roomId,
event_id: reactionEventId,
sender: localRtcMember.sender,
sender: localRtcMember.userId,
type: EventType.Reaction,
origin_server_ts: localTimestamp.getTime(),
content: {
@@ -183,7 +183,7 @@ test("handles waiting for event decryption", () => {
new MatrixEvent({
room_id: rtcSession.room.roomId,
event_id: reactionEventId,
sender: localRtcMember.sender,
sender: localRtcMember.userId,
type: EventType.Reaction,
origin_server_ts: localTimestamp.getTime(),
content: {
@@ -199,7 +199,7 @@ test("handles waiting for event decryption", () => {
expectObservable(raisedHands$).toBe("a-c", {
a: {},
c: {
[`${localRtcMember.sender}:${localRtcMember.deviceId}`]: {
[`${localRtcMember.userId}:${localRtcMember.deviceId}`]: {
reactionEventId,
membershipEventId: localRtcMember.eventId,
time: localTimestamp,
@@ -227,7 +227,7 @@ test("hands rejecting events without a proper membership", () => {
new MatrixEvent({
room_id: rtcSession.room.roomId,
event_id: reactionEventId,
sender: localRtcMember.sender,
sender: localRtcMember.userId,
type: EventType.Reaction,
origin_server_ts: localTimestamp.getTime(),
content: {
@@ -270,7 +270,7 @@ test("handles a reaction", () => {
new MatrixEvent({
room_id: rtcSession.room.roomId,
event_id: reactionEventId,
sender: localRtcMember.sender,
sender: localRtcMember.userId,
type: ElementCallReactionEventType,
content: {
emoji: reaction.emoji,
@@ -295,7 +295,7 @@ test("handles a reaction", () => {
{
a: {},
b: {
[`${localRtcMember.sender}:${localRtcMember.deviceId}`]: {
[`${localRtcMember.userId}:${localRtcMember.deviceId}`]: {
reactionOption: reaction,
expireAfter: new Date(REACTION_ACTIVE_TIME_MS),
},
@@ -327,7 +327,7 @@ test("ignores bad reaction events", () => {
new MatrixEvent({
room_id: rtcSession.room.roomId,
event_id: reactionEventId,
sender: localRtcMember.sender,
sender: localRtcMember.userId,
type: ElementCallReactionEventType,
content: {},
}),
@@ -342,7 +342,7 @@ test("ignores bad reaction events", () => {
new MatrixEvent({
room_id: rtcSession.room.roomId,
event_id: reactionEventId,
sender: localRtcMember.sender,
sender: localRtcMember.userId,
type: ElementCallReactionEventType,
content: {
emoji: reaction.emoji,
@@ -363,7 +363,7 @@ test("ignores bad reaction events", () => {
new MatrixEvent({
room_id: rtcSession.room.roomId,
event_id: reactionEventId,
sender: aliceRtcMember.sender,
sender: aliceRtcMember.userId,
type: ElementCallReactionEventType,
content: {
emoji: reaction.emoji,
@@ -384,7 +384,7 @@ test("ignores bad reaction events", () => {
new MatrixEvent({
room_id: rtcSession.room.roomId,
event_id: reactionEventId,
sender: localRtcMember.sender,
sender: localRtcMember.userId,
type: ElementCallReactionEventType,
content: {
name: reaction.name,
@@ -404,7 +404,7 @@ test("ignores bad reaction events", () => {
new MatrixEvent({
room_id: rtcSession.room.roomId,
event_id: reactionEventId,
sender: localRtcMember.sender,
sender: localRtcMember.userId,
type: ElementCallReactionEventType,
content: {
emoji: " ",
@@ -448,7 +448,7 @@ test("that reactions cannot be spammed", () => {
new MatrixEvent({
room_id: rtcSession.room.roomId,
event_id: reactionEventId,
sender: localRtcMember.sender,
sender: localRtcMember.userId,
type: ElementCallReactionEventType,
content: {
emoji: reactionA.emoji,
@@ -470,7 +470,7 @@ test("that reactions cannot be spammed", () => {
new MatrixEvent({
room_id: rtcSession.room.roomId,
event_id: reactionEventId,
sender: localRtcMember.sender,
sender: localRtcMember.userId,
type: ElementCallReactionEventType,
content: {
emoji: reactionB.emoji,
@@ -495,7 +495,7 @@ test("that reactions cannot be spammed", () => {
{
a: {},
b: {
[`${localRtcMember.sender}:${localRtcMember.deviceId}`]: {
[`${localRtcMember.userId}:${localRtcMember.deviceId}`]: {
reactionOption: reactionA,
expireAfter: new Date(REACTION_ACTIVE_TIME_MS),
},

View File

@@ -130,7 +130,7 @@ export class ReactionsReader {
private onMembershipsChanged = (oldMemberships: CallMembership[]): void => {
// Remove any raised hands for users no longer joined to the call.
for (const identifier of Object.keys(this.raisedHandsSubject$.value).filter(
(rhId) => oldMemberships.find((u) => u.sender == rhId),
(rhId) => oldMemberships.find((u) => u.userId == rhId),
)) {
this.removeRaisedHand(identifier);
}
@@ -138,10 +138,10 @@ export class ReactionsReader {
// For each member in the call, check to see if a reaction has
// been raised and adjust.
for (const m of this.rtcSession.memberships) {
if (!m.sender || !m.eventId) {
if (!m.userId || !m.eventId) {
continue;
}
const identifier = `${m.sender}:${m.deviceId}`;
const identifier = `${m.userId}:${m.deviceId}`;
if (
this.raisedHandsSubject$.value[identifier] &&
this.raisedHandsSubject$.value[identifier].membershipEventId !==
@@ -151,13 +151,13 @@ export class ReactionsReader {
// was raised, reset.
this.removeRaisedHand(identifier);
}
const reaction = this.getLastReactionEvent(m.eventId, m.sender);
const reaction = this.getLastReactionEvent(m.eventId, m.userId);
if (reaction) {
const eventId = reaction?.getId();
if (!eventId) {
continue;
}
this.addRaisedHand(`${m.sender}:${m.deviceId}`, {
this.addRaisedHand(`${m.userId}:${m.deviceId}`, {
membershipEventId: m.eventId,
reactionEventId: eventId,
time: new Date(reaction.localTimestamp),
@@ -219,7 +219,7 @@ export class ReactionsReader {
const membershipEventId = content?.["m.relates_to"]?.event_id;
const membershipEvent = this.rtcSession.memberships.find(
(e) => e.eventId === membershipEventId && e.sender === sender,
(e) => e.eventId === membershipEventId && e.userId === sender,
);
// Check to see if this reaction was made to a membership event (and the
// sender of the reaction matches the membership)
@@ -229,7 +229,7 @@ export class ReactionsReader {
);
return;
}
const identifier = `${membershipEvent.sender}:${membershipEvent.deviceId}`;
const identifier = `${membershipEvent.userId}:${membershipEvent.deviceId}`;
if (!content.emoji) {
logger.warn(`Reaction had no emoji from ${reactionEventId}`);
@@ -278,7 +278,7 @@ export class ReactionsReader {
// Check to see if this reaction was made to a membership event (and the
// sender of the reaction matches the membership)
const membershipEvent = this.rtcSession.memberships.find(
(e) => e.eventId === membershipEventId && e.sender === sender,
(e) => e.eventId === membershipEventId && e.userId === sender,
);
if (!membershipEvent) {
logger.warn(
@@ -289,7 +289,7 @@ export class ReactionsReader {
if (content?.["m.relates_to"].key === "🖐️") {
this.addRaisedHand(
`${membershipEvent.sender}:${membershipEvent.deviceId}`,
`${membershipEvent.userId}:${membershipEvent.deviceId}`,
{
reactionEventId,
membershipEventId,

View File

@@ -65,7 +65,7 @@ export const ReactionsSenderProvider = ({
const myMembershipEvent = useMemo(
() =>
memberships.find(
(m) => m.sender === myUserId && m.deviceId === myDeviceId,
(m) => m.userId === myUserId && m.deviceId === myDeviceId,
)?.eventId,
[memberships, myUserId, myDeviceId],
);

View File

@@ -156,7 +156,7 @@ test("plays one sound when a hand is raised", () => {
act(() => {
handRaisedSubject$.next({
// TODO: What is this string supposed to be?
[`${bobRtcMember.sender}:${bobRtcMember.deviceId}`]: {
[`${bobRtcMember.userId}:${bobRtcMember.deviceId}`]: {
time: new Date(),
membershipEventId: "",
reactionEventId: "",

View File

@@ -122,7 +122,7 @@ function createGroupCallView(
} {
const client = {
getUser: () => null,
getUserId: () => localRtcMember.sender,
getUserId: () => localRtcMember.userId,
getDeviceId: () => localRtcMember.deviceId,
getRoom: (rId) => (rId === roomId ? room : null),
} as Partial<MatrixClient> as MatrixClient;

View File

@@ -207,7 +207,7 @@ export const GroupCallView: FC<Props> = ({
// Count each member only once, regardless of how many devices they use
const participantCount = useMemo(
() => new Set<string>(memberships.map((m) => m.sender!)).size,
() => new Set<string>(memberships.map((m) => m.userId!)).size,
[memberships],
);

View File

@@ -110,7 +110,7 @@ function createInCallView(): RenderResult & {
} {
const client = {
getUser: () => null,
getUserId: () => localRtcMember.sender,
getUserId: () => localRtcMember.userId,
getDeviceId: () => localRtcMember.deviceId,
getRoom: (rId) => (rId === roomId ? room : null),
} as Partial<MatrixClient> as MatrixClient;

View File

@@ -59,11 +59,7 @@ import { type MuteStates } from "../state/MuteStates";
import { type MatrixInfo } from "./VideoPreview";
import { InviteButton } from "../button/InviteButton";
import { LayoutToggle } from "./LayoutToggle";
import {
CallViewModel,
type GridMode,
type Layout,
} from "../state/CallViewModel";
import { CallViewModel, type GridMode } from "../state/CallViewModel";
import { Grid, type TileProps } from "../grid/Grid";
import { useInitial } from "../useInitial";
import { SpotlightTile } from "../tile/SpotlightTile";
@@ -113,6 +109,7 @@ import { useAudioContext } from "../useAudioContext";
import ringtoneMp3 from "../sound/ringtone.mp3?url";
import ringtoneOgg from "../sound/ringtone.ogg?url";
import { useTrackProcessorObservable$ } from "../livekit/TrackProcessorContext.tsx";
import { type Layout } from "../state/layout-types.ts";
const maxTapDurationMs = 400;
@@ -300,6 +297,10 @@ export const InCallView: FC<InCallViewProps> = ({
const audioOutputSwitcher = useBehavior(vm.audioOutputSwitcher$);
const sharingScreen = useBehavior(vm.sharingScreen$);
const fatalCallError = useBehavior(vm.configError$);
// Stop the rendering and throw for the error boundary
if (fatalCallError) throw fatalCallError;
// We need to set the proper timings on the animation based upon the sound length.
const ringDuration = pickupPhaseAudio?.soundDuration["waiting"] ?? 1;
useEffect((): (() => void) => {

View File

@@ -13,8 +13,8 @@ import EventEmitter from "events";
import { enterRTCSession, leaveRTCSession } from "../src/rtcSessionHelpers";
import { mockConfig } from "./utils/test";
import { ElementWidgetActions, widget } from "./widget";
import { ErrorCode } from "./utils/errors.ts";
const USE_MUTI_SFU = false;
const getUrlParams = vi.hoisted(() => vi.fn(() => ({})));
vi.mock("./UrlParams", () => ({ getUrlParams }));
@@ -93,41 +93,26 @@ test("It joins the correct Session", async () => {
livekit_service_url: "http://my-well-known-service-url.com",
type: "livekit",
},
true,
{
encryptMedia: true,
useMultiSfu: USE_MUTI_SFU,
},
);
expect(mockedSession.joinRoomSession).toHaveBeenLastCalledWith(
[
{
livekit_alias: "my-oldest-member-service-alias",
livekit_service_url: "http://my-oldest-member-service-url.com",
type: "livekit",
},
{
livekit_alias: "roomId",
livekit_service_url: "http://my-well-known-service-url.com",
type: "livekit",
},
{
livekit_alias: "roomId",
livekit_service_url: "http://my-well-known-service-url2.com",
type: "livekit",
},
{
livekit_alias: "roomId",
livekit_service_url: "http://my-default-service-url.com",
type: "livekit",
},
],
{
focus_selection: "oldest_membership",
type: "livekit",
},
{
manageMediaKeys: false,
undefined,
expect.objectContaining({
manageMediaKeys: true,
useLegacyMemberEvents: false,
useExperimentalToDeviceTransport: false,
},
}),
);
});
@@ -172,37 +157,6 @@ test("leaveRTCSession doesn't close the widget when returning to lobby", async (
await testLeaveRTCSession("user", false);
});
test("It fails with configuration error if no live kit url config is set in fallback", async () => {
mockConfig({});
vi.spyOn(AutoDiscovery, "getRawClientConfig").mockResolvedValue({});
const mockedSession = vi.mocked({
room: {
roomId: "roomId",
client: {
getDomain: vi.fn().mockReturnValue("example.org"),
},
},
memberships: [],
getFocusInUse: vi.fn(),
joinRoomSession: vi.fn(),
}) as unknown as MatrixRTCSession;
await expect(
enterRTCSession(
mockedSession,
{
livekit_alias: "roomId",
livekit_service_url: "http://my-well-known-service-url.com",
type: "livekit",
},
true,
),
).rejects.toThrowError(
expect.objectContaining({ code: ErrorCode.MISSING_MATRIX_RTC_TRANSPORT }),
);
});
test("It should not fail with configuration error if homeserver config has livekit url but not fallback", async () => {
mockConfig({});
vi.spyOn(AutoDiscovery, "getRawClientConfig").mockResolvedValue({
@@ -239,6 +193,9 @@ test("It should not fail with configuration error if homeserver config has livek
livekit_service_url: "http://my-well-known-service-url.com",
type: "livekit",
},
true,
{
encryptMedia: true,
useMultiSfu: USE_MUTI_SFU,
},
);
});

View File

@@ -98,13 +98,37 @@ export async function makeTransport(
return transport;
}
export interface EnterRTCSessionOptions {
encryptMedia: boolean;
// TODO: remove this flag, the new membership manager is stable enough
useNewMembershipManager?: boolean;
// TODO: remove this flag, to-device transport is stable enough now
useExperimentalToDeviceTransport?: boolean;
/** EXPERIMENTAL: If true, will use the multi-sfu codepath where each member connects to its SFU instead of everyone connecting to an elected on. */
useMultiSfu?: boolean;
}
/**
* TODO! document this function properly
* @param rtcSession
* @param transport
* @param options
*/
export async function enterRTCSession(
rtcSession: MatrixRTCSession,
transport: LivekitTransport,
encryptMedia: boolean,
useExperimentalToDeviceTransport = false,
useMultiSfu = true,
options: EnterRTCSessionOptions = {
encryptMedia: true,
useExperimentalToDeviceTransport: false,
useMultiSfu: true,
},
): Promise<void> {
const {
encryptMedia,
useExperimentalToDeviceTransport = false,
useMultiSfu = true,
} = options;
PosthogAnalytics.instance.eventCallEnded.cacheStartCall(new Date());
PosthogAnalytics.instance.eventCallStarted.track(rtcSession.room.roomId);

View File

@@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { test, vi, onTestFinished, it, describe } from "vitest";
import { test, vi, onTestFinished, it, describe, expect } from "vitest";
import EventEmitter from "events";
import {
BehaviorSubject,
@@ -45,12 +45,10 @@ import {
MatrixRTCSessionEvent,
} from "matrix-js-sdk/lib/matrixrtc";
import { deepCompare } from "matrix-js-sdk/lib/utils";
import { AutoDiscovery } from "matrix-js-sdk/lib/autodiscovery";
import {
CallViewModel,
type CallViewModelOptions,
type Layout,
} from "./CallViewModel";
import { CallViewModel, type CallViewModelOptions } from "./CallViewModel";
import { type Layout } from "./layout-types";
import {
mockLocalParticipant,
mockMatrixRoom,
@@ -61,6 +59,7 @@ import {
MockRTCSession,
mockMediaDevices,
mockMuteStates,
mockConfig,
} from "../utils/test";
import {
ECAddonConnectionState,
@@ -95,6 +94,10 @@ import { MediaDevices } from "./MediaDevices";
import { getValue } from "../utils/observable";
import { type Behavior, constant } from "./Behavior";
import type { ProcessorState } from "../livekit/TrackProcessorContext.tsx";
import {
type ElementCallError,
MatrixRTCTransportMissingError,
} from "../utils/errors.ts";
const getUrlParams = vi.hoisted(() => vi.fn(() => ({})));
vi.mock("../UrlParams", () => ({ getUrlParams }));
@@ -302,7 +305,7 @@ function withCallViewModel(
const room = mockMatrixRoom({
client: new (class extends EventEmitter {
public getUserId(): string | undefined {
return localRtcMember.sender;
return localRtcMember.userId;
}
public getDeviceId(): string {
return localRtcMember.deviceId;
@@ -368,6 +371,61 @@ function withCallViewModel(
continuation(vm, rtcSession, { raisedHands$: raisedHands$ }, setSyncState);
}
test("test missing RTC config error", async () => {
const rtcMemberships$ = new BehaviorSubject<CallMembership[]>([]);
const emitter = new EventEmitter();
const client = vi.mocked<MatrixClient>({
on: emitter.on.bind(emitter),
off: emitter.off.bind(emitter),
getSyncState: vi.fn().mockReturnValue(SyncState.Syncing),
getUserId: vi.fn().mockReturnValue("@user:localhost"),
getUser: vi.fn().mockReturnValue(null),
getDeviceId: vi.fn().mockReturnValue("DEVICE"),
credentials: {
userId: "@user:localhost",
},
getCrypto: vi.fn().mockReturnValue(undefined),
getDomain: vi.fn().mockReturnValue("example.org"),
} as unknown as MatrixClient);
const matrixRoom = mockMatrixRoom({
roomId: "!myRoomId:example.com",
client,
getMember: vi.fn().mockReturnValue(undefined),
});
const fakeRtcSession = new MockRTCSession(matrixRoom).withMemberships(
rtcMemberships$,
);
mockConfig({});
vi.spyOn(AutoDiscovery, "getRawClientConfig").mockResolvedValue({});
const callVM = new CallViewModel(
fakeRtcSession.asMockedSession(),
matrixRoom,
mockMediaDevices({}),
mockMuteStates(),
{
encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
autoLeaveWhenOthersLeft: false,
},
new BehaviorSubject({} as Record<string, RaisedHandInfo>),
new BehaviorSubject({} as Record<string, ReactionInfo>),
of({ processor: undefined, supported: false }),
);
const failPromise = Promise.withResolvers<ElementCallError>();
callVM.configError$.subscribe((error) => {
if (error) {
failPromise.resolve(error);
}
});
const error = await failPromise.promise;
expect(error).toBeInstanceOf(MatrixRTCTransportMissingError);
});
test("participants are retained during a focus switch", () => {
withTestScheduler(({ behavior, expectObservable }) => {
// Participants disappear on frame 2 and come back on frame 3
@@ -1012,7 +1070,7 @@ it("should rank raised hands above video feeds and below speakers and presenters
},
b: () => {
raisedHands$.next({
[`${bobRtcMember.sender}:${bobRtcMember.deviceId}`]: {
[`${bobRtcMember.userId}:${bobRtcMember.deviceId}`]: {
time: new Date(),
reactionEventId: "",
membershipEventId: "",

View File

@@ -5,39 +5,33 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { observeParticipantEvents } from "@livekit/components-core";
import {
ConnectionState,
type BaseKeyProvider,
ConnectionState,
type E2EEOptions,
ExternalE2EEKeyProvider,
type Room as LivekitRoom,
type LocalParticipant,
ParticipantEvent,
RemoteParticipant,
type Participant,
type Room as LivekitRoom,
} from "livekit-client";
import E2EEWorker from "livekit-client/e2ee-worker?worker";
import {
ClientEvent,
type EventTimelineSetHandlerMap,
EventType,
type Room as MatrixRoom,
RoomEvent,
type RoomMember,
RoomStateEvent,
SyncState,
type Room as MatrixRoom,
type EventTimelineSetHandlerMap,
EventType,
RoomEvent,
} from "matrix-js-sdk";
import { deepCompare } from "matrix-js-sdk/lib/utils";
import {
BehaviorSubject,
EMPTY,
NEVER,
type Observable,
Subject,
combineLatest,
concat,
distinctUntilChanged,
EMPTY,
endWith,
filter,
from,
@@ -45,6 +39,8 @@ import {
ignoreElements,
map,
merge,
NEVER,
type Observable,
of,
pairwise,
race,
@@ -53,6 +49,7 @@ import {
skip,
skipWhile,
startWith,
Subject,
switchAll,
switchMap,
switchScan,
@@ -80,7 +77,7 @@ import { ViewModel } from "./ViewModel";
import {
LocalUserMediaViewModel,
type MediaViewModel,
RemoteUserMediaViewModel,
type RemoteUserMediaViewModel,
ScreenShareViewModel,
type UserMediaViewModel,
} from "./MediaViewModel";
@@ -90,7 +87,6 @@ import {
finalizeValue,
pauseWhen,
} from "../utils/observable";
import { ObservableScope } from "./ObservableScope";
import {
duplicateTiles,
multiSfu,
@@ -99,10 +95,6 @@ import {
} from "../settings/settings";
import { isFirefox } from "../Platform";
import { setPipEnabled$ } from "../controls";
import {
type GridTileViewModel,
type SpotlightTileViewModel,
} from "./TileViewModel";
import { TileStore } from "./TileStore";
import { gridLikeLayout } from "./GridLikeLayout";
import { spotlightExpandedLayout } from "./SpotlightExpandedLayout";
@@ -114,11 +106,10 @@ import {
type ReactionInfo,
type ReactionOption,
} from "../reactions";
import { observeSpeaker$ } from "./observeSpeaker";
import { shallowEquals } from "../utils/array";
import { calculateDisplayName, shouldDisambiguate } from "../utils/displayname";
import { type MediaDevices } from "./MediaDevices";
import { constant, type Behavior } from "./Behavior";
import { type Behavior, constant } from "./Behavior";
import {
enterRTCSession,
getLivekitAlias,
@@ -137,6 +128,19 @@ import { type ProcessorState } from "../livekit/TrackProcessorContext";
import { ElementWidgetActions, widget } from "../widget";
import { PublishConnection } from "./PublishConnection.ts";
import { type Async, async$, mapAsync, ready } from "./Async";
import { sharingScreen$, UserMedia } from "./UserMedia.ts";
import { ScreenShare } from "./ScreenShare.ts";
import {
type GridLayoutMedia,
type Layout,
type LayoutMedia,
type OneOnOneLayoutMedia,
type SpotlightExpandedLayoutMedia,
type SpotlightLandscapeLayoutMedia,
type SpotlightPortraitLayoutMedia,
} from "./layout-types.ts";
import { ElementCallError, UnknownCallError } from "../utils/errors.ts";
import { ObservableScope } from "./ObservableScope.ts";
export interface CallViewModelOptions {
encryptionSystem: EncryptionSystem;
@@ -161,287 +165,17 @@ const smallMobileCallThreshold = 3;
// with the interface
const showFooterMs = 4000;
export interface GridLayoutMedia {
type: "grid";
spotlight?: MediaViewModel[];
grid: UserMediaViewModel[];
}
export interface SpotlightLandscapeLayoutMedia {
type: "spotlight-landscape";
spotlight: MediaViewModel[];
grid: UserMediaViewModel[];
}
export interface SpotlightPortraitLayoutMedia {
type: "spotlight-portrait";
spotlight: MediaViewModel[];
grid: UserMediaViewModel[];
}
export interface SpotlightExpandedLayoutMedia {
type: "spotlight-expanded";
spotlight: MediaViewModel[];
pip?: UserMediaViewModel;
}
export interface OneOnOneLayoutMedia {
type: "one-on-one";
local: UserMediaViewModel;
remote: UserMediaViewModel;
}
export interface PipLayoutMedia {
type: "pip";
spotlight: MediaViewModel[];
}
export type LayoutMedia =
| GridLayoutMedia
| SpotlightLandscapeLayoutMedia
| SpotlightPortraitLayoutMedia
| SpotlightExpandedLayoutMedia
| OneOnOneLayoutMedia
| PipLayoutMedia;
export interface GridLayout {
type: "grid";
spotlight?: SpotlightTileViewModel;
grid: GridTileViewModel[];
setVisibleTiles: (value: number) => void;
}
export interface SpotlightLandscapeLayout {
type: "spotlight-landscape";
spotlight: SpotlightTileViewModel;
grid: GridTileViewModel[];
setVisibleTiles: (value: number) => void;
}
export interface SpotlightPortraitLayout {
type: "spotlight-portrait";
spotlight: SpotlightTileViewModel;
grid: GridTileViewModel[];
setVisibleTiles: (value: number) => void;
}
export interface SpotlightExpandedLayout {
type: "spotlight-expanded";
spotlight: SpotlightTileViewModel;
pip?: GridTileViewModel;
}
export interface OneOnOneLayout {
type: "one-on-one";
local: GridTileViewModel;
remote: GridTileViewModel;
}
export interface PipLayout {
type: "pip";
spotlight: SpotlightTileViewModel;
}
/**
* A layout defining the media tiles present on screen and their visual
* arrangement.
*/
export type Layout =
| GridLayout
| SpotlightLandscapeLayout
| SpotlightPortraitLayout
| SpotlightExpandedLayout
| OneOnOneLayout
| PipLayout;
export type GridMode = "grid" | "spotlight";
export type WindowMode = "normal" | "narrow" | "flat" | "pip";
/**
* Sorting bins defining the order in which media tiles appear in the layout.
*/
enum SortingBin {
/**
* Yourself, when the "always show self" option is on.
*/
SelfAlwaysShown,
/**
* Participants that are sharing their screen.
*/
Presenters,
/**
* Participants that have been speaking recently.
*/
Speakers,
/**
* Participants that have their hand raised.
*/
HandRaised,
/**
* Participants with video.
*/
Video,
/**
* Participants not sharing any video.
*/
NoVideo,
/**
* Yourself, when the "always show self" option is off.
*/
SelfNotAlwaysShown,
}
interface LayoutScanState {
layout: Layout | null;
tiles: TileStore;
}
class UserMedia {
private readonly scope = new ObservableScope();
public readonly vm: UserMediaViewModel;
private readonly participant$: BehaviorSubject<
LocalParticipant | RemoteParticipant | undefined
>;
public readonly speaker$: Behavior<boolean>;
public readonly presenter$: Behavior<boolean>;
public constructor(
public readonly id: string,
member: RoomMember,
participant: LocalParticipant | RemoteParticipant | undefined,
encryptionSystem: EncryptionSystem,
livekitRoom: LivekitRoom,
mediaDevices: MediaDevices,
pretendToBeDisconnected$: Behavior<boolean>,
displayname$: Observable<string>,
handRaised$: Observable<Date | null>,
reaction$: Observable<ReactionOption | null>,
) {
this.participant$ = new BehaviorSubject(participant);
if (participant?.isLocal) {
this.vm = new LocalUserMediaViewModel(
this.id,
member,
this.participant$ as Behavior<LocalParticipant>,
encryptionSystem,
livekitRoom,
mediaDevices,
this.scope.behavior(displayname$),
this.scope.behavior(handRaised$),
this.scope.behavior(reaction$),
);
} else {
this.vm = new RemoteUserMediaViewModel(
id,
member,
this.participant$.asObservable() as Observable<
RemoteParticipant | undefined
>,
encryptionSystem,
livekitRoom,
pretendToBeDisconnected$,
this.scope.behavior(displayname$),
this.scope.behavior(handRaised$),
this.scope.behavior(reaction$),
);
}
this.speaker$ = this.scope.behavior(observeSpeaker$(this.vm.speaking$));
this.presenter$ = this.scope.behavior(
this.participant$.pipe(
switchMap((p) => (p === undefined ? of(false) : sharingScreen$(p))),
),
);
}
public updateParticipant(
newParticipant: LocalParticipant | RemoteParticipant | undefined,
): void {
if (this.participant$.value !== newParticipant) {
// Update the BehaviourSubject in the UserMedia.
this.participant$.next(newParticipant);
}
}
public destroy(): void {
this.scope.end();
this.vm.destroy();
}
}
class ScreenShare {
private readonly scope = new ObservableScope();
public readonly vm: ScreenShareViewModel;
private readonly participant$: BehaviorSubject<
LocalParticipant | RemoteParticipant
>;
public constructor(
id: string,
member: RoomMember,
participant: LocalParticipant | RemoteParticipant,
encryptionSystem: EncryptionSystem,
livekitRoom: LivekitRoom,
pretendToBeDisconnected$: Behavior<boolean>,
displayName$: Observable<string>,
) {
this.participant$ = new BehaviorSubject(participant);
this.vm = new ScreenShareViewModel(
id,
member,
this.participant$.asObservable(),
encryptionSystem,
livekitRoom,
pretendToBeDisconnected$,
this.scope.behavior(displayName$),
participant.isLocal,
);
}
public destroy(): void {
this.scope.end();
this.vm.destroy();
}
}
type MediaItem = UserMedia | ScreenShare;
function getRoomMemberFromRtcMember(
rtcMember: CallMembership,
room: MatrixRoom,
): { id: string; member: RoomMember | undefined } {
// WARN! This is not exactly the sender but the user defined in the state key.
// This will be available once we change to the new "member as object" format in the MatrixRTC object.
let id = rtcMember.sender + ":" + rtcMember.deviceId;
if (!rtcMember.sender) {
return { id, member: undefined };
}
if (
rtcMember.sender === room.client.getUserId() &&
rtcMember.deviceId === room.client.getDeviceId()
) {
id = "local";
}
const member = room.getMember(rtcMember.sender) ?? undefined;
return { id, member };
}
function sharingScreen$(p: Participant): Observable<boolean> {
return observeParticipantEvents(
p,
ParticipantEvent.TrackPublished,
ParticipantEvent.TrackUnpublished,
ParticipantEvent.LocalTrackPublished,
ParticipantEvent.LocalTrackUnpublished,
).pipe(map((p) => p.isScreenShareEnabled));
}
export class CallViewModel extends ViewModel {
private readonly urlParams = getUrlParams();
@@ -459,6 +193,19 @@ export class CallViewModel extends ViewModel {
}
: undefined;
private readonly _configError$ = new BehaviorSubject<ElementCallError | null>(
null,
);
/**
* If there is a configuration error with the call (e.g. misconfigured E2EE).
* This is a fatal error that prevents the call from being created/joined.
* Should render a blocking error screen.
*/
public get configError$(): Behavior<ElementCallError | null> {
return this._configError$;
}
private readonly join$ = new Subject<void>();
public join(): void {
@@ -508,30 +255,34 @@ export class CallViewModel extends ViewModel {
* The transport that we would personally prefer to publish on (if not for the
* transport preferences of others, perhaps).
*/
private readonly preferredTransport = makeTransport(this.matrixRTCSession);
private readonly preferredTransport$ = this.scope.behavior(
async$(makeTransport(this.matrixRTCSession)),
);
/**
* Lists the transports used by ourselves, plus all other MatrixRTC session
* members.
* members. For completeness this also lists the preferred transport and
* whether we are in multi-SFU mode (because advertisedTransport$ wants to
* read them at the same time, and bundling data together when it might change
* together is what you have to do in RxJS to avoid reading inconsistent state
* or observing too many changes.)
*/
private readonly transports$: Behavior<{
local: Async<LivekitTransport>;
remote: { membership: CallMembership; transport: LivekitTransport }[];
preferred: Async<LivekitTransport>;
multiSfu: boolean;
} | null> = this.scope.behavior(
this.joined$.pipe(
switchMap((joined) =>
joined
? combineLatest(
[
async$(this.preferredTransport),
this.memberships$,
multiSfu.value$,
],
[this.preferredTransport$, this.memberships$, multiSfu.value$],
(preferred, memberships, multiSfu) => {
const oldestMembership =
this.matrixRTCSession.getOldestMembership();
const remote = memberships.flatMap((m) => {
if (m.sender === this.userId && m.deviceId === this.deviceId)
if (m.userId === this.userId && m.deviceId === this.deviceId)
return [];
const t = m.getTransport(oldestMembership ?? m);
return t && isLivekitTransport(t)
@@ -548,7 +299,14 @@ export class CallViewModel extends ViewModel {
local = ready(selection);
}
}
return { local, remote };
if (local.state === "error") {
this._configError$.next(
local.value instanceof ElementCallError
? local.value
: new UnknownCallError(local.value),
);
}
return { local, remote, preferred, multiSfu };
},
)
: of(null),
@@ -576,6 +334,35 @@ export class CallViewModel extends ViewModel {
),
);
/**
* The transport we should advertise in our MatrixRTC membership (plus whether
* it is a multi-SFU transport).
*/
private readonly advertisedTransport$: Behavior<{
multiSfu: boolean;
transport: LivekitTransport;
} | null> = this.scope.behavior(
this.transports$.pipe(
map((transports) =>
transports?.local.state === "ready" &&
transports.preferred.state === "ready"
? {
multiSfu: transports.multiSfu,
// In non-multi-SFU mode we should always advertise the preferred
// SFU to minimize the number of membership updates
transport: transports.multiSfu
? transports.local.value
: transports.preferred.value,
}
: null,
),
distinctUntilChanged<{
multiSfu: boolean;
transport: LivekitTransport;
} | null>(deepCompare),
),
);
/**
* The local connection over which we will publish our media. It could
* possibly also have some remote users' media available on it.
@@ -606,22 +393,23 @@ export class CallViewModel extends ViewModel {
),
);
public readonly livekitConnectionState$ = this.scope.behavior(
this.localConnection$.pipe(
switchMap((c) =>
c?.state === "ready"
? // TODO mapping to ConnectionState for compatibility, but we should use the full state?
c.value.focusConnectionState$.pipe(
map((s) => {
if (s.state === "ConnectedToLkRoom") return s.connectionState;
return ConnectionState.Disconnected;
}),
distinctUntilChanged(),
)
: of(ConnectionState.Disconnected),
public readonly livekitConnectionState$ =
this.scope.behavior<ConnectionState>(
this.localConnection$.pipe(
switchMap((c) =>
c?.state === "ready"
? // TODO mapping to ConnectionState for compatibility, but we should use the full state?
c.value.transportState$.pipe(
switchMap((s) => {
if (s.state === "ConnectedToLkRoom")
return s.connectionState$;
return of(ConnectionState.Disconnected);
}),
)
: of(ConnectionState.Disconnected),
),
),
),
);
);
/**
* Connections for each transport in use by one or more session members that
@@ -719,7 +507,7 @@ export class CallViewModel extends ViewModel {
map((connections) =>
[...connections.values()].map((c) => ({
room: c.livekitRoom,
url: c.localTransport.livekit_service_url,
url: c.transport.livekit_service_url,
isLocal: c instanceof PublishConnection,
})),
),
@@ -729,6 +517,9 @@ export class CallViewModel extends ViewModel {
private readonly userId = this.matrixRoom.client.getUserId()!;
private readonly deviceId = this.matrixRoom.client.getDeviceId()!;
/**
* Whether we are connected to the MatrixRTC session.
*/
private readonly matrixConnected$ = this.scope.behavior(
// To consider ourselves connected to MatrixRTC, we check the following:
and$(
@@ -763,6 +554,10 @@ export class CallViewModel extends ViewModel {
),
);
/**
* Whether we are "fully" connected to the call. Accounts for both the
* connection to the MatrixRTC session and the LiveKit publish connection.
*/
private readonly connected$ = this.scope.behavior(
and$(
this.matrixConnected$,
@@ -780,7 +575,7 @@ export class CallViewModel extends ViewModel {
// We are reconnecting if we previously had some successful initial
// connection but are now disconnected
scan(
({ connectedPreviously, reconnecting }, connectedNow) => ({
({ connectedPreviously }, connectedNow) => ({
connectedPreviously: connectedPreviously || connectedNow,
reconnecting: connectedPreviously && !connectedNow,
}),
@@ -807,7 +602,7 @@ export class CallViewModel extends ViewModel {
private readonly participantsByRoom$ = this.scope.behavior<
{
livekitRoom: LivekitRoom;
url: string;
url: string; // Included for use as a React key
participants: {
id: string;
participant: LocalParticipant | RemoteParticipant | undefined;
@@ -844,7 +639,7 @@ export class CallViewModel extends ViewModel {
| undefined;
member: RoomMember;
}[] = ps.map(({ participant, membership }) => ({
id: `${membership.sender}:${membership.deviceId}`,
id: `${membership.userId}:${membership.deviceId}`,
participant,
member:
getRoomMemberFromRtcMember(
@@ -857,7 +652,7 @@ export class CallViewModel extends ViewModel {
return {
livekitRoom: c.livekitRoom,
url: c.localTransport.livekit_service_url,
url: c.transport.livekit_service_url,
participants,
};
}),
@@ -921,7 +716,7 @@ export class CallViewModel extends ViewModel {
// We only consider RTC members for disambiguation as they are the only visible members.
for (const rtcMember of memberships) {
const matrixIdentifier = `${rtcMember.sender}:${rtcMember.deviceId}`;
const matrixIdentifier = `${rtcMember.userId}:${rtcMember.deviceId}`;
const { member } = getRoomMemberFromRtcMember(rtcMember, room);
if (!member) {
logger.error(
@@ -967,7 +762,11 @@ export class CallViewModel extends ViewModel {
scan((prevItems, [participantsByRoom, duplicateTiles]) => {
const newItems: Map<string, UserMedia | ScreenShare> = new Map(
function* (this: CallViewModel): Iterable<[string, MediaItem]> {
for (const { livekitRoom, participants } of participantsByRoom) {
for (const {
livekitRoom,
participants,
url,
} of participantsByRoom) {
for (const { id, participant, member } of participants) {
for (let i = 0; i < 1 + duplicateTiles; i++) {
const mediaId = `${id}:${i}`;
@@ -987,6 +786,7 @@ export class CallViewModel extends ViewModel {
participant,
this.options.encryptionSystem,
livekitRoom,
url,
this.mediaDevices,
this.pretendToBeDisconnected$,
this.memberDisplaynames$.pipe(
@@ -1008,6 +808,7 @@ export class CallViewModel extends ViewModel {
participant,
this.options.encryptionSystem,
livekitRoom,
url,
this.pretendToBeDisconnected$,
this.memberDisplaynames$.pipe(
map((m) => m.get(id) ?? "[👻]"),
@@ -1068,8 +869,8 @@ export class CallViewModel extends ViewModel {
pairwise(),
filter(
([prev, current]) =>
current.every((m) => m.sender === this.userId) &&
prev.some((m) => m.sender !== this.userId),
current.every((m) => m.userId === this.userId) &&
prev.some((m) => m.userId !== this.userId),
),
map(() => {}),
);
@@ -1144,7 +945,7 @@ export class CallViewModel extends ViewModel {
* Whether some Matrix user other than ourself is joined to the call.
*/
private readonly someoneElseJoined$ = this.memberships$.pipe(
map((ms) => ms.some((m) => m.sender !== this.userId)),
map((ms) => ms.some((m) => m.userId !== this.userId)),
) as Behavior<boolean>;
/**
@@ -1288,31 +1089,7 @@ export class CallViewModel extends ViewModel {
this.userMedia$.pipe(
switchMap((mediaItems) => {
const bins = mediaItems.map((m) =>
combineLatest(
[
m.speaker$,
m.presenter$,
m.vm.videoEnabled$,
m.vm.handRaised$,
m.vm instanceof LocalUserMediaViewModel
? m.vm.alwaysShow$
: of(false),
],
(speaker, presenter, video, handRaised, alwaysShow) => {
let bin: SortingBin;
if (m.vm.local)
bin = alwaysShow
? SortingBin.SelfAlwaysShown
: SortingBin.SelfNotAlwaysShown;
else if (presenter) bin = SortingBin.Presenters;
else if (speaker) bin = SortingBin.Speakers;
else if (handRaised) bin = SortingBin.HandRaised;
else if (video) bin = SortingBin.Video;
else bin = SortingBin.NoVideo;
return [m, bin] as const;
},
),
m.bin$.pipe(map((bin) => [m, bin] as const)),
);
// Sort the media by bin order and generate a tile for each one
return bins.length === 0
@@ -1983,13 +1760,11 @@ export class CallViewModel extends ViewModel {
.pipe(this.scope.bind())
.subscribe(({ start, stop }) => {
for (const c of stop) {
logger.info(
`Disconnecting from ${c.localTransport.livekit_service_url}`,
);
logger.info(`Disconnecting from ${c.transport.livekit_service_url}`);
c.stop().catch((err) => {
// TODO: better error handling
logger.error(
`Fail to stop connection to ${c.localTransport.livekit_service_url}`,
`Fail to stop connection to ${c.transport.livekit_service_url}`,
err,
);
});
@@ -1997,45 +1772,53 @@ export class CallViewModel extends ViewModel {
for (const c of start) {
c.start().then(
() =>
logger.info(
`Connected to ${c.localTransport.livekit_service_url}`,
),
(e) =>
logger.info(`Connected to ${c.transport.livekit_service_url}`),
(e) => {
// We only want to report fatal errors `_configError$` for the publish connection.
// If there is an error with another connection, it will not terminate the call and will be displayed
// on eacn tile.
if (
c instanceof PublishConnection &&
e instanceof ElementCallError
) {
this._configError$.next(e);
}
logger.error(
`Failed to start connection to ${c.localTransport.livekit_service_url}`,
`Failed to start connection to ${c.transport.livekit_service_url}`,
e,
),
);
},
);
}
});
// Start and stop session membership as needed
this.scope.reconcile(this.localTransport$, async (localTransport) => {
if (localTransport?.state === "ready") {
this.scope.reconcile(this.advertisedTransport$, async (advertised) => {
if (advertised !== null) {
try {
await enterRTCSession(
this.matrixRTCSession,
localTransport.value,
this.options.encryptionSystem.kind !== E2eeType.NONE,
true,
true,
multiSfu.value$.value,
);
this._configError$.next(null);
await enterRTCSession(this.matrixRTCSession, advertised.transport, {
encryptMedia: this.options.encryptionSystem.kind !== E2eeType.NONE,
useExperimentalToDeviceTransport: true,
useNewMembershipManager: true,
useMultiSfu: advertised.multiSfu,
});
} catch (e) {
logger.error("Error entering RTC session", e);
}
// Update our member event when our mute state changes.
const muteSubscription = this.muteStates.video.enabled$.subscribe(
(videoEnabled) =>
// TODO: Ensure that these calls are serialized in case of
// fast video toggling
void this.matrixRTCSession.updateCallIntent(
const intentScope = new ObservableScope();
intentScope.reconcile(
this.muteStates.video.enabled$,
async (videoEnabled) =>
this.matrixRTCSession.updateCallIntent(
videoEnabled ? "video" : "audio",
),
);
return async (): Promise<void> => {
muteSubscription.unsubscribe();
intentScope.end();
// Only sends Matrix leave event. The LiveKit session will disconnect
// as soon as either the stopConnection$ handler above gets to it or
// the view model is destroyed.
@@ -2131,3 +1914,23 @@ function getE2eeKeyProvider(
return keyProvider;
}
}
function getRoomMemberFromRtcMember(
rtcMember: CallMembership,
room: MatrixRoom,
): { id: string; member: RoomMember | undefined } {
let id = rtcMember.userId + ":" + rtcMember.deviceId;
if (!rtcMember.userId) {
return { id, member: undefined };
}
if (
rtcMember.userId === room.client.getUserId() &&
rtcMember.deviceId === room.client.getDeviceId()
) {
id = "local";
}
const member = room.getMember(rtcMember.userId) ?? undefined;
return { id, member };
}

View File

@@ -34,7 +34,8 @@ import type {
} from "matrix-js-sdk/lib/matrixrtc";
import {
type ConnectionOpts,
type FocusConnectionState,
type TransportState,
type PublishingParticipant,
RemoteConnection,
} from "./Connection.ts";
import { ObservableScope } from "./ObservableScope.ts";
@@ -160,9 +161,7 @@ describe("Start connection states", () => {
};
const connection = new RemoteConnection(opts, undefined);
expect(connection.focusConnectionState$.getValue().state).toEqual(
"Initialized",
);
expect(connection.transportState$.getValue().state).toEqual("Initialized");
});
it("fail to getOpenId token then error state", async () => {
@@ -179,8 +178,8 @@ describe("Start connection states", () => {
const connection = new RemoteConnection(opts, undefined);
const capturedStates: FocusConnectionState[] = [];
const s = connection.focusConnectionState$.subscribe((value) => {
const capturedStates: TransportState[] = [];
const s = connection.transportState$.subscribe((value) => {
capturedStates.push(value);
});
onTestFinished(() => s.unsubscribe());
@@ -208,7 +207,7 @@ describe("Start connection states", () => {
capturedState = capturedStates.pop();
if (capturedState!.state === "FailedToStart") {
expect(capturedState!.error.message).toEqual("Something went wrong");
expect(capturedState!.focus.livekit_alias).toEqual(
expect(capturedState!.transport.livekit_alias).toEqual(
livekitFocus.livekit_alias,
);
} else {
@@ -232,8 +231,8 @@ describe("Start connection states", () => {
const connection = new RemoteConnection(opts, undefined);
const capturedStates: FocusConnectionState[] = [];
const s = connection.focusConnectionState$.subscribe((value) => {
const capturedStates: TransportState[] = [];
const s = connection.transportState$.subscribe((value) => {
capturedStates.push(value);
});
onTestFinished(() => s.unsubscribe());
@@ -265,7 +264,7 @@ describe("Start connection states", () => {
expect(capturedState?.error.message).toContain(
"SFU Config fetch failed with exception Error",
);
expect(capturedState?.focus.livekit_alias).toEqual(
expect(capturedState?.transport.livekit_alias).toEqual(
livekitFocus.livekit_alias,
);
} else {
@@ -289,8 +288,8 @@ describe("Start connection states", () => {
const connection = new RemoteConnection(opts, undefined);
const capturedStates: FocusConnectionState[] = [];
const s = connection.focusConnectionState$.subscribe((value) => {
const capturedStates: TransportState[] = [];
const s = connection.transportState$.subscribe((value) => {
capturedStates.push(value);
});
onTestFinished(() => s.unsubscribe());
@@ -330,7 +329,7 @@ describe("Start connection states", () => {
expect(capturedState.error.message).toContain(
"Failed to connect to livekit",
);
expect(capturedState.focus.livekit_alias).toEqual(
expect(capturedState.transport.livekit_alias).toEqual(
livekitFocus.livekit_alias,
);
} else {
@@ -346,8 +345,8 @@ describe("Start connection states", () => {
const connection = setupRemoteConnection();
const capturedStates: FocusConnectionState[] = [];
const s = connection.focusConnectionState$.subscribe((value) => {
const capturedStates: TransportState[] = [];
const s = connection.transportState$.subscribe((value) => {
capturedStates.push(value);
});
onTestFinished(() => s.unsubscribe());
@@ -365,59 +364,6 @@ describe("Start connection states", () => {
expect(connectedState?.state).toEqual("ConnectedToLkRoom");
});
it("should relay livekit events once connected", async () => {
setupTest();
const connection = setupRemoteConnection();
await connection.start();
let capturedStates: FocusConnectionState[] = [];
const s = connection.focusConnectionState$.subscribe((value) => {
capturedStates.push(value);
});
onTestFinished(() => s.unsubscribe());
const states = [
ConnectionState.Disconnected,
ConnectionState.Connecting,
ConnectionState.Connected,
ConnectionState.SignalReconnecting,
ConnectionState.Connecting,
ConnectionState.Connected,
ConnectionState.Reconnecting,
];
for (const state of states) {
fakeRoomEventEmiter.emit(RoomEvent.ConnectionStateChanged, state);
}
for (const state of states) {
const s = capturedStates.shift();
expect(s?.state).toEqual("ConnectedToLkRoom");
const connectedState = s as FocusConnectionState & {
state: "ConnectedToLkRoom";
};
expect(connectedState.connectionState).toEqual(state);
// should always have the focus info
expect(connectedState.focus.livekit_alias).toEqual(
livekitFocus.livekit_alias,
);
expect(connectedState.focus.livekit_service_url).toEqual(
livekitFocus.livekit_service_url,
);
}
// If the state is not ConnectedToLkRoom, no events should be relayed anymore
await connection.stop();
capturedStates = [];
for (const state of states) {
fakeRoomEventEmiter.emit(RoomEvent.ConnectionStateChanged, state);
}
expect(capturedStates.length).toEqual(0);
});
it("shutting down the scope should stop the connection", async () => {
setupTest();
vi.useFakeTimers();
@@ -434,16 +380,16 @@ describe("Start connection states", () => {
});
function fakeRemoteLivekitParticipant(id: string): RemoteParticipant {
return vi.mocked<RemoteParticipant>({
return {
identity: id,
} as unknown as RemoteParticipant);
} as unknown as RemoteParticipant;
}
function fakeRtcMemberShip(userId: string, deviceId: string): CallMembership {
return vi.mocked<CallMembership>({
sender: userId,
deviceId: deviceId,
} as unknown as CallMembership);
return {
userId,
deviceId,
} as unknown as CallMembership;
}
describe("Publishing participants observations", () => {
@@ -454,22 +400,19 @@ describe("Publishing participants observations", () => {
const bobIsAPublisher = Promise.withResolvers<void>();
const danIsAPublisher = Promise.withResolvers<void>();
const observedPublishers: {
participant: RemoteParticipant;
membership: CallMembership;
}[][] = [];
const observedPublishers: PublishingParticipant[][] = [];
const s = connection.publishingParticipants$.subscribe((publishers) => {
observedPublishers.push(publishers);
if (
publishers.some(
(p) => p.participant.identity === "@bob:example.org:DEV111",
(p) => p.participant?.identity === "@bob:example.org:DEV111",
)
) {
bobIsAPublisher.resolve();
}
if (
publishers.some(
(p) => p.participant.identity === "@dan:example.org:DEV333",
(p) => p.participant?.identity === "@dan:example.org:DEV333",
)
) {
danIsAPublisher.resolve();
@@ -529,7 +472,7 @@ describe("Publishing participants observations", () => {
await bobIsAPublisher.promise;
const publishers = observedPublishers.pop();
expect(publishers?.length).toEqual(1);
expect(publishers?.[0].participant.identity).toEqual(
expect(publishers?.[0].participant?.identity).toEqual(
"@bob:example.org:DEV111",
);
@@ -546,12 +489,12 @@ describe("Publishing participants observations", () => {
expect(twoPublishers?.length).toEqual(2);
expect(
twoPublishers?.some(
(p) => p.participant.identity === "@bob:example.org:DEV111",
(p) => p.participant?.identity === "@bob:example.org:DEV111",
),
).toBeTruthy();
expect(
twoPublishers?.some(
(p) => p.participant.identity === "@dan:example.org:DEV333",
(p) => p.participant?.identity === "@dan:example.org:DEV333",
),
).toBeTruthy();
@@ -568,12 +511,25 @@ describe("Publishing participants observations", () => {
);
const updatedPublishers = observedPublishers.pop();
expect(updatedPublishers?.length).toEqual(1);
// Bob is not connected to the room but he is still in the rtc memberships declaring that
// he is using that focus to publish, so he should still appear as a publisher
expect(updatedPublishers?.length).toEqual(2);
const pp = updatedPublishers?.find(
(p) => p.membership.userId == "@bob:example.org",
);
expect(pp).toBeDefined();
expect(pp!.participant).not.toBeDefined();
expect(
updatedPublishers?.some(
(p) => p.participant.identity === "@dan:example.org:DEV333",
(p) => p.participant?.identity === "@dan:example.org:DEV333",
),
).toBeTruthy();
// Now if bob is not in the rtc memberships, he should disappear
const noBob = rtcMemberships.filter(
({ membership }) => membership.userId !== "@bob:example.org",
);
fakeMembershipsFocusMap$.next(noBob);
expect(observedPublishers.pop()?.length).toEqual(1);
});
it("should be scoped to parent scope", (): void => {
@@ -581,10 +537,7 @@ describe("Publishing participants observations", () => {
const connection = setupRemoteConnection();
let observedPublishers: {
participant: RemoteParticipant;
membership: CallMembership;
}[][] = [];
let observedPublishers: PublishingParticipant[][] = [];
const s = connection.publishingParticipants$.subscribe((publishers) => {
observedPublishers.push(publishers);
});
@@ -619,7 +572,7 @@ describe("Publishing participants observations", () => {
// We should have bob has a publisher now
const publishers = observedPublishers.pop();
expect(publishers?.length).toEqual(1);
expect(publishers?.[0].participant.identity).toEqual(
expect(publishers?.[0].participant?.identity).toEqual(
"@bob:example.org:DEV111",
);

View File

@@ -10,8 +10,10 @@ import {
connectionStateObserver,
} from "@livekit/components-core";
import {
ConnectionError,
type ConnectionState,
type E2EEOptions,
type RemoteParticipant,
Room as LivekitRoom,
type RoomOptions,
} from "livekit-client";
@@ -19,7 +21,7 @@ import {
type CallMembership,
type LivekitTransport,
} from "matrix-js-sdk/lib/matrixrtc";
import { BehaviorSubject, combineLatest } from "rxjs";
import { BehaviorSubject, combineLatest, type Observable } from "rxjs";
import {
getSFUConfigWithOpenID,
@@ -29,15 +31,19 @@ import {
import { type Behavior } from "./Behavior";
import { type ObservableScope } from "./ObservableScope";
import { defaultLiveKitOptions } from "../livekit/options";
import {
InsufficientCapacityError,
SFURoomCreationRestrictedError,
} from "../utils/errors.ts";
export interface ConnectionOpts {
/** The focus server to connect to. */
/** The media transport to connect to. */
transport: LivekitTransport;
/** The Matrix client to use for OpenID and SFU config requests. */
client: OpenIDClientParts;
/** The observable scope to use for this connection. */
scope: ObservableScope;
/** An observable of the current RTC call memberships and their associated focus. */
/** An observable of the current RTC call memberships and their associated transports. */
remoteTransports$: Behavior<
{ membership: CallMembership; transport: LivekitTransport }[]
>;
@@ -46,18 +52,33 @@ export interface ConnectionOpts {
livekitRoomFactory?: (options?: RoomOptions) => LivekitRoom;
}
export type FocusConnectionState =
export type TransportState =
| { state: "Initialized" }
| { state: "FetchingConfig"; focus: LivekitTransport }
| { state: "ConnectingToLkRoom"; focus: LivekitTransport }
| { state: "PublishingTracks"; focus: LivekitTransport }
| { state: "FailedToStart"; error: Error; focus: LivekitTransport }
| { state: "FetchingConfig"; transport: LivekitTransport }
| { state: "ConnectingToLkRoom"; transport: LivekitTransport }
| { state: "PublishingTracks"; transport: LivekitTransport }
| { state: "FailedToStart"; error: Error; transport: LivekitTransport }
| {
state: "ConnectedToLkRoom";
connectionState: ConnectionState;
focus: LivekitTransport;
connectionState$: Observable<ConnectionState>;
transport: LivekitTransport;
}
| { state: "Stopped"; focus: LivekitTransport };
| { state: "Stopped"; transport: LivekitTransport };
/**
* Represents participant publishing or expected to publish on the connection.
* It is paired with its associated rtc membership.
*/
export type PublishingParticipant = {
/**
* The LiveKit participant publishing on this connection, or undefined if the participant is not currently (yet) connected to the livekit room.
*/
participant: RemoteParticipant | undefined;
/**
* The rtc call membership associated with this participant.
*/
membership: CallMembership;
};
/**
* A connection to a Matrix RTC LiveKit backend.
@@ -66,14 +87,15 @@ export type FocusConnectionState =
*/
export class Connection {
// Private Behavior
private readonly _focusConnectionState$ =
new BehaviorSubject<FocusConnectionState>({ state: "Initialized" });
private readonly _transportState$ = new BehaviorSubject<TransportState>({
state: "Initialized",
});
/**
* The current state of the connection to the focus server.
* The current state of the connection to the media transport.
*/
public readonly focusConnectionState$: Behavior<FocusConnectionState> =
this._focusConnectionState$;
public readonly transportState$: Behavior<TransportState> =
this._transportState$;
/**
* Whether the connection has been stopped.
@@ -88,37 +110,62 @@ export class Connection {
* 1. Request an OpenId token `request_token` (allows matrix users to verify their identity with a third-party service.)
* 2. Use this token to request the SFU config to the MatrixRtc authentication service.
* 3. Connect to the configured LiveKit room.
*
* @throws {InsufficientCapacityError} if the LiveKit server indicates that it has insufficient capacity to accept the connection.
* @throws {SFURoomCreationRestrictedError} if the LiveKit server indicates that the room does not exist and cannot be created.
*/
public async start(): Promise<void> {
this.stopped = false;
try {
this._focusConnectionState$.next({
this._transportState$.next({
state: "FetchingConfig",
focus: this.localTransport,
transport: this.transport,
});
// TODO could this be loaded earlier to save time?
const { url, jwt } = await this.getSFUConfigWithOpenID();
// If we were stopped while fetching the config, don't proceed to connect
if (this.stopped) return;
this._focusConnectionState$.next({
this._transportState$.next({
state: "ConnectingToLkRoom",
focus: this.localTransport,
transport: this.transport,
});
await this.livekitRoom.connect(url, jwt);
try {
await this.livekitRoom.connect(url, jwt);
} catch (e) {
// LiveKit uses 503 to indicate that the server has hit its track limits.
// https://github.com/livekit/livekit/blob/fcb05e97c5a31812ecf0ca6f7efa57c485cea9fb/pkg/service/rtcservice.go#L171
// It also errors with a status code of 200 (yes, really) for room
// participant limits.
// LiveKit Cloud uses 429 for connection limits.
// Either way, all these errors can be explained as "insufficient capacity".
if (e instanceof ConnectionError) {
if (e.status === 503 || e.status === 200 || e.status === 429) {
throw new InsufficientCapacityError();
}
if (e.status === 404) {
// error msg is "Could not establish signal connection: requested room does not exist"
// The room does not exist. There are two different modes of operation for the SFU:
// - the room is created on the fly when connecting (livekit `auto_create` option)
// - Only authorized users can create rooms, so the room must exist before connecting (done by the auth jwt service)
// In the first case there will not be a 404, so we are in the second case.
throw new SFURoomCreationRestrictedError();
}
}
throw e;
}
// If we were stopped while connecting, don't proceed to update state.
if (this.stopped) return;
this._focusConnectionState$.next({
this._transportState$.next({
state: "ConnectedToLkRoom",
focus: this.localTransport,
connectionState: this.livekitRoom.state,
transport: this.transport,
connectionState$: connectionStateObserver(this.livekitRoom),
});
} catch (error) {
this._focusConnectionState$.next({
this._transportState$.next({
state: "FailedToStart",
error: error instanceof Error ? error : new Error(`${error}`),
focus: this.localTransport,
transport: this.transport,
});
throw error;
}
@@ -127,8 +174,8 @@ export class Connection {
protected async getSFUConfigWithOpenID(): Promise<SFUConfig> {
return await getSFUConfigWithOpenID(
this.client,
this.localTransport.livekit_service_url,
this.localTransport.livekit_alias,
this.transport.livekit_service_url,
this.transport.livekit_alias,
);
}
/**
@@ -140,9 +187,9 @@ export class Connection {
public async stop(): Promise<void> {
if (this.stopped) return;
await this.livekitRoom.disconnect();
this._focusConnectionState$.next({
this._transportState$.next({
state: "Stopped",
focus: this.localTransport,
transport: this.transport,
});
this.stopped = true;
}
@@ -152,12 +199,12 @@ export class Connection {
* This is derived from `participantsIncludingSubscribers$` and `remoteTransports$`.
* It filters the participants to only those that are associated with a membership that claims to publish on this connection.
*/
public readonly publishingParticipants$;
public readonly publishingParticipants$: Behavior<PublishingParticipant[]>;
/**
* The focus server to connect to.
* The media transport to connect to.
*/
public readonly localTransport: LivekitTransport;
public readonly transport: LivekitTransport;
private readonly client: OpenIDClientParts;
/**
@@ -173,7 +220,7 @@ export class Connection {
) {
const { transport, client, scope, remoteTransports$ } = opts;
this.localTransport = transport;
this.transport = transport;
this.client = client;
const participantsIncludingSubscribers$ = scope.behavior(
@@ -189,35 +236,20 @@ export class Connection {
// Find all members that claim to publish on this connection
.flatMap(({ membership, transport }) =>
transport.livekit_service_url ===
this.localTransport.livekit_service_url
this.transport.livekit_service_url
? [membership]
: [],
)
// Pair with their associated LiveKit participant (if any)
// Uses flatMap to filter out memberships with no associated rtc participant ([])
.flatMap((membership) => {
const id = `${membership.sender}:${membership.deviceId}`;
.map((membership) => {
const id = `${membership.userId}:${membership.deviceId}`;
const participant = participants.find((p) => p.identity === id);
return participant ? [{ participant, membership }] : [];
return { participant, membership };
}),
),
[],
);
scope
.behavior<ConnectionState>(connectionStateObserver(this.livekitRoom))
.subscribe((connectionState) => {
const current = this._focusConnectionState$.value;
// Only update the state if we are already connected to the LiveKit room.
if (current.state === "ConnectedToLkRoom") {
this._focusConnectionState$.next({
state: "ConnectedToLkRoom",
connectionState,
focus: current.focus,
});
}
});
scope.onEnd(() => void this.stop());
}
}

View File

@@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type Layout, type LayoutMedia } from "./CallViewModel";
import { type Layout, type LayoutMedia } from "./layout-types.ts";
import { type TileStore } from "./TileStore";
export type GridLikeLayoutType =

View File

@@ -266,6 +266,7 @@ abstract class BaseMediaViewModel extends ViewModel {
audioSource: AudioSource,
videoSource: VideoSource,
livekitRoom: LivekitRoom,
public readonly focusURL: string,
public readonly displayName$: Behavior<string>,
) {
super();
@@ -407,6 +408,7 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel {
participant$: Observable<LocalParticipant | RemoteParticipant | undefined>,
encryptionSystem: EncryptionSystem,
livekitRoom: LivekitRoom,
focusUrl: string,
displayName$: Behavior<string>,
public readonly handRaised$: Behavior<Date | null>,
public readonly reaction$: Behavior<ReactionOption | null>,
@@ -419,6 +421,7 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel {
Track.Source.Microphone,
Track.Source.Camera,
livekitRoom,
focusUrl,
displayName$,
);
@@ -539,6 +542,7 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel {
participant$: Behavior<LocalParticipant | undefined>,
encryptionSystem: EncryptionSystem,
livekitRoom: LivekitRoom,
focusURL: string,
private readonly mediaDevices: MediaDevices,
displayName$: Behavior<string>,
handRaised$: Behavior<Date | null>,
@@ -550,6 +554,7 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel {
participant$,
encryptionSystem,
livekitRoom,
focusURL,
displayName$,
handRaised$,
reaction$,
@@ -645,6 +650,7 @@ export class RemoteUserMediaViewModel extends BaseUserMediaViewModel {
participant$: Observable<RemoteParticipant | undefined>,
encryptionSystem: EncryptionSystem,
livekitRoom: LivekitRoom,
focusUrl: string,
private readonly pretendToBeDisconnected$: Behavior<boolean>,
displayname$: Behavior<string>,
handRaised$: Behavior<Date | null>,
@@ -656,6 +662,7 @@ export class RemoteUserMediaViewModel extends BaseUserMediaViewModel {
participant$,
encryptionSystem,
livekitRoom,
focusUrl,
displayname$,
handRaised$,
reaction$,
@@ -740,6 +747,7 @@ export class ScreenShareViewModel extends BaseMediaViewModel {
participant$: Observable<LocalParticipant | RemoteParticipant>,
encryptionSystem: EncryptionSystem,
livekitRoom: LivekitRoom,
focusUrl: string,
private readonly pretendToBeDisconnected$: Behavior<boolean>,
displayname$: Behavior<string>,
public readonly local: boolean,
@@ -752,6 +760,7 @@ export class ScreenShareViewModel extends BaseMediaViewModel {
Track.Source.ScreenShareAudio,
Track.Source.ScreenShare,
livekitRoom,
focusUrl,
displayname$,
);
}

View File

@@ -114,11 +114,11 @@ export class ObservableScope {
*/
public reconcile<T>(
value$: Behavior<T>,
callback: (value: T) => Promise<(() => Promise<void>) | undefined>,
callback: (value: T) => Promise<(() => Promise<void>) | void>,
): void {
let latestValue: T | typeof nothing = nothing;
let reconciledValue: T | typeof nothing = nothing;
let cleanUp: (() => Promise<void>) | undefined = undefined;
let cleanUp: (() => Promise<void>) | void = undefined;
value$
.pipe(
catchError(() => EMPTY), // Ignore errors

View File

@@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type OneOnOneLayout, type OneOnOneLayoutMedia } from "./CallViewModel";
import { type OneOnOneLayout, type OneOnOneLayoutMedia } from "./layout-types";
import { type TileStore } from "./TileStore";
/**

View File

@@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type PipLayout, type PipLayoutMedia } from "./CallViewModel";
import { type PipLayout, type PipLayoutMedia } from "./layout-types.ts";
import { type TileStore } from "./TileStore";
/**

View File

@@ -40,6 +40,8 @@ import { type ObservableScope } from "./ObservableScope.ts";
* This connection will publish the local user's audio and video tracks.
*/
export class PublishConnection extends Connection {
private readonly scope: ObservableScope;
/**
* Creates a new PublishConnection.
* @param args - The connection options. {@link ConnectionOpts}
@@ -75,11 +77,10 @@ export class PublishConnection extends Connection {
});
super(room, args);
this.scope = scope;
// Setup track processor syncing (blur)
this.observeTrackProcessors(scope, room, trackerProcessorState$);
// Observe mute state changes and update LiveKit microphone/camera states accordingly
this.observeMuteStates(scope);
// Observe media device changes and update LiveKit active devices accordingly
this.observeMediaDevices(scope, devices, controlledAudioDevices);
@@ -94,29 +95,53 @@ export class PublishConnection extends Connection {
* 2. Use this token to request the SFU config to the MatrixRtc authentication service.
* 3. Connect to the configured LiveKit room.
* 4. Create local audio and video tracks based on the current mute states and publish them to the room.
*
* @throws {InsufficientCapacityError} if the LiveKit server indicates that it has insufficient capacity to accept the connection.
* @throws {SFURoomCreationRestrictedError} if the LiveKit server indicates that the room does not exist and cannot be created.
*/
public async start(): Promise<void> {
this.stopped = false;
// Observe mute state changes and update LiveKit microphone/camera states accordingly
this.observeMuteStates(this.scope);
// TODO: This will fetch the JWT token. Perhaps we could keep it preloaded
// instead? This optimization would only be safe for a publish connection,
// because we don't want to leak the user's intent to perhaps join a call to
// remote servers before they actually commit to it.
await super.start();
if (this.stopped) return;
// TODO this can throw errors? It will also prompt for permissions if not already granted
const tracks = await this.livekitRoom.localParticipant.createTracks({
audio: this.muteStates.audio.enabled$.value,
video: this.muteStates.video.enabled$.value,
});
if (this.stopped) return;
for (const track of tracks) {
// TODO: handle errors? Needs the signaling connection to be up, but it has some retries internally
// with a timeout.
await this.livekitRoom.localParticipant.publishTrack(track);
// TODO-MULTI-SFU: Prepublish a microphone track
const audio = this.muteStates.audio.enabled$.value;
const video = this.muteStates.video.enabled$.value;
// createTracks throws if called with audio=false and video=false
if (audio || video) {
// TODO this can still throw errors? It will also prompt for permissions if not already granted
const tracks = await this.livekitRoom.localParticipant.createTracks({
audio,
video,
});
if (this.stopped) return;
// TODO: check if the connection is still active? and break the loop if not?
for (const track of tracks) {
// TODO: handle errors? Needs the signaling connection to be up, but it has some retries internally
// with a timeout.
await this.livekitRoom.localParticipant.publishTrack(track);
if (this.stopped) return;
// TODO: check if the connection is still active? and break the loop if not?
}
}
}
public async stop(): Promise<void> {
// TODO-MULTI-SFU: Move these calls back to ObservableScope.onEnd once scope
// actually has the right lifetime
this.muteStates.audio.unsetHandler();
this.muteStates.video.unsetHandler();
await super.stop();
}
/// Private methods
// Restart the audio input track whenever we detect that the active media
@@ -223,10 +248,6 @@ export class PublishConnection extends Connection {
}
return this.livekitRoom.localParticipant.isCameraEnabled;
});
scope.onEnd(() => {
this.muteStates.audio.unsetHandler();
this.muteStates.video.unsetHandler();
});
}
private observeTrackProcessors(

56
src/state/ScreenShare.ts Normal file
View File

@@ -0,0 +1,56 @@
/*
Copyright 2025 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { of, type Observable } from "rxjs";
import {
type LocalParticipant,
type RemoteParticipant,
type Room as LivekitRoom,
} from "livekit-client";
import { ObservableScope } from "./ObservableScope.ts";
import { ScreenShareViewModel } from "./MediaViewModel.ts";
import type { RoomMember } from "matrix-js-sdk";
import type { EncryptionSystem } from "../e2ee/sharedKeyManagement.ts";
import type { Behavior } from "./Behavior.ts";
/**
* A screen share media item to be presented in a tile. This is a thin wrapper
* around ScreenShareViewModel which essentially just establishes an
* ObservableScope for behaviors that the view model depends on.
*/
export class ScreenShare {
private readonly scope = new ObservableScope();
public readonly vm: ScreenShareViewModel;
public constructor(
id: string,
member: RoomMember,
participant: LocalParticipant | RemoteParticipant,
encryptionSystem: EncryptionSystem,
livekitRoom: LivekitRoom,
focusUrl: string,
pretendToBeDisconnected$: Behavior<boolean>,
displayName$: Observable<string>,
) {
this.vm = new ScreenShareViewModel(
id,
member,
of(participant),
encryptionSystem,
livekitRoom,
focusUrl,
pretendToBeDisconnected$,
this.scope.behavior(displayName$),
participant.isLocal,
);
}
public destroy(): void {
this.scope.end();
this.vm.destroy();
}
}

View File

@@ -8,7 +8,7 @@ Please see LICENSE in the repository root for full details.
import {
type SpotlightExpandedLayout,
type SpotlightExpandedLayoutMedia,
} from "./CallViewModel";
} from "./layout-types";
import { type TileStore } from "./TileStore";
/**

186
src/state/UserMedia.ts Normal file
View File

@@ -0,0 +1,186 @@
/*
Copyright 2025 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
BehaviorSubject,
combineLatest,
map,
type Observable,
of,
switchMap,
} from "rxjs";
import {
type LocalParticipant,
type Participant,
ParticipantEvent,
type RemoteParticipant,
type Room as LivekitRoom,
} from "livekit-client";
import { observeParticipantEvents } from "@livekit/components-core";
import { ObservableScope } from "./ObservableScope.ts";
import {
LocalUserMediaViewModel,
RemoteUserMediaViewModel,
type UserMediaViewModel,
} from "./MediaViewModel.ts";
import type { Behavior } from "./Behavior.ts";
import type { RoomMember } from "matrix-js-sdk";
import type { EncryptionSystem } from "../e2ee/sharedKeyManagement.ts";
import type { MediaDevices } from "./MediaDevices.ts";
import type { ReactionOption } from "../reactions";
import { observeSpeaker$ } from "./observeSpeaker.ts";
/**
* Sorting bins defining the order in which media tiles appear in the layout.
*/
enum SortingBin {
/**
* Yourself, when the "always show self" option is on.
*/
SelfAlwaysShown,
/**
* Participants that are sharing their screen.
*/
Presenters,
/**
* Participants that have been speaking recently.
*/
Speakers,
/**
* Participants that have their hand raised.
*/
HandRaised,
/**
* Participants with video.
*/
Video,
/**
* Participants not sharing any video.
*/
NoVideo,
/**
* Yourself, when the "always show self" option is off.
*/
SelfNotAlwaysShown,
}
/**
* A user media item to be presented in a tile. This is a thin wrapper around
* UserMediaViewModel which additionally determines the media item's sorting bin
* for inclusion in the call layout.
*/
export class UserMedia {
private readonly scope = new ObservableScope();
private readonly participant$ = new BehaviorSubject(this.initialParticipant);
public readonly vm: UserMediaViewModel = this.participant$.value?.isLocal
? new LocalUserMediaViewModel(
this.id,
this.member,
this.participant$ as Behavior<LocalParticipant>,
this.encryptionSystem,
this.livekitRoom,
this.focusURL,
this.mediaDevices,
this.scope.behavior(this.displayname$),
this.scope.behavior(this.handRaised$),
this.scope.behavior(this.reaction$),
)
: new RemoteUserMediaViewModel(
this.id,
this.member,
this.participant$ as Observable<RemoteParticipant | undefined>,
this.encryptionSystem,
this.livekitRoom,
this.focusURL,
this.pretendToBeDisconnected$,
this.scope.behavior(this.displayname$),
this.scope.behavior(this.handRaised$),
this.scope.behavior(this.reaction$),
);
private readonly speaker$ = this.scope.behavior(
observeSpeaker$(this.vm.speaking$),
);
private readonly presenter$ = this.scope.behavior(
this.participant$.pipe(
switchMap((p) => (p === undefined ? of(false) : sharingScreen$(p))),
),
);
/**
* Which sorting bin the media item should be placed in.
*/
// This is exposed here rather than by UserMediaViewModel because it's only
// relevant to the layout algorithms; the MediaView component should be
// ignorant of this value.
public readonly bin$ = combineLatest(
[
this.speaker$,
this.presenter$,
this.vm.videoEnabled$,
this.vm.handRaised$,
this.vm instanceof LocalUserMediaViewModel
? this.vm.alwaysShow$
: of(false),
],
(speaker, presenter, video, handRaised, alwaysShow) => {
if (this.vm.local)
return alwaysShow
? SortingBin.SelfAlwaysShown
: SortingBin.SelfNotAlwaysShown;
else if (presenter) return SortingBin.Presenters;
else if (speaker) return SortingBin.Speakers;
else if (handRaised) return SortingBin.HandRaised;
else if (video) return SortingBin.Video;
else return SortingBin.NoVideo;
},
);
public constructor(
public readonly id: string,
private readonly member: RoomMember,
private readonly initialParticipant:
| LocalParticipant
| RemoteParticipant
| undefined,
private readonly encryptionSystem: EncryptionSystem,
private readonly livekitRoom: LivekitRoom,
private readonly focusURL: string,
private readonly mediaDevices: MediaDevices,
private readonly pretendToBeDisconnected$: Behavior<boolean>,
private readonly displayname$: Observable<string>,
private readonly handRaised$: Observable<Date | null>,
private readonly reaction$: Observable<ReactionOption | null>,
) {}
public updateParticipant(
newParticipant: LocalParticipant | RemoteParticipant | undefined,
): void {
if (this.participant$.value !== newParticipant) {
// Update the BehaviourSubject in the UserMedia.
this.participant$.next(newParticipant);
}
}
public destroy(): void {
this.scope.end();
this.vm.destroy();
}
}
export function sharingScreen$(p: Participant): Observable<boolean> {
return observeParticipantEvents(
p,
ParticipantEvent.TrackPublished,
ParticipantEvent.TrackUnpublished,
ParticipantEvent.LocalTrackPublished,
ParticipantEvent.LocalTrackUnpublished,
).pipe(map((p) => p.isScreenShareEnabled));
}

108
src/state/layout-types.ts Normal file
View File

@@ -0,0 +1,108 @@
/*
Copyright 2025 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
type GridTileViewModel,
type SpotlightTileViewModel,
} from "./TileViewModel.ts";
import {
type MediaViewModel,
type UserMediaViewModel,
} from "./MediaViewModel.ts";
export interface GridLayoutMedia {
type: "grid";
spotlight?: MediaViewModel[];
grid: UserMediaViewModel[];
}
export interface SpotlightLandscapeLayoutMedia {
type: "spotlight-landscape";
spotlight: MediaViewModel[];
grid: UserMediaViewModel[];
}
export interface SpotlightPortraitLayoutMedia {
type: "spotlight-portrait";
spotlight: MediaViewModel[];
grid: UserMediaViewModel[];
}
export interface SpotlightExpandedLayoutMedia {
type: "spotlight-expanded";
spotlight: MediaViewModel[];
pip?: UserMediaViewModel;
}
export interface OneOnOneLayoutMedia {
type: "one-on-one";
local: UserMediaViewModel;
remote: UserMediaViewModel;
}
export interface PipLayoutMedia {
type: "pip";
spotlight: MediaViewModel[];
}
export type LayoutMedia =
| GridLayoutMedia
| SpotlightLandscapeLayoutMedia
| SpotlightPortraitLayoutMedia
| SpotlightExpandedLayoutMedia
| OneOnOneLayoutMedia
| PipLayoutMedia;
export interface GridLayout {
type: "grid";
spotlight?: SpotlightTileViewModel;
grid: GridTileViewModel[];
setVisibleTiles: (value: number) => void;
}
export interface SpotlightLandscapeLayout {
type: "spotlight-landscape";
spotlight: SpotlightTileViewModel;
grid: GridTileViewModel[];
setVisibleTiles: (value: number) => void;
}
export interface SpotlightPortraitLayout {
type: "spotlight-portrait";
spotlight: SpotlightTileViewModel;
grid: GridTileViewModel[];
setVisibleTiles: (value: number) => void;
}
export interface SpotlightExpandedLayout {
type: "spotlight-expanded";
spotlight: SpotlightTileViewModel;
pip?: GridTileViewModel;
}
export interface OneOnOneLayout {
type: "one-on-one";
local: GridTileViewModel;
remote: GridTileViewModel;
}
export interface PipLayout {
type: "pip";
spotlight: SpotlightTileViewModel;
}
/**
* A layout defining the media tiles present on screen and their visual
* arrangement.
*/
export type Layout =
| GridLayout
| SpotlightLandscapeLayout
| SpotlightPortraitLayout
| SpotlightExpandedLayout
| OneOnOneLayout
| PipLayout;

View File

@@ -190,6 +190,7 @@ const UserMediaTile: FC<UserMediaTileProps> = ({
currentReaction={reaction ?? undefined}
raisedHandOnClick={raisedHandOnClick}
localParticipant={vm.local}
focusUrl={vm.focusURL}
audioStreamStats={audioStreamStats}
videoStreamStats={videoStreamStats}
{...props}

View File

@@ -46,6 +46,8 @@ interface Props extends ComponentProps<typeof animated.div> {
localParticipant: boolean;
audioStreamStats?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats;
videoStreamStats?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats;
// The focus url, mainly for debugging purposes
focusUrl?: string;
}
export const MediaView: FC<Props> = ({
@@ -71,6 +73,7 @@ export const MediaView: FC<Props> = ({
localParticipant,
audioStreamStats,
videoStreamStats,
focusUrl,
...props
}) => {
const { t } = useTranslation();
@@ -134,6 +137,7 @@ export const MediaView: FC<Props> = ({
<RTCConnectionStats
audio={audioStreamStats}
video={videoStreamStats}
focusUrl={focusUrl}
/>
)}
{/* TODO: Bring this back once encryption status is less broken */}

View File

@@ -78,7 +78,7 @@ const SpotlightLocalUserMediaItem: FC<SpotlightLocalUserMediaItemProps> = ({
...props
}) => {
const mirror = useBehavior(vm.mirror$);
return <MediaView mirror={mirror} {...props} />;
return <MediaView mirror={mirror} focusUrl={vm.focusURL} {...props} />;
};
SpotlightLocalUserMediaItem.displayName = "SpotlightLocalUserMediaItem";

View File

@@ -65,7 +65,7 @@ export function shouldDisambiguate(
// displayname, after hidden character removal.
return (
memberships
.map((m) => m.sender && room.getMember(m.sender))
.map((m) => m.userId && room.getMember(m.userId))
// NOTE: We *should* have a room member for everyone.
.filter((m) => !!m)
.filter((m) => m.userId !== userId)

View File

@@ -49,7 +49,7 @@ export function getBasicRTCSession(
getChildEventsForEvent: vitest.fn(),
} as Partial<RelationsContainer> as RelationsContainer,
client: {
getUserId: () => localRtcMember.sender,
getUserId: () => localRtcMember.userId,
getDeviceId: () => localRtcMember.deviceId,
getSyncState: () => SyncState.Syncing,
sendEvent: vitest.fn().mockResolvedValue({ event_id: "$fake:event" }),

View File

@@ -196,7 +196,7 @@ export function mockRtcMembership(
content: data,
});
const cms = new CallMembership(event);
const cms = new CallMembership(event, data);
vi.mocked(cms).getTransport = vi.fn().mockReturnValue(fociPreferred[0]);
return cms;
}
@@ -210,11 +210,11 @@ export function mockMatrixRoomMember(
): RoomMember {
return {
...mockEmitter(),
userId: rtcMembership.sender,
userId: rtcMembership.userId,
getMxcAvatarUrl(): string | undefined {
return undefined;
},
rawDisplayName: rtcMembership.sender,
rawDisplayName: rtcMembership.userId,
...member,
} as RoomMember;
}
@@ -274,6 +274,7 @@ export async function withLocalMedia(
kind: E2eeType.PER_PARTICIPANT,
},
mockLivekitRoom({ localParticipant }),
"https://rtc-example.org",
mediaDevices,
constant(roomMember.rawDisplayName ?? "nodisplayname"),
constant(null),
@@ -314,6 +315,7 @@ export async function withRemoteMedia(
kind: E2eeType.PER_PARTICIPANT,
},
mockLivekitRoom({}, { remoteParticipants$: of([remoteParticipant]) }),
"https://rtc-example.org",
constant(false),
constant(roomMember.rawDisplayName ?? "nodisplayname"),
constant(null),

View File

@@ -97,6 +97,9 @@ export default ({
cert: fs.readFileSync("./backend/dev_tls_m.localhost.crt"),
},
},
worker: {
format: "es",
},
build: {
minify: mode === "production" ? true : false,
sourcemap: true,

View File

@@ -10355,11 +10355,6 @@ __metadata:
uuid: "npm:13"
checksum: 10c0/6eedb93865419ca375f550c66801cd8f331833aed80ef16c49ad23b3eab648d3963571a2124d9737deb6ec909211d716949ad78d127268e16a8e2cc5b18d9fe1
languageName: node
linkType: hard
"matrix-widget-api@npm:^1.10.0, matrix-widget-api@npm:^1.13.0":
version: 1.13.1
resolution: "matrix-widget-api@npm:1.13.1"
dependencies:
"@types/events": "npm:^3.0.0"
events: "npm:^3.2.0"