Matrix/element-call-Github
1
0
Fork 0
mirror of https://github.com/vector-im/element-call.git synced 2026-02-23 05:07:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'livekit' into toger5/otel-remove-experiment

Browse Source
This commit is contained in:
Timo
2026-01-05 14:22:11 +01:00
committed by GitHub
parent 8bf6518371 49cd980ddf
commit d1aabb05ff
96 changed files with 6364 additions and 2158 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
src/UrlParams.test.ts
View File

@@ -332,6 +332,42 @@ describe("UrlParams", () => {
expect(computeUrlParams("?intent=join_existing").skipLobby).toBe(false);
});
});
describe("noiseSuppression", () => {
it("defaults to true", () => {
expect(computeUrlParams().noiseSuppression).toBe(true);
});
it("is parsed", () => {
expect(
computeUrlParams("?intent=start_call&noiseSuppression=true")
.noiseSuppression,
).toBe(true);
expect(
computeUrlParams("?intent=start_call&noiseSuppression&bar=foo")
.noiseSuppression,
).toBe(true);
expect(computeUrlParams("?noiseSuppression=false").noiseSuppression).toBe(
false,
);
});
});
describe("echoCancellation", () => {
it("defaults to true", () => {
expect(computeUrlParams().echoCancellation).toBe(true);
});
it("is parsed", () => {
expect(computeUrlParams("?echoCancellation=true").echoCancellation).toBe(
true,
);
expect(computeUrlParams("?echoCancellation=false").echoCancellation).toBe(
false,
);
});
});
describe("header", () => {
it("uses header if provided", () => {
expect(computeUrlParams("?header=app_bar&hideHeader=true").header).toBe(

13
src/UrlParams.ts
View File

@@ -233,6 +233,17 @@ export interface UrlConfiguration {
*/
waitForCallPickup: boolean;
/**
* Whether to enable echo cancellation for audio capture.
* Defaults to true.
*/
echoCancellation?: boolean;
/**
* Whether to enable noise suppression for audio capture.
* Defaults to true.
*/
noiseSuppression?: boolean;
callIntent?: RTCCallIntent;
}
interface IntentAndPlatformDerivedConfiguration {
@@ -525,6 +536,8 @@ export const computeUrlParams = (search = "", hash = ""): UrlParams => {
]),
waitForCallPickup: parser.getFlag("waitForCallPickup"),
autoLeaveWhenOthersLeft: parser.getFlag("autoLeave"),
noiseSuppression: parser.getFlagParam("noiseSuppression", true),
echoCancellation: parser.getFlagParam("echoCancellation", true),
};
// Log the final configuration for debugging purposes.

32
src/analytics/PosthogAnalytics.ts
View File

@@ -247,9 +247,8 @@ export class PosthogAnalytics {
// wins, and the first writer will send tracking with an ID that doesn't match the one on the server
// until the next time account data is refreshed and this function is called (most likely on next
// page load). This will happen pretty infrequently, so we can tolerate the possibility.
const accountDataAnalyticsId = analyticsIdGenerator();
await this.setAccountAnalyticsId(accountDataAnalyticsId);
analyticsID = await this.hashedEcAnalyticsId(accountDataAnalyticsId);
analyticsID = analyticsIdGenerator();
await this.setAccountAnalyticsId(analyticsID);
}
} catch (e) {
// The above could fail due to network requests, but not essential to starting the application,
@@ -270,37 +269,14 @@ export class PosthogAnalytics {
private async getAnalyticsId(): Promise<string | null> {
const client: MatrixClient = window.matrixclient;
let accountAnalyticsId: string | null;
if (widget) {
accountAnalyticsId = getUrlParams().posthogUserId;
return getUrlParams().posthogUserId;
} else {
const accountData = await client.getAccountDataFromServer(
PosthogAnalytics.ANALYTICS_EVENT_TYPE,
);
accountAnalyticsId = accountData?.id ?? null;
return accountData?.id ?? null;
}
if (accountAnalyticsId) {
// we dont just use the element web analytics ID because that would allow to associate
// users between the two posthog instances. By using a hash from the username and the element web analytics id
// it is not possible to conclude the element web posthog user id from the element call user id and vice versa.
return await this.hashedEcAnalyticsId(accountAnalyticsId);
}
return null;
}
private async hashedEcAnalyticsId(
accountAnalyticsId: string,
): Promise<string> {
const client: MatrixClient = window.matrixclient;
const posthogIdMaterial = "ec" + accountAnalyticsId + client.getUserId();
const bufferForPosthogId = await crypto.subtle.digest(
"sha-256",
new TextEncoder().encode(posthogIdMaterial),
);
const view = new Int32Array(bufferForPosthogId);
return Array.from(view)
.map((b) => Math.abs(b).toString(16).padStart(2, "0"))
.join("");
}
private async setAccountAnalyticsId(analyticsID: string): Promise<void> {

4
src/e2ee/sharedKeyManagement.ts
View File

@@ -34,8 +34,8 @@ const getRoomSharedKeyLocalStorageKey = (roomId: string): string =>
`room-shared-key-${roomId}`;
/**
* An upto-date shared key for the room. Either from local storage or the value from `setInitialValue`.
* @param roomId
* An up-to-date shared key for the room. Either from local storage or the value from `setInitialValue`.
* @param roomId The room ID we want the shared key for.
* @param setInitialValue The value we get from the URL. The hook will overwrite the local storage value with this.
* @returns [roomSharedKey, setRoomSharedKey] like a react useState hook.
*/

6
src/livekit/MatrixAudioRenderer.tsx
View File

@@ -166,7 +166,11 @@ interface StereoPanAudioTrackProps {
* It main purpose is to remount the AudioTrack component when switching from
* audioContext to normal audio playback.
* As of now the AudioTrack component does not support adding audio nodes while being mounted.
* @param param0
* @param props The component props
* @param props.trackRef The track reference
* @param props.muted If the track should be muted
* @param props.audioContext The audio context to use
* @param props.audioNodes The audio nodes to use
* @returns
*/
function AudioTrackWithAudioNodes({

112
src/livekit/openIDSFU.test.ts Normal file
View File

@@ -0,0 +1,112 @@
/*
Copyright 2025 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
beforeEach,
afterEach,
describe,
expect,
it,
type MockedObject,
vitest,
} from "vitest";
import fetchMock from "fetch-mock";
import { getSFUConfigWithOpenID, type OpenIDClientParts } from "./openIDSFU";
import { testJWTToken } from "../utils/test-fixtures";
const sfuUrl = "https://sfu.example.org";
describe("getSFUConfigWithOpenID", () => {
let matrixClient: MockedObject<OpenIDClientParts>;
beforeEach(() => {
matrixClient = {
getOpenIdToken: vitest.fn(),
getDeviceId: vitest.fn(),
};
});
afterEach(() => {
vitest.clearAllMocks();
fetchMock.reset();
});
it("should handle fetching a token", async () => {
fetchMock.post("https://sfu.example.org/sfu/get", () => {
return {
status: 200,
body: { url: sfuUrl, jwt: testJWTToken },
};
});
const config = await getSFUConfigWithOpenID(
matrixClient,
"https://sfu.example.org",
"!example_room_id",
);
expect(config).toEqual({
jwt: testJWTToken,
url: sfuUrl,
livekitIdentity: "@me:example.org:ABCDEF",
livekitAlias: "!example_room_id",
});
void (await fetchMock.flush());
});
it("should fail if the SFU errors", async () => {
fetchMock.post("https://sfu.example.org/sfu/get", () => {
return {
status: 500,
body: { error: "Test failure" },
};
});
try {
await getSFUConfigWithOpenID(
matrixClient,
"https://sfu.example.org",
"!example_room_id",
);
} catch (ex) {
expect(((ex as Error).cause as Error).message).toEqual(
"SFU Config fetch failed with status code 500",
);
void (await fetchMock.flush());
return;
}
expect.fail("Expected test to throw;");
});
it("should retry fetching the openid token", async () => {
let count = 0;
matrixClient.getOpenIdToken.mockImplementation(async () => {
count++;
if (count < 2) {
throw Error("Test failure");
}
return Promise.resolve({
token_type: "Bearer",
access_token: "foobar",
matrix_server_name: "example.org",
expires_in: 30,
});
});
fetchMock.post("https://sfu.example.org/sfu/get", () => {
return {
status: 200,
body: { url: sfuUrl, jwt: testJWTToken },
};
});
const config = await getSFUConfigWithOpenID(
matrixClient,
"https://sfu.example.org",
"!example_room_id",
);
expect(config).toEqual({
jwt: testJWTToken,
url: sfuUrl,
livekitIdentity: "@me:example.org:ABCDEF",
livekitAlias: "!example_room_id",
});
void (await fetchMock.flush());
});
});

60
src/livekit/openIDSFU.ts
View File

@@ -11,9 +11,47 @@ import { logger } from "matrix-js-sdk/lib/logger";
import { FailToGetOpenIdToken } from "../utils/errors";
import { doNetworkOperationWithRetry } from "../utils/matrix";
/**
* Configuration and access tokens provided by the SFU on successful authentication.
*/
export interface SFUConfig {
url: string;
jwt: string;
livekitAlias: string;
livekitIdentity: string;
}
/**
* Decoded details from the JWT.
*/
interface SFUJWTPayload {
/**
* Expiration time for the JWT.
* Note: This value is in seconds since Unix epoch.
*/
exp: number;
/**
* Name of the instance which authored the JWT
*/
iss: string;
/**
* Time at which the JWT can start to be used.
* Note: This value is in seconds since Unix epoch.
*/
nbf: number;
/**
* Subject. The Livekit alias in this context.
*/
sub: string;
/**
* The set of permissions for the user.
*/
video: {
canPublish: boolean;
canSubscribe: boolean;
room: string;
roomJoin: boolean;
};
}
// The bits we need from MatrixClient
@@ -25,9 +63,9 @@ export type OpenIDClientParts = Pick<
* Gets a bearer token from the homeserver and then use it to authenticate
* to the matrix RTC backend in order to get acces to the SFU.
* It has built-in retry for calls to the homeserver with a backoff policy.
* @param client
* @param serviceUrl
* @param matrixRoomId
* @param client The Matrix client
* @param serviceUrl The URL of the livekit SFU service
* @param matrixRoomId The Matrix room ID for which to get the SFU config
* @returns Object containing the token information
* @throws FailToGetOpenIdToken
*/
@@ -57,7 +95,17 @@ export async function getSFUConfigWithOpenID(
);
logger.info(`Got JWT from call's active focus URL.`);
return sfuConfig;
// Pull the details from the JWT
const [, payloadStr] = sfuConfig.jwt.split(".");
// TODO: Prefer Uint8Array.fromBase64 when widely available
const payload = JSON.parse(global.atob(payloadStr)) as SFUJWTPayload;
return {
jwt: sfuConfig.jwt,
url: sfuConfig.url,
livekitAlias: payload.video.room,
// NOTE: Currently unused.
livekitIdentity: payload.sub,
};
}
async function getLiveKitJWT(
@@ -65,7 +113,7 @@ async function getLiveKitJWT(
livekitServiceURL: string,
roomName: string,
openIDToken: IOpenIDToken,
): Promise<SFUConfig> {
): Promise<{ url: string; jwt: string }> {
try {
const res = await fetch(livekitServiceURL + "/sfu/get", {
method: "POST",
@@ -83,6 +131,6 @@ async function getLiveKitJWT(
}
return await res.json();
} catch (e) {
throw new Error("SFU Config fetch failed with exception " + e);
throw new Error("SFU Config fetch failed with exception", { cause: e });
}
}

9
src/reactions/RaisedHandIndicator.test.tsx
View File

@@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { describe, expect, test } from "vitest";
import { beforeEach, describe, expect, test, vi } from "vitest";
import { render, configure } from "@testing-library/react";
import { RaisedHandIndicator } from "./RaisedHandIndicator";
@@ -15,6 +15,13 @@ configure({
});
describe("RaisedHandIndicator", () => {
const fixedTime = new Date("2025-01-01T12:00:00.000Z");
beforeEach(() => {
vi.useFakeTimers();
vi.setSystemTime(fixedTime);
});
test("renders nothing when no hand has been raised", () => {
const { container } = render(<RaisedHandIndicator />);
expect(container.firstChild).toBeNull();

6
src/reactions/ReactionsReader.ts
View File

@@ -135,10 +135,10 @@ export class ReactionsReader {
}
/**
* Fetchest any hand wave reactions by the given sender on the given
* Fetches any hand wave reactions by the given sender on the given
* membership event.
* @param membershipEventId
* @param expectedSender
* @param membershipEventId - The user membership event id.
* @param expectedSender - The expected sender of the reaction.
* @returns A MatrixEvent if one was found.
*/
private getLastReactionEvent(

6
src/reactions/__snapshots__/RaisedHandIndicator.test.tsx.snap
View File

@@ -15,7 +15,7 @@ exports[`RaisedHandIndicator > renders a smaller indicator when miniature is spe
</span>
</div>
<p>
00:01
00:00
</p>
</div>
`;
@@ -35,7 +35,7 @@ exports[`RaisedHandIndicator > renders an indicator when a hand has been raised
</span>
</div>
<p>
00:01
00:00
</p>
</div>
`;
@@ -55,7 +55,7 @@ exports[`RaisedHandIndicator > renders an indicator when a hand has been raised
</span>
</div>
<p>
01:01
01:00
</p>
</div>
`;

25
src/room/EarpieceOverlay.module.css
View File

@@ -7,34 +7,17 @@
align-items: center;
justify-content: center;
gap: var(--cpd-space-2x);
}
@keyframes fade-in {
from {
opacity: 0;
}
to {
opacity: 1;
}
transition: opacity 200ms;
}
.overlay[data-show="true"] {
animation: fade-in 200ms;
}
@keyframes fade-out {
from {
opacity: 1;
}
to {
opacity: 0;
display: none;
}
opacity: 1;
}
.overlay[data-show="false"] {
animation: fade-out 130ms forwards;
opacity: 0;
pointer-events: none;
transition-duration: 130ms;
}
.overlay::before {

2
src/room/GroupCallView.tsx
View File

@@ -160,6 +160,7 @@ export const GroupCallView: FC<Props> = ({
}, [rtcSession]);
// TODO move this into the callViewModel LocalMembership.ts
// We might actually not need this at all. Since we get into fatalError on those errors already?
useTypedEventEmitter(
rtcSession,
MatrixRTCSessionEvent.MembershipManagerError,
@@ -313,6 +314,7 @@ export const GroupCallView: FC<Props> = ({
const navigate = useNavigate();
// TODO split this into leave and onDisconnect
const onLeft = useCallback(
(
reason: "timeout" | "user" | "allOthersLeft" | "decline" | "error",

18
src/room/InCallView.tsx
View File

@@ -24,7 +24,7 @@ import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc";
import classNames from "classnames";
import { BehaviorSubject, map } from "rxjs";
import { useObservable } from "observable-hooks";
import { logger } from "matrix-js-sdk/lib/logger";
import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
import {
VoiceCallSolidIcon,
VolumeOnSolidIcon,
@@ -87,6 +87,7 @@ import { ReactionsOverlay } from "./ReactionsOverlay";
import { CallEventAudioRenderer } from "./CallEventAudioRenderer";
import {
debugTileLayout as debugTileLayoutSetting,
matrixRTCMode as matrixRTCModeSetting,
useSetting,
} from "../settings/settings";
import { ReactionsReader } from "../reactions/ReactionsReader";
@@ -108,6 +109,8 @@ import { useTrackProcessorObservable$ } from "../livekit/TrackProcessorContext.t
import { type Layout } from "../state/layout-types.ts";
import { ObservableScope } from "../state/ObservableScope.ts";
const logger = rootLogger.getChild("[InCallView]");
const maxTapDurationMs = 400;
export interface ActiveCallProps
@@ -126,6 +129,7 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
const mediaDevices = useMediaDevices();
const trackProcessorState$ = useTrackProcessorObservable$();
useEffect(() => {
logger.info("START CALL VIEW SCOPE");
const scope = new ObservableScope();
const reactionsReader = new ReactionsReader(scope, props.rtcSession);
const { autoLeaveWhenOthersLeft, waitForCallPickup, sendNotificationType } =
@@ -140,15 +144,20 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
encryptionSystem: props.e2eeSystem,
autoLeaveWhenOthersLeft,
waitForCallPickup: waitForCallPickup && sendNotificationType === "ring",
matrixRTCMode$: matrixRTCModeSetting.value$,
},
reactionsReader.raisedHands$,
reactionsReader.reactions$,
scope.behavior(trackProcessorState$),
);
// TODO move this somewhere else once we use the callViewModel in the lobby as well!
vm.join();
setVm(vm);
vm.leave$.pipe(scope.bind()).subscribe(props.onLeft);
return (): void => {
logger.info("END CALL VIEW SCOPE");
scope.end();
};
}, [
@@ -249,7 +258,7 @@ export const InCallView: FC<InCallViewProps> = ({
() => void toggleRaisedHand(),
);
const audioParticipants = useBehavior(vm.audioParticipants$);
const audioParticipants = useBehavior(vm.livekitRoomItems$);
const participantCount = useBehavior(vm.participantCount$);
const reconnecting = useBehavior(vm.reconnecting$);
const windowMode = useBehavior(vm.windowMode$);
@@ -266,7 +275,10 @@ export const InCallView: FC<InCallViewProps> = ({
const ringOverlay = useBehavior(vm.ringOverlay$);
const fatalCallError = useBehavior(vm.fatalError$);
// Stop the rendering and throw for the error boundary
if (fatalCallError) throw fatalCallError;
if (fatalCallError) {
logger.debug("fatalCallError stop rendering", fatalCallError);
throw fatalCallError;
}
// We need to set the proper timings on the animation based upon the sound length.
const ringDuration = pickupPhaseAudio?.soundDuration["waiting"] ?? 1;

4
src/room/LobbyView.tsx
View File

@@ -79,9 +79,9 @@ export const LobbyView: FC<Props> = ({
waitingForInvite,
}) => {
useEffect(() => {
logger.info("[Lifecycle] GroupCallView Component mounted");
logger.info("[Lifecycle] LobbyView Component mounted");
return (): void => {
logger.info("[Lifecycle] GroupCallView Component unmounted");
logger.info("[Lifecycle] LobbyView Component unmounted");
};
}, []);

16
src/room/useLoadGroupCall.ts
View File

@@ -106,22 +106,18 @@ async function joinRoomAfterInvite(
export class CallTerminatedMessage extends Error {
/**
* Creates a new CallTerminatedMessage.
*
* @param icon The icon to display with the message
* @param messageTitle The title of the call ended screen message (translated)
* @param messageBody The message explaining the kind of termination
* (kick, ban, knock reject, etc.) (translated)
* @param reason The user-provided reason for the termination (kick/ban)
*/
public constructor(
/**
* The icon to display with the message.
*/
public readonly icon: ComponentType<SVGAttributes<SVGElement>>,
messageTitle: string,
/**
* The message explaining the kind of termination (kick, ban, knock reject,
* etc.) (translated)
*/
public readonly messageBody: string,
/**
* The user-provided reason for the termination (kick/ban)
*/
public readonly reason?: string,
) {
super(messageTitle);

26
src/settings/rageshake.ts
View File

@@ -99,7 +99,7 @@ class ConsoleLogger extends EventEmitter {
/**
* Returns the log lines to flush to disk and empties the internal log buffer
* @return {string} \n delimited log lines
* @return \n delimited log lines
*/
public popLogs(): string {
const logsToFlush = this.logs;
@@ -109,7 +109,7 @@ class ConsoleLogger extends EventEmitter {
/**
* Returns lines currently in the log buffer without removing them
* @return {string} \n delimited log lines
* @return \n delimited log lines
*/
public peekLogs(): string {
return this.logs;
@@ -139,7 +139,7 @@ class IndexedDBLogStore {
}
/**
* @return {Promise} Resolves when the store is ready.
* @return Resolves when the store is ready.
*/
public async connect(): Promise<void> {
const req = this.indexedDB.open("logs");
@@ -219,7 +219,7 @@ class IndexedDBLogStore {
* This guarantees that we will always eventually do a flush when flush() is
* called.
*
* @return {Promise} Resolved when the logs have been flushed.
* @return Resolved when the logs have been flushed.
*/
public flush = async (): Promise<void> => {
// check if a flush() operation is ongoing
@@ -270,7 +270,7 @@ class IndexedDBLogStore {
* returned are deleted at the same time, so this can be called at startup
* to do house-keeping to keep the logs from growing too large.
*
* @return {Promise<Object[]>} Resolves to an array of objects. The array is
* @return Resolves to an array of objects. The array is
* sorted in time (oldest first) based on when the log file was created (the
* log ID). The objects have said log ID in an "id" field and "lines" which
* is a big string with all the new-line delimited logs.
@@ -421,12 +421,12 @@ class IndexedDBLogStore {
/**
* Helper method to collect results from a Cursor and promiseify it.
* @param {ObjectStore|Index} store The store to perform openCursor on.
* @param {IDBKeyRange=} keyRange Optional key range to apply on the cursor.
* @param {Function} resultMapper A function which is repeatedly called with a
* @param store - The store to perform openCursor on.
* @param keyRange - Optional key range to apply on the cursor.
* @param resultMapper - A function which is repeatedly called with a
* Cursor.
* Return the data you want to keep.
* @return {Promise<T[]>} Resolves to an array of whatever you returned from
* @return Resolves to an array of whatever you returned from
* resultMapper.
*/
async function selectQuery<T>(
@@ -464,9 +464,7 @@ declare global {
/**
* Configure rage shaking support for sending bug reports.
* Modifies globals.
* @param {boolean} setUpPersistence When true (default), the persistence will
* be set up immediately for the logs.
* @return {Promise} Resolves when set up.
* @return Resolves when set up.
*/
export async function init(): Promise<void> {
global.mx_rage_logger = new ConsoleLogger();
@@ -503,7 +501,7 @@ export async function init(): Promise<void> {
/**
* Try to start up the rageshake storage for logs. If not possible (client unsupported)
* then this no-ops.
* @return {Promise} Resolves when complete.
* @return Resolves when complete.
*/
async function tryInitStorage(): Promise<void> {
if (global.mx_rage_initStoragePromise) {
@@ -536,7 +534,7 @@ async function tryInitStorage(): Promise<void> {
/**
* Get a recent snapshot of the logs, ready for attaching to a bug report
*
* @return {LogEntry[]} list of log data
* @return list of log data
*/
export async function getLogsForReport(): Promise<LogEntry[]> {
if (!global.mx_rage_logger) {

2
src/state/CallViewModel/CallNotificationLifecycle.ts
View File

@@ -81,7 +81,7 @@ export interface Props {
localUser: { deviceId: string; userId: string };
}
/**
* @returns {callPickupState$, autoLeave$}
* @returns two observables:
* `callPickupState$` The current call pickup state of the call.
* - "unknown": The client has not yet sent the notification event. We don't know if it will because it first needs to send its own membership.
* Then we can conclude if we were the first one to join or not.

59
src/state/CallViewModel/CallViewModel.test.ts
View File

@@ -60,7 +60,8 @@ import {
import { MediaDevices } from "../MediaDevices.ts";
import { getValue } from "../../utils/observable.ts";
import { type Behavior, constant } from "../Behavior.ts";
import { withCallViewModel } from "./CallViewModelTestUtils.ts";
import { withCallViewModel as withCallViewModelInMode } from "./CallViewModelTestUtils.ts";
import { MatrixRTCMode } from "../../settings/settings.ts";
vi.mock("rxjs", async (importOriginal) => ({
...(await importOriginal()),
@@ -229,7 +230,13 @@ function mockRingEvent(
// need a value to fill in for them when emitting notifications
const mockLegacyRingEvent = {} as { event_id: string } & ICallNotifyContent;
describe("CallViewModel", () => {
describe.each([
[MatrixRTCMode.Legacy],
[MatrixRTCMode.Compatibil],
[MatrixRTCMode.Matrix_2_0],
])("CallViewModel (%s mode)", (mode) => {
const withCallViewModel = withCallViewModelInMode(mode);
test("participants are retained during a focus switch", () => {
withTestScheduler(({ behavior, expectObservable }) => {
// Participants disappear on frame 2 and come back on frame 3
@@ -267,7 +274,7 @@ describe("CallViewModel", () => {
});
});
it.skip("screen sharing activates spotlight layout", () => {
test("screen sharing activates spotlight layout", () => {
withTestScheduler(({ behavior, schedule, expectObservable }) => {
// Start with no screen shares, then have Alice and Bob share their screens,
// then return to no screen shares, then have just Alice share for a bit
@@ -502,6 +509,48 @@ describe("CallViewModel", () => {
});
});
test("layout reacts to window size", () => {
withTestScheduler(({ behavior, schedule, expectObservable }) => {
const windowSizeInputMarbles = "abc";
const expectedLayoutMarbles = " abc";
withCallViewModel(
{
remoteParticipants$: constant([aliceParticipant]),
rtcMembers$: constant([localRtcMember, aliceRtcMember]),
windowSize$: behavior(windowSizeInputMarbles, {
a: { width: 300, height: 600 }, // Start very narrow, like a phone
b: { width: 1000, height: 800 }, // Go to normal desktop window size
c: { width: 200, height: 180 }, // Go to PiP size
}),
},
(vm) => {
expectObservable(summarizeLayout$(vm.layout$)).toBe(
expectedLayoutMarbles,
{
a: {
// This is the expected one-on-one layout for a narrow window
type: "spotlight-expanded",
spotlight: [`${aliceId}:0`],
pip: `${localId}:0`,
},
b: {
// In a larger window, expect the normal one-on-one layout
type: "one-on-one",
local: `${localId}:0`,
remote: `${aliceId}:0`,
},
c: {
// In a PiP-sized window, we of course expect a PiP layout
type: "pip",
spotlight: [`${aliceId}:0`],
},
},
);
},
);
});
});
test("spotlight speakers swap places", () => {
withTestScheduler(({ behavior, schedule, expectObservable }) => {
// Go immediately into spotlight mode for the test
@@ -1207,7 +1256,9 @@ describe("CallViewModel", () => {
rtcSession.membershipStatus = Status.Connected;
},
n: () => {
rtcSession.membershipStatus = Status.Reconnecting;
// NOTE: This was removed in https://github.com/matrix-org/matrix-js-sdk/pull/5103 accidentally.
// eslint-disable-next-line @typescript-eslint/no-explicit-any
rtcSession.membershipStatus = "Reconnecting" as any;
},
});
schedule(probablyLeftMarbles, {

299
src/state/CallViewModel/CallViewModel.ts
View File

@@ -15,9 +15,9 @@ import {
} from "livekit-client";
import { type Room as MatrixRoom } from "matrix-js-sdk";
import {
catchError,
combineLatest,
distinctUntilChanged,
EMPTY,
filter,
fromEvent,
map,
@@ -28,7 +28,6 @@ import {
pairwise,
race,
scan,
skip,
skipWhile,
startWith,
Subject,
@@ -54,11 +53,15 @@ import {
ScreenShareViewModel,
type UserMediaViewModel,
} from "../MediaViewModel";
import { accumulate, generateItems, pauseWhen } from "../../utils/observable";
import {
accumulate,
filterBehavior,
generateItems,
pauseWhen,
} from "../../utils/observable";
import {
duplicateTiles,
MatrixRTCMode,
matrixRTCMode,
playReactionsSound,
showReactions,
} from "../../settings/settings";
@@ -77,7 +80,7 @@ import {
} from "../../reactions";
import { shallowEquals } from "../../utils/array";
import { type MediaDevices } from "../MediaDevices";
import { type Behavior } from "../Behavior";
import { constant, type Behavior } from "../Behavior";
import { E2eeType } from "../../e2ee/e2eeType";
import { MatrixKeyProvider } from "../../e2ee/matrixKeyProvider";
import { type MuteStates } from "../MuteStates";
@@ -95,15 +98,14 @@ import {
type SpotlightLandscapeLayoutMedia,
type SpotlightPortraitLayoutMedia,
} from "../layout-types.ts";
import { type ElementCallError } from "../../utils/errors.ts";
import { ElementCallError } from "../../utils/errors.ts";
import { type ObservableScope } from "../ObservableScope.ts";
import { createHomeserverConnected$ } from "./localMember/HomeserverConnected.ts";
import {
createLocalMembership$,
enterRTCSession,
LivekitState,
type LocalMemberConnectionState,
} from "./localMember/LocalMembership.ts";
TransportState,
} from "./localMember/LocalMember.ts";
import { createLocalTransport$ } from "./localMember/LocalTransport.ts";
import {
createMemberships$,
@@ -113,7 +115,9 @@ import { ECConnectionFactory } from "./remoteMembers/ConnectionFactory.ts";
import { createConnectionManager$ } from "./remoteMembers/ConnectionManager.ts";
import {
createMatrixLivekitMembers$,
type MatrixLivekitMember,
type TaggedParticipant,
type LocalMatrixLivekitMember,
type RemoteMatrixLivekitMember,
} from "./remoteMembers/MatrixLivekitMembers.ts";
import {
type AutoLeaveReason,
@@ -128,6 +132,7 @@ import {
} from "./remoteMembers/MatrixMemberMetadata.ts";
import { Publisher } from "./localMember/Publisher.ts";
import { type Connection } from "./remoteMembers/Connection.ts";
import { createLayoutModeSwitch } from "./LayoutSwitch.ts";
const logger = rootLogger.getChild("[CallViewModel]");
//TODO
@@ -149,6 +154,10 @@ export interface CallViewModelOptions {
livekitRoomFactory?: (options?: RoomOptions) => LivekitRoom;
/** Optional behavior overriding the local connection state, mainly for testing purposes. */
connectionState$?: Behavior<ConnectionState>;
/** Optional behavior overriding the computed window size, mainly for testing purposes. */
windowSize$?: Behavior<{ width: number; height: number }>;
/** The version & compatibility mode of MatrixRTC that we should use. */
matrixRTCMode$?: Behavior<MatrixRTCMode>;
}
// Do not play any sounds if the participant count has exceeded this
@@ -174,7 +183,7 @@ interface LayoutScanState {
}
type MediaItem = UserMedia | ScreenShare;
type AudioLivekitItem = {
export type LivekitRoomItem = {
livekitRoom: LivekitRoom;
participants: string[];
url: string;
@@ -197,12 +206,15 @@ export interface CallViewModel {
callPickupState$: Behavior<
"unknown" | "ringing" | "timeout" | "decline" | "success" | null
>;
/** Observable that emits when the user should leave the call (hangup pressed, widget action, error).
* THIS DOES NOT LEAVE THE CALL YET. The only way to leave the call (send the hangup event) is by ending the scope.
*/
leave$: Observable<"user" | AutoLeaveReason>;
/** Call to initiate hangup. Use in conbination with connectino state track the async hangup process. */
/** Call to initiate hangup. Use in conbination with reconnectino state track the async hangup process. */
hangup: () => void;
// joining
join: () => LocalMemberConnectionState;
join: () => void;
// screen sharing
/**
@@ -250,7 +262,11 @@ export interface CallViewModel {
*/
participantCount$: Behavior<number>;
/** Participants sorted by livekit room so they can be used in the audio rendering */
audioParticipants$: Behavior<AudioLivekitItem[]>;
livekitRoomItems$: Behavior<LivekitRoomItem[]>;
userMedia$: Behavior<UserMedia[]>;
/** use the layout instead, this is just for the sdk export. */
matrixLivekitMembers$: Behavior<RemoteMatrixLivekitMember[]>;
localMatrixLivekitMember$: Behavior<LocalMatrixLivekitMember | null>;
/** List of participants raising their hand */
handsRaised$: Behavior<Record<string, RaisedHandInfo>>;
/** List of reactions. Keys are: membership.membershipId (currently predefined as: `${membershipEvent.userId}:${membershipEvent.deviceId}`)*/
@@ -333,18 +349,17 @@ export interface CallViewModel {
switch: () => void;
} | null>;
// connection state
/**
* Whether various media/event sources should pretend to be disconnected from
* all network input, even if their connection still technically works.
* Whether the app is currently reconnecting to the LiveKit server and/or setting the matrix rtc room state.
*/
// We do this when the app is in the 'reconnecting' state, because it might be
// that the LiveKit connection is still functional while the homeserver is
// down, for example, and we want to avoid making people worry that the app is
// in a split-brained state.
// DISCUSSION own membership manager ALSO this probably can be simplifis
reconnecting$: Behavior<boolean>;
/**
* Shortcut for not requireing to parse and combine connectionState.matrix and connectionState.livekit
*/
connected$: Behavior<boolean>;
}
/**
* A view model providing all the application logic needed to show the in-call
* UI (may eventually be expanded to cover the lobby and feedback screens in the
@@ -372,6 +387,8 @@ export function createCallViewModel$(
options.encryptionSystem,
matrixRTCSession,
);
const matrixRTCMode$ =
options.matrixRTCMode$ ?? constant(MatrixRTCMode.Legacy);
// Each hbar seperates a block of input variables required for the CallViewModel to function.
// The outputs of this block is written under the hbar.
@@ -404,7 +421,7 @@ export function createCallViewModel$(
client,
roomId: matrixRoom.roomId,
useOldestMember$: scope.behavior(
matrixRTCMode.value$.pipe(map((v) => v === MatrixRTCMode.Legacy)),
matrixRTCMode$.pipe(map((v) => v === MatrixRTCMode.Legacy)),
),
});
@@ -415,6 +432,8 @@ export function createCallViewModel$(
livekitKeyProvider,
getUrlParams().controlledAudioDevices,
options.livekitRoomFactory,
getUrlParams().echoCancellation,
getUrlParams().noiseSuppression,
);
const connectionManager = createConnectionManager$({
@@ -422,7 +441,18 @@ export function createCallViewModel$(
connectionFactory: connectionFactory,
inputTransports$: scope.behavior(
combineLatest(
[localTransport$, membershipsAndTransports.transports$],
[
localTransport$.pipe(
catchError((e: unknown) => {
logger.info(
"dont pass local transport to createConnectionManager$. localTransport$ threw an error",
e,
);
return of(null);
}),
),
membershipsAndTransports.transports$,
],
(localTransport, transports) => {
const localTransportAsArray = localTransport ? [localTransport] : [];
return transports.mapInner((transports) => [
@@ -432,7 +462,7 @@ export function createCallViewModel$(
},
),
),
logger: logger,
logger,
});
const matrixLivekitMembers$ = createMatrixLivekitMembers$({
@@ -443,7 +473,7 @@ export function createCallViewModel$(
});
const connectOptions$ = scope.behavior(
matrixRTCMode.value$.pipe(
matrixRTCMode$.pipe(
map((mode) => ({
encryptMedia: livekitKeyProvider !== undefined,
// TODO. This might need to get called again on each change of matrixRTCMode...
@@ -454,13 +484,13 @@ export function createCallViewModel$(
const localMembership = createLocalMembership$({
scope: scope,
homeserverConnected$: createHomeserverConnected$(
homeserverConnected: createHomeserverConnected$(
scope,
client,
matrixRTCSession,
),
muteStates: muteStates,
joinMatrixRTC: async (transport: LivekitTransport) => {
joinMatrixRTC: (transport: LivekitTransport) => {
return enterRTCSession(
matrixRTCSession,
transport,
@@ -474,6 +504,9 @@ export function createCallViewModel$(
mediaDevices,
muteStates,
trackProcessorState$,
logger.getChild(
"[Publisher " + connection.transport.livekit_service_url + "]",
),
);
},
connectionManager: connectionManager,
@@ -494,22 +527,21 @@ export function createCallViewModel$(
),
);
const localMatrixLivekitMemberUninitialized = {
membership$: localRtcMembership$,
participant$: localMembership.participant$,
connection$: localMembership.connection$,
userId: userId,
};
const localMatrixLivekitMember$: Behavior<MatrixLivekitMember | null> =
const localMatrixLivekitMember$: Behavior<LocalMatrixLivekitMember | null> =
scope.behavior(
localRtcMembership$.pipe(
switchMap((membership) => {
if (!membership) return of(null);
return of(
// casting is save here since we know that localRtcMembership$ is !== null since we reached this case.
localMatrixLivekitMemberUninitialized as MatrixLivekitMember,
);
filterBehavior((membership) => membership !== null),
map((membership$) => {
if (membership$ === null) return null;
return {
membership$,
participant: {
type: "local" as const,
value$: localMembership.participant$,
},
connection$: localMembership.connection$,
userId,
};
}),
),
);
@@ -572,34 +604,16 @@ export function createCallViewModel$(
),
);
// CODESMELL?
// This is functionally the same Observable as leave$, except here it's
// hoisted to the top of the class. This enables the cyclic dependency between
// leave$ -> autoLeave$ -> callPickupState$ -> livekitConnectionState$ ->
// localConnection$ -> transports$ -> joined$ -> leave$.
const leaveHoisted$ = new Subject<
"user" | "timeout" | "decline" | "allOthersLeft"
>();
/**
* Whether various media/event sources should pretend to be disconnected from
* all network input, even if their connection still technically works.
*/
// We do this when the app is in the 'reconnecting' state, because it might be
// that the LiveKit connection is still functional while the homeserver is
// down, for example, and we want to avoid making people worry that the app is
// in a split-brained state.
// DISCUSSION own membership manager ALSO this probably can be simplifis
const reconnecting$ = localMembership.reconnecting$;
const pretendToBeDisconnected$ = reconnecting$;
const audioParticipants$ = scope.behavior(
const livekitRoomItems$ = scope.behavior(
matrixLivekitMembers$.pipe(
tap((val) => {
logger.debug("matrixLivekitMembers$ updated", val.value);
}),
switchMap((membersWithEpoch) => {
const members = membersWithEpoch.value;
const a$ = combineLatest(
members.map((member) =>
combineLatest([member.connection$, member.participant$]).pipe(
combineLatest([member.connection$, member.participant.value$]).pipe(
map(([connection, participant]) => {
// do not render audio for local participant
if (!connection || !participant || participant.isLocal)
@@ -619,7 +633,7 @@ export function createCallViewModel$(
return a$;
}),
map((members) =>
members.reduce<AudioLivekitItem[]>((acc, curr) => {
members.reduce<LivekitRoomItem[]>((acc, curr) => {
if (!curr) return acc;
const existing = acc.find((item) => item.url === curr.url);
@@ -640,7 +654,7 @@ export function createCallViewModel$(
);
const handsRaised$ = scope.behavior(
handsRaisedSubject$.pipe(pauseWhen(pretendToBeDisconnected$)),
handsRaisedSubject$.pipe(pauseWhen(localMembership.reconnecting$)),
);
const reactions$ = scope.behavior(
@@ -653,7 +667,7 @@ export function createCallViewModel$(
]),
),
),
pauseWhen(pretendToBeDisconnected$),
pauseWhen(localMembership.reconnecting$),
),
);
@@ -674,10 +688,10 @@ export function createCallViewModel$(
{ value: matrixLivekitMembers },
duplicateTiles,
]) {
let localParticipantId = undefined;
let localParticipantId: string | undefined = undefined;
// add local member if available
if (localMatrixLivekitMember) {
const { userId, participant$, connection$, membership$ } =
const { userId, participant, connection$, membership$ } =
localMatrixLivekitMember;
localParticipantId = `${userId}:${membership$.value.deviceId}`; // should be membership$.value.membershipID which is not optional
// const participantId = membership$.value.membershipID;
@@ -688,7 +702,7 @@ export function createCallViewModel$(
dup,
localParticipantId,
userId,
participant$,
participant satisfies TaggedParticipant as TaggedParticipant, // Widen the type safely
connection$,
],
data: undefined,
@@ -699,7 +713,7 @@ export function createCallViewModel$(
// add remote members that are available
for (const {
userId,
participant$,
participant,
connection$,
membership$,
} of matrixLivekitMembers) {
@@ -708,7 +722,7 @@ export function createCallViewModel$(
// const participantId = membership$.value?.identity;
for (let dup = 0; dup < 1 + duplicateTiles; dup++) {
yield {
keys: [dup, participantId, userId, participant$, connection$],
keys: [dup, participantId, userId, participant, connection$],
data: undefined,
};
}
@@ -720,7 +734,7 @@ export function createCallViewModel$(
dup,
participantId,
userId,
participant$,
participant,
connection$,
) => {
const livekitRoom$ = scope.behavior(
@@ -739,12 +753,12 @@ export function createCallViewModel$(
scope,
`${participantId}:${dup}`,
userId,
participant$,
participant,
options.encryptionSystem,
livekitRoom$,
focusUrl$,
mediaDevices,
pretendToBeDisconnected$,
localMembership.reconnecting$,
displayName$,
matrixMemberMetadataStore.createAvatarUrlBehavior$(userId),
handsRaised$.pipe(map((v) => v[participantId]?.time ?? null)),
@@ -840,10 +854,7 @@ export function createCallViewModel$(
merge(
autoLeave$,
merge(userHangup$, widgetHangup$).pipe(map(() => "user" as const)),
).pipe(
scope.share,
tap((reason) => leaveHoisted$.next(reason)),
);
).pipe(scope.share);
const spotlightSpeaker$ = scope.behavior<UserMediaViewModel | null>(
userMedia$.pipe(
@@ -952,20 +963,29 @@ export function createCallViewModel$(
),
);
const hasRemoteScreenShares$: Observable<boolean> = spotlight$.pipe(
map((spotlight) =>
spotlight.some((vm) => !vm.local && vm instanceof ScreenShareViewModel),
const hasRemoteScreenShares$ = scope.behavior<boolean>(
spotlight$.pipe(
map((spotlight) =>
spotlight.some((vm) => !vm.local && vm instanceof ScreenShareViewModel),
),
),
distinctUntilChanged(),
);
const pipEnabled$ = scope.behavior(setPipEnabled$, false);
const windowSize$ =
options.windowSize$ ??
scope.behavior<{ width: number; height: number }>(
fromEvent(window, "resize").pipe(
startWith(null),
map(() => ({ width: window.innerWidth, height: window.innerHeight })),
),
);
// A guess at what the window's mode should be based on its size and shape.
const naturalWindowMode$ = scope.behavior<WindowMode>(
fromEvent(window, "resize").pipe(
map(() => {
const height = window.innerHeight;
const width = window.innerWidth;
windowSize$.pipe(
map(({ width, height }) => {
if (height <= 400 && width <= 340) return "pip";
// Our layouts for flat windows are better at adapting to a small width
// than our layouts for narrow windows are at adapting to a small height,
@@ -975,7 +995,6 @@ export function createCallViewModel$(
return "normal";
}),
),
"normal",
);
/**
@@ -992,36 +1011,11 @@ export function createCallViewModel$(
spotlightExpandedToggle$.pipe(accumulate(false, (expanded) => !expanded)),
);
const gridModeUserSelection$ = new Subject<GridMode>();
/**
* The layout mode of the media tile grid.
*/
const gridMode$ =
// If the user hasn't selected spotlight and somebody starts screen sharing,
// automatically switch to spotlight mode and reset when screen sharing ends
scope.behavior<GridMode>(
gridModeUserSelection$.pipe(
switchMap((userSelection) =>
(userSelection === "spotlight"
? EMPTY
: combineLatest([hasRemoteScreenShares$, windowMode$]).pipe(
skip(userSelection === null ? 0 : 1),
map(
([hasScreenShares, windowMode]): GridMode =>
hasScreenShares || windowMode === "flat"
? "spotlight"
: "grid",
),
)
).pipe(startWith(userSelection ?? "grid")),
),
),
"grid",
);
const setGridMode = (value: GridMode): void => {
gridModeUserSelection$.next(value);
};
const { setGridMode, gridMode$ } = createLayoutModeSwitch(
scope,
windowMode$,
hasRemoteScreenShares$,
);
const gridLayoutMedia$: Observable<GridLayoutMedia> = combineLatest(
[grid$, spotlight$],
@@ -1448,16 +1442,44 @@ export function createCallViewModel$(
// reassigned here to make it publicly accessible
const toggleScreenSharing = localMembership.toggleScreenSharing;
const join = localMembership.requestConnect;
// TODO-MULTI-SFU: Use this view model for the lobby as well, and only call this once 'join' is clicked?
join();
const errors$ = scope.behavior<{
transportError?: ElementCallError;
matrixError?: ElementCallError;
connectionError?: ElementCallError;
publishError?: ElementCallError;
} | null>(
localMembership.localMemberState$.pipe(
map((value) => {
const returnObject: {
transportError?: ElementCallError;
matrixError?: ElementCallError;
connectionError?: ElementCallError;
publishError?: ElementCallError;
} = {};
if (value instanceof ElementCallError) return { transportError: value };
if (value === TransportState.Waiting) return null;
if (value.matrix instanceof ElementCallError)
returnObject.matrixError = value.matrix;
if (value.media instanceof ElementCallError)
returnObject.publishError = value.media;
else if (
typeof value.media === "object" &&
value.media.connection instanceof ElementCallError
)
returnObject.connectionError = value.media.connection;
return returnObject;
}),
),
null,
);
return {
autoLeave$: autoLeave$,
callPickupState$: callPickupState$,
ringOverlay$: ringOverlay$,
leave$: leave$,
hangup: (): void => userHangup$.next(),
join: join,
join: localMembership.requestJoinAndPublish,
toggleScreenSharing: toggleScreenSharing,
sharingScreen$: sharingScreen$,
@@ -1467,16 +1489,21 @@ export function createCallViewModel$(
unhoverScreen: (): void => screenUnhover$.next(),
fatalError$: scope.behavior(
localMembership.connectionState.livekit$.pipe(
filter((v) => v.state === LivekitState.Error),
map((s) => s.error),
errors$.pipe(
map((errors) => {
logger.debug("errors$ to compute any fatal errors:", errors);
return (
errors?.transportError ??
errors?.matrixError ??
errors?.connectionError ??
null
);
}),
filter((error) => error !== null),
),
null,
),
participantCount$: participantCount$,
audioParticipants$: audioParticipants$,
handsRaised$: handsRaised$,
reactions$: reactions$,
joinSoundEffect$: joinSoundEffect$,
@@ -1495,6 +1522,16 @@ export function createCallViewModel$(
spotlight$: spotlight$,
pip$: pip$,
layout$: layout$,
userMedia$,
localMatrixLivekitMember$,
matrixLivekitMembers$: scope.behavior(
matrixLivekitMembers$.pipe(
map((members) => members.value),
tap((v) => {
logger.debug("matrixLivekitMembers$ updated (exported)", v);
}),
),
),
tileStoreGeneration$: tileStoreGeneration$,
showSpotlightIndicators$: showSpotlightIndicators$,
showSpeakingIndicators$: showSpeakingIndicators$,
@@ -1502,7 +1539,9 @@ export function createCallViewModel$(
showFooter$: showFooter$,
earpieceMode$: earpieceMode$,
audioOutputSwitcher$: audioOutputSwitcher$,
reconnecting$: reconnecting$,
reconnecting$: localMembership.reconnecting$,
livekitRoomItems$,
connected$: localMembership.connected$,
};
}

222
src/state/CallViewModel/CallViewModelTestUtils.ts
View File

@@ -53,6 +53,7 @@ import {
import { type Behavior, constant } from "../Behavior";
import { type ProcessorState } from "../../livekit/TrackProcessorContext";
import { type MediaDevices } from "../MediaDevices";
import { type MatrixRTCMode } from "../../settings/settings";
mockConfig({
livekit: { livekit_service_url: "http://my-default-service-url.com" },
@@ -75,119 +76,130 @@ export interface CallViewModelInputs {
speaking: Map<Participant, Observable<boolean>>;
mediaDevices: MediaDevices;
initialSyncState: SyncState;
windowSize$: Behavior<{ width: number; height: number }>;
}
const localParticipant = mockLocalParticipant({ identity: "" });
export function withCallViewModel(
{
remoteParticipants$ = constant([]),
rtcMembers$ = constant([localRtcMember]),
livekitConnectionState$: connectionState$ = constant(
ConnectionState.Connected,
),
speaking = new Map(),
mediaDevices = mockMediaDevices({}),
initialSyncState = SyncState.Syncing,
}: Partial<CallViewModelInputs> = {},
continuation: (
vm: CallViewModel,
rtcSession: MockRTCSession,
subjects: { raisedHands$: BehaviorSubject<Record<string, RaisedHandInfo>> },
setSyncState: (value: SyncState) => void,
) => void,
options: CallViewModelOptions = {
encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
autoLeaveWhenOthersLeft: false,
},
): void {
let syncState = initialSyncState;
const setSyncState = (value: SyncState): void => {
const prev = syncState;
syncState = value;
room.client.emit(ClientEvent.Sync, value, prev);
};
const room = mockMatrixRoom({
client: new (class extends EventEmitter {
public getUserId(): string | undefined {
return localRtcMember.userId;
}
export function withCallViewModel(mode: MatrixRTCMode) {
return (
{
remoteParticipants$ = constant([]),
rtcMembers$ = constant([localRtcMember]),
livekitConnectionState$: connectionState$ = constant(
ConnectionState.Connected,
),
speaking = new Map(),
mediaDevices = mockMediaDevices({}),
initialSyncState = SyncState.Syncing,
windowSize$ = constant({ width: 1000, height: 800 }),
}: Partial<CallViewModelInputs> = {},
continuation: (
vm: CallViewModel,
rtcSession: MockRTCSession,
subjects: {
raisedHands$: BehaviorSubject<Record<string, RaisedHandInfo>>;
},
setSyncState: (value: SyncState) => void,
) => void,
options: Partial<CallViewModelOptions> = {},
): void => {
let syncState = initialSyncState;
const setSyncState = (value: SyncState): void => {
const prev = syncState;
syncState = value;
room.client.emit(ClientEvent.Sync, value, prev);
};
const room = mockMatrixRoom({
client: new (class extends EventEmitter {
public getUserId(): string | undefined {
return localRtcMember.userId;
}
public getDeviceId(): string {
return localRtcMember.deviceId;
}
public getDeviceId(): string {
return localRtcMember.deviceId;
}
public getDomain(): string {
return "example.com";
}
public getDomain(): string {
return "example.com";
}
public getSyncState(): SyncState {
return syncState;
}
})() as Partial<MatrixClient> as MatrixClient,
getMembers: () => Array.from(roomMembers.values()),
getMembersWithMembership: () => Array.from(roomMembers.values()),
});
const rtcSession = new MockRTCSession(room, []).withMemberships(rtcMembers$);
const participantsSpy = vi
.spyOn(ComponentsCore, "connectedParticipantsObserver")
.mockReturnValue(remoteParticipants$);
const mediaSpy = vi
.spyOn(ComponentsCore, "observeParticipantMedia")
.mockImplementation((p) =>
of({ participant: p } as Partial<
ComponentsCore.ParticipantMedia<LocalParticipant>
> as ComponentsCore.ParticipantMedia<LocalParticipant>),
public getSyncState(): SyncState {
return syncState;
}
})() as Partial<MatrixClient> as MatrixClient,
getMembers: () => Array.from(roomMembers.values()),
getMembersWithMembership: () => Array.from(roomMembers.values()),
});
const rtcSession = new MockRTCSession(room, []).withMemberships(
rtcMembers$,
);
const eventsSpy = vi
.spyOn(ComponentsCore, "observeParticipantEvents")
.mockImplementation((p, ...eventTypes) => {
if (eventTypes.includes(ParticipantEvent.IsSpeakingChanged)) {
return (speaking.get(p) ?? of(false)).pipe(
map((s): Participant => ({ ...p, isSpeaking: s }) as Participant),
);
} else {
return of(p);
}
const participantsSpy = vi
.spyOn(ComponentsCore, "connectedParticipantsObserver")
.mockReturnValue(remoteParticipants$);
const mediaSpy = vi
.spyOn(ComponentsCore, "observeParticipantMedia")
.mockImplementation((p) =>
of({ participant: p } as Partial<
ComponentsCore.ParticipantMedia<LocalParticipant>
> as ComponentsCore.ParticipantMedia<LocalParticipant>),
);
const eventsSpy = vi
.spyOn(ComponentsCore, "observeParticipantEvents")
.mockImplementation((p, ...eventTypes) => {
if (eventTypes.includes(ParticipantEvent.IsSpeakingChanged)) {
return (speaking.get(p) ?? of(false)).pipe(
map((s): Participant => ({ ...p, isSpeaking: s }) as Participant),
);
} else {
return of(p);
}
});
const roomEventSelectorSpy = vi
.spyOn(ComponentsCore, "roomEventSelector")
.mockImplementation((_room, _eventType) => of());
const muteStates = mockMuteStates();
const raisedHands$ = new BehaviorSubject<Record<string, RaisedHandInfo>>(
{},
);
const reactions$ = new BehaviorSubject<Record<string, ReactionInfo>>({});
const vm = createCallViewModel$(
testScope(),
rtcSession.asMockedSession(),
room,
mediaDevices,
muteStates,
{
encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
autoLeaveWhenOthersLeft: false,
livekitRoomFactory: (): LivekitRoom =>
mockLivekitRoom({
localParticipant,
disconnect: async () => Promise.resolve(),
setE2EEEnabled: async () => Promise.resolve(),
}),
connectionState$,
windowSize$,
matrixRTCMode$: constant(mode),
...options,
},
raisedHands$,
reactions$,
new BehaviorSubject<ProcessorState>({
processor: undefined,
supported: undefined,
}),
);
onTestFinished(() => {
participantsSpy.mockRestore();
mediaSpy.mockRestore();
eventsSpy.mockRestore();
roomEventSelectorSpy.mockRestore();
});
const roomEventSelectorSpy = vi
.spyOn(ComponentsCore, "roomEventSelector")
.mockImplementation((_room, _eventType) => of());
const muteStates = mockMuteStates();
const raisedHands$ = new BehaviorSubject<Record<string, RaisedHandInfo>>({});
const reactions$ = new BehaviorSubject<Record<string, ReactionInfo>>({});
const vm = createCallViewModel$(
testScope(),
rtcSession.asMockedSession(),
room,
mediaDevices,
muteStates,
{
...options,
livekitRoomFactory: (): LivekitRoom =>
mockLivekitRoom({
localParticipant,
disconnect: async () => Promise.resolve(),
setE2EEEnabled: async () => Promise.resolve(),
}),
connectionState$,
},
raisedHands$,
reactions$,
new BehaviorSubject<ProcessorState>({
processor: undefined,
supported: undefined,
}),
);
onTestFinished(() => {
participantsSpy.mockRestore();
mediaSpy.mockRestore();
eventsSpy.mockRestore();
roomEventSelectorSpy.mockRestore();
});
continuation(vm, rtcSession, { raisedHands$: raisedHands$ }, setSyncState);
continuation(vm, rtcSession, { raisedHands$: raisedHands$ }, setSyncState);
};
}

132
src/state/CallViewModel/LayoutSwitch.test.ts Normal file
View File

@@ -0,0 +1,132 @@
/*
Copyright 2025 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { describe, test } from "vitest";
import { createLayoutModeSwitch } from "./LayoutSwitch";
import { testScope, withTestScheduler } from "../../utils/test";
function testLayoutSwitch({
windowMode = "n",
hasScreenShares = "n",
userSelection = "",
expectedGridMode,
}: {
windowMode?: string;
hasScreenShares?: string;
userSelection?: string;
expectedGridMode: string;
}): void {
withTestScheduler(({ behavior, schedule, expectObservable }) => {
const { gridMode$, setGridMode } = createLayoutModeSwitch(
testScope(),
behavior(windowMode, { n: "normal", N: "narrow", f: "flat" }),
behavior(hasScreenShares, { y: true, n: false }),
);
schedule(userSelection, {
g: () => setGridMode("grid"),
s: () => setGridMode("spotlight"),
});
expectObservable(gridMode$).toBe(expectedGridMode, {
g: "grid",
s: "spotlight",
});
});
}
describe("default mode", () => {
test("uses grid layout by default", () =>
testLayoutSwitch({
expectedGridMode: "g",
}));
test("uses spotlight mode when window mode is flat", () =>
testLayoutSwitch({
windowMode: " f",
expectedGridMode: "s",
}));
});
test("allows switching modes manually", () =>
testLayoutSwitch({
userSelection: " --sgs",
expectedGridMode: "g-sgs",
}));
test("switches to spotlight mode when there is a remote screen share", () =>
testLayoutSwitch({
hasScreenShares: " n--y",
expectedGridMode: "g--s",
}));
test("can manually switch to grid when there is a screenshare", () =>
testLayoutSwitch({
hasScreenShares: " n-y",
userSelection: " ---g",
expectedGridMode: "g-sg",
}));
test("auto-switches after manually selecting grid", () =>
testLayoutSwitch({
// Two screenshares will happen in sequence. There is a screen share that
// forces spotlight, then the user manually switches back to grid.
hasScreenShares: " n-y-ny",
userSelection: " ---g",
expectedGridMode: "g-sg-s",
// If we did want to respect manual selection, the expectation would be: g-sg
}));
test("switches back to grid mode when the remote screen share ends", () =>
testLayoutSwitch({
hasScreenShares: " n--y--n",
expectedGridMode: "g--s--g",
}));
test("auto-switches to spotlight again after first screen share ends", () =>
testLayoutSwitch({
hasScreenShares: " nyny",
expectedGridMode: "gsgs",
}));
test("switches manually to grid after screen share while manually in spotlight", () =>
testLayoutSwitch({
// Initially, no one is sharing. Then the user manually switches to spotlight.
// After a screen share starts, the user manually switches to grid.
hasScreenShares: " n-y",
userSelection: " -s-g",
expectedGridMode: "gs-g",
}));
test("auto-switches to spotlight when in flat window mode", () =>
testLayoutSwitch({
// First normal, then narrow, then flat.
windowMode: " nNf",
expectedGridMode: "g-s",
}));
test("allows switching modes manually when in flat window mode", () =>
testLayoutSwitch({
// Window becomes flat, then user switches to grid and back.
// Finally the window returns to a normal shape.
windowMode: " nf--n",
userSelection: " --gs",
expectedGridMode: "gsgsg",
}));
test("stays in spotlight while there are screen shares even when window mode changes", () =>
testLayoutSwitch({
windowMode: " nfn",
hasScreenShares: " y",
expectedGridMode: "s",
}));
test("ignores end of screen share until window mode returns to normal", () =>
testLayoutSwitch({
windowMode: " nf-n",
hasScreenShares: " y-n",
expectedGridMode: "s--g",
}));

93
src/state/CallViewModel/LayoutSwitch.ts Normal file
View File

@@ -0,0 +1,93 @@
/*
Copyright 2025 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
combineLatest,
map,
Subject,
startWith,
skipWhile,
switchMap,
} from "rxjs";
import { type GridMode, type WindowMode } from "./CallViewModel.ts";
import { constant, type Behavior } from "../Behavior.ts";
import { type ObservableScope } from "../ObservableScope.ts";
/**
* Creates a layout mode switch that allows switching between grid and spotlight modes.
* The actual layout mode might switch automatically to spotlight if there is a
* remote screen share active or if the window mode is flat.
*
* @param scope - The observable scope to manage subscriptions.
* @param windowMode$ - The current window mode.
* @param hasRemoteScreenShares$ - A behavior indicating if there are remote screen shares active.
*/
export function createLayoutModeSwitch(
scope: ObservableScope,
windowMode$: Behavior<WindowMode>,
hasRemoteScreenShares$: Behavior<boolean>,
): {
gridMode$: Behavior<GridMode>;
setGridMode: (value: GridMode) => void;
} {
const userSelection$ = new Subject<GridMode>();
// Callback to set the grid mode desired by the user.
// Notice that this is only a preference, the actual grid mode can be overridden
// if there is a remote screen share active.
const setGridMode = (value: GridMode): void => userSelection$.next(value);
/**
* The natural grid mode - the mode that the grid would prefer to be in,
* not accounting for the user's manual selections.
*/
const naturalGridMode$ = scope.behavior<GridMode>(
combineLatest(
[hasRemoteScreenShares$, windowMode$],
(hasRemoteScreenShares, windowMode) =>
// When there are screen shares or the window is flat (as with a phone
// in landscape orientation), spotlight is a better experience.
// We want screen shares to be big and readable, and we want flipping
// your phone into landscape to be a quick way of maximising the
// spotlight tile.
hasRemoteScreenShares || windowMode === "flat" ? "spotlight" : "grid",
),
);
/**
* The layout mode of the media tile grid.
*/
const gridMode$ = scope.behavior<GridMode>(
// Whenever the user makes a selection, we enter a new mode of behavior:
userSelection$.pipe(
map((selection) => {
if (selection === "grid")
// The user has selected grid mode. Start by respecting their choice,
// but then follow the natural mode again as soon as it matches.
return naturalGridMode$.pipe(
skipWhile((naturalMode) => naturalMode !== selection),
startWith(selection),
);
// The user has selected spotlight mode. If this matches the natural
// mode, then follow the natural mode going forward.
return selection === naturalGridMode$.value
? naturalGridMode$
: constant(selection);
}),
// Initially the mode of behavior is to just follow the natural grid mode.
startWith(naturalGridMode$),
// Switch between each mode of behavior.
switchMap((mode$) => mode$),
),
);
return {
gridMode$,
setGridMode,
};
}

58
src/state/CallViewModel/localMember/HomeserverConnected.test.ts
View File

@@ -97,106 +97,106 @@ describe("createHomeserverConnected$", () => {
// LLM generated test cases. They are a bit overkill but I improved the mocking so it is
// easy enough to read them so I think they can stay.
it("is false when sync state is not Syncing", () => {
const hsConnected$ = createHomeserverConnected$(scope, client, session);
expect(hsConnected$.value).toBe(false);
const hsConnected = createHomeserverConnected$(scope, client, session);
expect(hsConnected.combined$.value).toBe(false);
});
it("remains false while membership status is not Connected even if sync is Syncing", () => {
const hsConnected$ = createHomeserverConnected$(scope, client, session);
const hsConnected = createHomeserverConnected$(scope, client, session);
client.setSyncState(SyncState.Syncing);
expect(hsConnected$.value).toBe(false); // membership still disconnected
expect(hsConnected.combined$.value).toBe(false); // membership still disconnected
});
it("is false when membership status transitions to Connected but ProbablyLeft is true", () => {
const hsConnected$ = createHomeserverConnected$(scope, client, session);
const hsConnected = createHomeserverConnected$(scope, client, session);
// Make sync loop OK
client.setSyncState(SyncState.Syncing);
// Indicate probable leave before connection
session.setProbablyLeft(true);
session.setMembershipStatus(Status.Connected);
expect(hsConnected$.value).toBe(false);
expect(hsConnected.combined$.value).toBe(false);
});
it("becomes true only when all three conditions are satisfied", () => {
const hsConnected$ = createHomeserverConnected$(scope, client, session);
const hsConnected = createHomeserverConnected$(scope, client, session);
// 1. Sync loop connected
client.setSyncState(SyncState.Syncing);
expect(hsConnected$.value).toBe(false); // not yet membership connected
expect(hsConnected.combined$.value).toBe(false); // not yet membership connected
// 2. Membership connected
session.setMembershipStatus(Status.Connected);
expect(hsConnected$.value).toBe(true); // probablyLeft is false
expect(hsConnected.combined$.value).toBe(true); // probablyLeft is false
});
it("drops back to false when sync loop leaves Syncing", () => {
const hsConnected$ = createHomeserverConnected$(scope, client, session);
const hsConnected = createHomeserverConnected$(scope, client, session);
// Reach connected state
client.setSyncState(SyncState.Syncing);
session.setMembershipStatus(Status.Connected);
expect(hsConnected$.value).toBe(true);
expect(hsConnected.combined$.value).toBe(true);
// Sync loop error => should flip false
client.setSyncState(SyncState.Error);
expect(hsConnected$.value).toBe(false);
expect(hsConnected.combined$.value).toBe(false);
});
it("drops back to false when membership status becomes disconnected", () => {
const hsConnected$ = createHomeserverConnected$(scope, client, session);
const hsConnected = createHomeserverConnected$(scope, client, session);
client.setSyncState(SyncState.Syncing);
session.setMembershipStatus(Status.Connected);
expect(hsConnected$.value).toBe(true);
expect(hsConnected.combined$.value).toBe(true);
session.setMembershipStatus(Status.Disconnected);
expect(hsConnected$.value).toBe(false);
expect(hsConnected.combined$.value).toBe(false);
});
it("drops to false when ProbablyLeft is emitted after being true", () => {
const hsConnected$ = createHomeserverConnected$(scope, client, session);
const hsConnected = createHomeserverConnected$(scope, client, session);
client.setSyncState(SyncState.Syncing);
session.setMembershipStatus(Status.Connected);
expect(hsConnected$.value).toBe(true);
expect(hsConnected.combined$.value).toBe(true);
session.setProbablyLeft(true);
expect(hsConnected$.value).toBe(false);
expect(hsConnected.combined$.value).toBe(false);
});
it("recovers to true if ProbablyLeft becomes false again while other conditions remain true", () => {
const hsConnected$ = createHomeserverConnected$(scope, client, session);
const hsConnected = createHomeserverConnected$(scope, client, session);
client.setSyncState(SyncState.Syncing);
session.setMembershipStatus(Status.Connected);
expect(hsConnected$.value).toBe(true);
expect(hsConnected.combined$.value).toBe(true);
session.setProbablyLeft(true);
expect(hsConnected$.value).toBe(false);
expect(hsConnected.combined$.value).toBe(false);
// Simulate clearing the flag (in realistic scenario membership manager would update)
session.setProbablyLeft(false);
expect(hsConnected$.value).toBe(true);
expect(hsConnected.combined$.value).toBe(true);
});
it("composite sequence reflects each individual failure reason", () => {
const hsConnected$ = createHomeserverConnected$(scope, client, session);
const hsConnected = createHomeserverConnected$(scope, client, session);
// Initially false (sync error + disconnected + not probably left)
expect(hsConnected$.value).toBe(false);
expect(hsConnected.combined$.value).toBe(false);
// Fix sync only
client.setSyncState(SyncState.Syncing);
expect(hsConnected$.value).toBe(false);
expect(hsConnected.combined$.value).toBe(false);
// Fix membership
session.setMembershipStatus(Status.Connected);
expect(hsConnected$.value).toBe(true);
expect(hsConnected.combined$.value).toBe(true);
// Introduce probablyLeft -> false
session.setProbablyLeft(true);
expect(hsConnected$.value).toBe(false);
expect(hsConnected.combined$.value).toBe(false);
// Restore notProbablyLeft -> true again
session.setProbablyLeft(false);
expect(hsConnected$.value).toBe(true);
expect(hsConnected.combined$.value).toBe(true);
// Drop sync -> false
client.setSyncState(SyncState.Error);
expect(hsConnected$.value).toBe(false);
expect(hsConnected.combined$.value).toBe(false);
});
});

38
src/state/CallViewModel/localMember/HomeserverConnected.ts
View File

@@ -25,6 +25,11 @@ import { type NodeStyleEventEmitter } from "../../../utils/test";
*/
const logger = rootLogger.getChild("[HomeserverConnected]");
export interface HomeserverConnected {
combined$: Behavior<boolean>;
rtsSession$: Behavior<Status>;
}
/**
* Behavior representing whether we consider ourselves connected to the Matrix homeserver
* for the purposes of a MatrixRTC session.
@@ -39,7 +44,7 @@ export function createHomeserverConnected$(
client: NodeStyleEventEmitter & Pick<MatrixClient, "getSyncState">,
matrixRTCSession: NodeStyleEventEmitter &
Pick<MatrixRTCSession, "membershipStatus" | "probablyLeft">,
): Behavior<boolean> {
): HomeserverConnected {
const syncing$ = (
fromEvent(client, ClientEvent.Sync) as Observable<[SyncState]>
).pipe(
@@ -47,12 +52,15 @@ export function createHomeserverConnected$(
map(([state]) => state === SyncState.Syncing),
);
const membershipConnected$ = fromEvent(
matrixRTCSession,
MembershipManagerEvent.StatusChanged,
).pipe(
startWith(null),
map(() => matrixRTCSession.membershipStatus === Status.Connected),
const rtsSession$ = scope.behavior<Status>(
fromEvent(matrixRTCSession, MembershipManagerEvent.StatusChanged).pipe(
map(() => matrixRTCSession.membershipStatus ?? Status.Unknown),
),
Status.Unknown,
);
const membershipConnected$ = rtsSession$.pipe(
map((status) => status === Status.Connected),
);
// This is basically notProbablyLeft$
@@ -71,15 +79,13 @@ export function createHomeserverConnected$(
map(() => matrixRTCSession.probablyLeft !== true),
);
const connectedCombined$ = and$(
syncing$,
membershipConnected$,
certainlyConnected$,
).pipe(
tap((connected) => {
logger.info(`Homeserver connected update: ${connected}`);
}),
const combined$ = scope.behavior(
and$(syncing$, membershipConnected$, certainlyConnected$).pipe(
tap((connected) => {
logger.info(`Homeserver connected update: ${connected}`);
}),
),
);
return scope.behavior(connectedCombined$);
return { combined$, rtsSession$ };
}

527
src/state/CallViewModel/localMember/LocalMember.test.ts Normal file
View File

@@ -0,0 +1,527 @@
/*
Copyright 2025 Element Creations Ltd.
Copyright 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
Status as RTCMemberStatus,
type LivekitTransport,
type MatrixRTCSession,
} from "matrix-js-sdk/lib/matrixrtc";
import { describe, expect, it, vi } from "vitest";
import { AutoDiscovery } from "matrix-js-sdk/lib/autodiscovery";
import { BehaviorSubject, map, of } from "rxjs";
import { logger } from "matrix-js-sdk/lib/logger";
import { type LocalParticipant, type LocalTrack } from "livekit-client";
import { MatrixRTCMode } from "../../../settings/settings";
import {
flushPromises,
mockConfig,
mockLivekitRoom,
mockMuteStates,
withTestScheduler,
} from "../../../utils/test";
import {
TransportState,
createLocalMembership$,
enterRTCSession,
PublishState,
TrackState,
} from "./LocalMember";
import { MatrixRTCTransportMissingError } from "../../../utils/errors";
import { Epoch, ObservableScope } from "../../ObservableScope";
import { constant } from "../../Behavior";
import { ConnectionManagerData } from "../remoteMembers/ConnectionManager";
import { ConnectionState, type Connection } from "../remoteMembers/Connection";
import { type Publisher } from "./Publisher";
const MATRIX_RTC_MODE = MatrixRTCMode.Legacy;
const getUrlParams = vi.hoisted(() => vi.fn(() => ({})));
vi.mock("../../../UrlParams", () => ({ getUrlParams }));
vi.mock("@livekit/components-core", () => ({
observeParticipantEvents: vi
.fn()
.mockReturnValue(of({ isScreenShareEnabled: false })),
}));
describe("LocalMembership", () => {
describe("enterRTCSession", () => {
it("It joins the correct Session", () => {
const focusFromOlderMembership = {
type: "livekit",
livekit_service_url: "http://my-oldest-member-service-url.com",
livekit_alias: "my-oldest-member-service-alias",
};
const focusConfigFromWellKnown = {
type: "livekit",
livekit_service_url: "http://my-well-known-service-url.com",
};
const focusConfigFromWellKnown2 = {
type: "livekit",
livekit_service_url: "http://my-well-known-service-url2.com",
};
const clientWellKnown = {
"org.matrix.msc4143.rtc_foci": [
focusConfigFromWellKnown,
focusConfigFromWellKnown2,
],
};
mockConfig({
livekit: { livekit_service_url: "http://my-default-service-url.com" },
});
vi.spyOn(AutoDiscovery, "getRawClientConfig").mockImplementation(
async (domain) => {
if (domain === "example.org") {
return Promise.resolve(clientWellKnown);
}
return Promise.resolve({});
},
);
const mockedSession = vi.mocked({
room: {
roomId: "roomId",
client: {
getDomain: vi.fn().mockReturnValue("example.org"),
getOpenIdToken: vi.fn().mockResolvedValue({
access_token: "ACCCESS_TOKEN",
token_type: "Bearer",
matrix_server_name: "localhost",
expires_in: 10000,
}),
},
},
memberships: [],
getFocusInUse: vi.fn().mockReturnValue(focusFromOlderMembership),
getOldestMembership: vi.fn().mockReturnValue({
getPreferredFoci: vi.fn().mockReturnValue([focusFromOlderMembership]),
}),
joinRoomSession: vi.fn(),
}) as unknown as MatrixRTCSession;
enterRTCSession(
mockedSession,
{
livekit_alias: "roomId",
livekit_service_url: "http://my-well-known-service-url.com",
type: "livekit",
},
{
encryptMedia: true,
matrixRTCMode: MATRIX_RTC_MODE,
},
);
expect(mockedSession.joinRoomSession).toHaveBeenLastCalledWith(
[
{
livekit_alias: "roomId",
livekit_service_url: "http://my-well-known-service-url.com",
type: "livekit",
},
],
undefined,
expect.objectContaining({
manageMediaKeys: true,
useLegacyMemberEvents: false,
}),
);
});
it("It should not fail with configuration error if homeserver config has livekit url but not fallback", () => {
mockConfig({});
vi.spyOn(AutoDiscovery, "getRawClientConfig").mockResolvedValue({
"org.matrix.msc4143.rtc_foci": [
{
type: "livekit",
livekit_service_url: "http://my-well-known-service-url.com",
},
],
});
const mockedSession = vi.mocked({
room: {
roomId: "roomId",
client: {
getDomain: vi.fn().mockReturnValue("example.org"),
getOpenIdToken: vi.fn().mockResolvedValue({
access_token: "ACCCESS_TOKEN",
token_type: "Bearer",
matrix_server_name: "localhost",
expires_in: 10000,
}),
},
},
memberships: [],
getFocusInUse: vi.fn(),
joinRoomSession: vi.fn(),
}) as unknown as MatrixRTCSession;
enterRTCSession(
mockedSession,
{
livekit_alias: "roomId",
livekit_service_url: "http://my-well-known-service-url.com",
type: "livekit",
},
{
encryptMedia: true,
matrixRTCMode: MATRIX_RTC_MODE,
},
);
});
});
const defaultCreateLocalMemberValues = {
options: constant({
encryptMedia: false,
matrixRTCMode: MatrixRTCMode.Matrix_2_0,
}),
matrixRTCSession: {
updateCallIntent: () => {},
leaveRoomSession: () => {},
} as unknown as MatrixRTCSession,
muteStates: mockMuteStates(),
trackProcessorState$: constant({
supported: false,
processor: undefined,
}),
logger: logger,
createPublisherFactory: vi.fn(),
joinMatrixRTC: async (): Promise<void> => {},
homeserverConnected: {
combined$: constant(true),
rtsSession$: constant(RTCMemberStatus.Connected),
},
};
it("throws error on missing RTC config error", () => {
withTestScheduler(({ scope, hot, expectObservable }) => {
const localTransport$ = scope.behavior<null | LivekitTransport>(
hot("1ms #", {}, new MatrixRTCTransportMissingError("domain.com")),
null,
);
// we do not need any connection data since we want to fail before reaching that.
const mockConnectionManager = {
transports$: scope.behavior(
localTransport$.pipe(map((t) => new Epoch([t]))),
),
connectionManagerData$: constant(
new Epoch(new ConnectionManagerData()),
),
};
const localMembership = createLocalMembership$({
scope,
...defaultCreateLocalMemberValues,
connectionManager: mockConnectionManager,
localTransport$,
});
localMembership.requestJoinAndPublish();
expectObservable(localMembership.localMemberState$).toBe("ne", {
n: TransportState.Waiting,
e: expect.toSatisfy((e) => e instanceof MatrixRTCTransportMissingError),
});
});
});
const aTransport = {
livekit_service_url: "a",
} as LivekitTransport;
const bTransport = {
livekit_service_url: "b",
} as LivekitTransport;
const connectionTransportAConnected = {
livekitRoom: mockLivekitRoom({
localParticipant: {
isScreenShareEnabled: false,
trackPublications: [],
} as unknown as LocalParticipant,
}),
state$: constant(ConnectionState.LivekitConnected),
transport: aTransport,
} as unknown as Connection;
const connectionTransportAConnecting = {
...connectionTransportAConnected,
state$: constant(ConnectionState.LivekitConnecting),
livekitRoom: mockLivekitRoom({}),
} as unknown as Connection;
const connectionTransportBConnected = {
state$: constant(ConnectionState.LivekitConnected),
transport: bTransport,
livekitRoom: mockLivekitRoom({}),
} as unknown as Connection;
it("recreates publisher if new connection is used and ENDS always unpublish and end tracks", async () => {
const scope = new ObservableScope();
const localTransport$ = new BehaviorSubject(aTransport);
const publishers: Publisher[] = [];
let seed = 0;
defaultCreateLocalMemberValues.createPublisherFactory.mockImplementation(
() => {
const a = seed;
seed += 1;
logger.info(`creating [${a}]`);
const p = {
stopPublishing: vi.fn().mockImplementation(() => {
logger.info(`stopPublishing [${a}]`);
}),
stopTracks: vi.fn(),
};
publishers.push(p as unknown as Publisher);
return p;
},
);
const publisherFactory =
defaultCreateLocalMemberValues.createPublisherFactory as ReturnType<
typeof vi.fn
>;
const connectionManagerData = new ConnectionManagerData();
connectionManagerData.add(connectionTransportAConnected, []);
connectionManagerData.add(connectionTransportBConnected, []);
createLocalMembership$({
scope,
...defaultCreateLocalMemberValues,
connectionManager: {
connectionManagerData$: constant(new Epoch(connectionManagerData)),
},
localTransport$,
});
await flushPromises();
localTransport$.next(bTransport);
await flushPromises();
expect(publisherFactory).toHaveBeenCalledTimes(2);
expect(publishers.length).toBe(2);
// stop the first Publisher and let the second one life.
expect(publishers[0].stopTracks).toHaveBeenCalled();
expect(publishers[1].stopTracks).not.toHaveBeenCalled();
expect(publishers[0].stopPublishing).toHaveBeenCalled();
expect(publishers[1].stopPublishing).not.toHaveBeenCalled();
expect(publisherFactory.mock.calls[0][0].transport).toBe(aTransport);
expect(publisherFactory.mock.calls[1][0].transport).toBe(bTransport);
scope.end();
await flushPromises();
// stop all tracks after ending scopes
expect(publishers[1].stopPublishing).toHaveBeenCalled();
// expect(publishers[1].stopTracks).toHaveBeenCalled();
defaultCreateLocalMemberValues.createPublisherFactory.mockReset();
});
it("only start tracks if requested", async () => {
const scope = new ObservableScope();
const localTransport$ = new BehaviorSubject(aTransport);
const publishers: Publisher[] = [];
const tracks$ = new BehaviorSubject<LocalTrack[]>([]);
const publishing$ = new BehaviorSubject<boolean>(false);
defaultCreateLocalMemberValues.createPublisherFactory.mockImplementation(
() => {
const p = {
stopPublishing: vi.fn(),
stopTracks: vi.fn(),
createAndSetupTracks: vi.fn().mockImplementation(async () => {
tracks$.next([{}, {}] as LocalTrack[]);
return Promise.resolve();
}),
tracks$,
publishing$,
};
publishers.push(p as unknown as Publisher);
return p;
},
);
const publisherFactory =
defaultCreateLocalMemberValues.createPublisherFactory as ReturnType<
typeof vi.fn
>;
const connectionManagerData = new ConnectionManagerData();
connectionManagerData.add(connectionTransportAConnected, []);
// connectionManagerData.add(connectionTransportB, []);
const localMembership = createLocalMembership$({
scope,
...defaultCreateLocalMemberValues,
connectionManager: {
connectionManagerData$: constant(new Epoch(connectionManagerData)),
},
localTransport$,
});
await flushPromises();
expect(publisherFactory).toHaveBeenCalledOnce();
// expect(localMembership.tracks$.value.length).toBe(0);
expect(publishers[0].createAndSetupTracks).not.toHaveBeenCalled();
localMembership.startTracks();
await flushPromises();
expect(publishers[0].createAndSetupTracks).toHaveBeenCalled();
// expect(localMembership.tracks$.value.length).toBe(2);
scope.end();
await flushPromises();
// stop all tracks after ending scopes
expect(publishers[0].stopPublishing).toHaveBeenCalled();
// expect(publishers[0].stopTracks).toHaveBeenCalled();
publisherFactory.mockClear();
});
// TODO add an integration test combining publisher and localMembership
//
it("tracks livekit state correctly", async () => {
const scope = new ObservableScope();
const connectionManagerData = new ConnectionManagerData();
const localTransport$ = new BehaviorSubject<null | LivekitTransport>(null);
const connectionManagerData$ = new BehaviorSubject(
new Epoch(connectionManagerData),
);
const publishers: Publisher[] = [];
const tracks$ = new BehaviorSubject<LocalTrack[]>([]);
const publishing$ = new BehaviorSubject<boolean>(false);
const createTrackResolver = Promise.withResolvers<void>();
const publishResolver = Promise.withResolvers<void>();
defaultCreateLocalMemberValues.createPublisherFactory.mockImplementation(
() => {
const p = {
stopPublishing: vi.fn(),
stopTracks: vi.fn().mockImplementation(() => {
logger.info("stopTracks");
tracks$.next([]);
}),
createAndSetupTracks: vi.fn().mockImplementation(async () => {
await createTrackResolver.promise;
tracks$.next([{}, {}] as LocalTrack[]);
}),
startPublishing: vi.fn().mockImplementation(async () => {
await publishResolver.promise;
publishing$.next(true);
}),
tracks$,
publishing$,
};
publishers.push(p as unknown as Publisher);
return p;
},
);
const publisherFactory =
defaultCreateLocalMemberValues.createPublisherFactory as ReturnType<
typeof vi.fn
>;
const localMembership = createLocalMembership$({
scope,
...defaultCreateLocalMemberValues,
connectionManager: {
connectionManagerData$,
},
localTransport$,
});
await flushPromises();
expect(localMembership.localMemberState$.value).toStrictEqual(
TransportState.Waiting,
);
localTransport$.next(aTransport);
await flushPromises();
expect(localMembership.localMemberState$.value).toStrictEqual({
matrix: RTCMemberStatus.Connected,
media: { connection: null, tracks: TrackState.WaitingForUser },
});
const connectionManagerData2 = new ConnectionManagerData();
connectionManagerData2.add(
// clone because we will mutate this later.
{ ...connectionTransportAConnecting } as unknown as Connection,
[],
);
connectionManagerData$.next(new Epoch(connectionManagerData2));
await flushPromises();
expect(localMembership.localMemberState$.value).toStrictEqual({
matrix: RTCMemberStatus.Connected,
media: {
connection: ConnectionState.LivekitConnecting,
tracks: TrackState.WaitingForUser,
},
});
(
connectionManagerData2.getConnectionForTransport(aTransport)!
.state$ as BehaviorSubject<ConnectionState>
).next(ConnectionState.LivekitConnected);
expect(localMembership.localMemberState$.value).toStrictEqual({
matrix: RTCMemberStatus.Connected,
media: {
connection: ConnectionState.LivekitConnected,
tracks: TrackState.WaitingForUser,
},
});
expect(publisherFactory).toHaveBeenCalledOnce();
// expect(localMembership.tracks$.value.length).toBe(0);
// -------
localMembership.startTracks();
// -------
await flushPromises();
// expect(localMembership.localMemberState$.value).toStrictEqual({
// matrix: RTCMemberStatus.Connected,
// media: {
// tracks: TrackState.Creating,
// connection: ConnectionState.LivekitConnected,
// },
// });
createTrackResolver.resolve();
await flushPromises();
expect(
// eslint-disable-next-line @typescript-eslint/no-explicit-any
(localMembership.localMemberState$.value as any).media,
).toStrictEqual(PublishState.WaitingForUser);
// -------
localMembership.requestJoinAndPublish();
// -------
expect(
// eslint-disable-next-line @typescript-eslint/no-explicit-any
(localMembership.localMemberState$.value as any).media,
).toStrictEqual(PublishState.Publishing);
publishResolver.resolve();
await flushPromises();
expect(
// eslint-disable-next-line @typescript-eslint/no-explicit-any
(localMembership.localMemberState$.value as any).media,
).toStrictEqual(PublishState.Publishing);
expect(publishers[0].stopPublishing).not.toHaveBeenCalled();
expect(localMembership.localMemberState$.isStopped).toBe(false);
scope.end();
await flushPromises();
// stays in connected state because it is stopped before the update to tracks update the state.
expect(
// eslint-disable-next-line @typescript-eslint/no-explicit-any
(localMembership.localMemberState$.value as any).media,
).toStrictEqual(PublishState.Publishing);
// stop all tracks after ending scopes
expect(publishers[0].stopPublishing).toHaveBeenCalled();
// expect(publishers[0].stopTracks).toHaveBeenCalled();
});
// TODO add tests for matrix local matrix participation.
});

739
src/state/CallViewModel/localMember/LocalMember.ts Normal file
View File

@@ -0,0 +1,739 @@
/*
Copyright 2025 Element Creations Ltd.
SPDX-License-IdFentifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
type Participant,
ParticipantEvent,
type LocalParticipant,
type ScreenShareCaptureOptions,
RoomEvent,
MediaDeviceFailure,
} from "livekit-client";
import { observeParticipantEvents } from "@livekit/components-core";
import {
Status as RTCSessionStatus,
type LivekitTransport,
type MatrixRTCSession,
} from "matrix-js-sdk/lib/matrixrtc";
import {
BehaviorSubject,
catchError,
combineLatest,
distinctUntilChanged,
from,
fromEvent,
map,
type Observable,
of,
pairwise,
startWith,
switchMap,
tap,
} from "rxjs";
import { type Logger } from "matrix-js-sdk/lib/logger";
import { deepCompare } from "matrix-js-sdk/lib/utils";
import { type Behavior } from "../../Behavior.ts";
import { type IConnectionManager } from "../remoteMembers/ConnectionManager.ts";
import { type ObservableScope } from "../../ObservableScope.ts";
import { type Publisher } from "./Publisher.ts";
import { type MuteStates } from "../../MuteStates.ts";
import {
ElementCallError,
FailToStartLivekitConnection,
MembershipManagerError,
UnknownCallError,
} from "../../../utils/errors.ts";
import { ElementWidgetActions, widget } from "../../../widget.ts";
import { getUrlParams } from "../../../UrlParams.ts";
import { PosthogAnalytics } from "../../../analytics/PosthogAnalytics.ts";
import { MatrixRTCMode } from "../../../settings/settings.ts";
import { Config } from "../../../config/Config.ts";
import {
ConnectionState,
type Connection,
type FailedToStartError,
} from "../remoteMembers/Connection.ts";
import { type HomeserverConnected } from "./HomeserverConnected.ts";
import { and$ } from "../../../utils/observable.ts";
export enum TransportState {
/** Not even a transport is available to the LocalMembership */
Waiting = "transport_waiting",
}
export enum PublishState {
WaitingForUser = "publish_waiting_for_user",
// XXX: This state is removed for now since we do not have full control over
// track publication anymore with the publisher abstraction, might come back in the future?
// /** Implies lk connection is connected */
// Starting = "publish_start_publishing",
/** Implies lk connection is connected */
Publishing = "publish_publishing",
}
// TODO not sure how to map that correctly with the
// new publisher that does not manage tracks itself anymore
export enum TrackState {
/** The track is waiting for user input to create tracks (waiting to call `startTracks()`) */
WaitingForUser = "tracks_waiting_for_user",
// XXX: This state is removed for now since we do not have full control over
// track creation anymore with the publisher abstraction, might come back in the future?
// /** Implies lk connection is connected */
// Creating = "tracks_creating",
/** Implies lk connection is connected */
Ready = "tracks_ready",
}
export type LocalMemberMediaState =
| {
tracks: TrackState;
connection: ConnectionState | FailedToStartError;
}
| PublishState
| ElementCallError;
export type LocalMemberState =
| ElementCallError
| TransportState.Waiting
| {
media: LocalMemberMediaState;
matrix: ElementCallError | RTCSessionStatus;
};
/*
* - get well known
* - get oldest membership
* - get transport to use
* - get openId + jwt token
* - wait for createTrack() call
* - create tracks
* - wait for join() call
* - Publisher.publishTracks()
* - send join state/sticky event
*/
interface Props {
// TODO add a comment into some code style readme or file header callviewmodel
// that the inputs for those createSomething$() functions should NOT contain any js-sdk objectes
scope: ObservableScope;
muteStates: MuteStates;
connectionManager: IConnectionManager;
createPublisherFactory: (connection: Connection) => Publisher;
joinMatrixRTC: (transport: LivekitTransport) => void;
homeserverConnected: HomeserverConnected;
localTransport$: Behavior<LivekitTransport | null>;
matrixRTCSession: Pick<
MatrixRTCSession,
"updateCallIntent" | "leaveRoomSession"
>;
logger: Logger;
}
/**
* This class is responsible for managing the own membership in a room.
* We want
* - a publisher
* -
* @param props The properties required to create the local membership.
* @param props.scope The observable scope to use.
* @param props.connectionManager The connection manager to get connections from.
* @param props.createPublisherFactory Factory to create a publisher once we have a connection.
* @param props.joinMatrixRTC Callback to join the matrix RTC session once we have a transport.
* @param props.homeserverConnected The homeserver connected state.
* @param props.localTransport$ The local transport to use for publishing.
* @param props.logger The logger to use.
* @param props.muteStates The mute states for video and audio.
* @param props.matrixRTCSession The matrix RTC session to join.
* @returns
* - publisher: The handle to create tracks and publish them to the room.
* - connected$: the current connection state. Including matrix server and livekit server connection. (only considering the livekit server we are using for our own media publication)
* - transport$: the transport object the ownMembership$ ended up using.
* - connectionState: the current connection state. Including matrix server and livekit server connection.
* - sharingScreen$: Whether we are sharing our screen. `undefined` if we cannot share the screen.
*/
export const createLocalMembership$ = ({
scope,
connectionManager,
localTransport$: localTransportCanThrow$,
homeserverConnected,
createPublisherFactory,
joinMatrixRTC,
logger: parentLogger,
muteStates,
matrixRTCSession,
}: Props): {
/**
* This request to start audio and video tracks.
* Can be called early to pre-emptively get media permissions and start devices.
*/
startTracks: () => void;
/**
* This sets a inner state (shouldPublish) to true and instructs the js-sdk and livekit to keep the user
* connected to matrix and livekit.
*/
requestJoinAndPublish: () => void;
requestDisconnect: () => void;
localMemberState$: Behavior<LocalMemberState>;
sharingScreen$: Behavior<boolean>;
/**
* Callback to toggle screen sharing. If null, screen sharing is not possible.
*/
toggleScreenSharing: (() => void) | null;
// tracks$: Behavior<LocalTrack[]>;
participant$: Behavior<LocalParticipant | null>;
connection$: Behavior<Connection | null>;
/**
* Tracks the homserver and livekit connected state and based on that computes reconnecting.
*/
reconnecting$: Behavior<boolean>;
/** Shorthand for homeserverConnected.rtcSession === Status.Disconnected
* Direct translation to the js-sdk membership manager connection `Status`.
*/
disconnected$: Behavior<boolean>;
/**
* Fully connected
*/
connected$: Behavior<boolean>;
} => {
const logger = parentLogger.getChild("[LocalMembership]");
logger.debug(`Creating local membership..`);
// Unwrap the local transport and set the state of the LocalMembership to error in case the transport is an error.
const localTransport$ = scope.behavior(
localTransportCanThrow$.pipe(
catchError((e: unknown) => {
let error: ElementCallError;
if (e instanceof ElementCallError) {
error = e;
} else {
error = new UnknownCallError(
e instanceof Error
? e
: new Error("Unknown error from localTransport"),
);
}
setTransportError(error);
return of(null);
}),
),
);
// Drop Epoch data here since we will not combine this anymore
const localConnection$ = scope.behavior(
combineLatest([
connectionManager.connectionManagerData$,
localTransport$,
]).pipe(
map(([{ value: connectionData }, localTransport]) => {
if (localTransport === null) {
return null;
}
return connectionData.getConnectionForTransport(localTransport);
}),
tap((connection) => {
logger.info(
`Local connection updated: ${connection?.transport?.livekit_service_url}`,
);
}),
),
);
// Tracks error that happen when creating the local tracks.
const mediaErrors$ = localConnection$.pipe(
switchMap((connection) => {
if (!connection) {
return of(null);
} else {
return fromEvent(
connection.livekitRoom,
RoomEvent.MediaDevicesError,
(error: Error) => {
return MediaDeviceFailure.getFailure(error) ?? null;
},
);
}
}),
);
mediaErrors$.pipe(scope.bind()).subscribe((error) => {
if (error) {
logger.error(`Failed to create local tracks:`, error);
setMatrixError(
// TODO is it fatal? Do we need to create a new Specialized Error?
new UnknownCallError(new Error(`Media device error: ${error}`)),
);
}
});
// MATRIX RELATED
// This should be used in a combineLatest with publisher$ to connect.
// to make it possible to call startTracks before the preferredTransport$ has resolved.
const trackStartRequested = Promise.withResolvers<void>();
// This should be used in a combineLatest with publisher$ to connect.
// to make it possible to call startTracks before the preferredTransport$ has resolved.
const joinAndPublishRequested$ = new BehaviorSubject(false);
/**
* The publisher is stored in here an abstracts creating and publishing tracks.
*/
const publisher$ = new BehaviorSubject<Publisher | null>(null);
const startTracks = (): void => {
trackStartRequested.resolve();
// This used to return the tracks, but now they are only accessible via the publisher.
};
const requestJoinAndPublish = (): void => {
trackStartRequested.resolve();
joinAndPublishRequested$.next(true);
};
const requestDisconnect = (): void => {
joinAndPublishRequested$.next(false);
};
// Take care of the publisher$
// create a new one as soon as a local Connection is available
//
// Recreate a new one once the local connection changes
// - stop publishing
// - destruct all current streams
// - overwrite current publisher
scope.reconcile(localConnection$, async (connection) => {
if (connection !== null) {
const publisher = createPublisherFactory(connection);
publisher$.next(publisher);
// Clean-up callback
return Promise.resolve(async (): Promise<void> => {
await publisher.stopPublishing();
await publisher.stopTracks();
});
}
});
// Use reconcile here to not run concurrent createAndSetupTracks calls
// `tracks$` will update once they are ready.
scope.reconcile(
scope.behavior(
combineLatest([
publisher$ /*, tracks$*/,
from(trackStartRequested.promise),
]),
null,
),
async (valueIfReady) => {
if (!valueIfReady) return;
const [publisher] = valueIfReady;
if (publisher) {
await publisher.createAndSetupTracks().catch((e) => logger.error(e));
}
},
);
// Based on `connectRequested$` we start publishing tracks. (once they are there!)
scope.reconcile(
scope.behavior(combineLatest([publisher$, joinAndPublishRequested$])),
async ([publisher, shouldJoinAndPublish]) => {
// Get the current publishing state to avoid redundant calls.
const isPublishing = publisher?.shouldPublish === true;
if (shouldJoinAndPublish && !isPublishing) {
try {
await publisher?.startPublishing();
} catch (error) {
const message =
error instanceof Error ? error.message : String(error);
setPublishError(new FailToStartLivekitConnection(message));
}
} else if (isPublishing) {
try {
await publisher?.stopPublishing();
} catch (error) {
setPublishError(new UnknownCallError(error as Error));
}
}
},
);
// STATE COMPUTATION
// These are non fatal since we can join a room and concume media even though publishing failed.
const publishError$ = new BehaviorSubject<ElementCallError | null>(null);
const setPublishError = (e: ElementCallError): void => {
if (publishError$.value !== null) {
logger.error("Multiple Media Errors:", e);
} else {
publishError$.next(e);
}
};
const fatalTransportError$ = new BehaviorSubject<ElementCallError | null>(
null,
);
const setTransportError = (e: ElementCallError): void => {
if (fatalTransportError$.value !== null) {
logger.error("Multiple Transport Errors:", e);
} else {
fatalTransportError$.next(e);
}
};
const localConnectionState$ = localConnection$.pipe(
switchMap((connection) => (connection ? connection.state$ : of(null))),
);
const mediaState$: Behavior<LocalMemberMediaState> = scope.behavior(
combineLatest([
localConnectionState$,
localTransport$,
joinAndPublishRequested$,
from(trackStartRequested.promise).pipe(
map(() => true),
startWith(false),
),
]).pipe(
map(
([
localConnectionState,
localTransport,
shouldPublish,
shouldStartTracks,
]) => {
if (!localTransport) return null;
const trackState: TrackState = shouldStartTracks
? TrackState.Ready
: TrackState.WaitingForUser;
if (
localConnectionState !== ConnectionState.LivekitConnected ||
trackState !== TrackState.Ready
)
return {
connection: localConnectionState,
tracks: trackState,
};
if (!shouldPublish) return PublishState.WaitingForUser;
// if (!publishing) return PublishState.Starting;
return PublishState.Publishing;
},
),
distinctUntilChanged(deepCompare),
),
);
const fatalMatrixError$ = new BehaviorSubject<ElementCallError | null>(null);
const setMatrixError = (e: ElementCallError): void => {
if (fatalMatrixError$.value !== null) {
logger.error("Multiple Matrix Errors:", e);
} else {
fatalMatrixError$.next(e);
}
};
const localMemberState$ = scope.behavior<LocalMemberState>(
combineLatest([
mediaState$,
homeserverConnected.rtsSession$,
fatalMatrixError$,
fatalTransportError$,
publishError$,
]).pipe(
map(
([
mediaState,
rtcSessionStatus,
fatalMatrixError,
fatalTransportError,
publishError,
]) => {
if (fatalTransportError !== null) return fatalTransportError;
// `mediaState` will be 'null' until the transport/connection appears.
if (mediaState && rtcSessionStatus)
return {
matrix: fatalMatrixError ?? rtcSessionStatus,
media: publishError ?? mediaState,
};
return TransportState.Waiting;
},
),
),
);
/**
* Whether we are "fully" connected to the call. Accounts for both the
* connection to the MatrixRTC session and the LiveKit publish connection.
*/
const matrixAndLivekitConnected$ = scope.behavior(
and$(
homeserverConnected.combined$,
localConnectionState$.pipe(
map((state) => state === ConnectionState.LivekitConnected),
),
).pipe(
tap((v) => logger.debug("livekit+matrix: Connected state changed", v)),
),
);
/**
* Whether we should tell the user that we're reconnecting to the call.
*/
const reconnecting$ = scope.behavior(
matrixAndLivekitConnected$.pipe(
pairwise(),
map(([prev, current]) => prev === true && current === false),
),
false,
);
// inform the widget about the connect and disconnect intent from the user.
scope
.behavior(joinAndPublishRequested$.pipe(pairwise(), scope.bind()), [
undefined,
joinAndPublishRequested$.value,
])
.subscribe(([prev, current]) => {
if (!widget) return;
// JOIN prev=false (was left) => current-true (now joiend)
if (!prev && current) {
widget.api.transport
.send(ElementWidgetActions.JoinCall, {})
.catch((e) => {
logger.error("Failed to send join action", e);
});
}
// LEAVE prev=false (was joined) => current-true (now left)
if (prev && !current) {
widget.api.transport
.send(ElementWidgetActions.HangupCall, {})
.catch((e) => {
logger.error("Failed to send hangup action", e);
});
}
});
combineLatest([muteStates.video.enabled$, homeserverConnected.combined$])
.pipe(scope.bind())
.subscribe(([videoEnabled, connected]) => {
if (!connected) return;
void matrixRTCSession.updateCallIntent(videoEnabled ? "video" : "audio");
});
// Keep matrix rtc session in sync with localTransport$, connectRequested$
scope.reconcile(
scope.behavior(combineLatest([localTransport$, joinAndPublishRequested$])),
async ([transport, shouldConnect]) => {
if (!transport) return;
// if shouldConnect=false we will do the disconnect as the cleanup from the previous reconcile iteration.
if (!shouldConnect) return;
try {
joinMatrixRTC(transport);
} catch (error) {
logger.error("Error entering RTC session", error);
if (error instanceof Error)
setMatrixError(new MembershipManagerError(error));
}
return Promise.resolve(async (): Promise<void> => {
try {
// TODO Update matrixRTCSession to allow udpating the transport without leaving the session!
await matrixRTCSession.leaveRoomSession(1000);
} catch (e) {
logger.error("Error leaving RTC session", e);
}
});
},
);
const participant$ = scope.behavior(
localConnection$.pipe(map((c) => c?.livekitRoom?.localParticipant ?? null)),
);
// Pause upstream of all local media tracks when we're disconnected from
// MatrixRTC, because it can be an unpleasant surprise for the app to say
// 'reconnecting' and yet still be transmitting your media to others.
// We use matrixConnected$ rather than reconnecting$ because we want to
// pause tracks during the initial joining sequence too until we're sure
// that our own media is displayed on screen.
// TODO refactor this based no livekitState$
combineLatest([participant$, homeserverConnected.combined$])
.pipe(scope.bind())
.subscribe(([participant, connected]) => {
if (!participant) return;
const publications = participant.trackPublications.values();
if (connected) {
for (const p of publications) {
if (p.track?.isUpstreamPaused === true) {
const kind = p.track.kind;
logger.info(
`Resuming ${kind} track (MatrixRTC connection present)`,
);
p.track
.resumeUpstream()
.catch((e) =>
logger.error(
`Failed to resume ${kind} track after MatrixRTC reconnection`,
e,
),
);
}
}
} else {
for (const p of publications) {
if (p.track?.isUpstreamPaused === false) {
const kind = p.track.kind;
logger.info(
`Pausing ${kind} track (uncertain MatrixRTC connection)`,
);
p.track
.pauseUpstream()
.catch((e) =>
logger.error(
`Failed to pause ${kind} track after entering uncertain MatrixRTC connection`,
e,
),
);
}
}
}
});
/**
* Whether the user is currently sharing their screen.
*/
const sharingScreen$ = scope.behavior(
participant$.pipe(
switchMap((p) => (p !== null ? observeSharingScreen$(p) : of(false))),
),
);
let toggleScreenSharing: (() => void) | null = null;
if (
"getDisplayMedia" in (navigator.mediaDevices ?? {}) &&
!getUrlParams().hideScreensharing
) {
toggleScreenSharing = (): void => {
const screenshareSettings: ScreenShareCaptureOptions = {
audio: true,
selfBrowserSurface: "include",
surfaceSwitching: "include",
systemAudio: "include",
};
const targetScreenshareState = !sharingScreen$.value;
logger.info(
`toggleScreenSharing called. Switching ${
targetScreenshareState ? "On" : "Off"
}`,
);
// If a connection is ready, toggle screen sharing.
// We deliberately do nothing in the case of a null connection because
// it looks nice for the call control buttons to all become available
// at once upon joining the call, rather than introducing a disabled
// state. The user can just click again.
// We also allow screen sharing to be toggled even if the connection
// is still initializing or publishing tracks, because there's no
// technical reason to disallow this. LiveKit will publish if it can.
participant$.value
?.setScreenShareEnabled(targetScreenshareState, screenshareSettings)
.catch(logger.error);
};
}
return {
startTracks,
requestJoinAndPublish,
requestDisconnect,
localMemberState$,
participant$,
reconnecting$,
connected$: matrixAndLivekitConnected$,
disconnected$: scope.behavior(
homeserverConnected.rtsSession$.pipe(
map((state) => state === RTCSessionStatus.Disconnected),
),
),
sharingScreen$,
toggleScreenSharing,
connection$: localConnection$,
};
};
export function observeSharingScreen$(p: Participant): Observable<boolean> {
return observeParticipantEvents(
p,
ParticipantEvent.TrackPublished,
ParticipantEvent.TrackUnpublished,
ParticipantEvent.LocalTrackPublished,
ParticipantEvent.LocalTrackUnpublished,
).pipe(map((p) => p.isScreenShareEnabled));
}
interface EnterRTCSessionOptions {
encryptMedia: boolean;
matrixRTCMode: MatrixRTCMode;
}
/**
* Does the necessary steps to enter the RTC session on the matrix side:
* - Preparing the membership info (FOCUS to use, options)
* - Sends the matrix event to join the call, and starts the membership manager:
* - Delay events management
* - Handles retries (fails only after several attempts)
*
* @param rtcSession - The MatrixRTCSession to join.
* @param transport - The LivekitTransport to use for this session.
* @param options - Options for entering the RTC session.
* @param options.encryptMedia - Whether to encrypt media.
* @param options.matrixRTCMode - The Matrix RTC mode to use.
* @throws If the widget could not send ElementWidgetActions.JoinCall action.
*/
// Exported for unit testing
export function enterRTCSession(
rtcSession: MatrixRTCSession,
transport: LivekitTransport,
{ encryptMedia, matrixRTCMode }: EnterRTCSessionOptions,
): void {
PosthogAnalytics.instance.eventCallEnded.cacheStartCall(new Date());
PosthogAnalytics.instance.eventCallStarted.track(rtcSession.room.roomId);
// This must be called before we start trying to join the call, as we need to
// have started tracking by the time calls start getting created.
// groupCallOTelMembership?.onJoinCall();
const { features, matrix_rtc_session: matrixRtcSessionConfig } = Config.get();
const useDeviceSessionMemberEvents =
features?.feature_use_device_session_member_events;
const { sendNotificationType: notificationType, callIntent } = getUrlParams();
const multiSFU = matrixRTCMode !== MatrixRTCMode.Legacy;
// Multi-sfu does not need a preferred foci list. just the focus that is actually used.
// TODO where/how do we track errors originating from the ongoing rtcSession?
rtcSession.joinRoomSession(
multiSFU ? [] : [transport],
multiSFU ? transport : undefined,
{
notificationType,
callIntent,
manageMediaKeys: encryptMedia,
...(useDeviceSessionMemberEvents !== undefined && {
useLegacyMemberEvents: !useDeviceSessionMemberEvents,
}),
delayedLeaveEventRestartMs:
matrixRtcSessionConfig?.delayed_leave_event_restart_ms,
delayedLeaveEventDelayMs:
matrixRtcSessionConfig?.delayed_leave_event_delay_ms,
delayedLeaveEventRestartLocalTimeoutMs:
matrixRtcSessionConfig?.delayed_leave_event_restart_local_timeout_ms,
networkErrorRetryMs: matrixRtcSessionConfig?.network_error_retry_ms,
makeKeyDelay: matrixRtcSessionConfig?.wait_for_key_rotation_ms,
membershipEventExpiryMs:
matrixRtcSessionConfig?.membership_event_expiry_ms,
useExperimentalToDeviceTransport: true,
unstableSendStickyEvents: matrixRTCMode === MatrixRTCMode.Matrix_2_0,
},
);
}

229
src/state/CallViewModel/localMember/LocalMembership.test.ts
View File

@@ -1,229 +0,0 @@
/*
Copyright 2025 Element Creations Ltd.
Copyright 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
type LivekitTransport,
type MatrixRTCSession,
} from "matrix-js-sdk/lib/matrixrtc";
import { describe, expect, it, vi } from "vitest";
import { AutoDiscovery } from "matrix-js-sdk/lib/autodiscovery";
import { map } from "rxjs";
import { logger } from "matrix-js-sdk/lib/logger";
import { MatrixRTCMode } from "../../../settings/settings";
import {
mockConfig,
mockMuteStates,
withTestScheduler,
} from "../../../utils/test";
import {
createLocalMembership$,
enterRTCSession,
LivekitState,
} from "./LocalMembership";
import { MatrixRTCTransportMissingError } from "../../../utils/errors";
import { Epoch } from "../../ObservableScope";
import { constant } from "../../Behavior";
import { ConnectionManagerData } from "../remoteMembers/ConnectionManager";
import { type Publisher } from "./Publisher";
const MATRIX_RTC_MODE = MatrixRTCMode.Legacy;
const getUrlParams = vi.hoisted(() => vi.fn(() => ({})));
vi.mock("../../../UrlParams", () => ({ getUrlParams }));
describe("LocalMembership", () => {
describe("enterRTCSession", () => {
it("It joins the correct Session", async () => {
const focusFromOlderMembership = {
type: "livekit",
livekit_service_url: "http://my-oldest-member-service-url.com",
livekit_alias: "my-oldest-member-service-alias",
};
const focusConfigFromWellKnown = {
type: "livekit",
livekit_service_url: "http://my-well-known-service-url.com",
};
const focusConfigFromWellKnown2 = {
type: "livekit",
livekit_service_url: "http://my-well-known-service-url2.com",
};
const clientWellKnown = {
"org.matrix.msc4143.rtc_foci": [
focusConfigFromWellKnown,
focusConfigFromWellKnown2,
],
};
mockConfig({
livekit: { livekit_service_url: "http://my-default-service-url.com" },
});
vi.spyOn(AutoDiscovery, "getRawClientConfig").mockImplementation(
async (domain) => {
if (domain === "example.org") {
return Promise.resolve(clientWellKnown);
}
return Promise.resolve({});
},
);
const mockedSession = vi.mocked({
room: {
roomId: "roomId",
client: {
getDomain: vi.fn().mockReturnValue("example.org"),
getOpenIdToken: vi.fn().mockResolvedValue({
access_token: "ACCCESS_TOKEN",
token_type: "Bearer",
matrix_server_name: "localhost",
expires_in: 10000,
}),
},
},
memberships: [],
getFocusInUse: vi.fn().mockReturnValue(focusFromOlderMembership),
getOldestMembership: vi.fn().mockReturnValue({
getPreferredFoci: vi.fn().mockReturnValue([focusFromOlderMembership]),
}),
joinRoomSession: vi.fn(),
}) as unknown as MatrixRTCSession;
await enterRTCSession(
mockedSession,
{
livekit_alias: "roomId",
livekit_service_url: "http://my-well-known-service-url.com",
type: "livekit",
},
{
encryptMedia: true,
matrixRTCMode: MATRIX_RTC_MODE,
},
);
expect(mockedSession.joinRoomSession).toHaveBeenLastCalledWith(
[
{
livekit_alias: "roomId",
livekit_service_url: "http://my-well-known-service-url.com",
type: "livekit",
},
],
undefined,
expect.objectContaining({
manageMediaKeys: true,
useLegacyMemberEvents: false,
}),
);
});
it("It should not fail with configuration error if homeserver config has livekit url but not fallback", async () => {
mockConfig({});
vi.spyOn(AutoDiscovery, "getRawClientConfig").mockResolvedValue({
"org.matrix.msc4143.rtc_foci": [
{
type: "livekit",
livekit_service_url: "http://my-well-known-service-url.com",
},
],
});
const mockedSession = vi.mocked({
room: {
roomId: "roomId",
client: {
getDomain: vi.fn().mockReturnValue("example.org"),
getOpenIdToken: vi.fn().mockResolvedValue({
access_token: "ACCCESS_TOKEN",
token_type: "Bearer",
matrix_server_name: "localhost",
expires_in: 10000,
}),
},
},
memberships: [],
getFocusInUse: vi.fn(),
joinRoomSession: vi.fn(),
}) as unknown as MatrixRTCSession;
await enterRTCSession(
mockedSession,
{
livekit_alias: "roomId",
livekit_service_url: "http://my-well-known-service-url.com",
type: "livekit",
},
{
encryptMedia: true,
matrixRTCMode: MATRIX_RTC_MODE,
},
);
});
});
const defaultCreateLocalMemberValues = {
options: constant({
encryptMedia: false,
matrixRTCMode: MatrixRTCMode.Matrix_2_0,
}),
matrixRTCSession: {
updateCallIntent: () => {},
leaveRoomSession: () => {},
} as unknown as MatrixRTCSession,
muteStates: mockMuteStates(),
isHomeserverConnected: constant(true),
trackProcessorState$: constant({
supported: false,
processor: undefined,
}),
logger: logger,
createPublisherFactory: (): Publisher => ({}) as unknown as Publisher,
joinMatrixRTC: async (): Promise<void> => {},
homeserverConnected$: constant(true),
};
it("throws error on missing RTC config error", () => {
withTestScheduler(({ scope, hot, expectObservable }) => {
const goodTransport = {
livekit_service_url: "other",
} as LivekitTransport;
const localTransport$ = scope.behavior<LivekitTransport>(
hot("1ms #", {}, new MatrixRTCTransportMissingError("domain.com")),
goodTransport,
);
const mockConnectionManager = {
transports$: scope.behavior(
localTransport$.pipe(map((t) => new Epoch([t]))),
),
connectionManagerData$: constant(
new Epoch(new ConnectionManagerData()),
),
};
const localMembership = createLocalMembership$({
scope,
...defaultCreateLocalMemberValues,
connectionManager: mockConnectionManager,
localTransport$,
});
expectObservable(localMembership.connectionState.livekit$).toBe("ne", {
n: { state: LivekitState.Uninitialized },
e: {
state: LivekitState.Error,
error: expect.toSatisfy(
(e) => e instanceof MatrixRTCTransportMissingError,
),
},
});
});
});
});

629
src/state/CallViewModel/localMember/LocalMembership.ts
View File

@@ -1,629 +0,0 @@
/*
Copyright 2025 Element Creations Ltd.
SPDX-License-IdFentifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
type LocalTrack,
type Participant,
ParticipantEvent,
type LocalParticipant,
type ScreenShareCaptureOptions,
} from "livekit-client";
import { observeParticipantEvents } from "@livekit/components-core";
import {
type LivekitTransport,
type MatrixRTCSession,
} from "matrix-js-sdk/lib/matrixrtc";
import {
BehaviorSubject,
catchError,
combineLatest,
distinctUntilChanged,
map,
type Observable,
of,
scan,
switchMap,
tap,
} from "rxjs";
import { type Logger } from "matrix-js-sdk/lib/logger";
import { type Behavior } from "../../Behavior";
import { type IConnectionManager } from "../remoteMembers/ConnectionManager";
import { ObservableScope } from "../../ObservableScope";
import { type Publisher } from "./Publisher";
import { type MuteStates } from "../../MuteStates";
import { and$ } from "../../../utils/observable";
import { ElementCallError, UnknownCallError } from "../../../utils/errors";
import { ElementWidgetActions, widget } from "../../../widget";
import { getUrlParams } from "../../../UrlParams.ts";
import { PosthogAnalytics } from "../../../analytics/PosthogAnalytics.ts";
import { MatrixRTCMode } from "../../../settings/settings.ts";
import { Config } from "../../../config/Config.ts";
import {
type Connection,
type ConnectionState,
} from "../remoteMembers/Connection.ts";
export enum LivekitState {
Uninitialized = "uninitialized",
Connecting = "connecting",
Connected = "connected",
Error = "error",
Disconnected = "disconnected",
Disconnecting = "disconnecting",
}
type LocalMemberLivekitState =
| { state: LivekitState.Error; error: ElementCallError }
| { state: LivekitState.Connected }
| { state: LivekitState.Connecting }
| { state: LivekitState.Uninitialized }
| { state: LivekitState.Disconnected }
| { state: LivekitState.Disconnecting };
export enum MatrixState {
Connected = "connected",
Disconnected = "disconnected",
Connecting = "connecting",
Error = "Error",
}
type LocalMemberMatrixState =
| { state: MatrixState.Connected }
| { state: MatrixState.Connecting }
| { state: MatrixState.Disconnected }
| { state: MatrixState.Error; error: Error };
export interface LocalMemberConnectionState {
livekit$: Behavior<LocalMemberLivekitState>;
matrix$: Behavior<LocalMemberMatrixState>;
}
/*
* - get well known
* - get oldest membership
* - get transport to use
* - get openId + jwt token
* - wait for createTrack() call
* - create tracks
* - wait for join() call
* - Publisher.publishTracks()
* - send join state/sticky event
*/
interface Props {
// TODO add a comment into some code style readme or file header callviewmodel
// that the inputs for those createSomething$() functions should NOT contain any js-sdk objectes
scope: ObservableScope;
muteStates: MuteStates;
connectionManager: IConnectionManager;
createPublisherFactory: (connection: Connection) => Publisher;
joinMatrixRTC: (trasnport: LivekitTransport) => Promise<void>;
homeserverConnected$: Behavior<boolean>;
localTransport$: Behavior<LivekitTransport | null>;
matrixRTCSession: Pick<
MatrixRTCSession,
"updateCallIntent" | "leaveRoomSession"
>;
logger: Logger;
}
/**
* This class is responsible for managing the own membership in a room.
* We want
* - a publisher
* -
* @param param0
* @returns
* - publisher: The handle to create tracks and publish them to the room.
* - connected$: the current connection state. Including matrix server and livekit server connection. (only considering the livekit server we are using for our own media publication)
* - transport$: the transport object the ownMembership$ ended up using.
* - connectionState: the current connection state. Including matrix server and livekit server connection.
* - sharingScreen$: Whether we are sharing our screen. `undefined` if we cannot share the screen.
*/
export const createLocalMembership$ = ({
scope,
connectionManager,
localTransport$: localTransportCanThrow$,
homeserverConnected$,
createPublisherFactory,
joinMatrixRTC,
logger: parentLogger,
muteStates,
matrixRTCSession,
}: Props): {
requestConnect: () => LocalMemberConnectionState;
startTracks: () => Behavior<LocalTrack[]>;
requestDisconnect: () => Observable<LocalMemberLivekitState> | null;
connectionState: LocalMemberConnectionState;
sharingScreen$: Behavior<boolean>;
/**
* Callback to toggle screen sharing. If null, screen sharing is not possible.
*/
toggleScreenSharing: (() => void) | null;
participant$: Behavior<LocalParticipant | null>;
connection$: Behavior<Connection | null>;
homeserverConnected$: Behavior<boolean>;
// deprecated fields
/** @deprecated use state instead*/
connected$: Behavior<boolean>;
// this needs to be discussed
/** @deprecated use state instead*/
reconnecting$: Behavior<boolean>;
} => {
const logger = parentLogger.getChild("[LocalMembership]");
logger.debug(`Creating local membership..`);
const state = {
livekit$: new BehaviorSubject<LocalMemberLivekitState>({
state: LivekitState.Uninitialized,
}),
matrix$: new BehaviorSubject<LocalMemberMatrixState>({
state: MatrixState.Disconnected,
}),
};
// This should be used in a combineLatest with publisher$ to connect.
// to make it possible to call startTracks before the preferredTransport$ has resolved.
const trackStartRequested$ = new BehaviorSubject(false);
// This should be used in a combineLatest with publisher$ to connect.
// to make it possible to call startTracks before the preferredTransport$ has resolved.
const connectRequested$ = new BehaviorSubject(false);
// This should be used in a combineLatest with publisher$ to connect.
const tracks$ = new BehaviorSubject<LocalTrack[]>([]);
// unwrap the local transport and set the state of the LocalMembership to error in case the transport is an error.
const localTransport$ = scope.behavior(
localTransportCanThrow$.pipe(
catchError((e: unknown) => {
let error: ElementCallError;
if (e instanceof ElementCallError) {
error = e;
} else {
error = new UnknownCallError(
e instanceof Error
? e
: new Error("Unknown error from localTransport"),
);
}
state.livekit$.next({ state: LivekitState.Error, error });
return of(null);
}),
),
);
// Drop Epoch data here since we will not combine this anymore
const localConnection$ = scope.behavior(
combineLatest([
connectionManager.connectionManagerData$,
localTransport$,
]).pipe(
map(([connectionData, localTransport]) => {
if (localTransport === null) {
return null;
}
return connectionData.value.getConnectionForTransport(localTransport);
}),
tap((connection) => {
logger.info(
`Local connection updated: ${connection?.transport?.livekit_service_url}`,
);
}),
),
);
// /**
// * Whether we are "fully" connected to the call. Accounts for both the
// * connection to the MatrixRTC session and the LiveKit publish connection.
// */
// // TODO use this in combination with the MemberState.
const connected$ = scope.behavior(
and$(
homeserverConnected$,
localConnection$.pipe(
switchMap((c) =>
c
? c.state$.pipe(map((state) => state.state === "ConnectedToLkRoom"))
: of(false),
),
),
),
);
const publisher$ = new BehaviorSubject<Publisher | null>(null);
localConnection$.pipe(scope.bind()).subscribe((connection) => {
if (connection !== null && publisher$.value === null) {
// TODO looks strange to not change publisher if connection changes.
// @toger5 will take care of this!
publisher$.next(createPublisherFactory(connection));
}
});
// const mutestate= publisher$.pipe(switchMap((publisher) => {
// return publisher.muteState$
// });
combineLatest([publisher$, trackStartRequested$]).subscribe(
([publisher, shouldStartTracks]) => {
if (publisher && shouldStartTracks) {
publisher
.createAndSetupTracks()
.then((tracks) => {
tracks$.next(tracks);
})
.catch((error) => {
logger.error("Error creating tracks:", error);
});
}
},
);
// MATRIX RELATED
// /**
// * Whether we should tell the user that we're reconnecting to the call.
// */
// DISCUSSION is there a better way to do this?
// sth that is more deriectly implied from the membership manager of the js sdk. (fromEvent(matrixRTCSession, Reconnecting)) ??? or similar
const reconnecting$ = scope.behavior(
connected$.pipe(
// We are reconnecting if we previously had some successful initial
// connection but are now disconnected
scan(
({ connectedPreviously }, connectedNow) => ({
connectedPreviously: connectedPreviously || connectedNow,
reconnecting: connectedPreviously && !connectedNow,
}),
{ connectedPreviously: false, reconnecting: false },
),
map(({ reconnecting }) => reconnecting),
),
);
const startTracks = (): Behavior<LocalTrack[]> => {
trackStartRequested$.next(true);
return tracks$;
};
combineLatest([publisher$, tracks$]).subscribe(([publisher, tracks]) => {
if (
tracks.length === 0 ||
// change this to !== Publishing
state.livekit$.value.state !== LivekitState.Uninitialized
) {
return;
}
state.livekit$.next({ state: LivekitState.Connecting });
publisher
?.startPublishing()
.then(() => {
state.livekit$.next({ state: LivekitState.Connected });
})
.catch((error) => {
state.livekit$.next({ state: LivekitState.Error, error });
});
});
combineLatest([localTransport$, connectRequested$]).subscribe(
// TODO reconnect when transport changes => create test.
([transport, connectRequested]) => {
if (
transport === null ||
!connectRequested ||
state.matrix$.value.state !== MatrixState.Disconnected
) {
logger.info(
"Not yet connecting because: ",
"transport === null:",
transport === null,
"!connectRequested:",
!connectRequested,
"state.matrix$.value.state !== MatrixState.Disconnected:",
state.matrix$.value.state !== MatrixState.Disconnected,
);
return;
}
state.matrix$.next({ state: MatrixState.Connecting });
logger.info("Matrix State connecting");
joinMatrixRTC(transport).catch((error) => {
logger.error(error);
state.matrix$.next({ state: MatrixState.Error, error });
});
},
);
// TODO add this and update `state.matrix$` based on it.
// useTypedEventEmitter(
// rtcSession,
// MatrixRTCSessionEvent.MembershipManagerError,
// (error) => setExternalError(new ConnectionLostError()),
// );
const requestConnect = (): LocalMemberConnectionState => {
trackStartRequested$.next(true);
connectRequested$.next(true);
return state;
};
const requestDisconnect = (): Behavior<LocalMemberLivekitState> | null => {
if (state.livekit$.value.state !== LivekitState.Connected) return null;
state.livekit$.next({ state: LivekitState.Disconnecting });
combineLatest([publisher$, tracks$], (publisher, tracks) => {
publisher
?.stopPublishing()
.then(() => {
tracks.forEach((track) => track.stop());
state.livekit$.next({ state: LivekitState.Disconnected });
})
.catch((error) => {
state.livekit$.next({ state: LivekitState.Error, error });
});
});
return state.livekit$;
};
// Pause upstream of all local media tracks when we're disconnected from
// MatrixRTC, because it can be an unpleasant surprise for the app to say
// 'reconnecting' and yet still be transmitting your media to others.
// We use matrixConnected$ rather than reconnecting$ because we want to
// pause tracks during the initial joining sequence too until we're sure
// that our own media is displayed on screen.
combineLatest([localConnection$, homeserverConnected$])
.pipe(scope.bind())
.subscribe(([connection, connected]) => {
if (connection?.state$.value.state !== "ConnectedToLkRoom") return;
const publications =
connection.livekitRoom.localParticipant.trackPublications.values();
if (connected) {
for (const p of publications) {
if (p.track?.isUpstreamPaused === true) {
const kind = p.track.kind;
logger.info(
`Resuming ${kind} track (MatrixRTC connection present)`,
);
p.track
.resumeUpstream()
.catch((e) =>
logger.error(
`Failed to resume ${kind} track after MatrixRTC reconnection`,
e,
),
);
}
}
} else {
for (const p of publications) {
if (p.track?.isUpstreamPaused === false) {
const kind = p.track.kind;
logger.info(
`Pausing ${kind} track (uncertain MatrixRTC connection)`,
);
p.track
.pauseUpstream()
.catch((e) =>
logger.error(
`Failed to pause ${kind} track after entering uncertain MatrixRTC connection`,
e,
),
);
}
}
}
});
// TODO: Refactor updateCallIntent to sth like this:
// combineLatest([muteStates.video.enabled$,localTransport$, state.matrix$]).pipe(map(()=>{
// matrixRTCSession.updateCallIntent(videoEnabled ? "video" : "audio"),
// }))
//
// TODO I do not fully understand what this does.
// Is it needed?
// Is this at the right place?
// Can this be simplified?
// Start and stop session membership as needed
// Discussed in statndup -> It seems we can remove this (there is another call to enterRTCSession in this file)
// MAKE SURE TO UNDERSTAND why reconcile is needed and what is potentially missing from the alternative enterRTCSession block.
// @toger5 will try to take care of this.
scope.reconcile(localTransport$, async (transport) => {
if (transport !== null && transport !== undefined) {
try {
state.matrix$.next({ state: MatrixState.Connecting });
await joinMatrixRTC(transport);
} catch (e) {
logger.error("Error entering RTC session", e);
}
// Update our member event when our mute state changes.
const intentScope = new ObservableScope();
intentScope.reconcile(muteStates.video.enabled$, async (videoEnabled) =>
matrixRTCSession.updateCallIntent(videoEnabled ? "video" : "audio"),
);
return async (): Promise<void> => {
intentScope.end();
// Only sends Matrix leave event. The LiveKit session will disconnect
// as soon as either the stopConnection$ handler above gets to it or
// the view model is destroyed.
try {
await matrixRTCSession.leaveRoomSession();
} catch (e) {
logger.error("Error leaving RTC session", e);
}
try {
await widget?.api.transport.send(ElementWidgetActions.HangupCall, {});
} catch (e) {
logger.error("Failed to send hangup action", e);
}
};
}
});
localConnection$
.pipe(
distinctUntilChanged(),
switchMap((c) =>
c === null ? of({ state: "Initialized" } as ConnectionState) : c.state$,
),
map((s) => {
logger.trace(`Local connection state update: ${s.state}`);
if (s.state == "FailedToStart") {
return s.error instanceof ElementCallError
? s.error
: new UnknownCallError(s.error);
}
}),
scope.bind(),
)
.subscribe((error) => {
if (error !== undefined)
state.livekit$.next({ state: LivekitState.Error, error });
});
/**
* Whether the user is currently sharing their screen.
*/
const sharingScreen$ = scope.behavior(
localConnection$.pipe(
switchMap((c) =>
c !== null
? observeSharingScreen$(c.livekitRoom.localParticipant)
: of(false),
),
),
);
let toggleScreenSharing = null;
if (
"getDisplayMedia" in (navigator.mediaDevices ?? {}) &&
!getUrlParams().hideScreensharing
) {
toggleScreenSharing = (): void => {
const screenshareSettings: ScreenShareCaptureOptions = {
audio: true,
selfBrowserSurface: "include",
surfaceSwitching: "include",
systemAudio: "include",
};
const targetScreenshareState = !sharingScreen$.value;
logger.info(
`toggleScreenSharing called. Switching ${
targetScreenshareState ? "On" : "Off"
}`,
);
// If a connection is ready, toggle screen sharing.
// We deliberately do nothing in the case of a null connection because
// it looks nice for the call control buttons to all become available
// at once upon joining the call, rather than introducing a disabled
// state. The user can just click again.
// We also allow screen sharing to be toggled even if the connection
// is still initializing or publishing tracks, because there's no
// technical reason to disallow this. LiveKit will publish if it can.
localConnection$.value?.livekitRoom.localParticipant
.setScreenShareEnabled(targetScreenshareState, screenshareSettings)
.catch(logger.error);
};
}
const participant$ = scope.behavior(
localConnection$.pipe(map((c) => c?.livekitRoom?.localParticipant ?? null)),
);
return {
startTracks,
requestConnect,
requestDisconnect,
connectionState: state,
homeserverConnected$,
connected$,
reconnecting$,
sharingScreen$,
toggleScreenSharing,
participant$,
connection$: localConnection$,
};
};
export function observeSharingScreen$(p: Participant): Observable<boolean> {
return observeParticipantEvents(
p,
ParticipantEvent.TrackPublished,
ParticipantEvent.TrackUnpublished,
ParticipantEvent.LocalTrackPublished,
ParticipantEvent.LocalTrackUnpublished,
).pipe(map((p) => p.isScreenShareEnabled));
}
interface EnterRTCSessionOptions {
encryptMedia: boolean;
matrixRTCMode: MatrixRTCMode;
}
/**
* Does the necessary steps to enter the RTC session on the matrix side:
* - Preparing the membership info (FOCUS to use, options)
* - Sends the matrix event to join the call, and starts the membership manager:
* - Delay events management
* - Handles retries (fails only after several attempts)
*
* @param rtcSession
* @param transport
* @param options
* @throws If the widget could not send ElementWidgetActions.JoinCall action.
*/
// Exported for unit testing
export async function enterRTCSession(
rtcSession: MatrixRTCSession,
transport: LivekitTransport,
{ encryptMedia, matrixRTCMode }: EnterRTCSessionOptions,
): Promise<void> {
PosthogAnalytics.instance.eventCallEnded.cacheStartCall(new Date());
PosthogAnalytics.instance.eventCallStarted.track(rtcSession.room.roomId);
// This must be called before we start trying to join the call, as we need to
// have started tracking by the time calls start getting created.
// groupCallOTelMembership?.onJoinCall();
const { features, matrix_rtc_session: matrixRtcSessionConfig } = Config.get();
const useDeviceSessionMemberEvents =
features?.feature_use_device_session_member_events;
const { sendNotificationType: notificationType, callIntent } = getUrlParams();
const multiSFU = matrixRTCMode !== MatrixRTCMode.Legacy;
// Multi-sfu does not need a preferred foci list. just the focus that is actually used.
// TODO where/how do we track errors originating from the ongoing rtcSession?
rtcSession.joinRoomSession(
multiSFU ? [] : [transport],
multiSFU ? transport : undefined,
{
notificationType,
callIntent,
manageMediaKeys: encryptMedia,
...(useDeviceSessionMemberEvents !== undefined && {
useLegacyMemberEvents: !useDeviceSessionMemberEvents,
}),
delayedLeaveEventRestartMs:
matrixRtcSessionConfig?.delayed_leave_event_restart_ms,
delayedLeaveEventDelayMs:
matrixRtcSessionConfig?.delayed_leave_event_delay_ms,
delayedLeaveEventRestartLocalTimeoutMs:
matrixRtcSessionConfig?.delayed_leave_event_restart_local_timeout_ms,
networkErrorRetryMs: matrixRtcSessionConfig?.network_error_retry_ms,
makeKeyDelay: matrixRtcSessionConfig?.wait_for_key_rotation_ms,
membershipEventExpiryMs:
matrixRtcSessionConfig?.membership_event_expiry_ms,
useExperimentalToDeviceTransport: true,
unstableSendStickyEvents: matrixRTCMode === MatrixRTCMode.Matrix_2_0,
},
);
if (widget) {
await widget.api.transport.send(ElementWidgetActions.JoinCall, {});
}
}

210
src/state/CallViewModel/localMember/LocalTransport.test.ts
View File

@@ -5,8 +5,18 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import {
afterEach,
beforeEach,
describe,
expect,
it,
type MockedObject,
vi,
} from "vitest";
import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc";
import { BehaviorSubject, lastValueFrom } from "rxjs";
import fetchMock from "fetch-mock";
import { mockConfig, flushPromises } from "../../../utils/test";
import { createLocalTransport$ } from "./LocalTransport";
@@ -17,31 +27,22 @@ import {
FailToGetOpenIdToken,
} from "../../../utils/errors";
import * as openIDSFU from "../../../livekit/openIDSFU";
import { customLivekitUrl } from "../../../settings/settings";
import { testJWTToken } from "../../../utils/test-fixtures";
describe("LocalTransport", () => {
const openIdResponse: openIDSFU.SFUConfig = {
url: "https://lk.example.org",
jwt: testJWTToken,
livekitAlias: "!example_room_id",
livekitIdentity: "@lk_user:ABCDEF",
};
let scope: ObservableScope;
beforeEach(() => (scope = new ObservableScope()));
afterEach(() => scope.end());
it("throws if config is missing", async () => {
const localTransport$ = createLocalTransport$({
scope,
roomId: "!room:example.org",
useOldestMember$: constant(false),
memberships$: constant(new Epoch<CallMembership[]>([])),
client: {
getDomain: () => "",
// These won't be called in this error path but satisfy the type
getOpenIdToken: vi.fn(),
getDeviceId: vi.fn(),
},
});
await flushPromises();
expect(() => localTransport$.value).toThrow(
new MatrixRTCTransportMissingError(""),
);
beforeEach(() => {
scope = new ObservableScope();
});
afterEach(() => scope.end());
it("throws FailToGetOpenIdToken when OpenID fetch fails", async () => {
// Provide a valid config so makeTransportInternal resolves a transport
@@ -60,12 +61,14 @@ describe("LocalTransport", () => {
const errors: Error[] = [];
const localTransport$ = createLocalTransport$({
scope,
roomId: "!room:example.org",
roomId: "!example_room_id",
useOldestMember$: constant(false),
memberships$: constant(new Epoch<CallMembership[]>([])),
client: {
// Use empty domain to skip .well-known and use config directly
getDomain: () => "",
// eslint-disable-next-line @typescript-eslint/naming-convention
_unstable_getRTCTransports: async () => Promise.resolve([]),
getOpenIdToken: vi.fn(),
getDeviceId: vi.fn(),
},
@@ -83,12 +86,12 @@ describe("LocalTransport", () => {
expect(() => localTransport$.value).toThrow(expectedError);
});
it("emits preferred transport after OpenID resolves", async () => {
it("updates local transport when oldest member changes", async () => {
// Use config so transport discovery succeeds, but delay OpenID JWT fetch
mockConfig({
livekit: { livekit_service_url: "https://lk.example.org" },
});
const memberships$ = new BehaviorSubject(new Epoch([]));
const openIdResolver = Promise.withResolvers<openIDSFU.SFUConfig>();
vi.spyOn(openIDSFU, "getSFUConfigWithOpenID").mockReturnValue(
@@ -97,24 +100,171 @@ describe("LocalTransport", () => {
const localTransport$ = createLocalTransport$({
scope,
roomId: "!room:example.org",
useOldestMember$: constant(false),
memberships$: constant(new Epoch<CallMembership[]>([])),
roomId: "!example_room_id",
useOldestMember$: constant(true),
memberships$,
client: {
getDomain: () => "",
// eslint-disable-next-line @typescript-eslint/naming-convention
_unstable_getRTCTransports: async () => Promise.resolve([]),
getOpenIdToken: vi.fn(),
getDeviceId: vi.fn(),
},
});
openIdResolver.resolve?.({ url: "https://lk.example.org", jwt: "jwt" });
openIdResolver.resolve?.(openIdResponse);
expect(localTransport$.value).toBe(null);
await flushPromises();
// final
expect(localTransport$.value).toStrictEqual({
livekit_alias: "!room:example.org",
livekit_alias: "!example_room_id",
livekit_service_url: "https://lk.example.org",
type: "livekit",
});
});
type LocalTransportProps = Parameters<typeof createLocalTransport$>[0];
describe("transport configuration mechanisms", () => {
let localTransportOpts: LocalTransportProps & {
client: MockedObject<LocalTransportProps["client"]>;
};
let openIdResolver: PromiseWithResolvers<openIDSFU.SFUConfig>;
beforeEach(() => {
mockConfig({});
customLivekitUrl.setValue(customLivekitUrl.defaultValue);
localTransportOpts = {
scope,
roomId: "!example_room_id",
useOldestMember$: constant(false),
memberships$: constant(new Epoch<CallMembership[]>([])),
client: {
getDomain: vi.fn().mockReturnValue(""),
// eslint-disable-next-line @typescript-eslint/naming-convention
_unstable_getRTCTransports: vi.fn().mockResolvedValue([]),
getOpenIdToken: vi.fn(),
getDeviceId: vi.fn(),
},
};
openIdResolver = Promise.withResolvers<openIDSFU.SFUConfig>();
vi.spyOn(openIDSFU, "getSFUConfigWithOpenID").mockReturnValue(
openIdResolver.promise,
);
});
afterEach(() => {
fetchMock.reset();
});
it("supports getting transport via application config", async () => {
mockConfig({
livekit: { livekit_service_url: "https://lk.example.org" },
});
const localTransport$ = createLocalTransport$(localTransportOpts);
openIdResolver.resolve?.(openIdResponse);
expect(localTransport$.value).toBe(null);
await flushPromises();
expect(localTransport$.value).toStrictEqual({
livekit_alias: "!example_room_id",
livekit_service_url: "https://lk.example.org",
type: "livekit",
});
});
it("supports getting transport via user settings", async () => {
customLivekitUrl.setValue("https://lk.example.org");
const localTransport$ = createLocalTransport$(localTransportOpts);
openIdResolver.resolve?.(openIdResponse);
expect(localTransport$.value).toBe(null);
await flushPromises();
expect(localTransport$.value).toStrictEqual({
livekit_alias: "!example_room_id",
livekit_service_url: "https://lk.example.org",
type: "livekit",
});
});
it("supports getting transport via backend", async () => {
localTransportOpts.client._unstable_getRTCTransports.mockResolvedValue([
{ type: "livekit", livekit_service_url: "https://lk.example.org" },
]);
const localTransport$ = createLocalTransport$(localTransportOpts);
openIdResolver.resolve?.(openIdResponse);
expect(localTransport$.value).toBe(null);
await flushPromises();
expect(localTransport$.value).toStrictEqual({
livekit_alias: "!example_room_id",
livekit_service_url: "https://lk.example.org",
type: "livekit",
});
});
it("fails fast if the openID request fails for backend config", async () => {
localTransportOpts.client._unstable_getRTCTransports.mockResolvedValue([
{ type: "livekit", livekit_service_url: "https://lk.example.org" },
]);
openIdResolver.reject(
new FailToGetOpenIdToken(new Error("Test driven error")),
);
try {
await lastValueFrom(createLocalTransport$(localTransportOpts));
throw Error("Expected test to throw");
} catch (ex) {
expect(ex).toBeInstanceOf(FailToGetOpenIdToken);
}
});
it("supports getting transport via well-known", async () => {
localTransportOpts.client.getDomain.mockReturnValue("example.org");
fetchMock.getOnce("https://example.org/.well-known/matrix/client", {
"org.matrix.msc4143.rtc_foci": [
{ type: "livekit", livekit_service_url: "https://lk.example.org" },
],
});
const localTransport$ = createLocalTransport$(localTransportOpts);
openIdResolver.resolve?.(openIdResponse);
expect(localTransport$.value).toBe(null);
await flushPromises();
expect(localTransport$.value).toStrictEqual({
livekit_alias: "!example_room_id",
livekit_service_url: "https://lk.example.org",
type: "livekit",
});
expect(fetchMock.done()).toEqual(true);
});
it("fails fast if the openId request fails for the well-known config", async () => {
localTransportOpts.client.getDomain.mockReturnValue("example.org");
fetchMock.getOnce("https://example.org/.well-known/matrix/client", {
"org.matrix.msc4143.rtc_foci": [
{ type: "livekit", livekit_service_url: "https://lk.example.org" },
],
});
openIdResolver.reject(
new FailToGetOpenIdToken(new Error("Test driven error")),
);
try {
await lastValueFrom(createLocalTransport$(localTransportOpts));
throw Error("Expected test to throw");
} catch (ex) {
expect(ex).toBeInstanceOf(FailToGetOpenIdToken);
}
});
it("throws if no options are available", async () => {
const localTransport$ = createLocalTransport$({
scope,
roomId: "!example_room_id",
useOldestMember$: constant(false),
memberships$: constant(new Epoch<CallMembership[]>([])),
client: {
getDomain: () => "",
// eslint-disable-next-line @typescript-eslint/naming-convention
_unstable_getRTCTransports: async () => Promise.resolve([]),
// These won't be called in this error path but satisfy the type
getOpenIdToken: vi.fn(),
getDeviceId: vi.fn(),
},
});
await flushPromises();
expect(() => localTransport$.value).toThrow(
new MatrixRTCTransportMissingError(""),
);
});
});
});

171
src/state/CallViewModel/localMember/LocalTransport.ts
View File

@@ -8,11 +8,11 @@ Please see LICENSE in the repository root for full details.
import {
type CallMembership,
isLivekitTransport,
type LivekitTransportConfig,
type LivekitTransport,
isLivekitTransportConfig,
type Transport,
} from "matrix-js-sdk/lib/matrixrtc";
import { type MatrixClient } from "matrix-js-sdk";
import { MatrixError, type MatrixClient } from "matrix-js-sdk";
import {
combineLatest,
distinctUntilChanged,
@@ -27,7 +27,10 @@ import { AutoDiscovery } from "matrix-js-sdk/lib/autodiscovery";
import { type Behavior } from "../../Behavior.ts";
import { type Epoch, type ObservableScope } from "../../ObservableScope.ts";
import { Config } from "../../../config/Config.ts";
import { MatrixRTCTransportMissingError } from "../../../utils/errors.ts";
import {
FailToGetOpenIdToken,
MatrixRTCTransportMissingError,
} from "../../../utils/errors.ts";
import {
getSFUConfigWithOpenID,
type OpenIDClientParts,
@@ -45,7 +48,8 @@ const logger = rootLogger.getChild("[LocalTransport]");
interface Props {
scope: ObservableScope;
memberships$: Behavior<Epoch<CallMembership[]>>;
client: Pick<MatrixClient, "getDomain"> & OpenIDClientParts;
client: Pick<MatrixClient, "getDomain" | "_unstable_getRTCTransports"> &
OpenIDClientParts;
roomId: string;
useOldestMember$: Behavior<boolean>;
}
@@ -85,7 +89,7 @@ export const createLocalTransport$ = ({
* The transport that we would personally prefer to publish on (if not for the
* transport preferences of others, perhaps).
*
* @throws
* @throws MatrixRTCTransportMissingError | FailToGetOpenIdToken
*/
const preferredTransport$: Behavior<LivekitTransport | null> = scope.behavior(
customLivekitUrl.value$.pipe(
@@ -116,73 +120,150 @@ export const createLocalTransport$ = ({
const FOCI_WK_KEY = "org.matrix.msc4143.rtc_foci";
/**
* Determine the correct Transport for the current session, including
* validating auth against the service to ensure it's correct.
* Prefers in order:
*
* @param client
* @param roomId
* @returns
* 1. The `urlFromDevSettings` value. If this cannot be validated, the function will throw.
* 2. The transports returned via the homeserver.
* 3. The transports returned via .well-known.
* 4. The transport configured in Element Call's config.
*
* @param client The authenticated Matrix client for the current user
* @param roomId The ID of the room to be connected to.
* @param urlFromDevSettings Override URL provided by the user's local config.
* @returns A fully validated transport config.
* @throws MatrixRTCTransportMissingError | FailToGetOpenIdToken
*/
async function makeTransport(
client: Pick<MatrixClient, "getDomain"> & OpenIDClientParts,
client: Pick<MatrixClient, "getDomain" | "_unstable_getRTCTransports"> &
OpenIDClientParts,
roomId: string,
urlFromDevSettings: string | null,
): Promise<LivekitTransport> {
let transport: LivekitTransport | undefined;
logger.trace("Searching for a preferred transport");
//TODO refactor this to use the jwt service returned alias.
const livekitAlias = roomId;
// We will call `getSFUConfigWithOpenID` once per transport here as it's our
// only mechanism of valiation. This means we will also ask the
// homeserver for a OpenID token a few times. Since OpenID tokens are single
// use we don't want to risk any issues by re-using a token.
//
// If the OpenID request were to fail then it's acceptable for us to fail
// this function early, as we assume the homeserver has got some problems.
// DEVTOOL: Highest priority: Load from devtool setting
if (urlFromDevSettings !== null) {
const transportFromStorage: LivekitTransport = {
logger.info("Using LiveKit transport from dev tools: ", urlFromDevSettings);
// Validate that the SFU is up. Otherwise, we want to fail on this
// as we don't permit other SFUs.
const config = await getSFUConfigWithOpenID(
client,
urlFromDevSettings,
roomId,
);
return {
type: "livekit",
livekit_service_url: urlFromDevSettings,
livekit_alias: livekitAlias,
livekit_alias: config.livekitAlias,
};
logger.info(
"Using LiveKit transport from dev tools: ",
transportFromStorage,
);
transport = transportFromStorage;
}
// WELL_KNOWN: Prioritize the .well-known/matrix/client, if available, over the configured SFU
async function getFirstUsableTransport(
transports: Transport[],
): Promise<LivekitTransport | null> {
for (const potentialTransport of transports) {
if (isLivekitTransportConfig(potentialTransport)) {
try {
const { livekitAlias } = await getSFUConfigWithOpenID(
client,
potentialTransport.livekit_service_url,
roomId,
);
return {
...potentialTransport,
livekit_alias: livekitAlias,
};
} catch (ex) {
if (ex instanceof FailToGetOpenIdToken) {
// Explictly throw these
throw ex;
}
logger.debug(
`Could not use SFU service "${potentialTransport.livekit_service_url}" as SFU`,
ex,
);
}
}
}
return null;
}
// MSC4143: Attempt to fetch transports from backend.
if ("_unstable_getRTCTransports" in client) {
try {
const selectedTransport = await getFirstUsableTransport(
await client._unstable_getRTCTransports(),
);
if (selectedTransport) {
logger.info("Using backend-configured SFU", selectedTransport);
return selectedTransport;
}
} catch (ex) {
if (ex instanceof MatrixError && ex.httpStatus === 404) {
// Expected, this is an unstable endpoint and it's not required.
logger.debug("Backend does not provide any RTC transports", ex);
} else if (ex instanceof FailToGetOpenIdToken) {
throw ex;
} else {
// We got an error that wasn't just missing support for the feature, so log it loudly.
logger.error(
"Unexpected error fetching RTC transports from backend",
ex,
);
}
}
}
// Legacy MSC4143 (to be removed) WELL_KNOWN: Prioritize the .well-known/matrix/client, if available.
const domain = client.getDomain();
if (domain && transport === undefined) {
if (domain) {
// we use AutoDiscovery instead of relying on the MatrixClient having already
// been fully configured and started
const wellKnownFoci = (await AutoDiscovery.getRawClientConfig(domain))?.[
FOCI_WK_KEY
];
if (Array.isArray(wellKnownFoci)) {
const wellKnownTransport: LivekitTransportConfig | undefined =
wellKnownFoci.find((f) => f && isLivekitTransportConfig(f));
if (wellKnownTransport !== undefined) {
logger.info("Using LiveKit transport from .well-known: ", transport);
transport = { ...wellKnownTransport, livekit_alias: livekitAlias };
}
const selectedTransport = Array.isArray(wellKnownFoci)
? await getFirstUsableTransport(wellKnownFoci)
: null;
if (selectedTransport) {
logger.info("Using .well-known SFU", selectedTransport);
return selectedTransport;
}
}
// CONFIG: Least prioritized; Load from config file
const urlFromConf = Config.get().livekit?.livekit_service_url;
if (urlFromConf && transport === undefined) {
const transportFromConf: LivekitTransport = {
type: "livekit",
livekit_service_url: urlFromConf,
livekit_alias: livekitAlias,
};
logger.info("Using LiveKit transport from config: ", transportFromConf);
transport = transportFromConf;
if (urlFromConf) {
try {
const { livekitAlias } = await getSFUConfigWithOpenID(
client,
urlFromConf,
roomId,
);
const selectedTransport: LivekitTransport = {
type: "livekit",
livekit_service_url: urlFromConf,
livekit_alias: livekitAlias,
};
logger.info("Using config SFU", selectedTransport);
return selectedTransport;
} catch (ex) {
if (ex instanceof FailToGetOpenIdToken) {
throw ex;
}
logger.error("Failed to validate config SFU", ex);
}
}
if (!transport) throw new MatrixRTCTransportMissingError(domain ?? ""); // this will call the jwt/sfu/get endpoint to pre create the livekit room.
await getSFUConfigWithOpenID(
client,
transport.livekit_service_url,
transport.livekit_alias,
);
return transport;
throw new MatrixRTCTransportMissingError(domain ?? "");
}

360
src/state/CallViewModel/localMember/Publisher.test.ts Normal file
View File

@@ -0,0 +1,360 @@
/*
Copyright 2025 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { afterEach, beforeEach, describe, expect, it, test, vi } from "vitest";
import {
ConnectionState as LivekitConnectionState,
LocalParticipant,
type LocalTrack,
type LocalTrackPublication,
ParticipantEvent,
Track,
} from "livekit-client";
import { BehaviorSubject } from "rxjs";
import { logger } from "matrix-js-sdk/lib/logger";
import { ObservableScope } from "../../ObservableScope";
import { constant } from "../../Behavior";
import {
flushPromises,
mockLivekitRoom,
mockMediaDevices,
} from "../../../utils/test";
import { Publisher } from "./Publisher";
import { type Connection } from "../remoteMembers/Connection";
import { type MuteStates } from "../../MuteStates";
let scope: ObservableScope;
beforeEach(() => {
scope = new ObservableScope();
});
afterEach(() => scope.end());
function createMockLocalTrack(source: Track.Source): LocalTrack {
const track = {
source,
isMuted: false,
isUpstreamPaused: false,
} as Partial<LocalTrack> as LocalTrack;
vi.mocked(track).mute = vi.fn().mockImplementation(() => {
track.isMuted = true;
});
vi.mocked(track).unmute = vi.fn().mockImplementation(() => {
track.isMuted = false;
});
vi.mocked(track).pauseUpstream = vi.fn().mockImplementation(() => {
// @ts-expect-error - for that test we want to set isUpstreamPaused directly
track.isUpstreamPaused = true;
});
vi.mocked(track).resumeUpstream = vi.fn().mockImplementation(() => {
// @ts-expect-error - for that test we want to set isUpstreamPaused directly
track.isUpstreamPaused = false;
});
return track;
}
function createMockMuteState(enabled$: BehaviorSubject<boolean>): {
enabled$: BehaviorSubject<boolean>;
setHandler: (h: (enabled: boolean) => void) => void;
unsetHandler: () => void;
} {
let currentHandler = (enabled: boolean): void => {};
const ms = {
enabled$,
setHandler: vi.fn().mockImplementation((h: (enabled: boolean) => void) => {
currentHandler = h;
}),
unsetHandler: vi.fn().mockImplementation(() => {
currentHandler = (enabled: boolean): void => {};
}),
};
// forward enabled$ emissions to the current handler
enabled$.subscribe((enabled) => {
logger.info(`MockMuteState: enabled changed to ${enabled}`);
currentHandler(enabled);
});
return ms;
}
let connection: Connection;
let muteStates: MuteStates;
let localParticipant: LocalParticipant;
let audioEnabled$: BehaviorSubject<boolean>;
let videoEnabled$: BehaviorSubject<boolean>;
let trackPublications: LocalTrackPublication[];
// use it to control when track creation resolves, default to resolved
let createTrackLock: Promise<void>;
beforeEach(() => {
trackPublications = [];
audioEnabled$ = new BehaviorSubject(false);
videoEnabled$ = new BehaviorSubject(false);
createTrackLock = Promise.resolve();
muteStates = {
audio: createMockMuteState(audioEnabled$),
video: createMockMuteState(videoEnabled$),
} as unknown as MuteStates;
const mockSendDataPacket = vi.fn();
const mockEngine = {
client: {
sendUpdateLocalMetadata: vi.fn(),
},
on: vi.fn().mockReturnThis(),
sendDataPacket: mockSendDataPacket,
};
localParticipant = new LocalParticipant(
"local-sid",
"local-identity",
// @ts-expect-error - for that test we want a real LocalParticipant to have the pending publications logic
mockEngine,
{
adaptiveStream: true,
dynacase: false,
audioCaptureDefaults: {},
videoCaptureDefaults: {},
stopLocalTrackOnUnpublish: true,
reconnectPolicy: "always",
disconnectOnPageLeave: true,
},
new Map(),
{},
);
vi.mocked(localParticipant).createTracks = vi
.fn()
.mockImplementation(async (opts) => {
const tracks: LocalTrack[] = [];
if (opts.audio) {
tracks.push(createMockLocalTrack(Track.Source.Microphone));
}
if (opts.video) {
tracks.push(createMockLocalTrack(Track.Source.Camera));
}
await createTrackLock;
return tracks;
});
vi.mocked(localParticipant).publishTrack = vi
.fn()
.mockImplementation(async (track: LocalTrack) => {
const pub = {
track,
source: track.source,
mute: track.mute,
unmute: track.unmute,
} as Partial<LocalTrackPublication> as LocalTrackPublication;
trackPublications.push(pub);
localParticipant.emit(ParticipantEvent.LocalTrackPublished, pub);
return Promise.resolve(pub);
});
vi.mocked(localParticipant).getTrackPublication = vi
.fn()
.mockImplementation((source: Track.Source) => {
return trackPublications.find((pub) => pub.track?.source === source);
});
connection = {
state$: constant({
state: "ConnectedToLkRoom",
livekitConnectionState$: constant(LivekitConnectionState.Connected),
}),
livekitRoom: mockLivekitRoom({
localParticipant: localParticipant,
}),
} as unknown as Connection;
});
describe("Publisher", () => {
let publisher: Publisher;
beforeEach(() => {
publisher = new Publisher(
scope,
connection,
mockMediaDevices({}),
muteStates,
constant({ supported: false, processor: undefined }),
logger,
);
});
afterEach(() => {});
it("Should not create tracks if started muted to avoid unneeded permission requests", async () => {
const createTracksSpy = vi.spyOn(
connection.livekitRoom.localParticipant,
"createTracks",
);
audioEnabled$.next(false);
videoEnabled$.next(false);
await publisher.createAndSetupTracks();
expect(createTracksSpy).not.toHaveBeenCalled();
});
it("Should minimize permission request by querying create at once", async () => {
const enableCameraAndMicrophoneSpy = vi.spyOn(
localParticipant,
"enableCameraAndMicrophone",
);
const createTracksSpy = vi.spyOn(localParticipant, "createTracks");
audioEnabled$.next(true);
videoEnabled$.next(true);
await publisher.createAndSetupTracks();
await flushPromises();
expect(enableCameraAndMicrophoneSpy).toHaveBeenCalled();
// It should create both at once
expect(createTracksSpy).toHaveBeenCalledWith({
audio: true,
video: true,
});
});
it("Ensure no data is streamed until publish has been called", async () => {
audioEnabled$.next(true);
await publisher.createAndSetupTracks();
// The track should be created and paused
expect(localParticipant.createTracks).toHaveBeenCalledWith({
audio: true,
video: undefined,
});
await flushPromises();
expect(localParticipant.publishTrack).toHaveBeenCalled();
await flushPromises();
const track = localParticipant.getTrackPublication(
Track.Source.Microphone,
)?.track;
expect(track).toBeDefined();
expect(track!.pauseUpstream).toHaveBeenCalled();
expect(track!.isUpstreamPaused).toBe(true);
});
it("Ensure resume upstream when published is called", async () => {
videoEnabled$.next(true);
await publisher.createAndSetupTracks();
// await flushPromises();
await publisher.startPublishing();
const track = localParticipant.getTrackPublication(
Track.Source.Camera,
)?.track;
expect(track).toBeDefined();
// expect(track.pauseUpstream).toHaveBeenCalled();
expect(track!.isUpstreamPaused).toBe(false);
});
describe("Mute states", () => {
let publisher: Publisher;
beforeEach(() => {
publisher = new Publisher(
scope,
connection,
mockMediaDevices({}),
muteStates,
constant({ supported: false, processor: undefined }),
logger,
);
});
test.each([
{ mutes: { audioEnabled: true, videoEnabled: false } },
{ mutes: { audioEnabled: true, videoEnabled: false } },
])("only create the tracks that are unmuted $mutes", async ({ mutes }) => {
// Ensure all muted
audioEnabled$.next(mutes.audioEnabled);
videoEnabled$.next(mutes.videoEnabled);
vi.mocked(connection.livekitRoom.localParticipant).createTracks = vi
.fn()
.mockResolvedValue([]);
await publisher.createAndSetupTracks();
expect(
connection.livekitRoom.localParticipant.createTracks,
).toHaveBeenCalledOnce();
expect(
connection.livekitRoom.localParticipant.createTracks,
).toHaveBeenCalledWith({
audio: mutes.audioEnabled ? true : undefined,
video: mutes.videoEnabled ? true : undefined,
});
});
});
it("does mute unmute audio", async () => {});
});
describe("Bug fix", () => {
// There is a race condition when creating and publishing tracks while the mute state changes.
// This race condition could cause tracks to be published even though they are muted at the
// beginning of a call coming from lobby.
// This is caused by our stack using manually the low level API to create and publish tracks,
// but also using the higher level setMicrophoneEnabled and setCameraEnabled functions that also create
// and publish tracks, and managing pending publications.
// Race is as follow, on creation of the Publisher we create the tracks then publish them.
// If in the middle of that process the mute state changes:
// - the `setMicrophoneEnabled` will be no-op because it is not aware of our created track and can't see any pending publication
// - If start publication is requested it will publish the track even though there was a mute request.
it("wrongly publish tracks while muted", async () => {
// setLogLevel(`debug`);
const publisher = new Publisher(
scope,
connection,
mockMediaDevices({}),
muteStates,
constant({ supported: false, processor: undefined }),
logger,
);
audioEnabled$.next(true);
const resolvers = Promise.withResolvers<void>();
createTrackLock = resolvers.promise;
// Initially the audio is unmuted, so creating tracks should publish the audio track
const createTracks = publisher.createAndSetupTracks();
void publisher.startPublishing();
void createTracks.then(() => {
void publisher.startPublishing();
});
// now mute the audio before allowing track creation to complete
audioEnabled$.next(false);
resolvers.resolve(undefined);
await createTracks;
await flushPromises();
const track = localParticipant.getTrackPublication(
Track.Source.Microphone,
)?.track;
expect(track).toBeDefined();
try {
expect(localParticipant.publishTrack).not.toHaveBeenCalled();
} catch {
expect(track!.mute).toHaveBeenCalled();
expect(track!.isMuted).toBe(true);
}
});
});

325
src/state/CallViewModel/localMember/Publisher.ts
View File

@@ -6,12 +6,12 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
ConnectionState as LivekitConnectionState,
type LocalTrackPublication,
LocalVideoTrack,
ParticipantEvent,
type Room as LivekitRoom,
Track,
type LocalTrack,
type LocalTrackPublication,
ConnectionState as LivekitConnectionState,
} from "livekit-client";
import {
map,
@@ -40,31 +40,36 @@ import { type ObservableScope } from "../../ObservableScope.ts";
* The Publisher is also responsible for creating the media tracks.
*/
export class Publisher {
public tracks: LocalTrack<Track.Kind>[] = [];
/**
* By default, livekit will start publishing tracks as soon as they are created.
* In the matrix RTC world, we want to control when tracks are published based
* on whether the user is part of the RTC session or not.
*/
public shouldPublish = false;
/**
* Creates a new Publisher.
* @param scope - The observable scope to use for managing the publisher.
* @param connection - The connection to use for publishing.
* @param devices - The media devices to use for audio and video input.
* @param muteStates - The mute states for audio and video.
* @param e2eeLivekitOptions - The E2EE options to use for the LiveKit room. Use to share the same key provider across connections!.
* @param trackerProcessorState$ - The processor state for the video track processor (e.g. background blur).
* @param logger - The logger to use for logging :D.
*/
public constructor(
private scope: ObservableScope,
private connection: Connection,
private connection: Pick<Connection, "livekitRoom" | "state$">, //setE2EEEnabled,
devices: MediaDevices,
private readonly muteStates: MuteStates,
trackerProcessorState$: Behavior<ProcessorState>,
private logger?: Logger,
private logger: Logger,
) {
this.logger?.info("[PublishConnection] Create LiveKit room");
const { controlledAudioDevices } = getUrlParams();
const room = connection.livekitRoom;
room.setE2EEEnabled(room.options.e2ee !== undefined)?.catch((e: Error) => {
this.logger?.error("Failed to set E2EE enabled on room", e);
this.logger.error("Failed to set E2EE enabled on room", e);
});
// Setup track processor syncing (blur)
@@ -74,117 +79,187 @@ export class Publisher {
this.workaroundRestartAudioInputTrackChrome(devices, scope);
this.scope.onEnd(() => {
this.logger?.info(
"[PublishConnection] Scope ended -> stop publishing all tracks",
);
this.logger.info("Scope ended -> stop publishing all tracks");
void this.stopPublishing();
muteStates.audio.unsetHandler();
muteStates.video.unsetHandler();
});
this.connection.livekitRoom.localParticipant.on(
ParticipantEvent.LocalTrackPublished,
this.onLocalTrackPublished.bind(this),
);
}
// LiveKit will publish the tracks as soon as they are created
// but we want to control when tracks are published.
// We cannot just mute the tracks, even if this will effectively stop the publishing,
// it would also prevent the user from seeing their own video/audio preview.
// So for that we use pauseUpStream(): Stops sending media to the server by replacing
// the sender track with null, but keeps the local MediaStreamTrack active.
// The user can still see/hear themselves locally, but remote participants see nothing.
private onLocalTrackPublished(
localTrackPublication: LocalTrackPublication,
): void {
this.logger.info("Local track published", localTrackPublication);
const lkRoom = this.connection.livekitRoom;
if (!this.shouldPublish) {
this.pauseUpstreams(lkRoom, [localTrackPublication.source]).catch((e) => {
this.logger.error(`Failed to pause upstreams`, e);
});
}
// also check the mute state and apply it
if (localTrackPublication.source === Track.Source.Microphone) {
const enabled = this.muteStates.audio.enabled$.value;
lkRoom.localParticipant.setMicrophoneEnabled(enabled).catch((e) => {
this.logger.error(
`Failed to enable microphone track, enabled:${enabled}`,
e,
);
});
} else if (localTrackPublication.source === Track.Source.Camera) {
const enabled = this.muteStates.video.enabled$.value;
lkRoom.localParticipant.setCameraEnabled(enabled).catch((e) => {
this.logger.error(
`Failed to enable camera track, enabled:${enabled}`,
e,
);
});
}
}
/**
* Create and setup local audio and video tracks based on the current mute states.
* It creates the tracks only if audio and/or video is enabled, to avoid unnecessary
* permission prompts.
*
* It also observes mute state changes to update LiveKit microphone/camera states accordingly.
* If a track is not created initially because disabled, it will be created when unmuting.
*
* This call is not blocking anymore, instead callers can listen to the
* `RoomEvent.MediaDevicesError` event in the LiveKit room to be notified of any errors.
*
*/
public async createAndSetupTracks(): Promise<void> {
this.logger.debug("createAndSetupTracks called");
const lkRoom = this.connection.livekitRoom;
// Observe mute state changes and update LiveKit microphone/camera states accordingly
this.observeMuteStates();
// Check if audio and/or video is enabled. We only create tracks if enabled,
// because it could prompt for permission, and we don't want to do that unnecessarily.
const audio = this.muteStates.audio.enabled$.value;
const video = this.muteStates.video.enabled$.value;
// We don't await the creation, because livekit could block until the tracks
// are fully published, and not only that they are created.
// We don't have control on that, localParticipant creates and publishes the tracks
// asap.
// We are using the `ParticipantEvent.LocalTrackPublished` to be notified
// when tracks are actually published, and at that point
// we can pause upstream if needed (depending on if startPublishing has been called).
if (audio && video) {
// Enable both at once in order to have a single permission prompt!
void lkRoom.localParticipant.enableCameraAndMicrophone();
} else if (audio) {
void lkRoom.localParticipant.setMicrophoneEnabled(true);
} else if (video) {
void lkRoom.localParticipant.setCameraEnabled(true);
}
return Promise.resolve();
}
private async pauseUpstreams(
lkRoom: LivekitRoom,
sources: Track.Source[],
): Promise<void> {
for (const source of sources) {
const track = lkRoom.localParticipant.getTrackPublication(source)?.track;
if (track) {
await track.pauseUpstream();
} else {
this.logger.warn(
`No track found for source ${source} to pause upstream`,
);
}
}
}
private async resumeUpstreams(
lkRoom: LivekitRoom,
sources: Track.Source[],
): Promise<void> {
for (const source of sources) {
const track = lkRoom.localParticipant.getTrackPublication(source)?.track;
if (track) {
await track.resumeUpstream();
} else {
this.logger.warn(
`No track found for source ${source} to resume upstream`,
);
}
}
}
/**
* Start the connection to LiveKit and publish local tracks.
*
* This will:
* wait for the connection to be ready.
// * 1. Request an OpenId token `request_token` (allows matrix users to verify their identity with a third-party service.)
// * 2. Use this token to request the SFU config to the MatrixRtc authentication service.
// * 3. Connect to the configured LiveKit room.
// * 4. Create local audio and video tracks based on the current mute states and publish them to the room.
* Request to publish local tracks to the LiveKit room.
* This will wait for the connection to be ready before publishing.
* Livekit also have some local retry logic for publishing tracks.
* Can be called multiple times, localparticipant manages the state of published tracks (or pending publications).
*
* @throws {InsufficientCapacityError} if the LiveKit server indicates that it has insufficient capacity to accept the connection.
* @throws {SFURoomCreationRestrictedError} if the LiveKit server indicates that the room does not exist and cannot be created.
* @returns
*/
public async createAndSetupTracks(): Promise<LocalTrack[]> {
const lkRoom = this.connection.livekitRoom;
// Observe mute state changes and update LiveKit microphone/camera states accordingly
this.observeMuteStates(this.scope);
// TODO: This should be an autostarted connection no need to start here. just check the connection state.
// TODO: This will fetch the JWT token. Perhaps we could keep it preloaded
// instead? This optimization would only be safe for a publish connection,
// because we don't want to leak the user's intent to perhaps join a call to
// remote servers before they actually commit to it.
// const { promise, resolve, reject } = Promise.withResolvers<void>();
// const sub = this.connection.state$.subscribe((s) => {
// if (s.state === "FailedToStart") {
// reject(new Error("Disconnected from LiveKit server"));
// } else if (s.state === "ConnectedToLkRoom") {
// resolve();
// }
// });
// try {
// await promise;
// } catch (e) {
// throw e;
// } finally {
// sub.unsubscribe();
// }
// TODO-MULTI-SFU: Prepublish a microphone track
const audio = this.muteStates.audio.enabled$.value;
const video = this.muteStates.video.enabled$.value;
// createTracks throws if called with audio=false and video=false
if (audio || video) {
// TODO this can still throw errors? It will also prompt for permissions if not already granted
this.tracks =
(await lkRoom.localParticipant
.createTracks({
audio,
video,
})
.catch((error) => {
this.logger?.error("Failed to create tracks", error);
})) ?? [];
public async startPublishing(): Promise<void> {
if (this.shouldPublish) {
this.logger.debug(`Already publishing, ignoring startPublishing call`);
return;
}
return this.tracks;
}
this.shouldPublish = true;
this.logger.debug("startPublishing called");
public async startPublishing(): Promise<LocalTrack[]> {
const lkRoom = this.connection.livekitRoom;
const { promise, resolve, reject } = Promise.withResolvers<void>();
const sub = this.connection.state$.subscribe((s) => {
switch (s.state) {
case "ConnectedToLkRoom":
resolve();
break;
case "FailedToStart":
reject(new Error("Failed to connect to LiveKit server"));
break;
default:
this.logger?.info("waiting for connection: ", s.state);
}
});
// Resume upstream for both audio and video tracks
// We need to call it explicitly because call setTrackEnabled does not always
// resume upstream. It will only if you switch the track from disabled to enabled,
// but if the track is already enabled but upstream is paused, it won't resume it.
// TODO what about screen share?
try {
await promise;
await this.resumeUpstreams(lkRoom, [
Track.Source.Microphone,
Track.Source.Camera,
]);
} catch (e) {
throw e;
} finally {
sub.unsubscribe();
this.logger.error(`Failed to resume upstreams`, e);
}
for (const track of this.tracks) {
// TODO: handle errors? Needs the signaling connection to be up, but it has some retries internally
// with a timeout.
await lkRoom.localParticipant.publishTrack(track).catch((error) => {
this.logger?.error("Failed to publish track", error);
});
// TODO: check if the connection is still active? and break the loop if not?
}
return this.tracks;
}
public async stopPublishing(): Promise<void> {
// TODO-MULTI-SFU: Move these calls back to ObservableScope.onEnd once scope
// actually has the right lifetime
this.muteStates.audio.unsetHandler();
this.muteStates.video.unsetHandler();
this.logger.debug("stopPublishing called");
this.shouldPublish = false;
// Pause upstream will stop sending media to the server, while keeping
// the local MediaStreamTrack active, so the user can still see themselves.
await this.pauseUpstreams(this.connection.livekitRoom, [
Track.Source.Microphone,
Track.Source.Camera,
Track.Source.ScreenShare,
]);
}
const localParticipant = this.connection.livekitRoom.localParticipant;
const tracks: LocalTrack[] = [];
const addToTracksIfDefined = (p: LocalTrackPublication): void => {
if (p.track !== undefined) tracks.push(p.track);
};
localParticipant.trackPublications.forEach(addToTracksIfDefined);
await localParticipant.unpublishTracks(tracks);
public async stopTracks(): Promise<void> {
const lkRoom = this.connection.livekitRoom;
for (const source of [
Track.Source.Microphone,
Track.Source.Camera,
Track.Source.ScreenShare,
]) {
const localPub = lkRoom.localParticipant.getTrackPublication(source);
if (localPub?.track) {
// stops and unpublishes the track
await lkRoom.localParticipant.unpublishTrack(localPub!.track, true);
}
}
}
/// Private methods
@@ -221,6 +296,9 @@ export class Publisher {
// the process of being restarted.
activeMicTrack.mediaStreamTrack.readyState !== "ended"
) {
this.logger?.info(
"Restarting audio device track due to active media device changed (workaroundRestartAudioInputTrackChrome)",
);
// Restart the track, which will cause Livekit to do another
// getUserMedia() call with deviceId: default to get the *new* default device.
// Note that room.switchActiveDevice() won't work: Livekit will ignore it because
@@ -229,7 +307,7 @@ export class Publisher {
.getTrackPublication(Track.Source.Microphone)
?.audioTrack?.restartTrack()
.catch((e) => {
this.logger?.error(`Failed to restart audio device track`, e);
this.logger.error(`Failed to restart audio device track`, e);
});
}
});
@@ -249,7 +327,7 @@ export class Publisher {
selected$.pipe(scope.bind()).subscribe((device) => {
if (lkRoom.state != LivekitConnectionState.Connected) return;
// if (this.connectionState$.value !== ConnectionState.Connected) return;
this.logger?.info(
this.logger.info(
"[LivekitRoom] syncDevice room.getActiveDevice(kind) !== d.id :",
lkRoom.getActiveDevice(kind),
" !== ",
@@ -262,7 +340,7 @@ export class Publisher {
lkRoom
.switchActiveDevice(kind, device.id)
.catch((e: Error) =>
this.logger?.error(
this.logger.error(
`Failed to sync ${kind} device with LiveKit`,
e,
),
@@ -278,30 +356,37 @@ export class Publisher {
/**
* Observe changes in the mute states and update the LiveKit room accordingly.
* @param scope
* @private
*/
private observeMuteStates(scope: ObservableScope): void {
private observeMuteStates(): void {
const lkRoom = this.connection.livekitRoom;
this.muteStates.audio.setHandler(async (desired) => {
this.muteStates.audio.setHandler(async (enable) => {
try {
await lkRoom.localParticipant.setMicrophoneEnabled(desired);
} catch (e) {
this.logger?.error(
"Failed to update LiveKit audio input mute state",
e,
this.logger.debug(
`handler: Setting LiveKit microphone enabled: ${enable}`,
);
await lkRoom.localParticipant.setMicrophoneEnabled(enable);
// Unmute will restart the track if it was paused upstream,
// but until explicitly requested, we want to keep it paused.
if (!this.shouldPublish && enable) {
await this.pauseUpstreams(lkRoom, [Track.Source.Microphone]);
}
} catch (e) {
this.logger.error("Failed to update LiveKit audio input mute state", e);
}
return lkRoom.localParticipant.isMicrophoneEnabled;
});
this.muteStates.video.setHandler(async (desired) => {
this.muteStates.video.setHandler(async (enable) => {
try {
await lkRoom.localParticipant.setCameraEnabled(desired);
this.logger.debug(`handler: Setting LiveKit camera enabled: ${enable}`);
await lkRoom.localParticipant.setCameraEnabled(enable);
// Unmute will restart the track if it was paused upstream,
// but until explicitly requested, we want to keep it paused.
if (!this.shouldPublish && enable) {
await this.pauseUpstreams(lkRoom, [Track.Source.Camera]);
}
} catch (e) {
this.logger?.error(
"Failed to update LiveKit video input mute state",
e,
);
this.logger.error("Failed to update LiveKit video input mute state", e);
}
return lkRoom.localParticipant.isCameraEnabled;
});
@@ -315,7 +400,7 @@ export class Publisher {
const track$ = scope.behavior(
observeTrackReference$(room.localParticipant, Track.Source.Camera).pipe(
map((trackRef) => {
const track = trackRef?.publication?.track;
const track = trackRef?.publication.track;
return track instanceof LocalVideoTrack ? track : null;
}),
),

214
src/state/CallViewModel/remoteMembers/Connection.test.ts
View File

@@ -30,13 +30,17 @@ import { logger } from "matrix-js-sdk/lib/logger";
import type { LivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
import {
Connection,
ConnectionState,
type ConnectionOpts,
type ConnectionState,
type PublishingParticipant,
} from "./Connection.ts";
import { ObservableScope } from "../../ObservableScope.ts";
import { type OpenIDClientParts } from "../../../livekit/openIDSFU.ts";
import { FailToGetOpenIdToken } from "../../../utils/errors.ts";
import {
ElementCallError,
FailToGetOpenIdToken,
} from "../../../utils/errors.ts";
import { testJWTToken } from "../../../utils/test-fixtures.ts";
import { mockRemoteParticipant } from "../../../utils/test.ts";
let testScope: ObservableScope;
@@ -47,11 +51,6 @@ let fakeLivekitRoom: MockedObject<LivekitRoom>;
let localParticipantEventEmiter: EventEmitter;
let fakeLocalParticipant: MockedObject<LocalParticipant>;
let fakeRoomEventEmiter: EventEmitter;
// let fakeMembershipsFocusMap$: BehaviorSubject<
// { membership: CallMembership; transport: LivekitTransport }[]
// >;
const livekitFocus: LivekitTransport = {
livekit_alias: "!roomID:example.org",
livekit_service_url: "https://matrix-rtc.example.org/livekit/jwt",
@@ -88,22 +87,25 @@ function setupTest(): void {
localParticipantEventEmiter,
),
} as unknown as LocalParticipant);
fakeRoomEventEmiter = new EventEmitter();
const fakeRoomEventEmitter = new EventEmitter();
fakeLivekitRoom = vi.mocked<LivekitRoom>({
connect: vi.fn(),
disconnect: vi.fn(),
remoteParticipants: new Map(),
localParticipant: fakeLocalParticipant,
state: LivekitConnectionState.Disconnected,
on: fakeRoomEventEmiter.on.bind(fakeRoomEventEmiter),
off: fakeRoomEventEmiter.off.bind(fakeRoomEventEmiter),
addListener: fakeRoomEventEmiter.addListener.bind(fakeRoomEventEmiter),
on: fakeRoomEventEmitter.on.bind(fakeRoomEventEmitter),
off: fakeRoomEventEmitter.off.bind(fakeRoomEventEmitter),
addListener: fakeRoomEventEmitter.addListener.bind(fakeRoomEventEmitter),
removeListener:
fakeRoomEventEmiter.removeListener.bind(fakeRoomEventEmiter),
fakeRoomEventEmitter.removeListener.bind(fakeRoomEventEmitter),
removeAllListeners:
fakeRoomEventEmiter.removeAllListeners.bind(fakeRoomEventEmiter),
fakeRoomEventEmitter.removeAllListeners.bind(fakeRoomEventEmitter),
setE2EEEnabled: vi.fn().mockResolvedValue(undefined),
emit: (eventName: string | symbol, ...args: unknown[]) => {
fakeRoomEventEmitter.emit(eventName, ...args);
},
} as unknown as LivekitRoom);
}
@@ -120,12 +122,21 @@ function setupRemoteConnection(): Connection {
status: 200,
body: {
url: "wss://matrix-rtc.m.localhost/livekit/sfu",
jwt: "ATOKEN",
jwt: testJWTToken,
},
};
});
fakeLivekitRoom.connect.mockResolvedValue(undefined);
fakeLivekitRoom.connect.mockImplementation(async (): Promise<void> => {
const changeEv = RoomEvent.ConnectionStateChanged;
fakeLivekitRoom.state = LivekitConnectionState.Connecting;
fakeLivekitRoom.emit(changeEv, fakeLivekitRoom.state);
fakeLivekitRoom.state = LivekitConnectionState.Connected;
fakeLivekitRoom.emit(changeEv, fakeLivekitRoom.state);
return Promise.resolve();
});
return new Connection(opts, logger);
}
@@ -148,7 +159,7 @@ describe("Start connection states", () => {
};
const connection = new Connection(opts, logger);
expect(connection.state$.getValue().state).toEqual("Initialized");
expect(connection.state$.getValue()).toEqual("Initialized");
});
it("fail to getOpenId token then error state", async () => {
@@ -164,7 +175,7 @@ describe("Start connection states", () => {
const connection = new Connection(opts, logger);
const capturedStates: ConnectionState[] = [];
const capturedStates: (ConnectionState | Error)[] = [];
const s = connection.state$.subscribe((value) => {
capturedStates.push(value);
});
@@ -184,22 +195,20 @@ describe("Start connection states", () => {
let capturedState = capturedStates.pop();
expect(capturedState).toBeDefined();
expect(capturedState!.state).toEqual("FetchingConfig");
expect(capturedState!).toEqual("FetchingConfig");
deferred.reject(new FailToGetOpenIdToken(new Error("Failed to get token")));
await vi.runAllTimersAsync();
capturedState = capturedStates.pop();
if (capturedState!.state === "FailedToStart") {
expect(capturedState!.error.message).toEqual("Something went wrong");
expect(capturedState!.transport.livekit_alias).toEqual(
if (capturedState instanceof Error) {
expect(capturedState.message).toEqual("Something went wrong");
expect(connection.transport.livekit_alias).toEqual(
livekitFocus.livekit_alias,
);
} else {
expect.fail(
"Expected FailedToStart state but got " + capturedState?.state,
);
expect.fail("Expected FailedToStart state but got " + capturedState);
}
});
@@ -216,7 +225,7 @@ describe("Start connection states", () => {
const connection = new Connection(opts, logger);
const capturedStates: ConnectionState[] = [];
const capturedStates: (ConnectionState | Error)[] = [];
const s = connection.state$.subscribe((value) => {
capturedStates.push(value);
});
@@ -238,24 +247,25 @@ describe("Start connection states", () => {
let capturedState = capturedStates.pop();
expect(capturedState).toBeDefined();
expect(capturedState?.state).toEqual("FetchingConfig");
expect(capturedState).toEqual(ConnectionState.FetchingConfig);
deferredSFU.resolve();
await vi.runAllTimersAsync();
capturedState = capturedStates.pop();
if (capturedState?.state === "FailedToStart") {
expect(capturedState?.error.message).toContain(
"SFU Config fetch failed with exception Error",
if (
capturedState instanceof ElementCallError &&
capturedState.cause instanceof Error
) {
expect(capturedState.cause.message).toContain(
"SFU Config fetch failed with exception",
);
expect(capturedState?.transport.livekit_alias).toEqual(
expect(connection.transport.livekit_alias).toEqual(
livekitFocus.livekit_alias,
);
} else {
expect.fail(
"Expected FailedToStart state but got " + capturedState?.state,
);
expect.fail("Expected FailedToStart state but got " + capturedState);
}
});
@@ -272,7 +282,7 @@ describe("Start connection states", () => {
const connection = new Connection(opts, logger);
const capturedStates: ConnectionState[] = [];
const capturedStates: (ConnectionState | Error)[] = [];
const s = connection.state$.subscribe((value) => {
capturedStates.push(value);
});
@@ -285,7 +295,7 @@ describe("Start connection states", () => {
status: 200,
body: {
url: "wss://matrix-rtc.m.localhost/livekit/sfu",
jwt: "ATOKEN",
jwt: testJWTToken,
},
};
});
@@ -302,18 +312,21 @@ describe("Start connection states", () => {
let capturedState = capturedStates.pop();
expect(capturedState).toBeDefined();
expect(capturedState?.state).toEqual("FetchingConfig");
expect(capturedState).toEqual(ConnectionState.FetchingConfig);
deferredSFU.resolve();
await vi.runAllTimersAsync();
capturedState = capturedStates.pop();
if (capturedState && capturedState?.state === "FailedToStart") {
expect(capturedState.error.message).toContain(
if (
capturedState instanceof ElementCallError &&
capturedState.cause instanceof Error
) {
expect(capturedState.cause.message).toContain(
"Failed to connect to livekit",
);
expect(capturedState.transport.livekit_alias).toEqual(
expect(connection.transport.livekit_alias).toEqual(
livekitFocus.livekit_alias,
);
} else {
@@ -329,7 +342,7 @@ describe("Start connection states", () => {
const connection = setupRemoteConnection();
const capturedStates: ConnectionState[] = [];
const capturedStates: (ConnectionState | Error)[] = [];
const s = connection.state$.subscribe((value) => {
capturedStates.push(value);
});
@@ -339,13 +352,15 @@ describe("Start connection states", () => {
await vi.runAllTimersAsync();
const initialState = capturedStates.shift();
expect(initialState?.state).toEqual("Initialized");
expect(initialState).toEqual(ConnectionState.Initialized);
const fetchingState = capturedStates.shift();
expect(fetchingState?.state).toEqual("FetchingConfig");
expect(fetchingState).toEqual(ConnectionState.FetchingConfig);
const disconnectedState = capturedStates.shift();
expect(disconnectedState).toEqual(ConnectionState.LivekitDisconnected);
const connectingState = capturedStates.shift();
expect(connectingState?.state).toEqual("ConnectingToLkRoom");
expect(connectingState).toEqual(ConnectionState.LivekitConnecting);
const connectedState = capturedStates.shift();
expect(connectedState?.state).toEqual("ConnectedToLkRoom");
expect(connectedState).toEqual(ConnectionState.LivekitConnected);
});
it("shutting down the scope should stop the connection", async () => {
@@ -363,46 +378,32 @@ describe("Start connection states", () => {
});
});
function fakeRemoteLivekitParticipant(
id: string,
publications: number = 1,
): RemoteParticipant {
return {
identity: id,
getTrackPublications: () => Array(publications),
} as unknown as RemoteParticipant;
}
describe("Publishing participants observations", () => {
it("should emit the list of publishing participants", () => {
describe("remote participants", () => {
it("emits the list of remote participants", () => {
setupTest();
const connection = setupRemoteConnection();
const bobIsAPublisher = Promise.withResolvers<void>();
const danIsAPublisher = Promise.withResolvers<void>();
const observedPublishers: PublishingParticipant[][] = [];
const s = connection.remoteParticipantsWithTracks$.subscribe(
(publishers) => {
observedPublishers.push(publishers);
if (publishers.some((p) => p.identity === "@bob:example.org:DEV111")) {
bobIsAPublisher.resolve();
}
if (publishers.some((p) => p.identity === "@dan:example.org:DEV333")) {
danIsAPublisher.resolve();
}
},
);
const observedParticipants: RemoteParticipant[][] = [];
const s = connection.remoteParticipants$.subscribe((participants) => {
observedParticipants.push(participants);
});
onTestFinished(() => s.unsubscribe());
// The publishingParticipants$ observable is derived from the current members of the
// The remoteParticipants$ observable is derived from the current members of the
// livekitRoom and the rtc membership in order to publish the members that are publishing
// on this connection.
let participants: RemoteParticipant[] = [
fakeRemoteLivekitParticipant("@alice:example.org:DEV000", 0),
fakeRemoteLivekitParticipant("@bob:example.org:DEV111", 0),
fakeRemoteLivekitParticipant("@carol:example.org:DEV222", 0),
fakeRemoteLivekitParticipant("@dan:example.org:DEV333", 0),
mockRemoteParticipant({ identity: "@alice:example.org:DEV000" }),
mockRemoteParticipant({ identity: "@bob:example.org:DEV111" }),
mockRemoteParticipant({ identity: "@carol:example.org:DEV222" }),
// Mock Dan to have no published tracks. We want him to still show show up
// in the participants list.
mockRemoteParticipant({
identity: "@dan:example.org:DEV333",
getTrackPublication: () => undefined,
getTrackPublications: () => [],
}),
];
// Let's simulate 3 members on the livekitRoom
@@ -411,24 +412,26 @@ describe("Publishing participants observations", () => {
);
participants.forEach((p) =>
fakeRoomEventEmiter.emit(RoomEvent.ParticipantConnected, p),
fakeLivekitRoom.emit(RoomEvent.ParticipantConnected, p),
);
// At this point there should be no publishers
expect(observedPublishers.pop()!.length).toEqual(0);
// At this point there should be ~~no~~ publishers
// We do have publisher now, since we do not filter for publishers anymore (to also have participants with only data tracks)
// The filtering we do is just based on the matrixRTC member events.
expect(observedParticipants.pop()!.length).toEqual(4);
participants = [
fakeRemoteLivekitParticipant("@alice:example.org:DEV000", 1),
fakeRemoteLivekitParticipant("@bob:example.org:DEV111", 1),
fakeRemoteLivekitParticipant("@carol:example.org:DEV222", 1),
fakeRemoteLivekitParticipant("@dan:example.org:DEV333", 2),
mockRemoteParticipant({ identity: "@alice:example.org:DEV000" }),
mockRemoteParticipant({ identity: "@bob:example.org:DEV111" }),
mockRemoteParticipant({ identity: "@carol:example.org:DEV222" }),
mockRemoteParticipant({ identity: "@dan:example.org:DEV333" }),
];
participants.forEach((p) =>
fakeRoomEventEmiter.emit(RoomEvent.ParticipantConnected, p),
fakeLivekitRoom.emit(RoomEvent.ParticipantConnected, p),
);
// At this point there should be no publishers
expect(observedPublishers.pop()!.length).toEqual(4);
expect(observedParticipants.pop()!.length).toEqual(4);
});
it("should be scoped to parent scope", (): void => {
@@ -436,16 +439,14 @@ describe("Publishing participants observations", () => {
const connection = setupRemoteConnection();
let observedPublishers: PublishingParticipant[][] = [];
const s = connection.remoteParticipantsWithTracks$.subscribe(
(publishers) => {
observedPublishers.push(publishers);
},
);
let observedParticipants: RemoteParticipant[][] = [];
const s = connection.remoteParticipants$.subscribe((participants) => {
observedParticipants.push(participants);
});
onTestFinished(() => s.unsubscribe());
let participants: RemoteParticipant[] = [
fakeRemoteLivekitParticipant("@bob:example.org:DEV111", 0),
mockRemoteParticipant({ identity: "@bob:example.org:DEV111" }),
];
// Let's simulate 3 members on the livekitRoom
@@ -454,38 +455,29 @@ describe("Publishing participants observations", () => {
);
for (const participant of participants) {
fakeRoomEventEmiter.emit(RoomEvent.ParticipantConnected, participant);
fakeLivekitRoom.emit(RoomEvent.ParticipantConnected, participant);
}
// At this point there should be no publishers
expect(observedPublishers.pop()!.length).toEqual(0);
participants = [fakeRemoteLivekitParticipant("@bob:example.org:DEV111", 1)];
for (const participant of participants) {
fakeRoomEventEmiter.emit(RoomEvent.ParticipantConnected, participant);
}
// We should have bob has a publisher now
const publishers = observedPublishers.pop();
expect(publishers?.length).toEqual(1);
expect(publishers?.[0]?.identity).toEqual("@bob:example.org:DEV111");
// We should have bob as a participant now
const ps = observedParticipants.pop();
expect(ps?.length).toEqual(1);
expect(ps?.[0]?.identity).toEqual("@bob:example.org:DEV111");
// end the parent scope
testScope.end();
observedPublishers = [];
observedParticipants = [];
// SHOULD NOT emit any more publishers as the scope is ended
// SHOULD NOT emit any more participants as the scope is ended
participants = participants.filter(
(p) => p.identity !== "@bob:example.org:DEV111",
);
fakeRoomEventEmiter.emit(
fakeLivekitRoom.emit(
RoomEvent.ParticipantDisconnected,
fakeRemoteLivekitParticipant("@bob:example.org:DEV111"),
mockRemoteParticipant({ identity: "@bob:example.org:DEV111" }),
);
expect(observedPublishers.length).toEqual(0);
expect(observedParticipants.length).toEqual(0);
});
});

161
src/state/CallViewModel/remoteMembers/Connection.ts
View File

@@ -12,14 +12,11 @@ import {
} from "@livekit/components-core";
import {
ConnectionError,
type ConnectionState as LivekitConenctionState,
type Room as LivekitRoom,
type LocalParticipant,
type RemoteParticipant,
RoomEvent,
} from "livekit-client";
import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
import { BehaviorSubject, map, type Observable } from "rxjs";
import { BehaviorSubject, map } from "rxjs";
import { type Logger } from "matrix-js-sdk/lib/logger";
import {
@@ -30,12 +27,12 @@ import {
import { type Behavior } from "../../Behavior.ts";
import { type ObservableScope } from "../../ObservableScope.ts";
import {
ElementCallError,
InsufficientCapacityError,
SFURoomCreationRestrictedError,
UnknownCallError,
} from "../../../utils/errors.ts";
export type PublishingParticipant = LocalParticipant | RemoteParticipant;
export interface ConnectionOpts {
/** The media transport to connect to. */
transport: LivekitTransport;
@@ -47,19 +44,30 @@ export interface ConnectionOpts {
/** Optional factory to create the LiveKit room, mainly for testing purposes. */
livekitRoomFactory: () => LivekitRoom;
}
export class FailedToStartError extends Error {
public constructor(message: string) {
super(message);
this.name = "FailedToStartError";
}
}
export type ConnectionState =
| { state: "Initialized" }
| { state: "FetchingConfig"; transport: LivekitTransport }
| { state: "ConnectingToLkRoom"; transport: LivekitTransport }
| { state: "PublishingTracks"; transport: LivekitTransport }
| { state: "FailedToStart"; error: Error; transport: LivekitTransport }
| {
state: "ConnectedToLkRoom";
livekitConnectionState$: Observable<LivekitConenctionState>;
transport: LivekitTransport;
}
| { state: "Stopped"; transport: LivekitTransport };
export enum ConnectionState {
/** The start state of a connection. It has been created but nothing has loaded yet. */
Initialized = "Initialized",
/** `start` has been called on the connection. It aquires the jwt info to conenct to the LK Room */
FetchingConfig = "FetchingConfig",
Stopped = "Stopped",
/** The same as ConnectionState.Disconnected from `livekit-client` */
LivekitDisconnected = "disconnected",
/** The same as ConnectionState.Connecting from `livekit-client` */
LivekitConnecting = "connecting",
/** The same as ConnectionState.Connected from `livekit-client` */
LivekitConnected = "connected",
/** The same as ConnectionState.Reconnecting from `livekit-client` */
LivekitReconnecting = "reconnecting",
/** The same as ConnectionState.SignalReconnecting from `livekit-client` */
LivekitSignalReconnecting = "signalReconnecting",
}
/**
* A connection to a Matrix RTC LiveKit backend.
@@ -68,14 +76,32 @@ export type ConnectionState =
*/
export class Connection {
// Private Behavior
private readonly _state$ = new BehaviorSubject<ConnectionState>({
state: "Initialized",
});
private readonly _state$ = new BehaviorSubject<
ConnectionState | ElementCallError
>(ConnectionState.Initialized);
/**
* The current state of the connection to the media transport.
*/
public readonly state$: Behavior<ConnectionState> = this._state$;
public readonly state$: Behavior<ConnectionState | Error> = this._state$;
/**
* The media transport to connect to.
*/
public readonly transport: LivekitTransport;
public readonly livekitRoom: LivekitRoom;
private scope: ObservableScope;
/**
* The remote LiveKit participants that are visible on this connection.
*
* Note that this may include participants that are connected only to
* subscribe, or publishers that are otherwise unattested in MatrixRTC state.
* It is therefore more low-level than what should be presented to the user.
*/
public readonly remoteParticipants$: Behavior<RemoteParticipant[]>;
/**
* Whether the connection has been stopped.
@@ -96,24 +122,29 @@ export class Connection {
* @throws {InsufficientCapacityError} if the LiveKit server indicates that it has insufficient capacity to accept the connection.
* @throws {SFURoomCreationRestrictedError} if the LiveKit server indicates that the room does not exist and cannot be created.
*/
// TODO dont make this throw and instead store a connection error state in this class?
// TODO consider an autostart pattern...
public async start(): Promise<void> {
this.logger.debug("Starting Connection");
this.stopped = false;
try {
this._state$.next({
state: "FetchingConfig",
transport: this.transport,
});
this._state$.next(ConnectionState.FetchingConfig);
// We should already have this information after creating the localTransport.
// It would probably be better to forward this here.
const { url, jwt } = await this.getSFUConfigWithOpenID();
// If we were stopped while fetching the config, don't proceed to connect
if (this.stopped) return;
this._state$.next({
state: "ConnectingToLkRoom",
transport: this.transport,
});
// Setup observer once we are done with getSFUConfigWithOpenID
connectionStateObserver(this.livekitRoom)
.pipe(
this.scope.bind(),
map((s) => s as unknown as ConnectionState),
)
.subscribe((lkState) => {
// It is save to cast lkState to ConnectionState as they are fully overlapping.
this._state$.next(lkState);
});
try {
await this.livekitRoom.connect(url, jwt);
} catch (e) {
@@ -128,7 +159,8 @@ export class Connection {
throw new InsufficientCapacityError();
}
if (e.status === 404) {
// error msg is "Could not establish signal connection: requested room does not exist"
// error msg is "Failed to create call"
// error description is "Call creation might be restricted to authorized users only. Try again later, or contact your server admin if the problem persists."
// The room does not exist. There are two different modes of operation for the SFU:
// - the room is created on the fly when connecting (livekit `auto_create` option)
// - Only authorized users can create rooms, so the room must exist before connecting (done by the auth jwt service)
@@ -140,19 +172,16 @@ export class Connection {
}
// If we were stopped while connecting, don't proceed to update state.
if (this.stopped) return;
this._state$.next({
state: "ConnectedToLkRoom",
transport: this.transport,
livekitConnectionState$: connectionStateObserver(this.livekitRoom),
});
} catch (error) {
this.logger.debug(`Failed to connect to LiveKit room: ${error}`);
this._state$.next({
state: "FailedToStart",
error: error instanceof Error ? error : new Error(`${error}`),
transport: this.transport,
});
this._state$.next(
error instanceof ElementCallError
? error
: error instanceof Error
? new UnknownCallError(error)
: new UnknownCallError(new Error(`${error}`)),
);
// Its okay to ignore the throw. The error is part of the state.
throw error;
}
}
@@ -177,30 +206,11 @@ export class Connection {
);
if (this.stopped) return;
await this.livekitRoom.disconnect();
this._state$.next({
state: "Stopped",
transport: this.transport,
});
this._state$.next(ConnectionState.Stopped);
this.stopped = true;
}
/**
* An observable of the participants that are publishing on this connection. (Excluding our local participant)
* This is derived from `participantsIncludingSubscribers$` and `remoteTransports$`.
* It filters the participants to only those that are associated with a membership that claims to publish on this connection.
*/
public readonly remoteParticipantsWithTracks$: Behavior<
PublishingParticipant[]
>;
/**
* The media transport to connect to.
*/
public readonly transport: LivekitTransport;
private readonly client: OpenIDClientParts;
public readonly livekitRoom: LivekitRoom;
private readonly logger: Logger;
/**
@@ -208,36 +218,23 @@ export class Connection {
*
* @param opts - Connection options {@link ConnectionOpts}.
*
* @param logger
* @param logger - The logger to use.
*/
public constructor(opts: ConnectionOpts, logger: Logger) {
this.logger = logger.getChild("[Connection]");
this.logger.info(
`[Connection] Creating new connection to ${opts.transport.livekit_service_url} ${opts.transport.livekit_alias}`,
`Creating new connection to ${opts.transport.livekit_service_url} ${opts.transport.livekit_alias}`,
);
const { transport, client, scope } = opts;
this.scope = scope;
this.livekitRoom = opts.livekitRoomFactory();
this.transport = transport;
this.client = client;
// REMOTE participants with track!!!
// this.remoteParticipantsWithTracks$
this.remoteParticipantsWithTracks$ = scope.behavior(
// only tracks remote participants
connectedParticipantsObserver(this.livekitRoom, {
additionalRoomEvents: [
RoomEvent.TrackPublished,
RoomEvent.TrackUnpublished,
],
}).pipe(
map((participants) => {
return participants.filter(
(participant) => participant.getTrackPublications().length > 0,
);
}),
),
[],
this.remoteParticipants$ = scope.behavior(
// Only tracks remote participants
connectedParticipantsObserver(this.livekitRoom),
);
scope.onEnd(() => {

52
src/state/CallViewModel/remoteMembers/ConnectionFactory.ts
View File

@@ -7,13 +7,15 @@ Please see LICENSE in the repository root for full details.
import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
import {
type E2EEOptions,
Room as LivekitRoom,
type RoomOptions,
type BaseKeyProvider,
type E2EEManagerOptions,
type BaseE2EEManager,
} from "livekit-client";
import { type Logger } from "matrix-js-sdk/lib/logger";
import E2EEWorker from "livekit-client/e2ee-worker?worker";
// imported as inline to support worker when loaded from a cdn (cross domain)
import E2EEWorker from "livekit-client/e2ee-worker?worker&inline";
import { type ObservableScope } from "../../ObservableScope.ts";
import { Connection } from "./Connection.ts";
@@ -41,9 +43,11 @@ export class ECConnectionFactory implements ConnectionFactory {
* @param client - The OpenID client parts for authentication, needed to get openID and JWT tokens.
* @param devices - Used for video/audio out/in capture options.
* @param processorState$ - Effects like background blur (only for publishing connection?)
* @param e2eeLivekitOptions - The E2EE options to use for the LiveKit Room.
* @param livekitKeyProvider - Optional key provider for end-to-end encryption.
* @param controlledAudioDevices - Option to indicate whether audio output device is controlled externally (native mobile app).
* @param livekitRoomFactory - Optional factory function (for testing) to create LivekitRoom instances. If not provided, a default factory is used.
* @param echoCancellation - Whether to enable echo cancellation for audio capture.
* @param noiseSuppression - Whether to enable noise suppression for audio capture.
*/
public constructor(
private client: OpenIDClientParts,
@@ -52,20 +56,24 @@ export class ECConnectionFactory implements ConnectionFactory {
livekitKeyProvider: BaseKeyProvider | undefined,
private controlledAudioDevices: boolean,
livekitRoomFactory?: () => LivekitRoom,
echoCancellation: boolean = true,
noiseSuppression: boolean = true,
) {
const defaultFactory = (): LivekitRoom =>
new LivekitRoom(
generateRoomOption(
this.devices,
this.processorState$.value,
livekitKeyProvider && {
generateRoomOption({
devices: this.devices,
processorState: this.processorState$.value,
e2eeLivekitOptions: livekitKeyProvider && {
keyProvider: livekitKeyProvider,
// It's important that every room use a separate E2EE worker.
// They get confused if given streams from multiple rooms.
worker: new E2EEWorker(),
},
this.controlledAudioDevices,
),
controlledAudioDevices: this.controlledAudioDevices,
echoCancellation,
noiseSuppression,
}),
);
this.livekitRoomFactory = livekitRoomFactory ?? defaultFactory;
}
@@ -90,12 +98,24 @@ export class ECConnectionFactory implements ConnectionFactory {
/**
* Generate the initial LiveKit RoomOptions based on the current media devices and processor state.
*/
function generateRoomOption(
devices: MediaDevices,
processorState: ProcessorState,
e2eeLivekitOptions: E2EEOptions | undefined,
controlledAudioDevices: boolean,
): RoomOptions {
function generateRoomOption({
devices,
processorState,
e2eeLivekitOptions,
controlledAudioDevices,
echoCancellation,
noiseSuppression,
}: {
devices: MediaDevices;
processorState: ProcessorState;
e2eeLivekitOptions:
| E2EEManagerOptions
| { e2eeManager: BaseE2EEManager }
| undefined;
controlledAudioDevices: boolean;
echoCancellation: boolean;
noiseSuppression: boolean;
}): RoomOptions {
return {
...defaultLiveKitOptions,
videoCaptureDefaults: {
@@ -106,6 +126,8 @@ function generateRoomOption(
audioCaptureDefaults: {
...defaultLiveKitOptions.audioCaptureDefaults,
deviceId: devices.audioInput.selected$.value?.id,
echoCancellation,
noiseSuppression,
},
audioOutput: {
// When using controlled audio devices, we don't want to set the

96
src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts
View File

@@ -8,7 +8,7 @@ Please see LICENSE in the repository root for full details.
import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";
import { BehaviorSubject } from "rxjs";
import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
import { type Participant as LivekitParticipant } from "livekit-client";
import { type RemoteParticipant } from "livekit-client";
import { logger } from "matrix-js-sdk/lib/logger";
import { Epoch, mapEpoch, ObservableScope } from "../../ObservableScope.ts";
@@ -52,7 +52,7 @@ beforeEach(() => {
(transport: LivekitTransport, scope: ObservableScope) => {
const mockConnection = {
transport,
remoteParticipantsWithTracks$: new BehaviorSubject([]),
remoteParticipants$: new BehaviorSubject([]),
} as unknown as Connection;
vi.mocked(mockConnection).start = vi.fn();
vi.mocked(mockConnection).stop = vi.fn();
@@ -200,24 +200,21 @@ describe("connections$ stream", () => {
});
describe("connectionManagerData$ stream", () => {
// Used in test to control fake connections' remoteParticipantsWithTracks$ streams
let fakePublishingParticipantsStreams: Map<
string,
Behavior<LivekitParticipant[]>
>;
// Used in test to control fake connections' remoteParticipants$ streams
let fakeRemoteParticipantsStreams: Map<string, Behavior<RemoteParticipant[]>>;
function keyForTransport(transport: LivekitTransport): string {
return `${transport.livekit_service_url}|${transport.livekit_alias}`;
}
beforeEach(() => {
fakePublishingParticipantsStreams = new Map();
fakeRemoteParticipantsStreams = new Map();
function getPublishingParticipantsFor(
function getRemoteParticipantsFor(
transport: LivekitTransport,
): Behavior<LivekitParticipant[]> {
): Behavior<RemoteParticipant[]> {
return (
fakePublishingParticipantsStreams.get(keyForTransport(transport)) ??
fakeRemoteParticipantsStreams.get(keyForTransport(transport)) ??
new BehaviorSubject([])
);
}
@@ -227,13 +224,12 @@ describe("connectionManagerData$ stream", () => {
.fn()
.mockImplementation(
(transport: LivekitTransport, scope: ObservableScope) => {
const fakePublishingParticipants$ = new BehaviorSubject<
LivekitParticipant[]
const fakeRemoteParticipants$ = new BehaviorSubject<
RemoteParticipant[]
>([]);
const mockConnection = {
transport,
remoteParticipantsWithTracks$:
getPublishingParticipantsFor(transport),
remoteParticipants$: getRemoteParticipantsFor(transport),
} as unknown as Connection;
vi.mocked(mockConnection).start = vi.fn();
vi.mocked(mockConnection).stop = vi.fn();
@@ -242,36 +238,36 @@ describe("connectionManagerData$ stream", () => {
void mockConnection.stop();
});
fakePublishingParticipantsStreams.set(
fakeRemoteParticipantsStreams.set(
keyForTransport(transport),
fakePublishingParticipants$,
fakeRemoteParticipants$,
);
return mockConnection;
},
);
});
test("Should report connections with the publishing participants", () => {
test("Should report connections with the remote participants", () => {
withTestScheduler(({ expectObservable, schedule, behavior }) => {
// Setup the fake participants streams behavior
// ==============================
fakePublishingParticipantsStreams.set(
fakeRemoteParticipantsStreams.set(
keyForTransport(TRANSPORT_1),
behavior("oa-b", {
o: [],
a: [{ identity: "user1A" } as LivekitParticipant],
a: [{ identity: "user1A" } as RemoteParticipant],
b: [
{ identity: "user1A" } as LivekitParticipant,
{ identity: "user1B" } as LivekitParticipant,
{ identity: "user1A" } as RemoteParticipant,
{ identity: "user1B" } as RemoteParticipant,
],
}),
);
fakePublishingParticipantsStreams.set(
fakeRemoteParticipantsStreams.set(
keyForTransport(TRANSPORT_2),
behavior("o-a", {
o: [],
a: [{ identity: "user2A" } as LivekitParticipant],
a: [{ identity: "user2A" } as RemoteParticipant],
}),
);
// ==============================
@@ -289,47 +285,47 @@ describe("connectionManagerData$ stream", () => {
a: expect.toSatisfy((e) => {
const data: ConnectionManagerData = e.value;
expect(data.getConnections().length).toBe(2);
expect(data.getParticipantForTransport(TRANSPORT_1).length).toBe(0);
expect(data.getParticipantForTransport(TRANSPORT_2).length).toBe(0);
expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(0);
expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(0);
return true;
}),
b: expect.toSatisfy((e) => {
const data: ConnectionManagerData = e.value;
expect(data.getConnections().length).toBe(2);
expect(data.getParticipantForTransport(TRANSPORT_1).length).toBe(1);
expect(data.getParticipantForTransport(TRANSPORT_2).length).toBe(0);
expect(data.getParticipantForTransport(TRANSPORT_1)[0].identity).toBe(
"user1A",
);
expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(1);
expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(0);
expect(
data.getParticipantsForTransport(TRANSPORT_1)[0].identity,
).toBe("user1A");
return true;
}),
c: expect.toSatisfy((e) => {
const data: ConnectionManagerData = e.value;
expect(data.getConnections().length).toBe(2);
expect(data.getParticipantForTransport(TRANSPORT_1).length).toBe(1);
expect(data.getParticipantForTransport(TRANSPORT_2).length).toBe(1);
expect(data.getParticipantForTransport(TRANSPORT_1)[0].identity).toBe(
"user1A",
);
expect(data.getParticipantForTransport(TRANSPORT_2)[0].identity).toBe(
"user2A",
);
expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(1);
expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(1);
expect(
data.getParticipantsForTransport(TRANSPORT_1)[0].identity,
).toBe("user1A");
expect(
data.getParticipantsForTransport(TRANSPORT_2)[0].identity,
).toBe("user2A");
return true;
}),
d: expect.toSatisfy((e) => {
const data: ConnectionManagerData = e.value;
expect(data.getConnections().length).toBe(2);
expect(data.getParticipantForTransport(TRANSPORT_1).length).toBe(2);
expect(data.getParticipantForTransport(TRANSPORT_2).length).toBe(1);
expect(data.getParticipantForTransport(TRANSPORT_1)[0].identity).toBe(
"user1A",
);
expect(data.getParticipantForTransport(TRANSPORT_1)[1].identity).toBe(
"user1B",
);
expect(data.getParticipantForTransport(TRANSPORT_2)[0].identity).toBe(
"user2A",
);
expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(2);
expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(1);
expect(
data.getParticipantsForTransport(TRANSPORT_1)[0].identity,
).toBe("user1A");
expect(
data.getParticipantsForTransport(TRANSPORT_1)[1].identity,
).toBe("user1B");
expect(
data.getParticipantsForTransport(TRANSPORT_2)[0].identity,
).toBe("user2A");
return true;
}),
});

83
src/state/CallViewModel/remoteMembers/ConnectionManager.ts
View File

@@ -6,13 +6,10 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
type LivekitTransport,
type ParticipantId,
} from "matrix-js-sdk/lib/matrixrtc";
import { BehaviorSubject, combineLatest, map, of, switchMap, tap } from "rxjs";
import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
import { combineLatest, map, of, switchMap, tap } from "rxjs";
import { type Logger } from "matrix-js-sdk/lib/logger";
import { type LocalParticipant, type RemoteParticipant } from "livekit-client";
import { type RemoteParticipant } from "livekit-client";
import { type Behavior } from "../../Behavior.ts";
import { type Connection } from "./Connection.ts";
@@ -24,21 +21,18 @@ import { type ConnectionFactory } from "./ConnectionFactory.ts";
export class ConnectionManagerData {
private readonly store: Map<
string,
[Connection, (LocalParticipant | RemoteParticipant)[]]
{ connection: Connection; participants: RemoteParticipant[] }
> = new Map();
public constructor() {}
public add(
connection: Connection,
participants: (LocalParticipant | RemoteParticipant)[],
): void {
public add(connection: Connection, participants: RemoteParticipant[]): void {
const key = this.getKey(connection.transport);
const existing = this.store.get(key);
if (!existing) {
this.store.set(key, [connection, participants]);
this.store.set(key, { connection, participants });
} else {
existing[1].push(...participants);
existing.participants.push(...participants);
}
}
@@ -47,59 +41,46 @@ export class ConnectionManagerData {
}
public getConnections(): Connection[] {
return Array.from(this.store.values()).map(([connection]) => connection);
return Array.from(this.store.values()).map(({ connection }) => connection);
}
public getConnectionForTransport(
transport: LivekitTransport,
): Connection | null {
return this.store.get(this.getKey(transport))?.[0] ?? null;
return this.store.get(this.getKey(transport))?.connection ?? null;
}
public getParticipantForTransport(
public getParticipantsForTransport(
transport: LivekitTransport,
): (LocalParticipant | RemoteParticipant)[] {
): RemoteParticipant[] {
const key = transport.livekit_service_url + "|" + transport.livekit_alias;
const existing = this.store.get(key);
if (existing) {
return existing[1];
return existing.participants;
}
return [];
}
/**
* Get all connections where the given participant is publishing.
* In theory, there could be several connections where the same participant is publishing but with
* only well behaving clients a participant should only be publishing on a single connection.
* @param participantId
*/
public getConnectionsForParticipant(
participantId: ParticipantId,
): Connection[] {
const connections: Connection[] = [];
for (const [connection, participants] of this.store.values()) {
if (participants.some((p) => p.identity === participantId)) {
connections.push(connection);
}
}
return connections;
}
}
interface Props {
scope: ObservableScope;
connectionFactory: ConnectionFactory;
inputTransports$: Behavior<Epoch<LivekitTransport[]>>;
logger: Logger;
}
// TODO - write test for scopes (do we really need to bind scope)
export interface IConnectionManager {
transports$: Behavior<Epoch<LivekitTransport[]>>;
connectionManagerData$: Behavior<Epoch<ConnectionManagerData>>;
}
/**
* Crete a `ConnectionManager`
* @param scope the observable scope used by this object.
* @param connectionFactory used to create new connections.
* @param _transportsSubscriptions$ A list of Behaviors each containing a LIST of LivekitTransport.
* @param props - Configuration object
* @param props.scope - The observable scope used by this object
* @param props.connectionFactory - Used to create new connections
* @param props.inputTransports$ - A list of Behaviors each containing a LIST of LivekitTransport.
* @param props.logger - The logger to use
* Each of these behaviors can be interpreted as subscribed list of transports.
*
* Using `registerTransports` independent external modules can control what connections
@@ -116,9 +97,6 @@ export function createConnectionManager$({
logger: parentLogger,
}: Props): IConnectionManager {
const logger = parentLogger.getChild("[ConnectionManager]");
const running$ = new BehaviorSubject(true);
scope.onEnd(() => running$.next(false));
// TODO logger: only construct one logger from the client and make it compatible via a EC specific sing
/**
@@ -130,10 +108,7 @@ export function createConnectionManager$({
* externally this is modified via `registerTransports()`.
*/
const transports$ = scope.behavior(
combineLatest([running$, inputTransports$]).pipe(
map(([running, transports]) =>
transports.mapInner((transport) => (running ? transport : [])),
),
inputTransports$.pipe(
map((transports) => transports.mapInner(removeDuplicateTransports)),
tap(({ value: transports }) => {
logger.trace(
@@ -183,23 +158,25 @@ export function createConnectionManager$({
const epoch = connections.epoch;
// Map the connections to list of {connection, participants}[]
const listOfConnectionsWithPublishingParticipants =
connections.value.map((connection) => {
return connection.remoteParticipantsWithTracks$.pipe(
const listOfConnectionsWithRemoteParticipants = connections.value.map(
(connection) => {
return connection.remoteParticipants$.pipe(
map((participants) => ({
connection,
participants,
})),
);
});
},
);
// probably not required
if (listOfConnectionsWithPublishingParticipants.length === 0) {
if (listOfConnectionsWithRemoteParticipants.length === 0) {
return of(new Epoch(new ConnectionManagerData(), epoch));
}
// combineLatest the several streams into a single stream with the ConnectionManagerData
return combineLatest(listOfConnectionsWithPublishingParticipants).pipe(
return combineLatest(listOfConnectionsWithRemoteParticipants).pipe(
map(
(lists) =>
new Epoch(
@@ -216,7 +193,7 @@ export function createConnectionManager$({
new Epoch(new ConnectionManagerData()),
);
return { transports$, connectionManagerData$ };
return { connectionManagerData$ };
}
function removeDuplicateTransports(

133
src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts Normal file
View File

@@ -0,0 +1,133 @@
/*
Copyright 2025 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";
import { Room as LivekitRoom } from "livekit-client";
import { BehaviorSubject } from "rxjs";
import fetchMock from "fetch-mock";
import { logger } from "matrix-js-sdk/lib/logger";
import EventEmitter from "events";
import { ObservableScope } from "../../ObservableScope.ts";
import { ECConnectionFactory } from "./ConnectionFactory.ts";
import type { OpenIDClientParts } from "../../../livekit/openIDSFU.ts";
import { exampleTransport, mockMediaDevices } from "../../../utils/test.ts";
import type { ProcessorState } from "../../../livekit/TrackProcessorContext.tsx";
import { constant } from "../../Behavior";
// At the top of your test file, after imports
vi.mock("livekit-client", async (importOriginal) => {
return {
...(await importOriginal()),
Room: vi.fn().mockImplementation(function (this: LivekitRoom, options) {
const emitter = new EventEmitter();
return {
on: emitter.on.bind(emitter),
off: emitter.off.bind(emitter),
emit: emitter.emit.bind(emitter),
disconnect: vi.fn(),
remoteParticipants: new Map(),
} as unknown as LivekitRoom;
}),
};
});
let testScope: ObservableScope;
let mockClient: OpenIDClientParts;
beforeEach(() => {
testScope = new ObservableScope();
mockClient = {
getOpenIdToken: vi.fn().mockReturnValue(""),
getDeviceId: vi.fn().mockReturnValue("DEV000"),
};
});
describe("ECConnectionFactory - Audio inputs options", () => {
test.each([
{ echo: true, noise: true },
{ echo: true, noise: false },
{ echo: false, noise: true },
{ echo: false, noise: false },
])(
"it sets echoCancellation=$echo and noiseSuppression=$noise based on constructor parameters",
({ echo, noise }) => {
// test("it sets echoCancellation and noiseSuppression based on constructor parameters", () => {
const RoomConstructor = vi.mocked(LivekitRoom);
const ecConnectionFactory = new ECConnectionFactory(
mockClient,
mockMediaDevices({}),
new BehaviorSubject<ProcessorState>({
supported: true,
processor: undefined,
}),
undefined,
false,
undefined,
echo,
noise,
);
ecConnectionFactory.createConnection(exampleTransport, testScope, logger);
// Check if Room was constructed with expected options
expect(RoomConstructor).toHaveBeenCalledWith(
expect.objectContaining({
audioCaptureDefaults: expect.objectContaining({
echoCancellation: echo,
noiseSuppression: noise,
}),
}),
);
},
);
});
describe("ECConnectionFactory - ControlledAudioDevice", () => {
test.each([{ controlled: true }, { controlled: false }])(
"it sets controlledAudioDevice=$controlled then uses deviceId accordingly",
({ controlled }) => {
// test("it sets echoCancellation and noiseSuppression based on constructor parameters", () => {
const RoomConstructor = vi.mocked(LivekitRoom);
const ecConnectionFactory = new ECConnectionFactory(
mockClient,
mockMediaDevices({
audioOutput: {
available$: constant(new Map<never, never>()),
selected$: constant({ id: "DEV00", virtualEarpiece: false }),
select: () => {},
},
}),
new BehaviorSubject<ProcessorState>({
supported: true,
processor: undefined,
}),
undefined,
controlled,
undefined,
false,
false,
);
ecConnectionFactory.createConnection(exampleTransport, testScope, logger);
// Check if Room was constructed with expected options
expect(RoomConstructor).toHaveBeenCalledWith(
expect.objectContaining({
audioOutput: expect.objectContaining({
deviceId: controlled ? undefined : "DEV00",
}),
}),
);
},
);
});
afterEach(() => {
testScope.end();
fetchMock.reset();
});

131
src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts
View File

@@ -15,7 +15,7 @@ import { combineLatest, map, type Observable } from "rxjs";
import { type IConnectionManager } from "./ConnectionManager.ts";
import {
type MatrixLivekitMember,
type RemoteMatrixLivekitMember,
createMatrixLivekitMembers$,
} from "./MatrixLivekitMembers.ts";
import {
@@ -91,7 +91,7 @@ test("should signal participant not yet connected to livekit", () => {
}),
);
const matrixLivekitMember$ = createMatrixLivekitMembers$({
const matrixLivekitMembers$ = createMatrixLivekitMembers$({
scope: testScope,
membershipsWithTransport$: testScope.behavior(membershipsWithTransport$),
connectionManager: {
@@ -99,21 +99,24 @@ test("should signal participant not yet connected to livekit", () => {
} as unknown as IConnectionManager,
});
expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe("a", {
a: expect.toSatisfy((data: MatrixLivekitMember[]) => {
expect(data.length).toEqual(1);
expectObservable(data[0].membership$).toBe("a", {
a: bobMembership,
});
expectObservable(data[0].participant$).toBe("a", {
a: null,
});
expectObservable(data[0].connection$).toBe("a", {
a: null,
});
return true;
}),
});
expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe(
"a",
{
a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => {
expect(data.length).toEqual(1);
expectObservable(data[0].membership$).toBe("a", {
a: bobMembership,
});
expectObservable(data[0].participant.value$).toBe("a", {
a: null,
});
expectObservable(data[0].connection$).toBe("a", {
a: null,
});
return true;
}),
},
);
});
});
@@ -171,7 +174,7 @@ test("should signal participant on a connection that is publishing", () => {
}),
);
const matrixLivekitMember$ = createMatrixLivekitMembers$({
const matrixLivekitMembers$ = createMatrixLivekitMembers$({
scope: testScope,
membershipsWithTransport$: testScope.behavior(membershipsWithTransport$),
connectionManager: {
@@ -179,25 +182,28 @@ test("should signal participant on a connection that is publishing", () => {
} as unknown as IConnectionManager,
});
expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe("a", {
a: expect.toSatisfy((data: MatrixLivekitMember[]) => {
expect(data.length).toEqual(1);
expectObservable(data[0].membership$).toBe("a", {
a: bobMembership,
});
expectObservable(data[0].participant$).toBe("a", {
a: expect.toSatisfy((participant) => {
expect(participant).toBeDefined();
expect(participant!.identity).toEqual(bobParticipantId);
return true;
}),
});
expectObservable(data[0].connection$).toBe("a", {
a: connection,
});
return true;
}),
});
expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe(
"a",
{
a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => {
expect(data.length).toEqual(1);
expectObservable(data[0].membership$).toBe("a", {
a: bobMembership,
});
expectObservable(data[0].participant.value$).toBe("a", {
a: expect.toSatisfy((participant) => {
expect(participant).toBeDefined();
expect(participant!.identity).toEqual(bobParticipantId);
return true;
}),
});
expectObservable(data[0].connection$).toBe("a", {
a: connection,
});
return true;
}),
},
);
});
});
@@ -222,7 +228,7 @@ test("should signal participant on a connection that is not publishing", () => {
}),
);
const matrixLivekitMember$ = createMatrixLivekitMembers$({
const matrixLivekitMembers$ = createMatrixLivekitMembers$({
scope: testScope,
membershipsWithTransport$: testScope.behavior(membershipsWithTransport$),
connectionManager: {
@@ -230,21 +236,24 @@ test("should signal participant on a connection that is not publishing", () => {
} as unknown as IConnectionManager,
});
expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe("a", {
a: expect.toSatisfy((data: MatrixLivekitMember[]) => {
expect(data.length).toEqual(1);
expectObservable(data[0].membership$).toBe("a", {
a: bobMembership,
});
expectObservable(data[0].participant$).toBe("a", {
a: null,
});
expectObservable(data[0].connection$).toBe("a", {
a: connection,
});
return true;
}),
});
expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe(
"a",
{
a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => {
expect(data.length).toEqual(1);
expectObservable(data[0].membership$).toBe("a", {
a: bobMembership,
});
expectObservable(data[0].participant.value$).toBe("a", {
a: null,
});
expectObservable(data[0].connection$).toBe("a", {
a: connection,
});
return true;
}),
},
);
});
});
@@ -283,7 +292,7 @@ describe("Publication edge case", () => {
}),
);
const matrixLivekitMember$ = createMatrixLivekitMembers$({
const matrixLivekitMembers$ = createMatrixLivekitMembers$({
scope: testScope,
membershipsWithTransport$: testScope.behavior(
membershipsWithTransport$,
@@ -293,10 +302,10 @@ describe("Publication edge case", () => {
} as unknown as IConnectionManager,
});
expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe(
expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe(
"a",
{
a: expect.toSatisfy((data: MatrixLivekitMember[]) => {
a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => {
expect(data.length).toEqual(2);
expectObservable(data[0].membership$).toBe("a", {
a: bobMembership,
@@ -305,7 +314,7 @@ describe("Publication edge case", () => {
// The real connection should be from transportA as per the membership
a: connectionA,
});
expectObservable(data[0].participant$).toBe("a", {
expectObservable(data[0].participant.value$).toBe("a", {
a: expect.toSatisfy((participant) => {
expect(participant).toBeDefined();
expect(participant!.identity).toEqual(bobParticipantId);
@@ -349,7 +358,7 @@ describe("Publication edge case", () => {
}),
);
const matrixLivekitMember$ = createMatrixLivekitMembers$({
const matrixLivekitMembers$ = createMatrixLivekitMembers$({
scope: testScope,
membershipsWithTransport$: testScope.behavior(
membershipsWithTransport$,
@@ -359,10 +368,10 @@ describe("Publication edge case", () => {
} as unknown as IConnectionManager,
});
expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe(
expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe(
"a",
{
a: expect.toSatisfy((data: MatrixLivekitMember[]) => {
a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => {
expect(data.length).toEqual(2);
expectObservable(data[0].membership$).toBe("a", {
a: bobMembership,
@@ -371,7 +380,7 @@ describe("Publication edge case", () => {
// The real connection should be from transportA as per the membership
a: connectionA,
});
expectObservable(data[0].participant$).toBe("a", {
expectObservable(data[0].participant.value$).toBe("a", {
// No participant as Bob is not publishing on his membership transport
a: null,
});

55
src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts
View File

@@ -5,10 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
type LocalParticipant as LocalLivekitParticipant,
type RemoteParticipant as RemoteLivekitParticipant,
} from "livekit-client";
import { type LocalParticipant, type RemoteParticipant } from "livekit-client";
import {
type LivekitTransport,
type CallMembership,
@@ -24,22 +21,44 @@ import { generateItemsWithEpoch } from "../../../utils/observable";
const logger = rootLogger.getChild("[MatrixLivekitMembers]");
/**
* Represents a Matrix call member and their associated LiveKit participation.
* `livekitParticipant` can be undefined if the member is not yet connected to the livekit room
* or if it has no livekit transport at all.
*/
export interface MatrixLivekitMember {
interface LocalTaggedParticipant {
type: "local";
value$: Behavior<LocalParticipant | null>;
}
interface RemoteTaggedParticipant {
type: "remote";
value$: Behavior<RemoteParticipant | null>;
}
export type TaggedParticipant =
| LocalTaggedParticipant
| RemoteTaggedParticipant;
interface MatrixLivekitMember {
membership$: Behavior<CallMembership>;
participant$: Behavior<
LocalLivekitParticipant | RemoteLivekitParticipant | null
>;
connection$: Behavior<Connection | null>;
// participantId: string; We do not want a participantId here since it will be generated by the jwt
// TODO decide if we can also drop the userId. Its in the matrix membership anyways.
userId: string;
}
/**
* Represents the local Matrix call member and their associated LiveKit participation.
* `livekitParticipant` can be null if the member is not yet connected to the livekit room
* or if it has no livekit transport at all.
*/
export interface LocalMatrixLivekitMember extends MatrixLivekitMember {
participant: LocalTaggedParticipant;
}
/**
* Represents a remote Matrix call member and their associated LiveKit participation.
* `livekitParticipant` can be null if the member is not yet connected to the livekit room
* or if it has no livekit transport at all.
*/
export interface RemoteMatrixLivekitMember extends MatrixLivekitMember {
participant: RemoteTaggedParticipant;
}
interface Props {
scope: ObservableScope;
membershipsWithTransport$: Behavior<
@@ -61,7 +80,7 @@ export function createMatrixLivekitMembers$({
scope,
membershipsWithTransport$,
connectionManager,
}: Props): Behavior<Epoch<MatrixLivekitMember[]>> {
}: Props): Behavior<Epoch<RemoteMatrixLivekitMember[]>> {
/**
* Stream of all the call members and their associated livekit data (if available).
*/
@@ -91,7 +110,7 @@ export function createMatrixLivekitMembers$({
const participantId = /*membership.membershipID*/ `${membership.userId}:${membership.deviceId}`;
const participants = transport
? managerData.getParticipantForTransport(transport)
? managerData.getParticipantsForTransport(transport)
: [];
const participant =
participants.find((p) => p.identity == participantId) ?? null;
@@ -108,14 +127,16 @@ export function createMatrixLivekitMembers$({
// Each update where the key of the generator array do not change will result in updates to the `data$` observable in the factory.
(scope, data$, participantId, userId) => {
logger.debug(
`Updating data$ for participantId: ${participantId}, userId: ${userId}`,
`Generating member for participantId: ${participantId}, userId: ${userId}`,
);
const { participant$, ...rest } = scope.splitBehavior(data$);
// will only get called once per `participantId, userId` pair.
// updates to data$ and as a result to displayName$ and mxcAvatarUrl$ are more frequent.
return {
participantId,
userId,
...scope.splitBehavior(data$),
participant: { type: "remote" as const, value$: participant$ },
...rest,
};
},
),

23
src/state/CallViewModel/remoteMembers/integration.test.ts
View File

@@ -29,10 +29,11 @@ import { type ProcessorState } from "../../../livekit/TrackProcessorContext.tsx"
import {
areLivekitTransportsEqual,
createMatrixLivekitMembers$,
type MatrixLivekitMember,
type RemoteMatrixLivekitMember,
} from "./MatrixLivekitMembers.ts";
import { createConnectionManager$ } from "./ConnectionManager.ts";
import { membershipsAndTransports$ } from "../../SessionBehaviors.ts";
import { testJWTToken } from "../../../utils/test-fixtures.ts";
// Test the integration of ConnectionManager and MatrixLivekitMerger
@@ -85,7 +86,7 @@ beforeEach(() => {
status: 200,
body: {
url: `wss://${domain}/livekit/sfu`,
jwt: "ATOKEN",
jwt: testJWTToken,
},
};
});
@@ -124,15 +125,15 @@ test("bob, carl, then bob joining no tracks yet", () => {
logger: logger,
});
const matrixLivekitItems$ = createMatrixLivekitMembers$({
const matrixLivekitMembers$ = createMatrixLivekitMembers$({
scope: testScope,
membershipsWithTransport$:
membershipsAndTransports.membershipsWithTransport$,
connectionManager,
});
expectObservable(matrixLivekitItems$).toBe(vMarble, {
a: expect.toSatisfy((e: Epoch<MatrixLivekitMember[]>) => {
expectObservable(matrixLivekitMembers$).toBe(vMarble, {
a: expect.toSatisfy((e: Epoch<RemoteMatrixLivekitMember[]>) => {
const items = e.value;
expect(items.length).toBe(1);
const item = items[0]!;
@@ -147,12 +148,12 @@ test("bob, carl, then bob joining no tracks yet", () => {
),
),
});
expectObservable(item.participant$).toBe("a", {
expectObservable(item.participant.value$).toBe("a", {
a: null,
});
return true;
}),
b: expect.toSatisfy((e: Epoch<MatrixLivekitMember[]>) => {
b: expect.toSatisfy((e: Epoch<RemoteMatrixLivekitMember[]>) => {
const items = e.value;
expect(items.length).toBe(2);
@@ -161,7 +162,7 @@ test("bob, carl, then bob joining no tracks yet", () => {
expectObservable(item.membership$).toBe("a", {
a: bobMembership,
});
expectObservable(item.participant$).toBe("a", {
expectObservable(item.participant.value$).toBe("a", {
a: null,
});
}
@@ -172,7 +173,7 @@ test("bob, carl, then bob joining no tracks yet", () => {
expectObservable(item.membership$).toBe("a", {
a: carlMembership,
});
expectObservable(item.participant$).toBe("a", {
expectObservable(item.participant.value$).toBe("a", {
a: null,
});
expectObservable(item.connection$).toBe("a", {
@@ -189,7 +190,7 @@ test("bob, carl, then bob joining no tracks yet", () => {
}
return true;
}),
c: expect.toSatisfy((e: Epoch<MatrixLivekitMember[]>) => {
c: expect.toSatisfy((e: Epoch<RemoteMatrixLivekitMember[]>) => {
const items = e.value;
expect(items.length).toBe(3);
@@ -216,7 +217,7 @@ test("bob, carl, then bob joining no tracks yet", () => {
return true;
}),
});
expectObservable(item.participant$).toBe("a", {
expectObservable(item.participant.value$).toBe("a", {
a: null,
});
}

70
src/state/CallViewModelWidget.test.ts
View File

@@ -15,6 +15,7 @@ import { constant } from "./Behavior.ts";
import { aliceParticipant, localRtcMember } from "../utils/test-fixtures.ts";
import { ElementWidgetActions, widget } from "../widget.ts";
import { E2eeType } from "../e2ee/e2eeType.ts";
import { MatrixRTCMode } from "../settings/settings.ts";
vi.mock("@livekit/components-core", { spy: true });
@@ -34,36 +35,43 @@ vi.mock("../widget", () => ({
},
}));
it("expect leave when ElementWidgetActions.HangupCall is called", async () => {
const pr = Promise.withResolvers<string>();
withCallViewModel(
{
remoteParticipants$: constant([aliceParticipant]),
rtcMembers$: constant([localRtcMember]),
},
(vm: CallViewModel) => {
vm.leave$.subscribe((s: string) => {
pr.resolve(s);
});
it.each([
[MatrixRTCMode.Legacy],
[MatrixRTCMode.Compatibil],
[MatrixRTCMode.Matrix_2_0],
])(
"expect leave when ElementWidgetActions.HangupCall is called (%s mode)",
async (mode) => {
const pr = Promise.withResolvers<string>();
withCallViewModel(mode)(
{
remoteParticipants$: constant([aliceParticipant]),
rtcMembers$: constant([localRtcMember]),
},
(vm: CallViewModel) => {
vm.leave$.subscribe((s: string) => {
pr.resolve(s);
});
widget!.lazyActions!.emit(
ElementWidgetActions.HangupCall,
new CustomEvent(ElementWidgetActions.HangupCall, {
detail: {
action: "im.vector.hangup",
api: "toWidget",
data: {},
requestId: "widgetapi-1761237395918",
widgetId: "mrUjS9T6uKUOWHMxXvLbSv0F",
},
}),
);
},
{
encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
},
);
widget!.lazyActions!.emit(
ElementWidgetActions.HangupCall,
new CustomEvent(ElementWidgetActions.HangupCall, {
detail: {
action: "im.vector.hangup",
api: "toWidget",
data: {},
requestId: "widgetapi-1761237395918",
widgetId: "mrUjS9T6uKUOWHMxXvLbSv0F",
},
}),
);
},
{
encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
},
);
const source = await pr.promise;
expect(source).toBe("user");
});
const source = await pr.promise;
expect(source).toBe("user");
},
);

41
src/state/MediaViewModel.test.ts
View File

@@ -20,6 +20,7 @@ import {
createLocalMedia,
createRemoteMedia,
withTestScheduler,
mockRemoteParticipant,
} from "../utils/test";
import { getValue } from "../utils/observable";
import { constant } from "./Behavior";
@@ -44,7 +45,11 @@ const rtcMembership = mockRtcMembership("@alice:example.org", "AAAA");
test("control a participant's volume", () => {
const setVolumeSpy = vi.fn();
const vm = createRemoteMedia(rtcMembership, {}, { setVolume: setVolumeSpy });
const vm = createRemoteMedia(
rtcMembership,
{},
mockRemoteParticipant({ setVolume: setVolumeSpy }),
);
withTestScheduler(({ expectObservable, schedule }) => {
schedule("-ab---c---d|", {
a() {
@@ -88,7 +93,7 @@ test("control a participant's volume", () => {
});
test("toggle fit/contain for a participant's video", () => {
const vm = createRemoteMedia(rtcMembership, {}, {});
const vm = createRemoteMedia(rtcMembership, {}, mockRemoteParticipant({}));
withTestScheduler(({ expectObservable, schedule }) => {
schedule("-ab|", {
a: () => vm.toggleFitContain(),
@@ -199,3 +204,35 @@ test("switch cameras", async () => {
});
expect(deviceId).toBe("front camera");
});
test("remote media is in waiting state when participant has not yet connected", () => {
const vm = createRemoteMedia(rtcMembership, {}, null); // null participant
expect(vm.waitingForMedia$.value).toBe(true);
});
test("remote media is not in waiting state when participant is connected", () => {
const vm = createRemoteMedia(rtcMembership, {}, mockRemoteParticipant({}));
expect(vm.waitingForMedia$.value).toBe(false);
});
test("remote media is not in waiting state when participant is connected with no publications", () => {
const vm = createRemoteMedia(
rtcMembership,
{},
mockRemoteParticipant({
getTrackPublication: () => undefined,
getTrackPublications: () => [],
}),
);
expect(vm.waitingForMedia$.value).toBe(false);
});
test("remote media is not in waiting state when user does not intend to publish anywhere", () => {
const vm = createRemoteMedia(
rtcMembership,
{},
mockRemoteParticipant({}),
undefined, // No room (no advertised transport)
);
expect(vm.waitingForMedia$.value).toBe(false);
});

45
src/state/MediaViewModel.ts
View File

@@ -7,8 +7,8 @@ Please see LICENSE in the repository root for full details.
import {
type AudioSource,
type TrackReferenceOrPlaceholder,
type VideoSource,
type TrackReference,
observeParticipantEvents,
observeParticipantMedia,
roomEventSelector,
@@ -33,7 +33,6 @@ import {
type Observable,
Subject,
combineLatest,
distinctUntilKeyChanged,
filter,
fromEvent,
interval,
@@ -60,14 +59,11 @@ import { type ObservableScope } from "./ObservableScope";
export function observeTrackReference$(
participant: Participant,
source: Track.Source,
): Observable<TrackReferenceOrPlaceholder> {
): Observable<TrackReference | undefined> {
return observeParticipantMedia(participant).pipe(
map(() => ({
participant: participant,
publication: participant.getTrackPublication(source),
source,
})),
distinctUntilKeyChanged("publication"),
map(() => participant.getTrackPublication(source)),
distinctUntilChanged(),
map((publication) => publication && { participant, publication, source }),
);
}
@@ -226,7 +222,7 @@ abstract class BaseMediaViewModel {
/**
* The LiveKit video track for this media.
*/
public readonly video$: Behavior<TrackReferenceOrPlaceholder | null>;
public readonly video$: Behavior<TrackReference | undefined>;
/**
* Whether there should be a warning that this media is unencrypted.
*/
@@ -241,10 +237,12 @@ abstract class BaseMediaViewModel {
private observeTrackReference$(
source: Track.Source,
): Behavior<TrackReferenceOrPlaceholder | null> {
): Behavior<TrackReference | undefined> {
return this.scope.behavior(
this.participant$.pipe(
switchMap((p) => (!p ? of(null) : observeTrackReference$(p, source))),
switchMap((p) =>
!p ? of(undefined) : observeTrackReference$(p, source),
),
),
);
}
@@ -268,7 +266,7 @@ abstract class BaseMediaViewModel {
encryptionSystem: EncryptionSystem,
audioSource: AudioSource,
videoSource: VideoSource,
livekitRoom$: Behavior<LivekitRoom | undefined>,
protected readonly livekitRoom$: Behavior<LivekitRoom | undefined>,
public readonly focusUrl$: Behavior<string | undefined>,
public readonly displayName$: Behavior<string>,
public readonly mxcAvatarUrl$: Behavior<string | undefined>,
@@ -281,8 +279,8 @@ abstract class BaseMediaViewModel {
[audio$, this.video$],
(a, v) =>
encryptionSystem.kind !== E2eeType.NONE &&
(a?.publication?.isEncrypted === false ||
v?.publication?.isEncrypted === false),
(a?.publication.isEncrypted === false ||
v?.publication.isEncrypted === false),
),
);
@@ -471,7 +469,7 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel {
private readonly videoTrack$: Observable<LocalVideoTrack | null> =
this.video$.pipe(
switchMap((v) => {
const track = v?.publication?.track;
const track = v?.publication.track;
if (!(track instanceof LocalVideoTrack)) return of(null);
return merge(
// Watch for track restarts because they indicate a camera switch.
@@ -596,6 +594,21 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel {
* A remote participant's user media.
*/
export class RemoteUserMediaViewModel extends BaseUserMediaViewModel {
/**
* Whether we are waiting for this user's LiveKit participant to exist. This
* could be because either we or the remote party are still connecting.
*/
public readonly waitingForMedia$ = this.scope.behavior<boolean>(
combineLatest(
[this.livekitRoom$, this.participant$],
(livekitRoom, participant) =>
// If livekitRoom is undefined, the user is not attempting to publish on
// any transport and so we shouldn't expect a participant. (They might
// be a subscribe-only bot for example.)
livekitRoom !== undefined && participant === null,
),
);
// This private field is used to override the value from the superclass
private __speaking$: Behavior<boolean>;
public get speaking$(): Behavior<boolean> {

212
src/state/MuteStates.test.ts Normal file
View File

@@ -0,0 +1,212 @@
/*
Copyright 2025 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";
import { BehaviorSubject } from "rxjs";
import { logger } from "matrix-js-sdk/lib/logger";
import { MuteStates, MuteState } from "./MuteStates";
import {
type AudioOutputDeviceLabel,
type DeviceLabel,
type MediaDevice,
type SelectedAudioOutputDevice,
type SelectedDevice,
} from "./MediaDevices";
import { constant } from "./Behavior";
import { ObservableScope } from "./ObservableScope";
import { flushPromises, mockMediaDevices } from "../utils/test";
const getUrlParams = vi.hoisted(() => vi.fn(() => ({})));
vi.mock("../UrlParams", () => ({ getUrlParams }));
let testScope: ObservableScope;
beforeEach(() => {
testScope = new ObservableScope();
});
afterEach(() => {
testScope.end();
});
describe("MuteState", () => {
test("should automatically mute if force mute is set", async () => {
const forceMute$ = new BehaviorSubject<boolean>(false);
const deviceStub = {
available$: constant(
new Map<string, DeviceLabel>([
["fbac11", { type: "name", name: "HD Camera" }],
]),
),
selected$: constant({ id: "fbac11" }),
select(): void {},
} as unknown as MediaDevice<DeviceLabel, SelectedDevice>;
const muteState = new MuteState(
testScope,
deviceStub,
constant(true),
true,
forceMute$,
);
let lastEnabled: boolean = false;
muteState.enabled$.subscribe((enabled) => {
lastEnabled = enabled;
});
let setEnabled: ((enabled: boolean) => void) | null = null;
muteState.setEnabled$.subscribe((setter) => {
setEnabled = setter;
});
await flushPromises();
setEnabled!(true);
await flushPromises();
expect(lastEnabled).toBe(true);
// Now force mute
forceMute$.next(true);
await flushPromises();
// Should automatically mute
expect(lastEnabled).toBe(false);
// Try to unmute can not work
expect(setEnabled).toBeNull();
// Disable force mute
forceMute$.next(false);
await flushPromises();
// TODO I'd expect it to go back to previous state (enabled)
// but actually it goes back to the initial state from construction (disabled)
// Should go back to previous state (enabled)
// Skip for now
// expect(lastEnabled).toBe(true);
// But yet it can be unmuted now
expect(setEnabled).not.toBeNull();
setEnabled!(true);
await flushPromises();
expect(lastEnabled).toBe(true);
});
});
describe("MuteStates", () => {
function aAudioOutputDevices(): MediaDevice<
AudioOutputDeviceLabel,
SelectedAudioOutputDevice
> {
const selected$ = new BehaviorSubject<
SelectedAudioOutputDevice | undefined
>({
id: "default",
virtualEarpiece: false,
});
return {
available$: constant(
new Map<string, AudioOutputDeviceLabel>([
["default", { type: "speaker" }],
["0000", { type: "speaker" }],
["1111", { type: "earpiece" }],
["222", { type: "name", name: "Bluetooth Speaker" }],
]),
),
selected$,
select(id: string): void {
if (!this.available$.getValue().has(id)) {
logger.warn(`Attempted to select unknown device id: ${id}`);
return;
}
selected$.next({
id,
/** For test purposes we ignore this */
virtualEarpiece: false,
});
},
};
}
function aVideoInput(): MediaDevice<DeviceLabel, SelectedDevice> {
const selected$ = new BehaviorSubject<SelectedDevice | undefined>(
undefined,
);
return {
available$: constant(
new Map<string, DeviceLabel>([
["0000", { type: "name", name: "HD Camera" }],
["1111", { type: "name", name: "WebCam Pro" }],
]),
),
selected$,
select(id: string): void {
if (!this.available$.getValue().has(id)) {
logger.warn(`Attempted to select unknown device id: ${id}`);
return;
}
selected$.next({ id });
},
};
}
test("should mute camera when in earpiece mode", async () => {
const audioOutputDevice = aAudioOutputDevices();
const mediaDevices = mockMediaDevices({
audioOutput: audioOutputDevice,
videoInput: aVideoInput(),
// other devices are not relevant for this test
});
const muteStates = new MuteStates(
testScope,
mediaDevices,
// consider joined
constant(true),
);
let latestSyncedState: boolean | null = null;
muteStates.video.setHandler(async (enabled: boolean): Promise<boolean> => {
logger.info(`Video mute state set to: ${enabled}`);
latestSyncedState = enabled;
return Promise.resolve(enabled);
});
let lastVideoEnabled: boolean = false;
muteStates.video.enabled$.subscribe((enabled) => {
lastVideoEnabled = enabled;
});
expect(muteStates.video.setEnabled$.value).toBeDefined();
muteStates.video.setEnabled$.value?.(true);
await flushPromises();
expect(lastVideoEnabled).toBe(true);
// Select earpiece audio output
audioOutputDevice.select("1111");
await flushPromises();
// Video should be automatically muted
expect(lastVideoEnabled).toBe(false);
expect(latestSyncedState).toBe(false);
// Try to switch to speaker
audioOutputDevice.select("0000");
await flushPromises();
// TODO I'd expect it to go back to previous state (enabled)??
// But maybe not? If you move the phone away from your ear you may not want it
// to automatically enable video?
expect(lastVideoEnabled).toBe(false);
// But yet it can be unmuted now
expect(muteStates.video.setEnabled$.value).toBeDefined();
muteStates.video.setEnabled$.value?.(true);
await flushPromises();
expect(lastVideoEnabled).toBe(true);
});
});

63
src/state/MuteStates.ts
View File

@@ -27,7 +27,7 @@ import { ElementWidgetActions, widget } from "../widget";
import { Config } from "../config/Config";
import { getUrlParams } from "../UrlParams";
import { type ObservableScope } from "./ObservableScope";
import { type Behavior } from "./Behavior";
import { type Behavior, constant } from "./Behavior";
interface MuteStateData {
enabled$: Observable<boolean>;
@@ -38,31 +38,58 @@ interface MuteStateData {
export type Handler = (desired: boolean) => Promise<boolean>;
const defaultHandler: Handler = async (desired) => Promise.resolve(desired);
class MuteState<Label, Selected> {
/**
* Internal class - exported only for testing purposes.
* Do not use directly outside of tests.
*/
export class MuteState<Label, Selected> {
// TODO: rewrite this to explain behavior, it is not understandable, and cannot add logging
private readonly enabledByDefault$ =
this.enabledByConfig && !getUrlParams().skipLobby
? this.joined$.pipe(map((isJoined) => !isJoined))
: of(false);
private readonly handler$ = new BehaviorSubject(defaultHandler);
public setHandler(handler: Handler): void {
if (this.handler$.value !== defaultHandler)
throw new Error("Multiple mute state handlers are not supported");
this.handler$.next(handler);
}
public unsetHandler(): void {
this.handler$.next(defaultHandler);
}
private readonly canControlDevices$ = combineLatest([
this.device.available$,
this.forceMute$,
]).pipe(
map(([available, forceMute]) => {
return !forceMute && available.size > 0;
}),
);
private readonly data$ = this.scope.behavior<MuteStateData>(
this.device.available$.pipe(
map((available) => available.size > 0),
this.canControlDevices$.pipe(
distinctUntilChanged(),
withLatestFrom(
this.enabledByDefault$,
(devicesConnected, enabledByDefault) => {
if (!devicesConnected)
(canControlDevices, enabledByDefault) => {
logger.info(
`MuteState: canControlDevices: ${canControlDevices}, enabled by default: ${enabledByDefault}`,
);
if (!canControlDevices) {
logger.info(
`MuteState: devices connected: ${canControlDevices}, disabling`,
);
// We need to sync the mute state with the handler
// to ensure nothing is beeing published.
this.handler$.value(false).catch((err) => {
logger.error("MuteState-disable: handler error", err);
});
return { enabled$: of(false), set: null, toggle: null };
}
// Assume the default value only once devices are actually connected
let enabled = enabledByDefault;
@@ -135,21 +162,45 @@ class MuteState<Label, Selected> {
private readonly device: MediaDevice<Label, Selected>,
private readonly joined$: Observable<boolean>,
private readonly enabledByConfig: boolean,
/**
* An optional observable which, when it emits `true`, will force the mute.
* Used for video to stop camera when earpiece mode is on.
* @private
*/
private readonly forceMute$: Observable<boolean>,
) {}
}
export class MuteStates {
/**
* True if the selected audio output device is an earpiece.
* Used to force-disable video when on earpiece.
*/
private readonly isEarpiece$ = combineLatest(
this.mediaDevices.audioOutput.available$,
this.mediaDevices.audioOutput.selected$,
).pipe(
map(([available, selected]) => {
if (!selected?.id) return false;
const device = available.get(selected.id);
logger.info(`MuteStates: selected audio output device:`, device);
return device?.type === "earpiece";
}),
);
public readonly audio = new MuteState(
this.scope,
this.mediaDevices.audioInput,
this.joined$,
Config.get().media_devices.enable_audio,
constant(false),
);
public readonly video = new MuteState(
this.scope,
this.mediaDevices.videoInput,
this.joined$,
Config.get().media_devices.enable_video,
this.isEarpiece$,
);
public constructor(

137
src/state/ObservableScope.test.ts
View File

@@ -5,9 +5,10 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { describe, expect, it } from "vitest";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { BehaviorSubject, combineLatest, Subject } from "rxjs";
import { logger } from "matrix-js-sdk/lib/logger";
import { sleep } from "matrix-js-sdk/lib/utils";
import {
Epoch,
@@ -102,3 +103,137 @@ describe("Epoch", () => {
s$.complete();
});
});
describe("Reconcile", () => {
beforeEach(() => {
vi.useFakeTimers();
});
afterEach(() => {
vi.useRealTimers();
});
it("should wait clean up before processing next", async () => {
vi.useFakeTimers();
const scope = new ObservableScope();
const behavior$ = new BehaviorSubject<number>(0);
const setup = vi.fn().mockImplementation(async () => await sleep(100));
const cleanup = vi
.fn()
.mockImplementation(async (n: number) => await sleep(100));
scope.reconcile(behavior$, async (value) => {
await setup();
return async (): Promise<void> => {
await cleanup(value);
};
});
// Let the initial setup process
await vi.advanceTimersByTimeAsync(120);
expect(setup).toHaveBeenCalledTimes(1);
expect(cleanup).toHaveBeenCalledTimes(0);
// Send next value
behavior$.next(1);
await vi.advanceTimersByTimeAsync(50);
// Should not have started setup for 1 yet
expect(setup).toHaveBeenCalledTimes(1);
expect(cleanup).toHaveBeenCalledTimes(1);
expect(cleanup).toHaveBeenCalledWith(0);
// Let cleanup finish
await vi.advanceTimersByTimeAsync(50);
// Now setup for 1 should have started
expect(setup).toHaveBeenCalledTimes(2);
});
it("should skip intermediates values that are not setup", async () => {
vi.useFakeTimers();
const scope = new ObservableScope();
const behavior$ = new BehaviorSubject<number>(0);
const setup = vi
.fn()
.mockImplementation(async (n: number) => await sleep(100));
const cleanupLock = Promise.withResolvers();
const cleanup = vi
.fn()
.mockImplementation(async (n: number) => await cleanupLock.promise);
scope.reconcile(behavior$, async (value) => {
await setup(value);
return async (): Promise<void> => {
await cleanup(value);
};
});
// Let the initial setup process (0)
await vi.advanceTimersByTimeAsync(120);
// Send 4 next values quickly
behavior$.next(1);
behavior$.next(2);
behavior$.next(3);
behavior$.next(4);
await vi.advanceTimersByTimeAsync(3000);
// should have only called cleanup for 0
expect(cleanup).toHaveBeenCalledTimes(1);
expect(cleanup).toHaveBeenCalledWith(0);
// Let cleanup finish
cleanupLock.resolve(undefined);
await vi.advanceTimersByTimeAsync(120);
// Now setup for 4 should have started, skipping 1,2,3
expect(setup).toHaveBeenCalledTimes(2);
expect(setup).toHaveBeenCalledWith(4);
expect(setup).not.toHaveBeenCalledWith(1);
expect(setup).not.toHaveBeenCalledWith(2);
expect(setup).not.toHaveBeenCalledWith(3);
});
it("should wait for setup to complete before starting cleanup", async () => {
vi.useFakeTimers();
const scope = new ObservableScope();
const behavior$ = new BehaviorSubject<number>(0);
const setup = vi
.fn()
.mockImplementation(async (n: number) => await sleep(3000));
const cleanupLock = Promise.withResolvers();
const cleanup = vi
.fn()
.mockImplementation(async (n: number) => await cleanupLock.promise);
scope.reconcile(behavior$, async (value) => {
await setup(value);
return async (): Promise<void> => {
await cleanup(value);
};
});
await vi.advanceTimersByTimeAsync(500);
// Setup for 0 should be in progress
expect(setup).toHaveBeenCalledTimes(1);
behavior$.next(1);
await vi.advanceTimersByTimeAsync(500);
// Should not have started setup for 1 yet
expect(setup).not.toHaveBeenCalledWith(1);
// Should not have called cleanup yet, because the setup for 0 is not done
expect(cleanup).toHaveBeenCalledTimes(0);
// Let setup for 0 finish
await vi.advanceTimersByTimeAsync(2500 + 100);
// Now cleanup for 0 should have started
expect(cleanup).toHaveBeenCalledTimes(1);
expect(cleanup).toHaveBeenCalledWith(0);
cleanupLock.resolve(undefined);
await vi.advanceTimersByTimeAsync(100);
// Now setup for 1 should have started
expect(setup).toHaveBeenCalledWith(1);
});
});

42
src/state/ObservableScope.ts
View File

@@ -123,8 +123,22 @@ export class ObservableScope {
callback: (value: T) => Promise<(() => Promise<void>) | void>,
): void {
let latestValue: T | typeof nothing = nothing;
let reconciledValue: T | typeof nothing = nothing;
let reconcilePromise: Promise<void> | undefined = undefined;
let cleanUp: (() => Promise<void>) | void = undefined;
let prevVal: T | typeof nothing = nothing;
// While this loop runs it will process the latest from `value$` until it caught up with the updates.
// It might skip updates from `value$` and only process the newest value after callback has resolved.
const reconcileLoop = async (): Promise<void> => {
while (latestValue !== prevVal) {
await cleanUp?.(); // Call the previous value's clean-up handler
prevVal = latestValue;
if (latestValue !== nothing) cleanUp = await callback(latestValue); // Sync current value...
// `latestValue` might have gotten updated during the `await callback`. That is why we loop here
}
};
value$
.pipe(
catchError(() => EMPTY), // Ignore errors
@@ -132,23 +146,15 @@ export class ObservableScope {
endWith(nothing), // Clean up when the scope ends
)
.subscribe((value) => {
void (async (): Promise<void> => {
if (latestValue === nothing) {
latestValue = value;
while (latestValue !== reconciledValue) {
await cleanUp?.(); // Call the previous value's clean-up handler
reconciledValue = latestValue;
if (latestValue !== nothing)
cleanUp = await callback(latestValue); // Sync current value
}
// Reset to signal that reconciliation is done for now
latestValue = nothing;
} else {
// There's already an instance of the above 'while' loop running
// concurrently. Just update the latest value and let it be handled.
latestValue = value;
}
})();
// Always track the latest value! The `reconcileLoop` will run until it "processed" the "last" `latestValue`.
latestValue = value;
// There's already an instance of the below 'reconcileLoop' loop running
// concurrently. So lets let the loop handle it. NEVER instanciate two `reconcileLoop`s.
if (reconcilePromise) return;
reconcilePromise = reconcileLoop().finally(() => {
reconcilePromise = undefined;
});
});
}

69
src/state/UserMedia.ts
View File

@@ -27,6 +27,7 @@ import type { ReactionOption } from "../reactions";
import { observeSpeaker$ } from "./observeSpeaker.ts";
import { generateItems } from "../utils/observable.ts";
import { ScreenShare } from "./ScreenShare.ts";
import { type TaggedParticipant } from "./CallViewModel/remoteMembers/MatrixLivekitMembers.ts";
/**
* Sorting bins defining the order in which media tiles appear in the layout.
@@ -68,40 +69,46 @@ enum SortingBin {
* for inclusion in the call layout and tracks associated screen shares.
*/
export class UserMedia {
public readonly vm: UserMediaViewModel = this.participant$.value?.isLocal
? new LocalUserMediaViewModel(
this.scope,
this.id,
this.userId,
this.participant$ as Behavior<LocalParticipant | null>,
this.encryptionSystem,
this.livekitRoom$,
this.focusUrl$,
this.mediaDevices,
this.displayName$,
this.mxcAvatarUrl$,
this.scope.behavior(this.handRaised$),
this.scope.behavior(this.reaction$),
)
: new RemoteUserMediaViewModel(
this.scope,
this.id,
this.userId,
this.participant$ as Behavior<RemoteParticipant | null>,
this.encryptionSystem,
this.livekitRoom$,
this.focusUrl$,
this.pretendToBeDisconnected$,
this.displayName$,
this.mxcAvatarUrl$,
this.scope.behavior(this.handRaised$),
this.scope.behavior(this.reaction$),
);
public readonly vm: UserMediaViewModel =
this.participant.type === "local"
? new LocalUserMediaViewModel(
this.scope,
this.id,
this.userId,
this.participant.value$,
this.encryptionSystem,
this.livekitRoom$,
this.focusUrl$,
this.mediaDevices,
this.displayName$,
this.mxcAvatarUrl$,
this.scope.behavior(this.handRaised$),
this.scope.behavior(this.reaction$),
)
: new RemoteUserMediaViewModel(
this.scope,
this.id,
this.userId,
this.participant.value$,
this.encryptionSystem,
this.livekitRoom$,
this.focusUrl$,
this.pretendToBeDisconnected$,
this.displayName$,
this.mxcAvatarUrl$,
this.scope.behavior(this.handRaised$),
this.scope.behavior(this.reaction$),
);
private readonly speaker$ = this.scope.behavior(
observeSpeaker$(this.vm.speaking$),
);
// TypeScript needs this widening of the type to happen in a separate statement
private readonly participant$: Behavior<
LocalParticipant | RemoteParticipant | null
> = this.participant.value$;
/**
* All screen share media associated with this user media.
*/
@@ -184,9 +191,7 @@ export class UserMedia {
private readonly scope: ObservableScope,
public readonly id: string,
private readonly userId: string,
private readonly participant$: Behavior<
LocalParticipant | RemoteParticipant | null
>,
private readonly participant: TaggedParticipant,
private readonly encryptionSystem: EncryptionSystem,
private readonly livekitRoom$: Behavior<LivekitRoom | undefined>,
private readonly focusUrl$: Behavior<string | undefined>,

10
src/tile/GridTile.test.tsx
View File

@@ -12,7 +12,11 @@ import { axe } from "vitest-axe";
import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc";
import { GridTile } from "./GridTile";
import { mockRtcMembership, createRemoteMedia } from "../utils/test";
import {
mockRtcMembership,
createRemoteMedia,
mockRemoteParticipant,
} from "../utils/test";
import { GridTileViewModel } from "../state/TileViewModel";
import { ReactionsSenderProvider } from "../reactions/useReactionsSender";
import type { CallViewModel } from "../state/CallViewModel/CallViewModel";
@@ -31,11 +35,11 @@ test("GridTile is accessible", async () => {
rawDisplayName: "Alice",
getMxcAvatarUrl: () => "mxc://adfsg",
},
{
mockRemoteParticipant({
setVolume() {},
getTrackPublication: () =>
({}) as Partial<RemoteTrackPublication> as RemoteTrackPublication,
},
}),
);
const fakeRtcSession = {

8
src/tile/GridTile.tsx
View File

@@ -69,6 +69,7 @@ interface UserMediaTileProps extends TileProps {
vm: UserMediaViewModel;
mirror: boolean;
locallyMuted: boolean;
waitingForMedia?: boolean;
primaryButton?: ReactNode;
menuStart?: ReactNode;
menuEnd?: ReactNode;
@@ -79,6 +80,7 @@ const UserMediaTile: FC<UserMediaTileProps> = ({
vm,
showSpeakingIndicators,
locallyMuted,
waitingForMedia,
primaryButton,
menuStart,
menuEnd,
@@ -148,7 +150,7 @@ const UserMediaTile: FC<UserMediaTileProps> = ({
const tile = (
<MediaView
ref={ref}
video={video ?? undefined}
video={video}
userId={vm.userId}
unencryptedWarning={unencryptedWarning}
encryptionStatus={encryptionStatus}
@@ -194,7 +196,7 @@ const UserMediaTile: FC<UserMediaTileProps> = ({
raisedHandTime={handRaised ?? undefined}
currentReaction={reaction ?? undefined}
raisedHandOnClick={raisedHandOnClick}
localParticipant={vm.local}
waitingForMedia={waitingForMedia}
focusUrl={focusUrl}
audioStreamStats={audioStreamStats}
videoStreamStats={videoStreamStats}
@@ -290,6 +292,7 @@ const RemoteUserMediaTile: FC<RemoteUserMediaTileProps> = ({
...props
}) => {
const { t } = useTranslation();
const waitingForMedia = useBehavior(vm.waitingForMedia$);
const locallyMuted = useBehavior(vm.locallyMuted$);
const localVolume = useBehavior(vm.localVolume$);
const onSelectMute = useCallback(
@@ -311,6 +314,7 @@ const RemoteUserMediaTile: FC<RemoteUserMediaTileProps> = ({
<UserMediaTile
ref={ref}
vm={vm}
waitingForMedia={waitingForMedia}
locallyMuted={locallyMuted}
mirror={false}
menuStart={

36
src/tile/MediaView.test.tsx
View File

@@ -47,7 +47,6 @@ describe("MediaView", () => {
video: trackReference,
userId: "@alice:example.com",
mxcAvatarUrl: undefined,
localParticipant: false,
focusable: true,
};
@@ -66,24 +65,13 @@ describe("MediaView", () => {
});
});
describe("with no participant", () => {
it("shows avatar for local user", () => {
render(
<MediaView {...baseProps} video={undefined} localParticipant={true} />,
);
describe("with no video", () => {
it("shows avatar", () => {
render(<MediaView {...baseProps} video={undefined} />);
expect(
screen.getByRole("img", { name: "@alice:example.com" }),
).toBeVisible();
expect(screen.queryAllByText("Waiting for media...").length).toBe(0);
});
it("shows avatar and label for remote user", () => {
render(
<MediaView {...baseProps} video={undefined} localParticipant={false} />,
);
expect(
screen.getByRole("img", { name: "@alice:example.com" }),
).toBeVisible();
expect(screen.getByText("Waiting for media...")).toBeVisible();
expect(screen.queryByTestId("video")).toBe(null);
});
});
@@ -94,6 +82,22 @@ describe("MediaView", () => {
});
});
describe("waitingForMedia", () => {
test("defaults to false", () => {
render(<MediaView {...baseProps} />);
expect(screen.queryAllByText("Waiting for media...").length).toBe(0);
});
test("shows and is accessible", async () => {
const { container } = render(
<TooltipProvider>
<MediaView {...baseProps} waitingForMedia={true} />
</TooltipProvider>,
);
expect(await axe(container)).toHaveNoViolations();
expect(screen.getByText("Waiting for media...")).toBeVisible();
});
});
describe("unencryptedWarning", () => {
test("is shown and accessible", async () => {
const { container } = render(

6
src/tile/MediaView.tsx
View File

@@ -43,7 +43,7 @@ interface Props extends ComponentProps<typeof animated.div> {
raisedHandTime?: Date;
currentReaction?: ReactionOption;
raisedHandOnClick?: () => void;
localParticipant: boolean;
waitingForMedia?: boolean;
audioStreamStats?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats;
videoStreamStats?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats;
// The focus url, mainly for debugging purposes
@@ -71,7 +71,7 @@ export const MediaView: FC<Props> = ({
raisedHandTime,
currentReaction,
raisedHandOnClick,
localParticipant,
waitingForMedia,
audioStreamStats,
videoStreamStats,
focusUrl,
@@ -129,7 +129,7 @@ export const MediaView: FC<Props> = ({
/>
)}
</div>
{!video && !localParticipant && (
{waitingForMedia && (
<div className={styles.status}>
{t("video_tile.waiting_for_media")}
</div>

3
src/tile/SpotlightTile.test.tsx
View File

@@ -17,6 +17,7 @@ import {
mockRtcMembership,
createLocalMedia,
createRemoteMedia,
mockRemoteParticipant,
} from "../utils/test";
import { SpotlightTileViewModel } from "../state/TileViewModel";
import { constant } from "../state/Behavior";
@@ -33,7 +34,7 @@ test("SpotlightTile is accessible", async () => {
rawDisplayName: "Alice",
getMxcAvatarUrl: () => "mxc://adfsg",
},
{},
mockRemoteParticipant({}),
);
const vm2 = createLocalMedia(

18
src/tile/SpotlightTile.tsx
View File

@@ -38,6 +38,7 @@ import {
type MediaViewModel,
ScreenShareViewModel,
type UserMediaViewModel,
type RemoteUserMediaViewModel,
} from "../state/MediaViewModel";
import { useInitial } from "../useInitial";
import { useMergedRefs } from "../useMergedRefs";
@@ -84,6 +85,21 @@ const SpotlightLocalUserMediaItem: FC<SpotlightLocalUserMediaItemProps> = ({
SpotlightLocalUserMediaItem.displayName = "SpotlightLocalUserMediaItem";
interface SpotlightRemoteUserMediaItemProps
extends SpotlightUserMediaItemBaseProps {
vm: RemoteUserMediaViewModel;
}
const SpotlightRemoteUserMediaItem: FC<SpotlightRemoteUserMediaItemProps> = ({
vm,
...props
}) => {
const waitingForMedia = useBehavior(vm.waitingForMedia$);
return (
<MediaView waitingForMedia={waitingForMedia} mirror={false} {...props} />
);
};
interface SpotlightUserMediaItemProps extends SpotlightItemBaseProps {
vm: UserMediaViewModel;
}
@@ -103,7 +119,7 @@ const SpotlightUserMediaItem: FC<SpotlightUserMediaItemProps> = ({
return vm instanceof LocalUserMediaViewModel ? (
<SpotlightLocalUserMediaItem vm={vm} {...baseProps} />
) : (
<MediaView mirror={false} {...baseProps} />
<SpotlightRemoteUserMediaItem vm={vm} {...baseProps} />
);
};

11
src/useAudioContext.tsx
View File

@@ -22,9 +22,12 @@ import * as controls from "./controls";
* Play a sound though a given AudioContext. Will take
* care of connecting the correct buffer and gating
* through gain.
* @param volume The volume to play at.
* @param ctx The context to play through.
* @param buffer The buffer to play.
* @param volume The volume to play at.
* @param stereoPan The stereo pan to apply.
* @param delayS Delay in seconds before starting playing.
* @param abort Optional AbortController that can be used to stop playback.
* @returns A promise that resolves when the sound has finished playing.
*/
async function playSound(
@@ -55,9 +58,11 @@ async function playSound(
* Play a sound though a given AudioContext, looping until stopped. Will take
* care of connecting the correct buffer and gating
* through gain.
* @param volume The volume to play at.
* @param ctx The context to play through.
* @param buffer The buffer to play.
* @param volume The volume to play at.
* @param stereoPan The stereo pan to apply.
* @param delayS Delay in seconds between each loop.
* @returns A function used to end the sound. This function will return a promise when the sound has stopped.
*/
function playSoundLooping(
@@ -120,7 +125,7 @@ interface UseAudioContext<S extends string> {
/**
* Add an audio context which can be used to play
* a set of preloaded sounds.
* @param props
* @param props The properties for the audio context.
* @returns Either an instance that can be used to play sounds, or null if not ready.
*/
export function useAudioContext<S extends string>(

7
src/utils/displayname.ts
View File

@@ -77,6 +77,13 @@ export function shouldDisambiguate(
);
}
/**
* Calculates a display name for a member, optionally disambiguating it.
* @param member - The member to calculate the display name for.
* @param member.rawDisplayName - The raw display name of the member
* @param member.userId - The user ID of the member
* @param disambiguate - Whether to disambiguate the display name.
*/
export function calculateDisplayName(
member: { rawDisplayName?: string; userId: string },
disambiguate: boolean,

76
src/utils/errors.ts
View File

@@ -13,6 +13,8 @@ export enum ErrorCode {
*/
MISSING_MATRIX_RTC_TRANSPORT = "MISSING_MATRIX_RTC_TRANSPORT",
CONNECTION_LOST_ERROR = "CONNECTION_LOST_ERROR",
INTERNAL_MEMBERSHIP_MANAGER = "INTERNAL_MEMBERSHIP_MANAGER",
FAILED_TO_START_LIVEKIT = "FAILED_TO_START_LIVEKIT",
/** LiveKit indicates that the server has hit its track limits */
INSUFFICIENT_CAPACITY_ERROR = "INSUFFICIENT_CAPACITY_ERROR",
E2EE_NOT_SUPPORTED = "E2EE_NOT_SUPPORTED",
@@ -27,6 +29,7 @@ export enum ErrorCategory {
NETWORK_CONNECTIVITY = "NETWORK_CONNECTIVITY",
CLIENT_CONFIGURATION = "CLIENT_CONFIGURATION",
UNKNOWN = "UNKNOWN",
SYSTEM_FAILURE = "SYSTEM_FAILURE",
// SYSTEM_FAILURE / FEDERATION_FAILURE ..
}
@@ -54,9 +57,16 @@ export class ElementCallError extends Error {
}
}
/**
* Configuration problem due to no MatrixRTC backend/SFU is exposed via .well-known and no fallback configured.
*/
export class MatrixRTCTransportMissingError extends ElementCallError {
public domain: string;
/**
* Creates an instance of MatrixRTCTransportMissingError.
* @param domain - The domain where the MatrixRTC transport is missing.
*/
public constructor(domain: string) {
super(
t("error.call_is_not_supported"),
@@ -72,6 +82,9 @@ export class MatrixRTCTransportMissingError extends ElementCallError {
}
}
/**
* Error indicating that the connection to the call was lost and could not be re-established.
*/
export class ConnectionLostError extends ElementCallError {
public constructor() {
super(
@@ -83,6 +96,30 @@ export class ConnectionLostError extends ElementCallError {
}
}
/**
* Error indicating a failure in the membership manager causing the join call
* operation to fail.
*/
export class MembershipManagerError extends ElementCallError {
/**
* Creates an instance of MembershipManagerError.
*
* @param error - The underlying error that caused the membership manager failure.
*/
public constructor(error: Error) {
super(
t("error.membership_manager"),
ErrorCode.INTERNAL_MEMBERSHIP_MANAGER,
ErrorCategory.SYSTEM_FAILURE,
t("error.membership_manager_description"),
error,
);
}
}
/**
* Error indicating that end-to-end encryption is not supported in the current environment.
*/
export class E2EENotSupportedError extends ElementCallError {
public constructor() {
super(
@@ -94,7 +131,14 @@ export class E2EENotSupportedError extends ElementCallError {
}
}
/**
* Error indicating an unknown issue occurred during a call operation.
*/
export class UnknownCallError extends ElementCallError {
/**
* Creates an instance of UnknownCallError.
* @param error - The underlying error that caused the unknown issue.
*/
public constructor(error: Error) {
super(
t("error.generic"),
@@ -107,7 +151,14 @@ export class UnknownCallError extends ElementCallError {
}
}
/**
* Error indicating a failure to obtain an OpenID token.
*/
export class FailToGetOpenIdToken extends ElementCallError {
/**
* Creates an instance of FailToGetOpenIdToken.
* @param error - The underlying error that caused the failure.
*/
public constructor(error: Error) {
super(
t("error.generic"),
@@ -120,6 +171,27 @@ export class FailToGetOpenIdToken extends ElementCallError {
}
}
/**
* Error indicating a failure to start publishing on a LiveKit connection.
*/
export class FailToStartLivekitConnection extends ElementCallError {
/**
* Creates an instance of FailToStartLivekitConnection.
* @param e - An optional error message providing additional context.
*/
public constructor(e?: string) {
super(
t("error.failed_to_start_livekit"),
ErrorCode.FAILED_TO_START_LIVEKIT,
ErrorCategory.NETWORK_CONNECTIVITY,
e,
);
}
}
/**
* Error indicating that a LiveKit's server has hit its track limits.
*/
export class InsufficientCapacityError extends ElementCallError {
public constructor() {
super(
@@ -131,6 +203,10 @@ export class InsufficientCapacityError extends ElementCallError {
}
}
/**
* Error indicating that room creation is restricted by the SFU.
* Only authorized users can create rooms, so the room must exist before connecting (done by the auth jwt service)
*/
export class SFURoomCreationRestrictedError extends ElementCallError {
public constructor() {
super(

7
src/utils/matrix.ts
View File

@@ -188,7 +188,6 @@ function fullAliasFromRoomName(roomName: string, client: MatrixClient): string {
* Applies some basic sanitisation to a room name that the user
* has given us
* @param input The room name from the user
* @param client A matrix client object
*/
export function sanitiseRoomNameInput(input: string): string {
// check to see if the user has entered a fully qualified room
@@ -304,8 +303,9 @@ export async function createRoom(
/**
* Returns an absolute URL to that will load Element Call with the given room
* @param roomId ID of the room
* @param roomName Name of the room
* @param encryptionSystem what encryption (or EncryptionSystem.Unencrypted) the room uses
* @param roomName Name of the room
* @param viaServers Optional list of servers to include as 'via' parameters in the URL
*/
export function getAbsoluteRoomUrl(
roomId: string,
@@ -321,8 +321,9 @@ export function getAbsoluteRoomUrl(
/**
* Returns a relative URL to that will load Element Call with the given room
* @param roomId ID of the room
* @param roomName Name of the room
* @param encryptionSystem what encryption (or EncryptionSystem.Unencrypted) the room uses
* @param roomName Name of the room
* @param viaServers Optional list of servers to include as 'via' parameters in the URL
*/
export function getRelativeRoomUrl(
roomId: string,

1
src/utils/media.ts
View File

@@ -8,6 +8,7 @@ Please see LICENSE in the repository root for full details.
/**
* Finds a media device with label matching 'deviceName'
* @param deviceName The label of the device to look for
* @param kind The kind of media device to look for
* @param devices The list of devices to search
* @returns A matching media device or undefined if no matching device was found
*/

35
src/utils/observable.test.ts
View File

@@ -5,11 +5,12 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { test } from "vitest";
import { Subject } from "rxjs";
import { expect, test } from "vitest";
import { type Observable, of, Subject, switchMap } from "rxjs";
import { withTestScheduler } from "./test";
import { generateItems, pauseWhen } from "./observable";
import { filterBehavior, generateItems, pauseWhen } from "./observable";
import { type Behavior } from "../state/Behavior";
test("pauseWhen", () => {
withTestScheduler(({ behavior, expectObservable }) => {
@@ -72,3 +73,31 @@ test("generateItems", () => {
expectObservable(scope4$).toBe(scope4Marbles);
});
});
test("filterBehavior", () => {
withTestScheduler(({ behavior, expectObservable }) => {
// Filtering the input should segment it into 2 modes of non-null behavior.
const inputMarbles = " abcxabx";
const filteredMarbles = "a--xa-x";
const input$ = behavior(inputMarbles, {
a: "a",
b: "b",
c: "c",
x: null,
});
const filtered$: Observable<Behavior<string> | null> = input$.pipe(
filterBehavior((value) => typeof value === "string"),
);
expectObservable(filtered$).toBe(filteredMarbles, {
a: expect.any(Object),
x: null,
});
expectObservable(
filtered$.pipe(
switchMap((value$) => (value$ === null ? of(null) : value$)),
),
).toBe(inputMarbles, { a: "a", b: "b", c: "c", x: null });
});
});

24
src/utils/observable.ts
View File

@@ -22,6 +22,7 @@ import {
withLatestFrom,
BehaviorSubject,
type OperatorFunction,
distinctUntilChanged,
} from "rxjs";
import { type Behavior } from "../state/Behavior";
@@ -134,7 +135,6 @@ interface ItemHandle<Data, Item> {
* requested at a later time, and destroyed (have their scope ended) when the
* key is no longer requested.
*
* @param input$ The input value to be mapped.
* @param generator A generator function yielding a tuple of keys and the
* currently associated data for each item that it wants to exist.
* @param factory A function constructing an individual item, given the item's key,
@@ -185,6 +185,28 @@ export function generateItemsWithEpoch<
);
}
/**
* Segments a behavior into periods during which its value matches the filter
* (outputting a behavior with a narrowed type) and periods during which it does
* not match (outputting null).
*/
export function filterBehavior<T, S extends T>(
predicate: (value: T) => value is S,
): OperatorFunction<T, Behavior<S> | null> {
return (input$) =>
input$.pipe(
scan<T, BehaviorSubject<S> | null>((acc$, input) => {
if (predicate(input)) {
const output$ = acc$ ?? new BehaviorSubject(input);
output$.next(input);
return output$;
}
return null;
}, null),
distinctUntilChanged(),
);
}
function generateItemsInternal<
Input,
Keys extends [unknown, ...unknown[]],

14
src/utils/test-fixtures.ts
View File

@@ -59,3 +59,17 @@ export const daveRTLRtcMember = mockRtcMembership("@dave2:example.org", "DDDD");
export const daveRTL = mockMatrixRoomMember(daveRTLRtcMember, {
rawDisplayName: "\u202eevaD",
});
export const testJWTToken = [
{}, // header
{
// payload
sub: "@me:example.org:ABCDEF",
video: {
room: "!example_room_id",
},
},
{}, // signature
]
.map((d) => global.btoa(JSON.stringify(d)))
.join(".");

2
src/utils/test-viewmodel.ts
View File

@@ -37,6 +37,7 @@ import {
import { aliceRtcMember, localRtcMember } from "./test-fixtures";
import { type RaisedHandInfo, type ReactionInfo } from "../reactions";
import { constant } from "../state/Behavior";
import { MatrixRTCMode } from "../settings/settings";
mockConfig({ livekit: { livekit_service_url: "https://example.com" } });
@@ -162,6 +163,7 @@ export function getBasicCallViewModelEnvironment(
setE2EEEnabled: async () => Promise.resolve(),
}),
connectionState$: constant(ConnectionState.Connected),
matrixRTCMode$: constant(MatrixRTCMode.Legacy),
...callViewModelOptions,
},
handRaisedSubject$,

45
src/utils/test.ts
View File

@@ -44,12 +44,12 @@ import {
Track,
} from "livekit-client";
import { randomUUID } from "crypto";
import {
type RoomAndToDeviceEvents,
type RoomAndToDeviceEventsHandlerMap,
} from "matrix-js-sdk/lib/matrixrtc/RoomAndToDeviceKeyTransport";
import { type TrackReference } from "@livekit/components-core";
import EventEmitter from "events";
import {
type KeyTransportEvents,
type KeyTransportEventsHandlerMap,
} from "matrix-js-sdk/lib/matrixrtc/IKeyTransport";
import {
LocalUserMediaViewModel,
@@ -284,6 +284,8 @@ export function mockLivekitRoom(
): LivekitRoom {
const livekitRoom = {
options: {},
setE2EEEnabled: vi.fn(),
...mockEmitter(),
...room,
} as Partial<LivekitRoom> as LivekitRoom;
@@ -306,7 +308,11 @@ export function mockLocalParticipant(
return {
isLocal: true,
trackPublications: new Map(),
unpublishTracks: async () => Promise.resolve(),
publishTrack: vi.fn(),
unpublishTracks: vi.fn().mockResolvedValue([]),
createTracks: vi.fn(),
setMicrophoneEnabled: vi.fn(),
setCameraEnabled: vi.fn(),
getTrackPublication: () =>
({}) as Partial<LocalTrackPublication> as LocalTrackPublication,
...mockEmitter(),
@@ -315,12 +321,12 @@ export function mockLocalParticipant(
}
export function createLocalMedia(
localRtcMember: CallMembership,
rtcMember: CallMembership,
roomMember: Partial<RoomMember>,
localParticipant: LocalParticipant,
mediaDevices: MediaDevices,
): LocalUserMediaViewModel {
const member = mockMatrixRoomMember(localRtcMember, roomMember);
const member = mockMatrixRoomMember(rtcMember, roomMember);
return new LocalUserMediaViewModel(
testScope(),
"local",
@@ -355,23 +361,26 @@ export function mockRemoteParticipant(
}
export function createRemoteMedia(
localRtcMember: CallMembership,
rtcMember: CallMembership,
roomMember: Partial<RoomMember>,
participant: Partial<RemoteParticipant>,
participant: RemoteParticipant | null,
livekitRoom: LivekitRoom | undefined = mockLivekitRoom(
{},
{
remoteParticipants$: of(participant ? [participant] : []),
},
),
): RemoteUserMediaViewModel {
const member = mockMatrixRoomMember(localRtcMember, roomMember);
const remoteParticipant = mockRemoteParticipant(participant);
const member = mockMatrixRoomMember(rtcMember, roomMember);
return new RemoteUserMediaViewModel(
testScope(),
"remote",
member.userId,
of(remoteParticipant),
constant(participant),
{
kind: E2eeType.PER_PARTICIPANT,
},
constant(
mockLivekitRoom({}, { remoteParticipants$: of([remoteParticipant]) }),
),
constant(livekitRoom),
constant("https://rtc-example.org"),
constant(false),
constant(member.rawDisplayName ?? "nodisplayname"),
@@ -394,9 +403,9 @@ export function mockConfig(
}
export class MockRTCSession extends TypedEventEmitter<
MatrixRTCSessionEvent | RoomAndToDeviceEvents | MembershipManagerEvent,
MatrixRTCSessionEventHandlerMap &
RoomAndToDeviceEventsHandlerMap &
MatrixRTCSessionEvent | MembershipManagerEvent | KeyTransportEvents,
KeyTransportEventsHandlerMap &
MatrixRTCSessionEventHandlerMap &
MembershipManagerEventHandlerMap
> {
public asMockedSession(): MockedObject<MatrixRTCSession> {

19
src/widget.ts
View File

@@ -64,6 +64,12 @@ export const widget = ((): WidgetHelpers | null => {
try {
const { widgetId, parentUrl } = getUrlParams();
const { roomId, userId, deviceId, baseUrl, e2eEnabled, allowIceFallback } =
getUrlParams();
if (!roomId) throw new Error("Room ID must be supplied");
if (!userId) throw new Error("User ID must be supplied");
if (!deviceId) throw new Error("Device ID must be supplied");
if (!baseUrl) throw new Error("Base URL must be supplied");
if (widgetId && parentUrl) {
const parentOrigin = new URL(parentUrl).origin;
logger.info("Widget API is available");
@@ -92,19 +98,6 @@ export const widget = ((): WidgetHelpers | null => {
// We need to do this now rather than later because it has capabilities to
// request, and is responsible for starting the transport (should it be?)
const {
roomId,
userId,
deviceId,
baseUrl,
e2eEnabled,
allowIceFallback,
} = getUrlParams();
if (!roomId) throw new Error("Room ID must be supplied");
if (!userId) throw new Error("User ID must be supplied");
if (!deviceId) throw new Error("Device ID must be supplied");
if (!baseUrl) throw new Error("Base URL must be supplied");
// These are all the event types the app uses
const sendEvent = [
EventType.CallNotify, // Sent as a deprecated fallback