Matrix/element-call-Github
1
0
Fork 0
mirror of https://github.com/vector-im/element-call.git synced 2026-02-05 04:15:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'livekit' into toger5/delayed-event-delegation

Browse Source
This commit is contained in:
Timo K
2026-01-05 21:08:21 +01:00
parent 0f5c5d8be5 49cd980ddf
commit df2bd22005
46 changed files with 2380 additions and 245 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/e2ee/sharedKeyManagement.ts
View File

@@ -34,8 +34,8 @@ const getRoomSharedKeyLocalStorageKey = (roomId: string): string =>
`room-shared-key-${roomId}`;
/**
* An upto-date shared key for the room. Either from local storage or the value from `setInitialValue`.
* @param roomId
* An up-to-date shared key for the room. Either from local storage or the value from `setInitialValue`.
* @param roomId The room ID we want the shared key for.
* @param setInitialValue The value we get from the URL. The hook will overwrite the local storage value with this.
* @returns [roomSharedKey, setRoomSharedKey] like a react useState hook.
*/

6
src/livekit/MatrixAudioRenderer.tsx
View File

@@ -165,7 +165,11 @@ interface StereoPanAudioTrackProps {
* It main purpose is to remount the AudioTrack component when switching from
* audioContext to normal audio playback.
* As of now the AudioTrack component does not support adding audio nodes while being mounted.
* @param param0
* @param props The component props
* @param props.trackRef The track reference
* @param props.muted If the track should be muted
* @param props.audioContext The audio context to use
* @param props.audioNodes The audio nodes to use
* @returns
*/
function AudioTrackWithAudioNodes({

112
src/livekit/openIDSFU.test.ts Normal file
View File

@@ -0,0 +1,112 @@
/*
Copyright 2025 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
beforeEach,
afterEach,
describe,
expect,
it,
type MockedObject,
vitest,
} from "vitest";
import fetchMock from "fetch-mock";
import { getSFUConfigWithOpenID, type OpenIDClientParts } from "./openIDSFU";
import { testJWTToken } from "../utils/test-fixtures";
const sfuUrl = "https://sfu.example.org";
describe("getSFUConfigWithOpenID", () => {
let matrixClient: MockedObject<OpenIDClientParts>;
beforeEach(() => {
matrixClient = {
getOpenIdToken: vitest.fn(),
getDeviceId: vitest.fn(),
};
});
afterEach(() => {
vitest.clearAllMocks();
fetchMock.reset();
});
it("should handle fetching a token", async () => {
fetchMock.post("https://sfu.example.org/sfu/get", () => {
return {
status: 200,
body: { url: sfuUrl, jwt: testJWTToken },
};
});
const config = await getSFUConfigWithOpenID(
matrixClient,
"https://sfu.example.org",
"!example_room_id",
);
expect(config).toEqual({
jwt: testJWTToken,
url: sfuUrl,
livekitIdentity: "@me:example.org:ABCDEF",
livekitAlias: "!example_room_id",
});
void (await fetchMock.flush());
});
it("should fail if the SFU errors", async () => {
fetchMock.post("https://sfu.example.org/sfu/get", () => {
return {
status: 500,
body: { error: "Test failure" },
};
});
try {
await getSFUConfigWithOpenID(
matrixClient,
"https://sfu.example.org",
"!example_room_id",
);
} catch (ex) {
expect(((ex as Error).cause as Error).message).toEqual(
"SFU Config fetch failed with status code 500",
);
void (await fetchMock.flush());
return;
}
expect.fail("Expected test to throw;");
});
it("should retry fetching the openid token", async () => {
let count = 0;
matrixClient.getOpenIdToken.mockImplementation(async () => {
count++;
if (count < 2) {
throw Error("Test failure");
}
return Promise.resolve({
token_type: "Bearer",
access_token: "foobar",
matrix_server_name: "example.org",
expires_in: 30,
});
});
fetchMock.post("https://sfu.example.org/sfu/get", () => {
return {
status: 200,
body: { url: sfuUrl, jwt: testJWTToken },
};
});
const config = await getSFUConfigWithOpenID(
matrixClient,
"https://sfu.example.org",
"!example_room_id",
);
expect(config).toEqual({
jwt: testJWTToken,
url: sfuUrl,
livekitIdentity: "@me:example.org:ABCDEF",
livekitAlias: "!example_room_id",
});
void (await fetchMock.flush());
});
});

83
src/livekit/openIDSFU.ts
View File

@@ -13,9 +13,47 @@ import { FailToGetOpenIdToken } from "../utils/errors";
import { doNetworkOperationWithRetry } from "../utils/matrix";
import { Config } from "../config/Config";
/**
* Configuration and access tokens provided by the SFU on successful authentication.
*/
export interface SFUConfig {
url: string;
jwt: string;
livekitAlias: string;
livekitIdentity: string;
}
/**
* Decoded details from the JWT.
*/
interface SFUJWTPayload {
/**
* Expiration time for the JWT.
* Note: This value is in seconds since Unix epoch.
*/
exp: number;
/**
* Name of the instance which authored the JWT
*/
iss: string;
/**
* Time at which the JWT can start to be used.
* Note: This value is in seconds since Unix epoch.
*/
nbf: number;
/**
* Subject. The Livekit alias in this context.
*/
sub: string;
/**
* The set of permissions for the user.
*/
video: {
canPublish: boolean;
canSubscribe: boolean;
room: string;
roomJoin: boolean;
};
}
// The bits we need from MatrixClient
@@ -27,15 +65,15 @@ export type OpenIDClientParts = Pick<
* Gets a bearer token from the homeserver and then use it to authenticate
* to the matrix RTC backend in order to get acces to the SFU.
* It has built-in retry for calls to the homeserver with a backoff policy.
* @param client
* @param client The Matrix client
* @param membership
* @param serviceUrl
* @param serviceUrl The URL of the livekit SFU service
* @param forceOldEndpoint This will use the old jwt endpoint which will create the rtc backend identity based on string concatination
* instead of a hash.
* This function by default uses whatever is possible with the current jwt service installed next to the SFU.
* For remote connections this does not matter, since we will not publish there we can rely on the newest option.
* For our own connection we can only use the hashed version if we also send the new matrix2.0 sticky events.
* @param livekitRoomAlias
* @param roomId The room id used in the jwt request. This is NOT the livekit_alias. The jwt service will provide the alias. It maps matrix room ids <-> Livekit aliases.
* @param delayEndpointBaseUrl
* @param delayId
* @param logger
@@ -47,7 +85,7 @@ export async function getSFUConfigWithOpenID(
membership: CallMembershipIdentityParts,
serviceUrl: string,
forceOldJwtEndpoint: boolean,
livekitRoomAlias: string,
roomId: string,
delayEndpointBaseUrl?: string,
delayId?: string,
logger?: Logger,
@@ -68,39 +106,49 @@ export async function getSFUConfigWithOpenID(
const args: [CallMembershipIdentityParts, string, string, IOpenIDToken] = [
membership,
serviceUrl,
livekitRoomAlias,
roomId,
openIdToken,
];
let sfuConfig: { url: string; jwt: string };
try {
// we do not want to try the old endpoint, since we are not sending the new matrix2.0 sticky events (no hashed identity in the event)
if (forceOldJwtEndpoint) throw new Error("Force old jwt endpoint");
if (!delayId)
throw new Error("No delayId, Won't try matrix 2.0 jwt endpoint.");
const sfuConfig = await getLiveKitJWTWithDelayDelegation(
sfuConfig = await getLiveKitJWTWithDelayDelegation(
...args,
delayEndpointBaseUrl,
delayId,
);
logger?.info(`Got JWT from call's active focus URL.`);
return sfuConfig;
} catch (e) {
logger?.warn(
`Failed fetching jwt with matrix 2.0 endpoint (retry with legacy)`,
e,
);
const sfuConfig = await getLiveKitJWT(...args);
sfuConfig = await getLiveKitJWT(...args);
logger?.info(`Got JWT from call's active focus URL.`);
return sfuConfig;
}
} // Pull the details from the JWT
const [, payloadStr] = sfuConfig.jwt.split(".");
const payload = JSON.parse(global.atob(payloadStr)) as SFUJWTPayload;
return {
jwt: sfuConfig.jwt,
url: sfuConfig.url,
livekitAlias: payload.video.room,
// NOTE: Currently unused.
livekitIdentity: payload.sub,
};
}
async function getLiveKitJWT(
membership: CallMembershipIdentityParts,
livekitServiceURL: string,
livekitRoomAlias: string,
matrixRoomId: string,
openIDToken: IOpenIDToken,
): Promise<SFUConfig> {
): Promise<{ url: string; jwt: string }> {
try {
const res = await fetch(livekitServiceURL + "/sfu/get", {
method: "POST",
@@ -108,7 +156,8 @@ async function getLiveKitJWT(
"Content-Type": "application/json",
},
body: JSON.stringify({
room: livekitRoomAlias,
// This is the actual livekit room alias. For the legacy jwt endpoint simply the room id was used.
room: matrixRoomId,
openid_token: openIDToken,
device_id: membership.deviceId,
}),
@@ -118,22 +167,22 @@ async function getLiveKitJWT(
}
return await res.json();
} catch (e) {
throw new Error("SFU Config fetch failed with exception " + e);
throw new Error("SFU Config fetch failed with exception", { cause: e });
}
}
export async function getLiveKitJWTWithDelayDelegation(
membership: CallMembershipIdentityParts,
livekitServiceURL: string,
livekitRoomAlias: string,
matrixRoomId: string,
openIDToken: IOpenIDToken,
delayEndpointBaseUrl?: string,
delayId?: string,
): Promise<SFUConfig> {
): Promise<{ url: string; jwt: string }> {
const { userId, deviceId, memberId } = membership;
const body = {
room_id: livekitRoomAlias,
room_id: matrixRoomId,
slot_id: "m.call#ROOM",
openid_token: openIDToken,
member: {

6
src/reactions/ReactionsReader.ts
View File

@@ -135,10 +135,10 @@ export class ReactionsReader {
}
/**
* Fetchest any hand wave reactions by the given sender on the given
* Fetches any hand wave reactions by the given sender on the given
* membership event.
* @param membershipEventId
* @param expectedSender
* @param membershipEventId - The user membership event id.
* @param expectedSender - The expected sender of the reaction.
* @returns A MatrixEvent if one was found.
*/
private getLastReactionEvent(

2
src/room/InCallView.tsx
View File

@@ -260,7 +260,7 @@ export const InCallView: FC<InCallViewProps> = ({
() => void toggleRaisedHand(),
);
const audioParticipants = useBehavior(vm.audioParticipants$);
const audioParticipants = useBehavior(vm.livekitRoomItems$);
const participantCount = useBehavior(vm.participantCount$);
const reconnecting = useBehavior(vm.reconnecting$);
const windowMode = useBehavior(vm.windowMode$);

16
src/room/useLoadGroupCall.ts
View File

@@ -106,22 +106,18 @@ async function joinRoomAfterInvite(
export class CallTerminatedMessage extends Error {
/**
* Creates a new CallTerminatedMessage.
*
* @param icon The icon to display with the message
* @param messageTitle The title of the call ended screen message (translated)
* @param messageBody The message explaining the kind of termination
* (kick, ban, knock reject, etc.) (translated)
* @param reason The user-provided reason for the termination (kick/ban)
*/
public constructor(
/**
* The icon to display with the message.
*/
public readonly icon: ComponentType<SVGAttributes<SVGElement>>,
messageTitle: string,
/**
* The message explaining the kind of termination (kick, ban, knock reject,
* etc.) (translated)
*/
public readonly messageBody: string,
/**
* The user-provided reason for the termination (kick/ban)
*/
public readonly reason?: string,
) {
super(messageTitle);

26
src/settings/rageshake.ts
View File

@@ -99,7 +99,7 @@ class ConsoleLogger extends EventEmitter {
/**
* Returns the log lines to flush to disk and empties the internal log buffer
* @return {string} \n delimited log lines
* @return \n delimited log lines
*/
public popLogs(): string {
const logsToFlush = this.logs;
@@ -109,7 +109,7 @@ class ConsoleLogger extends EventEmitter {
/**
* Returns lines currently in the log buffer without removing them
* @return {string} \n delimited log lines
* @return \n delimited log lines
*/
public peekLogs(): string {
return this.logs;
@@ -139,7 +139,7 @@ class IndexedDBLogStore {
}
/**
* @return {Promise} Resolves when the store is ready.
* @return Resolves when the store is ready.
*/
public async connect(): Promise<void> {
const req = this.indexedDB.open("logs");
@@ -219,7 +219,7 @@ class IndexedDBLogStore {
* This guarantees that we will always eventually do a flush when flush() is
* called.
*
* @return {Promise} Resolved when the logs have been flushed.
* @return Resolved when the logs have been flushed.
*/
public flush = async (): Promise<void> => {
// check if a flush() operation is ongoing
@@ -270,7 +270,7 @@ class IndexedDBLogStore {
* returned are deleted at the same time, so this can be called at startup
* to do house-keeping to keep the logs from growing too large.
*
* @return {Promise<Object[]>} Resolves to an array of objects. The array is
* @return Resolves to an array of objects. The array is
* sorted in time (oldest first) based on when the log file was created (the
* log ID). The objects have said log ID in an "id" field and "lines" which
* is a big string with all the new-line delimited logs.
@@ -421,12 +421,12 @@ class IndexedDBLogStore {
/**
* Helper method to collect results from a Cursor and promiseify it.
* @param {ObjectStore|Index} store The store to perform openCursor on.
* @param {IDBKeyRange=} keyRange Optional key range to apply on the cursor.
* @param {Function} resultMapper A function which is repeatedly called with a
* @param store - The store to perform openCursor on.
* @param keyRange - Optional key range to apply on the cursor.
* @param resultMapper - A function which is repeatedly called with a
* Cursor.
* Return the data you want to keep.
* @return {Promise<T[]>} Resolves to an array of whatever you returned from
* @return Resolves to an array of whatever you returned from
* resultMapper.
*/
async function selectQuery<T>(
@@ -464,9 +464,7 @@ declare global {
/**
* Configure rage shaking support for sending bug reports.
* Modifies globals.
* @param {boolean} setUpPersistence When true (default), the persistence will
* be set up immediately for the logs.
* @return {Promise} Resolves when set up.
* @return Resolves when set up.
*/
export async function init(): Promise<void> {
global.mx_rage_logger = new ConsoleLogger();
@@ -503,7 +501,7 @@ export async function init(): Promise<void> {
/**
* Try to start up the rageshake storage for logs. If not possible (client unsupported)
* then this no-ops.
* @return {Promise} Resolves when complete.
* @return Resolves when complete.
*/
async function tryInitStorage(): Promise<void> {
if (global.mx_rage_initStoragePromise) {
@@ -536,7 +534,7 @@ async function tryInitStorage(): Promise<void> {
/**
* Get a recent snapshot of the logs, ready for attaching to a bug report
*
* @return {LogEntry[]} list of log data
* @return list of log data
*/
export async function getLogsForReport(): Promise<LogEntry[]> {
if (!global.mx_rage_logger) {

2
src/state/CallViewModel/CallNotificationLifecycle.ts
View File

@@ -81,7 +81,7 @@ export interface Props {
localUser: { deviceId: string; userId: string };
}
/**
* @returns {callPickupState$, autoLeave$}
* @returns two observables:
* `callPickupState$` The current call pickup state of the call.
* - "unknown": The client has not yet sent the notification event. We don't know if it will because it first needs to send its own membership.
* Then we can conclude if we were the first one to join or not.

61
src/state/CallViewModel/CallViewModel.ts
View File

@@ -82,7 +82,7 @@ import {
} from "../../reactions";
import { shallowEquals } from "../../utils/array";
import { type MediaDevices } from "../MediaDevices";
import { type Behavior } from "../Behavior";
import { constant, type Behavior } from "../Behavior";
import { E2eeType } from "../../e2ee/e2eeType";
import { MatrixKeyProvider } from "../../e2ee/matrixKeyProvider";
import { type MuteStates } from "../MuteStates";
@@ -119,6 +119,7 @@ import {
createMatrixLivekitMembers$,
type TaggedParticipant,
type LocalMatrixLivekitMember,
type RemoteMatrixLivekitMember,
} from "./remoteMembers/MatrixLivekitMembers.ts";
import {
type AutoLeaveReason,
@@ -158,7 +159,7 @@ export interface CallViewModelOptions {
/** Optional behavior overriding the computed window size, mainly for testing purposes. */
windowSize$?: Behavior<{ width: number; height: number }>;
/** The version & compatibility mode of MatrixRTC that we should use. */
matrixRTCMode$: Behavior<MatrixRTCMode>;
matrixRTCMode$?: Behavior<MatrixRTCMode>;
}
// Do not play any sounds if the participant count has exceeded this
@@ -184,7 +185,7 @@ interface LayoutScanState {
}
type MediaItem = UserMedia | ScreenShare;
type AudioLivekitItem = {
export type LivekitRoomItem = {
livekitRoom: LivekitRoom;
participants: string[];
url: string;
@@ -207,8 +208,11 @@ export interface CallViewModel {
callPickupState$: Behavior<
"unknown" | "ringing" | "timeout" | "decline" | "success" | null
>;
/** Observable that emits when the user should leave the call (hangup pressed, widget action, error).
* THIS DOES NOT LEAVE THE CALL YET. The only way to leave the call (send the hangup event) is by ending the scope.
*/
leave$: Observable<"user" | AutoLeaveReason>;
/** Call to initiate hangup. Use in conbination with connectino state track the async hangup process. */
/** Call to initiate hangup. Use in conbination with reconnectino state track the async hangup process. */
hangup: () => void;
// joining
@@ -260,7 +264,11 @@ export interface CallViewModel {
*/
participantCount$: Behavior<number>;
/** Participants sorted by livekit room so they can be used in the audio rendering */
audioParticipants$: Behavior<AudioLivekitItem[]>;
livekitRoomItems$: Behavior<LivekitRoomItem[]>;
userMedia$: Behavior<UserMedia[]>;
/** use the layout instead, this is just for the sdk export. */
matrixLivekitMembers$: Behavior<RemoteMatrixLivekitMember[]>;
localMatrixLivekitMember$: Behavior<LocalMatrixLivekitMember | null>;
/** List of participants raising their hand */
handsRaised$: Behavior<Record<string, RaisedHandInfo>>;
/** List of reactions. Keys are: membership.membershipId (currently predefined as: `${membershipEvent.userId}:${membershipEvent.deviceId}`)*/
@@ -343,17 +351,15 @@ export interface CallViewModel {
switch: () => void;
} | null>;
// connection state
/**
* Whether various media/event sources should pretend to be disconnected from
* all network input, even if their connection still technically works.
* Whether the app is currently reconnecting to the LiveKit server and/or setting the matrix rtc room state.
*/
// We do this when the app is in the 'reconnecting' state, because it might be
// that the LiveKit connection is still functional while the homeserver is
// down, for example, and we want to avoid making people worry that the app is
// in a split-brained state.
// DISCUSSION own membership manager ALSO this probably can be simplifis
reconnecting$: Behavior<boolean>;
/**
* Shortcut for not requireing to parse and combine connectionState.matrix and connectionState.livekit
*/
connected$: Behavior<boolean>;
}
/**
@@ -386,6 +392,8 @@ export function createCallViewModel$(
options.encryptionSystem,
matrixRTCSession,
);
const matrixRTCMode$ =
options.matrixRTCMode$ ?? constant(MatrixRTCMode.Legacy);
// Each hbar seperates a block of input variables required for the CallViewModel to function.
// The outputs of this block is written under the hbar.
@@ -420,7 +428,7 @@ export function createCallViewModel$(
};
const useOldJwtEndpoint$ = scope.behavior(
options.matrixRTCMode$.pipe(map((v) => v !== MatrixRTCMode.Matrix_2_0)),
matrixRTCMode$.pipe(map((v) => v !== MatrixRTCMode.Matrix_2_0)),
);
const localTransport$ = createLocalTransport$({
scope: scope,
@@ -439,7 +447,7 @@ export function createCallViewModel$(
roomId: matrixRoom.roomId,
useOldJwtEndpoint$,
useOldestMember$: scope.behavior(
options.matrixRTCMode$.pipe(map((v) => v === MatrixRTCMode.Legacy)),
matrixRTCMode$.pipe(map((v) => v === MatrixRTCMode.Legacy)),
),
});
@@ -482,7 +490,7 @@ export function createCallViewModel$(
});
const connectOptions$ = scope.behavior(
options.matrixRTCMode$.pipe(
matrixRTCMode$.pipe(
map((mode) => ({
encryptMedia: livekitKeyProvider !== undefined,
// TODO. This might need to get called again on each change of matrixRTCMode...
@@ -515,7 +523,7 @@ export function createCallViewModel$(
muteStates,
trackProcessorState$,
logger.getChild(
"[Publisher" + connection.transport.livekit_service_url + "]",
"[Publisher " + connection.transport.livekit_service_url + "]",
),
);
},
@@ -614,7 +622,7 @@ export function createCallViewModel$(
),
);
const audioParticipants$ = scope.behavior(
const livekitRoomItems$ = scope.behavior(
matrixLivekitMembers$.pipe(
switchMap((members) => {
const a$ = combineLatest(
@@ -639,7 +647,7 @@ export function createCallViewModel$(
return a$;
}),
map((members) =>
members.reduce<AudioLivekitItem[]>((acc, curr) => {
members.reduce<LivekitRoomItem[]>((acc, curr) => {
if (!curr) return acc;
const existing = acc.find((item) => item.url === curr.url);
@@ -1509,10 +1517,7 @@ export function createCallViewModel$(
),
null,
),
participantCount$: participantCount$,
audioParticipants$: audioParticipants$,
handsRaised$: handsRaised$,
reactions$: reactions$,
joinSoundEffect$: joinSoundEffect$,
@@ -1531,6 +1536,16 @@ export function createCallViewModel$(
spotlight$: spotlight$,
pip$: pip$,
layout$: layout$,
userMedia$,
localMatrixLivekitMember$,
matrixLivekitMembers$: scope.behavior(
matrixLivekitMembers$.pipe(
map((members) => members.value),
tap((v) => {
logger.debug("matrixLivekitMembers$ updated (exported)", v);
}),
),
),
tileStoreGeneration$: tileStoreGeneration$,
showSpotlightIndicators$: showSpotlightIndicators$,
showSpeakingIndicators$: showSpeakingIndicators$,
@@ -1539,6 +1554,8 @@ export function createCallViewModel$(
earpieceMode$: earpieceMode$,
audioOutputSwitcher$: audioOutputSwitcher$,
reconnecting$: localMembership.reconnecting$,
livekitRoomItems$,
connected$: localMembership.connected$,
};
}

28
src/state/CallViewModel/localMember/LocalMember.ts
View File

@@ -139,7 +139,16 @@ interface Props {
* We want
* - a publisher
* -
* @param param0
* @param props The properties required to create the local membership.
* @param props.scope The observable scope to use.
* @param props.connectionManager The connection manager to get connections from.
* @param props.createPublisherFactory Factory to create a publisher once we have a connection.
* @param props.joinMatrixRTC Callback to join the matrix RTC session once we have a transport.
* @param props.homeserverConnected The homeserver connected state.
* @param props.localTransport$ The local transport to use for publishing.
* @param props.logger The logger to use.
* @param props.muteStates The mute states for video and audio.
* @param props.matrixRTCSession The matrix RTC session to join.
* @returns
* - publisher: The handle to create tracks and publish them to the room.
* - connected$: the current connection state. Including matrix server and livekit server connection. (only considering the livekit server we are using for our own media publication)
@@ -178,14 +187,18 @@ export const createLocalMembership$ = ({
// tracks$: Behavior<LocalTrack[]>;
participant$: Behavior<LocalParticipant | null>;
connection$: Behavior<Connection | null>;
/** Shorthand for homeserverConnected.rtcSession === Status.Reconnecting
* Direct translation to the js-sdk membership manager connection `Status`.
/**
* Tracks the homserver and livekit connected state and based on that computes reconnecting.
*/
reconnecting$: Behavior<boolean>;
/** Shorthand for homeserverConnected.rtcSession === Status.Disconnected
* Direct translation to the js-sdk membership manager connection `Status`.
*/
disconnected$: Behavior<boolean>;
/**
* Fully connected
*/
connected$: Behavior<boolean>;
} => {
const logger = parentLogger.getChild("[LocalMembership]");
logger.debug(`Creating local membership..`);
@@ -639,6 +652,7 @@ export const createLocalMembership$ = ({
localMemberState$,
participant$,
reconnecting$,
connected$: matrixAndLivekitConnected$,
disconnected$: scope.behavior(
homeserverConnected.rtsSession$.pipe(
map((state) => state === RTCSessionStatus.Disconnected),
@@ -672,9 +686,11 @@ interface EnterRTCSessionOptions {
* - Delay events management
* - Handles retries (fails only after several attempts)
*
* @param rtcSession
* @param transport
* @param options
* @param rtcSession - The MatrixRTCSession to join.
* @param transport - The LivekitTransport to use for this session.
* @param options - Options for entering the RTC session.
* @param options.encryptMedia - Whether to encrypt media.
* @param options.matrixRTCMode - The Matrix RTC mode to use.
* @throws If the widget could not send ElementWidgetActions.JoinCall action.
*/
// Exported for unit testing

179
src/state/CallViewModel/localMember/LocalTransport.test.ts
View File

@@ -5,9 +5,18 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import {
afterEach,
beforeEach,
describe,
expect,
it,
type MockedObject,
vi,
} from "vitest";
import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc";
import { BehaviorSubject } from "rxjs";
import { BehaviorSubject, lastValueFrom } from "rxjs";
import fetchMock from "fetch-mock";
import { mockConfig, flushPromises, ownMemberMock } from "../../../utils/test";
import { createLocalTransport$ } from "./LocalTransport";
@@ -18,8 +27,17 @@ import {
FailToGetOpenIdToken,
} from "../../../utils/errors";
import * as openIDSFU from "../../../livekit/openIDSFU";
import { customLivekitUrl } from "../../../settings/settings";
import { testJWTToken } from "../../../utils/test-fixtures";
describe("LocalTransport", () => {
const openIdResponse: openIDSFU.SFUConfig = {
url: "https://lk.example.org",
jwt: testJWTToken,
livekitAlias: "!example_room_id",
livekitIdentity: "@lk_user:ABCDEF",
};
let scope: ObservableScope;
beforeEach(() => (scope = new ObservableScope()));
afterEach(() => scope.end());
@@ -65,13 +83,15 @@ describe("LocalTransport", () => {
const errors: Error[] = [];
const localTransport$ = createLocalTransport$({
scope,
roomId: "!room:example.org",
roomId: "!example_room_id",
useOldestMember$: constant(false),
memberships$: constant(new Epoch<CallMembership[]>([])),
client: {
baseUrl: "https://lk.example.org",
// Use empty domain to skip .well-known and use config directly
getDomain: () => "",
// eslint-disable-next-line @typescript-eslint/naming-convention
_unstable_getRTCTransports: async () => Promise.resolve([]),
getOpenIdToken: vi.fn(),
getDeviceId: vi.fn(),
},
@@ -145,11 +165,13 @@ describe("LocalTransport", () => {
const localTransport$ = createLocalTransport$({
scope,
roomId: "!room:example.org",
roomId: "!example_room_id",
useOldestMember$: constant(true),
memberships$,
client: {
getDomain: () => "",
// eslint-disable-next-line @typescript-eslint/naming-convention
_unstable_getRTCTransports: async () => Promise.resolve([]),
getOpenIdToken: vi.fn(),
getDeviceId: vi.fn(),
baseUrl: "https://lk.example.org",
@@ -159,14 +181,159 @@ describe("LocalTransport", () => {
delayId$: constant("delay_id_mock"),
});
openIdResolver.resolve?.({ url: "https://lk.example.org", jwt: "jwt" });
openIdResolver.resolve?.(openIdResponse);
expect(localTransport$.value).toBe(null);
await flushPromises();
// final
expect(localTransport$.value).toStrictEqual({
livekit_alias: "!room:example.org",
livekit_alias: "!example_room_id",
livekit_service_url: "https://lk.example.org",
type: "livekit",
});
});
type LocalTransportProps = Parameters<typeof createLocalTransport$>[0];
describe("transport configuration mechanisms", () => {
let localTransportOpts: LocalTransportProps & {
client: MockedObject<LocalTransportProps["client"]>;
};
let openIdResolver: PromiseWithResolvers<openIDSFU.SFUConfig>;
beforeEach(() => {
mockConfig({});
customLivekitUrl.setValue(customLivekitUrl.defaultValue);
localTransportOpts = {
scope,
roomId: "!example_room_id",
useOldestMember$: constant(false),
memberships$: constant(new Epoch<CallMembership[]>([])),
client: {
getDomain: vi.fn().mockReturnValue(""),
// eslint-disable-next-line @typescript-eslint/naming-convention
_unstable_getRTCTransports: vi.fn().mockResolvedValue([]),
getOpenIdToken: vi.fn(),
getDeviceId: vi.fn(),
},
};
openIdResolver = Promise.withResolvers<openIDSFU.SFUConfig>();
vi.spyOn(openIDSFU, "getSFUConfigWithOpenID").mockReturnValue(
openIdResolver.promise,
);
});
afterEach(() => {
fetchMock.reset();
});
it("supports getting transport via application config", async () => {
mockConfig({
livekit: { livekit_service_url: "https://lk.example.org" },
});
const localTransport$ = createLocalTransport$(localTransportOpts);
openIdResolver.resolve?.(openIdResponse);
expect(localTransport$.value).toBe(null);
await flushPromises();
expect(localTransport$.value).toStrictEqual({
livekit_alias: "!example_room_id",
livekit_service_url: "https://lk.example.org",
type: "livekit",
});
});
it("supports getting transport via user settings", async () => {
customLivekitUrl.setValue("https://lk.example.org");
const localTransport$ = createLocalTransport$(localTransportOpts);
openIdResolver.resolve?.(openIdResponse);
expect(localTransport$.value).toBe(null);
await flushPromises();
expect(localTransport$.value).toStrictEqual({
livekit_alias: "!example_room_id",
livekit_service_url: "https://lk.example.org",
type: "livekit",
});
});
it("supports getting transport via backend", async () => {
localTransportOpts.client._unstable_getRTCTransports.mockResolvedValue([
{ type: "livekit", livekit_service_url: "https://lk.example.org" },
]);
const localTransport$ = createLocalTransport$(localTransportOpts);
openIdResolver.resolve?.(openIdResponse);
expect(localTransport$.value).toBe(null);
await flushPromises();
expect(localTransport$.value).toStrictEqual({
livekit_alias: "!example_room_id",
livekit_service_url: "https://lk.example.org",
type: "livekit",
});
});
it("fails fast if the openID request fails for backend config", async () => {
localTransportOpts.client._unstable_getRTCTransports.mockResolvedValue([
{ type: "livekit", livekit_service_url: "https://lk.example.org" },
]);
openIdResolver.reject(
new FailToGetOpenIdToken(new Error("Test driven error")),
);
try {
await lastValueFrom(createLocalTransport$(localTransportOpts));
throw Error("Expected test to throw");
} catch (ex) {
expect(ex).toBeInstanceOf(FailToGetOpenIdToken);
}
});
it("supports getting transport via well-known", async () => {
localTransportOpts.client.getDomain.mockReturnValue("example.org");
fetchMock.getOnce("https://example.org/.well-known/matrix/client", {
"org.matrix.msc4143.rtc_foci": [
{ type: "livekit", livekit_service_url: "https://lk.example.org" },
],
});
const localTransport$ = createLocalTransport$(localTransportOpts);
openIdResolver.resolve?.(openIdResponse);
expect(localTransport$.value).toBe(null);
await flushPromises();
expect(localTransport$.value).toStrictEqual({
livekit_alias: "!example_room_id",
livekit_service_url: "https://lk.example.org",
type: "livekit",
});
expect(fetchMock.done()).toEqual(true);
});
it("fails fast if the openId request fails for the well-known config", async () => {
localTransportOpts.client.getDomain.mockReturnValue("example.org");
fetchMock.getOnce("https://example.org/.well-known/matrix/client", {
"org.matrix.msc4143.rtc_foci": [
{ type: "livekit", livekit_service_url: "https://lk.example.org" },
],
});
openIdResolver.reject(
new FailToGetOpenIdToken(new Error("Test driven error")),
);
try {
await lastValueFrom(createLocalTransport$(localTransportOpts));
throw Error("Expected test to throw");
} catch (ex) {
expect(ex).toBeInstanceOf(FailToGetOpenIdToken);
}
});
it("throws if no options are available", async () => {
const localTransport$ = createLocalTransport$({
scope,
roomId: "!example_room_id",
useOldestMember$: constant(false),
memberships$: constant(new Epoch<CallMembership[]>([])),
client: {
getDomain: () => "",
// eslint-disable-next-line @typescript-eslint/naming-convention
_unstable_getRTCTransports: async () => Promise.resolve([]),
// These won't be called in this error path but satisfy the type
getOpenIdToken: vi.fn(),
getDeviceId: vi.fn(),
},
});
await flushPromises();
expect(() => localTransport$.value).toThrow(
new MatrixRTCTransportMissingError(""),
);
});
});
});

165
src/state/CallViewModel/localMember/LocalTransport.ts
View File

@@ -8,11 +8,11 @@ Please see LICENSE in the repository root for full details.
import {
type CallMembership,
isLivekitTransport,
type LivekitTransportConfig,
type LivekitTransport,
isLivekitTransportConfig,
type Transport,
} from "matrix-js-sdk/lib/matrixrtc";
import { type MatrixClient } from "matrix-js-sdk";
import { MatrixError, type MatrixClient } from "matrix-js-sdk";
import {
combineLatest,
distinctUntilChanged,
@@ -28,7 +28,10 @@ import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/En
import { type Behavior } from "../../Behavior.ts";
import { type Epoch, type ObservableScope } from "../../ObservableScope.ts";
import { Config } from "../../../config/Config.ts";
import { MatrixRTCTransportMissingError } from "../../../utils/errors.ts";
import {
FailToGetOpenIdToken,
MatrixRTCTransportMissingError,
} from "../../../utils/errors.ts";
import {
getSFUConfigWithOpenID,
type OpenIDClientParts,
@@ -47,7 +50,11 @@ interface Props {
scope: ObservableScope;
ownMembershipIdentity: CallMembershipIdentityParts;
memberships$: Behavior<Epoch<CallMembership[]>>;
client: Pick<MatrixClient, "getDomain" | "baseUrl"> & OpenIDClientParts;
client: Pick<
MatrixClient,
"getDomain" | "baseUrl" | "_unstable_getRTCTransports"
> &
OpenIDClientParts;
roomId: string;
useOldestMember$: Behavior<boolean>;
useOldJwtEndpoint$: Behavior<boolean>;
@@ -141,16 +148,30 @@ export const createLocalTransport$ = ({
const FOCI_WK_KEY = "org.matrix.msc4143.rtc_foci";
/**
* Determine the correct Transport for the current session, including
* validating auth against the service to ensure it's correct.
* Prefers in order:
*
* @param client
* @param roomId
* 1. The `urlFromDevSettings` value. If this cannot be validated, the function will throw.
* 2. The transports returned via the homeserver.
* 3. The transports returned via .well-known.
* 4. The transport configured in Element Call's config.
*
* @param client The authenticated Matrix client for the current user
* @param roomId The ID of the room to be connected to.
* @param urlFromDevSettings Override URL provided by the user's local config.
* @param useMatrix2 This implies using the matrix2 jwt endpoint (including delayed event delegation of the jwt token)
* @param delayId
* @returns
* @returns A fully validated transport config.
* @throws MatrixRTCTransportMissingError | FailToGetOpenIdToken
*/
async function makeTransport(
client: Pick<MatrixClient, "getDomain" | "baseUrl"> & OpenIDClientParts,
client: Pick<
MatrixClient,
"getDomain" | "baseUrl" | "_unstable_getRTCTransports"
> &
OpenIDClientParts,
membership: CallMembershipIdentityParts,
roomId: string,
urlFromDevSettings: string | null,
@@ -159,51 +180,127 @@ async function makeTransport(
): Promise<LivekitTransport & { forceOldJwtEndpoint: boolean }> {
let transport: LivekitTransport | undefined;
logger.trace("Searching for a preferred transport");
//TODO refactor this to use the jwt service returned alias.
const livekitAlias = roomId;
// We will call `getSFUConfigWithOpenID` once per transport here as it's our
// only mechanism of valiation. This means we will also ask the
// homeserver for a OpenID token a few times. Since OpenID tokens are single
// use we don't want to risk any issues by re-using a token.
//
// If the OpenID request were to fail then it's acceptable for us to fail
// this function early, as we assume the homeserver has got some problems.
// DEVTOOL: Highest priority: Load from devtool setting
if (urlFromDevSettings !== null) {
const transportFromStorage: LivekitTransport = {
logger.info("Using LiveKit transport from dev tools: ", urlFromDevSettings);
// Validate that the SFU is up. Otherwise, we want to fail on this
// as we don't permit other SFUs.
const config = await getSFUConfigWithOpenID(
client,
urlFromDevSettings,
roomId,
);
return {
type: "livekit",
livekit_service_url: urlFromDevSettings,
livekit_alias: livekitAlias,
livekit_alias: config.livekitAlias,
};
logger.info(
"Using LiveKit transport from dev tools: ",
transportFromStorage,
);
transport = transportFromStorage;
}
// WELL_KNOWN: Prioritize the .well-known/matrix/client, if available, over the configured SFU
async function getFirstUsableTransport(
transports: Transport[],
): Promise<LivekitTransport | null> {
for (const potentialTransport of transports) {
if (isLivekitTransportConfig(potentialTransport)) {
try {
const { livekitAlias } = await getSFUConfigWithOpenID(
client,
potentialTransport.livekit_service_url,
roomId,
);
return {
...potentialTransport,
livekit_alias: livekitAlias,
};
} catch (ex) {
if (ex instanceof FailToGetOpenIdToken) {
// Explictly throw these
throw ex;
}
logger.debug(
`Could not use SFU service "${potentialTransport.livekit_service_url}" as SFU`,
ex,
);
}
}
}
return null;
}
// MSC4143: Attempt to fetch transports from backend.
if ("_unstable_getRTCTransports" in client) {
try {
const selectedTransport = await getFirstUsableTransport(
await client._unstable_getRTCTransports(),
);
if (selectedTransport) {
logger.info("Using backend-configured SFU", selectedTransport);
return selectedTransport;
}
} catch (ex) {
if (ex instanceof MatrixError && ex.httpStatus === 404) {
// Expected, this is an unstable endpoint and it's not required.
logger.debug("Backend does not provide any RTC transports", ex);
} else if (ex instanceof FailToGetOpenIdToken) {
throw ex;
} else {
// We got an error that wasn't just missing support for the feature, so log it loudly.
logger.error(
"Unexpected error fetching RTC transports from backend",
ex,
);
}
}
}
// Legacy MSC4143 (to be removed) WELL_KNOWN: Prioritize the .well-known/matrix/client, if available.
const domain = client.getDomain();
if (domain && transport === undefined) {
if (domain) {
// we use AutoDiscovery instead of relying on the MatrixClient having already
// been fully configured and started
const wellKnownFoci = (await AutoDiscovery.getRawClientConfig(domain))?.[
FOCI_WK_KEY
];
if (Array.isArray(wellKnownFoci)) {
const wellKnownTransport: LivekitTransportConfig | undefined =
wellKnownFoci.find((f) => f && isLivekitTransportConfig(f));
if (wellKnownTransport !== undefined) {
logger.info("Using LiveKit transport from .well-known: ", transport);
transport = { ...wellKnownTransport, livekit_alias: livekitAlias };
}
const selectedTransport = Array.isArray(wellKnownFoci)
? await getFirstUsableTransport(wellKnownFoci)
: null;
if (selectedTransport) {
logger.info("Using .well-known SFU", selectedTransport);
return selectedTransport;
}
}
// CONFIG: Least prioritized; Load from config file
const urlFromConf = Config.get().livekit?.livekit_service_url;
if (urlFromConf && transport === undefined) {
const transportFromConf: LivekitTransport = {
type: "livekit",
livekit_service_url: urlFromConf,
livekit_alias: livekitAlias,
};
logger.info("Using LiveKit transport from config: ", transportFromConf);
transport = transportFromConf;
if (urlFromConf) {
try {
const { livekitAlias } = await getSFUConfigWithOpenID(
client,
urlFromConf,
roomId,
);
const selectedTransport: LivekitTransport = {
type: "livekit",
livekit_service_url: urlFromConf,
livekit_alias: livekitAlias,
};
logger.info("Using config SFU", selectedTransport);
return selectedTransport;
} catch (ex) {
if (ex instanceof FailToGetOpenIdToken) {
throw ex;
}
logger.error("Failed to validate config SFU", ex);
}
}
if (!transport) throw new MatrixRTCTransportMissingError(domain ?? "");

5
src/state/CallViewModel/localMember/Publisher.ts
View File

@@ -143,7 +143,7 @@ export class Publisher {
this.logger.debug("createAndSetupTracks called");
const lkRoom = this.connection.livekitRoom;
// Observe mute state changes and update LiveKit microphone/camera states accordingly
this.observeMuteStates(this.scope);
this.observeMuteStates();
// Check if audio and/or video is enabled. We only create tracks if enabled,
// because it could prompt for permission, and we don't want to do that unnecessarily.
@@ -356,10 +356,9 @@ export class Publisher {
/**
* Observe changes in the mute states and update the LiveKit room accordingly.
* @param scope
* @private
*/
private observeMuteStates(scope: ObservableScope): void {
private observeMuteStates(): void {
const lkRoom = this.connection.livekitRoom;
this.muteStates.audio.setHandler(async (enable) => {
try {

26
src/state/CallViewModel/remoteMembers/Connection.test.ts
View File

@@ -39,6 +39,7 @@ import {
ElementCallError,
FailToGetOpenIdToken,
} from "../../../utils/errors.ts";
import { testJWTToken } from "../../../utils/test-fixtures.ts";
import { mockRemoteParticipant, ownMemberMock } from "../../../utils/test.ts";
let testScope: ObservableScope;
@@ -122,7 +123,7 @@ function setupRemoteConnection(): Connection {
status: 200,
body: {
url: "wss://matrix-rtc.m.localhost/livekit/sfu",
jwt: "ATOKEN",
jwt: testJWTToken,
},
};
});
@@ -259,7 +260,7 @@ describe("Start connection states", () => {
capturedState.cause instanceof Error
) {
expect(capturedState.cause.message).toContain(
"SFU Config fetch failed with exception Error",
"SFU Config fetch failed with exception",
);
expect(connection.transport.livekit_alias).toEqual(
livekitFocus.livekit_alias,
@@ -295,7 +296,7 @@ describe("Start connection states", () => {
status: 200,
body: {
url: "wss://matrix-rtc.m.localhost/livekit/sfu",
jwt: "ATOKEN",
jwt: testJWTToken,
},
};
});
@@ -393,7 +394,7 @@ describe("remote participants", () => {
// livekitRoom and the rtc membership in order to publish the members that are publishing
// on this connection.
const participants: RemoteParticipant[] = [
let participants: RemoteParticipant[] = [
mockRemoteParticipant({ identity: "@alice:example.org:DEV000" }),
mockRemoteParticipant({ identity: "@bob:example.org:DEV111" }),
mockRemoteParticipant({ identity: "@carol:example.org:DEV222" }),
@@ -415,7 +416,22 @@ describe("remote participants", () => {
fakeLivekitRoom.emit(RoomEvent.ParticipantConnected, p),
);
// All remote participants should be present
// At this point there should be ~~no~~ publishers
// We do have publisher now, since we do not filter for publishers anymore (to also have participants with only data tracks)
// The filtering we do is just based on the matrixRTC member events.
expect(observedParticipants.pop()!.length).toEqual(4);
participants = [
mockRemoteParticipant({ identity: "@alice:example.org:DEV000" }),
mockRemoteParticipant({ identity: "@bob:example.org:DEV111" }),
mockRemoteParticipant({ identity: "@carol:example.org:DEV222" }),
mockRemoteParticipant({ identity: "@dan:example.org:DEV333" }),
];
participants.forEach((p) =>
fakeLivekitRoom.emit(RoomEvent.ParticipantConnected, p),
);
// At this point there should be no publishers
expect(observedParticipants.pop()!.length).toEqual(4);
});

4
src/state/CallViewModel/remoteMembers/Connection.ts
View File

@@ -228,7 +228,7 @@ export class Connection {
*
* @param opts - Connection options {@link ConnectionOpts}.
*
* @param logger
* @param logger - The logger to use.
*/
public constructor(
opts: ConnectionOpts,
@@ -238,7 +238,7 @@ export class Connection {
this.forceOldJwtEndpoint = opts.forceOldJwtEndpoint ?? false;
this.logger = logger.getChild("[Connection]");
this.logger.info(
`[Connection] Creating new connection to ${opts.transport.livekit_service_url} ${opts.transport.livekit_alias}`,
`Creating new connection to ${opts.transport.livekit_service_url} ${opts.transport.livekit_alias}`,
);
const { transport, client, scope } = opts;

7
src/state/CallViewModel/remoteMembers/ConnectionFactory.ts
View File

@@ -13,7 +13,8 @@ import {
type BaseE2EEManager,
} from "livekit-client";
import { type Logger } from "matrix-js-sdk/lib/logger";
import E2EEWorker from "livekit-client/e2ee-worker?worker";
// imported as inline to support worker when loaded from a cdn (cross domain)
import E2EEWorker from "livekit-client/e2ee-worker?worker&inline";
import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc/LivekitTransport";
@@ -45,11 +46,11 @@ export class ECConnectionFactory implements ConnectionFactory {
* @param client - The OpenID client parts for authentication, needed to get openID and JWT tokens.
* @param devices - Used for video/audio out/in capture options.
* @param processorState$ - Effects like background blur (only for publishing connection?)
* @param livekitKeyProvider
* @param livekitKeyProvider - Optional key provider for end-to-end encryption.
* @param controlledAudioDevices - Option to indicate whether audio output device is controlled externally (native mobile app).
* @param livekitRoomFactory - Optional factory function (for testing) to create LivekitRoom instances. If not provided, a default factory is used.
* @param echoCancellation - Whether to enable echo cancellation for audio capture.
* @param noiseSuppression - Whether to enable noise suppression for audio capture.
* @param livekitRoomFactory - Optional factory function (for testing) to create LivekitRoom instances. If not provided, a default factory is used.
*/
public constructor(
private client: OpenIDClientParts,

52
src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts
View File

@@ -293,47 +293,47 @@ describe("connectionManagerData$ stream", () => {
a: expect.toSatisfy((e) => {
const data: ConnectionManagerData = e.value;
expect(data.getConnections().length).toBe(2);
expect(data.getParticipantForTransport(TRANSPORT_1).length).toBe(0);
expect(data.getParticipantForTransport(TRANSPORT_2).length).toBe(0);
expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(0);
expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(0);
return true;
}),
b: expect.toSatisfy((e) => {
const data: ConnectionManagerData = e.value;
expect(data.getConnections().length).toBe(2);
expect(data.getParticipantForTransport(TRANSPORT_1).length).toBe(1);
expect(data.getParticipantForTransport(TRANSPORT_2).length).toBe(0);
expect(data.getParticipantForTransport(TRANSPORT_1)[0].identity).toBe(
"user1A",
);
expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(1);
expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(0);
expect(
data.getParticipantsForTransport(TRANSPORT_1)[0].identity,
).toBe("user1A");
return true;
}),
c: expect.toSatisfy((e) => {
const data: ConnectionManagerData = e.value;
expect(data.getConnections().length).toBe(2);
expect(data.getParticipantForTransport(TRANSPORT_1).length).toBe(1);
expect(data.getParticipantForTransport(TRANSPORT_2).length).toBe(1);
expect(data.getParticipantForTransport(TRANSPORT_1)[0].identity).toBe(
"user1A",
);
expect(data.getParticipantForTransport(TRANSPORT_2)[0].identity).toBe(
"user2A",
);
expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(1);
expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(1);
expect(
data.getParticipantsForTransport(TRANSPORT_1)[0].identity,
).toBe("user1A");
expect(
data.getParticipantsForTransport(TRANSPORT_2)[0].identity,
).toBe("user2A");
return true;
}),
d: expect.toSatisfy((e) => {
const data: ConnectionManagerData = e.value;
expect(data.getConnections().length).toBe(2);
expect(data.getParticipantForTransport(TRANSPORT_1).length).toBe(2);
expect(data.getParticipantForTransport(TRANSPORT_2).length).toBe(1);
expect(data.getParticipantForTransport(TRANSPORT_1)[0].identity).toBe(
"user1A",
);
expect(data.getParticipantForTransport(TRANSPORT_1)[1].identity).toBe(
"user1B",
);
expect(data.getParticipantForTransport(TRANSPORT_2)[0].identity).toBe(
"user2A",
);
expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(2);
expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(1);
expect(
data.getParticipantsForTransport(TRANSPORT_1)[0].identity,
).toBe("user1A");
expect(
data.getParticipantsForTransport(TRANSPORT_1)[1].identity,
).toBe("user1B");
expect(
data.getParticipantsForTransport(TRANSPORT_2)[0].identity,
).toBe("user2A");
return true;
}),
});

31
src/state/CallViewModel/remoteMembers/ConnectionManager.ts
View File

@@ -20,8 +20,10 @@ import { areLivekitTransportsEqual } from "./MatrixLivekitMembers.ts";
import { type ConnectionFactory } from "./ConnectionFactory.ts";
export class ConnectionManagerData {
private readonly store: Map<string, [Connection, RemoteParticipant[]]> =
new Map();
private readonly store: Map<
string,
{ connection: Connection; participants: RemoteParticipant[] }
> = new Map();
public constructor() {}
@@ -29,9 +31,9 @@ export class ConnectionManagerData {
const key = this.getKey(connection.transport);
const existing = this.store.get(key);
if (!existing) {
this.store.set(key, [connection, participants]);
this.store.set(key, { connection, participants });
} else {
existing[1].push(...participants);
existing.participants.push(...participants);
}
}
@@ -40,20 +42,24 @@ export class ConnectionManagerData {
}
public getConnections(): Connection[] {
return Array.from(this.store.values()).map(([connection]) => connection);
return Array.from(this.store.values()).map(({ connection }) => connection);
}
public getConnectionForTransport(
transport: LivekitTransport,
): Connection | null {
return this.store.get(this.getKey(transport))?.[0] ?? null;
return this.store.get(this.getKey(transport))?.connection ?? null;
}
public getParticipantForTransport(
public getParticipantsForTransport(
transport: LivekitTransport,
): RemoteParticipant[] {
const key = transport.livekit_service_url + "|" + transport.livekit_alias;
return this.store.get(key)?.[1] ?? [];
const existing = this.store.get(key);
if (existing) {
return existing.participants;
}
return [];
}
}
@@ -74,9 +80,11 @@ export interface IConnectionManager {
/**
* Crete a `ConnectionManager`
* @param scope the observable scope used by this object.
* @param connectionFactory used to create new connections.
* @param _transportsSubscriptions$ A list of Behaviors each containing a LIST of LivekitTransport.
* @param props - Configuration object
* @param props.scope - The observable scope used by this object
* @param props.connectionFactory - Used to create new connections
* @param props.inputTransports$ - A list of Behaviors each containing a LIST of LivekitTransport.
* @param props.logger - The logger to use
* Each of these behaviors can be interpreted as subscribed list of transports.
*
* Using `registerTransports` independent external modules can control what connections
@@ -207,6 +215,7 @@ export function createConnectionManager$({
);
// probably not required
if (listOfConnectionsWithRemoteParticipants.length === 0) {
return of(new Epoch(new ConnectionManagerData(), epoch));
}

4
src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts
View File

@@ -249,7 +249,7 @@ describe("Publication edge case", () => {
constant(connectionWithPublisher),
);
const matrixLivekitMember$ = createMatrixLivekitMembers$({
const matrixLivekitMembers$ = createMatrixLivekitMembers$({
scope: testScope,
membershipsWithTransport$: testScope.behavior(membershipsWithTransport$),
connectionManager: {
@@ -257,7 +257,7 @@ describe("Publication edge case", () => {
} as unknown as IConnectionManager,
});
await flushPromises();
expect(matrixLivekitMember$.value.value).toSatisfy(
expect(matrixLivekitMembers$.value.value).toSatisfy(
(data: RemoteMatrixLivekitMember[]) => {
expect(data.length).toEqual(2);
expect(data[0].membership$.value).toBe(bobMembership);

2
src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts
View File

@@ -100,7 +100,7 @@ export function createMatrixLivekitMembers$({
function* ([membershipsWithTransport, managerData]) {
for (const { membership, transport } of membershipsWithTransport) {
const participants = transport
? managerData.getParticipantForTransport(transport)
? managerData.getParticipantsForTransport(transport)
: [];
const participant =
participants.find(

13
src/state/CallViewModel/remoteMembers/integration.test.ts
View File

@@ -13,7 +13,11 @@ import fetchMock from "fetch-mock";
import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
import { logger } from "matrix-js-sdk/lib/logger";
import { type Epoch, ObservableScope, trackEpoch } from "../../ObservableScope.ts";
import {
type Epoch,
ObservableScope,
trackEpoch,
} from "../../ObservableScope.ts";
import { ECConnectionFactory } from "./ConnectionFactory.ts";
import { type OpenIDClientParts } from "../../../livekit/openIDSFU.ts";
import {
@@ -31,6 +35,7 @@ import {
import { createConnectionManager$ } from "./ConnectionManager.ts";
import { membershipsAndTransports$ } from "../../SessionBehaviors.ts";
import { constant } from "../../Behavior.ts";
import { testJWTToken } from "../../../utils/test-fixtures.ts";
// Test the integration of ConnectionManager and MatrixLivekitMerger
@@ -83,7 +88,7 @@ beforeEach(() => {
status: 200,
body: {
url: `wss://${domain}/livekit/sfu`,
jwt: "ATOKEN",
jwt: testJWTToken,
},
};
});
@@ -124,14 +129,14 @@ test("bob, carl, then bob joining no tracks yet", () => {
ownMembershipIdentity: ownMemberMock,
});
const matrixLivekitItems$ = createMatrixLivekitMembers$({
const matrixLivekitMembers$ = createMatrixLivekitMembers$({
scope: testScope,
membershipsWithTransport$:
membershipsAndTransports.membershipsWithTransport$,
connectionManager,
});
expectObservable(matrixLivekitItems$).toBe(vMarble, {
expectObservable(matrixLivekitMembers$).toBe(vMarble, {
a: expect.toSatisfy((e: Epoch<RemoteMatrixLivekitMember[]>) => {
const items = e.value;
expect(items.length).toBe(1);

11
src/useAudioContext.tsx
View File

@@ -22,9 +22,12 @@ import * as controls from "./controls";
* Play a sound though a given AudioContext. Will take
* care of connecting the correct buffer and gating
* through gain.
* @param volume The volume to play at.
* @param ctx The context to play through.
* @param buffer The buffer to play.
* @param volume The volume to play at.
* @param stereoPan The stereo pan to apply.
* @param delayS Delay in seconds before starting playing.
* @param abort Optional AbortController that can be used to stop playback.
* @returns A promise that resolves when the sound has finished playing.
*/
async function playSound(
@@ -55,9 +58,11 @@ async function playSound(
* Play a sound though a given AudioContext, looping until stopped. Will take
* care of connecting the correct buffer and gating
* through gain.
* @param volume The volume to play at.
* @param ctx The context to play through.
* @param buffer The buffer to play.
* @param volume The volume to play at.
* @param stereoPan The stereo pan to apply.
* @param delayS Delay in seconds between each loop.
* @returns A function used to end the sound. This function will return a promise when the sound has stopped.
*/
function playSoundLooping(
@@ -120,7 +125,7 @@ interface UseAudioContext<S extends string> {
/**
* Add an audio context which can be used to play
* a set of preloaded sounds.
* @param props
* @param props The properties for the audio context.
* @returns Either an instance that can be used to play sounds, or null if not ready.
*/
export function useAudioContext<S extends string>(

7
src/utils/displayname.ts
View File

@@ -77,6 +77,13 @@ export function shouldDisambiguate(
);
}
/**
* Calculates a display name for a member, optionally disambiguating it.
* @param member - The member to calculate the display name for.
* @param member.rawDisplayName - The raw display name of the member
* @param member.userId - The user ID of the member
* @param disambiguate - Whether to disambiguate the display name.
*/
export function calculateDisplayName(
member: { rawDisplayName?: string; userId: string },
disambiguate: boolean,

50
src/utils/errors.ts
View File

@@ -57,9 +57,16 @@ export class ElementCallError extends Error {
}
}
/**
* Configuration problem due to no MatrixRTC backend/SFU is exposed via .well-known and no fallback configured.
*/
export class MatrixRTCTransportMissingError extends ElementCallError {
public domain: string;
/**
* Creates an instance of MatrixRTCTransportMissingError.
* @param domain - The domain where the MatrixRTC transport is missing.
*/
public constructor(domain: string) {
super(
t("error.call_is_not_supported"),
@@ -75,6 +82,9 @@ export class MatrixRTCTransportMissingError extends ElementCallError {
}
}
/**
* Error indicating that the connection to the call was lost and could not be re-established.
*/
export class ConnectionLostError extends ElementCallError {
public constructor() {
super(
@@ -86,7 +96,16 @@ export class ConnectionLostError extends ElementCallError {
}
}
/**
* Error indicating a failure in the membership manager causing the join call
* operation to fail.
*/
export class MembershipManagerError extends ElementCallError {
/**
* Creates an instance of MembershipManagerError.
*
* @param error - The underlying error that caused the membership manager failure.
*/
public constructor(error: Error) {
super(
t("error.membership_manager"),
@@ -98,6 +117,9 @@ export class MembershipManagerError extends ElementCallError {
}
}
/**
* Error indicating that end-to-end encryption is not supported in the current environment.
*/
export class E2EENotSupportedError extends ElementCallError {
public constructor() {
super(
@@ -109,7 +131,14 @@ export class E2EENotSupportedError extends ElementCallError {
}
}
/**
* Error indicating an unknown issue occurred during a call operation.
*/
export class UnknownCallError extends ElementCallError {
/**
* Creates an instance of UnknownCallError.
* @param error - The underlying error that caused the unknown issue.
*/
public constructor(error: Error) {
super(
t("error.generic"),
@@ -122,7 +151,14 @@ export class UnknownCallError extends ElementCallError {
}
}
/**
* Error indicating a failure to obtain an OpenID token.
*/
export class FailToGetOpenIdToken extends ElementCallError {
/**
* Creates an instance of FailToGetOpenIdToken.
* @param error - The underlying error that caused the failure.
*/
public constructor(error: Error) {
super(
t("error.generic"),
@@ -135,7 +171,14 @@ export class FailToGetOpenIdToken extends ElementCallError {
}
}
/**
* Error indicating a failure to start publishing on a LiveKit connection.
*/
export class FailToStartLivekitConnection extends ElementCallError {
/**
* Creates an instance of FailToStartLivekitConnection.
* @param e - An optional error message providing additional context.
*/
public constructor(e?: string) {
super(
t("error.failed_to_start_livekit"),
@@ -146,6 +189,9 @@ export class FailToStartLivekitConnection extends ElementCallError {
}
}
/**
* Error indicating that a LiveKit's server has hit its track limits.
*/
export class InsufficientCapacityError extends ElementCallError {
public constructor() {
super(
@@ -157,6 +203,10 @@ export class InsufficientCapacityError extends ElementCallError {
}
}
/**
* Error indicating that room creation is restricted by the SFU.
* Only authorized users can create rooms, so the room must exist before connecting (done by the auth jwt service)
*/
export class SFURoomCreationRestrictedError extends ElementCallError {
public constructor() {
super(

7
src/utils/matrix.ts
View File

@@ -188,7 +188,6 @@ function fullAliasFromRoomName(roomName: string, client: MatrixClient): string {
* Applies some basic sanitisation to a room name that the user
* has given us
* @param input The room name from the user
* @param client A matrix client object
*/
export function sanitiseRoomNameInput(input: string): string {
// check to see if the user has entered a fully qualified room
@@ -304,8 +303,9 @@ export async function createRoom(
/**
* Returns an absolute URL to that will load Element Call with the given room
* @param roomId ID of the room
* @param roomName Name of the room
* @param encryptionSystem what encryption (or EncryptionSystem.Unencrypted) the room uses
* @param roomName Name of the room
* @param viaServers Optional list of servers to include as 'via' parameters in the URL
*/
export function getAbsoluteRoomUrl(
roomId: string,
@@ -321,8 +321,9 @@ export function getAbsoluteRoomUrl(
/**
* Returns a relative URL to that will load Element Call with the given room
* @param roomId ID of the room
* @param roomName Name of the room
* @param encryptionSystem what encryption (or EncryptionSystem.Unencrypted) the room uses
* @param roomName Name of the room
* @param viaServers Optional list of servers to include as 'via' parameters in the URL
*/
export function getRelativeRoomUrl(
roomId: string,

1
src/utils/media.ts
View File

@@ -8,6 +8,7 @@ Please see LICENSE in the repository root for full details.
/**
* Finds a media device with label matching 'deviceName'
* @param deviceName The label of the device to look for
* @param kind The kind of media device to look for
* @param devices The list of devices to search
* @returns A matching media device or undefined if no matching device was found
*/

1
src/utils/observable.ts
View File

@@ -135,7 +135,6 @@ interface ItemHandle<Data, Item> {
* requested at a later time, and destroyed (have their scope ended) when the
* key is no longer requested.
*
* @param input$ The input value to be mapped.
* @param generator A generator function yielding a tuple of keys and the
* currently associated data for each item that it wants to exist.
* @param factory A function constructing an individual item, given the item's key,

14
src/utils/test-fixtures.ts
View File

@@ -59,3 +59,17 @@ export const daveRTLRtcMember = mockRtcMembership("@dave2:example.org", "DDDD");
export const daveRTL = mockMatrixRoomMember(daveRTLRtcMember, {
rawDisplayName: "\u202eevaD",
});
export const testJWTToken = [
{}, // header
{
// payload
sub: "@me:example.org:ABCDEF",
video: {
room: "!example_room_id",
},
},
{}, // signature
]
.map((d) => global.btoa(JSON.stringify(d)))
.join(".");

19
src/widget.ts
View File

@@ -64,6 +64,12 @@ export const widget = ((): WidgetHelpers | null => {
try {
const { widgetId, parentUrl } = getUrlParams();
const { roomId, userId, deviceId, baseUrl, e2eEnabled, allowIceFallback } =
getUrlParams();
if (!roomId) throw new Error("Room ID must be supplied");
if (!userId) throw new Error("User ID must be supplied");
if (!deviceId) throw new Error("Device ID must be supplied");
if (!baseUrl) throw new Error("Base URL must be supplied");
if (widgetId && parentUrl) {
const parentOrigin = new URL(parentUrl).origin;
logger.info("Widget API is available");
@@ -92,19 +98,6 @@ export const widget = ((): WidgetHelpers | null => {
// We need to do this now rather than later because it has capabilities to
// request, and is responsible for starting the transport (should it be?)
const {
roomId,
userId,
deviceId,
baseUrl,
e2eEnabled,
allowIceFallback,
} = getUrlParams();
if (!roomId) throw new Error("Room ID must be supplied");
if (!userId) throw new Error("User ID must be supplied");
if (!deviceId) throw new Error("Device ID must be supplied");
if (!baseUrl) throw new Error("Base URL must be supplied");
// These are all the event types the app uses
const sendEvent = [
EventType.CallNotify, // Sent as a deprecated fallback