Merge pull request #3638 from element-hq/toger5/delayed-event-delegation

Pseudonomous identity and use the new jwt service endpoint (with delayed event delegation)
This commit is contained in:
Timo
2026-01-09 19:58:30 +01:00
committed by GitHub
45 changed files with 1447 additions and 683 deletions

View File

@@ -38,6 +38,8 @@ experimental_features:
# MSC4222 needed for syncv2 state_after. This allow clients to
# correctly track the state of the room.
msc4222_enabled: true
# sticky events for MatrixRTC user state
msc4354_enabled: true
# The maximum allowed duration by which sent events can be delayed, as
# per MSC4140. Must be a positive value if set. Defaults to no

View File

@@ -38,7 +38,7 @@ experimental_features:
# MSC4222 needed for syncv2 state_after. This allow clients to
# correctly track the state of the room.
msc4222_enabled: true
# sticky events for matrixRTC user state
# sticky events for MatrixRTC user state
msc4354_enabled: true
# The maximum allowed duration by which sent events can be delayed, as

View File

@@ -38,6 +38,8 @@ experimental_features:
# MSC4222 needed for syncv2 state_after. This allow clients to
# correctly track the state of the room.
msc4222_enabled: true
# sticky events for MatrixRTC user state
msc4354_enabled: true
# The maximum allowed duration by which sent events can be delayed, as
# per MSC4140. Must be a positive value if set. Defaults to no

View File

@@ -38,6 +38,8 @@ experimental_features:
# MSC4222 needed for syncv2 state_after. This allow clients to
# correctly track the state of the room.
msc4222_enabled: true
# sticky events for MatrixRTC user state
msc4354_enabled: true
# The maximum allowed duration by which sent events can be delayed, as
# per MSC4140. Must be a positive value if set. Defaults to no

View File

@@ -3,7 +3,7 @@ networks:
services:
auth-service:
image: ghcr.io/element-hq/lk-jwt-service:latest-ci
image: ghcr.io/element-hq/lk-jwt-service:pr_139
pull_policy: always
hostname: auth-server
environment:
@@ -25,7 +25,7 @@ services:
- ecbackend
auth-service-1:
image: ghcr.io/element-hq/lk-jwt-service:latest-ci
image: ghcr.io/element-hq/lk-jwt-service:pr_139
pull_policy: always
hostname: auth-server-1
environment:
@@ -88,7 +88,7 @@ services:
synapse:
hostname: homeserver
image: docker.io/matrixdotorg/synapse:latest
image: ghcr.io/element-hq/synapse:pr-18968-dcb7678281bc02d4551043a6338fe5b7e6aa47ce
pull_policy: always
environment:
- SYNAPSE_CONFIG_PATH=/data/cfg/homeserver.yaml
@@ -106,7 +106,7 @@ services:
synapse-1:
hostname: homeserver-1
image: docker.io/matrixdotorg/synapse:latest
image: ghcr.io/element-hq/synapse:pr-18968-dcb7678281bc02d4551043a6338fe5b7e6aa47ce
pull_policy: always
environment:
- SYNAPSE_CONFIG_PATH=/data/cfg/homeserver.yaml

View File

@@ -116,6 +116,7 @@
"matrix_rtc_transport_missing": "The server is not configured to work with {{brand}}. Please contact your server admin (Domain: {{domain}}, Error Code: {{ errorCode }}).",
"membership_manager": "Membership Manager Error",
"membership_manager_description": "The Membership Manager had to shut down. This is caused by many consequtive failed network requests.",
"no_matrix_2_authorization_service": "Your authorization service for you media server (SFU) is not on the newest version",
"open_elsewhere": "Opened in another tab",
"open_elsewhere_description": "{{brand}} has been opened in another tab. If that doesn't sound right, try reloading the page.",
"room_creation_restricted": "Failed to create call",

View File

@@ -125,6 +125,7 @@
"typescript": "^5.8.3",
"typescript-eslint-language-service": "^5.0.5",
"unique-names-generator": "^4.6.0",
"uuid": "^13.0.0",
"vaul": "^1.0.0",
"vite": "^7.0.0",
"vite-plugin-generate-file": "^0.3.0",

View File

@@ -101,12 +101,7 @@ export async function createMatrixRTCSdk(
const mediaDevices = new MediaDevices(scope);
const muteStates = new MuteStates(scope, mediaDevices, constant(true));
const slot = { application, id };
const rtcSession = new MatrixRTCSession(
client,
room,
MatrixRTCSession.sessionMembershipsForSlot(room, slot),
slot,
);
const rtcSession = new MatrixRTCSession(client, room, slot);
const callViewModel = createCallViewModel$(
scope,
rtcSession,

View File

@@ -20,6 +20,7 @@ interface Props {
audio?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats;
video?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats;
focusUrl?: string;
rtcBackendIdentity?: string;
}
const extractDomain = (url: string): string => {
@@ -37,6 +38,7 @@ export const RTCConnectionStats: FC<Props> = ({
audio,
video,
focusUrl,
rtcBackendIdentity,
...rest
}) => {
const [showModal, setShowModal] = useState(false);
@@ -71,6 +73,9 @@ export const RTCConnectionStats: FC<Props> = ({
</pre>
</div>
</Modal>
<Text as="span" size="xs" title="rtcBackendIdentity">
rtcBackendIdentity:{rtcBackendIdentity}
</Text>
{focusUrl && (
<div>
<Text as="span" size="xs" title="focusURL">

View File

@@ -6,11 +6,13 @@ Please see LICENSE in the repository root for full details.
*/
import { BaseKeyProvider } from "livekit-client";
import { logger } from "matrix-js-sdk/lib/logger";
import {
type MatrixRTCSession,
MatrixRTCSessionEvent,
} from "matrix-js-sdk/lib/matrixrtc";
import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
const logger = rootLogger.getChild("[MatrixKeyProvider]");
export class MatrixKeyProvider extends BaseKeyProvider {
private rtcSession?: MatrixRTCSession;
@@ -42,7 +44,8 @@ export class MatrixKeyProvider extends BaseKeyProvider {
private onEncryptionKeyChanged = (
encryptionKey: Uint8Array<ArrayBuffer>,
encryptionKeyIndex: number,
participantId: string,
membershipParts: CallMembershipIdentityParts,
rtcBackendIdentity: string,
): void => {
crypto.subtle
.importKey("raw", encryptionKey, "HKDF", false, [
@@ -53,17 +56,17 @@ export class MatrixKeyProvider extends BaseKeyProvider {
(keyMaterial) => {
this.onSetEncryptionKey(
keyMaterial,
participantId,
rtcBackendIdentity,
encryptionKeyIndex,
);
logger.debug(
`Sent new key to livekit room=${this.rtcSession?.room.roomId} participantId=${participantId} encryptionKeyIndex=${encryptionKeyIndex}`,
`Sent new key to livekit room=${this.rtcSession?.room.roomId} participantId=${rtcBackendIdentity} (before hash: ${membershipParts.userId}:${membershipParts.deviceId}) encryptionKeyIndex=${encryptionKeyIndex}`,
);
},
(e) => {
logger.error(
`Failed to create key material from buffer for livekit room=${this.rtcSession?.room.roomId} participantId=${participantId} encryptionKeyIndex=${encryptionKeyIndex}`,
`Failed to create key material from buffer for livekit room=${this.rtcSession?.room.roomId} participantId before hash=${membershipParts.userId}:${membershipParts.deviceId} encryptionKeyIndex=${encryptionKeyIndex}`,
e,
);
},

View File

@@ -15,7 +15,6 @@ import {
type AudioTrackProps,
} from "@livekit/components-react";
import { logger } from "matrix-js-sdk/lib/logger";
import { type ParticipantId } from "matrix-js-sdk/lib/matrixrtc";
import { useEarpieceAudioConfig } from "../MediaDevicesContext";
import { useReactiveState } from "../useReactiveState";
@@ -32,7 +31,7 @@ export interface MatrixAudioRendererProps {
* This list needs to be composed based on the matrixRTC members so that we do not play audio from users
* that are not expected to be in the rtc session (local user is excluded).
*/
validIdentities: ParticipantId[];
validIdentities: string[];
/**
* If set to `true`, mutes all audio tracks rendered by the component.
* @remarks

View File

@@ -18,6 +18,7 @@ import fetchMock from "fetch-mock";
import { getSFUConfigWithOpenID, type OpenIDClientParts } from "./openIDSFU";
import { testJWTToken } from "../utils/test-fixtures";
import { ownMemberMock } from "../utils/test";
const sfuUrl = "https://sfu.example.org";
@@ -33,6 +34,7 @@ describe("getSFUConfigWithOpenID", () => {
vitest.clearAllMocks();
fetchMock.reset();
});
it("should handle fetching a token", async () => {
fetchMock.post("https://sfu.example.org/sfu/get", () => {
return {
@@ -42,6 +44,7 @@ describe("getSFUConfigWithOpenID", () => {
});
const config = await getSFUConfigWithOpenID(
matrixClient,
ownMemberMock,
"https://sfu.example.org",
"!example_room_id",
);
@@ -53,6 +56,7 @@ describe("getSFUConfigWithOpenID", () => {
});
void (await fetchMock.flush());
});
it("should fail if the SFU errors", async () => {
fetchMock.post("https://sfu.example.org/sfu/get", () => {
return {
@@ -63,11 +67,12 @@ describe("getSFUConfigWithOpenID", () => {
try {
await getSFUConfigWithOpenID(
matrixClient,
ownMemberMock,
"https://sfu.example.org",
"!example_room_id",
);
} catch (ex) {
expect(((ex as Error).cause as Error).message).toEqual(
expect((ex as Error).message).toEqual(
"SFU Config fetch failed with status code 500",
);
void (await fetchMock.flush());
@@ -76,6 +81,104 @@ describe("getSFUConfigWithOpenID", () => {
expect.fail("Expected test to throw;");
});
it("should try legacy and then new endpoint with delay delegation", async () => {
fetchMock.post("https://sfu.example.org/get_token", () => {
return {
status: 500,
body: { error: "Test failure" },
};
});
fetchMock.post("https://sfu.example.org/sfu/get", () => {
return {
status: 500,
body: { error: "Test failure" },
};
});
try {
await getSFUConfigWithOpenID(
matrixClient,
ownMemberMock,
"https://sfu.example.org",
"!example_room_id",
{
delayEndpointBaseUrl: "https://matrix.homeserverserver.org",
delayId: "mock_delay_id",
},
);
} catch (ex) {
expect((ex as Error).message).toEqual(
"SFU Config fetch failed with status code 500",
);
void (await fetchMock.flush());
}
const calls = fetchMock.calls();
expect(calls.length).toBe(2);
expect(calls[0][0]).toStrictEqual("https://sfu.example.org/get_token");
expect(calls[0][1]).toStrictEqual({
// check if it uses correct delayID!
body: '{"room_id":"!example_room_id","slot_id":"m.call#ROOM","member":{"id":"@alice:example.org:DEVICE","claimed_user_id":"@alice:example.org","claimed_device_id":"DEVICE"},"delay_id":"mock_delay_id","delay_timeout":1000,"delay_cs_api_url":"https://matrix.homeserverserver.org"}',
method: "POST",
headers: {
"Content-Type": "application/json",
},
});
expect(calls[1][0]).toStrictEqual("https://sfu.example.org/sfu/get");
expect(calls[1][1]).toStrictEqual({
body: '{"room":"!example_room_id","device_id":"DEVICE"}',
headers: {
"Content-Type": "application/json",
},
method: "POST",
});
});
it("dont try legacy if endpoint with delay delegation is sucessful", async () => {
fetchMock.post("https://sfu.example.org/get_token", () => {
return {
status: 200,
body: { url: sfuUrl, jwt: testJWTToken },
};
});
fetchMock.post("https://sfu.example.org/sfu/get", () => {
return {
status: 500,
body: { error: "Test failure" },
};
});
try {
await getSFUConfigWithOpenID(
matrixClient,
ownMemberMock,
"https://sfu.example.org",
"!example_room_id",
{
delayEndpointBaseUrl: "https://matrix.homeserverserver.org",
delayId: "mock_delay_id",
},
);
} catch (ex) {
expect((ex as Error).message).toEqual(
"SFU Config fetch failed with status code 500",
);
void (await fetchMock.flush());
}
const calls = fetchMock.calls();
expect(calls.length).toBe(1);
expect(calls[0][0]).toStrictEqual("https://sfu.example.org/get_token");
expect(calls[0][1]).toStrictEqual({
// check if it uses correct delayID!
body: '{"room_id":"!example_room_id","slot_id":"m.call#ROOM","member":{"id":"@alice:example.org:DEVICE","claimed_user_id":"@alice:example.org","claimed_device_id":"DEVICE"},"delay_id":"mock_delay_id","delay_timeout":1000,"delay_cs_api_url":"https://matrix.homeserverserver.org"}',
method: "POST",
headers: {
"Content-Type": "application/json",
},
});
});
it("should retry fetching the openid token", async () => {
let count = 0;
matrixClient.getOpenIdToken.mockImplementation(async () => {
@@ -98,6 +201,7 @@ describe("getSFUConfigWithOpenID", () => {
});
const config = await getSFUConfigWithOpenID(
matrixClient,
ownMemberMock,
"https://sfu.example.org",
"!example_room_id",
);

View File

@@ -5,11 +5,21 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type IOpenIDToken, type MatrixClient } from "matrix-js-sdk";
import { logger } from "matrix-js-sdk/lib/logger";
import {
retryNetworkOperation,
type IOpenIDToken,
type MatrixClient,
} from "matrix-js-sdk";
import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
import { type Logger } from "matrix-js-sdk/lib/logger";
import { FailToGetOpenIdToken } from "../utils/errors";
import {
FailToGetOpenIdToken,
NoMatrix2AuthorizationService,
} from "../utils/errors";
import { doNetworkOperationWithRetry } from "../utils/matrix";
import { Config } from "../config/Config";
import { JwtEndpointVersion } from "../state/CallViewModel/localMember/LocalTransport";
/**
* Configuration and access tokens provided by the SFU on successful authentication.
@@ -18,6 +28,7 @@ export interface SFUConfig {
url: string;
jwt: string;
livekitAlias: string;
// NOTE: Currently unused.
livekitIdentity: string;
}
@@ -64,15 +75,32 @@ export type OpenIDClientParts = Pick<
* to the matrix RTC backend in order to get acces to the SFU.
* It has built-in retry for calls to the homeserver with a backoff policy.
* @param client The Matrix client
* @param membership Our own membership identity parts used to send to jwt service.
* @param serviceUrl The URL of the livekit SFU service
* @param matrixRoomId The Matrix room ID for which to get the SFU config
* @param roomId The room id used in the jwt request. This is NOT the livekit_alias. The jwt service will provide the alias. It maps matrix room ids <-> Livekit aliases.
* @param opts Additional options to modify which endpoint with which data will be used to aquire the jwt token.
* @param opts.forceJwtEndpoint This will use the old jwt endpoint which will create the rtc backend identity based on string concatination
* instead of a hash.
* This function by default uses whatever is possible with the current jwt service installed next to the SFU.
* For remote connections this does not matter, since we will not publish there we can rely on the newest option.
* For our own connection we can only use the hashed version if we also send the new matrix2.0 sticky events.
* @param opts.delayEndpointBaseUrl The URL of the matrix homeserver.
* @param opts.delayId The delay id used for the jwt service to manage.
* @param logger optional logger.
* @returns Object containing the token information
* @throws FailToGetOpenIdToken
*/
export async function getSFUConfigWithOpenID(
client: OpenIDClientParts,
membership: CallMembershipIdentityParts,
serviceUrl: string,
matrixRoomId: string,
roomId: string,
opts?: {
forceJwtEndpoint?: JwtEndpointVersion;
delayEndpointBaseUrl?: string;
delayId?: string;
},
logger?: Logger,
): Promise<SFUConfig> {
let openIdToken: IOpenIDToken;
try {
@@ -84,16 +112,67 @@ export async function getSFUConfigWithOpenID(
error instanceof Error ? error : new Error("Unknown error"),
);
}
logger.debug("Got openID token", openIdToken);
logger?.debug("Got openID token", openIdToken);
logger.info(`Trying to get JWT for focus ${serviceUrl}...`);
const sfuConfig = await getLiveKitJWT(
client,
serviceUrl,
matrixRoomId,
openIdToken,
);
logger.info(`Got JWT from call's active focus URL.`);
logger?.info(`Trying to get JWT for focus ${serviceUrl}...`);
let sfuConfig: { url: string; jwt: string } | undefined;
const tryBothJwtEndpoints = opts?.forceJwtEndpoint === undefined; // This is for SFUs where we do not publish.
const forceMatrix2Jwt =
opts?.forceJwtEndpoint === JwtEndpointVersion.Matrix_2_0;
// We want to start using the new endpoint (with optional delay delegation)
// if we can use both or if we are forced to use the new one.
if (tryBothJwtEndpoints || forceMatrix2Jwt) {
try {
sfuConfig = await getLiveKitJWTWithDelayDelegation(
membership,
serviceUrl,
roomId,
openIdToken,
opts?.delayEndpointBaseUrl,
opts?.delayId,
);
logger?.info(`Got JWT from call's active focus URL.`);
} catch (e) {
if (e instanceof NotSupportedError) {
logger?.warn(
`Failed fetching jwt with matrix 2.0 endpoint (retry with legacy) Not supported`,
e,
);
sfuConfig = undefined;
} else {
logger?.warn(
`Failed fetching jwt with matrix 2.0 endpoint other issues ->`,
`(not going to try with legacy endpoint: forceOldJwtEndpoint is set to false, we did not get a not supported error from the sfu)`,
e,
);
// Make this throw a hard error in case we force the matrix2.0 endpoint.
if (forceMatrix2Jwt)
throw new NoMatrix2AuthorizationService(e as Error);
// NEVER get bejond this point if we forceMatrix2 and it failed!
}
}
}
// DEPRECATED
// here we either have a sfuConfig or we alredy exited because of `if (forceMatrix2) throw ...`
// The only case we can get into this condition is, if `forceMatrix2` is `false`
if (sfuConfig === undefined) {
sfuConfig = await getLiveKitJWT(
membership.deviceId,
serviceUrl,
roomId,
openIdToken,
);
logger?.info(`Got JWT from call's active focus URL.`);
}
if (!sfuConfig) {
throw new Error("No `sfuConfig` after trying with old and new endpoints");
}
// Pull the details from the JWT
const [, payloadStr] = sfuConfig.jwt.split(".");
@@ -104,33 +183,108 @@ export async function getSFUConfigWithOpenID(
url: sfuConfig.url,
livekitAlias: payload.video.room,
// NOTE: Currently unused.
// Probably also not helpful since we now compute the backendIdentity on joining the call so we can use it for the encryption manager.
// The only reason for us to know it locally is to connect the right users with the lk world. (and to set our own keys)
livekitIdentity: payload.sub,
};
}
const RETRIES = 4;
async function getLiveKitJWT(
client: OpenIDClientParts,
deviceId: string,
livekitServiceURL: string,
roomName: string,
matrixRoomId: string,
openIDToken: IOpenIDToken,
): Promise<{ url: string; jwt: string }> {
try {
const res = await fetch(livekitServiceURL + "/sfu/get", {
let res: Response | undefined;
await retryNetworkOperation(RETRIES, async () => {
res = await fetch(livekitServiceURL + "/sfu/get", {
method: "POST",
headers: {
"Content-Type": "application/json",
},
body: JSON.stringify({
room: roomName,
// This is the actual livekit room alias. For the legacy jwt endpoint simply the room id was used.
room: matrixRoomId,
openid_token: openIDToken,
device_id: client.getDeviceId(),
device_id: deviceId,
}),
});
if (!res.ok) {
throw new Error("SFU Config fetch failed with status code " + res.status);
}
return await res.json();
} catch (e) {
throw new Error("SFU Config fetch failed with exception", { cause: e });
});
if (!res) {
throw new Error(
`Network error while connecting to jwt service after ${RETRIES} retries`,
);
}
if (!res.ok) {
throw new Error("SFU Config fetch failed with status code " + res.status);
}
return await res.json();
}
class NotSupportedError extends Error {
public constructor(message: string) {
super(message);
this.name = "NotSupported";
}
}
export async function getLiveKitJWTWithDelayDelegation(
membership: CallMembershipIdentityParts,
livekitServiceURL: string,
matrixRoomId: string,
openIDToken: IOpenIDToken,
delayEndpointBaseUrl?: string,
delayId?: string,
): Promise<{ url: string; jwt: string }> {
const { userId, deviceId, memberId } = membership;
const body = {
room_id: matrixRoomId,
slot_id: "m.call#ROOM",
openid_token: openIDToken,
member: {
id: memberId,
claimed_user_id: userId,
claimed_device_id: deviceId,
},
};
let bodyDalayParts = {};
// Also check for empty string
if (delayId && delayEndpointBaseUrl) {
const delayTimeoutMs =
Config.get().matrix_rtc_session?.delayed_leave_event_delay_ms ?? 1000;
bodyDalayParts = {
delay_id: delayId,
delay_timeout: delayTimeoutMs,
delay_cs_api_url: delayEndpointBaseUrl,
};
}
let res: Response | undefined;
await retryNetworkOperation(RETRIES, async () => {
res = await fetch(livekitServiceURL + "/get_token", {
method: "POST",
headers: {
"Content-Type": "application/json",
},
body: JSON.stringify({ ...body, ...bodyDalayParts }),
});
});
if (!res) {
throw new Error(
`Network error while connecting to jwt service after ${RETRIES} retries`,
);
}
if (!res.ok) {
const msg = "SFU Config fetch failed with status code " + res.status;
if (res.status === 404) {
throw new NotSupportedError(msg);
} else {
throw new Error(msg);
}
}
return await res.json();
}

View File

@@ -798,6 +798,8 @@ export const InCallView: FC<InCallViewProps> = ({
</div>
);
const allConnections = useBehavior(vm.allConnections$);
return (
<div
className={styles.inRoom}
@@ -836,8 +838,14 @@ export const InCallView: FC<InCallViewProps> = ({
onDismiss={closeSettings}
tab={settingsTab}
onTabChange={setSettingsTab}
// TODO expose correct data to setttings modal
livekitRooms={[]}
livekitRooms={allConnections
.getConnections()
.map((connectionItem) => ({
room: connectionItem.livekitRoom,
// TODO compute is local or tag it in the livekit room items already
isLocal: undefined,
url: connectionItem.transport.livekit_service_url,
}))}
/>
</>
)}

View File

@@ -8,3 +8,14 @@ Please see LICENSE in the repository root for full details.
pre {
font-size: var(--font-size-micro);
}
.livekit_room_box {
border: 3px solid var(--cpd-color-bg-subtle-secondary);
border-radius: var(--cpd-space-8x);
padding: var(--cpd-space-4x);
margin-bottom: var(--cpd-space-4x);
margin-top: var(--cpd-space-4x);
li {
font-size: var(--font-size-micro);
}
}

View File

@@ -7,9 +7,9 @@ Please see LICENSE in the repository root for full details.
import { describe, expect, it, vi } from "vitest";
import { render, waitFor } from "@testing-library/react";
import { type Room as LivekitRoom } from "livekit-client";
import type { MatrixClient } from "matrix-js-sdk";
import type { Room as LivekitRoom } from "livekit-client";
import { DeveloperSettingsTab } from "./DeveloperSettingsTab";
// Mock url params hook to avoid environment-dependent snapshot churn.
@@ -30,6 +30,8 @@ function createMockLivekitRoom(
serverInfo,
metadata,
engine: { client: { ws: { url: wsUrl } } },
localParticipant: { identity: "localParticipantIdentity" },
remoteParticipants: new Map(),
} as unknown as LivekitRoom;
return {
@@ -69,6 +71,8 @@ describe("DeveloperSettingsTab", () => {
isLocal: false,
url: "wss://remote-sfu.example.org",
room: {
localParticipant: { identity: "localParticipantIdentity" },
remoteParticipants: new Map(),
serverInfo: { region: "remote", version: "4.5.6" },
metadata: "remote-metadata",
engine: { client: { ws: { url: "wss://remote-sfu.example.org" } } },

View File

@@ -29,6 +29,7 @@ import {
Label,
RadioControl,
} from "@vector-im/compound-web";
import { type Room as LivekitRoom } from "livekit-client";
import { FieldRow, InputField } from "../input/Input";
import {
@@ -42,7 +43,6 @@ import {
customLivekitUrl as customLivekitUrlSetting,
MatrixRTCMode,
} from "./settings";
import type { Room as LivekitRoom } from "livekit-client";
import styles from "./DeveloperSettingsTab.module.css";
import { useUrlParams } from "../UrlParams";
@@ -275,8 +275,8 @@ export const DeveloperSettingsTab: FC<Props> = ({
name={matrixRTCModeRadioGroup}
control={
<RadioControl
checked={matrixRTCMode === MatrixRTCMode.Compatibil}
value={MatrixRTCMode.Compatibil}
checked={matrixRTCMode === MatrixRTCMode.Compatibility}
value={MatrixRTCMode.Compatibility}
onChange={onMatrixRTCModeChange}
/>
}
@@ -304,12 +304,12 @@ export const DeveloperSettingsTab: FC<Props> = ({
</InlineField>
</Form>
{livekitRooms?.map((livekitRoom) => (
<>
<h3>
<div className={styles.livekit_room_box}>
<h4>
{t("developer_mode.livekit_sfu", {
url: livekitRoom.url || "unknown",
})}
</h3>
</h4>
{livekitRoom.isLocal && <p>ws-url: {localSfuUrl?.href}</p>}
<p>
{t("developer_mode.livekit_server_info")}(
@@ -321,7 +321,19 @@ export const DeveloperSettingsTab: FC<Props> = ({
: "undefined"}
{livekitRoom.room.metadata}
</pre>
</>
<p>Local Participant</p>
<pre className={styles.pre}>
{livekitRoom.room.localParticipant.identity}
</pre>
<p>Remote Participants</p>
<ul>
{Array.from(livekitRoom.room.remoteParticipants.keys()).map(
(id) => (
<li key={id}>{id}</li>
),
)}
</ul>
</div>
))}
<p>{t("developer_mode.environment_variables")}</p>
<pre>{JSON.stringify(env, null, 2)}</pre>

View File

@@ -284,7 +284,7 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
name="_r_0_"
title=""
type="radio"
value="compatibil"
value="compatibility"
/>
<div
class="_ui_1qhtc_19"
@@ -349,46 +349,78 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
</div>
</div>
</form>
<h3>
LiveKit SFU: wss://local-sfu.example.org
</h3>
<p>
ws-url:
wss://local-sfu.example.org/
</p>
<p>
LiveKit Server Info
(
local
)
</p>
<pre
class="pre"
<div
class="livekit_room_box"
>
{
<h4>
LiveKit SFU: wss://local-sfu.example.org
</h4>
<p>
ws-url:
wss://local-sfu.example.org/
</p>
<p>
LiveKit Server Info
(
local
)
</p>
<pre
class="pre"
>
{
"region": "local",
"version": "1.2.3"
}
local-metadata
</pre>
<h3>
LiveKit SFU: wss://remote-sfu.example.org
</h3>
<p>
LiveKit Server Info
(
remote
)
</p>
<pre
class="pre"
local-metadata
</pre>
<p>
Local Participant
</p>
<pre
class="pre"
>
localParticipantIdentity
</pre>
<p>
Remote Participants
</p>
<ul />
</div>
<div
class="livekit_room_box"
>
{
<h4>
LiveKit SFU: wss://remote-sfu.example.org
</h4>
<p>
LiveKit Server Info
(
remote
)
</p>
<pre
class="pre"
>
{
"region": "remote",
"version": "4.5.6"
}
remote-metadata
</pre>
remote-metadata
</pre>
<p>
Local Participant
</p>
<pre
class="pre"
>
localParticipantIdentity
</pre>
<p>
Remote Participants
</p>
<ul />
</div>
<p>
Environment variables
</p>

View File

@@ -126,7 +126,13 @@ export const alwaysShowIphoneEarpiece = new Setting<boolean>(
export enum MatrixRTCMode {
Legacy = "legacy",
Compatibil = "compatibil",
Compatibility = "compatibility",
/** This implies using
* - sticky events
* - hashed RTC backend identity
* - the new endpoint for the jwt token on the local membership (remote memberships will always try the new jwt endpoint first -> then the legacy one)
* - use the hashed identity for the local membership
*/
Matrix_2_0 = "matrix_2_0",
}

View File

@@ -78,11 +78,14 @@ vi.mock("../e2ee/matrixKeyProvider");
const getUrlParams = vi.hoisted(() => vi.fn(() => ({})));
vi.mock("../UrlParams", () => ({ getUrlParams }));
vi.mock("../rtcSessionHelpers", async (importOriginal) => ({
...(await importOriginal()),
makeTransport: async (): Promise<LivekitTransport> =>
Promise.resolve(exampleTransport),
}));
vi.mock(
"../state/CallViewModel/localMember/localTransport",
async (importOriginal) => ({
...(await importOriginal()),
makeTransport: async (): Promise<LivekitTransport> =>
Promise.resolve(exampleTransport),
}),
);
const yesNo = {
y: true,
@@ -232,7 +235,7 @@ const mockLegacyRingEvent = {} as { event_id: string } & ICallNotifyContent;
describe.each([
[MatrixRTCMode.Legacy],
[MatrixRTCMode.Compatibil],
[MatrixRTCMode.Compatibility],
[MatrixRTCMode.Matrix_2_0],
])("CallViewModel (%s mode)", (mode) => {
const withCallViewModel = withCallViewModelInMode(mode);
@@ -1255,11 +1258,6 @@ describe.each([
y: () => {
rtcSession.membershipStatus = Status.Connected;
},
n: () => {
// NOTE: This was removed in https://github.com/matrix-org/matrix-js-sdk/pull/5103 accidentally.
// eslint-disable-next-line @typescript-eslint/no-explicit-any
rtcSession.membershipStatus = "Reconnecting" as any;
},
});
schedule(probablyLeftMarbles, {
y: () => {

View File

@@ -41,10 +41,13 @@ import {
} from "rxjs";
import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
import {
MembershipManagerEvent,
type LivekitTransport,
type MatrixRTCSession,
} from "matrix-js-sdk/lib/matrixrtc";
import { type IWidgetApiRequest } from "matrix-widget-api";
import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
import { v4 as uuidv4 } from "uuid";
import {
LocalUserMediaViewModel,
@@ -98,7 +101,7 @@ import {
type SpotlightLandscapeLayoutMedia,
type SpotlightPortraitLayoutMedia,
} from "../layout-types.ts";
import { ElementCallError } from "../../utils/errors.ts";
import { ElementCallError, UnknownCallError } from "../../utils/errors.ts";
import { type ObservableScope } from "../ObservableScope.ts";
import { createHomeserverConnected$ } from "./localMember/HomeserverConnected.ts";
import {
@@ -106,13 +109,19 @@ import {
enterRTCSession,
TransportState,
} from "./localMember/LocalMember.ts";
import { createLocalTransport$ } from "./localMember/LocalTransport.ts";
import {
createLocalTransport$,
JwtEndpointVersion,
} from "./localMember/LocalTransport.ts";
import {
createMemberships$,
membershipsAndTransports$,
} from "../SessionBehaviors.ts";
import { ECConnectionFactory } from "./remoteMembers/ConnectionFactory.ts";
import { createConnectionManager$ } from "./remoteMembers/ConnectionManager.ts";
import {
type ConnectionManagerData,
createConnectionManager$,
} from "./remoteMembers/ConnectionManager.ts";
import {
createMatrixLivekitMembers$,
type TaggedParticipant,
@@ -261,6 +270,7 @@ export interface CallViewModel {
* multiple devices.
*/
participantCount$: Behavior<number>;
allConnections$: Behavior<ConnectionManagerData>;
/** Participants sorted by livekit room so they can be used in the audio rendering */
livekitRoomItems$: Behavior<LivekitRoomItem[]>;
userMedia$: Behavior<UserMedia[]>;
@@ -381,8 +391,11 @@ export function createCallViewModel$(
trackProcessorState$: Behavior<ProcessorState>,
): CallViewModel {
const client = matrixRoom.client;
const userId = client.getUserId()!;
const deviceId = client.getDeviceId()!;
const userId = client.getUserId();
const deviceId = client.getDeviceId();
if (!(userId && deviceId))
throw new UnknownCallError(new Error("userId and deviceId are required"));
const livekitKeyProvider = getE2eeKeyProvider(
options.encryptionSystem,
matrixRTCSession,
@@ -415,11 +428,37 @@ export function createCallViewModel$(
memberships$,
);
const ownMembershipIdentity: CallMembershipIdentityParts = {
userId,
deviceId,
// This will only be consumed by the sticky membership manager. So it has no impact on legacy calls.
memberId: uuidv4(),
};
const localTransport$ = createLocalTransport$({
scope: scope,
memberships$: memberships$,
ownMembershipIdentity,
client,
delayId$: scope.behavior(
(
fromEvent(
matrixRTCSession,
MembershipManagerEvent.DelayIdChanged,
) as Observable<string | undefined>
).pipe(map((v) => v ?? null)),
matrixRTCSession.delayId ?? null,
),
roomId: matrixRoom.roomId,
forceJwtEndpoint$: scope.behavior(
matrixRTCMode$.pipe(
map((v) =>
v === MatrixRTCMode.Matrix_2_0
? JwtEndpointVersion.Matrix_2_0
: JwtEndpointVersion.Legacy,
),
),
),
useOldestMember$: scope.behavior(
matrixRTCMode$.pipe(map((v) => v === MatrixRTCMode.Legacy)),
),
@@ -439,30 +478,20 @@ export function createCallViewModel$(
const connectionManager = createConnectionManager$({
scope: scope,
connectionFactory: connectionFactory,
inputTransports$: scope.behavior(
combineLatest(
[
localTransport$.pipe(
catchError((e: unknown) => {
logger.info(
"dont pass local transport to createConnectionManager$. localTransport$ threw an error",
e,
);
return of(null);
}),
),
membershipsAndTransports.transports$,
],
(localTransport, transports) => {
const localTransportAsArray = localTransport ? [localTransport] : [];
return transports.mapInner((transports) => [
...localTransportAsArray,
...transports,
]);
},
localTransport$: scope.behavior(
localTransport$.pipe(
catchError((e: unknown) => {
logger.info(
"could not pass local transport to createConnectionManager$. localTransport$ threw an error",
e,
);
return of(null);
}),
),
),
logger,
remoteTransports$: membershipsAndTransports.transports$,
logger: logger,
ownMembershipIdentity,
});
const matrixLivekitMembers$ = createMatrixLivekitMembers$({
@@ -493,6 +522,7 @@ export function createCallViewModel$(
joinMatrixRTC: (transport: LivekitTransport) => {
return enterRTCSession(
matrixRTCSession,
ownMembershipIdentity,
transport,
connectOptions$.value,
);
@@ -604,15 +634,14 @@ export function createCallViewModel$(
),
);
const allConnections$ = scope.behavior(
connectionManager.connectionManagerData$.pipe(map((d) => d.value)),
);
const livekitRoomItems$ = scope.behavior(
matrixLivekitMembers$.pipe(
tap((val) => {
logger.debug("matrixLivekitMembers$ updated", val.value);
}),
switchMap((membersWithEpoch) => {
const members = membersWithEpoch.value;
switchMap((members) => {
const a$ = combineLatest(
members.map((member) =>
members.value.map((member) =>
combineLatest([member.connection$, member.participant.value$]).pipe(
map(([connection, participant]) => {
// do not render audio for local participant
@@ -685,29 +714,29 @@ export function createCallViewModel$(
generateItems(
function* ([
localMatrixLivekitMember,
{ value: matrixLivekitMembers },
matrixLivekitMembers,
duplicateTiles,
]) {
let localParticipantId: string | undefined = undefined;
let localUserMediaId: string | undefined = undefined;
// add local member if available
if (localMatrixLivekitMember) {
const { userId, participant, connection$, membership$ } =
localMatrixLivekitMember;
localParticipantId = `${userId}:${membership$.value.deviceId}`; // should be membership$.value.membershipID which is not optional
// const participantId = membership$.value.membershipID;
if (localParticipantId) {
for (let dup = 0; dup < 1 + duplicateTiles; dup++) {
yield {
keys: [
dup,
localParticipantId,
userId,
participant satisfies TaggedParticipant as TaggedParticipant, // Widen the type safely
connection$,
],
data: undefined,
};
}
localUserMediaId = `${userId}:${membership$.value.deviceId}`;
const rtcBackendIdentity = membership$.value.rtcBackendIdentity;
for (let dup = 0; dup < 1 + duplicateTiles; dup++) {
yield {
keys: [
dup,
localUserMediaId,
userId,
participant satisfies TaggedParticipant as TaggedParticipant, // Widen the type safely
connection$,
rtcBackendIdentity,
],
data: undefined,
};
}
}
// add remote members that are available
@@ -716,13 +745,22 @@ export function createCallViewModel$(
participant,
connection$,
membership$,
} of matrixLivekitMembers) {
const participantId = `${userId}:${membership$.value.deviceId}`;
if (participantId === localParticipantId) continue;
// const participantId = membership$.value?.identity;
} of matrixLivekitMembers.value) {
const userMediaId = `${userId}:${membership$.value.deviceId}`;
const rtcBackendIdentity = membership$.value.rtcBackendIdentity;
// skip local user as we added them manually before
if (userMediaId === localUserMediaId) continue;
for (let dup = 0; dup < 1 + duplicateTiles; dup++) {
yield {
keys: [dup, participantId, userId, participant, connection$],
keys: [
dup,
userMediaId,
userId,
participant,
connection$,
rtcBackendIdentity,
],
data: undefined,
};
}
@@ -732,10 +770,11 @@ export function createCallViewModel$(
scope,
_data$,
dup,
participantId,
userMediaId,
userId,
participant,
connection$,
rtcBackendIdentity,
) => {
const livekitRoom$ = scope.behavior(
connection$.pipe(map((c) => c?.livekitRoom)),
@@ -751,8 +790,9 @@ export function createCallViewModel$(
return new UserMedia(
scope,
`${participantId}:${dup}`,
`${userMediaId}:${dup}`,
userId,
rtcBackendIdentity,
participant,
options.encryptionSystem,
livekitRoom$,
@@ -761,8 +801,8 @@ export function createCallViewModel$(
localMembership.reconnecting$,
displayName$,
matrixMemberMetadataStore.createAvatarUrlBehavior$(userId),
handsRaised$.pipe(map((v) => v[participantId]?.time ?? null)),
reactions$.pipe(map((v) => v[participantId] ?? undefined)),
handsRaised$.pipe(map((v) => v[userMediaId]?.time ?? null)),
reactions$.pipe(map((v) => v[userMediaId] ?? undefined)),
);
},
),
@@ -1503,6 +1543,7 @@ export function createCallViewModel$(
),
null,
),
allConnections$,
participantCount$: participantCount$,
handsRaised$: handsRaised$,
reactions$: reactions$,

View File

@@ -24,6 +24,7 @@ import {
mockLivekitRoom,
mockMuteStates,
withTestScheduler,
ownMemberMock,
} from "../../../utils/test";
import {
TransportState,
@@ -38,6 +39,7 @@ import { constant } from "../../Behavior";
import { ConnectionManagerData } from "../remoteMembers/ConnectionManager";
import { ConnectionState, type Connection } from "../remoteMembers/Connection";
import { type Publisher } from "./Publisher";
import { type LocalTransportWithSFUConfig } from "./LocalTransport";
const MATRIX_RTC_MODE = MatrixRTCMode.Legacy;
const getUrlParams = vi.hoisted(() => vi.fn(() => ({})));
@@ -103,11 +105,12 @@ describe("LocalMembership", () => {
getOldestMembership: vi.fn().mockReturnValue({
getPreferredFoci: vi.fn().mockReturnValue([focusFromOlderMembership]),
}),
joinRoomSession: vi.fn(),
joinRTCSession: vi.fn(),
}) as unknown as MatrixRTCSession;
enterRTCSession(
mockedSession,
ownMemberMock,
{
livekit_alias: "roomId",
livekit_service_url: "http://my-well-known-service-url.com",
@@ -119,7 +122,12 @@ describe("LocalMembership", () => {
},
);
expect(mockedSession.joinRoomSession).toHaveBeenLastCalledWith(
expect(mockedSession.joinRTCSession).toHaveBeenLastCalledWith(
{
deviceId: "DEVICE",
memberId: "@alice:example.org:DEVICE",
userId: "@alice:example.org",
},
[
{
livekit_alias: "roomId",
@@ -161,11 +169,12 @@ describe("LocalMembership", () => {
},
memberships: [],
getFocusInUse: vi.fn(),
joinRoomSession: vi.fn(),
joinRTCSession: vi.fn(),
}) as unknown as MatrixRTCSession;
enterRTCSession(
mockedSession,
ownMemberMock,
{
livekit_alias: "roomId",
livekit_service_url: "http://my-well-known-service-url.com",
@@ -204,10 +213,11 @@ describe("LocalMembership", () => {
it("throws error on missing RTC config error", () => {
withTestScheduler(({ scope, hot, expectObservable }) => {
const localTransport$ = scope.behavior<null | LivekitTransport>(
hot("1ms #", {}, new MatrixRTCTransportMissingError("domain.com")),
null,
);
const localTransport$ =
scope.behavior<null | LocalTransportWithSFUConfig>(
hot("1ms #", {}, new MatrixRTCTransportMissingError("domain.com")),
null,
);
// we do not need any connection data since we want to fail before reaching that.
const mockConnectionManager = {
@@ -235,11 +245,23 @@ describe("LocalMembership", () => {
});
const aTransport = {
livekit_service_url: "a",
} as LivekitTransport;
transport: {
livekit_service_url: "a",
} as LivekitTransport,
sfuConfig: {
url: "sfu-url",
jwt: "sfu-token",
},
} as LocalTransportWithSFUConfig;
const bTransport = {
livekit_service_url: "b",
} as LivekitTransport;
transport: {
livekit_service_url: "b",
} as LivekitTransport,
sfuConfig: {
url: "sfu-url",
jwt: "sfu-token",
},
} as LocalTransportWithSFUConfig;
const connectionTransportAConnected = {
livekitRoom: mockLivekitRoom({
@@ -249,7 +271,7 @@ describe("LocalMembership", () => {
} as unknown as LocalParticipant,
}),
state$: constant(ConnectionState.LivekitConnected),
transport: aTransport,
transport: aTransport.transport,
} as unknown as Connection;
const connectionTransportAConnecting = {
...connectionTransportAConnected,
@@ -258,11 +280,11 @@ describe("LocalMembership", () => {
} as unknown as Connection;
const connectionTransportBConnected = {
state$: constant(ConnectionState.LivekitConnected),
transport: bTransport,
transport: bTransport.transport,
livekitRoom: mockLivekitRoom({}),
} as unknown as Connection;
it("recreates publisher if new connection is used and ENDS always unpublish and end tracks", async () => {
it("recreates publisher if new connection is used, always unpublish and end tracks", async () => {
const scope = new ObservableScope();
const localTransport$ = new BehaviorSubject(aTransport);
@@ -310,8 +332,12 @@ describe("LocalMembership", () => {
expect(publishers[1].stopTracks).not.toHaveBeenCalled();
expect(publishers[0].stopPublishing).toHaveBeenCalled();
expect(publishers[1].stopPublishing).not.toHaveBeenCalled();
expect(publisherFactory.mock.calls[0][0].transport).toBe(aTransport);
expect(publisherFactory.mock.calls[1][0].transport).toBe(bTransport);
expect(publisherFactory.mock.calls[0][0].transport).toBe(
aTransport.transport,
);
expect(publisherFactory.mock.calls[1][0].transport).toBe(
bTransport.transport,
);
scope.end();
await flushPromises();
// stop all tracks after ending scopes
@@ -383,7 +409,8 @@ describe("LocalMembership", () => {
const scope = new ObservableScope();
const connectionManagerData = new ConnectionManagerData();
const localTransport$ = new BehaviorSubject<null | LivekitTransport>(null);
const localTransport$ =
new BehaviorSubject<null | LocalTransportWithSFUConfig>(null);
const connectionManagerData$ = new BehaviorSubject(
new Epoch(connectionManagerData),
);
@@ -460,7 +487,7 @@ describe("LocalMembership", () => {
});
(
connectionManagerData2.getConnectionForTransport(aTransport)!
connectionManagerData2.getConnectionForTransport(aTransport.transport)!
.state$ as BehaviorSubject<ConnectionState>
).next(ConnectionState.LivekitConnected);
expect(localMembership.localMemberState$.value).toStrictEqual({

View File

@@ -36,6 +36,7 @@ import {
} from "rxjs";
import { type Logger } from "matrix-js-sdk/lib/logger";
import { deepCompare } from "matrix-js-sdk/lib/utils";
import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
import { type Behavior } from "../../Behavior.ts";
import { type IConnectionManager } from "../remoteMembers/ConnectionManager.ts";
@@ -60,6 +61,7 @@ import {
} from "../remoteMembers/Connection.ts";
import { type HomeserverConnected } from "./HomeserverConnected.ts";
import { and$ } from "../../../utils/observable.ts";
import { type LocalTransportWithSFUConfig } from "./LocalTransport.ts";
export enum TransportState {
/** Not even a transport is available to the LocalMembership */
@@ -125,7 +127,7 @@ interface Props {
createPublisherFactory: (connection: Connection) => Publisher;
joinMatrixRTC: (transport: LivekitTransport) => void;
homeserverConnected: HomeserverConnected;
localTransport$: Behavior<LivekitTransport | null>;
localTransport$: Behavior<LocalTransportWithSFUConfig | null>;
matrixRTCSession: Pick<
MatrixRTCSession,
"updateCallIntent" | "leaveRoomSession"
@@ -233,7 +235,9 @@ export const createLocalMembership$ = ({
return null;
}
return connectionData.getConnectionForTransport(localTransport);
return connectionData.getConnectionForTransport(
localTransport.transport,
);
}),
tap((connection) => {
logger.info(
@@ -532,7 +536,7 @@ export const createLocalMembership$ = ({
if (!shouldConnect) return;
try {
joinMatrixRTC(transport);
joinMatrixRTC(transport.transport);
} catch (error) {
logger.error("Error entering RTC session", error);
if (error instanceof Error)
@@ -551,7 +555,12 @@ export const createLocalMembership$ = ({
);
const participant$ = scope.behavior(
localConnection$.pipe(map((c) => c?.livekitRoom?.localParticipant ?? null)),
localConnection$.pipe(
map((c) => c?.livekitRoom?.localParticipant ?? null),
tap((p) => {
logger.debug("participant$ updated:", p?.identity);
}),
),
);
// Pause upstream of all local media tracks when we're disconnected from
@@ -686,18 +695,19 @@ interface EnterRTCSessionOptions {
* - Handles retries (fails only after several attempts)
*
* @param rtcSession - The MatrixRTCSession to join.
* @param ownMembershipIdentity - Options for entering the RTC session.
* @param transport - The LivekitTransport to use for this session.
* @param options - Options for entering the RTC session.
* @param options.encryptMedia - Whether to encrypt media.
* @param options.matrixRTCMode - The Matrix RTC mode to use.
* @param options - `encryptMedia`: Whether to encrypt media `matrixRTCMode`: The Matrix RTC mode to use.
* @throws If the widget could not send ElementWidgetActions.JoinCall action.
*/
// Exported for unit testing
export function enterRTCSession(
rtcSession: MatrixRTCSession,
ownMembershipIdentity: CallMembershipIdentityParts,
transport: LivekitTransport,
{ encryptMedia, matrixRTCMode }: EnterRTCSessionOptions,
options: EnterRTCSessionOptions,
): void {
const { encryptMedia, matrixRTCMode } = options;
PosthogAnalytics.instance.eventCallEnded.cacheStartCall(new Date());
PosthogAnalytics.instance.eventCallStarted.track(rtcSession.room.roomId);
@@ -709,10 +719,13 @@ export function enterRTCSession(
const useDeviceSessionMemberEvents =
features?.feature_use_device_session_member_events;
const { sendNotificationType: notificationType, callIntent } = getUrlParams();
const multiSFU = matrixRTCMode !== MatrixRTCMode.Legacy;
const multiSFU =
matrixRTCMode === MatrixRTCMode.Compatibility ||
matrixRTCMode === MatrixRTCMode.Matrix_2_0;
// Multi-sfu does not need a preferred foci list. just the focus that is actually used.
// TODO where/how do we track errors originating from the ongoing rtcSession?
rtcSession.joinRoomSession(
rtcSession.joinRTCSession(
ownMembershipIdentity,
multiSFU ? [] : [transport],
multiSFU ? transport : undefined,
{

View File

@@ -18,8 +18,8 @@ import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc";
import { BehaviorSubject, lastValueFrom } from "rxjs";
import fetchMock from "fetch-mock";
import { mockConfig, flushPromises } from "../../../utils/test";
import { createLocalTransport$ } from "./LocalTransport";
import { mockConfig, flushPromises, ownMemberMock } from "../../../utils/test";
import { createLocalTransport$, JwtEndpointVersion } from "./LocalTransport";
import { constant } from "../../Behavior";
import { Epoch, ObservableScope } from "../../ObservableScope";
import {
@@ -39,11 +39,35 @@ describe("LocalTransport", () => {
};
let scope: ObservableScope;
beforeEach(() => {
scope = new ObservableScope();
});
beforeEach(() => (scope = new ObservableScope()));
afterEach(() => scope.end());
it("throws if config is missing", async () => {
const localTransport$ = createLocalTransport$({
scope,
roomId: "!room:example.org",
useOldestMember$: constant(false),
memberships$: constant(new Epoch<CallMembership[]>([])),
client: {
// eslint-disable-next-line @typescript-eslint/naming-convention
_unstable_getRTCTransports: async () => Promise.resolve([]),
getDomain: () => "",
baseUrl: "example.org",
// These won't be called in this error path but satisfy the type
getOpenIdToken: vi.fn(),
getDeviceId: vi.fn(),
},
ownMembershipIdentity: ownMemberMock,
forceJwtEndpoint$: constant(JwtEndpointVersion.Legacy),
delayId$: constant("delay_id_mock"),
});
await flushPromises();
expect(() => localTransport$.value).toThrow(
new MatrixRTCTransportMissingError(""),
);
});
it("throws FailToGetOpenIdToken when OpenID fetch fails", async () => {
// Provide a valid config so makeTransportInternal resolves a transport
const scope = new ObservableScope();
@@ -65,6 +89,7 @@ describe("LocalTransport", () => {
useOldestMember$: constant(false),
memberships$: constant(new Epoch<CallMembership[]>([])),
client: {
baseUrl: "https://lk.example.org",
// Use empty domain to skip .well-known and use config directly
getDomain: () => "",
// eslint-disable-next-line @typescript-eslint/naming-convention
@@ -72,6 +97,9 @@ describe("LocalTransport", () => {
getOpenIdToken: vi.fn(),
getDeviceId: vi.fn(),
},
ownMembershipIdentity: ownMemberMock,
forceJwtEndpoint$: constant(JwtEndpointVersion.Legacy),
delayId$: constant("delay_id_mock"),
});
localTransport$.subscribe(
(o) => observations.push(o),
@@ -86,6 +114,60 @@ describe("LocalTransport", () => {
expect(() => localTransport$.value).toThrow(expectedError);
});
it("emits preferred transport after OpenID resolves", async () => {
// Use config so transport discovery succeeds, but delay OpenID JWT fetch
mockConfig({
livekit: { livekit_service_url: "https://lk.example.org" },
});
const openIdResolver = Promise.withResolvers<openIDSFU.SFUConfig>();
vi.spyOn(openIDSFU, "getSFUConfigWithOpenID").mockReturnValue(
openIdResolver.promise,
);
const localTransport$ = createLocalTransport$({
scope,
roomId: "!room:example.org",
useOldestMember$: constant(false),
memberships$: constant(new Epoch<CallMembership[]>([])),
client: {
// eslint-disable-next-line @typescript-eslint/naming-convention
_unstable_getRTCTransports: async () => Promise.resolve([]),
getDomain: () => "",
getOpenIdToken: vi.fn(),
getDeviceId: vi.fn(),
baseUrl: "https://lk.example.org",
},
ownMembershipIdentity: ownMemberMock,
forceJwtEndpoint$: constant(JwtEndpointVersion.Legacy),
delayId$: constant("delay_id_mock"),
});
openIdResolver.resolve?.({
url: "https://lk.example.org",
jwt: "jwt",
livekitAlias: "!room:example.org",
livekitIdentity: ownMemberMock.userId + ":" + ownMemberMock.deviceId,
});
expect(localTransport$.value).toBe(null);
await flushPromises();
// final
expect(localTransport$.value).toStrictEqual({
transport: {
livekit_alias: "!room:example.org",
livekit_service_url: "https://lk.example.org",
type: "livekit",
},
sfuConfig: {
jwt: "jwt",
livekitAlias: "!room:example.org",
livekitIdentity: "@alice:example.org:DEVICE",
url: "https://lk.example.org",
},
});
});
it("updates local transport when oldest member changes", async () => {
// Use config so transport discovery succeeds, but delay OpenID JWT fetch
mockConfig({
@@ -109,7 +191,11 @@ describe("LocalTransport", () => {
_unstable_getRTCTransports: async () => Promise.resolve([]),
getOpenIdToken: vi.fn(),
getDeviceId: vi.fn(),
baseUrl: "https://lk.example.org",
},
ownMembershipIdentity: ownMemberMock,
forceJwtEndpoint$: constant(JwtEndpointVersion.Legacy),
delayId$: constant("delay_id_mock"),
});
openIdResolver.resolve?.(openIdResponse);
@@ -117,9 +203,17 @@ describe("LocalTransport", () => {
await flushPromises();
// final
expect(localTransport$.value).toStrictEqual({
livekit_alias: "!example_room_id",
livekit_service_url: "https://lk.example.org",
type: "livekit",
transport: {
livekit_alias: "!example_room_id",
livekit_service_url: "https://lk.example.org",
type: "livekit",
},
sfuConfig: {
jwt: "e30=.eyJzdWIiOiJAbWU6ZXhhbXBsZS5vcmc6QUJDREVGIiwidmlkZW8iOnsicm9vbSI6IiFleGFtcGxlX3Jvb21faWQifX0=.e30=",
livekitAlias: "!example_room_id",
livekitIdentity: "@lk_user:ABCDEF",
url: "https://lk.example.org",
},
});
});
@@ -134,11 +228,15 @@ describe("LocalTransport", () => {
mockConfig({});
customLivekitUrl.setValue(customLivekitUrl.defaultValue);
localTransportOpts = {
ownMembershipIdentity: ownMemberMock,
scope,
roomId: "!example_room_id",
useOldestMember$: constant(false),
forceJwtEndpoint$: constant(JwtEndpointVersion.Legacy),
delayId$: constant(null),
memberships$: constant(new Epoch<CallMembership[]>([])),
client: {
baseUrl: "https://example.org",
getDomain: vi.fn().mockReturnValue(""),
// eslint-disable-next-line @typescript-eslint/naming-convention
_unstable_getRTCTransports: vi.fn().mockResolvedValue([]),
@@ -165,9 +263,17 @@ describe("LocalTransport", () => {
expect(localTransport$.value).toBe(null);
await flushPromises();
expect(localTransport$.value).toStrictEqual({
livekit_alias: "!example_room_id",
livekit_service_url: "https://lk.example.org",
type: "livekit",
transport: {
livekit_alias: "!example_room_id",
livekit_service_url: "https://lk.example.org",
type: "livekit",
},
sfuConfig: {
jwt: "e30=.eyJzdWIiOiJAbWU6ZXhhbXBsZS5vcmc6QUJDREVGIiwidmlkZW8iOnsicm9vbSI6IiFleGFtcGxlX3Jvb21faWQifX0=.e30=",
livekitAlias: "!example_room_id",
livekitIdentity: "@lk_user:ABCDEF",
url: "https://lk.example.org",
},
});
});
it("supports getting transport via user settings", async () => {
@@ -177,9 +283,17 @@ describe("LocalTransport", () => {
expect(localTransport$.value).toBe(null);
await flushPromises();
expect(localTransport$.value).toStrictEqual({
livekit_alias: "!example_room_id",
livekit_service_url: "https://lk.example.org",
type: "livekit",
transport: {
livekit_alias: "!example_room_id",
livekit_service_url: "https://lk.example.org",
type: "livekit",
},
sfuConfig: {
jwt: "e30=.eyJzdWIiOiJAbWU6ZXhhbXBsZS5vcmc6QUJDREVGIiwidmlkZW8iOnsicm9vbSI6IiFleGFtcGxlX3Jvb21faWQifX0=.e30=",
livekitAlias: "!example_room_id",
livekitIdentity: "@lk_user:ABCDEF",
url: "https://lk.example.org",
},
});
});
it("supports getting transport via backend", async () => {
@@ -191,9 +305,17 @@ describe("LocalTransport", () => {
expect(localTransport$.value).toBe(null);
await flushPromises();
expect(localTransport$.value).toStrictEqual({
livekit_alias: "!example_room_id",
livekit_service_url: "https://lk.example.org",
type: "livekit",
transport: {
livekit_alias: "!example_room_id",
livekit_service_url: "https://lk.example.org",
type: "livekit",
},
sfuConfig: {
jwt: "e30=.eyJzdWIiOiJAbWU6ZXhhbXBsZS5vcmc6QUJDREVGIiwidmlkZW8iOnsicm9vbSI6IiFleGFtcGxlX3Jvb21faWQifX0=.e30=",
livekitAlias: "!example_room_id",
livekitIdentity: "@lk_user:ABCDEF",
url: "https://lk.example.org",
},
});
});
it("fails fast if the openID request fails for backend config", async () => {
@@ -222,9 +344,17 @@ describe("LocalTransport", () => {
expect(localTransport$.value).toBe(null);
await flushPromises();
expect(localTransport$.value).toStrictEqual({
livekit_alias: "!example_room_id",
livekit_service_url: "https://lk.example.org",
type: "livekit",
transport: {
livekit_alias: "!example_room_id",
livekit_service_url: "https://lk.example.org",
type: "livekit",
},
sfuConfig: {
jwt: "e30=.eyJzdWIiOiJAbWU6ZXhhbXBsZS5vcmc6QUJDREVGIiwidmlkZW8iOnsicm9vbSI6IiFleGFtcGxlX3Jvb21faWQifX0=.e30=",
livekitAlias: "!example_room_id",
livekitIdentity: "@lk_user:ABCDEF",
url: "https://lk.example.org",
},
});
expect(fetchMock.done()).toEqual(true);
});
@@ -248,11 +378,15 @@ describe("LocalTransport", () => {
it("throws if no options are available", async () => {
const localTransport$ = createLocalTransport$({
scope,
ownMembershipIdentity: ownMemberMock,
roomId: "!example_room_id",
useOldestMember$: constant(false),
forceJwtEndpoint$: constant(JwtEndpointVersion.Legacy),
delayId$: constant(null),
memberships$: constant(new Epoch<CallMembership[]>([])),
client: {
getDomain: () => "",
baseUrl: "https://example.org",
// eslint-disable-next-line @typescript-eslint/naming-convention
_unstable_getRTCTransports: async () => Promise.resolve([]),
// These won't be called in this error path but satisfy the type

View File

@@ -23,6 +23,7 @@ import {
} from "rxjs";
import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
import { AutoDiscovery } from "matrix-js-sdk/lib/autodiscovery";
import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
import { type Behavior } from "../../Behavior.ts";
import { type Epoch, type ObservableScope } from "../../ObservableScope.ts";
@@ -30,9 +31,11 @@ import { Config } from "../../../config/Config.ts";
import {
FailToGetOpenIdToken,
MatrixRTCTransportMissingError,
NoMatrix2AuthorizationService,
} from "../../../utils/errors.ts";
import {
getSFUConfigWithOpenID,
type SFUConfig,
type OpenIDClientParts,
} from "../../../livekit/openIDSFU.ts";
import { areLivekitTransportsEqual } from "../remoteMembers/MatrixLivekitMembers.ts";
@@ -47,11 +50,32 @@ const logger = rootLogger.getChild("[LocalTransport]");
*/
interface Props {
scope: ObservableScope;
ownMembershipIdentity: CallMembershipIdentityParts;
memberships$: Behavior<Epoch<CallMembership[]>>;
client: Pick<MatrixClient, "getDomain" | "_unstable_getRTCTransports"> &
client: Pick<
MatrixClient,
"getDomain" | "baseUrl" | "_unstable_getRTCTransports"
> &
OpenIDClientParts;
roomId: string;
useOldestMember$: Behavior<boolean>;
forceJwtEndpoint$: Behavior<JwtEndpointVersion>;
delayId$: Behavior<string | null>;
}
export enum JwtEndpointVersion {
Legacy = "legacy",
Matrix_2_0 = "matrix_2_0",
}
export interface LocalTransportWithSFUConfig {
transport: LivekitTransport;
sfuConfig: SFUConfig;
}
export function isLocalTransportWithSFUConfig(
obj: LivekitTransport | LocalTransportWithSFUConfig,
): obj is LocalTransportWithSFUConfig {
return "transport" in obj && "sfuConfig" in obj;
}
/**
@@ -61,26 +85,53 @@ interface Props {
* @prop useOldestMember Whether to use the same transport as the oldest member.
* This will only update once the first oldest member appears. Will not recompute if the oldest member leaves.
*
* @prop useOldJwtEndpoint$ Whether to set forceOldJwtEndpoint on the returned transport and to use the old JWT endpoint.
* This is used when the connection manager needs to know if it has to use the legacy endpoint which implies a string concatenated rtcBackendIdentity.
* (which is expected for non sticky event based rtc member events)
* @returns The local transport. It will be created using the correct sfu endpoint based on the useOldJwtEndpoint$ value.
* @throws MatrixRTCTransportMissingError | FailToGetOpenIdToken
*/
export const createLocalTransport$ = ({
scope,
memberships$,
ownMembershipIdentity,
client,
roomId,
useOldestMember$,
}: Props): Behavior<LivekitTransport | null> => {
forceJwtEndpoint$,
delayId$,
}: Props): Behavior<LocalTransportWithSFUConfig | null> => {
/**
* The transport over which we should be actively publishing our media.
* undefined when not joined.
*/
const oldestMemberTransport$ = scope.behavior(
memberships$.pipe(
map(
(memberships) =>
memberships.value[0]?.getTransport(memberships.value[0]) ?? null,
),
combineLatest([memberships$]).pipe(
map(([memberships]) => {
const oldestMember = memberships.value[0];
const transport = oldestMember?.getTransport(memberships.value[0]);
if (!transport) return null;
return transport;
}),
first((t) => t != null && isLivekitTransport(t)),
switchMap((transport) => {
// Get the open jwt token to connect to the sfu
const computeLocalTransportWithSFUConfig =
async (): Promise<LocalTransportWithSFUConfig> => {
return {
transport,
sfuConfig: await getSFUConfigWithOpenID(
client,
ownMembershipIdentity,
transport.livekit_service_url,
roomId,
{ forceJwtEndpoint: JwtEndpointVersion.Legacy },
logger,
),
};
};
return from(computeLocalTransportWithSFUConfig());
}),
),
null,
);
@@ -91,9 +142,30 @@ export const createLocalTransport$ = ({
*
* @throws MatrixRTCTransportMissingError | FailToGetOpenIdToken
*/
const preferredTransport$: Behavior<LivekitTransport | null> = scope.behavior(
customLivekitUrl.value$.pipe(
switchMap((customUrl) => from(makeTransport(client, roomId, customUrl))),
const preferredTransport$ = scope.behavior(
// preferredTransport$ (used for multi sfu) needs to know if we are using the old or new
// jwt endpoint (`get_token` vs `sfu/get`) based on that the jwt endpoint will compute the rtcBackendIdentity
// differently. (sha(`${userId}|${deviceId}|${memberId}`) vs `${userId}|${deviceId}|${memberId}`)
// When using sticky events (we need to use the new endpoint).
combineLatest([customLivekitUrl.value$, delayId$, forceJwtEndpoint$]).pipe(
switchMap(([customUrl, delayId, forceEndpoint]) => {
logger.info(
"Creating preferred transport based on: ",
customUrl,
delayId,
forceEndpoint,
);
return from(
makeTransport(
client,
ownMembershipIdentity,
roomId,
customUrl,
forceEndpoint,
delayId ?? undefined,
),
);
}),
),
null,
);
@@ -112,7 +184,9 @@ export const createLocalTransport$ = ({
? (oldestMemberTransport ?? preferredTransport)
: preferredTransport,
),
distinctUntilChanged(areLivekitTransportsEqual),
distinctUntilChanged((t1, t2) =>
areLivekitTransportsEqual(t1?.transport ?? null, t2?.transport ?? null),
),
),
);
};
@@ -124,25 +198,63 @@ const FOCI_WK_KEY = "org.matrix.msc4143.rtc_foci";
* validating auth against the service to ensure it's correct.
* Prefers in order:
*
* 1. The `urlFromDevSettings` value. If this cannot be validated, the function will throw.
* 2. The transports returned via the homeserver.
* 3. The transports returned via .well-known.
* 4. The transport configured in Element Call's config.
*
* @param client The authenticated Matrix client for the current user
* @param membership The membership identity of the user.
* @param roomId The ID of the room to be connected to.
* @param urlFromDevSettings Override URL provided by the user's local config.
* @param forceJwtEndpoint Whether to force a specific JWT endpoint
* - `Legacy` / `Matrix_2_0`
* - `get_token` / `sfu/get`
* - not hashing / hashing the backendIdentity
* @param delayId the delay id passed to the jwt service.
*
* @returns A fully validated transport config.
* @throws MatrixRTCTransportMissingError | FailToGetOpenIdToken
*/
async function makeTransport(
client: Pick<MatrixClient, "getDomain" | "_unstable_getRTCTransports"> &
client: Pick<
MatrixClient,
"getDomain" | "baseUrl" | "_unstable_getRTCTransports"
> &
OpenIDClientParts,
membership: CallMembershipIdentityParts,
roomId: string,
urlFromDevSettings: string | null,
): Promise<LivekitTransport> {
forceJwtEndpoint: JwtEndpointVersion,
delayId?: string,
): Promise<LocalTransportWithSFUConfig> {
logger.trace("Searching for a preferred transport");
async function doOpenIdAndJWTFromUrl(
url: string,
): Promise<LocalTransportWithSFUConfig> {
const sfuConfig = await getSFUConfigWithOpenID(
client,
membership,
url,
roomId,
{
forceJwtEndpoint: forceJwtEndpoint,
delayEndpointBaseUrl: client.baseUrl,
delayId,
},
logger,
);
return {
transport: {
type: "livekit",
livekit_service_url: url,
livekit_alias: sfuConfig.livekitAlias,
},
sfuConfig,
};
}
// We will call `getSFUConfigWithOpenID` once per transport here as it's our
// only mechanism of valiation. This means we will also ask the
// homeserver for a OpenID token a few times. Since OpenID tokens are single
@@ -153,39 +265,29 @@ async function makeTransport(
// DEVTOOL: Highest priority: Load from devtool setting
if (urlFromDevSettings !== null) {
logger.info("Using LiveKit transport from dev tools: ", urlFromDevSettings);
// Validate that the SFU is up. Otherwise, we want to fail on this
// as we don't permit other SFUs.
const config = await getSFUConfigWithOpenID(
client,
urlFromDevSettings,
roomId,
);
return {
type: "livekit",
livekit_service_url: urlFromDevSettings,
livekit_alias: config.livekitAlias,
};
// This will call the jwt/sfu/get endpoint to pre create the livekit room.
logger.info("Using LiveKit transport from dev tools: ", urlFromDevSettings);
return await doOpenIdAndJWTFromUrl(urlFromDevSettings);
}
async function getFirstUsableTransport(
transports: Transport[],
): Promise<LivekitTransport | null> {
): Promise<LocalTransportWithSFUConfig | null> {
for (const potentialTransport of transports) {
if (isLivekitTransportConfig(potentialTransport)) {
try {
const { livekitAlias } = await getSFUConfigWithOpenID(
client,
// This will call the jwt/sfu/get endpoint to pre create the livekit room.
return await doOpenIdAndJWTFromUrl(
potentialTransport.livekit_service_url,
roomId,
);
return {
...potentialTransport,
livekit_alias: livekitAlias,
};
} catch (ex) {
// Explictly throw these
if (ex instanceof FailToGetOpenIdToken) {
// Explictly throw these
throw ex;
}
if (ex instanceof NoMatrix2AuthorizationService) {
throw ex;
}
logger.debug(
@@ -245,18 +347,9 @@ async function makeTransport(
const urlFromConf = Config.get().livekit?.livekit_service_url;
if (urlFromConf) {
try {
const { livekitAlias } = await getSFUConfigWithOpenID(
client,
urlFromConf,
roomId,
);
const selectedTransport: LivekitTransport = {
type: "livekit",
livekit_service_url: urlFromConf,
livekit_alias: livekitAlias,
};
logger.info("Using config SFU", selectedTransport);
return selectedTransport;
// This will call the jwt/sfu/get endpoint to pre create the livekit room.
logger.info("Using config SFU", urlFromConf);
return await doOpenIdAndJWTFromUrl(urlFromConf);
} catch (ex) {
if (ex instanceof FailToGetOpenIdToken) {
throw ex;
@@ -265,5 +358,6 @@ async function makeTransport(
}
}
// If we do not have returned a transport by now we throw an error
throw new MatrixRTCTransportMissingError(domain ?? "");
}

View File

@@ -26,8 +26,8 @@ import fetchMock from "fetch-mock";
import EventEmitter from "events";
import { type IOpenIDToken } from "matrix-js-sdk";
import { logger } from "matrix-js-sdk/lib/logger";
import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc/LivekitTransport";
import type { LivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
import {
Connection,
ConnectionState,
@@ -40,7 +40,7 @@ import {
FailToGetOpenIdToken,
} from "../../../utils/errors.ts";
import { testJWTToken } from "../../../utils/test-fixtures.ts";
import { mockRemoteParticipant } from "../../../utils/test.ts";
import { mockRemoteParticipant, ownMemberMock } from "../../../utils/test.ts";
let testScope: ObservableScope;
@@ -114,6 +114,7 @@ function setupRemoteConnection(): Connection {
client: client,
transport: livekitFocus,
scope: testScope,
ownMembershipIdentity: ownMemberMock,
livekitRoomFactory: () => fakeLivekitRoom,
};
@@ -155,6 +156,7 @@ describe("Start connection states", () => {
client: client,
transport: livekitFocus,
scope: testScope,
ownMembershipIdentity: ownMemberMock,
livekitRoomFactory: () => fakeLivekitRoom,
};
const connection = new Connection(opts, logger);
@@ -170,6 +172,7 @@ describe("Start connection states", () => {
client: client,
transport: livekitFocus,
scope: testScope,
ownMembershipIdentity: ownMemberMock,
livekitRoomFactory: () => fakeLivekitRoom,
};
@@ -220,6 +223,7 @@ describe("Start connection states", () => {
client: client,
transport: livekitFocus,
scope: testScope,
ownMembershipIdentity: ownMemberMock,
livekitRoomFactory: () => fakeLivekitRoom,
};
@@ -259,7 +263,7 @@ describe("Start connection states", () => {
capturedState.cause instanceof Error
) {
expect(capturedState.cause.message).toContain(
"SFU Config fetch failed with exception",
"SFU Config fetch failed with status code 500",
);
expect(connection.transport.livekit_alias).toEqual(
livekitFocus.livekit_alias,
@@ -277,6 +281,7 @@ describe("Start connection states", () => {
client: client,
transport: livekitFocus,
scope: testScope,
ownMembershipIdentity: ownMemberMock,
livekitRoomFactory: () => fakeLivekitRoom,
};

View File

@@ -18,6 +18,7 @@ import {
import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
import { BehaviorSubject, map } from "rxjs";
import { type Logger } from "matrix-js-sdk/lib/logger";
import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
import {
getSFUConfigWithOpenID,
@@ -32,8 +33,21 @@ import {
SFURoomCreationRestrictedError,
UnknownCallError,
} from "../../../utils/errors.ts";
import { type JwtEndpointVersion } from "../localMember/LocalTransport.ts";
export interface ConnectionOpts {
/**
* For the local transport we already do know the jwt token and url. We can reuse it.
* On top the local transport will send additional data to the jwt server to use delayed event delegation.
*/
existingSFUConfig?: SFUConfig;
/**
* For local connections that use the oldest member pattern. here we have not prefetched the sfuConfig
* and hence we need to let the connection do the jwt token fetching.
*/
forceJwtEndpoint?: JwtEndpointVersion;
/** The identity parts to use on this connection */
ownMembershipIdentity: CallMembershipIdentityParts;
/** The media transport to connect to. */
transport: LivekitTransport;
/** The Matrix client to use for OpenID and SFU config requests. */
@@ -129,8 +143,10 @@ export class Connection {
try {
this._state$.next(ConnectionState.FetchingConfig);
// We should already have this information after creating the localTransport.
// It would probably be better to forward this here.
const { url, jwt } = await this.getSFUConfigWithOpenID();
// only call getSFUConfigWithOpenID for connections where we do not have a token yet. (existingJwtTokenData === undefined)
const { url, jwt } =
this.existingSFUConfig ??
(await this.getSFUConfigForRemoteConnection());
// If we were stopped while fetching the config, don't proceed to connect
if (this.stopped) return;
@@ -186,11 +202,17 @@ export class Connection {
}
}
protected async getSFUConfigWithOpenID(): Promise<SFUConfig> {
protected async getSFUConfigForRemoteConnection(): Promise<SFUConfig> {
// This will only be called for sfu's where we do not publish ourselves.
// For the local connection we will use the existingJwtTokenData
return await getSFUConfigWithOpenID(
this.client,
this.ownMembershipIdentity,
this.transport.livekit_service_url,
this.transport.livekit_alias,
// dont pass any custom opts for the subscribe only connections
{},
this.logger,
);
}
@@ -212,7 +234,8 @@ export class Connection {
private readonly client: OpenIDClientParts;
private readonly logger: Logger;
private readonly ownMembershipIdentity: CallMembershipIdentityParts;
private readonly existingSFUConfig?: SFUConfig;
/**
* Creates a new connection to a matrix RTC LiveKit backend.
*
@@ -221,6 +244,8 @@ export class Connection {
* @param logger - The logger to use.
*/
public constructor(opts: ConnectionOpts, logger: Logger) {
this.ownMembershipIdentity = opts.ownMembershipIdentity;
this.existingSFUConfig = opts.existingSFUConfig;
this.logger = logger.getChild("[Connection]");
this.logger.info(
`Creating new connection to ${opts.transport.livekit_service_url} ${opts.transport.livekit_alias}`,

View File

@@ -5,7 +5,6 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
import {
Room as LivekitRoom,
type RoomOptions,
@@ -16,10 +15,15 @@ import {
import { type Logger } from "matrix-js-sdk/lib/logger";
// imported as inline to support worker when loaded from a cdn (cross domain)
import E2EEWorker from "livekit-client/e2ee-worker?worker&inline";
import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc/LivekitTransport";
import { type ObservableScope } from "../../ObservableScope.ts";
import { Connection } from "./Connection.ts";
import type { OpenIDClientParts } from "../../../livekit/openIDSFU.ts";
import type {
OpenIDClientParts,
SFUConfig,
} from "../../../livekit/openIDSFU.ts";
import type { MediaDevices } from "../../MediaDevices.ts";
import type { Behavior } from "../../Behavior.ts";
import type { ProcessorState } from "../../../livekit/TrackProcessorContext.tsx";
@@ -28,9 +32,11 @@ import { defaultLiveKitOptions } from "../../../livekit/options.ts";
// TODO evaluate if this should be done like the Publisher Factory
export interface ConnectionFactory {
createConnection(
transport: LivekitTransport,
scope: ObservableScope,
transport: LivekitTransport,
ownMembershipIdentity: CallMembershipIdentityParts,
logger: Logger,
sfuConfig?: SFUConfig,
): Connection;
}
@@ -78,17 +84,30 @@ export class ECConnectionFactory implements ConnectionFactory {
this.livekitRoomFactory = livekitRoomFactory ?? defaultFactory;
}
/**
*
* @param scope The observable scope (used for clean-up)
* @param transport The transport to use for this connection.
* @param ownMembershipIdentity required to connect (using the jwt service) with the SFU.
* @param logger The logger instance to use for this connection.
* @param sfuConfig optional config in case we already have a token for this connection.
* @returns
*/
public createConnection(
transport: LivekitTransport,
scope: ObservableScope,
transport: LivekitTransport,
ownMembershipIdentity: CallMembershipIdentityParts,
logger: Logger,
sfuConfig?: SFUConfig,
): Connection {
return new Connection(
{
existingSFUConfig: sfuConfig,
transport,
client: this.client,
scope: scope,
livekitRoomFactory: this.livekitRoomFactory,
ownMembershipIdentity,
},
logger,
);

View File

@@ -18,9 +18,9 @@ import {
} from "./ConnectionManager.ts";
import { type ConnectionFactory } from "./ConnectionFactory.ts";
import { type Connection } from "./Connection.ts";
import { withTestScheduler } from "../../../utils/test.ts";
import { ownMemberMock, withTestScheduler } from "../../../utils/test.ts";
import { areLivekitTransportsEqual } from "./MatrixLivekitMembers.ts";
import { type Behavior } from "../../Behavior.ts";
import { constant, type Behavior } from "../../Behavior.ts";
// Some test constants
@@ -49,7 +49,7 @@ beforeEach(() => {
vi.mocked(fakeConnectionFactory).createConnection = vi
.fn()
.mockImplementation(
(transport: LivekitTransport, scope: ObservableScope) => {
(scope: ObservableScope, transport: LivekitTransport) => {
const mockConnection = {
transport,
remoteParticipants$: new BehaviorSubject([]),
@@ -76,10 +76,12 @@ describe("connections$ stream", () => {
const { connectionManagerData$ } = createConnectionManager$({
scope: testScope,
connectionFactory: fakeConnectionFactory,
inputTransports$: behavior("a", {
localTransport$: constant(null),
remoteTransports$: behavior("a", {
a: new Epoch([TRANSPORT_1, TRANSPORT_2], 0),
}),
logger: logger,
ownMembershipIdentity: ownMemberMock,
});
expectObservable(
@@ -115,7 +117,8 @@ describe("connections$ stream", () => {
const { connectionManagerData$ } = createConnectionManager$({
scope: testScope,
connectionFactory: fakeConnectionFactory,
inputTransports$: behavior("abcdef", {
localTransport$: constant(null),
remoteTransports$: behavior("abcdef", {
a: new Epoch([TRANSPORT_1], 0),
b: new Epoch([TRANSPORT_1], 1),
c: new Epoch([TRANSPORT_1], 2),
@@ -124,6 +127,7 @@ describe("connections$ stream", () => {
f: new Epoch([TRANSPORT_1, TRANSPORT_2], 5),
}),
logger: logger,
ownMembershipIdentity: ownMemberMock,
});
expectObservable(
@@ -160,12 +164,14 @@ describe("connections$ stream", () => {
const { connectionManagerData$ } = createConnectionManager$({
scope: testScope,
connectionFactory: fakeConnectionFactory,
inputTransports$: behavior("abc", {
localTransport$: constant(null),
remoteTransports$: behavior("abc", {
a: new Epoch([TRANSPORT_1], 0),
b: new Epoch([TRANSPORT_1, TRANSPORT_2], 1),
c: new Epoch([TRANSPORT_1], 2),
}),
logger: logger,
ownMembershipIdentity: ownMemberMock,
});
expectObservable(
@@ -223,7 +229,7 @@ describe("connectionManagerData$ stream", () => {
vi.mocked(fakeConnectionFactory).createConnection = vi
.fn()
.mockImplementation(
(transport: LivekitTransport, scope: ObservableScope) => {
(scope: ObservableScope, transport: LivekitTransport) => {
const fakeRemoteParticipants$ = new BehaviorSubject<
RemoteParticipant[]
>([]);
@@ -275,10 +281,12 @@ describe("connectionManagerData$ stream", () => {
const { connectionManagerData$ } = createConnectionManager$({
scope: testScope,
connectionFactory: fakeConnectionFactory,
inputTransports$: behavior("a", {
localTransport$: constant(null),
remoteTransports$: behavior("a", {
a: new Epoch([TRANSPORT_1, TRANSPORT_2], 0),
}),
logger,
ownMembershipIdentity: ownMemberMock,
});
expectObservable(connectionManagerData$).toBe("abcd", {

View File

@@ -7,9 +7,10 @@ Please see LICENSE in the repository root for full details.
*/
import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
import { combineLatest, map, of, switchMap, tap } from "rxjs";
import { combineLatest, map, of, switchMap } from "rxjs";
import { type Logger } from "matrix-js-sdk/lib/logger";
import { type RemoteParticipant } from "livekit-client";
import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
import { type Behavior } from "../../Behavior.ts";
import { type Connection } from "./Connection.ts";
@@ -17,6 +18,11 @@ import { Epoch, type ObservableScope } from "../../ObservableScope.ts";
import { generateItemsWithEpoch } from "../../../utils/observable.ts";
import { areLivekitTransportsEqual } from "./MatrixLivekitMembers.ts";
import { type ConnectionFactory } from "./ConnectionFactory.ts";
import {
isLocalTransportWithSFUConfig,
type LocalTransportWithSFUConfig,
} from "../localMember/LocalTransport.ts";
import { type SFUConfig } from "../../../livekit/openIDSFU.ts";
export class ConnectionManagerData {
private readonly store: Map<
@@ -65,8 +71,11 @@ export class ConnectionManagerData {
interface Props {
scope: ObservableScope;
connectionFactory: ConnectionFactory;
inputTransports$: Behavior<Epoch<LivekitTransport[]>>;
localTransport$: Behavior<LocalTransportWithSFUConfig | null>;
remoteTransports$: Behavior<Epoch<LivekitTransport[]>>;
logger: Logger;
ownMembershipIdentity: CallMembershipIdentityParts;
}
// TODO - write test for scopes (do we really need to bind scope)
@@ -79,8 +88,12 @@ export interface IConnectionManager {
* @param props - Configuration object
* @param props.scope - The observable scope used by this object
* @param props.connectionFactory - Used to create new connections
* @param props.inputTransports$ - A list of Behaviors each containing a LIST of LivekitTransport.
* @param props.logger - The logger to use
* @param props.localTransport$ - The local transport to use. (deduplicated with remoteTransports$)
* @param props.remoteTransports$ - All other transports. The connection manager will create connections for each transport. (deduplicated with localTransport$)
* @param props.ownMembershipIdentity - The own membership identity to use.
* @param props.logger - The logger to use.
*
* Each of these behaviors can be interpreted as subscribed list of transports.
*
* Using `registerTransports` independent external modules can control what connections
@@ -93,8 +106,10 @@ export interface IConnectionManager {
export function createConnectionManager$({
scope,
connectionFactory,
inputTransports$,
localTransport$,
remoteTransports$,
logger: parentLogger,
ownMembershipIdentity,
}: Props): IConnectionManager {
const logger = parentLogger.getChild("[ConnectionManager]");
// TODO logger: only construct one logger from the client and make it compatible via a EC specific sing
@@ -107,12 +122,33 @@ export function createConnectionManager$({
* It is build based on the list of subscribed transports (`transportsSubscriptions$`).
* externally this is modified via `registerTransports()`.
*/
const transports$ = scope.behavior(
inputTransports$.pipe(
map((transports) => transports.mapInner(removeDuplicateTransports)),
tap(({ value: transports }) => {
logger.trace(
`Managing transports: ${transports.map((t) => t.livekit_service_url).join(", ")}`,
const localAndRemoteTransports$: Behavior<
Epoch<(LivekitTransport | LocalTransportWithSFUConfig)[]>
> = scope.behavior(
combineLatest([remoteTransports$, localTransport$]).pipe(
// Combine local and remote transports into one transport array
// and set the forceOldJwtEndpoint property on the local transport
map(([remoteTransports, localTransport]) => {
let localTransportAsArray: LocalTransportWithSFUConfig[] = [];
if (localTransport) {
localTransportAsArray = [localTransport];
}
const dedupedRemote = removeDuplicateTransports(remoteTransports.value);
const remoteWithoutLocal = dedupedRemote.filter(
(transport) =>
!localTransportAsArray.find((l) =>
areLivekitTransportsEqual(l.transport, transport),
),
);
logger.debug(
"remoteWithoutLocal",
remoteWithoutLocal,
"localTransportAsArray",
localTransportAsArray,
);
return new Epoch(
[...localTransportAsArray, ...remoteWithoutLocal],
remoteTransports.epoch,
);
}),
),
@@ -122,25 +158,51 @@ export function createConnectionManager$({
* Connections for each transport in use by one or more session members.
*/
const connections$ = scope.behavior(
transports$.pipe(
localAndRemoteTransports$.pipe(
generateItemsWithEpoch(
function* (transports) {
for (const transport of transports)
yield {
keys: [transport.livekit_service_url, transport.livekit_alias],
data: undefined,
};
for (const transportWithOrWithoutSfuConfig of transports) {
if (
isLocalTransportWithSFUConfig(transportWithOrWithoutSfuConfig)
) {
// This is the local transport only the `LocalTransportWithSFUConfig` has a `sfuConfig` field
const { transport, sfuConfig } = transportWithOrWithoutSfuConfig;
yield {
keys: [
transport.livekit_service_url,
transport.livekit_alias,
sfuConfig,
],
data: undefined,
};
} else {
const transport = transportWithOrWithoutSfuConfig;
yield {
keys: [
transport.livekit_service_url,
transport.livekit_alias,
undefined as undefined | SFUConfig,
],
data: undefined,
};
}
}
},
(scope, _data$, serviceUrl, alias) => {
logger.debug(`Creating connection to ${serviceUrl} (${alias})`);
(scope, _data$, serviceUrl, alias, sfuConfig) => {
logger.debug(
`Creating connection to ${serviceUrl} (${alias}, withSfuConfig (local connection?): ${JSON.stringify(sfuConfig) ?? "no config->remote connection"})`,
);
const connection = connectionFactory.createConnection(
scope,
{
type: "livekit",
livekit_service_url: serviceUrl,
livekit_alias: alias,
},
scope,
ownMembershipIdentity,
logger,
sfuConfig,
);
// Start the connection immediately
// Use connection state to track connection progress
@@ -190,18 +252,18 @@ export function createConnectionManager$({
);
}),
),
new Epoch(new ConnectionManagerData()),
new Epoch(new ConnectionManagerData(), -1),
);
return { connectionManagerData$ };
}
function removeDuplicateTransports(
transports: LivekitTransport[],
): LivekitTransport[] {
function removeDuplicateTransports<T extends LivekitTransport>(
transports: T[],
): T[] {
return transports.reduce((acc, transport) => {
if (!acc.some((t) => areLivekitTransportsEqual(t, transport)))
acc.push(transport);
return acc;
}, [] as LivekitTransport[]);
}, [] as T[]);
}

View File

@@ -15,7 +15,11 @@ import EventEmitter from "events";
import { ObservableScope } from "../../ObservableScope.ts";
import { ECConnectionFactory } from "./ConnectionFactory.ts";
import type { OpenIDClientParts } from "../../../livekit/openIDSFU.ts";
import { exampleTransport, mockMediaDevices } from "../../../utils/test.ts";
import {
exampleTransport,
mockMediaDevices,
ownMemberMock,
} from "../../../utils/test.ts";
import type { ProcessorState } from "../../../livekit/TrackProcessorContext.tsx";
import { constant } from "../../Behavior";
@@ -72,7 +76,12 @@ describe("ECConnectionFactory - Audio inputs options", () => {
echo,
noise,
);
ecConnectionFactory.createConnection(exampleTransport, testScope, logger);
ecConnectionFactory.createConnection(
testScope,
exampleTransport,
ownMemberMock,
logger,
);
// Check if Room was constructed with expected options
expect(RoomConstructor).toHaveBeenCalledWith(
@@ -113,7 +122,12 @@ describe("ECConnectionFactory - ControlledAudioDevice", () => {
false,
false,
);
ecConnectionFactory.createConnection(exampleTransport, testScope, logger);
ecConnectionFactory.createConnection(
testScope,
exampleTransport,
ownMemberMock,
logger,
);
// Check if Room was constructed with expected options
expect(RoomConstructor).toHaveBeenCalledWith(

View File

@@ -10,8 +10,7 @@ import {
type CallMembership,
type LivekitTransport,
} from "matrix-js-sdk/lib/matrixrtc";
import { getParticipantId } from "matrix-js-sdk/lib/matrixrtc/utils";
import { combineLatest, map, type Observable } from "rxjs";
import { BehaviorSubject, combineLatest, map, type Observable } from "rxjs";
import { type IConnectionManager } from "./ConnectionManager.ts";
import {
@@ -26,14 +25,18 @@ import {
} from "../../ObservableScope.ts";
import { ConnectionManagerData } from "./ConnectionManager.ts";
import {
mockCallMembership,
flushPromises,
mockRtcMembership,
mockRemoteParticipant,
withTestScheduler,
} from "../../../utils/test.ts";
import { type Connection } from "./Connection.ts";
import { constant } from "../../Behavior.ts";
let testScope: ObservableScope;
const fallbackMemberId = (userId: string, deviceId: string): string =>
`${userId}:${deviceId}`;
const transportA: LivekitTransport = {
type: "livekit",
livekit_service_url: "https://lk.example.org",
@@ -46,16 +49,12 @@ const transportB: LivekitTransport = {
livekit_alias: "!alias:sample.com",
};
const bobMembership = mockCallMembership(
"@bob:example.org",
"DEV000",
transportA,
);
const carlMembership = mockCallMembership(
"@carl:sample.com",
"DEV111",
transportB,
);
const bobMembership = mockRtcMembership("@bob:example.org", "DEV000", {
fociPreferred: [transportA],
});
const carlMembership = mockRtcMembership("@carl:sample.com", "DEV111", {
fociPreferred: [transportB],
});
beforeEach(() => {
testScope = new ObservableScope();
@@ -76,52 +75,41 @@ function epochMeWith$<T, U>(
);
}
test("should signal participant not yet connected to livekit", () => {
withTestScheduler(({ behavior, expectObservable }) => {
const { memberships$, membershipsWithTransport$ } = fromMemberships$(
behavior("a", {
a: [bobMembership],
}),
);
test("should signal participant not yet connected to livekit", async () => {
const mockedMemberships$ = new BehaviorSubject([bobMembership]);
const mockConnectionManagerData$ = new BehaviorSubject(
new ConnectionManagerData(),
);
const { memberships$, membershipsWithTransport$ } =
createEpochedMemberships$(mockedMemberships$);
const connectionManagerData$ = epochMeWith$(
memberships$,
behavior("a", {
a: new ConnectionManagerData(),
}),
);
const connectionManagerData$ = epochMeWith$(
memberships$,
mockConnectionManagerData$,
);
const matrixLivekitMembers$ = createMatrixLivekitMembers$({
scope: testScope,
membershipsWithTransport$: testScope.behavior(membershipsWithTransport$),
connectionManager: {
connectionManagerData$: connectionManagerData$,
} as unknown as IConnectionManager,
});
expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe(
"a",
{
a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => {
expect(data.length).toEqual(1);
expectObservable(data[0].membership$).toBe("a", {
a: bobMembership,
});
expectObservable(data[0].participant.value$).toBe("a", {
a: null,
});
expectObservable(data[0].connection$).toBe("a", {
a: null,
});
return true;
}),
},
);
const matrixLivekitMember$ = createMatrixLivekitMembers$({
scope: testScope,
membershipsWithTransport$: testScope.behavior(membershipsWithTransport$),
connectionManager: {
connectionManagerData$: connectionManagerData$,
} as unknown as IConnectionManager,
});
await flushPromises();
expect(matrixLivekitMember$.value.value).toSatisfy(
(data: RemoteMatrixLivekitMember[]) => {
expect(data.length).toEqual(1);
expect(data[0].membership$.value).toBe(bobMembership);
expect(data[0].participant.value$.value).toBe(null);
expect(data[0].connection$.value).toBe(null);
return true;
},
);
});
// Helper to create epoch'ed memberships$ and membershipsWithTransport$ from memberships observable.
function fromMemberships$(m$: Observable<CallMembership[]>): {
function createEpochedMemberships$(m$: Observable<CallMembership[]>): {
memberships$: Observable<Epoch<CallMembership[]>>;
membershipsWithTransport$: Observable<
Epoch<{ membership: CallMembership; transport?: LivekitTransport }[]>
@@ -146,32 +134,115 @@ function fromMemberships$(m$: Observable<CallMembership[]>): {
};
}
test("should signal participant on a connection that is publishing", () => {
withTestScheduler(({ behavior, expectObservable }) => {
const bobParticipantId = getParticipantId(
test("should signal participant on a connection that is publishing", async () => {
const bobParticipantId = fallbackMemberId(
bobMembership.userId,
bobMembership.deviceId,
);
const { memberships$, membershipsWithTransport$ } = createEpochedMemberships$(
constant([bobMembership]),
);
const connection = {
transport: bobMembership.getTransport(bobMembership),
} as unknown as Connection;
const dataWithPublisher = new ConnectionManagerData();
dataWithPublisher.add(connection, [
mockRemoteParticipant({ identity: bobParticipantId }),
]);
const connectionManagerData$ = epochMeWith$(
memberships$,
constant(dataWithPublisher),
);
const matrixLivekitMember$ = createMatrixLivekitMembers$({
scope: testScope,
membershipsWithTransport$: testScope.behavior(membershipsWithTransport$),
connectionManager: {
connectionManagerData$: connectionManagerData$,
} as unknown as IConnectionManager,
});
await flushPromises();
expect(matrixLivekitMember$.value.value).toSatisfy(
(data: RemoteMatrixLivekitMember[]) => {
expect(data.length).toEqual(1);
expect(data[0].membership$.value).toBe(bobMembership);
expect(data[0].participant.value$.value).toSatisfy((participant) => {
expect(participant).toBeDefined();
expect(participant!.identity).toEqual(bobParticipantId);
return true;
});
expect(data[0].connection$.value).toBe(connection);
return true;
},
);
});
test("should signal participant on a connection that is not publishing", async () => {
const { memberships$, membershipsWithTransport$ } = createEpochedMemberships$(
constant([bobMembership]),
);
const connection = {
transport: bobMembership.getTransport(bobMembership),
} as unknown as Connection;
const dataWithPublisher = new ConnectionManagerData();
dataWithPublisher.add(connection, []);
const connectionManagerData$ = epochMeWith$(
memberships$,
constant(dataWithPublisher),
);
const matrixLivekitMember$ = createMatrixLivekitMembers$({
scope: testScope,
membershipsWithTransport$: testScope.behavior(membershipsWithTransport$),
connectionManager: {
connectionManagerData$: connectionManagerData$,
} as unknown as IConnectionManager,
});
await flushPromises();
expect(matrixLivekitMember$.value.value).toSatisfy(
(data: RemoteMatrixLivekitMember[]) => {
expect(data.length).toEqual(1);
expect(data[0].membership$.value).toBe(bobMembership);
expect(data[0].participant.value$.value).toBe(null);
expect(data[0].connection$.value).toBe(connection);
return true;
},
);
});
describe("Publication edge case", () => {
test("bob is publishing in several connections", async () => {
const { memberships$, membershipsWithTransport$ } =
createEpochedMemberships$(constant([bobMembership, carlMembership]));
const connectionWithPublisher = new ConnectionManagerData();
const bobParticipantId = fallbackMemberId(
bobMembership.userId,
bobMembership.deviceId,
);
const { memberships$, membershipsWithTransport$ } = fromMemberships$(
behavior("a", {
a: [bobMembership],
}),
);
const connection = {
transport: bobMembership.getTransport(bobMembership),
const connectionA = {
transport: transportA,
} as unknown as Connection;
const dataWithPublisher = new ConnectionManagerData();
dataWithPublisher.add(connection, [
const connectionB = {
transport: transportB,
} as unknown as Connection;
connectionWithPublisher.add(connectionA, [
mockRemoteParticipant({ identity: bobParticipantId }),
]);
connectionWithPublisher.add(connectionB, [
mockRemoteParticipant({ identity: bobParticipantId }),
]);
const connectionManagerData$ = epochMeWith$(
memberships$,
behavior("a", {
a: dataWithPublisher,
}),
constant(connectionWithPublisher),
);
const matrixLivekitMembers$ = createMatrixLivekitMembers$({
@@ -181,213 +252,73 @@ test("should signal participant on a connection that is publishing", () => {
connectionManagerData$: connectionManagerData$,
} as unknown as IConnectionManager,
});
expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe(
"a",
{
a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => {
expect(data.length).toEqual(1);
expectObservable(data[0].membership$).toBe("a", {
a: bobMembership,
});
expectObservable(data[0].participant.value$).toBe("a", {
a: expect.toSatisfy((participant) => {
expect(participant).toBeDefined();
expect(participant!.identity).toEqual(bobParticipantId);
return true;
}),
});
expectObservable(data[0].connection$).toBe("a", {
a: connection,
});
await flushPromises();
expect(matrixLivekitMembers$.value.value).toSatisfy(
(data: RemoteMatrixLivekitMember[]) => {
expect(data.length).toEqual(2);
expect(data[0].membership$.value).toBe(bobMembership);
expect(data[0].connection$.value).toBe(connectionA);
expect(data[0].participant.value$.value).toSatisfy((participant) => {
expect(participant).toBeDefined();
expect(participant!.identity).toEqual(bobParticipantId);
return true;
}),
});
return true;
},
);
});
});
test("should signal participant on a connection that is not publishing", () => {
withTestScheduler(({ behavior, expectObservable }) => {
const { memberships$, membershipsWithTransport$ } = fromMemberships$(
behavior("a", {
a: [bobMembership],
}),
);
test("bob is publishing in the wrong connection", async () => {
const mockedMemberships$ = new BehaviorSubject([
bobMembership,
carlMembership,
]);
const connection = {
transport: bobMembership.getTransport(bobMembership),
} as unknown as Connection;
const dataWithPublisher = new ConnectionManagerData();
dataWithPublisher.add(connection, []);
const { memberships$, membershipsWithTransport$ } =
createEpochedMemberships$(mockedMemberships$);
const connectionManagerData$ = epochMeWith$(
memberships$,
behavior("a", {
a: dataWithPublisher,
}),
);
const connectionWithPublisher = new ConnectionManagerData();
const matrixLivekitMembers$ = createMatrixLivekitMembers$({
scope: testScope,
membershipsWithTransport$: testScope.behavior(membershipsWithTransport$),
connectionManager: {
connectionManagerData$: connectionManagerData$,
} as unknown as IConnectionManager,
});
const bobParticipantId = fallbackMemberId(
bobMembership.userId,
bobMembership.deviceId,
);
const connectionA = { transport: transportA } as unknown as Connection;
const connectionB = { transport: transportB } as unknown as Connection;
expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe(
"a",
{
a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => {
expect(data.length).toEqual(1);
expectObservable(data[0].membership$).toBe("a", {
a: bobMembership,
});
expectObservable(data[0].participant.value$).toBe("a", {
a: null,
});
expectObservable(data[0].connection$).toBe("a", {
a: connection,
});
return true;
}),
},
);
});
});
describe("Publication edge case", () => {
test("bob is publishing in several connections", () => {
withTestScheduler(({ behavior, expectObservable }) => {
const { memberships$, membershipsWithTransport$ } = fromMemberships$(
behavior("a", {
a: [bobMembership, carlMembership],
}),
);
const connectionWithPublisher = new ConnectionManagerData();
const bobParticipantId = getParticipantId(
bobMembership.userId,
bobMembership.deviceId,
);
const connectionA = {
transport: transportA,
} as unknown as Connection;
const connectionB = {
transport: transportB,
} as unknown as Connection;
connectionWithPublisher.add(connectionA, [
mockRemoteParticipant({ identity: bobParticipantId }),
]);
connectionWithPublisher.add(connectionB, [
mockRemoteParticipant({ identity: bobParticipantId }),
]);
const connectionManagerData$ = epochMeWith$(
memberships$,
behavior("a", {
a: connectionWithPublisher,
}),
);
const matrixLivekitMembers$ = createMatrixLivekitMembers$({
scope: testScope,
membershipsWithTransport$: testScope.behavior(
membershipsWithTransport$,
),
connectionManager: {
connectionManagerData$: connectionManagerData$,
} as unknown as IConnectionManager,
});
expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe(
"a",
{
a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => {
expect(data.length).toEqual(2);
expectObservable(data[0].membership$).toBe("a", {
a: bobMembership,
});
expectObservable(data[0].connection$).toBe("a", {
// The real connection should be from transportA as per the membership
a: connectionA,
});
expectObservable(data[0].participant.value$).toBe("a", {
a: expect.toSatisfy((participant) => {
expect(participant).toBeDefined();
expect(participant!.identity).toEqual(bobParticipantId);
return true;
}),
});
return true;
}),
},
);
});
});
test("bob is publishing in the wrong connection", () => {
withTestScheduler(({ behavior, expectObservable }) => {
const { memberships$, membershipsWithTransport$ } = fromMemberships$(
behavior("a", {
a: [bobMembership, carlMembership],
}),
);
const connectionWithPublisher = new ConnectionManagerData();
const bobParticipantId = getParticipantId(
bobMembership.userId,
bobMembership.deviceId,
);
const connectionA = { transport: transportA } as unknown as Connection;
const connectionB = { transport: transportB } as unknown as Connection;
// Bob is not publishing on A
connectionWithPublisher.add(connectionA, []);
// Bob is publishing on B but his membership says A
connectionWithPublisher.add(connectionB, [
mockRemoteParticipant({ identity: bobParticipantId }),
]);
const connectionManagerData$ = epochMeWith$(
memberships$,
behavior("a", {
a: connectionWithPublisher,
}),
);
const matrixLivekitMembers$ = createMatrixLivekitMembers$({
scope: testScope,
membershipsWithTransport$: testScope.behavior(
membershipsWithTransport$,
),
connectionManager: {
connectionManagerData$: connectionManagerData$,
} as unknown as IConnectionManager,
});
expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe(
"a",
{
a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => {
expect(data.length).toEqual(2);
expectObservable(data[0].membership$).toBe("a", {
a: bobMembership,
});
expectObservable(data[0].connection$).toBe("a", {
// The real connection should be from transportA as per the membership
a: connectionA,
});
expectObservable(data[0].participant.value$).toBe("a", {
// No participant as Bob is not publishing on his membership transport
a: null,
});
return true;
}),
},
);
});
// Bob is not publishing on A
connectionWithPublisher.add(connectionA, []);
// Bob is publishing on B but his membership says A
connectionWithPublisher.add(connectionB, [
mockRemoteParticipant({ identity: bobParticipantId }),
]);
const connectionsWithPublisher$ = new BehaviorSubject(
connectionWithPublisher,
);
const connectionManagerData$ = epochMeWith$(
memberships$,
connectionsWithPublisher$,
);
const matrixLivekitMember$ = createMatrixLivekitMembers$({
scope: testScope,
membershipsWithTransport$: testScope.behavior(membershipsWithTransport$),
connectionManager: {
connectionManagerData$: connectionManagerData$,
} as unknown as IConnectionManager,
});
await flushPromises();
expect(matrixLivekitMember$.value.value).toSatisfy(
(data: RemoteMatrixLivekitMember[]) => {
expect(data.length).toEqual(2);
expect(data[0].membership$.value).toBe(bobMembership);
expect(data[0].connection$.value).toBe(connectionA);
expect(data[0].participant.value$.value).toBe(null);
return true;
},
);
});

View File

@@ -84,7 +84,6 @@ export function createMatrixLivekitMembers$({
/**
* Stream of all the call members and their associated livekit data (if available).
*/
return scope.behavior(
combineLatest([
membershipsWithTransport$,
@@ -93,47 +92,39 @@ export function createMatrixLivekitMembers$({
filter((values) =>
values.every((value) => value.epoch === values[0].epoch),
),
map(
([
{ value: membershipsWithTransports, epoch },
{ value: managerData },
]) =>
new Epoch([membershipsWithTransports, managerData] as const, epoch),
),
map(([ms, data]) => new Epoch([ms.value, data.value] as const, ms.epoch)),
generateItemsWithEpoch(
// Generator function.
// creates an array of `{key, data}[]`
// Each change in the keys (new key, missing key) will result in a call to the factory function.
function* ([membershipsWithTransports, managerData]) {
for (const { membership, transport } of membershipsWithTransports) {
// TODO! cannot use membership.membershipID yet, Currently its hardcoded by the jwt service to
const participantId = /*membership.membershipID*/ `${membership.userId}:${membership.deviceId}`;
function* ([membershipsWithTransport, managerData]) {
for (const { membership, transport } of membershipsWithTransport) {
const participants = transport
? managerData.getParticipantsForTransport(transport)
: [];
const participant =
participants.find((p) => p.identity == participantId) ?? null;
participants.find(
(p) => p.identity == membership.rtcBackendIdentity,
) ?? null;
const connection = transport
? managerData.getConnectionForTransport(transport)
: null;
yield {
keys: [participantId, membership.userId],
keys: [membership.userId, membership.deviceId],
data: { membership, participant, connection },
};
}
},
// Each update where the key of the generator array do not change will result in updates to the `data$` observable in the factory.
(scope, data$, participantId, userId) => {
(scope, data$, userId, deviceId) => {
logger.debug(
`Generating member for participantId: ${participantId}, userId: ${userId}`,
`Generating member for livekitIdentity: ${data$.value.membership.rtcBackendIdentity}, userId:deviceId: ${userId}${deviceId}`,
);
const { participant$, ...rest } = scope.splitBehavior(data$);
// will only get called once per `participantId, userId` pair.
// updates to data$ and as a result to displayName$ and mxcAvatarUrl$ are more frequent.
return {
participantId,
userId,
participant: { type: "remote" as const, value$: participant$ },
...rest,
@@ -141,15 +132,16 @@ export function createMatrixLivekitMembers$({
},
),
),
new Epoch([], -1),
);
}
// TODO add back in the callviewmodel pauseWhen(this.pretendToBeDisconnected$)
// TODO add this to the JS-SDK
export function areLivekitTransportsEqual(
t1: LivekitTransport | null,
t2: LivekitTransport | null,
export function areLivekitTransportsEqual<T extends LivekitTransport>(
t1: T | null,
t2: T | null,
): boolean {
if (t1 && t2) return t1.livekit_service_url === t2.livekit_service_url;
// In case we have different lk rooms in the same SFU (depends on the livekit authorization service)

View File

@@ -18,7 +18,7 @@ import { it } from "vitest";
import { ObservableScope } from "../../ObservableScope.ts";
import type { Room as MatrixRoom } from "matrix-js-sdk/lib/models/room";
import {
mockCallMembership,
mockRtcMembership,
mockMatrixRoomMember,
withTestScheduler,
} from "../../../utils/test.ts";
@@ -111,7 +111,7 @@ describe("MatrixMemberMetadata", () => {
rawDisplayName: "it's a me",
});
const memberships$ = behavior("a", {
a: [mockCallMembership("@local:example.com", "DEVICE1")],
a: [mockRtcMembership("@local:example.com", "DEVICE1")],
});
const metadataStore = createMatrixMemberMetadata$(
testScope,
@@ -149,8 +149,8 @@ describe("MatrixMemberMetadata", () => {
withTestScheduler(({ behavior, expectObservable }) => {
const memberships$ = behavior("a", {
a: [
mockCallMembership("@alice:example.com", "DEVICE1"),
mockCallMembership("@bob:example.com", "DEVICE1"),
mockRtcMembership("@alice:example.com", "DEVICE1"),
mockRtcMembership("@bob:example.com", "DEVICE1"),
],
});
const metadataStore = createMatrixMemberMetadata$(
@@ -179,7 +179,7 @@ describe("MatrixMemberMetadata", () => {
setUpBasicRoom();
const memberships$ = behavior("a", {
a: [mockCallMembership("@no-name:foo.bar", "D000")],
a: [mockRtcMembership("@no-name:foo.bar", "D000")],
});
const metadataStore = createMatrixMemberMetadata$(
testScope,
@@ -201,11 +201,11 @@ describe("MatrixMemberMetadata", () => {
const memberships$ = behavior("a", {
a: [
mockCallMembership("@bob:example.com", "DEVICE1"),
mockCallMembership("@bob:example.com", "DEVICE2"),
mockCallMembership("@bob:foo.bar", "BOB000"),
mockCallMembership("@carl:example.com", "C000"),
mockCallMembership("@evil:example.com", "E000"),
mockRtcMembership("@bob:example.com", "DEVICE1"),
mockRtcMembership("@bob:example.com", "DEVICE2"),
mockRtcMembership("@bob:foo.bar", "BOB000"),
mockRtcMembership("@carl:example.com", "C000"),
mockRtcMembership("@evil:example.com", "E000"),
],
});
@@ -233,10 +233,10 @@ describe("MatrixMemberMetadata", () => {
setUpBasicRoom();
const memberships$ = behavior("ab", {
a: [mockCallMembership("@bob:example.com", "DEVICE1")],
a: [mockRtcMembership("@bob:example.com", "DEVICE1")],
b: [
mockCallMembership("@bob:example.com", "DEVICE1"),
mockCallMembership("@bob:foo.bar", "BOB000"),
mockRtcMembership("@bob:example.com", "DEVICE1"),
mockRtcMembership("@bob:foo.bar", "BOB000"),
],
});
@@ -262,10 +262,10 @@ describe("MatrixMemberMetadata", () => {
const memberships$ = behavior("ab", {
a: [
mockCallMembership("@bob:example.com", "DEVICE1"),
mockCallMembership("@bob:foo.bar", "BOB000"),
mockRtcMembership("@bob:example.com", "DEVICE1"),
mockRtcMembership("@bob:foo.bar", "BOB000"),
],
b: [mockCallMembership("@bob:example.com", "DEVICE1")],
b: [mockRtcMembership("@bob:example.com", "DEVICE1")],
});
const metadataStore = createMatrixMemberMetadata$(
@@ -292,8 +292,8 @@ describe("MatrixMemberMetadata", () => {
const memberships$ = behavior("a", {
a: [
mockCallMembership("@bob:example.com", "B000"),
mockCallMembership("@carl:example.com", "C000"),
mockRtcMembership("@bob:example.com", "B000"),
mockRtcMembership("@carl:example.com", "C000"),
],
});
const metadataStore = createMatrixMemberMetadata$(
@@ -331,16 +331,16 @@ describe("MatrixMemberMetadata", () => {
// - room join/leave
// - disambiguate
const memberships$ = behavior("ab-d", {
a: [mockCallMembership(CARL, "C000")],
a: [mockRtcMembership(CARL, "C000")],
b: [
mockCallMembership(CARL, "C000"),
mockRtcMembership(CARL, "C000"),
// bob joins
mockCallMembership(BOB, "B000"),
mockRtcMembership(BOB, "B000"),
],
// c carl gets renamed to BOB
d: [
// carl leaves
mockCallMembership(BOB, "B000"),
mockRtcMembership(BOB, "B000"),
],
});
schedule("--a-", {
@@ -379,8 +379,8 @@ describe("MatrixMemberMetadata", () => {
it("should disambiguate users with invisible characters", () => {
withTestScheduler(({ behavior, expectObservable }) => {
const bobRtcMember = mockCallMembership("@bob:example.org", "BBBB");
const bobZeroWidthSpaceRtcMember = mockCallMembership(
const bobRtcMember = mockRtcMembership("@bob:example.org", "BBBB");
const bobZeroWidthSpaceRtcMember = mockRtcMembership(
"@bob2:example.org",
"BBBB",
);
@@ -397,9 +397,9 @@ describe("MatrixMemberMetadata", () => {
fakeMemberWith(bobZeroWidthSpace);
fakeMemberWith({ userId: "@carol:example.org" });
const memberships$ = behavior("ab", {
a: [mockCallMembership("@carol:example.org", "1111"), bobRtcMember],
a: [mockRtcMembership("@carol:example.org", "1111"), bobRtcMember],
b: [
mockCallMembership("@carol:example.org", "1111"),
mockRtcMembership("@carol:example.org", "1111"),
bobRtcMember,
bobZeroWidthSpaceRtcMember,
],
@@ -450,8 +450,8 @@ describe("MatrixMemberMetadata", () => {
it("should strip RTL characters from displayname", () => {
withTestScheduler(({ behavior, expectObservable }) => {
const daveRtcMember = mockCallMembership("@dave:example.org", "DDDD");
const daveRTLRtcMember = mockCallMembership(
const daveRtcMember = mockRtcMembership("@dave:example.org", "DDDD");
const daveRTLRtcMember = mockRtcMembership(
"@dave2:example.org",
"DDDD",
);
@@ -466,9 +466,9 @@ describe("MatrixMemberMetadata", () => {
fakeMemberWith(daveRTL);
fakeMemberWith(dave);
const memberships$ = behavior("ab", {
a: [mockCallMembership("@carol:example.org", "DDDD")],
a: [mockRtcMembership("@carol:example.org", "DDDD")],
b: [
mockCallMembership("@carol:example.org", "DDDD"),
mockRtcMembership("@carol:example.org", "DDDD"),
daveRtcMember,
daveRTLRtcMember,
],
@@ -527,8 +527,8 @@ describe("MatrixMemberMetadata", () => {
});
const memberships$ = behavior("a", {
a: [
mockCallMembership("@local:example.com", "DEVICE1"),
mockCallMembership("@alice:example.com", "DEVICE1"),
mockRtcMembership("@local:example.com", "DEVICE1"),
mockRtcMembership("@alice:example.com", "DEVICE1"),
],
});
const metadataStore = createMatrixMemberMetadata$(
@@ -562,12 +562,12 @@ describe("MatrixMemberMetadata", () => {
fakeMemberWith({ userId: "@carl:example.com" });
fakeMemberWith({ userId: "@bob:example.com" });
const memberships$ = behavior("ab-d", {
a: [mockCallMembership("@bob:example.com", "B000")],
a: [mockRtcMembership("@bob:example.com", "B000")],
b: [
mockCallMembership("@bob:example.com", "B000"),
mockCallMembership("@carl:example.com", "C000"),
mockRtcMembership("@bob:example.com", "B000"),
mockRtcMembership("@carl:example.com", "C000"),
],
d: [mockCallMembership("@carl:example.com", "C000")],
d: [mockRtcMembership("@carl:example.com", "C000")],
});
const metadataStore = createMatrixMemberMetadata$(

View File

@@ -21,8 +21,9 @@ import {
import { ECConnectionFactory } from "./ConnectionFactory.ts";
import { type OpenIDClientParts } from "../../../livekit/openIDSFU.ts";
import {
mockCallMembership,
mockMediaDevices,
mockRtcMembership,
ownMemberMock,
withTestScheduler,
} from "../../../utils/test.ts";
import { type ProcessorState } from "../../../livekit/TrackProcessorContext.tsx";
@@ -33,6 +34,7 @@ import {
} from "./MatrixLivekitMembers.ts";
import { createConnectionManager$ } from "./ConnectionManager.ts";
import { membershipsAndTransports$ } from "../../SessionBehaviors.ts";
import { constant } from "../../Behavior.ts";
import { testJWTToken } from "../../../utils/test-fixtures.ts";
// Test the integration of ConnectionManager and MatrixLivekitMerger
@@ -99,9 +101,9 @@ afterEach(() => {
test("bob, carl, then bob joining no tracks yet", () => {
withTestScheduler(({ expectObservable, behavior, scope }) => {
const bobMembership = mockCallMembership("@bob:example.com", "BDEV000");
const carlMembership = mockCallMembership("@carl:example.com", "CDEV000");
const daveMembership = mockCallMembership("@dave:foo.bar", "DDEV000");
const bobMembership = mockRtcMembership("@bob:example.com", "BDEV000");
const carlMembership = mockRtcMembership("@carl:example.com", "CDEV000");
const daveMembership = mockRtcMembership("@dave:foo.bar", "DDEV000");
const eMarble = "abc";
const vMarble = "abc";
@@ -121,8 +123,10 @@ test("bob, carl, then bob joining no tracks yet", () => {
const connectionManager = createConnectionManager$({
scope: testScope,
connectionFactory: ecConnectionFactory,
inputTransports$: membershipsAndTransports.transports$,
localTransport$: constant(null),
remoteTransports$: membershipsAndTransports.transports$,
logger: logger,
ownMembershipIdentity: ownMemberMock,
});
const matrixLivekitMembers$ = createMatrixLivekitMembers$({

View File

@@ -37,7 +37,7 @@ vi.mock("../widget", () => ({
it.each([
[MatrixRTCMode.Legacy],
[MatrixRTCMode.Compatibil],
[MatrixRTCMode.Compatibility],
[MatrixRTCMode.Matrix_2_0],
])(
"expect leave when ElementWidgetActions.HangupCall is called (%s mode)",

View File

@@ -257,6 +257,7 @@ abstract class BaseMediaViewModel {
* The Matrix user to which this media belongs.
*/
public readonly userId: string,
public readonly rtcBackendIdentity: string,
// We don't necessarily have a participant if a user connects via MatrixRTC but not (yet) through
// livekit.
protected readonly participant$: Observable<
@@ -406,6 +407,7 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel {
scope: ObservableScope,
id: string,
userId: string,
rtcBackendIdentity: string,
participant$: Observable<LocalParticipant | RemoteParticipant | null>,
encryptionSystem: EncryptionSystem,
livekitRoom$: Behavior<LivekitRoom | undefined>,
@@ -419,6 +421,7 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel {
scope,
id,
userId,
rtcBackendIdentity,
participant$,
encryptionSystem,
Track.Source.Microphone,
@@ -544,6 +547,7 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel {
scope: ObservableScope,
id: string,
userId: string,
rtcBackendIdentity: string,
participant$: Behavior<LocalParticipant | null>,
encryptionSystem: EncryptionSystem,
livekitRoom$: Behavior<LivekitRoom | undefined>,
@@ -558,6 +562,7 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel {
scope,
id,
userId,
rtcBackendIdentity,
participant$,
encryptionSystem,
livekitRoom$,
@@ -671,6 +676,7 @@ export class RemoteUserMediaViewModel extends BaseUserMediaViewModel {
scope: ObservableScope,
id: string,
userId: string,
rtcBackendIdentity: string,
participant$: Observable<RemoteParticipant | null>,
encryptionSystem: EncryptionSystem,
livekitRoom$: Behavior<LivekitRoom | undefined>,
@@ -685,6 +691,7 @@ export class RemoteUserMediaViewModel extends BaseUserMediaViewModel {
scope,
id,
userId,
rtcBackendIdentity,
participant$,
encryptionSystem,
livekitRoom$,
@@ -772,6 +779,7 @@ export class ScreenShareViewModel extends BaseMediaViewModel {
scope: ObservableScope,
id: string,
userId: string,
rtcBackendIdentity: string,
participant$: Observable<LocalParticipant | RemoteParticipant>,
encryptionSystem: EncryptionSystem,
livekitRoom$: Behavior<LivekitRoom | undefined>,
@@ -785,6 +793,7 @@ export class ScreenShareViewModel extends BaseMediaViewModel {
scope,
id,
userId,
rtcBackendIdentity,
participant$,
encryptionSystem,
Track.Source.ScreenShareAudio,

View File

@@ -28,6 +28,7 @@ export class ScreenShare {
private readonly scope: ObservableScope,
id: string,
userId: string,
rtcBackendIdentity: string,
participant: LocalParticipant | RemoteParticipant,
encryptionSystem: EncryptionSystem,
livekitRoom$: Behavior<LivekitRoom | undefined>,
@@ -40,6 +41,7 @@ export class ScreenShare {
this.scope,
id,
userId,
rtcBackendIdentity,
of(participant),
encryptionSystem,
livekitRoom$,

View File

@@ -75,6 +75,7 @@ export class UserMedia {
this.scope,
this.id,
this.userId,
this.rtcBackendIdentity,
this.participant.value$,
this.encryptionSystem,
this.livekitRoom$,
@@ -89,6 +90,7 @@ export class UserMedia {
this.scope,
this.id,
this.userId,
this.rtcBackendIdentity,
this.participant.value$,
this.encryptionSystem,
this.livekitRoom$,
@@ -140,6 +142,7 @@ export class UserMedia {
scope,
`${this.id}:${key}`,
this.userId,
this.rtcBackendIdentity,
p,
this.encryptionSystem,
this.livekitRoom$,
@@ -191,6 +194,7 @@ export class UserMedia {
private readonly scope: ObservableScope,
public readonly id: string,
private readonly userId: string,
private readonly rtcBackendIdentity: string,
private readonly participant: TaggedParticipant,
private readonly encryptionSystem: EncryptionSystem,
private readonly livekitRoom$: Behavior<LivekitRoom | undefined>,

View File

@@ -113,6 +113,7 @@ const UserMediaTile: FC<UserMediaTileProps> = ({
},
[vm],
);
const rtcBackendIdentity = vm.rtcBackendIdentity;
const handRaised = useBehavior(vm.handRaised$);
const reaction = useBehavior(vm.reaction$);
@@ -200,6 +201,7 @@ const UserMediaTile: FC<UserMediaTileProps> = ({
focusUrl={focusUrl}
audioStreamStats={audioStreamStats}
videoStreamStats={videoStreamStats}
rtcBackendIdentity={rtcBackendIdentity}
{...props}
/>
);

View File

@@ -18,7 +18,11 @@ import styles from "./MediaView.module.css";
import { Avatar } from "../Avatar";
import { type EncryptionStatus } from "../state/MediaViewModel";
import { RaisedHandIndicator } from "../reactions/RaisedHandIndicator";
import { showHandRaisedTimer, useSetting } from "../settings/settings";
import {
showConnectionStats,
showHandRaisedTimer,
useSetting,
} from "../settings/settings";
import { type ReactionOption } from "../reactions";
import { ReactionIndicator } from "../reactions/ReactionIndicator";
import { RTCConnectionStats } from "../RTCConnectionStats";
@@ -46,6 +50,7 @@ interface Props extends ComponentProps<typeof animated.div> {
waitingForMedia?: boolean;
audioStreamStats?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats;
videoStreamStats?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats;
rtcBackendIdentity?: string;
// The focus url, mainly for debugging purposes
focusUrl?: string;
}
@@ -74,11 +79,13 @@ export const MediaView: FC<Props> = ({
waitingForMedia,
audioStreamStats,
videoStreamStats,
rtcBackendIdentity,
focusUrl,
...props
}) => {
const { t } = useTranslation();
const [handRaiseTimerVisible] = useSetting(showHandRaisedTimer);
const [showConnectioStats] = useSetting(showConnectionStats);
const avatarSize = Math.round(Math.min(targetWidth, targetHeight) / 2);
@@ -132,14 +139,18 @@ export const MediaView: FC<Props> = ({
{waitingForMedia && (
<div className={styles.status}>
{t("video_tile.waiting_for_media")}
{showConnectioStats ? " " + rtcBackendIdentity : ""}
</div>
)}
{(audioStreamStats || videoStreamStats) && (
<RTCConnectionStats
audio={audioStreamStats}
video={videoStreamStats}
focusUrl={focusUrl}
/>
<>
<RTCConnectionStats
audio={audioStreamStats}
video={videoStreamStats}
focusUrl={focusUrl}
rtcBackendIdentity={rtcBackendIdentity}
/>
</>
)}
{/* TODO: Bring this back once encryption status is less broken */}
{/*encryptionStatus !== EncryptionStatus.Okay && (

View File

@@ -19,6 +19,7 @@ export enum ErrorCode {
INSUFFICIENT_CAPACITY_ERROR = "INSUFFICIENT_CAPACITY_ERROR",
E2EE_NOT_SUPPORTED = "E2EE_NOT_SUPPORTED",
OPEN_ID_ERROR = "OPEN_ID_ERROR",
NO_MATRIX_2_AUTHORIZATION_SERVICE = "NO_MATRIX_2_0_AUTHORIZATION_SERVICE",
SFU_ERROR = "SFU_ERROR",
UNKNOWN_ERROR = "UNKNOWN_ERROR",
}
@@ -171,6 +172,23 @@ export class FailToGetOpenIdToken extends ElementCallError {
}
}
export class NoMatrix2AuthorizationService extends ElementCallError {
/**
* Creates an instance of NoMatrix2_0AuthorizationService.
* @param error - The underlying error that caused the failure.
*/
public constructor(error: Error) {
super(
t("error.generic"),
ErrorCode.NO_MATRIX_2_AUTHORIZATION_SERVICE,
ErrorCategory.CONFIGURATION_ISSUE,
t("error.no_matrix_2_authorization_service"),
// Properly set it as a cause for a better reporting on sentry
error,
);
}
}
/**
* Error indicating a failure to start publishing on a LiveKit connection.
*/

View File

@@ -17,14 +17,16 @@ export const localRtcMemberDevice2 = mockRtcMembership(
"2222",
);
export const local = mockMatrixRoomMember(localRtcMember);
// export const localParticipant = mockLocalParticipant({ identity: "" });
export const localId = `${local.userId}:${localRtcMember.deviceId}`;
export const aliceRtcMember = mockRtcMembership("@alice:example.org", "AAAA");
export const aliceDeviceId = "AAAA";
export const aliceUserId = "@alice:example.org";
export const aliceId = `${aliceUserId}:${aliceDeviceId}`;
export const aliceRtcMember = mockRtcMembership(aliceUserId, aliceDeviceId);
export const alice = mockMatrixRoomMember(aliceRtcMember, {
rawDisplayName: "Alice",
});
export const aliceId = `${alice.userId}:${aliceRtcMember.deviceId}`;
export const aliceParticipant = mockRemoteParticipant({ identity: aliceId });
export const aliceDoppelgangerRtcMember = mockRtcMembership(
@@ -38,11 +40,13 @@ export const aliceDoppelganger = mockMatrixRoomMember(
},
);
export const bobRtcMember = mockRtcMembership("@bob:example.org", "BBBB");
export const bobDeviceId = "BBBB";
export const bobUserId = "@bob:example.org";
export const bobId = `${bobUserId}:${bobDeviceId}`;
export const bobRtcMember = mockRtcMembership(bobUserId, bobDeviceId);
export const bob = mockMatrixRoomMember(bobRtcMember, {
rawDisplayName: "Bob",
});
export const bobId = `${bob.userId}:${bobRtcMember.deviceId}`;
export const bobZeroWidthSpaceRtcMember = mockRtcMembership(
"@bob2:example.org",

View File

@@ -50,6 +50,7 @@ import {
type KeyTransportEvents,
type KeyTransportEventsHandlerMap,
} from "matrix-js-sdk/lib/matrixrtc/IKeyTransport";
import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
import {
LocalUserMediaViewModel,
@@ -201,40 +202,30 @@ export const exampleTransport: LivekitTransport = {
livekit_alias: "!alias:example.org",
};
export function mockCallMembership(
userId: string,
deviceId: string,
transport?: Transport,
): CallMembership {
const t = transport ?? transportForUser(userId);
return {
userId: userId,
deviceId: deviceId,
getTransport: vi.fn().mockReturnValue(t),
transports: [t],
} as unknown as CallMembership;
}
function transportForUser(userId: string): Transport {
const domain = userId.split(":")[1];
return {
type: "livekit",
livekit_service_url: `https://lk.${domain}`,
livekit_alias: `!alias:${domain}`,
};
}
export function mockRtcMembership(
user: string | RoomMember,
deviceId: string,
callId = "",
fociPreferred: Transport[] = [exampleTransport],
focusActive: LivekitFocusSelection = {
type: "livekit",
focus_selection: "oldest_membership",
customOverwrites?: {
rtcBackendIdentity?: string;
callId?: string;
fociPreferred?: Transport[];
focusActive?: LivekitFocusSelection;
membership?: Partial<SessionMembershipData>;
},
membership: Partial<SessionMembershipData> = {},
): CallMembership {
// setup defaults based on overwrites and fallback values.
const { rtcBackendIdentity, callId, fociPreferred, focusActive, membership } =
{
fociPreferred: [exampleTransport],
focusActive: {
type: "livekit" as const,
focus_selection: "oldest_membership" as const,
},
callId: "",
membership: {},
...customOverwrites,
};
const data: SessionMembershipData = {
application: "m.call",
call_id: callId,
@@ -243,17 +234,29 @@ export function mockRtcMembership(
focus_active: focusActive,
...membership,
};
const userId = typeof user === "string" ? user : user.userId;
const event = new MatrixEvent({
sender: typeof user === "string" ? user : user.userId,
sender: userId,
event_id: `$-ev-${randomUUID()}:example.org`,
content: data,
});
const cms = new CallMembership(event, data);
const membershipData = CallMembership.membershipDataFromMatrixEvent(event);
const cms = new CallMembership(
event,
membershipData,
rtcBackendIdentity ?? `${userId}:${deviceId}`,
);
vi.mocked(cms).getTransport = vi.fn().mockReturnValue(fociPreferred[0]);
return cms;
}
export const ownMemberMock: CallMembershipIdentityParts = {
userId: "@alice:example.org",
deviceId: "DEVICE",
memberId: "@alice:example.org:DEVICE",
};
// Maybe it'd be good to move this to matrix-js-sdk? Our testing needs are
// rather simple, but if one util to mock a member is good enough for us, maybe
// it's useful for matrix-js-sdk consumers in general.
@@ -331,6 +334,7 @@ export function createLocalMedia(
testScope(),
"local",
member.userId,
rtcMember.rtcBackendIdentity,
constant(localParticipant),
{
kind: E2eeType.PER_PARTICIPANT,
@@ -376,6 +380,7 @@ export function createRemoteMedia(
testScope(),
"remote",
member.userId,
rtcMember.rtcBackendIdentity,
constant(participant),
{
kind: E2eeType.PER_PARTICIPANT,
@@ -478,7 +483,7 @@ export class MockRTCSession extends TypedEventEmitter<
if (value !== prev) this.emit(MembershipManagerEvent.ProbablyLeft, value);
}
public async joinRoomSession(): Promise<void> {
public async joinRTCSession(): Promise<void> {
return Promise.resolve();
}
}

View File

@@ -3300,10 +3300,10 @@ __metadata:
languageName: node
linkType: hard
"@matrix-org/matrix-sdk-crypto-wasm@npm:^16.0.0":
version: 16.0.0
resolution: "@matrix-org/matrix-sdk-crypto-wasm@npm:16.0.0"
checksum: 10c0/13b4ede3e618da819957abff778afefcf3baf9a2faac04a36bb5a07a44fae2ea05fbfa072eb3408d48b2b7b9aaf27242ce52c594c8ce9bf1fb8b3aade2832be1
"@matrix-org/matrix-sdk-crypto-wasm@npm:^17.0.0":
version: 17.0.0
resolution: "@matrix-org/matrix-sdk-crypto-wasm@npm:17.0.0"
checksum: 10c0/fa97e3111099057e0953e7550d6556b6e7553f3badd5b25a6988d2fcc94d22288a27e63cb204771b74ff24388d770c83f2cf5aec583f05c6ecf46509b8020570
languageName: node
linkType: hard
@@ -8399,6 +8399,7 @@ __metadata:
typescript: "npm:^5.8.3"
typescript-eslint-language-service: "npm:^5.0.5"
unique-names-generator: "npm:^4.6.0"
uuid: "npm:^13.0.0"
vaul: "npm:^1.0.0"
vite: "npm:^7.0.0"
vite-plugin-generate-file: "npm:^0.3.0"
@@ -11467,10 +11468,10 @@ __metadata:
"matrix-js-sdk@matrix-org/matrix-js-sdk#develop":
version: 39.4.0
resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=174439c2f0c09cf9926c28435ba4db1345df4aee"
resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=4d0d32307eb4f1ce1fb65080fcca704f5bdedc31"
dependencies:
"@babel/runtime": "npm:^7.12.5"
"@matrix-org/matrix-sdk-crypto-wasm": "npm:^16.0.0"
"@matrix-org/matrix-sdk-crypto-wasm": "npm:^17.0.0"
another-json: "npm:^0.2.0"
bs58: "npm:^6.0.0"
content-type: "npm:^1.0.4"
@@ -11483,7 +11484,7 @@ __metadata:
sdp-transform: "npm:^3.0.0"
unhomoglyph: "npm:^1.0.6"
uuid: "npm:13"
checksum: 10c0/5178de27bb618aed6f80632a72c5582542ceedb51ef15534493360a624b072e0c276693ad9e37d83f2ddb06716f9eb6d02960e158e029f7a005676873778c745
checksum: 10c0/59c9d81ccf823584dc783502cb5c928562e3490c63f5ce98ee3232a603545d6278e90dc951c1fd0bae2792ba732ec5171e03596fd396bb2150d596cebb7fbac9
languageName: node
linkType: hard
@@ -15346,7 +15347,7 @@ __metadata:
languageName: node
linkType: hard
"uuid@npm:13":
"uuid@npm:13, uuid@npm:^13.0.0":
version: 13.0.0
resolution: "uuid@npm:13.0.0"
bin: