From 7dbbd763b965c47e65772ead3d5707e8b6242202 Mon Sep 17 00:00:00 2001 From: Timo K Date: Fri, 9 Jan 2026 13:38:26 +0100 Subject: [PATCH] Refactor how we aquire the jwt token for the local user. (only fetch it once) The local jwt token needs to be aquired via the right endpoint. The endpoint defines how our rtcBackendIdentity is computed. Based on us using sticky events or state events we also need to use the right endpoint. This cannot be done generically in the connection manager. The jwt token now is computed in the localTransport and the resolved sfu config is passed to the connection manager. Add JWT endpoint version and SFU config support Pin matrix-js-sdk to a specific commit and update dev auth image tag. Propagate SFU config and JWT endpoint choice through local transport, ConnectionManager and Connection; add JwtEndpointVersion enum and LocalTransportWithSFUConfig type. Add NO_MATRIX_2 auth error and locale string, thread rtcBackendIdentity through UI props, and include related test, CSS and minor imports updates --- dev-backend-docker-compose.yml | 4 +- locales/en/app.json | 1 + package.json | 2 +- src/RTCConnectionStats.tsx | 5 + src/livekit/openIDSFU.test.ts | 17 ++- src/livekit/openIDSFU.ts | 43 ++++-- src/room/InCallView.tsx | 18 ++- src/settings/DeveloperSettingsTab.module.css | 11 ++ src/settings/DeveloperSettingsTab.tsx | 22 +++- src/state/CallViewModel/CallViewModel.ts | 46 +++++-- .../localMember/LocalMember.test.ts | 35 +++-- .../CallViewModel/localMember/LocalMember.ts | 9 +- .../localMember/LocalTransport.test.ts | 14 +- .../localMember/LocalTransport.ts | 110 ++++++++++++---- .../remoteMembers/Connection.test.ts | 15 ++- .../CallViewModel/remoteMembers/Connection.ts | 44 ++++--- .../remoteMembers/ConnectionFactory.ts | 21 +-- .../remoteMembers/ConnectionManager.ts | 123 ++++++++++-------- .../remoteMembers/ECConnectionFactory.test.ts | 4 +- src/state/MediaViewModel.ts | 12 ++ src/state/ScreenShare.ts | 3 + src/state/UserMedia.ts | 6 + src/tile/GridTile.tsx | 2 + src/tile/MediaView.tsx | 15 ++- src/utils/errors.ts | 18 +++ src/utils/test.ts | 2 + yarn.lock | 11 +- 27 files changed, 421 insertions(+), 192 deletions(-) diff --git a/dev-backend-docker-compose.yml b/dev-backend-docker-compose.yml index 5e955831..0efefd07 100644 --- a/dev-backend-docker-compose.yml +++ b/dev-backend-docker-compose.yml @@ -3,7 +3,7 @@ networks: services: auth-service: - image: ghcr.io/element-hq/lk-jwt-service:latest-ci + image: ghcr.io/element-hq/lk-jwt-service:pr_139 pull_policy: always hostname: auth-server environment: @@ -25,7 +25,7 @@ services: - ecbackend auth-service-1: - image: ghcr.io/element-hq/lk-jwt-service:latest-ci + image: ghcr.io/element-hq/lk-jwt-service:pr_139 pull_policy: always hostname: auth-server-1 environment: diff --git a/locales/en/app.json b/locales/en/app.json index 1ff066ea..c70a0a49 100644 --- a/locales/en/app.json +++ b/locales/en/app.json @@ -116,6 +116,7 @@ "matrix_rtc_transport_missing": "The server is not configured to work with {{brand}}. Please contact your server admin (Domain: {{domain}}, Error Code: {{ errorCode }}).", "membership_manager": "Membership Manager Error", "membership_manager_description": "The Membership Manager had to shut down. This is caused by many consequtive failed network requests.", + "no_matrix_2_authorization_service": "Your authorization service for you media server (SFU) is not on the newest version", "open_elsewhere": "Opened in another tab", "open_elsewhere_description": "{{brand}} has been opened in another tab. If that doesn't sound right, try reloading the page.", "room_creation_restricted": "Failed to create call", diff --git a/package.json b/package.json index 944fc43b..6a8d3c76 100644 --- a/package.json +++ b/package.json @@ -104,7 +104,7 @@ "livekit-client": "^2.13.0", "lodash-es": "^4.17.21", "loglevel": "^1.9.1", - "matrix-js-sdk": "matrix-org/matrix-js-sdk#develop", + "matrix-js-sdk": "matrix-org/matrix-js-sdk#ff5fab7722e54be2d69a380cd6182ecb262d7859", "matrix-widget-api": "^1.14.0", "node-stdlib-browser": "^1.3.1", "normalize.css": "^8.0.1", diff --git a/src/RTCConnectionStats.tsx b/src/RTCConnectionStats.tsx index d51089cf..f62a1e4d 100644 --- a/src/RTCConnectionStats.tsx +++ b/src/RTCConnectionStats.tsx @@ -20,6 +20,7 @@ interface Props { audio?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats; video?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats; focusUrl?: string; + rtcBackendIdentity?: string; } const extractDomain = (url: string): string => { @@ -37,6 +38,7 @@ export const RTCConnectionStats: FC = ({ audio, video, focusUrl, + rtcBackendIdentity, ...rest }) => { const [showModal, setShowModal] = useState(false); @@ -71,6 +73,9 @@ export const RTCConnectionStats: FC = ({ + + rtcBackendIdentity:{rtcBackendIdentity} + {focusUrl && (
diff --git a/src/livekit/openIDSFU.test.ts b/src/livekit/openIDSFU.test.ts index 5f286958..20820748 100644 --- a/src/livekit/openIDSFU.test.ts +++ b/src/livekit/openIDSFU.test.ts @@ -46,7 +46,6 @@ describe("getSFUConfigWithOpenID", () => { matrixClient, ownMemberMock, "https://sfu.example.org", - false, "!example_room_id", ); expect(config).toEqual({ @@ -70,7 +69,6 @@ describe("getSFUConfigWithOpenID", () => { matrixClient, ownMemberMock, "https://sfu.example.org", - false, "!example_room_id", ); } catch (ex) { @@ -101,10 +99,11 @@ describe("getSFUConfigWithOpenID", () => { matrixClient, ownMemberMock, "https://sfu.example.org", - false, "!example_room_id", - "https://matrix.homeserverserver.org", - "mock_delay_id", + { + delayEndpointBaseUrl: "https://matrix.homeserverserver.org", + delayId: "mock_delay_id", + }, ); } catch (ex) { expect((ex as Error).message).toEqual( @@ -154,10 +153,11 @@ describe("getSFUConfigWithOpenID", () => { matrixClient, ownMemberMock, "https://sfu.example.org", - false, "!example_room_id", - "https://matrix.homeserverserver.org", - "mock_delay_id", + { + delayEndpointBaseUrl: "https://matrix.homeserverserver.org", + delayId: "mock_delay_id", + }, ); } catch (ex) { expect((ex as Error).message).toEqual( @@ -203,7 +203,6 @@ describe("getSFUConfigWithOpenID", () => { matrixClient, ownMemberMock, "https://sfu.example.org", - false, "!example_room_id", ); expect(config).toEqual({ diff --git a/src/livekit/openIDSFU.ts b/src/livekit/openIDSFU.ts index df1d02a5..a5a8adc5 100644 --- a/src/livekit/openIDSFU.ts +++ b/src/livekit/openIDSFU.ts @@ -13,9 +13,13 @@ import { import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; import { type Logger } from "matrix-js-sdk/lib/logger"; -import { FailToGetOpenIdToken } from "../utils/errors"; +import { + FailToGetOpenIdToken, + NoMatrix2AuthorizationService, +} from "../utils/errors"; import { doNetworkOperationWithRetry } from "../utils/matrix"; import { Config } from "../config/Config"; +import { JwtEndpointVersion } from "../state/CallViewModel/localMember/LocalTransport"; /** * Configuration and access tokens provided by the SFU on successful authentication. @@ -73,14 +77,15 @@ export type OpenIDClientParts = Pick< * @param client The Matrix client * @param membership Our own membership identity parts used to send to jwt service. * @param serviceUrl The URL of the livekit SFU service - * @param forceOldJwtEndpoint This will use the old jwt endpoint which will create the rtc backend identity based on string concatination + * @param roomId The room id used in the jwt request. This is NOT the livekit_alias. The jwt service will provide the alias. It maps matrix room ids <-> Livekit aliases. + * @param opts Additional options to modify which endpoint with which data will be used to aquire the jwt token. + * @param opts.forceJwtEndpoint This will use the old jwt endpoint which will create the rtc backend identity based on string concatination * instead of a hash. * This function by default uses whatever is possible with the current jwt service installed next to the SFU. * For remote connections this does not matter, since we will not publish there we can rely on the newest option. * For our own connection we can only use the hashed version if we also send the new matrix2.0 sticky events. - * @param roomId The room id used in the jwt request. This is NOT the livekit_alias. The jwt service will provide the alias. It maps matrix room ids <-> Livekit aliases. - * @param delayEndpointBaseUrl The URL of the matrix homeserver. - * @param delayId The delay id used for the jwt service to manage. + * @param opts.delayEndpointBaseUrl The URL of the matrix homeserver. + * @param opts.delayId The delay id used for the jwt service to manage. * @param logger optional logger. * @returns Object containing the token information * @throws FailToGetOpenIdToken @@ -89,10 +94,12 @@ export async function getSFUConfigWithOpenID( client: OpenIDClientParts, membership: CallMembershipIdentityParts, serviceUrl: string, - forceOldJwtEndpoint: boolean, roomId: string, - delayEndpointBaseUrl?: string, - delayId?: string, + opts?: { + forceJwtEndpoint?: JwtEndpointVersion; + delayEndpointBaseUrl?: string; + delayId?: string; + }, logger?: Logger, ): Promise { let openIdToken: IOpenIDToken; @@ -113,15 +120,21 @@ export async function getSFUConfigWithOpenID( // If forceOldJwtEndpoint is set we indicate that we do not want to try the new endpoint, // since we are not sending the new matrix2.0 sticky events (no hashed identity in the event) - if (forceOldJwtEndpoint === false) { + if ( + // we do not force anything. Try with new first (remote connections) + !opts?.forceJwtEndpoint || + // we do force the matrix2.0 endpoint + (opts?.forceJwtEndpoint && + opts?.forceJwtEndpoint === JwtEndpointVersion.Matrix_2_0) + ) { try { sfuConfig = await getLiveKitJWTWithDelayDelegation( membership, serviceUrl, roomId, openIdToken, - delayEndpointBaseUrl, - delayId, + opts?.delayEndpointBaseUrl, + opts?.delayId, ); logger?.info(`Got JWT from call's active focus URL.`); } catch (e) { @@ -137,12 +150,16 @@ export async function getSFUConfigWithOpenID( `(not going to try with legacy endpoint: forceOldJwtEndpoint is set to false, we did not get a not supported error from the sfu)`, e, ); + // Make this throw a hard error in case we force the matrix2.0 endpoint. + if (opts?.forceJwtEndpoint === JwtEndpointVersion.Matrix_2_0) + throw new NoMatrix2AuthorizationService(e as Error); } } } // DEPRECATED - // Either forceOldJwtEndpoint = true or getLiveKitJWTWithDelayDelegation throws -> reset sfuConfig = undefined + // here we either have a sfuConfig or we alredy exited becuause of `if (opts?.forceEndpoint === MatrixRTCMode.Matrix_2_0) throw e;` + // The only case we can get into this if is, if `opts?.forceEndpoint !== MatrixRTCMode.Matrix_2_0` if (sfuConfig === undefined) { sfuConfig = await getLiveKitJWT( membership.deviceId, @@ -235,7 +252,7 @@ export async function getLiveKitJWTWithDelayDelegation( let bodyDalayParts = {}; // Also check for empty string - if (delayId && delayEndpointBaseUrl && false) { + if (delayId && delayEndpointBaseUrl) { const delayTimeoutMs = Config.get().matrix_rtc_session?.delayed_leave_event_delay_ms ?? 1000; bodyDalayParts = { diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index fdbd4461..54f189f0 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -113,10 +113,8 @@ const logger = rootLogger.getChild("[InCallView]"); const maxTapDurationMs = 400; -export interface ActiveCallProps extends Omit< - InCallViewProps, - "vm" | "livekitRoom" | "connState" -> { +export interface ActiveCallProps + extends Omit { e2eeSystem: EncryptionSystem; // TODO refactor those reasons into an enum onLeft: ( @@ -798,6 +796,8 @@ export const InCallView: FC = ({
); + const allConnections = useBehavior(vm.allConnections$); + return (
= ({ onDismiss={closeSettings} tab={settingsTab} onTabChange={setSettingsTab} - // TODO expose correct data to setttings modal - livekitRooms={[]} + livekitRooms={allConnections + .getConnections() + .map((connectionItem) => ({ + room: connectionItem.livekitRoom, + // TODO compute is local or tag it in the livekit room items already + isLocal: undefined, + url: connectionItem.transport.livekit_service_url, + }))} /> )} diff --git a/src/settings/DeveloperSettingsTab.module.css b/src/settings/DeveloperSettingsTab.module.css index 7b83eb6c..29f4211b 100644 --- a/src/settings/DeveloperSettingsTab.module.css +++ b/src/settings/DeveloperSettingsTab.module.css @@ -8,3 +8,14 @@ Please see LICENSE in the repository root for full details. pre { font-size: var(--font-size-micro); } + +.livekit_room_box { + border: 3px solid var(--cpd-color-bg-subtle-secondary); + border-radius: var(--cpd-space-8x); + padding: var(--cpd-space-4x); + margin-bottom: var(--cpd-space-4x); + margin-top: var(--cpd-space-4x); + li { + font-size: var(--font-size-micro); + } +} diff --git a/src/settings/DeveloperSettingsTab.tsx b/src/settings/DeveloperSettingsTab.tsx index c88eadf0..a187e4b5 100644 --- a/src/settings/DeveloperSettingsTab.tsx +++ b/src/settings/DeveloperSettingsTab.tsx @@ -29,6 +29,7 @@ import { Label, RadioControl, } from "@vector-im/compound-web"; +import { type Room as LivekitRoom } from "livekit-client"; import { FieldRow, InputField } from "../input/Input"; import { @@ -42,7 +43,6 @@ import { customLivekitUrl as customLivekitUrlSetting, MatrixRTCMode, } from "./settings"; -import type { Room as LivekitRoom } from "livekit-client"; import styles from "./DeveloperSettingsTab.module.css"; import { useUrlParams } from "../UrlParams"; @@ -304,12 +304,12 @@ export const DeveloperSettingsTab: FC = ({ {livekitRooms?.map((livekitRoom) => ( - <> -

+
+

{t("developer_mode.livekit_sfu", { url: livekitRoom.url || "unknown", })} -

+

{livekitRoom.isLocal &&

ws-url: {localSfuUrl?.href}

}

{t("developer_mode.livekit_server_info")}( @@ -321,7 +321,19 @@ export const DeveloperSettingsTab: FC = ({ : "undefined"} {livekitRoom.room.metadata} - +

Local Participant

+ 
+            {livekitRoom.room.localParticipant.identity}
+
+

Remote Participants

+
    + {Array.from(livekitRoom.room.remoteParticipants.keys()).map( + (id) => ( +
  • {id}
    • + ), + )} +
+
))}

{t("developer_mode.environment_variables")}

{JSON.stringify(env, null, 2)}
diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index c75b3ec4..8fc2d7e6 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -108,13 +108,19 @@ import { enterRTCSession, TransportState, } from "./localMember/LocalMember.ts"; -import { createLocalTransport$ } from "./localMember/LocalTransport.ts"; +import { + createLocalTransport$, + JwtEndpointVersion, +} from "./localMember/LocalTransport.ts"; import { createMemberships$, membershipsAndTransports$, } from "../SessionBehaviors.ts"; import { ECConnectionFactory } from "./remoteMembers/ConnectionFactory.ts"; -import { createConnectionManager$ } from "./remoteMembers/ConnectionManager.ts"; +import { + type ConnectionManagerData, + createConnectionManager$, +} from "./remoteMembers/ConnectionManager.ts"; import { createMatrixLivekitMembers$, type TaggedParticipant, @@ -263,6 +269,7 @@ export interface CallViewModel { * multiple devices. */ participantCount$: Behavior; + allConnections$: Behavior; /** Participants sorted by livekit room so they can be used in the audio rendering */ livekitRoomItems$: Behavior; userMedia$: Behavior; @@ -428,14 +435,6 @@ export function createCallViewModel$( memberId: `${userId}:${deviceId}`, }; - const useOldJwtEndpoint$ = scope.behavior( - matrixRTCMode$.pipe( - map( - (v) => v === MatrixRTCMode.Legacy || v === MatrixRTCMode.Compatibility, - ), - ), - ); - const localTransport$ = createLocalTransport$({ scope: scope, memberships$: memberships$, @@ -451,7 +450,15 @@ export function createCallViewModel$( matrixRTCSession.delayId ?? null, ), roomId: matrixRoom.roomId, - useOldJwtEndpoint$, + forceJwtEndpoint$: scope.behavior( + matrixRTCMode$.pipe( + map((v) => + v === MatrixRTCMode.Matrix_2_0 + ? JwtEndpointVersion.Matrix_2_0 + : JwtEndpointVersion.Legacy, + ), + ), + ), useOldestMember$: scope.behavior( matrixRTCMode$.pipe(map((v) => v === MatrixRTCMode.Legacy)), ), @@ -483,7 +490,6 @@ export function createCallViewModel$( ), ), remoteTransports$: membershipsAndTransports.transports$, - forceOldJwtEndpointForLocalTransport$: useOldJwtEndpoint$, logger: logger, ownMembershipIdentity, }); @@ -628,6 +634,9 @@ export function createCallViewModel$( ), ); + const allConnections$ = scope.behavior( + connectionManager.connectionManagerData$.pipe(map((d) => d.value)), + ); const livekitRoomItems$ = scope.behavior( matrixLivekitMembers$.pipe( switchMap((members) => { @@ -724,6 +733,7 @@ export function createCallViewModel$( userId, participant satisfies TaggedParticipant as TaggedParticipant, // Widen the type safely connection$, + membership$.value, ], data: undefined, }; @@ -742,7 +752,14 @@ export function createCallViewModel$( // const participantId = membership$.value?.identity; for (let dup = 0; dup < 1 + duplicateTiles; dup++) { yield { - keys: [dup, userMediaId, userId, participant, connection$], + keys: [ + dup, + userMediaId, + userId, + participant, + connection$, + membership$.value, + ], data: undefined, }; } @@ -756,6 +773,7 @@ export function createCallViewModel$( userId, participant, connection$, + membership, ) => { const livekitRoom$ = scope.behavior( connection$.pipe(map((c) => c?.livekitRoom)), @@ -773,6 +791,7 @@ export function createCallViewModel$( scope, `${participantId}:${dup}`, userId, + membership, participant, options.encryptionSystem, livekitRoom$, @@ -1523,6 +1542,7 @@ export function createCallViewModel$( ), null, ), + allConnections$, participantCount$: participantCount$, handsRaised$: handsRaised$, reactions$: reactions$, diff --git a/src/state/CallViewModel/localMember/LocalMember.test.ts b/src/state/CallViewModel/localMember/LocalMember.test.ts index 8a7505eb..6b750785 100644 --- a/src/state/CallViewModel/localMember/LocalMember.test.ts +++ b/src/state/CallViewModel/localMember/LocalMember.test.ts @@ -39,6 +39,7 @@ import { constant } from "../../Behavior"; import { ConnectionManagerData } from "../remoteMembers/ConnectionManager"; import { ConnectionState, type Connection } from "../remoteMembers/Connection"; import { type Publisher } from "./Publisher"; +import { type LocalTransportWithSFUConfig } from "./LocalTransport"; const MATRIX_RTC_MODE = MatrixRTCMode.Legacy; const getUrlParams = vi.hoisted(() => vi.fn(() => ({}))); @@ -212,10 +213,11 @@ describe("LocalMembership", () => { it("throws error on missing RTC config error", () => { withTestScheduler(({ scope, hot, expectObservable }) => { - const localTransport$ = scope.behavior( - hot("1ms #", {}, new MatrixRTCTransportMissingError("domain.com")), - null, - ); + const localTransport$ = + scope.behavior( + hot("1ms #", {}, new MatrixRTCTransportMissingError("domain.com")), + null, + ); // we do not need any connection data since we want to fail before reaching that. const mockConnectionManager = { @@ -243,11 +245,23 @@ describe("LocalMembership", () => { }); const aTransport = { - livekit_service_url: "a", - } as LivekitTransport; + transport: { + livekit_service_url: "a", + } as LivekitTransport, + sfuConfig: { + url: "sfu-url", + jwt: "sfu-token", + }, + } as LocalTransportWithSFUConfig; const bTransport = { - livekit_service_url: "b", - } as LivekitTransport; + transport: { + livekit_service_url: "b", + } as LivekitTransport, + sfuConfig: { + url: "sfu-url", + jwt: "sfu-token", + }, + } as LocalTransportWithSFUConfig; const connectionTransportAConnected = { livekitRoom: mockLivekitRoom({ @@ -391,7 +405,8 @@ describe("LocalMembership", () => { const scope = new ObservableScope(); const connectionManagerData = new ConnectionManagerData(); - const localTransport$ = new BehaviorSubject(null); + const localTransport$ = + new BehaviorSubject(null); const connectionManagerData$ = new BehaviorSubject( new Epoch(connectionManagerData), ); @@ -468,7 +483,7 @@ describe("LocalMembership", () => { }); ( - connectionManagerData2.getConnectionForTransport(aTransport)! + connectionManagerData2.getConnectionForTransport(aTransport.transport)! .state$ as BehaviorSubject ).next(ConnectionState.LivekitConnected); expect(localMembership.localMemberState$.value).toStrictEqual({ diff --git a/src/state/CallViewModel/localMember/LocalMember.ts b/src/state/CallViewModel/localMember/LocalMember.ts index eb506132..4749e942 100644 --- a/src/state/CallViewModel/localMember/LocalMember.ts +++ b/src/state/CallViewModel/localMember/LocalMember.ts @@ -61,6 +61,7 @@ import { } from "../remoteMembers/Connection.ts"; import { type HomeserverConnected } from "./HomeserverConnected.ts"; import { and$ } from "../../../utils/observable.ts"; +import { type LocalTransportWithSFUConfig } from "./LocalTransport.ts"; export enum TransportState { /** Not even a transport is available to the LocalMembership */ @@ -126,7 +127,7 @@ interface Props { createPublisherFactory: (connection: Connection) => Publisher; joinMatrixRTC: (transport: LivekitTransport) => void; homeserverConnected: HomeserverConnected; - localTransport$: Behavior; + localTransport$: Behavior; matrixRTCSession: Pick< MatrixRTCSession, "updateCallIntent" | "leaveRoomSession" @@ -234,7 +235,9 @@ export const createLocalMembership$ = ({ return null; } - return connectionData.getConnectionForTransport(localTransport); + return connectionData.getConnectionForTransport( + localTransport.transport, + ); }), tap((connection) => { logger.info( @@ -533,7 +536,7 @@ export const createLocalMembership$ = ({ if (!shouldConnect) return; try { - joinMatrixRTC(transport); + joinMatrixRTC(transport.transport); } catch (error) { logger.error("Error entering RTC session", error); if (error instanceof Error) diff --git a/src/state/CallViewModel/localMember/LocalTransport.test.ts b/src/state/CallViewModel/localMember/LocalTransport.test.ts index 2199ca94..8da0db7f 100644 --- a/src/state/CallViewModel/localMember/LocalTransport.test.ts +++ b/src/state/CallViewModel/localMember/LocalTransport.test.ts @@ -19,7 +19,7 @@ import { BehaviorSubject, lastValueFrom } from "rxjs"; import fetchMock from "fetch-mock"; import { mockConfig, flushPromises, ownMemberMock } from "../../../utils/test"; -import { createLocalTransport$ } from "./LocalTransport"; +import { createLocalTransport$, JwtEndpointVersion } from "./LocalTransport"; import { constant } from "../../Behavior"; import { Epoch, ObservableScope } from "../../ObservableScope"; import { @@ -58,7 +58,7 @@ describe("LocalTransport", () => { getDeviceId: vi.fn(), }, ownMembershipIdentity: ownMemberMock, - useOldJwtEndpoint$: constant(false), + forceJwtEndpoint$: constant(JwtEndpointVersion.Legacy), delayId$: constant("delay_id_mock"), }); await flushPromises(); @@ -98,7 +98,7 @@ describe("LocalTransport", () => { getDeviceId: vi.fn(), }, ownMembershipIdentity: ownMemberMock, - useOldJwtEndpoint$: constant(false), + forceJwtEndpoint$: constant(JwtEndpointVersion.Legacy), delayId$: constant("delay_id_mock"), }); localTransport$.subscribe( @@ -140,7 +140,7 @@ describe("LocalTransport", () => { baseUrl: "https://lk.example.org", }, ownMembershipIdentity: ownMemberMock, - useOldJwtEndpoint$: constant(false), + forceJwtEndpoint$: constant(JwtEndpointVersion.Legacy), delayId$: constant("delay_id_mock"), }); @@ -186,7 +186,7 @@ describe("LocalTransport", () => { baseUrl: "https://lk.example.org", }, ownMembershipIdentity: ownMemberMock, - useOldJwtEndpoint$: constant(false), + forceJwtEndpoint$: constant(JwtEndpointVersion.Legacy), delayId$: constant("delay_id_mock"), }); @@ -216,7 +216,7 @@ describe("LocalTransport", () => { scope, roomId: "!example_room_id", useOldestMember$: constant(false), - useOldJwtEndpoint$: constant(false), + forceJwtEndpoint$: constant(JwtEndpointVersion.Legacy), delayId$: constant(null), memberships$: constant(new Epoch([])), client: { @@ -333,7 +333,7 @@ describe("LocalTransport", () => { ownMembershipIdentity: ownMemberMock, roomId: "!example_room_id", useOldestMember$: constant(false), - useOldJwtEndpoint$: constant(false), + forceJwtEndpoint$: constant(JwtEndpointVersion.Legacy), delayId$: constant(null), memberships$: constant(new Epoch([])), client: { diff --git a/src/state/CallViewModel/localMember/LocalTransport.ts b/src/state/CallViewModel/localMember/LocalTransport.ts index 8a6a750b..6e0e56a3 100644 --- a/src/state/CallViewModel/localMember/LocalTransport.ts +++ b/src/state/CallViewModel/localMember/LocalTransport.ts @@ -31,9 +31,11 @@ import { Config } from "../../../config/Config.ts"; import { FailToGetOpenIdToken, MatrixRTCTransportMissingError, + NoMatrix2AuthorizationService, } from "../../../utils/errors.ts"; import { getSFUConfigWithOpenID, + type SFUConfig, type OpenIDClientParts, } from "../../../livekit/openIDSFU.ts"; import { areLivekitTransportsEqual } from "../remoteMembers/MatrixLivekitMembers.ts"; @@ -57,10 +59,25 @@ interface Props { OpenIDClientParts; roomId: string; useOldestMember$: Behavior; - useOldJwtEndpoint$: Behavior; + forceJwtEndpoint$: Behavior; delayId$: Behavior; } +export enum JwtEndpointVersion { + Legacy = "legacy", + Matrix_2_0 = "matrix_2_0", +} + +export interface LocalTransportWithSFUConfig { + transport: LivekitTransport; + sfuConfig: SFUConfig; +} +export function isLocalTransportWithSFUConfig( + obj: LivekitTransport | LocalTransportWithSFUConfig, +): obj is LocalTransportWithSFUConfig { + return "transport" in obj && "sfuConfig" in obj; +} + /** * This class is responsible for managing the local transport. * "Which transport is the local member going to use" @@ -81,22 +98,40 @@ export const createLocalTransport$ = ({ client, roomId, useOldestMember$, - useOldJwtEndpoint$, + forceJwtEndpoint$, delayId$, -}: Props): Behavior => { +}: Props): Behavior => { /** * The transport over which we should be actively publishing our media. * undefined when not joined. */ const oldestMemberTransport$ = scope.behavior( - combineLatest([memberships$, useOldJwtEndpoint$]).pipe( - map(([memberships, forceOldJwtEndpoint]) => { + combineLatest([memberships$]).pipe( + map(([memberships]) => { const oldestMember = memberships.value[0]; const transport = oldestMember?.getTransport(memberships.value[0]); if (!transport) return null; return transport; }), first((t) => t != null && isLivekitTransport(t)), + switchMap((transport) => { + // Get the open jwt token to connect to the sfu + const computeLocalTransportWithSFUConfig = + async (): Promise => { + return { + transport, + sfuConfig: await getSFUConfigWithOpenID( + client, + ownMembershipIdentity, + transport.livekit_service_url, + roomId, + { forceJwtEndpoint: JwtEndpointVersion.Legacy }, + logger, + ), + }; + }; + return from(computeLocalTransportWithSFUConfig()); + }), ), null, ); @@ -108,19 +143,29 @@ export const createLocalTransport$ = ({ * @throws MatrixRTCTransportMissingError | FailToGetOpenIdToken */ const preferredTransport$ = scope.behavior( - combineLatest([customLivekitUrl.value$, delayId$, useOldJwtEndpoint$]).pipe( - switchMap(([customUrl, delayId, forceOldJwtEndpoint]) => - from( + // preferredTransport$ (used for multi sfu) needs to know if we are using the old or new + // jwt endpoint (`get_token` vs `sfu/get`) based on that the jwt endpoint will compute the rtcBackendIdentity + // differently. (sha(`${userId}|${deviceId}|${memberId}`) vs `${userId}|${deviceId}|${memberId}`) + // When using sticky events (we need to use the new endpoint). + combineLatest([customLivekitUrl.value$, delayId$, forceJwtEndpoint$]).pipe( + switchMap(([customUrl, delayId, forceEndpoint]) => { + logger.info( + "Creating preferred transport based on: ", + customUrl, + delayId, + forceEndpoint, + ); + return from( makeTransport( client, ownMembershipIdentity, roomId, customUrl, - forceOldJwtEndpoint, + forceEndpoint, delayId ?? undefined, ), - ), - ), + ); + }), ), null, ); @@ -139,7 +184,9 @@ export const createLocalTransport$ = ({ ? (oldestMemberTransport ?? preferredTransport) : preferredTransport, ), - distinctUntilChanged((t1, t2) => areLivekitTransportsEqual(t1, t2)), + distinctUntilChanged((t1, t2) => + areLivekitTransportsEqual(t1?.transport ?? null, t2?.transport ?? null), + ), ), ); }; @@ -161,7 +208,10 @@ const FOCI_WK_KEY = "org.matrix.msc4143.rtc_foci"; * @param membership The membership identity of the user. * @param roomId The ID of the room to be connected to. * @param urlFromDevSettings Override URL provided by the user's local config. - * @param forceOldJwtEndpoint Whether to force the old JWT endpoint (not hashing the backendIdentity). + * @param forceJwtEndpoint Whether to force a specific JWT endpoint + * - `Legacy` / `Matrix_2_0` + * - `get_token` / `sfu/get` + * - not hashing / hashing the backendIdentity * @param delayId the delay id passed to the jwt service. * * @returns A fully validated transport config. @@ -176,26 +226,33 @@ async function makeTransport( membership: CallMembershipIdentityParts, roomId: string, urlFromDevSettings: string | null, - forceOldJwtEndpoint: boolean, + forceJwtEndpoint: JwtEndpointVersion, delayId?: string, -): Promise { +): Promise { logger.trace("Searching for a preferred transport"); - async function doOpenIdAndJWTFromUrl(url: string): Promise { - const { livekitAlias } = await getSFUConfigWithOpenID( + async function doOpenIdAndJWTFromUrl( + url: string, + ): Promise { + const sfuConfig = await getSFUConfigWithOpenID( client, membership, url, - forceOldJwtEndpoint, roomId, - client.baseUrl, - delayId, + { + forceJwtEndpoint: forceJwtEndpoint, + delayEndpointBaseUrl: client.baseUrl, + delayId, + }, logger, ); return { - type: "livekit", - livekit_service_url: url, - livekit_alias: livekitAlias, + transport: { + type: "livekit", + livekit_service_url: url, + livekit_alias: sfuConfig.livekitAlias, + }, + sfuConfig, }; } // We will call `getSFUConfigWithOpenID` once per transport here as it's our @@ -217,7 +274,7 @@ async function makeTransport( async function getFirstUsableTransport( transports: Transport[], - ): Promise { + ): Promise { for (const potentialTransport of transports) { if (isLivekitTransportConfig(potentialTransport)) { try { @@ -226,8 +283,11 @@ async function makeTransport( potentialTransport.livekit_service_url, ); } catch (ex) { + // Explictly throw these if (ex instanceof FailToGetOpenIdToken) { - // Explictly throw these + throw ex; + } + if (ex instanceof NoMatrix2AuthorizationService) { throw ex; } logger.debug( diff --git a/src/state/CallViewModel/remoteMembers/Connection.test.ts b/src/state/CallViewModel/remoteMembers/Connection.test.ts index 0130a5ce..cc430645 100644 --- a/src/state/CallViewModel/remoteMembers/Connection.test.ts +++ b/src/state/CallViewModel/remoteMembers/Connection.test.ts @@ -114,6 +114,7 @@ function setupRemoteConnection(): Connection { client: client, transport: livekitFocus, scope: testScope, + ownMembershipIdentity: ownMemberMock, livekitRoomFactory: () => fakeLivekitRoom, }; @@ -138,7 +139,7 @@ function setupRemoteConnection(): Connection { return Promise.resolve(); }); - return new Connection(opts, logger, ownMemberMock); + return new Connection(opts, logger); } afterEach(() => { @@ -155,9 +156,10 @@ describe("Start connection states", () => { client: client, transport: livekitFocus, scope: testScope, + ownMembershipIdentity: ownMemberMock, livekitRoomFactory: () => fakeLivekitRoom, }; - const connection = new Connection(opts, logger, ownMemberMock); + const connection = new Connection(opts, logger); expect(connection.state$.getValue()).toEqual("Initialized"); }); @@ -170,10 +172,11 @@ describe("Start connection states", () => { client: client, transport: livekitFocus, scope: testScope, + ownMembershipIdentity: ownMemberMock, livekitRoomFactory: () => fakeLivekitRoom, }; - const connection = new Connection(opts, logger, ownMemberMock); + const connection = new Connection(opts, logger); const capturedStates: (ConnectionState | Error)[] = []; const s = connection.state$.subscribe((value) => { @@ -220,10 +223,11 @@ describe("Start connection states", () => { client: client, transport: livekitFocus, scope: testScope, + ownMembershipIdentity: ownMemberMock, livekitRoomFactory: () => fakeLivekitRoom, }; - const connection = new Connection(opts, logger, ownMemberMock); + const connection = new Connection(opts, logger); const capturedStates: (ConnectionState | Error)[] = []; const s = connection.state$.subscribe((value) => { @@ -277,10 +281,11 @@ describe("Start connection states", () => { client: client, transport: livekitFocus, scope: testScope, + ownMembershipIdentity: ownMemberMock, livekitRoomFactory: () => fakeLivekitRoom, }; - const connection = new Connection(opts, logger, ownMemberMock); + const connection = new Connection(opts, logger); const capturedStates: (ConnectionState | Error)[] = []; const s = connection.state$.subscribe((value) => { diff --git a/src/state/CallViewModel/remoteMembers/Connection.ts b/src/state/CallViewModel/remoteMembers/Connection.ts index 269fd4f1..f286b0cd 100644 --- a/src/state/CallViewModel/remoteMembers/Connection.ts +++ b/src/state/CallViewModel/remoteMembers/Connection.ts @@ -33,10 +33,21 @@ import { SFURoomCreationRestrictedError, UnknownCallError, } from "../../../utils/errors.ts"; +import { type JwtEndpointVersion } from "../localMember/LocalTransport.ts"; export interface ConnectionOpts { - /** Whether we always try to connect to this connection via the legacy jwt endpoint. (no hash identity) */ - forceOldJwtEndpoint?: boolean; + /** + * For the local transport we already do know the jwt token and url. We can reuse it. + * On top the local transport will send additional data to the jwt server to use delayed event delegation. + */ + existingSFUConfig?: SFUConfig; + /** + * For local connections that use the oldest member pattern. here we have not prefetched the sfuConfig + * and hence we need to let the connection do the jwt token fetching. + */ + forceJwtEndpoint?: JwtEndpointVersion; + /** The identity parts to use on this connection */ + ownMembershipIdentity: CallMembershipIdentityParts; /** The media transport to connect to. */ transport: LivekitTransport; /** The Matrix client to use for OpenID and SFU config requests. */ @@ -132,8 +143,10 @@ export class Connection { try { this._state$.next(ConnectionState.FetchingConfig); // We should already have this information after creating the localTransport. - // It would probably be better to forward this here. - const { url, jwt } = await this.getSFUConfigWithOpenID(); + // only call getSFUConfigWithOpenID for connections where we do not have a token yet. (existingJwtTokenData === undefined) + const { url, jwt } = + this.existingSFUConfig ?? + (await this.getSFUConfigForRemoteConnection()); // If we were stopped while fetching the config, don't proceed to connect if (this.stopped) return; @@ -189,17 +202,16 @@ export class Connection { } } - protected async getSFUConfigWithOpenID(): Promise { + protected async getSFUConfigForRemoteConnection(): Promise { + // This will only be called for sfu's where we do not publish ourselves. + // For the local connection we will use the existingJwtTokenData return await getSFUConfigWithOpenID( this.client, this.ownMembershipIdentity, this.transport.livekit_service_url, - this.forceOldJwtEndpoint, this.transport.livekit_alias, - // For the remote members we intentionally do not pass a delayEndpointBaseUrl. - undefined, - // and no delayId. - undefined, + // dont pass any custom opts for the subscribe only connections + {}, this.logger, ); } @@ -222,7 +234,8 @@ export class Connection { private readonly client: OpenIDClientParts; private readonly logger: Logger; - private readonly forceOldJwtEndpoint: boolean; + private readonly ownMembershipIdentity: CallMembershipIdentityParts; + private readonly existingSFUConfig?: SFUConfig; /** * Creates a new connection to a matrix RTC LiveKit backend. * @@ -230,12 +243,9 @@ export class Connection { * * @param logger - The logger to use. */ - public constructor( - opts: ConnectionOpts, - logger: Logger, - private ownMembershipIdentity: CallMembershipIdentityParts, - ) { - this.forceOldJwtEndpoint = opts.forceOldJwtEndpoint ?? false; + public constructor(opts: ConnectionOpts, logger: Logger) { + this.ownMembershipIdentity = opts.ownMembershipIdentity; + this.existingSFUConfig = opts.existingSFUConfig; this.logger = logger.getChild("[Connection]"); this.logger.info( `Creating new connection to ${opts.transport.livekit_service_url} ${opts.transport.livekit_alias}`, diff --git a/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts b/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts index 668538ac..aa20037c 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts @@ -20,7 +20,10 @@ import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc/LivekitTransp import { type ObservableScope } from "../../ObservableScope.ts"; import { Connection } from "./Connection.ts"; -import type { OpenIDClientParts } from "../../../livekit/openIDSFU.ts"; +import type { + OpenIDClientParts, + SFUConfig, +} from "../../../livekit/openIDSFU.ts"; import type { MediaDevices } from "../../MediaDevices.ts"; import type { Behavior } from "../../Behavior.ts"; import type { ProcessorState } from "../../../livekit/TrackProcessorContext.tsx"; @@ -29,11 +32,11 @@ import { defaultLiveKitOptions } from "../../../livekit/options.ts"; // TODO evaluate if this should be done like the Publisher Factory export interface ConnectionFactory { createConnection( - transport: LivekitTransport, scope: ObservableScope, + transport: LivekitTransport, ownMembershipIdentity: CallMembershipIdentityParts, logger: Logger, - forceOldJwtEndpoint?: boolean, + sfuConfig?: SFUConfig, ): Connection; } @@ -83,30 +86,30 @@ export class ECConnectionFactory implements ConnectionFactory { /** * - * @param transport The transport to use for this connection. * @param scope The observable scope (used for clean-up) + * @param transport The transport to use for this connection. * @param ownMembershipIdentity required to connect (using the jwt service) with the SFU. * @param logger The logger instance to use for this connection. - * @param forceOldJwtEndpoint Use the old JWT endpoint independent of what the sfu supports. + * @param sfuConfig optional config in case we already have a token for this connection. * @returns */ public createConnection( - transport: LivekitTransport, scope: ObservableScope, + transport: LivekitTransport, ownMembershipIdentity: CallMembershipIdentityParts, logger: Logger, - forceOldJwtEndpoint?: boolean, + sfuConfig?: SFUConfig, ): Connection { return new Connection( { + existingSFUConfig: sfuConfig, transport, client: this.client, scope: scope, livekitRoomFactory: this.livekitRoomFactory, - forceOldJwtEndpoint, + ownMembershipIdentity, }, logger, - ownMembershipIdentity, ); } } diff --git a/src/state/CallViewModel/remoteMembers/ConnectionManager.ts b/src/state/CallViewModel/remoteMembers/ConnectionManager.ts index 5c50f0cd..4295c5f2 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionManager.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionManager.ts @@ -12,12 +12,17 @@ import { type Logger } from "matrix-js-sdk/lib/logger"; import { type RemoteParticipant } from "livekit-client"; import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; -import { constant, type Behavior } from "../../Behavior.ts"; +import { type Behavior } from "../../Behavior.ts"; import { type Connection } from "./Connection.ts"; import { Epoch, type ObservableScope } from "../../ObservableScope.ts"; import { generateItemsWithEpoch } from "../../../utils/observable.ts"; import { areLivekitTransportsEqual } from "./MatrixLivekitMembers.ts"; import { type ConnectionFactory } from "./ConnectionFactory.ts"; +import { + isLocalTransportWithSFUConfig, + type LocalTransportWithSFUConfig, +} from "../localMember/LocalTransport.ts"; +import { type SFUConfig } from "../../../livekit/openIDSFU.ts"; export class ConnectionManagerData { private readonly store: Map< @@ -66,9 +71,9 @@ export class ConnectionManagerData { interface Props { scope: ObservableScope; connectionFactory: ConnectionFactory; - localTransport$: Behavior; + localTransport$: Behavior; remoteTransports$: Behavior>; - forceOldJwtEndpointForLocalTransport$?: Behavior; + logger: Logger; ownMembershipIdentity: CallMembershipIdentityParts; } @@ -87,7 +92,7 @@ export interface IConnectionManager { * @param props.remoteTransports$ - All other transports. The connection manager will create connections for each transport. (deduplicated with localTransport$) * @param props.ownMembershipIdentity - The own membership identity to use. * @param props.logger - The logger to use. - * @param props.forceOldJwtEndpointForLocalTransport$ - Use the old JWT endpoint independent of what the sfu supports. Only applies for localTransport$. + * * Each of these behaviors can be interpreted as subscribed list of transports. * @@ -103,7 +108,6 @@ export function createConnectionManager$({ connectionFactory, localTransport$, remoteTransports$, - forceOldJwtEndpointForLocalTransport$ = constant(false), logger: parentLogger, ownMembershipIdentity, }: Props): IConnectionManager { @@ -118,42 +122,35 @@ export function createConnectionManager$({ * It is build based on the list of subscribed transports (`transportsSubscriptions$`). * externally this is modified via `registerTransports()`. */ - const transportsWithJwtTag$ = scope.behavior( - combineLatest([ - remoteTransports$, - localTransport$, - forceOldJwtEndpointForLocalTransport$, - ]).pipe( - // combine local and remote transports into one transport array + const localAndRemoteTransports$: Behavior< + Epoch<(LivekitTransport | LocalTransportWithSFUConfig)[]> + > = scope.behavior( + combineLatest([remoteTransports$, localTransport$]).pipe( + // Combine local and remote transports into one transport array // and set the forceOldJwtEndpoint property on the local transport - map( - ([ - remoteTransports, - localTransport, - forceOldJwtEndpointForLocalTransport, - ]) => { - let localTransportAsArray: (LivekitTransport & { - forceOldJwtEndpoint: boolean; - })[] = []; - if (localTransport) { - localTransportAsArray = [ - { - ...localTransport, - forceOldJwtEndpoint: forceOldJwtEndpointForLocalTransport, - }, - ]; - } - return new Epoch( - removeDuplicateTransports([ - ...localTransportAsArray, - ...remoteTransports.value, - ]) as (LivekitTransport & { - forceOldJwtEndpoint?: boolean; - })[], - remoteTransports.epoch, - ); - }, - ), + map(([remoteTransports, localTransport]) => { + let localTransportAsArray: LocalTransportWithSFUConfig[] = []; + if (localTransport) { + localTransportAsArray = [localTransport]; + } + const dedupedRemote = removeDuplicateTransports(remoteTransports.value); + const remoteWithoutLocal = dedupedRemote.filter( + (transport) => + !localTransportAsArray.find((l) => + areLivekitTransportsEqual(l.transport, transport), + ), + ); + logger.debug( + "remoteWithoutLocal", + remoteWithoutLocal, + "localTransportAsArray", + localTransportAsArray, + ); + return new Epoch( + [...localTransportAsArray, ...remoteWithoutLocal], + remoteTransports.epoch, + ); + }), ), ); @@ -161,33 +158,51 @@ export function createConnectionManager$({ * Connections for each transport in use by one or more session members. */ const connections$ = scope.behavior( - transportsWithJwtTag$.pipe( + localAndRemoteTransports$.pipe( generateItemsWithEpoch( function* (transports) { - for (const transport of transports) - yield { - keys: [ - transport.livekit_service_url, - transport.livekit_alias, - transport.forceOldJwtEndpoint, - ], - data: undefined, - }; + for (const transportWithOrWithoutSfuConfig of transports) { + if ( + isLocalTransportWithSFUConfig(transportWithOrWithoutSfuConfig) + ) { + // This is the local transport only the `LocalTransportWithSFUConfig` has a `sfuConfig` field + const { transport, sfuConfig } = transportWithOrWithoutSfuConfig; + yield { + keys: [ + transport.livekit_service_url, + transport.livekit_alias, + sfuConfig, + ], + data: undefined, + }; + } else { + const transport = transportWithOrWithoutSfuConfig; + yield { + keys: [ + transport.livekit_service_url, + transport.livekit_alias, + undefined as undefined | SFUConfig, + ], + data: undefined, + }; + } + } }, - (scope, _data$, serviceUrl, alias, forceOldJwtEndpoint) => { + (scope, _data$, serviceUrl, alias, sfuConfig) => { logger.debug( - `Creating connection to ${serviceUrl} (${alias}, forceOldJwtEndpoint: ${forceOldJwtEndpoint})`, + `Creating connection to ${serviceUrl} (${alias}, withSfuConfig (local connection?): ${JSON.stringify(sfuConfig) ?? "no config->remote connection"})`, ); + const connection = connectionFactory.createConnection( + scope, { type: "livekit", livekit_service_url: serviceUrl, livekit_alias: alias, }, - scope, ownMembershipIdentity, logger, - forceOldJwtEndpoint, + sfuConfig, ); // Start the connection immediately // Use connection state to track connection progress diff --git a/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts b/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts index f28bd158..b56a17f7 100644 --- a/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts +++ b/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts @@ -77,8 +77,8 @@ describe("ECConnectionFactory - Audio inputs options", () => { noise, ); ecConnectionFactory.createConnection( - exampleTransport, testScope, + exampleTransport, ownMemberMock, logger, ); @@ -123,8 +123,8 @@ describe("ECConnectionFactory - ControlledAudioDevice", () => { false, ); ecConnectionFactory.createConnection( - exampleTransport, testScope, + exampleTransport, ownMemberMock, logger, ); diff --git a/src/state/MediaViewModel.ts b/src/state/MediaViewModel.ts index 9888d6bf..6b020057 100644 --- a/src/state/MediaViewModel.ts +++ b/src/state/MediaViewModel.ts @@ -44,6 +44,7 @@ import { throttleTime, distinctUntilChanged, } from "rxjs"; +import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc"; import { alwaysShowSelf } from "../settings/settings"; import { showConnectionStats } from "../settings/settings"; @@ -257,6 +258,7 @@ abstract class BaseMediaViewModel { * The Matrix user to which this media belongs. */ public readonly userId: string, + public readonly rtcMembership: CallMembership, // We don't necessarily have a participant if a user connects via MatrixRTC but not (yet) through // livekit. protected readonly participant$: Observable< @@ -402,10 +404,13 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel { */ public readonly cropVideo$: Behavior = this._cropVideo$; + public readonly rtcBackendIdentity = this.rtcMembership.rtcBackendIdentity; + public constructor( scope: ObservableScope, id: string, userId: string, + rtcMembership: CallMembership, participant$: Observable, encryptionSystem: EncryptionSystem, livekitRoom$: Behavior, @@ -419,6 +424,7 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel { scope, id, userId, + rtcMembership, participant$, encryptionSystem, Track.Source.Microphone, @@ -544,6 +550,7 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel { scope: ObservableScope, id: string, userId: string, + rtcMembership: CallMembership, participant$: Behavior, encryptionSystem: EncryptionSystem, livekitRoom$: Behavior, @@ -558,6 +565,7 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel { scope, id, userId, + rtcMembership, participant$, encryptionSystem, livekitRoom$, @@ -671,6 +679,7 @@ export class RemoteUserMediaViewModel extends BaseUserMediaViewModel { scope: ObservableScope, id: string, userId: string, + rtcMembership: CallMembership, participant$: Observable, encryptionSystem: EncryptionSystem, livekitRoom$: Behavior, @@ -685,6 +694,7 @@ export class RemoteUserMediaViewModel extends BaseUserMediaViewModel { scope, id, userId, + rtcMembership, participant$, encryptionSystem, livekitRoom$, @@ -772,6 +782,7 @@ export class ScreenShareViewModel extends BaseMediaViewModel { scope: ObservableScope, id: string, userId: string, + rtcMembership: CallMembership, participant$: Observable, encryptionSystem: EncryptionSystem, livekitRoom$: Behavior, @@ -785,6 +796,7 @@ export class ScreenShareViewModel extends BaseMediaViewModel { scope, id, userId, + rtcMembership, participant$, encryptionSystem, Track.Source.ScreenShareAudio, diff --git a/src/state/ScreenShare.ts b/src/state/ScreenShare.ts index 0a241cdf..67f97a93 100644 --- a/src/state/ScreenShare.ts +++ b/src/state/ScreenShare.ts @@ -10,6 +10,7 @@ import { type RemoteParticipant, type Room as LivekitRoom, } from "livekit-client"; +import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc/CallMembership"; import { type ObservableScope } from "./ObservableScope.ts"; import { ScreenShareViewModel } from "./MediaViewModel.ts"; @@ -28,6 +29,7 @@ export class ScreenShare { private readonly scope: ObservableScope, id: string, userId: string, + rtcMember: CallMembership, participant: LocalParticipant | RemoteParticipant, encryptionSystem: EncryptionSystem, livekitRoom$: Behavior, @@ -40,6 +42,7 @@ export class ScreenShare { this.scope, id, userId, + rtcMember, of(participant), encryptionSystem, livekitRoom$, diff --git a/src/state/UserMedia.ts b/src/state/UserMedia.ts index 690870e6..6435fb17 100644 --- a/src/state/UserMedia.ts +++ b/src/state/UserMedia.ts @@ -13,6 +13,7 @@ import { type Room as LivekitRoom, } from "livekit-client"; import { observeParticipantEvents } from "@livekit/components-core"; +import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc/CallMembership"; import { type ObservableScope } from "./ObservableScope.ts"; import { @@ -75,6 +76,7 @@ export class UserMedia { this.scope, this.id, this.userId, + this.rtcMembership, this.participant.value$, this.encryptionSystem, this.livekitRoom$, @@ -89,6 +91,7 @@ export class UserMedia { this.scope, this.id, this.userId, + this.rtcMembership, this.participant.value$, this.encryptionSystem, this.livekitRoom$, @@ -140,6 +143,7 @@ export class UserMedia { scope, `${this.id}:${key}`, this.userId, + this.rtcMembership, p, this.encryptionSystem, this.livekitRoom$, @@ -191,6 +195,8 @@ export class UserMedia { private readonly scope: ObservableScope, public readonly id: string, private readonly userId: string, + // TODO evaluate if this should just be the rtcBackendIdentity + private readonly rtcMembership: CallMembership, private readonly participant: TaggedParticipant, private readonly encryptionSystem: EncryptionSystem, private readonly livekitRoom$: Behavior, diff --git a/src/tile/GridTile.tsx b/src/tile/GridTile.tsx index 2f750c50..92262f05 100644 --- a/src/tile/GridTile.tsx +++ b/src/tile/GridTile.tsx @@ -113,6 +113,7 @@ const UserMediaTile: FC = ({ }, [vm], ); + const rtcBackendIdentity = vm.rtcBackendIdentity; const handRaised = useBehavior(vm.handRaised$); const reaction = useBehavior(vm.reaction$); @@ -200,6 +201,7 @@ const UserMediaTile: FC = ({ focusUrl={focusUrl} audioStreamStats={audioStreamStats} videoStreamStats={videoStreamStats} + rtcBackendIdentity={rtcBackendIdentity} {...props} /> ); diff --git a/src/tile/MediaView.tsx b/src/tile/MediaView.tsx index 8bb38d94..7847a30a 100644 --- a/src/tile/MediaView.tsx +++ b/src/tile/MediaView.tsx @@ -46,6 +46,7 @@ interface Props extends ComponentProps { waitingForMedia?: boolean; audioStreamStats?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats; videoStreamStats?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats; + rtcBackendIdentity?: string; // The focus url, mainly for debugging purposes focusUrl?: string; } @@ -74,6 +75,7 @@ export const MediaView: FC = ({ waitingForMedia, audioStreamStats, videoStreamStats, + rtcBackendIdentity, focusUrl, ...props }) => { @@ -135,11 +137,14 @@ export const MediaView: FC = ({ )} {(audioStreamStats || videoStreamStats) && ( - + <> + + )} {/* TODO: Bring this back once encryption status is less broken */} {/*encryptionStatus !== EncryptionStatus.Okay && ( diff --git a/src/utils/errors.ts b/src/utils/errors.ts index cddf90de..20a282e6 100644 --- a/src/utils/errors.ts +++ b/src/utils/errors.ts @@ -19,6 +19,7 @@ export enum ErrorCode { INSUFFICIENT_CAPACITY_ERROR = "INSUFFICIENT_CAPACITY_ERROR", E2EE_NOT_SUPPORTED = "E2EE_NOT_SUPPORTED", OPEN_ID_ERROR = "OPEN_ID_ERROR", + NO_MATRIX_2_AUTHORIZATION_SERVICE = "NO_MATRIX_2_0_AUTHORIZATION_SERVICE", SFU_ERROR = "SFU_ERROR", UNKNOWN_ERROR = "UNKNOWN_ERROR", } @@ -171,6 +172,23 @@ export class FailToGetOpenIdToken extends ElementCallError { } } +export class NoMatrix2AuthorizationService extends ElementCallError { + /** + * Creates an instance of NoMatrix2_0AuthorizationService. + * @param error - The underlying error that caused the failure. + */ + public constructor(error: Error) { + super( + t("error.generic"), + ErrorCode.NO_MATRIX_2_AUTHORIZATION_SERVICE, + ErrorCategory.CONFIGURATION_ISSUE, + t("error.no_matrix_2_authorization_service"), + // Properly set it as a cause for a better reporting on sentry + error, + ); + } +} + /** * Error indicating a failure to start publishing on a LiveKit connection. */ diff --git a/src/utils/test.ts b/src/utils/test.ts index d24ad130..9f9ae4b7 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -334,6 +334,7 @@ export function createLocalMedia( testScope(), "local", member.userId, + rtcMember, constant(localParticipant), { kind: E2eeType.PER_PARTICIPANT, @@ -379,6 +380,7 @@ export function createRemoteMedia( testScope(), "remote", member.userId, + rtcMember, constant(participant), { kind: E2eeType.PER_PARTICIPANT, diff --git a/yarn.lock b/yarn.lock index 54832a25..b32cf3c8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8246,7 +8246,7 @@ __metadata: livekit-client: "npm:^2.13.0" lodash-es: "npm:^4.17.21" loglevel: "npm:^1.9.1" - matrix-js-sdk: "matrix-org/matrix-js-sdk#develop" + matrix-js-sdk: "matrix-org/matrix-js-sdk#ff5fab7722e54be2d69a380cd6182ecb262d7859" matrix-widget-api: "npm:^1.14.0" node-stdlib-browser: "npm:^1.3.1" normalize.css: "npm:^8.0.1" @@ -11333,9 +11333,9 @@ __metadata: languageName: node linkType: hard -"matrix-js-sdk@matrix-org/matrix-js-sdk#develop": - version: 39.4.0 - resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=4b89fb23c54aaf7826bd127d8fa21cc7bb87688f" +"matrix-js-sdk@portal:/Users/timo/Projects/matrix-js-sdk::locator=element-call%40workspace%3A.": + version: 0.0.0-use.local + resolution: "matrix-js-sdk@portal:/Users/timo/Projects/matrix-js-sdk::locator=element-call%40workspace%3A." dependencies: "@babel/runtime": "npm:^7.12.5" "@matrix-org/matrix-sdk-crypto-wasm": "npm:^16.0.0" @@ -11351,9 +11351,8 @@ __metadata: sdp-transform: "npm:^3.0.0" unhomoglyph: "npm:^1.0.6" uuid: "npm:13" - checksum: 10c0/bc7443bf67822e9bc7b8e531b4e61e6ebac41c2fd8047ac0567456c264ae0d1911fbef6e437d312a3adeead86cd5e7134944e3fd73d28002777618bc0ebaa1ca languageName: node - linkType: hard + linkType: soft "matrix-widget-api@npm:^1.14.0": version: 1.15.0