diff --git a/package.json b/package.json index 2611587a..346c12cf 100644 --- a/package.json +++ b/package.json @@ -109,7 +109,7 @@ "livekit-client": "^2.13.0", "lodash-es": "^4.17.21", "loglevel": "^1.9.1", - "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#head=toger5/use-membershipID-for-session-state-events&commit=8cea2c05839ebcaa90945504a453b9b1e1092fc4", + "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#head=toger5/use-membershipID-for-session-state-events&commit=6f0815930a008eff8f86e6e5748d447be0e7c25e", "matrix-widget-api": "^1.14.0", "normalize.css": "^8.0.1", "observable-hooks": "^4.2.3", diff --git a/src/e2ee/matrixKeyProvider.ts b/src/e2ee/matrixKeyProvider.ts index 962c9bc6..a9b0865f 100644 --- a/src/e2ee/matrixKeyProvider.ts +++ b/src/e2ee/matrixKeyProvider.ts @@ -11,6 +11,7 @@ import { MatrixRTCSessionEvent, } from "matrix-js-sdk/lib/matrixrtc"; import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; +import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; const logger = rootLogger.getChild("[MatrixKeyProvider]"); export class MatrixKeyProvider extends BaseKeyProvider { @@ -43,6 +44,7 @@ export class MatrixKeyProvider extends BaseKeyProvider { private onEncryptionKeyChanged = ( encryptionKey: Uint8Array, encryptionKeyIndex: number, + membershipParts: CallMembershipIdentityParts, rtcBackendIdentity: string, ): void => { crypto.subtle @@ -59,12 +61,12 @@ export class MatrixKeyProvider extends BaseKeyProvider { ); logger.debug( - `Sent new key to livekit room=${this.rtcSession?.room.roomId} participantId=${membershipFull.rtcBackendIdentity} (before hash: ${membershipFull.userId}) encryptionKeyIndex=${encryptionKeyIndex}`, + `Sent new key to livekit room=${this.rtcSession?.room.roomId} participantId=${rtcBackendIdentity} (before hash: ${membershipParts.userId}) encryptionKeyIndex=${encryptionKeyIndex}`, ); }, (e) => { logger.error( - `Failed to create key material from buffer for livekit room=${this.rtcSession?.room.roomId} participantId before hash=${membershipFull.userId} encryptionKeyIndex=${encryptionKeyIndex}`, + `Failed to create key material from buffer for livekit room=${this.rtcSession?.room.roomId} participantId before hash=${membershipParts.userId} encryptionKeyIndex=${encryptionKeyIndex}`, e, ); }, diff --git a/src/livekit/openIDSFU.ts b/src/livekit/openIDSFU.ts index f07bb035..aaf07615 100644 --- a/src/livekit/openIDSFU.ts +++ b/src/livekit/openIDSFU.ts @@ -6,8 +6,8 @@ Please see LICENSE in the repository root for full details. */ import { type IOpenIDToken, type MatrixClient } from "matrix-js-sdk"; -import { logger } from "matrix-js-sdk/lib/logger"; import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; +import { type Logger } from "matrix-js-sdk/lib/logger"; import { FailToGetOpenIdToken } from "../utils/errors"; import { doNetworkOperationWithRetry } from "../utils/matrix"; @@ -28,8 +28,17 @@ export type OpenIDClientParts = Pick< * to the matrix RTC backend in order to get acces to the SFU. * It has built-in retry for calls to the homeserver with a backoff policy. * @param client + * @param membership * @param serviceUrl - * @param matrixRoomId + * @param forceOldEndpoint This will use the old jwt endpoint which will create the rtc backend identity based on string concatination + * instead of a hash. + * This function by default uses whatever is possible with the current jwt service installed next to the SFU. + * For remote connections this does not matter, since we will not publish there we can rely on the newest option. + * For our own connection we can only use the hashed version if we also send the new matrix2.0 sticky events. + * @param livekitRoomAlias + * @param delayEndpointBaseUrl + * @param delayId + * @param logger * @returns Object containing the token information * @throws FailToGetOpenIdToken */ @@ -37,10 +46,11 @@ export async function getSFUConfigWithOpenID( client: OpenIDClientParts, membership: CallMembershipIdentityParts, serviceUrl: string, + forceOldJwtEndpoint: boolean, livekitRoomAlias: string, - matrix2jwt: boolean, delayEndpointBaseUrl?: string, delayId?: string, + logger?: Logger, ): Promise { let openIdToken: IOpenIDToken; try { @@ -52,26 +62,35 @@ export async function getSFUConfigWithOpenID( error instanceof Error ? error : new Error("Unknown error"), ); } - logger.debug("Got openID token", openIdToken); + logger?.debug("Got openID token", openIdToken); - logger.info(`Trying to get JWT for focus ${serviceUrl}...`); + logger?.info(`Trying to get JWT for focus ${serviceUrl}...`); const args: [CallMembershipIdentityParts, string, string, IOpenIDToken] = [ membership, serviceUrl, livekitRoomAlias, openIdToken, ]; - if (matrix2jwt) { + try { + // we do not want to try the old endpoint, since we are not sending the new matrix2.0 sticky events (no hashed identity in the event) + if (forceOldJwtEndpoint) throw new Error("Force old jwt endpoint"); + if (!delayId) + throw new Error("No delayId, Won't try matrix 2.0 jwt endpoint."); + const sfuConfig = await getLiveKitJWTWithDelayDelegation( ...args, delayEndpointBaseUrl, delayId, ); - logger.info(`Got JWT from call's active focus URL.`); + logger?.info(`Got JWT from call's active focus URL.`); return sfuConfig; - } else { + } catch (e) { + logger?.warn( + `Failed fetching jwt with matrix 2.0 endpoint (retry with legacy)`, + e, + ); const sfuConfig = await getLiveKitJWT(...args); - logger.info(`Got JWT from call's active focus URL.`); + logger?.info(`Got JWT from call's active focus URL.`); return sfuConfig; } } diff --git a/src/room/CallEventAudioRenderer.test.tsx b/src/room/CallEventAudioRenderer.test.tsx index 38f56b14..733346eb 100644 --- a/src/room/CallEventAudioRenderer.test.tsx +++ b/src/room/CallEventAudioRenderer.test.tsx @@ -23,7 +23,6 @@ import { import { exampleTransport, - mockComputeLivekitParticipantIdentity$, mockMatrixRoomMember, mockRtcMembership, } from "../utils/test"; @@ -48,13 +47,6 @@ vitest.mock("../rtcSessionHelpers", async (importOriginal) => ({ ...(await importOriginal()), makeTransport: (): [LivekitTransport] => [exampleTransport], })); -vitest.mock( - import("../state/CallViewModel/remoteMembers/LivekitParticipantIdentity.ts"), - async (importOriginal) => ({ - ...(await importOriginal()), - computeLivekitParticipantIdentity$: mockComputeLivekitParticipantIdentity$, - }), -); afterEach(() => { vitest.clearAllMocks(); diff --git a/src/room/InCallView.test.tsx b/src/room/InCallView.test.tsx index cd0af547..8ac4bccb 100644 --- a/src/room/InCallView.test.tsx +++ b/src/room/InCallView.test.tsx @@ -23,7 +23,6 @@ import { TooltipProvider } from "@vector-im/compound-web"; import { RoomContext, useLocalParticipant } from "@livekit/components-react"; import { - mockComputeLivekitParticipantIdentity$, mockLivekitRoom, mockLocalParticipant, mockMatrixRoom, @@ -62,13 +61,6 @@ vi.mock("../livekit/MatrixAudioRenderer"); vi.mock("react-use-measure", () => ({ default: (): [() => void, object] => [(): void => {}, {}], })); -vi.mock( - import("../state/CallViewModel/remoteMembers/LivekitParticipantIdentity.ts"), - async (importOriginal) => ({ - ...(await importOriginal()), - computeLivekitParticipantIdentity$: mockComputeLivekitParticipantIdentity$, - }), -); const localRtcMember = mockRtcMembership("@carol:example.org", "CCCC"); const localParticipant = mockLocalParticipant({ diff --git a/src/settings/settings.ts b/src/settings/settings.ts index f85e1414..33408fd9 100644 --- a/src/settings/settings.ts +++ b/src/settings/settings.ts @@ -127,6 +127,12 @@ export const alwaysShowIphoneEarpiece = new Setting( export enum MatrixRTCMode { Legacy = "legacy", Compatibil = "compatibil", + /** This implies using + * - sticky events + * - hashed RTC backend identity + * - the new endpoint for the jwt token on the local membership (remote memberships will always try the new jwt endpoint first -> then the legacy one) + * - use the hashed identity for the local membership + */ Matrix_2_0 = "matrix_2_0", } diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index 23c58268..922a390e 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -415,17 +415,18 @@ export function createCallViewModel$( const ownMembershipIdentity: CallMembershipIdentityParts = { userId, deviceId, + // TODO look into this!!! memberId: `${userId}:${deviceId}`, }; + const useOldJwtEndpoint$ = scope.behavior( + options.matrixRTCMode$.pipe(map((v) => v !== MatrixRTCMode.Matrix_2_0)), + ); const localTransport$ = createLocalTransport$({ scope: scope, memberships$: memberships$, ownMembershipIdentity, client, - useMatrix2$: scope.behavior( - options.matrixRTCMode$.pipe(map((v) => v === MatrixRTCMode.Matrix_2_0)), - ), delayId$: scope.behavior( ( fromEvent( @@ -436,6 +437,7 @@ export function createCallViewModel$( matrixRTCSession.delayId ?? null, ), roomId: matrixRoom.roomId, + useOldJwtEndpoint$, useOldestMember$: scope.behavior( options.matrixRTCMode$.pipe(map((v) => v === MatrixRTCMode.Legacy)), ), @@ -455,29 +457,19 @@ export function createCallViewModel$( const connectionManager = createConnectionManager$({ scope: scope, connectionFactory: connectionFactory, - inputTransports$: scope.behavior( - combineLatest( - [ - localTransport$.pipe( - catchError((e: unknown) => { - logger.info( - "dont pass local transport to createConnectionManager$. localTransport$ threw an error", - e, - ); - return of(null); - }), - ), - membershipsAndTransports.transports$, - ], - (localTransport, transports) => { - const localTransportAsArray = localTransport ? [localTransport] : []; - return transports.mapInner((transports) => [ - ...localTransportAsArray, - ...transports, - ]); - }, + localTransport$: scope.behavior( + localTransport$.pipe( + catchError((e: unknown) => { + logger.info( + "could not pass local transport to createConnectionManager$. localTransport$ threw an error", + e, + ); + return of(null); + }), ), ), + remoteTransports$: membershipsAndTransports.transports$, + forceOldJwtEndpointForLocalTransport$: useOldJwtEndpoint$, logger: logger, ownMembershipIdentity, }); diff --git a/src/state/CallViewModel/localMember/LocalTransport.test.ts b/src/state/CallViewModel/localMember/LocalTransport.test.ts index ba030757..e7df6e33 100644 --- a/src/state/CallViewModel/localMember/LocalTransport.test.ts +++ b/src/state/CallViewModel/localMember/LocalTransport.test.ts @@ -38,7 +38,7 @@ describe("LocalTransport", () => { getDeviceId: vi.fn(), }, ownMembershipIdentity: ownMemberMock, - useMatrix2$: constant(false), + useOldJwtEndpoint$: constant(false), delayId$: constant("delay_id_mock"), }); await flushPromises(); @@ -76,7 +76,7 @@ describe("LocalTransport", () => { getDeviceId: vi.fn(), }, ownMembershipIdentity: ownMemberMock, - useMatrix2$: constant(false), + useOldJwtEndpoint$: constant(false), delayId$: constant("delay_id_mock"), }); localTransport$.subscribe( @@ -116,7 +116,7 @@ describe("LocalTransport", () => { baseUrl: "https://lk.example.org", }, ownMembershipIdentity: ownMemberMock, - useMatrix2$: constant(false), + useOldJwtEndpoint$: constant(false), delayId$: constant("delay_id_mock"), }); @@ -155,7 +155,7 @@ describe("LocalTransport", () => { baseUrl: "https://lk.example.org", }, ownMembershipIdentity: ownMemberMock, - useMatrix2$: constant(false), + useOldJwtEndpoint$: constant(false), delayId$: constant("delay_id_mock"), }); diff --git a/src/state/CallViewModel/localMember/LocalTransport.ts b/src/state/CallViewModel/localMember/LocalTransport.ts index 6c3e1cd0..b44cf967 100644 --- a/src/state/CallViewModel/localMember/LocalTransport.ts +++ b/src/state/CallViewModel/localMember/LocalTransport.ts @@ -35,7 +35,6 @@ import { } from "../../../livekit/openIDSFU.ts"; import { areLivekitTransportsEqual } from "../remoteMembers/MatrixLivekitMembers.ts"; import { customLivekitUrl } from "../../../settings/settings.ts"; -import { type LivekitTransportWithVersion } from "../remoteMembers/ConnectionManager.ts"; const logger = rootLogger.getChild("[LocalTransport]"); @@ -51,7 +50,7 @@ interface Props { client: Pick & OpenIDClientParts; roomId: string; useOldestMember$: Behavior; - useMatrix2$: Behavior; + useOldJwtEndpoint$: Behavior; delayId$: Behavior; } @@ -62,6 +61,11 @@ interface Props { * @prop useOldestMember Whether to use the same transport as the oldest member. * This will only update once the first oldest member appears. Will not recompute if the oldest member leaves. * + * @prop useOldJwtEndpoint$ Whether to set forceOldJwtEndpoint the use the old JWT endpoint. + * This is used when the connection manager needs to know if it has to use the legacy endpoint which implies a string concatenated rtcBackendIdentity. + * (which is expected for non sticky event based rtc member events) + * @returns Behavior<(LivekitTransport & { forceOldJwtEndpoint: boolean }) | null> The `forceOldJwtEndpoint` field is added to let the connection EncryptionManager + * know that this transport is for the local member and it IS RELEVANT which jwt endpoint to use. (for the local member transport, we need to know which jwt endpoint to use) * @throws MatrixRTCTransportMissingError | FailToGetOpenIdToken */ export const createLocalTransport$ = ({ @@ -71,21 +75,20 @@ export const createLocalTransport$ = ({ client, roomId, useOldestMember$, - useMatrix2$, + useOldJwtEndpoint$, delayId$, -}: Props): Behavior => { +}: Props): Behavior => { /** * The transport over which we should be actively publishing our media. * undefined when not joined. */ const oldestMemberTransport$ = scope.behavior( - memberships$.pipe( - map((memberships) => { + combineLatest([memberships$, useOldJwtEndpoint$]).pipe( + map(([memberships, forceOldJwtEndpoint]) => { const oldestMember = memberships.value[0]; - const t = oldestMember?.getTransport(memberships.value[0]); - if (!t) return null; - // Here we will use the matrix2 information from the oldest member transport. - return { ...t, useMatrix2: oldestMember.kind === "rtc" }; + const transport = oldestMember?.getTransport(memberships.value[0]); + if (!transport) return null; + return { ...transport, forceOldJwtEndpoint }; }), first((t) => t != null && isLivekitTransport(t)), ), @@ -98,24 +101,23 @@ export const createLocalTransport$ = ({ * * @throws MatrixRTCTransportMissingError | FailToGetOpenIdToken */ - const preferredTransport$: Behavior = - scope.behavior( - combineLatest([customLivekitUrl.value$, useMatrix2$, delayId$]).pipe( - switchMap(([customUrl, useMatrix2, delayId]) => - from( - makeTransport( - client, - ownMembershipIdentity, - roomId, - customUrl, - useMatrix2, - delayId ?? undefined, - ), + const preferredTransport$: Behavior = scope.behavior( + combineLatest([customLivekitUrl.value$, delayId$, useOldJwtEndpoint$]).pipe( + switchMap(([customUrl, delayId, forceOldJwtEndpoint]) => + from( + makeTransport( + client, + ownMembershipIdentity, + roomId, + customUrl, + forceOldJwtEndpoint, + delayId ?? undefined, ), ), ), - null, - ); + ), + null, + ); /** * The chosen transport we should advertise in our MatrixRTC membership. @@ -131,7 +133,7 @@ export const createLocalTransport$ = ({ ? (oldestMemberTransport ?? preferredTransport) : preferredTransport, ), - distinctUntilChanged(areLivekitTransportsEqual), + distinctUntilChanged((t1, t2) => areLivekitTransportsEqual(t1, t2)), ), ); }; @@ -142,6 +144,8 @@ const FOCI_WK_KEY = "org.matrix.msc4143.rtc_foci"; * * @param client * @param roomId + * @param useMatrix2 This implies using the matrix2 jwt endpoint (including delayed event delegation of the jwt token) + * @param delayId * @returns * @throws MatrixRTCTransportMissingError | FailToGetOpenIdToken */ @@ -150,9 +154,9 @@ async function makeTransport( membership: CallMembershipIdentityParts, roomId: string, urlFromDevSettings: string | null, - matrix2jwt = false, + forceOldJwtEndpoint: boolean, delayId?: string, -): Promise { +): Promise { let transport: LivekitTransport | undefined; logger.trace("Searching for a preferred transport"); //TODO refactor this to use the jwt service returned alias. @@ -209,11 +213,12 @@ async function makeTransport( client, membership, transport.livekit_service_url, + forceOldJwtEndpoint, transport.livekit_alias, - matrix2jwt, client.baseUrl, delayId, + logger, ); - return { ...transport, useMatrix2: matrix2jwt }; + return { ...transport, forceOldJwtEndpoint }; } diff --git a/src/state/CallViewModel/remoteMembers/Connection.test.ts b/src/state/CallViewModel/remoteMembers/Connection.test.ts index 533f451a..57578641 100644 --- a/src/state/CallViewModel/remoteMembers/Connection.test.ts +++ b/src/state/CallViewModel/remoteMembers/Connection.test.ts @@ -26,6 +26,7 @@ import fetchMock from "fetch-mock"; import EventEmitter from "events"; import { type IOpenIDToken } from "matrix-js-sdk"; import { logger } from "matrix-js-sdk/lib/logger"; +import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc/LivekitTransport"; import { Connection, @@ -39,7 +40,6 @@ import { FailToGetOpenIdToken, } from "../../../utils/errors.ts"; import { mockRemoteParticipant, ownMemberMock } from "../../../utils/test.ts"; -import { type LivekitTransportWithVersion } from "./ConnectionManager.ts"; let testScope: ObservableScope; @@ -50,7 +50,7 @@ let fakeLivekitRoom: MockedObject; let localParticipantEventEmiter: EventEmitter; let fakeLocalParticipant: MockedObject; -const livekitFocus: LivekitTransportWithVersion = { +const livekitFocus: LivekitTransport = { livekit_alias: "!roomID:example.org", livekit_service_url: "https://matrix-rtc.example.org/livekit/jwt", type: "livekit", diff --git a/src/state/CallViewModel/remoteMembers/Connection.ts b/src/state/CallViewModel/remoteMembers/Connection.ts index d32bbce6..e070b56b 100644 --- a/src/state/CallViewModel/remoteMembers/Connection.ts +++ b/src/state/CallViewModel/remoteMembers/Connection.ts @@ -35,8 +35,10 @@ import { } from "../../../utils/errors.ts"; export interface ConnectionOpts { + /** Whether we always try to connect to this connection via the legacy jwt endpoint. (no hash identity) */ + forceOldJwtEndpoint?: boolean; /** The media transport to connect to. */ - transport: LivekitTransport & { useMatrix2: boolean }; + transport: LivekitTransport; /** The Matrix client to use for OpenID and SFU config requests. */ client: OpenIDClientParts; /** The observable scope to use for this connection. */ @@ -89,7 +91,7 @@ export class Connection { /** * The media transport to connect to. */ - public readonly transport: LivekitTransport & { useMatrix2: boolean }; + public readonly transport: LivekitTransport; public readonly livekitRoom: LivekitRoom; @@ -192,16 +194,14 @@ export class Connection { this.client, this.ownMembershipIdentity, this.transport.livekit_service_url, + this.forceOldJwtEndpoint, this.transport.livekit_alias, - this.transport.useMatrix2, + // For the remote members we intentionally do not pass a delayEndpointBaseUrl. + undefined, + // and no delayId. + undefined, + this.logger, ); - // client: OpenIDClientParts, - // membership: CallMembershipIdentityParts, - // serviceUrl: string, - // livekitRoomAlias: string, - // matrix2jwt: boolean, - // delayEndpointBaseUrl?: string, - // delayId?: string, } /** @@ -222,7 +222,7 @@ export class Connection { private readonly client: OpenIDClientParts; private readonly logger: Logger; - + private readonly forceOldJwtEndpoint: boolean; /** * Creates a new connection to a matrix RTC LiveKit backend. * @@ -235,6 +235,7 @@ export class Connection { logger: Logger, private ownMembershipIdentity: CallMembershipIdentityParts, ) { + this.forceOldJwtEndpoint = opts.forceOldJwtEndpoint ?? false; this.logger = logger.getChild("[Connection]"); this.logger.info( `[Connection] Creating new connection to ${opts.transport.livekit_service_url} ${opts.transport.livekit_alias}`, diff --git a/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts b/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts index 82a1a78a..94652d16 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts @@ -15,6 +15,7 @@ import { import { type Logger } from "matrix-js-sdk/lib/logger"; import E2EEWorker from "livekit-client/e2ee-worker?worker"; import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; +import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc/LivekitTransport"; import { type ObservableScope } from "../../ObservableScope.ts"; import { Connection } from "./Connection.ts"; @@ -23,15 +24,15 @@ import type { MediaDevices } from "../../MediaDevices.ts"; import type { Behavior } from "../../Behavior.ts"; import type { ProcessorState } from "../../../livekit/TrackProcessorContext.tsx"; import { defaultLiveKitOptions } from "../../../livekit/options.ts"; -import { type LivekitTransportWithVersion } from "./ConnectionManager.ts"; // TODO evaluate if this should be done like the Publisher Factory export interface ConnectionFactory { createConnection( - transport: LivekitTransportWithVersion, + transport: LivekitTransport, scope: ObservableScope, logger: Logger, ownMembershipIdentity: CallMembershipIdentityParts, + forceOldJwtEndpoint?: boolean, ): Connection; } @@ -88,10 +89,11 @@ export class ECConnectionFactory implements ConnectionFactory { * @returns */ public createConnection( - transport: LivekitTransportWithVersion, + transport: LivekitTransport, scope: ObservableScope, logger: Logger, ownMembershipIdentity: CallMembershipIdentityParts, + forceOldJwtEndpoint?: boolean, ): Connection { return new Connection( { @@ -99,6 +101,7 @@ export class ECConnectionFactory implements ConnectionFactory { client: this.client, scope: scope, livekitRoomFactory: this.livekitRoomFactory, + forceOldJwtEndpoint, }, logger, ownMembershipIdentity, diff --git a/src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts b/src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts index 4ab91646..088bf41b 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts @@ -14,29 +14,26 @@ import { logger } from "matrix-js-sdk/lib/logger"; import { Epoch, mapEpoch, ObservableScope } from "../../ObservableScope.ts"; import { createConnectionManager$, - type LivekitTransportWithVersion, type ConnectionManagerData, } from "./ConnectionManager.ts"; import { type ConnectionFactory } from "./ConnectionFactory.ts"; import { type Connection } from "./Connection.ts"; import { ownMemberMock, withTestScheduler } from "../../../utils/test.ts"; import { areLivekitTransportsEqual } from "./MatrixLivekitMembers.ts"; -import { type Behavior } from "../../Behavior.ts"; +import { constant, type Behavior } from "../../Behavior.ts"; // Some test constants -const TRANSPORT_1: LivekitTransportWithVersion = { +const TRANSPORT_1: LivekitTransport = { type: "livekit", livekit_service_url: "https://lk.example.org", livekit_alias: "!alias:example.org", - useMatrix2: false, }; -const TRANSPORT_2: LivekitTransportWithVersion = { +const TRANSPORT_2: LivekitTransport = { type: "livekit", livekit_service_url: "https://lk.sample.com", livekit_alias: "!alias:sample.com", - useMatrix2: false, }; let fakeConnectionFactory: ConnectionFactory; @@ -79,7 +76,8 @@ describe("connections$ stream", () => { const { connectionManagerData$ } = createConnectionManager$({ scope: testScope, connectionFactory: fakeConnectionFactory, - inputTransports$: behavior("a", { + localTransport$: constant(null), + remoteTransports$: behavior("a", { a: new Epoch([TRANSPORT_1, TRANSPORT_2], 0), }), logger: logger, @@ -119,7 +117,8 @@ describe("connections$ stream", () => { const { connectionManagerData$ } = createConnectionManager$({ scope: testScope, connectionFactory: fakeConnectionFactory, - inputTransports$: behavior("abcdef", { + localTransport$: constant(null), + remoteTransports$: behavior("abcdef", { a: new Epoch([TRANSPORT_1], 0), b: new Epoch([TRANSPORT_1], 1), c: new Epoch([TRANSPORT_1], 2), @@ -165,7 +164,8 @@ describe("connections$ stream", () => { const { connectionManagerData$ } = createConnectionManager$({ scope: testScope, connectionFactory: fakeConnectionFactory, - inputTransports$: behavior("abc", { + localTransport$: constant(null), + remoteTransports$: behavior("abc", { a: new Epoch([TRANSPORT_1], 0), b: new Epoch([TRANSPORT_1, TRANSPORT_2], 1), c: new Epoch([TRANSPORT_1], 2), @@ -281,7 +281,8 @@ describe("connectionManagerData$ stream", () => { const { connectionManagerData$ } = createConnectionManager$({ scope: testScope, connectionFactory: fakeConnectionFactory, - inputTransports$: behavior("a", { + localTransport$: constant(null), + remoteTransports$: behavior("a", { a: new Epoch([TRANSPORT_1, TRANSPORT_2], 0), }), logger, diff --git a/src/state/CallViewModel/remoteMembers/ConnectionManager.ts b/src/state/CallViewModel/remoteMembers/ConnectionManager.ts index d5852d84..6101f79b 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionManager.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionManager.ts @@ -7,22 +7,18 @@ Please see LICENSE in the repository root for full details. */ import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; -import { combineLatest, map, of, switchMap, tap } from "rxjs"; +import { combineLatest, map, of, switchMap } from "rxjs"; import { type Logger } from "matrix-js-sdk/lib/logger"; import { type RemoteParticipant } from "livekit-client"; import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; -import { type Behavior } from "../../Behavior.ts"; +import { constant, type Behavior } from "../../Behavior.ts"; import { type Connection } from "./Connection.ts"; import { Epoch, type ObservableScope } from "../../ObservableScope.ts"; import { generateItemsWithEpoch } from "../../../utils/observable.ts"; import { areLivekitTransportsEqual } from "./MatrixLivekitMembers.ts"; import { type ConnectionFactory } from "./ConnectionFactory.ts"; -export type LivekitTransportWithVersion = LivekitTransport & { - useMatrix2: boolean; -}; - export class ConnectionManagerData { private readonly store: Map = new Map(); @@ -64,7 +60,9 @@ export class ConnectionManagerData { interface Props { scope: ObservableScope; connectionFactory: ConnectionFactory; - inputTransports$: Behavior>; + localTransport$: Behavior; + remoteTransports$: Behavior>; + forceOldJwtEndpointForLocalTransport$?: Behavior; logger: Logger; ownMembershipIdentity: CallMembershipIdentityParts; } @@ -91,13 +89,29 @@ export interface IConnectionManager { export function createConnectionManager$({ scope, connectionFactory, - inputTransports$, + localTransport$, + remoteTransports$, + forceOldJwtEndpointForLocalTransport$ = constant(false), logger: parentLogger, ownMembershipIdentity, }: Props): IConnectionManager { const logger = parentLogger.getChild("[ConnectionManager]"); // TODO logger: only construct one logger from the client and make it compatible via a EC specific sing + const allInputTransports$ = combineLatest([ + localTransport$, + remoteTransports$, + ]).pipe( + map(([localTransport, transports]) => { + const localTransportAsArray = localTransport ? [localTransport] : []; + return transports.mapInner((transports) => [ + ...localTransportAsArray, + ...transports, + ]); + }), + map((transports) => transports.mapInner(removeDuplicateTransports)), + ); + /** * All transports currently managed by the ConnectionManager. * @@ -106,14 +120,32 @@ export function createConnectionManager$({ * It is build based on the list of subscribed transports (`transportsSubscriptions$`). * externally this is modified via `registerTransports()`. */ - const transports$ = scope.behavior( - inputTransports$.pipe( - map((transports) => transports.mapInner(removeDuplicateTransports)), - tap(({ value: transports }) => { - logger.trace( - `Managing transports: ${transports.map((t) => t.livekit_service_url).join(", ")}`, - ); - }), + const transportsWithJwtTag$ = scope.behavior( + combineLatest([ + allInputTransports$, + localTransport$, + forceOldJwtEndpointForLocalTransport$, + ]).pipe( + map( + ([ + transports, + localTransport, + forceOldJwtEndpointForLocalTransport, + ]) => { + // nmodify only the local transport with forceOldJwtEndpointForLocalTransport + const index = transports.value.findIndex((t) => + areLivekitTransportsEqual(localTransport, t), + ); + transports.value[index].forceOldJwtEndpoint = + forceOldJwtEndpointForLocalTransport; + logger.trace( + `Managing transports: ${transports.value.map((t) => t.livekit_service_url).join(", ")}`, + ); + return transports as Epoch< + (LivekitTransport & { forceOldJwtEndpoint?: boolean })[] + >; + }, + ), ), ); @@ -121,7 +153,7 @@ export function createConnectionManager$({ * Connections for each transport in use by one or more session members. */ const connections$ = scope.behavior( - transports$.pipe( + transportsWithJwtTag$.pipe( generateItemsWithEpoch( function* (transports) { for (const transport of transports) @@ -129,23 +161,23 @@ export function createConnectionManager$({ keys: [ transport.livekit_service_url, transport.livekit_alias, - transport.useMatrix2, + transport.forceOldJwtEndpoint, ], data: undefined, }; }, - (scope, _data$, serviceUrl, alias, useMatrix2) => { + (scope, _data$, serviceUrl, alias, forceOldJwtEndpoint) => { logger.debug(`Creating connection to ${serviceUrl} (${alias})`); const connection = connectionFactory.createConnection( { type: "livekit", livekit_service_url: serviceUrl, livekit_alias: alias, - useMatrix2, }, scope, logger, ownMembershipIdentity, + forceOldJwtEndpoint, ); // Start the connection immediately // Use connection state to track connection progress diff --git a/src/state/CallViewModel/remoteMembers/integration.test.ts b/src/state/CallViewModel/remoteMembers/integration.test.ts index 00062c60..df10c861 100644 --- a/src/state/CallViewModel/remoteMembers/integration.test.ts +++ b/src/state/CallViewModel/remoteMembers/integration.test.ts @@ -13,11 +13,7 @@ import fetchMock from "fetch-mock"; import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; import { logger } from "matrix-js-sdk/lib/logger"; -import { - type Epoch, - ObservableScope, - trackEpoch, -} from "../../ObservableScope.ts"; +import { type Epoch, ObservableScope, trackEpoch } from "../../ObservableScope.ts"; import { ECConnectionFactory } from "./ConnectionFactory.ts"; import { type OpenIDClientParts } from "../../../livekit/openIDSFU.ts"; import { @@ -34,6 +30,7 @@ import { } from "./MatrixLivekitMembers.ts"; import { createConnectionManager$ } from "./ConnectionManager.ts"; import { membershipsAndTransports$ } from "../../SessionBehaviors.ts"; +import { constant } from "../../Behavior.ts"; // Test the integration of ConnectionManager and MatrixLivekitMerger @@ -121,7 +118,8 @@ test("bob, carl, then bob joining no tracks yet", () => { const connectionManager = createConnectionManager$({ scope: testScope, connectionFactory: ecConnectionFactory, - inputTransports$: membershipsAndTransports.transports$, + localTransport$: constant(null), + remoteTransports$: membershipsAndTransports.transports$, logger: logger, ownMembershipIdentity: ownMemberMock, }); diff --git a/src/state/SessionBehaviors.ts b/src/state/SessionBehaviors.ts index b61d2fe6..e174a1cc 100644 --- a/src/state/SessionBehaviors.ts +++ b/src/state/SessionBehaviors.ts @@ -8,6 +8,7 @@ Please see LICENSE in the repository root for full details. import { type CallMembership, isLivekitTransport, + type LivekitTransport, type MatrixRTCSession, MatrixRTCSessionEvent, } from "matrix-js-sdk/lib/matrixrtc"; @@ -20,18 +21,15 @@ import { type ObservableScope, } from "./ObservableScope"; import { type Behavior } from "./Behavior"; -import { type LivekitTransportWithVersion } from "./CallViewModel/remoteMembers/ConnectionManager"; export const membershipsAndTransports$ = ( scope: ObservableScope, memberships$: Behavior>, ): { membershipsWithTransport$: Behavior< - Epoch< - { membership: CallMembership; transport?: LivekitTransportWithVersion }[] - > + Epoch<{ membership: CallMembership; transport?: LivekitTransport }[]> >; - transports$: Behavior>; + transports$: Behavior>; } => { /** * Lists the transports used by ourselves, plus all other MatrixRTC session @@ -49,12 +47,7 @@ export const membershipsAndTransports$ = ( const transport = membership.getTransport(oldestMembership); return { membership, - transport: isLivekitTransport(transport) - ? { - ...transport, - useMatrix2: membership.kind === "rtc", - } - : undefined, + transport: isLivekitTransport(transport) ? transport : undefined, }; }); }), diff --git a/src/utils/test.ts b/src/utils/test.ts index a860bde0..02277af0 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -25,6 +25,7 @@ import { import { CallMembership, type LivekitFocusSelection, + type LivekitTransport, type MatrixRTCSession, MatrixRTCSessionEvent, type MatrixRTCSessionEventHandlerMap, @@ -66,7 +67,6 @@ import { type MediaDevices } from "../state/MediaDevices"; import { type Behavior, constant } from "../state/Behavior"; import { ObservableScope } from "../state/ObservableScope"; import { MuteStates } from "../state/MuteStates"; -import { type LivekitTransportWithVersion } from "../state/CallViewModel/remoteMembers/ConnectionManager"; export function withFakeTimers(continuation: () => void): void { vi.useFakeTimers(); @@ -197,11 +197,10 @@ export function mockEmitter(): EmitterMock { }; } -export const exampleTransport: LivekitTransportWithVersion = { +export const exampleTransport: LivekitTransport = { type: "livekit", livekit_service_url: "https://lk.example.org", livekit_alias: "!alias:example.org", - useMatrix2: false, }; export function mockCallMembership( diff --git a/yarn.lock b/yarn.lock index 0ddaad61..83555527 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7533,7 +7533,7 @@ __metadata: livekit-client: "npm:^2.13.0" lodash-es: "npm:^4.17.21" loglevel: "npm:^1.9.1" - matrix-js-sdk: "github:matrix-org/matrix-js-sdk#head=toger5/use-membershipID-for-session-state-events&commit=8cea2c05839ebcaa90945504a453b9b1e1092fc4" + matrix-js-sdk: "github:matrix-org/matrix-js-sdk#head=toger5/use-membershipID-for-session-state-events&commit=6f0815930a008eff8f86e6e5748d447be0e7c25e" matrix-widget-api: "npm:^1.14.0" normalize.css: "npm:^8.0.1" observable-hooks: "npm:^4.2.3" @@ -10338,9 +10338,9 @@ __metadata: languageName: node linkType: hard -"matrix-js-sdk@github:matrix-org/matrix-js-sdk#head=toger5/use-membershipID-for-session-state-events&commit=8cea2c05839ebcaa90945504a453b9b1e1092fc4": +"matrix-js-sdk@github:matrix-org/matrix-js-sdk#head=toger5/use-membershipID-for-session-state-events&commit=6f0815930a008eff8f86e6e5748d447be0e7c25e": version: 39.4.0 - resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=8cea2c05839ebcaa90945504a453b9b1e1092fc4" + resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=6f0815930a008eff8f86e6e5748d447be0e7c25e" dependencies: "@babel/runtime": "npm:^7.12.5" "@matrix-org/matrix-sdk-crypto-wasm": "npm:^16.0.0" @@ -10356,7 +10356,7 @@ __metadata: sdp-transform: "npm:^3.0.0" unhomoglyph: "npm:^1.0.6" uuid: "npm:13" - checksum: 10c0/2375dd3d9191f78fe589b0d3170f3da7792ed469a81d3ba3cd12f4915fd33a859f8af3491edb9cf0cdaa1f881a3ea7c1bf7539e850ad0360ec9981271f462c81 + checksum: 10c0/a5a904a79f3660d1f6fe217195e662adf82af4a445681e47f292772d9d4d63ce60aaca209f40c41e2d659bee2b17cd5b3345bbad77795032057f2c0e3129cc77 languageName: node linkType: hard