diff --git a/dev-backend-docker-compose.yml b/dev-backend-docker-compose.yml index 5e955831..0efefd07 100644 --- a/dev-backend-docker-compose.yml +++ b/dev-backend-docker-compose.yml @@ -3,7 +3,7 @@ networks: services: auth-service: - image: ghcr.io/element-hq/lk-jwt-service:latest-ci + image: ghcr.io/element-hq/lk-jwt-service:pr_139 pull_policy: always hostname: auth-server environment: @@ -25,7 +25,7 @@ services: - ecbackend auth-service-1: - image: ghcr.io/element-hq/lk-jwt-service:latest-ci + image: ghcr.io/element-hq/lk-jwt-service:pr_139 pull_policy: always hostname: auth-server-1 environment: diff --git a/locales/en/app.json b/locales/en/app.json index 1ff066ea..c70a0a49 100644 --- a/locales/en/app.json +++ b/locales/en/app.json @@ -116,6 +116,7 @@ "matrix_rtc_transport_missing": "The server is not configured to work with {{brand}}. Please contact your server admin (Domain: {{domain}}, Error Code: {{ errorCode }}).", "membership_manager": "Membership Manager Error", "membership_manager_description": "The Membership Manager had to shut down. This is caused by many consequtive failed network requests.", + "no_matrix_2_authorization_service": "Your authorization service for you media server (SFU) is not on the newest version", "open_elsewhere": "Opened in another tab", "open_elsewhere_description": "{{brand}} has been opened in another tab. If that doesn't sound right, try reloading the page.", "room_creation_restricted": "Failed to create call", diff --git a/package.json b/package.json index 944fc43b..6a8d3c76 100644 --- a/package.json +++ b/package.json @@ -104,7 +104,7 @@ "livekit-client": "^2.13.0", "lodash-es": "^4.17.21", "loglevel": "^1.9.1", - "matrix-js-sdk": "matrix-org/matrix-js-sdk#develop", + "matrix-js-sdk": "matrix-org/matrix-js-sdk#ff5fab7722e54be2d69a380cd6182ecb262d7859", "matrix-widget-api": "^1.14.0", "node-stdlib-browser": "^1.3.1", "normalize.css": "^8.0.1", diff --git a/src/RTCConnectionStats.tsx b/src/RTCConnectionStats.tsx index d51089cf..f62a1e4d 100644 --- a/src/RTCConnectionStats.tsx +++ b/src/RTCConnectionStats.tsx @@ -20,6 +20,7 @@ interface Props { audio?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats; video?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats; focusUrl?: string; + rtcBackendIdentity?: string; } const extractDomain = (url: string): string => { @@ -37,6 +38,7 @@ export const RTCConnectionStats: FC = ({ audio, video, focusUrl, + rtcBackendIdentity, ...rest }) => { const [showModal, setShowModal] = useState(false); @@ -71,6 +73,9 @@ export const RTCConnectionStats: FC = ({ + + rtcBackendIdentity:{rtcBackendIdentity} + {focusUrl && (
diff --git a/src/livekit/openIDSFU.test.ts b/src/livekit/openIDSFU.test.ts index 5f286958..20820748 100644 --- a/src/livekit/openIDSFU.test.ts +++ b/src/livekit/openIDSFU.test.ts @@ -46,7 +46,6 @@ describe("getSFUConfigWithOpenID", () => { matrixClient, ownMemberMock, "https://sfu.example.org", - false, "!example_room_id", ); expect(config).toEqual({ @@ -70,7 +69,6 @@ describe("getSFUConfigWithOpenID", () => { matrixClient, ownMemberMock, "https://sfu.example.org", - false, "!example_room_id", ); } catch (ex) { @@ -101,10 +99,11 @@ describe("getSFUConfigWithOpenID", () => { matrixClient, ownMemberMock, "https://sfu.example.org", - false, "!example_room_id", - "https://matrix.homeserverserver.org", - "mock_delay_id", + { + delayEndpointBaseUrl: "https://matrix.homeserverserver.org", + delayId: "mock_delay_id", + }, ); } catch (ex) { expect((ex as Error).message).toEqual( @@ -154,10 +153,11 @@ describe("getSFUConfigWithOpenID", () => { matrixClient, ownMemberMock, "https://sfu.example.org", - false, "!example_room_id", - "https://matrix.homeserverserver.org", - "mock_delay_id", + { + delayEndpointBaseUrl: "https://matrix.homeserverserver.org", + delayId: "mock_delay_id", + }, ); } catch (ex) { expect((ex as Error).message).toEqual( @@ -203,7 +203,6 @@ describe("getSFUConfigWithOpenID", () => { matrixClient, ownMemberMock, "https://sfu.example.org", - false, "!example_room_id", ); expect(config).toEqual({ diff --git a/src/livekit/openIDSFU.ts b/src/livekit/openIDSFU.ts index df1d02a5..a5a8adc5 100644 --- a/src/livekit/openIDSFU.ts +++ b/src/livekit/openIDSFU.ts @@ -13,9 +13,13 @@ import { import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; import { type Logger } from "matrix-js-sdk/lib/logger"; -import { FailToGetOpenIdToken } from "../utils/errors"; +import { + FailToGetOpenIdToken, + NoMatrix2AuthorizationService, +} from "../utils/errors"; import { doNetworkOperationWithRetry } from "../utils/matrix"; import { Config } from "../config/Config"; +import { JwtEndpointVersion } from "../state/CallViewModel/localMember/LocalTransport"; /** * Configuration and access tokens provided by the SFU on successful authentication. @@ -73,14 +77,15 @@ export type OpenIDClientParts = Pick< * @param client The Matrix client * @param membership Our own membership identity parts used to send to jwt service. * @param serviceUrl The URL of the livekit SFU service - * @param forceOldJwtEndpoint This will use the old jwt endpoint which will create the rtc backend identity based on string concatination + * @param roomId The room id used in the jwt request. This is NOT the livekit_alias. The jwt service will provide the alias. It maps matrix room ids <-> Livekit aliases. + * @param opts Additional options to modify which endpoint with which data will be used to aquire the jwt token. + * @param opts.forceJwtEndpoint This will use the old jwt endpoint which will create the rtc backend identity based on string concatination * instead of a hash. * This function by default uses whatever is possible with the current jwt service installed next to the SFU. * For remote connections this does not matter, since we will not publish there we can rely on the newest option. * For our own connection we can only use the hashed version if we also send the new matrix2.0 sticky events. - * @param roomId The room id used in the jwt request. This is NOT the livekit_alias. The jwt service will provide the alias. It maps matrix room ids <-> Livekit aliases. - * @param delayEndpointBaseUrl The URL of the matrix homeserver. - * @param delayId The delay id used for the jwt service to manage. + * @param opts.delayEndpointBaseUrl The URL of the matrix homeserver. + * @param opts.delayId The delay id used for the jwt service to manage. * @param logger optional logger. * @returns Object containing the token information * @throws FailToGetOpenIdToken @@ -89,10 +94,12 @@ export async function getSFUConfigWithOpenID( client: OpenIDClientParts, membership: CallMembershipIdentityParts, serviceUrl: string, - forceOldJwtEndpoint: boolean, roomId: string, - delayEndpointBaseUrl?: string, - delayId?: string, + opts?: { + forceJwtEndpoint?: JwtEndpointVersion; + delayEndpointBaseUrl?: string; + delayId?: string; + }, logger?: Logger, ): Promise { let openIdToken: IOpenIDToken; @@ -113,15 +120,21 @@ export async function getSFUConfigWithOpenID( // If forceOldJwtEndpoint is set we indicate that we do not want to try the new endpoint, // since we are not sending the new matrix2.0 sticky events (no hashed identity in the event) - if (forceOldJwtEndpoint === false) { + if ( + // we do not force anything. Try with new first (remote connections) + !opts?.forceJwtEndpoint || + // we do force the matrix2.0 endpoint + (opts?.forceJwtEndpoint && + opts?.forceJwtEndpoint === JwtEndpointVersion.Matrix_2_0) + ) { try { sfuConfig = await getLiveKitJWTWithDelayDelegation( membership, serviceUrl, roomId, openIdToken, - delayEndpointBaseUrl, - delayId, + opts?.delayEndpointBaseUrl, + opts?.delayId, ); logger?.info(`Got JWT from call's active focus URL.`); } catch (e) { @@ -137,12 +150,16 @@ export async function getSFUConfigWithOpenID( `(not going to try with legacy endpoint: forceOldJwtEndpoint is set to false, we did not get a not supported error from the sfu)`, e, ); + // Make this throw a hard error in case we force the matrix2.0 endpoint. + if (opts?.forceJwtEndpoint === JwtEndpointVersion.Matrix_2_0) + throw new NoMatrix2AuthorizationService(e as Error); } } } // DEPRECATED - // Either forceOldJwtEndpoint = true or getLiveKitJWTWithDelayDelegation throws -> reset sfuConfig = undefined + // here we either have a sfuConfig or we alredy exited becuause of `if (opts?.forceEndpoint === MatrixRTCMode.Matrix_2_0) throw e;` + // The only case we can get into this if is, if `opts?.forceEndpoint !== MatrixRTCMode.Matrix_2_0` if (sfuConfig === undefined) { sfuConfig = await getLiveKitJWT( membership.deviceId, @@ -235,7 +252,7 @@ export async function getLiveKitJWTWithDelayDelegation( let bodyDalayParts = {}; // Also check for empty string - if (delayId && delayEndpointBaseUrl && false) { + if (delayId && delayEndpointBaseUrl) { const delayTimeoutMs = Config.get().matrix_rtc_session?.delayed_leave_event_delay_ms ?? 1000; bodyDalayParts = { diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index fdbd4461..54f189f0 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -113,10 +113,8 @@ const logger = rootLogger.getChild("[InCallView]"); const maxTapDurationMs = 400; -export interface ActiveCallProps extends Omit< - InCallViewProps, - "vm" | "livekitRoom" | "connState" -> { +export interface ActiveCallProps + extends Omit { e2eeSystem: EncryptionSystem; // TODO refactor those reasons into an enum onLeft: ( @@ -798,6 +796,8 @@ export const InCallView: FC = ({
); + const allConnections = useBehavior(vm.allConnections$); + return (
= ({ onDismiss={closeSettings} tab={settingsTab} onTabChange={setSettingsTab} - // TODO expose correct data to setttings modal - livekitRooms={[]} + livekitRooms={allConnections + .getConnections() + .map((connectionItem) => ({ + room: connectionItem.livekitRoom, + // TODO compute is local or tag it in the livekit room items already + isLocal: undefined, + url: connectionItem.transport.livekit_service_url, + }))} /> )} diff --git a/src/settings/DeveloperSettingsTab.module.css b/src/settings/DeveloperSettingsTab.module.css index 7b83eb6c..29f4211b 100644 --- a/src/settings/DeveloperSettingsTab.module.css +++ b/src/settings/DeveloperSettingsTab.module.css @@ -8,3 +8,14 @@ Please see LICENSE in the repository root for full details. pre { font-size: var(--font-size-micro); } + +.livekit_room_box { + border: 3px solid var(--cpd-color-bg-subtle-secondary); + border-radius: var(--cpd-space-8x); + padding: var(--cpd-space-4x); + margin-bottom: var(--cpd-space-4x); + margin-top: var(--cpd-space-4x); + li { + font-size: var(--font-size-micro); + } +} diff --git a/src/settings/DeveloperSettingsTab.tsx b/src/settings/DeveloperSettingsTab.tsx index c88eadf0..a187e4b5 100644 --- a/src/settings/DeveloperSettingsTab.tsx +++ b/src/settings/DeveloperSettingsTab.tsx @@ -29,6 +29,7 @@ import { Label, RadioControl, } from "@vector-im/compound-web"; +import { type Room as LivekitRoom } from "livekit-client"; import { FieldRow, InputField } from "../input/Input"; import { @@ -42,7 +43,6 @@ import { customLivekitUrl as customLivekitUrlSetting, MatrixRTCMode, } from "./settings"; -import type { Room as LivekitRoom } from "livekit-client"; import styles from "./DeveloperSettingsTab.module.css"; import { useUrlParams } from "../UrlParams"; @@ -304,12 +304,12 @@ export const DeveloperSettingsTab: FC = ({ {livekitRooms?.map((livekitRoom) => ( - <> -

+
+

{t("developer_mode.livekit_sfu", { url: livekitRoom.url || "unknown", })} -

+

{livekitRoom.isLocal &&

ws-url: {localSfuUrl?.href}

}

{t("developer_mode.livekit_server_info")}( @@ -321,7 +321,19 @@ export const DeveloperSettingsTab: FC = ({ : "undefined"} {livekitRoom.room.metadata} - +

Local Participant

+ 
+            {livekitRoom.room.localParticipant.identity}
+
+

Remote Participants

+
    + {Array.from(livekitRoom.room.remoteParticipants.keys()).map( + (id) => ( +
  • {id}
    • + ), + )} +
+
))}

{t("developer_mode.environment_variables")}

{JSON.stringify(env, null, 2)}
diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index c75b3ec4..8fc2d7e6 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -108,13 +108,19 @@ import { enterRTCSession, TransportState, } from "./localMember/LocalMember.ts"; -import { createLocalTransport$ } from "./localMember/LocalTransport.ts"; +import { + createLocalTransport$, + JwtEndpointVersion, +} from "./localMember/LocalTransport.ts"; import { createMemberships$, membershipsAndTransports$, } from "../SessionBehaviors.ts"; import { ECConnectionFactory } from "./remoteMembers/ConnectionFactory.ts"; -import { createConnectionManager$ } from "./remoteMembers/ConnectionManager.ts"; +import { + type ConnectionManagerData, + createConnectionManager$, +} from "./remoteMembers/ConnectionManager.ts"; import { createMatrixLivekitMembers$, type TaggedParticipant, @@ -263,6 +269,7 @@ export interface CallViewModel { * multiple devices. */ participantCount$: Behavior; + allConnections$: Behavior; /** Participants sorted by livekit room so they can be used in the audio rendering */ livekitRoomItems$: Behavior; userMedia$: Behavior; @@ -428,14 +435,6 @@ export function createCallViewModel$( memberId: `${userId}:${deviceId}`, }; - const useOldJwtEndpoint$ = scope.behavior( - matrixRTCMode$.pipe( - map( - (v) => v === MatrixRTCMode.Legacy || v === MatrixRTCMode.Compatibility, - ), - ), - ); - const localTransport$ = createLocalTransport$({ scope: scope, memberships$: memberships$, @@ -451,7 +450,15 @@ export function createCallViewModel$( matrixRTCSession.delayId ?? null, ), roomId: matrixRoom.roomId, - useOldJwtEndpoint$, + forceJwtEndpoint$: scope.behavior( + matrixRTCMode$.pipe( + map((v) => + v === MatrixRTCMode.Matrix_2_0 + ? JwtEndpointVersion.Matrix_2_0 + : JwtEndpointVersion.Legacy, + ), + ), + ), useOldestMember$: scope.behavior( matrixRTCMode$.pipe(map((v) => v === MatrixRTCMode.Legacy)), ), @@ -483,7 +490,6 @@ export function createCallViewModel$( ), ), remoteTransports$: membershipsAndTransports.transports$, - forceOldJwtEndpointForLocalTransport$: useOldJwtEndpoint$, logger: logger, ownMembershipIdentity, }); @@ -628,6 +634,9 @@ export function createCallViewModel$( ), ); + const allConnections$ = scope.behavior( + connectionManager.connectionManagerData$.pipe(map((d) => d.value)), + ); const livekitRoomItems$ = scope.behavior( matrixLivekitMembers$.pipe( switchMap((members) => { @@ -724,6 +733,7 @@ export function createCallViewModel$( userId, participant satisfies TaggedParticipant as TaggedParticipant, // Widen the type safely connection$, + membership$.value, ], data: undefined, }; @@ -742,7 +752,14 @@ export function createCallViewModel$( // const participantId = membership$.value?.identity; for (let dup = 0; dup < 1 + duplicateTiles; dup++) { yield { - keys: [dup, userMediaId, userId, participant, connection$], + keys: [ + dup, + userMediaId, + userId, + participant, + connection$, + membership$.value, + ], data: undefined, }; } @@ -756,6 +773,7 @@ export function createCallViewModel$( userId, participant, connection$, + membership, ) => { const livekitRoom$ = scope.behavior( connection$.pipe(map((c) => c?.livekitRoom)), @@ -773,6 +791,7 @@ export function createCallViewModel$( scope, `${participantId}:${dup}`, userId, + membership, participant, options.encryptionSystem, livekitRoom$, @@ -1523,6 +1542,7 @@ export function createCallViewModel$( ), null, ), + allConnections$, participantCount$: participantCount$, handsRaised$: handsRaised$, reactions$: reactions$, diff --git a/src/state/CallViewModel/localMember/LocalMember.test.ts b/src/state/CallViewModel/localMember/LocalMember.test.ts index 8a7505eb..6b750785 100644 --- a/src/state/CallViewModel/localMember/LocalMember.test.ts +++ b/src/state/CallViewModel/localMember/LocalMember.test.ts @@ -39,6 +39,7 @@ import { constant } from "../../Behavior"; import { ConnectionManagerData } from "../remoteMembers/ConnectionManager"; import { ConnectionState, type Connection } from "../remoteMembers/Connection"; import { type Publisher } from "./Publisher"; +import { type LocalTransportWithSFUConfig } from "./LocalTransport"; const MATRIX_RTC_MODE = MatrixRTCMode.Legacy; const getUrlParams = vi.hoisted(() => vi.fn(() => ({}))); @@ -212,10 +213,11 @@ describe("LocalMembership", () => { it("throws error on missing RTC config error", () => { withTestScheduler(({ scope, hot, expectObservable }) => { - const localTransport$ = scope.behavior( - hot("1ms #", {}, new MatrixRTCTransportMissingError("domain.com")), - null, - ); + const localTransport$ = + scope.behavior( + hot("1ms #", {}, new MatrixRTCTransportMissingError("domain.com")), + null, + ); // we do not need any connection data since we want to fail before reaching that. const mockConnectionManager = { @@ -243,11 +245,23 @@ describe("LocalMembership", () => { }); const aTransport = { - livekit_service_url: "a", - } as LivekitTransport; + transport: { + livekit_service_url: "a", + } as LivekitTransport, + sfuConfig: { + url: "sfu-url", + jwt: "sfu-token", + }, + } as LocalTransportWithSFUConfig; const bTransport = { - livekit_service_url: "b", - } as LivekitTransport; + transport: { + livekit_service_url: "b", + } as LivekitTransport, + sfuConfig: { + url: "sfu-url", + jwt: "sfu-token", + }, + } as LocalTransportWithSFUConfig; const connectionTransportAConnected = { livekitRoom: mockLivekitRoom({ @@ -391,7 +405,8 @@ describe("LocalMembership", () => { const scope = new ObservableScope(); const connectionManagerData = new ConnectionManagerData(); - const localTransport$ = new BehaviorSubject(null); + const localTransport$ = + new BehaviorSubject(null); const connectionManagerData$ = new BehaviorSubject( new Epoch(connectionManagerData), ); @@ -468,7 +483,7 @@ describe("LocalMembership", () => { }); ( - connectionManagerData2.getConnectionForTransport(aTransport)! + connectionManagerData2.getConnectionForTransport(aTransport.transport)! .state$ as BehaviorSubject ).next(ConnectionState.LivekitConnected); expect(localMembership.localMemberState$.value).toStrictEqual({ diff --git a/src/state/CallViewModel/localMember/LocalMember.ts b/src/state/CallViewModel/localMember/LocalMember.ts index eb506132..4749e942 100644 --- a/src/state/CallViewModel/localMember/LocalMember.ts +++ b/src/state/CallViewModel/localMember/LocalMember.ts @@ -61,6 +61,7 @@ import { } from "../remoteMembers/Connection.ts"; import { type HomeserverConnected } from "./HomeserverConnected.ts"; import { and$ } from "../../../utils/observable.ts"; +import { type LocalTransportWithSFUConfig } from "./LocalTransport.ts"; export enum TransportState { /** Not even a transport is available to the LocalMembership */ @@ -126,7 +127,7 @@ interface Props { createPublisherFactory: (connection: Connection) => Publisher; joinMatrixRTC: (transport: LivekitTransport) => void; homeserverConnected: HomeserverConnected; - localTransport$: Behavior; + localTransport$: Behavior; matrixRTCSession: Pick< MatrixRTCSession, "updateCallIntent" | "leaveRoomSession" @@ -234,7 +235,9 @@ export const createLocalMembership$ = ({ return null; } - return connectionData.getConnectionForTransport(localTransport); + return connectionData.getConnectionForTransport( + localTransport.transport, + ); }), tap((connection) => { logger.info( @@ -533,7 +536,7 @@ export const createLocalMembership$ = ({ if (!shouldConnect) return; try { - joinMatrixRTC(transport); + joinMatrixRTC(transport.transport); } catch (error) { logger.error("Error entering RTC session", error); if (error instanceof Error) diff --git a/src/state/CallViewModel/localMember/LocalTransport.test.ts b/src/state/CallViewModel/localMember/LocalTransport.test.ts index 2199ca94..8da0db7f 100644 --- a/src/state/CallViewModel/localMember/LocalTransport.test.ts +++ b/src/state/CallViewModel/localMember/LocalTransport.test.ts @@ -19,7 +19,7 @@ import { BehaviorSubject, lastValueFrom } from "rxjs"; import fetchMock from "fetch-mock"; import { mockConfig, flushPromises, ownMemberMock } from "../../../utils/test"; -import { createLocalTransport$ } from "./LocalTransport"; +import { createLocalTransport$, JwtEndpointVersion } from "./LocalTransport"; import { constant } from "../../Behavior"; import { Epoch, ObservableScope } from "../../ObservableScope"; import { @@ -58,7 +58,7 @@ describe("LocalTransport", () => { getDeviceId: vi.fn(), }, ownMembershipIdentity: ownMemberMock, - useOldJwtEndpoint$: constant(false), + forceJwtEndpoint$: constant(JwtEndpointVersion.Legacy), delayId$: constant("delay_id_mock"), }); await flushPromises(); @@ -98,7 +98,7 @@ describe("LocalTransport", () => { getDeviceId: vi.fn(), }, ownMembershipIdentity: ownMemberMock, - useOldJwtEndpoint$: constant(false), + forceJwtEndpoint$: constant(JwtEndpointVersion.Legacy), delayId$: constant("delay_id_mock"), }); localTransport$.subscribe( @@ -140,7 +140,7 @@ describe("LocalTransport", () => { baseUrl: "https://lk.example.org", }, ownMembershipIdentity: ownMemberMock, - useOldJwtEndpoint$: constant(false), + forceJwtEndpoint$: constant(JwtEndpointVersion.Legacy), delayId$: constant("delay_id_mock"), }); @@ -186,7 +186,7 @@ describe("LocalTransport", () => { baseUrl: "https://lk.example.org", }, ownMembershipIdentity: ownMemberMock, - useOldJwtEndpoint$: constant(false), + forceJwtEndpoint$: constant(JwtEndpointVersion.Legacy), delayId$: constant("delay_id_mock"), }); @@ -216,7 +216,7 @@ describe("LocalTransport", () => { scope, roomId: "!example_room_id", useOldestMember$: constant(false), - useOldJwtEndpoint$: constant(false), + forceJwtEndpoint$: constant(JwtEndpointVersion.Legacy), delayId$: constant(null), memberships$: constant(new Epoch([])), client: { @@ -333,7 +333,7 @@ describe("LocalTransport", () => { ownMembershipIdentity: ownMemberMock, roomId: "!example_room_id", useOldestMember$: constant(false), - useOldJwtEndpoint$: constant(false), + forceJwtEndpoint$: constant(JwtEndpointVersion.Legacy), delayId$: constant(null), memberships$: constant(new Epoch([])), client: { diff --git a/src/state/CallViewModel/localMember/LocalTransport.ts b/src/state/CallViewModel/localMember/LocalTransport.ts index 8a6a750b..6e0e56a3 100644 --- a/src/state/CallViewModel/localMember/LocalTransport.ts +++ b/src/state/CallViewModel/localMember/LocalTransport.ts @@ -31,9 +31,11 @@ import { Config } from "../../../config/Config.ts"; import { FailToGetOpenIdToken, MatrixRTCTransportMissingError, + NoMatrix2AuthorizationService, } from "../../../utils/errors.ts"; import { getSFUConfigWithOpenID, + type SFUConfig, type OpenIDClientParts, } from "../../../livekit/openIDSFU.ts"; import { areLivekitTransportsEqual } from "../remoteMembers/MatrixLivekitMembers.ts"; @@ -57,10 +59,25 @@ interface Props { OpenIDClientParts; roomId: string; useOldestMember$: Behavior; - useOldJwtEndpoint$: Behavior; + forceJwtEndpoint$: Behavior; delayId$: Behavior; } +export enum JwtEndpointVersion { + Legacy = "legacy", + Matrix_2_0 = "matrix_2_0", +} + +export interface LocalTransportWithSFUConfig { + transport: LivekitTransport; + sfuConfig: SFUConfig; +} +export function isLocalTransportWithSFUConfig( + obj: LivekitTransport | LocalTransportWithSFUConfig, +): obj is LocalTransportWithSFUConfig { + return "transport" in obj && "sfuConfig" in obj; +} + /** * This class is responsible for managing the local transport. * "Which transport is the local member going to use" @@ -81,22 +98,40 @@ export const createLocalTransport$ = ({ client, roomId, useOldestMember$, - useOldJwtEndpoint$, + forceJwtEndpoint$, delayId$, -}: Props): Behavior => { +}: Props): Behavior => { /** * The transport over which we should be actively publishing our media. * undefined when not joined. */ const oldestMemberTransport$ = scope.behavior( - combineLatest([memberships$, useOldJwtEndpoint$]).pipe( - map(([memberships, forceOldJwtEndpoint]) => { + combineLatest([memberships$]).pipe( + map(([memberships]) => { const oldestMember = memberships.value[0]; const transport = oldestMember?.getTransport(memberships.value[0]); if (!transport) return null; return transport; }), first((t) => t != null && isLivekitTransport(t)), + switchMap((transport) => { + // Get the open jwt token to connect to the sfu + const computeLocalTransportWithSFUConfig = + async (): Promise => { + return { + transport, + sfuConfig: await getSFUConfigWithOpenID( + client, + ownMembershipIdentity, + transport.livekit_service_url, + roomId, + { forceJwtEndpoint: JwtEndpointVersion.Legacy }, + logger, + ), + }; + }; + return from(computeLocalTransportWithSFUConfig()); + }), ), null, ); @@ -108,19 +143,29 @@ export const createLocalTransport$ = ({ * @throws MatrixRTCTransportMissingError | FailToGetOpenIdToken */ const preferredTransport$ = scope.behavior( - combineLatest([customLivekitUrl.value$, delayId$, useOldJwtEndpoint$]).pipe( - switchMap(([customUrl, delayId, forceOldJwtEndpoint]) => - from( + // preferredTransport$ (used for multi sfu) needs to know if we are using the old or new + // jwt endpoint (`get_token` vs `sfu/get`) based on that the jwt endpoint will compute the rtcBackendIdentity + // differently. (sha(`${userId}|${deviceId}|${memberId}`) vs `${userId}|${deviceId}|${memberId}`) + // When using sticky events (we need to use the new endpoint). + combineLatest([customLivekitUrl.value$, delayId$, forceJwtEndpoint$]).pipe( + switchMap(([customUrl, delayId, forceEndpoint]) => { + logger.info( + "Creating preferred transport based on: ", + customUrl, + delayId, + forceEndpoint, + ); + return from( makeTransport( client, ownMembershipIdentity, roomId, customUrl, - forceOldJwtEndpoint, + forceEndpoint, delayId ?? undefined, ), - ), - ), + ); + }), ), null, ); @@ -139,7 +184,9 @@ export const createLocalTransport$ = ({ ? (oldestMemberTransport ?? preferredTransport) : preferredTransport, ), - distinctUntilChanged((t1, t2) => areLivekitTransportsEqual(t1, t2)), + distinctUntilChanged((t1, t2) => + areLivekitTransportsEqual(t1?.transport ?? null, t2?.transport ?? null), + ), ), ); }; @@ -161,7 +208,10 @@ const FOCI_WK_KEY = "org.matrix.msc4143.rtc_foci"; * @param membership The membership identity of the user. * @param roomId The ID of the room to be connected to. * @param urlFromDevSettings Override URL provided by the user's local config. - * @param forceOldJwtEndpoint Whether to force the old JWT endpoint (not hashing the backendIdentity). + * @param forceJwtEndpoint Whether to force a specific JWT endpoint + * - `Legacy` / `Matrix_2_0` + * - `get_token` / `sfu/get` + * - not hashing / hashing the backendIdentity * @param delayId the delay id passed to the jwt service. * * @returns A fully validated transport config. @@ -176,26 +226,33 @@ async function makeTransport( membership: CallMembershipIdentityParts, roomId: string, urlFromDevSettings: string | null, - forceOldJwtEndpoint: boolean, + forceJwtEndpoint: JwtEndpointVersion, delayId?: string, -): Promise { +): Promise { logger.trace("Searching for a preferred transport"); - async function doOpenIdAndJWTFromUrl(url: string): Promise { - const { livekitAlias } = await getSFUConfigWithOpenID( + async function doOpenIdAndJWTFromUrl( + url: string, + ): Promise { + const sfuConfig = await getSFUConfigWithOpenID( client, membership, url, - forceOldJwtEndpoint, roomId, - client.baseUrl, - delayId, + { + forceJwtEndpoint: forceJwtEndpoint, + delayEndpointBaseUrl: client.baseUrl, + delayId, + }, logger, ); return { - type: "livekit", - livekit_service_url: url, - livekit_alias: livekitAlias, + transport: { + type: "livekit", + livekit_service_url: url, + livekit_alias: sfuConfig.livekitAlias, + }, + sfuConfig, }; } // We will call `getSFUConfigWithOpenID` once per transport here as it's our @@ -217,7 +274,7 @@ async function makeTransport( async function getFirstUsableTransport( transports: Transport[], - ): Promise { + ): Promise { for (const potentialTransport of transports) { if (isLivekitTransportConfig(potentialTransport)) { try { @@ -226,8 +283,11 @@ async function makeTransport( potentialTransport.livekit_service_url, ); } catch (ex) { + // Explictly throw these if (ex instanceof FailToGetOpenIdToken) { - // Explictly throw these + throw ex; + } + if (ex instanceof NoMatrix2AuthorizationService) { throw ex; } logger.debug( diff --git a/src/state/CallViewModel/remoteMembers/Connection.test.ts b/src/state/CallViewModel/remoteMembers/Connection.test.ts index 0130a5ce..cc430645 100644 --- a/src/state/CallViewModel/remoteMembers/Connection.test.ts +++ b/src/state/CallViewModel/remoteMembers/Connection.test.ts @@ -114,6 +114,7 @@ function setupRemoteConnection(): Connection { client: client, transport: livekitFocus, scope: testScope, + ownMembershipIdentity: ownMemberMock, livekitRoomFactory: () => fakeLivekitRoom, }; @@ -138,7 +139,7 @@ function setupRemoteConnection(): Connection { return Promise.resolve(); }); - return new Connection(opts, logger, ownMemberMock); + return new Connection(opts, logger); } afterEach(() => { @@ -155,9 +156,10 @@ describe("Start connection states", () => { client: client, transport: livekitFocus, scope: testScope, + ownMembershipIdentity: ownMemberMock, livekitRoomFactory: () => fakeLivekitRoom, }; - const connection = new Connection(opts, logger, ownMemberMock); + const connection = new Connection(opts, logger); expect(connection.state$.getValue()).toEqual("Initialized"); }); @@ -170,10 +172,11 @@ describe("Start connection states", () => { client: client, transport: livekitFocus, scope: testScope, + ownMembershipIdentity: ownMemberMock, livekitRoomFactory: () => fakeLivekitRoom, }; - const connection = new Connection(opts, logger, ownMemberMock); + const connection = new Connection(opts, logger); const capturedStates: (ConnectionState | Error)[] = []; const s = connection.state$.subscribe((value) => { @@ -220,10 +223,11 @@ describe("Start connection states", () => { client: client, transport: livekitFocus, scope: testScope, + ownMembershipIdentity: ownMemberMock, livekitRoomFactory: () => fakeLivekitRoom, }; - const connection = new Connection(opts, logger, ownMemberMock); + const connection = new Connection(opts, logger); const capturedStates: (ConnectionState | Error)[] = []; const s = connection.state$.subscribe((value) => { @@ -277,10 +281,11 @@ describe("Start connection states", () => { client: client, transport: livekitFocus, scope: testScope, + ownMembershipIdentity: ownMemberMock, livekitRoomFactory: () => fakeLivekitRoom, }; - const connection = new Connection(opts, logger, ownMemberMock); + const connection = new Connection(opts, logger); const capturedStates: (ConnectionState | Error)[] = []; const s = connection.state$.subscribe((value) => { diff --git a/src/state/CallViewModel/remoteMembers/Connection.ts b/src/state/CallViewModel/remoteMembers/Connection.ts index 269fd4f1..f286b0cd 100644 --- a/src/state/CallViewModel/remoteMembers/Connection.ts +++ b/src/state/CallViewModel/remoteMembers/Connection.ts @@ -33,10 +33,21 @@ import { SFURoomCreationRestrictedError, UnknownCallError, } from "../../../utils/errors.ts"; +import { type JwtEndpointVersion } from "../localMember/LocalTransport.ts"; export interface ConnectionOpts { - /** Whether we always try to connect to this connection via the legacy jwt endpoint. (no hash identity) */ - forceOldJwtEndpoint?: boolean; + /** + * For the local transport we already do know the jwt token and url. We can reuse it. + * On top the local transport will send additional data to the jwt server to use delayed event delegation. + */ + existingSFUConfig?: SFUConfig; + /** + * For local connections that use the oldest member pattern. here we have not prefetched the sfuConfig + * and hence we need to let the connection do the jwt token fetching. + */ + forceJwtEndpoint?: JwtEndpointVersion; + /** The identity parts to use on this connection */ + ownMembershipIdentity: CallMembershipIdentityParts; /** The media transport to connect to. */ transport: LivekitTransport; /** The Matrix client to use for OpenID and SFU config requests. */ @@ -132,8 +143,10 @@ export class Connection { try { this._state$.next(ConnectionState.FetchingConfig); // We should already have this information after creating the localTransport. - // It would probably be better to forward this here. - const { url, jwt } = await this.getSFUConfigWithOpenID(); + // only call getSFUConfigWithOpenID for connections where we do not have a token yet. (existingJwtTokenData === undefined) + const { url, jwt } = + this.existingSFUConfig ?? + (await this.getSFUConfigForRemoteConnection()); // If we were stopped while fetching the config, don't proceed to connect if (this.stopped) return; @@ -189,17 +202,16 @@ export class Connection { } } - protected async getSFUConfigWithOpenID(): Promise { + protected async getSFUConfigForRemoteConnection(): Promise { + // This will only be called for sfu's where we do not publish ourselves. + // For the local connection we will use the existingJwtTokenData return await getSFUConfigWithOpenID( this.client, this.ownMembershipIdentity, this.transport.livekit_service_url, - this.forceOldJwtEndpoint, this.transport.livekit_alias, - // For the remote members we intentionally do not pass a delayEndpointBaseUrl. - undefined, - // and no delayId. - undefined, + // dont pass any custom opts for the subscribe only connections + {}, this.logger, ); } @@ -222,7 +234,8 @@ export class Connection { private readonly client: OpenIDClientParts; private readonly logger: Logger; - private readonly forceOldJwtEndpoint: boolean; + private readonly ownMembershipIdentity: CallMembershipIdentityParts; + private readonly existingSFUConfig?: SFUConfig; /** * Creates a new connection to a matrix RTC LiveKit backend. * @@ -230,12 +243,9 @@ export class Connection { * * @param logger - The logger to use. */ - public constructor( - opts: ConnectionOpts, - logger: Logger, - private ownMembershipIdentity: CallMembershipIdentityParts, - ) { - this.forceOldJwtEndpoint = opts.forceOldJwtEndpoint ?? false; + public constructor(opts: ConnectionOpts, logger: Logger) { + this.ownMembershipIdentity = opts.ownMembershipIdentity; + this.existingSFUConfig = opts.existingSFUConfig; this.logger = logger.getChild("[Connection]"); this.logger.info( `Creating new connection to ${opts.transport.livekit_service_url} ${opts.transport.livekit_alias}`, diff --git a/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts b/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts index 668538ac..aa20037c 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts @@ -20,7 +20,10 @@ import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc/LivekitTransp import { type ObservableScope } from "../../ObservableScope.ts"; import { Connection } from "./Connection.ts"; -import type { OpenIDClientParts } from "../../../livekit/openIDSFU.ts"; +import type { + OpenIDClientParts, + SFUConfig, +} from "../../../livekit/openIDSFU.ts"; import type { MediaDevices } from "../../MediaDevices.ts"; import type { Behavior } from "../../Behavior.ts"; import type { ProcessorState } from "../../../livekit/TrackProcessorContext.tsx"; @@ -29,11 +32,11 @@ import { defaultLiveKitOptions } from "../../../livekit/options.ts"; // TODO evaluate if this should be done like the Publisher Factory export interface ConnectionFactory { createConnection( - transport: LivekitTransport, scope: ObservableScope, + transport: LivekitTransport, ownMembershipIdentity: CallMembershipIdentityParts, logger: Logger, - forceOldJwtEndpoint?: boolean, + sfuConfig?: SFUConfig, ): Connection; } @@ -83,30 +86,30 @@ export class ECConnectionFactory implements ConnectionFactory { /** * - * @param transport The transport to use for this connection. * @param scope The observable scope (used for clean-up) + * @param transport The transport to use for this connection. * @param ownMembershipIdentity required to connect (using the jwt service) with the SFU. * @param logger The logger instance to use for this connection. - * @param forceOldJwtEndpoint Use the old JWT endpoint independent of what the sfu supports. + * @param sfuConfig optional config in case we already have a token for this connection. * @returns */ public createConnection( - transport: LivekitTransport, scope: ObservableScope, + transport: LivekitTransport, ownMembershipIdentity: CallMembershipIdentityParts, logger: Logger, - forceOldJwtEndpoint?: boolean, + sfuConfig?: SFUConfig, ): Connection { return new Connection( { + existingSFUConfig: sfuConfig, transport, client: this.client, scope: scope, livekitRoomFactory: this.livekitRoomFactory, - forceOldJwtEndpoint, + ownMembershipIdentity, }, logger, - ownMembershipIdentity, ); } } diff --git a/src/state/CallViewModel/remoteMembers/ConnectionManager.ts b/src/state/CallViewModel/remoteMembers/ConnectionManager.ts index 5c50f0cd..4295c5f2 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionManager.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionManager.ts @@ -12,12 +12,17 @@ import { type Logger } from "matrix-js-sdk/lib/logger"; import { type RemoteParticipant } from "livekit-client"; import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; -import { constant, type Behavior } from "../../Behavior.ts"; +import { type Behavior } from "../../Behavior.ts"; import { type Connection } from "./Connection.ts"; import { Epoch, type ObservableScope } from "../../ObservableScope.ts"; import { generateItemsWithEpoch } from "../../../utils/observable.ts"; import { areLivekitTransportsEqual } from "./MatrixLivekitMembers.ts"; import { type ConnectionFactory } from "./ConnectionFactory.ts"; +import { + isLocalTransportWithSFUConfig, + type LocalTransportWithSFUConfig, +} from "../localMember/LocalTransport.ts"; +import { type SFUConfig } from "../../../livekit/openIDSFU.ts"; export class ConnectionManagerData { private readonly store: Map< @@ -66,9 +71,9 @@ export class ConnectionManagerData { interface Props { scope: ObservableScope; connectionFactory: ConnectionFactory; - localTransport$: Behavior; + localTransport$: Behavior; remoteTransports$: Behavior>; - forceOldJwtEndpointForLocalTransport$?: Behavior; + logger: Logger; ownMembershipIdentity: CallMembershipIdentityParts; } @@ -87,7 +92,7 @@ export interface IConnectionManager { * @param props.remoteTransports$ - All other transports. The connection manager will create connections for each transport. (deduplicated with localTransport$) * @param props.ownMembershipIdentity - The own membership identity to use. * @param props.logger - The logger to use. - * @param props.forceOldJwtEndpointForLocalTransport$ - Use the old JWT endpoint independent of what the sfu supports. Only applies for localTransport$. + * * Each of these behaviors can be interpreted as subscribed list of transports. * @@ -103,7 +108,6 @@ export function createConnectionManager$({ connectionFactory, localTransport$, remoteTransports$, - forceOldJwtEndpointForLocalTransport$ = constant(false), logger: parentLogger, ownMembershipIdentity, }: Props): IConnectionManager { @@ -118,42 +122,35 @@ export function createConnectionManager$({ * It is build based on the list of subscribed transports (`transportsSubscriptions$`). * externally this is modified via `registerTransports()`. */ - const transportsWithJwtTag$ = scope.behavior( - combineLatest([ - remoteTransports$, - localTransport$, - forceOldJwtEndpointForLocalTransport$, - ]).pipe( - // combine local and remote transports into one transport array + const localAndRemoteTransports$: Behavior< + Epoch<(LivekitTransport | LocalTransportWithSFUConfig)[]> + > = scope.behavior( + combineLatest([remoteTransports$, localTransport$]).pipe( + // Combine local and remote transports into one transport array // and set the forceOldJwtEndpoint property on the local transport - map( - ([ - remoteTransports, - localTransport, - forceOldJwtEndpointForLocalTransport, - ]) => { - let localTransportAsArray: (LivekitTransport & { - forceOldJwtEndpoint: boolean; - })[] = []; - if (localTransport) { - localTransportAsArray = [ - { - ...localTransport, - forceOldJwtEndpoint: forceOldJwtEndpointForLocalTransport, - }, - ]; - } - return new Epoch( - removeDuplicateTransports([ - ...localTransportAsArray, - ...remoteTransports.value, - ]) as (LivekitTransport & { - forceOldJwtEndpoint?: boolean; - })[], - remoteTransports.epoch, - ); - }, - ), + map(([remoteTransports, localTransport]) => { + let localTransportAsArray: LocalTransportWithSFUConfig[] = []; + if (localTransport) { + localTransportAsArray = [localTransport]; + } + const dedupedRemote = removeDuplicateTransports(remoteTransports.value); + const remoteWithoutLocal = dedupedRemote.filter( + (transport) => + !localTransportAsArray.find((l) => + areLivekitTransportsEqual(l.transport, transport), + ), + ); + logger.debug( + "remoteWithoutLocal", + remoteWithoutLocal, + "localTransportAsArray", + localTransportAsArray, + ); + return new Epoch( + [...localTransportAsArray, ...remoteWithoutLocal], + remoteTransports.epoch, + ); + }), ), ); @@ -161,33 +158,51 @@ export function createConnectionManager$({ * Connections for each transport in use by one or more session members. */ const connections$ = scope.behavior( - transportsWithJwtTag$.pipe( + localAndRemoteTransports$.pipe( generateItemsWithEpoch( function* (transports) { - for (const transport of transports) - yield { - keys: [ - transport.livekit_service_url, - transport.livekit_alias, - transport.forceOldJwtEndpoint, - ], - data: undefined, - }; + for (const transportWithOrWithoutSfuConfig of transports) { + if ( + isLocalTransportWithSFUConfig(transportWithOrWithoutSfuConfig) + ) { + // This is the local transport only the `LocalTransportWithSFUConfig` has a `sfuConfig` field + const { transport, sfuConfig } = transportWithOrWithoutSfuConfig; + yield { + keys: [ + transport.livekit_service_url, + transport.livekit_alias, + sfuConfig, + ], + data: undefined, + }; + } else { + const transport = transportWithOrWithoutSfuConfig; + yield { + keys: [ + transport.livekit_service_url, + transport.livekit_alias, + undefined as undefined | SFUConfig, + ], + data: undefined, + }; + } + } }, - (scope, _data$, serviceUrl, alias, forceOldJwtEndpoint) => { + (scope, _data$, serviceUrl, alias, sfuConfig) => { logger.debug( - `Creating connection to ${serviceUrl} (${alias}, forceOldJwtEndpoint: ${forceOldJwtEndpoint})`, + `Creating connection to ${serviceUrl} (${alias}, withSfuConfig (local connection?): ${JSON.stringify(sfuConfig) ?? "no config->remote connection"})`, ); + const connection = connectionFactory.createConnection( + scope, { type: "livekit", livekit_service_url: serviceUrl, livekit_alias: alias, }, - scope, ownMembershipIdentity, logger, - forceOldJwtEndpoint, + sfuConfig, ); // Start the connection immediately // Use connection state to track connection progress diff --git a/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts b/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts index f28bd158..b56a17f7 100644 --- a/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts +++ b/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts @@ -77,8 +77,8 @@ describe("ECConnectionFactory - Audio inputs options", () => { noise, ); ecConnectionFactory.createConnection( - exampleTransport, testScope, + exampleTransport, ownMemberMock, logger, ); @@ -123,8 +123,8 @@ describe("ECConnectionFactory - ControlledAudioDevice", () => { false, ); ecConnectionFactory.createConnection( - exampleTransport, testScope, + exampleTransport, ownMemberMock, logger, ); diff --git a/src/state/MediaViewModel.ts b/src/state/MediaViewModel.ts index 9888d6bf..6b020057 100644 --- a/src/state/MediaViewModel.ts +++ b/src/state/MediaViewModel.ts @@ -44,6 +44,7 @@ import { throttleTime, distinctUntilChanged, } from "rxjs"; +import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc"; import { alwaysShowSelf } from "../settings/settings"; import { showConnectionStats } from "../settings/settings"; @@ -257,6 +258,7 @@ abstract class BaseMediaViewModel { * The Matrix user to which this media belongs. */ public readonly userId: string, + public readonly rtcMembership: CallMembership, // We don't necessarily have a participant if a user connects via MatrixRTC but not (yet) through // livekit. protected readonly participant$: Observable< @@ -402,10 +404,13 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel { */ public readonly cropVideo$: Behavior = this._cropVideo$; + public readonly rtcBackendIdentity = this.rtcMembership.rtcBackendIdentity; + public constructor( scope: ObservableScope, id: string, userId: string, + rtcMembership: CallMembership, participant$: Observable, encryptionSystem: EncryptionSystem, livekitRoom$: Behavior, @@ -419,6 +424,7 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel { scope, id, userId, + rtcMembership, participant$, encryptionSystem, Track.Source.Microphone, @@ -544,6 +550,7 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel { scope: ObservableScope, id: string, userId: string, + rtcMembership: CallMembership, participant$: Behavior, encryptionSystem: EncryptionSystem, livekitRoom$: Behavior, @@ -558,6 +565,7 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel { scope, id, userId, + rtcMembership, participant$, encryptionSystem, livekitRoom$, @@ -671,6 +679,7 @@ export class RemoteUserMediaViewModel extends BaseUserMediaViewModel { scope: ObservableScope, id: string, userId: string, + rtcMembership: CallMembership, participant$: Observable, encryptionSystem: EncryptionSystem, livekitRoom$: Behavior, @@ -685,6 +694,7 @@ export class RemoteUserMediaViewModel extends BaseUserMediaViewModel { scope, id, userId, + rtcMembership, participant$, encryptionSystem, livekitRoom$, @@ -772,6 +782,7 @@ export class ScreenShareViewModel extends BaseMediaViewModel { scope: ObservableScope, id: string, userId: string, + rtcMembership: CallMembership, participant$: Observable, encryptionSystem: EncryptionSystem, livekitRoom$: Behavior, @@ -785,6 +796,7 @@ export class ScreenShareViewModel extends BaseMediaViewModel { scope, id, userId, + rtcMembership, participant$, encryptionSystem, Track.Source.ScreenShareAudio, diff --git a/src/state/ScreenShare.ts b/src/state/ScreenShare.ts index 0a241cdf..67f97a93 100644 --- a/src/state/ScreenShare.ts +++ b/src/state/ScreenShare.ts @@ -10,6 +10,7 @@ import { type RemoteParticipant, type Room as LivekitRoom, } from "livekit-client"; +import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc/CallMembership"; import { type ObservableScope } from "./ObservableScope.ts"; import { ScreenShareViewModel } from "./MediaViewModel.ts"; @@ -28,6 +29,7 @@ export class ScreenShare { private readonly scope: ObservableScope, id: string, userId: string, + rtcMember: CallMembership, participant: LocalParticipant | RemoteParticipant, encryptionSystem: EncryptionSystem, livekitRoom$: Behavior, @@ -40,6 +42,7 @@ export class ScreenShare { this.scope, id, userId, + rtcMember, of(participant), encryptionSystem, livekitRoom$, diff --git a/src/state/UserMedia.ts b/src/state/UserMedia.ts index 690870e6..6435fb17 100644 --- a/src/state/UserMedia.ts +++ b/src/state/UserMedia.ts @@ -13,6 +13,7 @@ import { type Room as LivekitRoom, } from "livekit-client"; import { observeParticipantEvents } from "@livekit/components-core"; +import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc/CallMembership"; import { type ObservableScope } from "./ObservableScope.ts"; import { @@ -75,6 +76,7 @@ export class UserMedia { this.scope, this.id, this.userId, + this.rtcMembership, this.participant.value$, this.encryptionSystem, this.livekitRoom$, @@ -89,6 +91,7 @@ export class UserMedia { this.scope, this.id, this.userId, + this.rtcMembership, this.participant.value$, this.encryptionSystem, this.livekitRoom$, @@ -140,6 +143,7 @@ export class UserMedia { scope, `${this.id}:${key}`, this.userId, + this.rtcMembership, p, this.encryptionSystem, this.livekitRoom$, @@ -191,6 +195,8 @@ export class UserMedia { private readonly scope: ObservableScope, public readonly id: string, private readonly userId: string, + // TODO evaluate if this should just be the rtcBackendIdentity + private readonly rtcMembership: CallMembership, private readonly participant: TaggedParticipant, private readonly encryptionSystem: EncryptionSystem, private readonly livekitRoom$: Behavior, diff --git a/src/tile/GridTile.tsx b/src/tile/GridTile.tsx index 2f750c50..92262f05 100644 --- a/src/tile/GridTile.tsx +++ b/src/tile/GridTile.tsx @@ -113,6 +113,7 @@ const UserMediaTile: FC = ({ }, [vm], ); + const rtcBackendIdentity = vm.rtcBackendIdentity; const handRaised = useBehavior(vm.handRaised$); const reaction = useBehavior(vm.reaction$); @@ -200,6 +201,7 @@ const UserMediaTile: FC = ({ focusUrl={focusUrl} audioStreamStats={audioStreamStats} videoStreamStats={videoStreamStats} + rtcBackendIdentity={rtcBackendIdentity} {...props} /> ); diff --git a/src/tile/MediaView.tsx b/src/tile/MediaView.tsx index 8bb38d94..7847a30a 100644 --- a/src/tile/MediaView.tsx +++ b/src/tile/MediaView.tsx @@ -46,6 +46,7 @@ interface Props extends ComponentProps { waitingForMedia?: boolean; audioStreamStats?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats; videoStreamStats?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats; + rtcBackendIdentity?: string; // The focus url, mainly for debugging purposes focusUrl?: string; } @@ -74,6 +75,7 @@ export const MediaView: FC = ({ waitingForMedia, audioStreamStats, videoStreamStats, + rtcBackendIdentity, focusUrl, ...props }) => { @@ -135,11 +137,14 @@ export const MediaView: FC = ({ )} {(audioStreamStats || videoStreamStats) && ( - + <> + + )} {/* TODO: Bring this back once encryption status is less broken */} {/*encryptionStatus !== EncryptionStatus.Okay && ( diff --git a/src/utils/errors.ts b/src/utils/errors.ts index cddf90de..20a282e6 100644 --- a/src/utils/errors.ts +++ b/src/utils/errors.ts @@ -19,6 +19,7 @@ export enum ErrorCode { INSUFFICIENT_CAPACITY_ERROR = "INSUFFICIENT_CAPACITY_ERROR", E2EE_NOT_SUPPORTED = "E2EE_NOT_SUPPORTED", OPEN_ID_ERROR = "OPEN_ID_ERROR", + NO_MATRIX_2_AUTHORIZATION_SERVICE = "NO_MATRIX_2_0_AUTHORIZATION_SERVICE", SFU_ERROR = "SFU_ERROR", UNKNOWN_ERROR = "UNKNOWN_ERROR", } @@ -171,6 +172,23 @@ export class FailToGetOpenIdToken extends ElementCallError { } } +export class NoMatrix2AuthorizationService extends ElementCallError { + /** + * Creates an instance of NoMatrix2_0AuthorizationService. + * @param error - The underlying error that caused the failure. + */ + public constructor(error: Error) { + super( + t("error.generic"), + ErrorCode.NO_MATRIX_2_AUTHORIZATION_SERVICE, + ErrorCategory.CONFIGURATION_ISSUE, + t("error.no_matrix_2_authorization_service"), + // Properly set it as a cause for a better reporting on sentry + error, + ); + } +} + /** * Error indicating a failure to start publishing on a LiveKit connection. */ diff --git a/src/utils/test.ts b/src/utils/test.ts index d24ad130..9f9ae4b7 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -334,6 +334,7 @@ export function createLocalMedia( testScope(), "local", member.userId, + rtcMember, constant(localParticipant), { kind: E2eeType.PER_PARTICIPANT, @@ -379,6 +380,7 @@ export function createRemoteMedia( testScope(), "remote", member.userId, + rtcMember, constant(participant), { kind: E2eeType.PER_PARTICIPANT, diff --git a/yarn.lock b/yarn.lock index 54832a25..b32cf3c8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8246,7 +8246,7 @@ __metadata: livekit-client: "npm:^2.13.0" lodash-es: "npm:^4.17.21" loglevel: "npm:^1.9.1" - matrix-js-sdk: "matrix-org/matrix-js-sdk#develop" + matrix-js-sdk: "matrix-org/matrix-js-sdk#ff5fab7722e54be2d69a380cd6182ecb262d7859" matrix-widget-api: "npm:^1.14.0" node-stdlib-browser: "npm:^1.3.1" normalize.css: "npm:^8.0.1" @@ -11333,9 +11333,9 @@ __metadata: languageName: node linkType: hard -"matrix-js-sdk@matrix-org/matrix-js-sdk#develop": - version: 39.4.0 - resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=4b89fb23c54aaf7826bd127d8fa21cc7bb87688f" +"matrix-js-sdk@portal:/Users/timo/Projects/matrix-js-sdk::locator=element-call%40workspace%3A.": + version: 0.0.0-use.local + resolution: "matrix-js-sdk@portal:/Users/timo/Projects/matrix-js-sdk::locator=element-call%40workspace%3A." dependencies: "@babel/runtime": "npm:^7.12.5" "@matrix-org/matrix-sdk-crypto-wasm": "npm:^16.0.0" @@ -11351,9 +11351,8 @@ __metadata: sdp-transform: "npm:^3.0.0" unhomoglyph: "npm:^1.0.6" uuid: "npm:13" - checksum: 10c0/bc7443bf67822e9bc7b8e531b4e61e6ebac41c2fd8047ac0567456c264ae0d1911fbef6e437d312a3adeead86cd5e7134944e3fd73d28002777618bc0ebaa1ca languageName: node - linkType: hard + linkType: soft "matrix-widget-api@npm:^1.14.0": version: 1.15.0