/* Copyright 2025 Element Creations Ltd. SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ import { type LocalParticipant, type RemoteParticipant } from "livekit-client"; import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc"; import { combineLatest, filter, map } from "rxjs"; import { logger } from "matrix-js-sdk/lib/logger"; import { type Behavior } from "../../Behavior"; import { type ConnectionMap } from "./ConnectionMap"; import { Epoch, type ObservableScope } from "../../ObservableScope"; import { type Connection } from "./Connection"; import { generateItemsWithEpoch } from "../../../utils/observable"; import { type TransportLocator } from "../../../livekit/auth"; interface LocalTaggedParticipant { type: "local"; value$: Behavior; } interface RemoteTaggedParticipant { type: "remote"; value$: Behavior; } export type TaggedParticipant = | LocalTaggedParticipant | RemoteTaggedParticipant; export interface MatrixLivekitMember { membership$: Behavior; connection$: Behavior; // participantId: string; We do not want a participantId here since it will be generated by the jwt // TODO decide if we can also drop the userId. Its in the matrix membership anyways. userId: string; } /** * Represents the local Matrix call member and their associated LiveKit participation. * `livekitParticipant` can be null if the member is not yet connected to the livekit room * or if it has no livekit transport at all. */ export interface LocalMatrixLivekitMember extends MatrixLivekitMember { participant: LocalTaggedParticipant; } /** * Represents a remote Matrix call member and their associated LiveKit participation. * `livekitParticipant` can be null if the member is not yet connected to the livekit room * or if it has no livekit transport at all. */ export interface RemoteMatrixLivekitMember extends MatrixLivekitMember { participant: RemoteTaggedParticipant; } interface Props { scope: ObservableScope; membershipsWithTransport$: Behavior< Epoch< { membership: CallMembership; transport: TransportLocator | undefined }[] > >; connectionMap$: Behavior>; localUser: { deviceId: string; userId: string }; } /** * Combines MatrixRTC and Livekit worlds. * * It has a small public interface: * - in (via constructor): * - an observable of CallMembership[] to track the call members (The matrix side) * - a `ConnectionMap` for the lk rooms (The livekit side) * - out (via public Observable): * - `remoteMatrixLivekitMember` an observable of MatrixLivekitMember[] to track the remote members and associated livekit data. */ export function createRemoteMatrixLivekitMembers$({ scope, membershipsWithTransport$, connectionMap$, localUser, }: Props): Behavior> { /** * Behavior of all the remote call members and their associated livekit data (if available). */ return scope.behavior( combineLatest([membershipsWithTransport$, connectionMap$]).pipe( filter((values) => values.every((value) => value.epoch === values[0].epoch), ), map(([ms, data]) => new Epoch([ms.value, data.value] as const, ms.epoch)), generateItemsWithEpoch( "RemoteMatrixLivekitMembers", // Generator function. // creates an array of `{key, data}[]` // Each change in the keys (new key) will result in a call to the factory function. function* ([membershipsWithTransport, managerData]) { for (const { membership, transport } of membershipsWithTransport) { // Exclude the local membership if ( membership.userId === localUser.userId && membership.deviceId === localUser.deviceId ) continue; const participants = transport ? managerData.getParticipantsForTransport(transport) : []; const matches = participants.filter((p) => membership.backendIdentities.includes(p.identity), ); const participant = matches[0] ?? null; const connection = transport ? managerData.getConnectionForTransport(transport) : null; if (matches.length > 1) logger.warn( `[RemoteMatrixLivekitMembers] ${membership.rtcBackendIdentity}: ${matches.length} LiveKit participants match (sids ${matches.map((p) => p.sid).join(", ")}), using ${participant?.sid}`, ); yield { // This could just be the backend identity without the other keys. // The user ID, device ID, and member ID are included however so // they show up in debug logs. keys: [ membership.userId, membership.deviceId, membership.memberId, membership.rtcBackendIdentity, ], data: { membership, participant, connection }, }; } }, // Each update where the key of the generator array do not change will result in updates to the `data$` behavior. (scope, data$, userId, _deviceId, _memberId, rtcBackendIdentity) => { const { participant$, ...rest } = scope.splitBehavior(data$); // Log whether the member could be matched to a LiveKit participant, // since a tile shows "waiting for media" for as long as it cannot. participant$.pipe(scope.bind()).subscribe((p) => { const transport = data$.value.connection?.transport; logger.info( `[RemoteMatrixLivekitMembers] ${rtcBackendIdentity}: LiveKit participant ${p ? `matched (${p.sid})` : "missing"} on ${transport ? JSON.stringify(transport) : "no connection"}`, ); }); // will only get called once per backend identity. // updates to data$ and as a result to displayName$ and mxcAvatarUrl$ are more frequent. return { userId, participant: { type: "remote" as const, value$: participant$ }, ...rest, }; }, ), ), new Epoch([], -1), ); } // TODO add back in the callviewmodel pauseWhen(this.pretendToBeDisconnected$)