element-call-Github/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts

/*
Copyright 2025 Element Creations Ltd.

SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/

import { type LocalParticipant, type RemoteParticipant } from "livekit-client";
import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc";
import { combineLatest, filter, map } from "rxjs";
import { logger } from "matrix-js-sdk/lib/logger";

import { type Behavior } from "../../Behavior";
import { type ConnectionMap } from "./ConnectionMap";
import { Epoch, type ObservableScope } from "../../ObservableScope";
import { type Connection } from "./Connection";
import { generateItemsWithEpoch } from "../../../utils/observable";
import { type TransportLocator } from "../../../livekit/auth";

interface LocalTaggedParticipant {
  type: "local";
  value$: Behavior<LocalParticipant | null>;
}
interface RemoteTaggedParticipant {
  type: "remote";
  value$: Behavior<RemoteParticipant | null>;
}
export type TaggedParticipant =
  | LocalTaggedParticipant
  | RemoteTaggedParticipant;

export interface MatrixLivekitMember {
  membership$: Behavior<CallMembership>;
  connection$: Behavior<Connection | null>;
  // participantId: string; We do not want a participantId here since it will be generated by the jwt
  // TODO decide if we can also drop the userId. Its in the matrix membership anyways.
  userId: string;
}

/**
 * Represents the local Matrix call member and their associated LiveKit participation.
 * `livekitParticipant` can be null if the member is not yet connected to the livekit room
 * or if it has no livekit transport at all.
 */
export interface LocalMatrixLivekitMember extends MatrixLivekitMember {
  participant: LocalTaggedParticipant;
}

/**
 * Represents a remote Matrix call member and their associated LiveKit participation.
 * `livekitParticipant` can be null if the member is not yet connected to the livekit room
 * or if it has no livekit transport at all.
 */
export interface RemoteMatrixLivekitMember extends MatrixLivekitMember {
  participant: RemoteTaggedParticipant;
}

interface Props {
  scope: ObservableScope;
  membershipsWithTransport$: Behavior<
    Epoch<
      { membership: CallMembership; transport: TransportLocator | undefined }[]
    >
  >;
  connectionMap$: Behavior<Epoch<ConnectionMap>>;
  localUser: { deviceId: string; userId: string };
}

/**
 * Combines MatrixRTC and Livekit worlds.
 *
 * It has a small public interface:
 *  - in (via constructor):
 *    - an observable of CallMembership[] to track the call members (The matrix side)
 *    - a `ConnectionMap` for the lk rooms (The livekit side)
 *  - out (via public Observable):
 *    - `remoteMatrixLivekitMember` an observable of MatrixLivekitMember[] to track the remote members and associated livekit data.
 */
export function createRemoteMatrixLivekitMembers$({
  scope,
  membershipsWithTransport$,
  connectionMap$,
  localUser,
}: Props): Behavior<Epoch<RemoteMatrixLivekitMember[]>> {
  /**
   * Behavior of all the remote call members and their associated livekit data (if available).
   */
  return scope.behavior(
    combineLatest([membershipsWithTransport$, connectionMap$]).pipe(
      filter((values) =>
        values.every((value) => value.epoch === values[0].epoch),
      ),
      map(([ms, data]) => new Epoch([ms.value, data.value] as const, ms.epoch)),
      generateItemsWithEpoch(
        "RemoteMatrixLivekitMembers",
        // Generator function.
        // creates an array of `{key, data}[]`
        // Each change in the keys (new key) will result in a call to the factory function.
        function* ([membershipsWithTransport, managerData]) {
          for (const { membership, transport } of membershipsWithTransport) {
            // Exclude the local membership
            if (
              membership.userId === localUser.userId &&
              membership.deviceId === localUser.deviceId
            )
              continue;

            const participants = transport
              ? managerData.getParticipantsForTransport(transport)
              : [];
            const matches = participants.filter((p) =>
              membership.backendIdentities.includes(p.identity),
            );
            const participant = matches[0] ?? null;
            const connection = transport
              ? managerData.getConnectionForTransport(transport)
              : null;
            if (matches.length > 1)
              logger.warn(
                `[RemoteMatrixLivekitMembers] ${membership.rtcBackendIdentity}: ${matches.length} LiveKit participants match (sids ${matches.map((p) => p.sid).join(", ")}), using ${participant?.sid}`,
              );

            yield {
              // This could just be the backend identity without the other keys.
              // The user ID, device ID, and member ID are included however so
              // they show up in debug logs.
              keys: [
                membership.userId,
                membership.deviceId,
                membership.memberId,
                membership.rtcBackendIdentity,
              ],
              data: { membership, participant, connection },
            };
          }
        },
        // Each update where the key of the generator array do not change will result in updates to the `data$` behavior.
        (scope, data$, userId, _deviceId, _memberId, rtcBackendIdentity) => {
          const { participant$, ...rest } = scope.splitBehavior(data$);
          // Log whether the member could be matched to a LiveKit participant,
          // since a tile shows "waiting for media" for as long as it cannot.
          participant$.pipe(scope.bind()).subscribe((p) => {
            const transport = data$.value.connection?.transport;
            logger.info(
              `[RemoteMatrixLivekitMembers] ${rtcBackendIdentity}: LiveKit participant ${p ? `matched (${p.sid})` : "missing"} on ${transport ? JSON.stringify(transport) : "no connection"}`,
            );
          });
          // will only get called once per backend identity.
          // updates to data$ and as a result to displayName$ and mxcAvatarUrl$ are more frequent.
          return {
            userId,
            participant: { type: "remote" as const, value$: participant$ },
            ...rest,
          };
        },
      ),
    ),
    new Epoch([], -1),
  );
}

// TODO add back in the callviewmodel pauseWhen(this.pretendToBeDisconnected$)