diff --git a/src/livekit/MatrixAudioRenderer.tsx b/src/livekit/MatrixAudioRenderer.tsx index 0b87b27b..f402b32d 100644 --- a/src/livekit/MatrixAudioRenderer.tsx +++ b/src/livekit/MatrixAudioRenderer.tsx @@ -6,7 +6,7 @@ Please see LICENSE in the repository root for full details. */ import { getTrackReferenceId } from "@livekit/components-core"; -import { type Room as LivekitRoom } from "livekit-client"; +import { type Room as LivekitRoom, type Participant } from "livekit-client"; import { type RemoteAudioTrack, Track } from "livekit-client"; import { useEffect, useMemo, useRef, useState, type ReactNode } from "react"; import { @@ -14,7 +14,7 @@ import { AudioTrack, type AudioTrackProps, } from "@livekit/components-react"; -import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc"; +import { type RoomMember } from "matrix-js-sdk"; import { logger } from "matrix-js-sdk/lib/logger"; import { useEarpieceAudioConfig } from "../MediaDevicesContext"; @@ -22,13 +22,20 @@ import { useReactiveState } from "../useReactiveState"; import * as controls from "../controls"; import {} from "@livekit/components-core"; export interface MatrixAudioRendererProps { + /** + * The service URL of the LiveKit room. + */ + url: string; + livekitRoom: LivekitRoom; /** * The list of participants to render audio for. * This list needs to be composed based on the matrixRTC members so that we do not play audio from users * that are not expected to be in the rtc session. */ - members: CallMembership[]; - livekitRoom: LivekitRoom; + participants: { + participant: Participant; + member: RoomMember; + }[]; /** * If set to `true`, mutes all audio tracks rendered by the component. * @remarks @@ -52,14 +59,14 @@ export interface MatrixAudioRendererProps { * @public */ export function LivekitRoomAudioRenderer({ - members, - muted, + url, livekitRoom, + participants, + muted, }: MatrixAudioRendererProps): ReactNode { - const validIdentities = useMemo( - () => - new Set(members?.map((member) => `${member.sender}:${member.deviceId}`)), - [members], + const participantSet = useMemo( + () => new Set(participants.map(({ participant }) => participant)), + [participants], ); const loggedInvalidIdentities = useRef(new Set()); @@ -71,11 +78,11 @@ export function LivekitRoomAudioRenderer({ * @param identity The identity of the track that is invalid * @param validIdentities The list of valid identities */ - const logInvalid = (identity: string, validIdentities: Set): void => { + const logInvalid = (identity: string): void => { if (loggedInvalidIdentities.current.has(identity)) return; logger.warn( - `[MatrixAudioRenderer] Audio track ${identity} has no matching matrix call member`, - `current members: ${Array.from(validIdentities.values())}`, + `[MatrixAudioRenderer] Audio track ${identity} from ${url} has no matching matrix call member`, + `current members: ${participants.map((p) => p.participant.identity)}`, `track will not get rendered`, ); loggedInvalidIdentities.current.add(identity); @@ -93,23 +100,27 @@ export function LivekitRoomAudioRenderer({ room: livekitRoom, }, ).filter((ref) => { - const isValid = validIdentities?.has(ref.participant.identity); + const isValid = participantSet?.has(ref.participant); if (!isValid && !ref.participant.isLocal) - logInvalid(ref.participant.identity, validIdentities); + logInvalid(ref.participant.identity); return ( !ref.participant.isLocal && ref.publication.kind === Track.Kind.Audio && isValid ); }); + useEffect(() => { - if (!tracks.some((t) => !validIdentities.has(t.participant.identity))) { + if ( + loggedInvalidIdentities.current.size && + tracks.every((t) => participantSet.has(t.participant)) + ) { logger.debug( - `[MatrixAudioRenderer] All audio tracks have a matching matrix call member identity.`, + `[MatrixAudioRenderer] All audio tracks from ${url} have a matching matrix call member identity.`, ); loggedInvalidIdentities.current.clear(); } - }, [tracks, validIdentities]); + }, [tracks, participantSet, url]); // This component is also (in addition to the "only play audio for connected members" logic above) // responsible for mimicking earpiece audio on iPhones. diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 20d53b3a..db2c0f2a 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -286,6 +286,8 @@ export const InCallView: FC = ({ () => void toggleRaisedHand(), ); + const allLivekitRooms = useBehavior(vm.allLivekitRooms$); + const participantsByRoom = useBehavior(vm.participantsByRoom$); const participantCount = useBehavior(vm.participantCount$); const reconnecting = useBehavior(vm.reconnecting$); const windowMode = useBehavior(vm.windowMode$); @@ -739,9 +741,6 @@ export const InCallView: FC = ({ matrixRoom.roomId, ); - const allLivekitRooms = useBehavior(vm.allLivekitRooms$); - const memberships = useBehavior(vm.memberships$); - const buttons: JSX.Element[] = []; buttons.push( @@ -862,11 +861,12 @@ export const InCallView: FC = ({ ) } - {allLivekitRooms.map((roomItem) => ( + {participantsByRoom.map(({ livekitRoom, url, participants }) => ( ))} diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 3b1cdc96..3dff08d3 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -480,7 +480,7 @@ export class CallViewModel extends ViewModel { */ // Note that MatrixRTCSession already filters the call memberships by users // that are joined to the room; we don't need to perform extra filtering here. - public readonly memberships$ = this.scope.behavior( + private readonly memberships$ = this.scope.behavior( fromEvent( this.matrixRTCSession, MatrixRTCSessionEvent.MembershipsChanged, @@ -679,16 +679,19 @@ export class CallViewModel extends ViewModel { // in a split-brained state. private readonly pretendToBeDisconnected$ = this.reconnecting$; - private readonly participants$ = this.scope.behavior< + public readonly participantsByRoom$ = this.scope.behavior< { - participant: LocalParticipant | RemoteParticipant; - member: RoomMember; livekitRoom: LivekitRoom; + url: string; + participants: { + participant: LocalParticipant | RemoteParticipant; + member: RoomMember; + }[]; }[] >( - from(this.localConnection) + combineLatest([this.localConnection, this.localFocus]) .pipe( - switchMap((localConnection) => { + switchMap(([localConnection, localFocus]) => { const memberError = (): never => { throw new Error("No room member for call membership"); }; @@ -696,32 +699,41 @@ export class CallViewModel extends ViewModel { participant: localConnection.livekitRoom.localParticipant, member: this.matrixRoom.getMember(this.userId ?? "") ?? memberError(), - livekitRoom: localConnection.livekitRoom, }; + return this.remoteConnections$.pipe( switchMap((connections) => combineLatest( - [localConnection, ...connections.values()].map((c) => + [ + [localFocus.livekit_service_url, localConnection] as const, + ...connections, + ].map(([url, c]) => c.publishingParticipants$.pipe( - map((ps) => - ps.map(({ participant, membership }) => ({ + map((ps) => { + const participants: { + participant: LocalParticipant | RemoteParticipant; + member: RoomMember; + }[] = ps.map(({ participant, membership }) => ({ participant, member: getRoomMemberFromRtcMember( membership, this.matrixRoom, )?.member ?? memberError(), + })); + if (c === localConnection) + participants.push(localParticipant); + + return { livekitRoom: c.livekitRoom, - })), - ), + url, + participants, + }; + }), ), ), ), ), - map((remoteParticipants) => [ - localParticipant, - ...remoteParticipants.flat(1), - ]), ); }), ) @@ -798,7 +810,7 @@ export class CallViewModel extends ViewModel { */ private readonly mediaItems$ = this.scope.behavior( combineLatest([ - this.participants$, + this.participantsByRoom$, duplicateTiles.value$, this.memberships$, showNonMemberTiles.value$, @@ -806,71 +818,75 @@ export class CallViewModel extends ViewModel { scan( ( prevItems, - [participants, duplicateTiles, memberships, showNonMemberTiles], + [participantsByRoom, duplicateTiles, memberships, showNonMemberTiles], ) => { const newItems: Map = new Map( function* (this: CallViewModel): Iterable<[string, MediaItem]> { - for (const { participant, member, livekitRoom } of participants) { - const matrixId = participant.isLocal - ? "local" - : participant.identity; - for (let i = 0; i < 1 + duplicateTiles; i++) { - const mediaId = `${matrixId}:${i}`; - let prevMedia = prevItems.get(mediaId); - if (prevMedia && prevMedia instanceof UserMedia) { - prevMedia.updateParticipant(participant); - if (prevMedia.vm.member === undefined) { - // We have a previous media created because of the `debugShowNonMember` flag. - // In this case we actually replace the media item. - // This "hack" never occurs if we do not use the `debugShowNonMember` debugging - // option and if we always find a room member for each rtc member (which also - // only fails if we have a fundamental problem) - prevMedia = undefined; - } - } - yield [ - mediaId, - // We create UserMedia with or without a participant. - // This will be the initial value of a BehaviourSubject. - // Once a participant appears we will update the BehaviourSubject. (see above) - prevMedia ?? - new UserMedia( - mediaId, - member, - participant, - this.options.encryptionSystem, - livekitRoom, - this.mediaDevices, - this.pretendToBeDisconnected$, - this.memberDisplaynames$.pipe( - map((m) => m.get(matrixId) ?? "[👻]"), - ), - this.handsRaised$.pipe( - map((v) => v[matrixId]?.time ?? null), - ), - this.reactions$.pipe( - map((v) => v[matrixId] ?? undefined), - ), - ), - ]; + for (const { livekitRoom, participants } of participantsByRoom) { + for (const { participant, member } of participants) { + const matrixId = participant.isLocal + ? "local" + : participant.identity; + + for (let i = 0; i < 1 + duplicateTiles; i++) { + const mediaId = `${matrixId}:${i}`; + let prevMedia = prevItems.get(mediaId); + if (prevMedia && prevMedia instanceof UserMedia) { + prevMedia.updateParticipant(participant); + if (prevMedia.vm.member === undefined) { + // We have a previous media created because of the `debugShowNonMember` flag. + // In this case we actually replace the media item. + // This "hack" never occurs if we do not use the `debugShowNonMember` debugging + // option and if we always find a room member for each rtc member (which also + // only fails if we have a fundamental problem) + prevMedia = undefined; + } + } - if (participant?.isScreenShareEnabled) { - const screenShareId = `${mediaId}:screen-share`; yield [ - screenShareId, - prevItems.get(screenShareId) ?? - new ScreenShare( - screenShareId, + mediaId, + // We create UserMedia with or without a participant. + // This will be the initial value of a BehaviourSubject. + // Once a participant appears we will update the BehaviourSubject. (see above) + prevMedia ?? + new UserMedia( + mediaId, member, participant, this.options.encryptionSystem, livekitRoom, + this.mediaDevices, this.pretendToBeDisconnected$, this.memberDisplaynames$.pipe( map((m) => m.get(matrixId) ?? "[👻]"), ), + this.handsRaised$.pipe( + map((v) => v[matrixId]?.time ?? null), + ), + this.reactions$.pipe( + map((v) => v[matrixId] ?? undefined), + ), ), ]; + + if (participant?.isScreenShareEnabled) { + const screenShareId = `${mediaId}:screen-share`; + yield [ + screenShareId, + prevItems.get(screenShareId) ?? + new ScreenShare( + screenShareId, + member, + participant, + this.options.encryptionSystem, + livekitRoom, + this.pretendToBeDisconnected$, + this.memberDisplaynames$.pipe( + map((m) => m.get(matrixId) ?? "[👻]"), + ), + ), + ]; + } } } } diff --git a/src/state/Connection.ts b/src/state/Connection.ts index e14ddc9a..2513382c 100644 --- a/src/state/Connection.ts +++ b/src/state/Connection.ts @@ -93,11 +93,9 @@ export class Connection { ); this.publishingParticipants$ = this.scope.behavior( - combineLatest([ - this.participantsIncludingSubscribers$, - this.membershipsFocusMap$, - ]).pipe( - map(([participants, membershipsFocusMap]) => + combineLatest( + [this.participantsIncludingSubscribers$, this.membershipsFocusMap$], + (participants, membershipsFocusMap) => membershipsFocusMap // Find all members that claim to publish on this connection .flatMap(({ membership, focus }) => @@ -113,7 +111,6 @@ export class Connection { ); return participant ? [{ participant, membership }] : []; }), - ), ), [], );