Don't render audio from participants that aren't meant to be publishing

src/livekit/MatrixAudioRenderer.tsx

@@ -6,7 +6,7 @@ Please see LICENSE in the repository root for full details.
 */
 
 import { getTrackReferenceId } from "@livekit/components-core";
-import { type Room as LivekitRoom } from "livekit-client";
+import { type Room as LivekitRoom, type Participant } from "livekit-client";
 import { type RemoteAudioTrack, Track } from "livekit-client";
 import { useEffect, useMemo, useRef, useState, type ReactNode } from "react";
 import {
@@ -14,7 +14,7 @@ import {
   AudioTrack,
   type AudioTrackProps,
 } from "@livekit/components-react";
-import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc";
+import { type RoomMember } from "matrix-js-sdk";
 import { logger } from "matrix-js-sdk/lib/logger";
 
 import { useEarpieceAudioConfig } from "../MediaDevicesContext";
@@ -22,13 +22,20 @@ import { useReactiveState } from "../useReactiveState";
 import * as controls from "../controls";
 import {} from "@livekit/components-core";
 export interface MatrixAudioRendererProps {
+  /**
+   * The service URL of the LiveKit room.
+   */
+  url: string;
+  livekitRoom: LivekitRoom;
   /**
    * The list of participants to render audio for.
    * This list needs to be composed based on the matrixRTC members so that we do not play audio from users
    * that are not expected to be in the rtc session.
    */
-  members: CallMembership[];
-  livekitRoom: LivekitRoom;
+  participants: {
+    participant: Participant;
+    member: RoomMember;
+  }[];
   /**
    * If set to `true`, mutes all audio tracks rendered by the component.
    * @remarks
@@ -52,14 +59,14 @@ export interface MatrixAudioRendererProps {
  * @public
  */
 export function LivekitRoomAudioRenderer({
-  members,
-  muted,
+  url,
   livekitRoom,
+  participants,
+  muted,
 }: MatrixAudioRendererProps): ReactNode {
-  const validIdentities = useMemo(
-    () =>
-      new Set(members?.map((member) => `${member.sender}:${member.deviceId}`)),
-    [members],
+  const participantSet = useMemo(
+    () => new Set(participants.map(({ participant }) => participant)),
+    [participants],
   );
 
   const loggedInvalidIdentities = useRef(new Set<string>());
@@ -71,11 +78,11 @@ export function LivekitRoomAudioRenderer({
    * @param identity The identity of the track that is invalid
    * @param validIdentities The list of valid identities
    */
-  const logInvalid = (identity: string, validIdentities: Set<string>): void => {
+  const logInvalid = (identity: string): void => {
     if (loggedInvalidIdentities.current.has(identity)) return;
     logger.warn(
-      `[MatrixAudioRenderer] Audio track ${identity} has no matching matrix call member`,
-      `current members: ${Array.from(validIdentities.values())}`,
+      `[MatrixAudioRenderer] Audio track ${identity} from ${url} has no matching matrix call member`,
+      `current members: ${participants.map((p) => p.participant.identity)}`,
       `track will not get rendered`,
     );
     loggedInvalidIdentities.current.add(identity);
@@ -93,23 +100,27 @@ export function LivekitRoomAudioRenderer({
       room: livekitRoom,
     },
   ).filter((ref) => {
-    const isValid = validIdentities?.has(ref.participant.identity);
+    const isValid = participantSet?.has(ref.participant);
     if (!isValid && !ref.participant.isLocal)
-      logInvalid(ref.participant.identity, validIdentities);
+      logInvalid(ref.participant.identity);
     return (
       !ref.participant.isLocal &&
       ref.publication.kind === Track.Kind.Audio &&
       isValid
     );
   });
+
   useEffect(() => {
-    if (!tracks.some((t) => !validIdentities.has(t.participant.identity))) {
+    if (
+      loggedInvalidIdentities.current.size &&
+      tracks.every((t) => participantSet.has(t.participant))
+    ) {
       logger.debug(
-        `[MatrixAudioRenderer] All audio tracks have a matching matrix call member identity.`,
+        `[MatrixAudioRenderer] All audio tracks from ${url} have a matching matrix call member identity.`,
       );
       loggedInvalidIdentities.current.clear();
     }
-  }, [tracks, validIdentities]);
+  }, [tracks, participantSet, url]);
 
   // This component is also (in addition to the "only play audio for connected members" logic above)
   // responsible for mimicking earpiece audio on iPhones.

src/room/InCallView.tsx

@@ -286,6 +286,8 @@ export const InCallView: FC<InCallViewProps> = ({
     () => void toggleRaisedHand(),
   );
 
+  const allLivekitRooms = useBehavior(vm.allLivekitRooms$);
+  const participantsByRoom = useBehavior(vm.participantsByRoom$);
   const participantCount = useBehavior(vm.participantCount$);
   const reconnecting = useBehavior(vm.reconnecting$);
   const windowMode = useBehavior(vm.windowMode$);
@@ -739,9 +741,6 @@ export const InCallView: FC<InCallViewProps> = ({
     matrixRoom.roomId,
   );
 
-  const allLivekitRooms = useBehavior(vm.allLivekitRooms$);
-  const memberships = useBehavior(vm.memberships$);
-
   const buttons: JSX.Element[] = [];
 
   buttons.push(
@@ -862,11 +861,12 @@ export const InCallView: FC<InCallViewProps> = ({
           </Text>
         )
       }
-      {allLivekitRooms.map((roomItem) => (
+      {participantsByRoom.map(({ livekitRoom, url, participants }) => (
         <LivekitRoomAudioRenderer
-          key={roomItem.url}
-          livekitRoom={roomItem.room}
-          members={memberships}
+          key={url}
+          url={url}
+          livekitRoom={livekitRoom}
+          participants={participants}
           muted={muteAllAudio}
         />
       ))}

src/state/CallViewModel.ts

@@ -480,7 +480,7 @@ export class CallViewModel extends ViewModel {
    */
   // Note that MatrixRTCSession already filters the call memberships by users
   // that are joined to the room; we don't need to perform extra filtering here.
-  public readonly memberships$ = this.scope.behavior(
+  private readonly memberships$ = this.scope.behavior(
     fromEvent(
       this.matrixRTCSession,
       MatrixRTCSessionEvent.MembershipsChanged,
@@ -679,16 +679,19 @@ export class CallViewModel extends ViewModel {
   // in a split-brained state.
   private readonly pretendToBeDisconnected$ = this.reconnecting$;
 
-  private readonly participants$ = this.scope.behavior<
+  public readonly participantsByRoom$ = this.scope.behavior<
     {
-      participant: LocalParticipant | RemoteParticipant;
-      member: RoomMember;
       livekitRoom: LivekitRoom;
+      url: string;
+      participants: {
+        participant: LocalParticipant | RemoteParticipant;
+        member: RoomMember;
+      }[];
     }[]
   >(
-    from(this.localConnection)
+    combineLatest([this.localConnection, this.localFocus])
       .pipe(
-        switchMap((localConnection) => {
+        switchMap(([localConnection, localFocus]) => {
           const memberError = (): never => {
             throw new Error("No room member for call membership");
           };
@@ -696,32 +699,41 @@ export class CallViewModel extends ViewModel {
             participant: localConnection.livekitRoom.localParticipant,
             member:
               this.matrixRoom.getMember(this.userId ?? "") ?? memberError(),
-            livekitRoom: localConnection.livekitRoom,
           };
+
           return this.remoteConnections$.pipe(
             switchMap((connections) =>
               combineLatest(
-                [localConnection, ...connections.values()].map((c) =>
+                [
+                  [localFocus.livekit_service_url, localConnection] as const,
+                  ...connections,
+                ].map(([url, c]) =>
                   c.publishingParticipants$.pipe(
-                    map((ps) =>
-                      ps.map(({ participant, membership }) => ({
+                    map((ps) => {
+                      const participants: {
+                        participant: LocalParticipant | RemoteParticipant;
+                        member: RoomMember;
+                      }[] = ps.map(({ participant, membership }) => ({
                         participant,
                         member:
                           getRoomMemberFromRtcMember(
                             membership,
                             this.matrixRoom,
                           )?.member ?? memberError(),
+                      }));
+                      if (c === localConnection)
+                        participants.push(localParticipant);
+
+                      return {
                         livekitRoom: c.livekitRoom,
-                      })),
-                    ),
+                        url,
+                        participants,
+                      };
+                    }),
                   ),
                 ),
               ),
             ),
-            map((remoteParticipants) => [
-              localParticipant,
-              ...remoteParticipants.flat(1),
-            ]),
           );
         }),
       )
@@ -798,7 +810,7 @@ export class CallViewModel extends ViewModel {
    */
   private readonly mediaItems$ = this.scope.behavior<MediaItem[]>(
     combineLatest([
-      this.participants$,
+      this.participantsByRoom$,
       duplicateTiles.value$,
       this.memberships$,
       showNonMemberTiles.value$,
@@ -806,71 +818,75 @@ export class CallViewModel extends ViewModel {
       scan(
         (
           prevItems,
-          [participants, duplicateTiles, memberships, showNonMemberTiles],
+          [participantsByRoom, duplicateTiles, memberships, showNonMemberTiles],
         ) => {
           const newItems: Map<string, UserMedia | ScreenShare> = new Map(
             function* (this: CallViewModel): Iterable<[string, MediaItem]> {
-              for (const { participant, member, livekitRoom } of participants) {
-                const matrixId = participant.isLocal
-                  ? "local"
-                  : participant.identity;
-                for (let i = 0; i < 1 + duplicateTiles; i++) {
-                  const mediaId = `${matrixId}:${i}`;
-                  let prevMedia = prevItems.get(mediaId);
-                  if (prevMedia && prevMedia instanceof UserMedia) {
-                    prevMedia.updateParticipant(participant);
-                    if (prevMedia.vm.member === undefined) {
-                      // We have a previous media created because of the `debugShowNonMember` flag.
-                      // In this case we actually replace the media item.
-                      // This "hack" never occurs if we do not use the `debugShowNonMember` debugging
-                      // option and if we always find a room member for each rtc member (which also
-                      // only fails if we have a fundamental problem)
-                      prevMedia = undefined;
-                    }
-                  }
-                  yield [
-                    mediaId,
-                    // We create UserMedia with or without a participant.
-                    // This will be the initial value of a BehaviourSubject.
-                    // Once a participant appears we will update the BehaviourSubject. (see above)
-                    prevMedia ??
-                      new UserMedia(
-                        mediaId,
-                        member,
-                        participant,
-                        this.options.encryptionSystem,
-                        livekitRoom,
-                        this.mediaDevices,
-                        this.pretendToBeDisconnected$,
-                        this.memberDisplaynames$.pipe(
-                          map((m) => m.get(matrixId) ?? "[👻]"),
-                        ),
-                        this.handsRaised$.pipe(
-                          map((v) => v[matrixId]?.time ?? null),
-                        ),
-                        this.reactions$.pipe(
-                          map((v) => v[matrixId] ?? undefined),
-                        ),
-                      ),
-                  ];
+              for (const { livekitRoom, participants } of participantsByRoom) {
+                for (const { participant, member } of participants) {
+                  const matrixId = participant.isLocal
+                    ? "local"
+                    : participant.identity;
+
+                  for (let i = 0; i < 1 + duplicateTiles; i++) {
+                    const mediaId = `${matrixId}:${i}`;
+                    let prevMedia = prevItems.get(mediaId);
+                    if (prevMedia && prevMedia instanceof UserMedia) {
+                      prevMedia.updateParticipant(participant);
+                      if (prevMedia.vm.member === undefined) {
+                        // We have a previous media created because of the `debugShowNonMember` flag.
+                        // In this case we actually replace the media item.
+                        // This "hack" never occurs if we do not use the `debugShowNonMember` debugging
+                        // option and if we always find a room member for each rtc member (which also
+                        // only fails if we have a fundamental problem)
+                        prevMedia = undefined;
+                      }
+                    }
 
-                  if (participant?.isScreenShareEnabled) {
-                    const screenShareId = `${mediaId}:screen-share`;
                     yield [
-                      screenShareId,
-                      prevItems.get(screenShareId) ??
-                        new ScreenShare(
-                          screenShareId,
+                      mediaId,
+                      // We create UserMedia with or without a participant.
+                      // This will be the initial value of a BehaviourSubject.
+                      // Once a participant appears we will update the BehaviourSubject. (see above)
+                      prevMedia ??
+                        new UserMedia(
+                          mediaId,
                           member,
                           participant,
                           this.options.encryptionSystem,
                           livekitRoom,
+                          this.mediaDevices,
                           this.pretendToBeDisconnected$,
                           this.memberDisplaynames$.pipe(
                             map((m) => m.get(matrixId) ?? "[👻]"),
                           ),
+                          this.handsRaised$.pipe(
+                            map((v) => v[matrixId]?.time ?? null),
+                          ),
+                          this.reactions$.pipe(
+                            map((v) => v[matrixId] ?? undefined),
+                          ),
                         ),
                     ];
+
+                    if (participant?.isScreenShareEnabled) {
+                      const screenShareId = `${mediaId}:screen-share`;
+                      yield [
+                        screenShareId,
+                        prevItems.get(screenShareId) ??
+                          new ScreenShare(
+                            screenShareId,
+                            member,
+                            participant,
+                            this.options.encryptionSystem,
+                            livekitRoom,
+                            this.pretendToBeDisconnected$,
+                            this.memberDisplaynames$.pipe(
+                              map((m) => m.get(matrixId) ?? "[👻]"),
+                            ),
+                          ),
+                      ];
+                    }
                   }
                 }
               }

src/state/Connection.ts

@@ -93,11 +93,9 @@ export class Connection {
     );
 
     this.publishingParticipants$ = this.scope.behavior(
-      combineLatest([
-        this.participantsIncludingSubscribers$,
-        this.membershipsFocusMap$,
-      ]).pipe(
-        map(([participants, membershipsFocusMap]) =>
+      combineLatest(
+        [this.participantsIncludingSubscribers$, this.membershipsFocusMap$],
+        (participants, membershipsFocusMap) =>
           membershipsFocusMap
             // Find all members that claim to publish on this connection
             .flatMap(({ membership, focus }) =>
@@ -113,7 +111,6 @@ export class Connection {
               );
               return participant ? [{ participant, membership }] : [];
             }),
-        ),
       ),
       [],
     );