make audio work

Signed-off-by: Timo K <toger5@hotmail.de>

src/livekit/MatrixAudioRenderer.test.tsx

@@ -18,7 +18,7 @@ import { useTracks } from "@livekit/components-react";
 
 import { testAudioContext } from "../useAudioContext.test";
 import * as MediaDevicesContext from "../MediaDevicesContext";
-import { MatrixAudioRenderer } from "./MatrixAudioRenderer";
+import { LivekitRoomAudioRenderer } from "./MatrixAudioRenderer";
 import { mockMediaDevices, mockTrack } from "../utils/test";
 
 export const TestAudioContextConstructor = vi.fn(() => testAudioContext);
@@ -54,7 +54,7 @@ vi.mocked(useTracks).mockReturnValue(tracks);
 it("should render for member", () => {
   const { container, queryAllByTestId } = render(
     <MediaDevicesProvider value={mockMediaDevices({})}>
-      <MatrixAudioRenderer
+      <LivekitRoomAudioRenderer
         members={[{ sender: "test", deviceId: "123" }] as CallMembership[]}
       />
     </MediaDevicesProvider>,
@@ -69,7 +69,7 @@ it("should not render without member", () => {
   ] as CallMembership[];
   const { container, queryAllByTestId } = render(
     <MediaDevicesProvider value={mockMediaDevices({})}>
-      <MatrixAudioRenderer members={memberships} />
+      <LivekitRoomAudioRenderer members={memberships} />
     </MediaDevicesProvider>,
   );
   expect(container).toBeTruthy();
@@ -79,7 +79,7 @@ it("should not render without member", () => {
 it("should not setup audioContext gain and pan if there is no need to.", () => {
   render(
     <MediaDevicesProvider value={mockMediaDevices({})}>
-      <MatrixAudioRenderer
+      <LivekitRoomAudioRenderer
         members={[{ sender: "test", deviceId: "123" }] as CallMembership[]}
       />
     </MediaDevicesProvider>,
@@ -102,7 +102,7 @@ it("should setup audioContext gain and pan", () => {
   });
   render(
     <MediaDevicesProvider value={mockMediaDevices({})}>
-      <MatrixAudioRenderer
+      <LivekitRoomAudioRenderer
         members={[{ sender: "test", deviceId: "123" }] as CallMembership[]}
       />
     </MediaDevicesProvider>,

src/livekit/MatrixAudioRenderer.tsx

@@ -6,6 +6,7 @@ Please see LICENSE in the repository root for full details.
 */
 
 import { getTrackReferenceId } from "@livekit/components-core";
+import { type Room as LivekitRoom } from "livekit-client";
 import { type RemoteAudioTrack, Track } from "livekit-client";
 import { useEffect, useMemo, useRef, useState, type ReactNode } from "react";
 import {
@@ -19,7 +20,7 @@ import { logger } from "matrix-js-sdk/lib/logger";
 import { useEarpieceAudioConfig } from "../MediaDevicesContext";
 import { useReactiveState } from "../useReactiveState";
 import * as controls from "../controls";
-
+import {} from "@livekit/components-core";
 export interface MatrixAudioRendererProps {
   /**
    * The list of participants to render audio for.
@@ -27,6 +28,7 @@ export interface MatrixAudioRendererProps {
    * that are not expected to be in the rtc session.
    */
   members: CallMembership[];
+  livekitRoom: LivekitRoom;
   /**
    * If set to `true`, mutes all audio tracks rendered by the component.
    * @remarks
@@ -49,9 +51,10 @@ export interface MatrixAudioRendererProps {
  * ```
  * @public
  */
-export function MatrixAudioRenderer({
+export function LivekitRoomAudioRenderer({
   members,
   muted,
+  livekitRoom,
 }: MatrixAudioRendererProps): ReactNode {
   const validIdentities = useMemo(
     () =>
@@ -89,6 +92,7 @@ export function MatrixAudioRenderer({
     {
       updateOnlyOn: [],
       onlySubscribed: true,
+      room: livekitRoom,
     },
   ).filter((ref) => {
     const isValid = validIdentities?.has(ref.participant.identity);

src/room/InCallView.test.tsx

@@ -45,7 +45,7 @@ import {
 } from "../settings/settings";
 import { ReactionsSenderProvider } from "../reactions/useReactionsSender";
 import { useRoomEncryptionSystem } from "../e2ee/sharedKeyManagement";
-import { MatrixAudioRenderer } from "../livekit/MatrixAudioRenderer";
+import { LivekitRoomAudioRenderer } from "../livekit/MatrixAudioRenderer";
 import { MediaDevicesContext } from "../MediaDevicesContext";
 import { HeaderStyle } from "../UrlParams";
 
@@ -88,7 +88,7 @@ beforeEach(() => {
 
   // MatrixAudioRenderer is tested separately.
   (
-    MatrixAudioRenderer as MockedFunction<typeof MatrixAudioRenderer>
+    LivekitRoomAudioRenderer as MockedFunction<typeof LivekitRoomAudioRenderer>
   ).mockImplementation((_props) => {
     return <div>mocked: MatrixAudioRenderer</div>;
   });

src/room/InCallView.tsx

@@ -106,6 +106,7 @@ import {
 } from "../settings/settings";
 import { ReactionsReader } from "../reactions/ReactionsReader";
 import { useTypedEventEmitter } from "../useEvents.ts";
+import { LivekitRoomAudioRenderer } from "../livekit/MatrixAudioRenderer.tsx";
 import { muteAllAudio$ } from "../state/MuteAllAudioModel.ts";
 import { useMediaDevices } from "../MediaDevicesContext.ts";
 import { EarpieceOverlay } from "./EarpieceOverlay.tsx";
@@ -151,7 +152,6 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
       },
       reactionsReader.raisedHands$,
       reactionsReader.reactions$,
-      props.e2eeSystem,
     );
     setVm(vm);
     return (): void => {
@@ -746,6 +746,8 @@ export const InCallView: FC<InCallViewProps> = ({
     matrixRoom.roomId,
   );
 
+  const allLivekitRooms = useBehavior(vm.allLivekitRooms$);
+  const memberships = useBehavior(vm.memberships$);
   const toggleScreensharing = useCallback(() => {
     throw new Error("TODO-MULTI-SFU");
     // localParticipant
@@ -878,7 +880,14 @@ export const InCallView: FC<InCallViewProps> = ({
           </Text>
         )
       }
-      {/* TODO-MULTI-SFU: <MatrixAudioRenderer members={memberships} muted={muteAllAudio} /> */}
+      {allLivekitRooms.map((roomItem) => (
+        <LivekitRoomAudioRenderer
+          key={roomItem.url}
+          livekitRoom={roomItem.room}
+          members={memberships}
+          muted={muteAllAudio}
+        />
+      ))}
       {renderContent()}
       <CallEventAudioRenderer vm={vm} muted={muteAllAudio} />
       <ReactionsAudioRenderer vm={vm} muted={muteAllAudio} />

src/state/CallViewModel.ts

@@ -116,7 +116,6 @@ import { type Behavior } from "./Behavior";
 import {
   enterRTCSession,
   getLivekitAlias,
-  leaveRTCSession,
   makeFocus,
 } from "../rtcSessionHelpers";
 import { E2eeType } from "../e2ee/e2eeType";
@@ -462,7 +461,10 @@ export class CallViewModel extends ViewModel {
       ),
   );
 
-  private readonly memberships$ = this.scope.behavior(
+  // TODO-MULTI-SFU make sure that we consider the room memberships here as well (so that here we only have valid memberships)
+  // this also makes it possible to use this memberships$ list in all observables based on it.
+  // there should be no other call to: this.matrixRTCSession.memberships!
+  public readonly memberships$ = this.scope.behavior(
     fromEvent(
       this.matrixRTCSession,
       MatrixRTCSessionEvent.MembershipsChanged,
@@ -567,6 +569,26 @@ export class CallViewModel extends ViewModel {
     concatMap(({ stop }) => stop),
   );
 
+  public readonly allLivekitRooms$ = this.scope.behavior(
+    combineLatest([
+      this.remoteConnections$,
+      this.localConnection,
+      this.localFocus,
+    ]).pipe(
+      map(([remoteConnections, localConnection, localFocus]) =>
+        Array.from(remoteConnections.entries())
+          .map(([index, c]) => ({ room: c.livekitRoom, url: index }))
+          .concat([
+            {
+              room: localConnection.livekitRoom,
+              url: localFocus.livekit_service_url,
+            },
+          ]),
+      ),
+      startWith([]),
+    ),
+  );
+
   private readonly userId = this.matrixRoom.client.getUserId();
 
   private readonly matrixConnected$ = this.scope.behavior(