after merge cleanup

sdk/main.ts

@@ -50,7 +50,6 @@ import { getUrlParams } from "../src/UrlParams";
 import { MuteStates } from "../src/state/MuteStates";
 import { MediaDevices } from "../src/state/MediaDevices";
 import { E2eeType } from "../src/e2ee/e2eeType";
-import { type LocalMemberConnectionState } from "../src/state/CallViewModel/localMember/LocalMembership";
 import {
   currentAndPrev,
   logger,
@@ -62,7 +61,11 @@ import { ElementWidgetActions } from "../src/widget";
 import { type Connection } from "../src/state/CallViewModel/remoteMembers/Connection";
 
 interface MatrixRTCSdk {
-  join: () => LocalMemberConnectionState;
+  /**
+   * observe connected$ to track the state.
+   * @returns
+   */
+  join: () => void;
   /** @throws on leave errors */
   leave: () => void;
   data$: Observable<{ sender: string; data: string }>;
@@ -201,7 +204,7 @@ export async function createMatrixRTCSdk(
             return of((data: string): never => {
               throw Error("local membership not yet ready.");
             });
-          return m.participant$.pipe(
+          return m.participant.value$.pipe(
             map((p) => {
               if (p === null) {
                 return (data: string): never => {
@@ -264,11 +267,10 @@ export async function createMatrixRTCSdk(
   logger.info("createMatrixRTCSdk done");
 
   return {
-    join: (): LocalMemberConnectionState => {
+    join: (): void => {
       // first lets try making the widget sticky
       tryMakeSticky();
       callViewModel.join();
-      return callViewModel.connectionState;
     },
     leave: (): void => {
       callViewModel.hangup();
@@ -284,7 +286,7 @@ export async function createMatrixRTCSdk(
             combineLatest([
               member.connection$,
               member.membership$,
-              member.participant$,
+              member.participant.value$,
             ]).pipe(
               map(([connection, membership, participant]) => ({
                 connection,

src/state/CallViewModel/CallViewModel.ts

@@ -12,7 +12,6 @@ import {
   ExternalE2EEKeyProvider,
   type Room as LivekitRoom,
   type RoomOptions,
-  type LocalParticipant as LocalLivekitParticipant,
 } from "livekit-client";
 import { type Room as MatrixRoom } from "matrix-js-sdk";
 import {
@@ -81,7 +80,7 @@ import {
 } from "../../reactions";
 import { shallowEquals } from "../../utils/array";
 import { type MediaDevices } from "../MediaDevices";
-import { type Behavior } from "../Behavior";
+import { constant, type Behavior } from "../Behavior";
 import { E2eeType } from "../../e2ee/e2eeType";
 import { MatrixKeyProvider } from "../../e2ee/matrixKeyProvider";
 import { type MuteStates } from "../MuteStates";
@@ -105,9 +104,8 @@ import { createHomeserverConnected$ } from "./localMember/HomeserverConnected.ts
 import {
   createLocalMembership$,
   enterRTCSession,
-  type LocalMemberConnectionState,
-  RTCBackendState,
-} from "./localMember/LocalMembership.ts";
+  TransportState,
+} from "./localMember/LocalMember.ts";
 import { createLocalTransport$ } from "./localMember/LocalTransport.ts";
 import {
   createMemberships$,
@@ -119,6 +117,7 @@ import {
   createMatrixLivekitMembers$,
   type TaggedParticipant,
   type LocalMatrixLivekitMember,
+  type RemoteMatrixLivekitMember,
 } from "./remoteMembers/MatrixLivekitMembers.ts";
 import {
   type AutoLeaveReason,
@@ -158,7 +157,7 @@ export interface CallViewModelOptions {
   /** Optional behavior overriding the computed window size, mainly for testing purposes. */
   windowSize$?: Behavior<{ width: number; height: number }>;
   /** The version & compatibility mode of MatrixRTC that we should use. */
-  matrixRTCMode$: Behavior<MatrixRTCMode>;
+  matrixRTCMode$?: Behavior<MatrixRTCMode>;
 }
 
 // Do not play any sounds if the participant count has exceeded this
@@ -190,13 +189,6 @@ export type LivekitRoomItem = {
   url: string;
 };
 
-export type LocalMatrixLivekitMember = Pick<
-  MatrixLivekitMember,
-  "userId" | "membership$" | "connection$"
-> & {
-  participant$: Behavior<LocalLivekitParticipant | null>;
-};
-
 /**
  * The return of createCallViewModel$
  * this interface represents the root source of data for the call view.
@@ -273,7 +265,7 @@ export interface CallViewModel {
   livekitRoomItems$: Behavior<LivekitRoomItem[]>;
   userMedia$: Behavior<UserMedia[]>;
   /** use the layout instead, this is just for the sdk export. */
-  matrixLivekitMembers$: Behavior<MatrixLivekitMember[]>;
+  matrixLivekitMembers$: Behavior<RemoteMatrixLivekitMember[]>;
   localMatrixLivekitMember$: Behavior<LocalMatrixLivekitMember | null>;
   /** List of participants raising their hand */
   handsRaised$: Behavior<Record<string, RaisedHandInfo>>;
@@ -357,26 +349,15 @@ export interface CallViewModel {
     switch: () => void;
   } | null>;
 
-  // connection state
   /**
-   * Whether various media/event sources should pretend to be disconnected from
-   * all network input, even if their connection still technically works.
+   * Whether the app is currently reconnecting to the LiveKit server and/or setting the matrix rtc room state.
    */
-  // We do this when the app is in the 'reconnecting' state, because it might be
-  // that the LiveKit connection is still functional while the homeserver is
-  // down, for example, and we want to avoid making people worry that the app is
-  // in a split-brained state.
-  // DISCUSSION own membership manager ALSO this probably can be simplifis
   reconnecting$: Behavior<boolean>;
 
   /**
    * Shortcut for not requireing to parse and combine connectionState.matrix and connectionState.livekit
    */
   connected$: Behavior<boolean>;
-  /**
-   *
-   */
-  connectionState: LocalMemberConnectionState;
 }
 
 /**
@@ -406,6 +387,8 @@ export function createCallViewModel$(
     options.encryptionSystem,
     matrixRTCSession,
   );
+  const matrixRTCMode$ =
+    options.matrixRTCMode$ ?? constant(MatrixRTCMode.Legacy);
 
   // Each hbar seperates a block of input variables required for the CallViewModel to function.
   // The outputs of this block is written under the hbar.
@@ -438,7 +421,7 @@ export function createCallViewModel$(
     client,
     roomId: matrixRoom.roomId,
     useOldestMember$: scope.behavior(
-      options.matrixRTCMode$.pipe(map((v) => v === MatrixRTCMode.Legacy)),
+      matrixRTCMode$.pipe(map((v) => v === MatrixRTCMode.Legacy)),
     ),
   });
 
@@ -482,7 +465,7 @@ export function createCallViewModel$(
     logger,
   });
 
-  const { matrixLivekitMembers$ } = createMatrixLivekitMembers$({
+  const matrixLivekitMembers$ = createMatrixLivekitMembers$({
     scope: scope,
     membershipsWithTransport$:
       membershipsAndTransports.membershipsWithTransport$,
@@ -490,7 +473,7 @@ export function createCallViewModel$(
   });
 
   const connectOptions$ = scope.behavior(
-    options.matrixRTCMode$.pipe(
+    matrixRTCMode$.pipe(
       map((mode) => ({
         encryptMedia: livekitKeyProvider !== undefined,
         // TODO. This might need to get called again on each change of matrixRTCMode...
@@ -1527,17 +1510,6 @@ export function createCallViewModel$(
       null,
     ),
 
-    participantCount$,
-    livekitRoomItems$,
-    handsRaised$,
-    reactions$,
-    joinSoundEffect$,
-    leaveSoundEffect$,
-    newHandRaised$,
-    newScreenShare$,
-    audibleReactions$,
-    visibleReactions$,
-
     handsRaised$: handsRaised$,
     reactions$: reactions$,
     joinSoundEffect$: joinSoundEffect$,
@@ -1546,7 +1518,6 @@ export function createCallViewModel$(
     newScreenShare$: newScreenShare$,
     audibleReactions$: audibleReactions$,
     visibleReactions$: visibleReactions$,
-
     windowMode$: windowMode$,
     spotlightExpanded$: spotlightExpanded$,
     toggleSpotlightExpanded$: toggleSpotlightExpanded$,
@@ -1574,6 +1545,9 @@ export function createCallViewModel$(
     earpieceMode$: earpieceMode$,
     audioOutputSwitcher$: audioOutputSwitcher$,
     reconnecting$: localMembership.reconnecting$,
+    participantCount$,
+    livekitRoomItems$,
+    connected$: localMembership.connected$,
   };
 }
 

src/state/CallViewModel/localMember/LocalMember.ts

@@ -177,14 +177,18 @@ export const createLocalMembership$ = ({
   // tracks$: Behavior<LocalTrack[]>;
   participant$: Behavior<LocalParticipant | null>;
   connection$: Behavior<Connection | null>;
-  /** Shorthand for homeserverConnected.rtcSession === Status.Reconnecting
-   * Direct translation to the js-sdk membership manager connection `Status`.
+  /**
+   * Tracks the homserver and livekit connected state and based on that computes reconnecting.
    */
   reconnecting$: Behavior<boolean>;
   /** Shorthand for homeserverConnected.rtcSession === Status.Disconnected
    * Direct translation to the js-sdk membership manager connection `Status`.
    */
   disconnected$: Behavior<boolean>;
+  /**
+   * Fully connected
+   */
+  connected$: Behavior<boolean>;
 } => {
   const logger = parentLogger.getChild("[LocalMembership]");
   logger.debug(`Creating local membership..`);
@@ -637,11 +641,8 @@ export const createLocalMembership$ = ({
     requestDisconnect,
     localMemberState$,
     participant$,
-<<<<<<< HEAD:src/state/CallViewModel/localMember/LocalMembership.ts
-    connected$,
-=======
->>>>>>> livekit:src/state/CallViewModel/localMember/LocalMember.ts
     reconnecting$,
+    connected$: matrixAndLivekitConnected$,
     disconnected$: scope.behavior(
       homeserverConnected.rtsSession$.pipe(
         map((state) => state === RTCSessionStatus.Disconnected),

src/state/CallViewModel/remoteMembers/Connection.test.ts

@@ -392,7 +392,7 @@ describe("remote participants", () => {
     // livekitRoom and the rtc membership in order to publish the members that are publishing
     // on this connection.
 
-    const participants: RemoteParticipant[] = [
+    let participants: RemoteParticipant[] = [
       mockRemoteParticipant({ identity: "@alice:example.org:DEV000" }),
       mockRemoteParticipant({ identity: "@bob:example.org:DEV111" }),
       mockRemoteParticipant({ identity: "@carol:example.org:DEV222" }),
@@ -414,28 +414,23 @@ describe("remote participants", () => {
       fakeLivekitRoom.emit(RoomEvent.ParticipantConnected, p),
     );
 
-<<<<<<< HEAD
     // At this point there should be ~~no~~ publishers
     // We do have publisher now, since we do not filter for publishers anymore (to also have participants with only data tracks)
     // The filtering we do is just based on the matrixRTC member events.
-    expect(observedPublishers.pop()!.length).toEqual(4);
+    expect(observedParticipants.pop()!.length).toEqual(4);
 
     participants = [
-      fakeRemoteLivekitParticipant("@alice:example.org:DEV000", 1),
-      fakeRemoteLivekitParticipant("@bob:example.org:DEV111", 1),
-      fakeRemoteLivekitParticipant("@carol:example.org:DEV222", 1),
-      fakeRemoteLivekitParticipant("@dan:example.org:DEV333", 2),
+      mockRemoteParticipant({ identity: "@alice:example.org:DEV000" }),
+      mockRemoteParticipant({ identity: "@bob:example.org:DEV111" }),
+      mockRemoteParticipant({ identity: "@carol:example.org:DEV222" }),
+      mockRemoteParticipant({ identity: "@dan:example.org:DEV333" }),
     ];
     participants.forEach((p) =>
-      fakeRoomEventEmiter.emit(RoomEvent.ParticipantConnected, p),
+      fakeLivekitRoom.emit(RoomEvent.ParticipantConnected, p),
     );
 
     // At this point there should be no publishers
-    expect(observedPublishers.pop()!.length).toEqual(4);
-=======
-    // All remote participants should be present
     expect(observedParticipants.pop()!.length).toEqual(4);
->>>>>>> livekit
   });
 
   it("should be scoped to parent scope", (): void => {
@@ -443,15 +438,9 @@ describe("remote participants", () => {
 
     const connection = setupRemoteConnection();
 
-<<<<<<< HEAD
-    let observedPublishers: PublishingParticipant[][] = [];
-    const s = connection.remoteParticipants$.subscribe((publishers) => {
-      observedPublishers.push(publishers);
-=======
     let observedParticipants: RemoteParticipant[][] = [];
     const s = connection.remoteParticipants$.subscribe((participants) => {
       observedParticipants.push(participants);
->>>>>>> livekit
     });
     onTestFinished(() => s.unsubscribe());
 
@@ -468,28 +457,10 @@ describe("remote participants", () => {
       fakeLivekitRoom.emit(RoomEvent.ParticipantConnected, participant);
     }
 
-<<<<<<< HEAD
-    // At this point there should be ~~no~~ publishers
-    // We do have publisher now, since we do not filter for publishers anymore (to also have participants with only data tracks)
-    // The filtering we do is just based on the matrixRTC member events.
-    expect(observedPublishers.pop()!.length).toEqual(1);
-
-    participants = [fakeRemoteLivekitParticipant("@bob:example.org:DEV111", 1)];
-
-    for (const participant of participants) {
-      fakeRoomEventEmiter.emit(RoomEvent.ParticipantConnected, participant);
-    }
-
-    // We should have bob has a publisher now
-    const publishers = observedPublishers.pop();
-    expect(publishers?.length).toEqual(1);
-    expect(publishers?.[0]?.identity).toEqual("@bob:example.org:DEV111");
-=======
     // We should have bob as a participant now
     const ps = observedParticipants.pop();
     expect(ps?.length).toEqual(1);
     expect(ps?.[0]?.identity).toEqual("@bob:example.org:DEV111");
->>>>>>> livekit
 
     // end the parent scope
     testScope.end();

src/state/CallViewModel/remoteMembers/Connection.ts

@@ -16,7 +16,7 @@ import {
   type RemoteParticipant,
 } from "livekit-client";
 import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
-import { BehaviorSubject } from "rxjs";
+import { BehaviorSubject, map } from "rxjs";
 import { type Logger } from "matrix-js-sdk/lib/logger";
 
 import {

src/state/CallViewModel/remoteMembers/ConnectionManager.ts

@@ -19,8 +19,10 @@ import { areLivekitTransportsEqual } from "./MatrixLivekitMembers.ts";
 import { type ConnectionFactory } from "./ConnectionFactory.ts";
 
 export class ConnectionManagerData {
-  private readonly store: Map<string, [Connection, RemoteParticipant[]]> =
-    new Map();
+  private readonly store: Map<
+    string,
+    { connection: Connection; participants: RemoteParticipant[] }
+  > = new Map();
 
   public constructor() {}
 
@@ -166,11 +168,8 @@ export function createConnectionManager$({
         );
 
         // probably not required
-<<<<<<< HEAD
-        if (listOfConnectionsWithParticipants.length === 0) {
-=======
+
         if (listOfConnectionsWithRemoteParticipants.length === 0) {
->>>>>>> livekit
           return of(new Epoch(new ConnectionManagerData(), epoch));
         }
 

src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts

@@ -91,7 +91,7 @@ test("should signal participant not yet connected to livekit", () => {
       }),
     );
 
-    const { matrixLivekitMembers$ } = createMatrixLivekitMembers$({
+    const matrixLivekitMembers$ = createMatrixLivekitMembers$({
       scope: testScope,
       membershipsWithTransport$: testScope.behavior(membershipsWithTransport$),
       connectionManager: {
@@ -99,21 +99,24 @@ test("should signal participant not yet connected to livekit", () => {
       } as unknown as IConnectionManager,
     });
 
-    expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe("a", {
-      a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => {
-        expect(data.length).toEqual(1);
-        expectObservable(data[0].membership$).toBe("a", {
-          a: bobMembership,
-        });
-        expectObservable(data[0].participant.value$).toBe("a", {
-          a: null,
-        });
-        expectObservable(data[0].connection$).toBe("a", {
-          a: null,
-        });
-        return true;
-      }),
-    });
+    expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe(
+      "a",
+      {
+        a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => {
+          expect(data.length).toEqual(1);
+          expectObservable(data[0].membership$).toBe("a", {
+            a: bobMembership,
+          });
+          expectObservable(data[0].participant.value$).toBe("a", {
+            a: null,
+          });
+          expectObservable(data[0].connection$).toBe("a", {
+            a: null,
+          });
+          return true;
+        }),
+      },
+    );
   });
 });
 
@@ -171,7 +174,7 @@ test("should signal participant on a connection that is publishing", () => {
       }),
     );
 
-    const { matrixLivekitMembers$ } = createMatrixLivekitMembers$({
+    const matrixLivekitMembers$ = createMatrixLivekitMembers$({
       scope: testScope,
       membershipsWithTransport$: testScope.behavior(membershipsWithTransport$),
       connectionManager: {
@@ -179,25 +182,28 @@ test("should signal participant on a connection that is publishing", () => {
       } as unknown as IConnectionManager,
     });
 
-    expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe("a", {
-      a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => {
-        expect(data.length).toEqual(1);
-        expectObservable(data[0].membership$).toBe("a", {
-          a: bobMembership,
-        });
-        expectObservable(data[0].participant.value$).toBe("a", {
-          a: expect.toSatisfy((participant) => {
-            expect(participant).toBeDefined();
-            expect(participant!.identity).toEqual(bobParticipantId);
-            return true;
-          }),
-        });
-        expectObservable(data[0].connection$).toBe("a", {
-          a: connection,
-        });
-        return true;
-      }),
-    });
+    expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe(
+      "a",
+      {
+        a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => {
+          expect(data.length).toEqual(1);
+          expectObservable(data[0].membership$).toBe("a", {
+            a: bobMembership,
+          });
+          expectObservable(data[0].participant.value$).toBe("a", {
+            a: expect.toSatisfy((participant) => {
+              expect(participant).toBeDefined();
+              expect(participant!.identity).toEqual(bobParticipantId);
+              return true;
+            }),
+          });
+          expectObservable(data[0].connection$).toBe("a", {
+            a: connection,
+          });
+          return true;
+        }),
+      },
+    );
   });
 });
 
@@ -222,7 +228,7 @@ test("should signal participant on a connection that is not publishing", () => {
       }),
     );
 
-    const { matrixLivekitMembers$ } = createMatrixLivekitMembers$({
+    const matrixLivekitMembers$ = createMatrixLivekitMembers$({
       scope: testScope,
       membershipsWithTransport$: testScope.behavior(membershipsWithTransport$),
       connectionManager: {
@@ -230,21 +236,24 @@ test("should signal participant on a connection that is not publishing", () => {
       } as unknown as IConnectionManager,
     });
 
-    expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe("a", {
-      a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => {
-        expect(data.length).toEqual(1);
-        expectObservable(data[0].membership$).toBe("a", {
-          a: bobMembership,
-        });
-        expectObservable(data[0].participant.value$).toBe("a", {
-          a: null,
-        });
-        expectObservable(data[0].connection$).toBe("a", {
-          a: connection,
-        });
-        return true;
-      }),
-    });
+    expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe(
+      "a",
+      {
+        a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => {
+          expect(data.length).toEqual(1);
+          expectObservable(data[0].membership$).toBe("a", {
+            a: bobMembership,
+          });
+          expectObservable(data[0].participant.value$).toBe("a", {
+            a: null,
+          });
+          expectObservable(data[0].connection$).toBe("a", {
+            a: connection,
+          });
+          return true;
+        }),
+      },
+    );
   });
 });
 
@@ -283,7 +292,7 @@ describe("Publication edge case", () => {
         }),
       );
 
-      const { matrixLivekitMembers$ } = createMatrixLivekitMembers$({
+      const matrixLivekitMembers$ = createMatrixLivekitMembers$({
         scope: testScope,
         membershipsWithTransport$: testScope.behavior(
           membershipsWithTransport$,
@@ -349,7 +358,7 @@ describe("Publication edge case", () => {
         }),
       );
 
-      const { matrixLivekitMembers$ } = createMatrixLivekitMembers$({
+      const matrixLivekitMembers$ = createMatrixLivekitMembers$({
         scope: testScope,
         membershipsWithTransport$: testScope.behavior(
           membershipsWithTransport$,

src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts

@@ -85,7 +85,7 @@ export function createMatrixLivekitMembers$({
    * Stream of all the call members and their associated livekit data (if available).
    */
 
-  const matrixLivekitMembers$ = scope.behavior(
+  return scope.behavior(
     combineLatest([
       membershipsWithTransport$,
       connectionManager.connectionManagerData$,
@@ -142,11 +142,6 @@ export function createMatrixLivekitMembers$({
       ),
     ),
   );
-  return {
-    matrixLivekitMembers$,
-    // TODO add only publishing participants... maybe. disucss at least
-    // scope.behavior(matrixLivekitMembers$.pipe(map((items) => items.value.map((i)=>{ i.}))))
-  };
 }
 
 // TODO add back in the callviewmodel pauseWhen(this.pretendToBeDisconnected$)

src/state/CallViewModel/remoteMembers/integration.test.ts

@@ -124,14 +124,14 @@ test("bob, carl, then bob joining no tracks yet", () => {
       logger: logger,
     });
 
-    const { matrixLivekitMembers$ } = createMatrixLivekitMembers$({
+    const matrixLivekitMembers$ = createMatrixLivekitMembers$({
       scope: testScope,
       membershipsWithTransport$:
         membershipsAndTransports.membershipsWithTransport$,
       connectionManager,
     });
 
-    expectObservable(matrixLivekitItems$).toBe(vMarble, {
+    expectObservable(matrixLivekitMembers$).toBe(vMarble, {
       a: expect.toSatisfy((e: Epoch<RemoteMatrixLivekitMember[]>) => {
         const items = e.value;
         expect(items.length).toBe(1);

yarn.lock

@@ -11735,6 +11735,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"pkg-dir@npm:^5.0.0":
+  version: 5.0.0
+  resolution: "pkg-dir@npm:5.0.0"
+  dependencies:
+    find-up: "npm:^5.0.0"
+  checksum: 10c0/793a496d685dc55bbbdbbb22d884535c3b29241e48e3e8d37e448113a71b9e42f5481a61fdc672d7322de12fbb2c584dd3a68bf89b18fffce5c48a390f911bc5
+  languageName: node
+  linkType: hard
+
 "playwright-core@npm:1.57.0":
   version: 1.57.0
   resolution: "playwright-core@npm:1.57.0"