From 852d2ee37540ae335526fedf1772f1715989e803 Mon Sep 17 00:00:00 2001 From: Timo K Date: Mon, 22 Dec 2025 13:35:40 +0100 Subject: [PATCH] after merge cleanup --- sdk/main.ts | 14 ++- src/state/CallViewModel/CallViewModel.ts | 56 +++------ .../CallViewModel/localMember/LocalMember.ts | 13 +- .../remoteMembers/Connection.test.ts | 43 ++----- .../CallViewModel/remoteMembers/Connection.ts | 2 +- .../remoteMembers/ConnectionManager.ts | 11 +- .../MatrixLivekitMembers.test.ts | 117 ++++++++++-------- .../remoteMembers/MatrixLivekitMembers.ts | 7 +- .../remoteMembers/integration.test.ts | 4 +- yarn.lock | 9 ++ 10 files changed, 118 insertions(+), 158 deletions(-) diff --git a/sdk/main.ts b/sdk/main.ts index d2683277..5b23a700 100644 --- a/sdk/main.ts +++ b/sdk/main.ts @@ -50,7 +50,6 @@ import { getUrlParams } from "../src/UrlParams"; import { MuteStates } from "../src/state/MuteStates"; import { MediaDevices } from "../src/state/MediaDevices"; import { E2eeType } from "../src/e2ee/e2eeType"; -import { type LocalMemberConnectionState } from "../src/state/CallViewModel/localMember/LocalMembership"; import { currentAndPrev, logger, @@ -62,7 +61,11 @@ import { ElementWidgetActions } from "../src/widget"; import { type Connection } from "../src/state/CallViewModel/remoteMembers/Connection"; interface MatrixRTCSdk { - join: () => LocalMemberConnectionState; + /** + * observe connected$ to track the state. + * @returns + */ + join: () => void; /** @throws on leave errors */ leave: () => void; data$: Observable<{ sender: string; data: string }>; @@ -201,7 +204,7 @@ export async function createMatrixRTCSdk( return of((data: string): never => { throw Error("local membership not yet ready."); }); - return m.participant$.pipe( + return m.participant.value$.pipe( map((p) => { if (p === null) { return (data: string): never => { @@ -264,11 +267,10 @@ export async function createMatrixRTCSdk( logger.info("createMatrixRTCSdk done"); return { - join: (): LocalMemberConnectionState => { + join: (): void => { // first lets try making the widget sticky tryMakeSticky(); callViewModel.join(); - return callViewModel.connectionState; }, leave: (): void => { callViewModel.hangup(); @@ -284,7 +286,7 @@ export async function createMatrixRTCSdk( combineLatest([ member.connection$, member.membership$, - member.participant$, + member.participant.value$, ]).pipe( map(([connection, membership, participant]) => ({ connection, diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index 2bd51cb9..00a7d3e9 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -12,7 +12,6 @@ import { ExternalE2EEKeyProvider, type Room as LivekitRoom, type RoomOptions, - type LocalParticipant as LocalLivekitParticipant, } from "livekit-client"; import { type Room as MatrixRoom } from "matrix-js-sdk"; import { @@ -81,7 +80,7 @@ import { } from "../../reactions"; import { shallowEquals } from "../../utils/array"; import { type MediaDevices } from "../MediaDevices"; -import { type Behavior } from "../Behavior"; +import { constant, type Behavior } from "../Behavior"; import { E2eeType } from "../../e2ee/e2eeType"; import { MatrixKeyProvider } from "../../e2ee/matrixKeyProvider"; import { type MuteStates } from "../MuteStates"; @@ -105,9 +104,8 @@ import { createHomeserverConnected$ } from "./localMember/HomeserverConnected.ts import { createLocalMembership$, enterRTCSession, - type LocalMemberConnectionState, - RTCBackendState, -} from "./localMember/LocalMembership.ts"; + TransportState, +} from "./localMember/LocalMember.ts"; import { createLocalTransport$ } from "./localMember/LocalTransport.ts"; import { createMemberships$, @@ -119,6 +117,7 @@ import { createMatrixLivekitMembers$, type TaggedParticipant, type LocalMatrixLivekitMember, + type RemoteMatrixLivekitMember, } from "./remoteMembers/MatrixLivekitMembers.ts"; import { type AutoLeaveReason, @@ -158,7 +157,7 @@ export interface CallViewModelOptions { /** Optional behavior overriding the computed window size, mainly for testing purposes. */ windowSize$?: Behavior<{ width: number; height: number }>; /** The version & compatibility mode of MatrixRTC that we should use. */ - matrixRTCMode$: Behavior; + matrixRTCMode$?: Behavior; } // Do not play any sounds if the participant count has exceeded this @@ -190,13 +189,6 @@ export type LivekitRoomItem = { url: string; }; -export type LocalMatrixLivekitMember = Pick< - MatrixLivekitMember, - "userId" | "membership$" | "connection$" -> & { - participant$: Behavior; -}; - /** * The return of createCallViewModel$ * this interface represents the root source of data for the call view. @@ -273,7 +265,7 @@ export interface CallViewModel { livekitRoomItems$: Behavior; userMedia$: Behavior; /** use the layout instead, this is just for the sdk export. */ - matrixLivekitMembers$: Behavior; + matrixLivekitMembers$: Behavior; localMatrixLivekitMember$: Behavior; /** List of participants raising their hand */ handsRaised$: Behavior>; @@ -357,26 +349,15 @@ export interface CallViewModel { switch: () => void; } | null>; - // connection state /** - * Whether various media/event sources should pretend to be disconnected from - * all network input, even if their connection still technically works. + * Whether the app is currently reconnecting to the LiveKit server and/or setting the matrix rtc room state. */ - // We do this when the app is in the 'reconnecting' state, because it might be - // that the LiveKit connection is still functional while the homeserver is - // down, for example, and we want to avoid making people worry that the app is - // in a split-brained state. - // DISCUSSION own membership manager ALSO this probably can be simplifis reconnecting$: Behavior; /** * Shortcut for not requireing to parse and combine connectionState.matrix and connectionState.livekit */ connected$: Behavior; - /** - * - */ - connectionState: LocalMemberConnectionState; } /** @@ -406,6 +387,8 @@ export function createCallViewModel$( options.encryptionSystem, matrixRTCSession, ); + const matrixRTCMode$ = + options.matrixRTCMode$ ?? constant(MatrixRTCMode.Legacy); // Each hbar seperates a block of input variables required for the CallViewModel to function. // The outputs of this block is written under the hbar. @@ -438,7 +421,7 @@ export function createCallViewModel$( client, roomId: matrixRoom.roomId, useOldestMember$: scope.behavior( - options.matrixRTCMode$.pipe(map((v) => v === MatrixRTCMode.Legacy)), + matrixRTCMode$.pipe(map((v) => v === MatrixRTCMode.Legacy)), ), }); @@ -482,7 +465,7 @@ export function createCallViewModel$( logger, }); - const { matrixLivekitMembers$ } = createMatrixLivekitMembers$({ + const matrixLivekitMembers$ = createMatrixLivekitMembers$({ scope: scope, membershipsWithTransport$: membershipsAndTransports.membershipsWithTransport$, @@ -490,7 +473,7 @@ export function createCallViewModel$( }); const connectOptions$ = scope.behavior( - options.matrixRTCMode$.pipe( + matrixRTCMode$.pipe( map((mode) => ({ encryptMedia: livekitKeyProvider !== undefined, // TODO. This might need to get called again on each change of matrixRTCMode... @@ -1527,17 +1510,6 @@ export function createCallViewModel$( null, ), - participantCount$, - livekitRoomItems$, - handsRaised$, - reactions$, - joinSoundEffect$, - leaveSoundEffect$, - newHandRaised$, - newScreenShare$, - audibleReactions$, - visibleReactions$, - handsRaised$: handsRaised$, reactions$: reactions$, joinSoundEffect$: joinSoundEffect$, @@ -1546,7 +1518,6 @@ export function createCallViewModel$( newScreenShare$: newScreenShare$, audibleReactions$: audibleReactions$, visibleReactions$: visibleReactions$, - windowMode$: windowMode$, spotlightExpanded$: spotlightExpanded$, toggleSpotlightExpanded$: toggleSpotlightExpanded$, @@ -1574,6 +1545,9 @@ export function createCallViewModel$( earpieceMode$: earpieceMode$, audioOutputSwitcher$: audioOutputSwitcher$, reconnecting$: localMembership.reconnecting$, + participantCount$, + livekitRoomItems$, + connected$: localMembership.connected$, }; } diff --git a/src/state/CallViewModel/localMember/LocalMember.ts b/src/state/CallViewModel/localMember/LocalMember.ts index 0c6516ad..6d28bc56 100644 --- a/src/state/CallViewModel/localMember/LocalMember.ts +++ b/src/state/CallViewModel/localMember/LocalMember.ts @@ -177,14 +177,18 @@ export const createLocalMembership$ = ({ // tracks$: Behavior; participant$: Behavior; connection$: Behavior; - /** Shorthand for homeserverConnected.rtcSession === Status.Reconnecting - * Direct translation to the js-sdk membership manager connection `Status`. + /** + * Tracks the homserver and livekit connected state and based on that computes reconnecting. */ reconnecting$: Behavior; /** Shorthand for homeserverConnected.rtcSession === Status.Disconnected * Direct translation to the js-sdk membership manager connection `Status`. */ disconnected$: Behavior; + /** + * Fully connected + */ + connected$: Behavior; } => { const logger = parentLogger.getChild("[LocalMembership]"); logger.debug(`Creating local membership..`); @@ -637,11 +641,8 @@ export const createLocalMembership$ = ({ requestDisconnect, localMemberState$, participant$, -<<<<<<< HEAD:src/state/CallViewModel/localMember/LocalMembership.ts - connected$, -======= ->>>>>>> livekit:src/state/CallViewModel/localMember/LocalMember.ts reconnecting$, + connected$: matrixAndLivekitConnected$, disconnected$: scope.behavior( homeserverConnected.rtsSession$.pipe( map((state) => state === RTCSessionStatus.Disconnected), diff --git a/src/state/CallViewModel/remoteMembers/Connection.test.ts b/src/state/CallViewModel/remoteMembers/Connection.test.ts index 044902f9..bcc0bac2 100644 --- a/src/state/CallViewModel/remoteMembers/Connection.test.ts +++ b/src/state/CallViewModel/remoteMembers/Connection.test.ts @@ -392,7 +392,7 @@ describe("remote participants", () => { // livekitRoom and the rtc membership in order to publish the members that are publishing // on this connection. - const participants: RemoteParticipant[] = [ + let participants: RemoteParticipant[] = [ mockRemoteParticipant({ identity: "@alice:example.org:DEV000" }), mockRemoteParticipant({ identity: "@bob:example.org:DEV111" }), mockRemoteParticipant({ identity: "@carol:example.org:DEV222" }), @@ -414,28 +414,23 @@ describe("remote participants", () => { fakeLivekitRoom.emit(RoomEvent.ParticipantConnected, p), ); -<<<<<<< HEAD // At this point there should be ~~no~~ publishers // We do have publisher now, since we do not filter for publishers anymore (to also have participants with only data tracks) // The filtering we do is just based on the matrixRTC member events. - expect(observedPublishers.pop()!.length).toEqual(4); + expect(observedParticipants.pop()!.length).toEqual(4); participants = [ - fakeRemoteLivekitParticipant("@alice:example.org:DEV000", 1), - fakeRemoteLivekitParticipant("@bob:example.org:DEV111", 1), - fakeRemoteLivekitParticipant("@carol:example.org:DEV222", 1), - fakeRemoteLivekitParticipant("@dan:example.org:DEV333", 2), + mockRemoteParticipant({ identity: "@alice:example.org:DEV000" }), + mockRemoteParticipant({ identity: "@bob:example.org:DEV111" }), + mockRemoteParticipant({ identity: "@carol:example.org:DEV222" }), + mockRemoteParticipant({ identity: "@dan:example.org:DEV333" }), ]; participants.forEach((p) => - fakeRoomEventEmiter.emit(RoomEvent.ParticipantConnected, p), + fakeLivekitRoom.emit(RoomEvent.ParticipantConnected, p), ); // At this point there should be no publishers - expect(observedPublishers.pop()!.length).toEqual(4); -======= - // All remote participants should be present expect(observedParticipants.pop()!.length).toEqual(4); ->>>>>>> livekit }); it("should be scoped to parent scope", (): void => { @@ -443,15 +438,9 @@ describe("remote participants", () => { const connection = setupRemoteConnection(); -<<<<<<< HEAD - let observedPublishers: PublishingParticipant[][] = []; - const s = connection.remoteParticipants$.subscribe((publishers) => { - observedPublishers.push(publishers); -======= let observedParticipants: RemoteParticipant[][] = []; const s = connection.remoteParticipants$.subscribe((participants) => { observedParticipants.push(participants); ->>>>>>> livekit }); onTestFinished(() => s.unsubscribe()); @@ -468,28 +457,10 @@ describe("remote participants", () => { fakeLivekitRoom.emit(RoomEvent.ParticipantConnected, participant); } -<<<<<<< HEAD - // At this point there should be ~~no~~ publishers - // We do have publisher now, since we do not filter for publishers anymore (to also have participants with only data tracks) - // The filtering we do is just based on the matrixRTC member events. - expect(observedPublishers.pop()!.length).toEqual(1); - - participants = [fakeRemoteLivekitParticipant("@bob:example.org:DEV111", 1)]; - - for (const participant of participants) { - fakeRoomEventEmiter.emit(RoomEvent.ParticipantConnected, participant); - } - - // We should have bob has a publisher now - const publishers = observedPublishers.pop(); - expect(publishers?.length).toEqual(1); - expect(publishers?.[0]?.identity).toEqual("@bob:example.org:DEV111"); -======= // We should have bob as a participant now const ps = observedParticipants.pop(); expect(ps?.length).toEqual(1); expect(ps?.[0]?.identity).toEqual("@bob:example.org:DEV111"); ->>>>>>> livekit // end the parent scope testScope.end(); diff --git a/src/state/CallViewModel/remoteMembers/Connection.ts b/src/state/CallViewModel/remoteMembers/Connection.ts index 9f3f562e..cf92e2a6 100644 --- a/src/state/CallViewModel/remoteMembers/Connection.ts +++ b/src/state/CallViewModel/remoteMembers/Connection.ts @@ -16,7 +16,7 @@ import { type RemoteParticipant, } from "livekit-client"; import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; -import { BehaviorSubject } from "rxjs"; +import { BehaviorSubject, map } from "rxjs"; import { type Logger } from "matrix-js-sdk/lib/logger"; import { diff --git a/src/state/CallViewModel/remoteMembers/ConnectionManager.ts b/src/state/CallViewModel/remoteMembers/ConnectionManager.ts index 5db80d08..c1b4af59 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionManager.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionManager.ts @@ -19,8 +19,10 @@ import { areLivekitTransportsEqual } from "./MatrixLivekitMembers.ts"; import { type ConnectionFactory } from "./ConnectionFactory.ts"; export class ConnectionManagerData { - private readonly store: Map = - new Map(); + private readonly store: Map< + string, + { connection: Connection; participants: RemoteParticipant[] } + > = new Map(); public constructor() {} @@ -166,11 +168,8 @@ export function createConnectionManager$({ ); // probably not required -<<<<<<< HEAD - if (listOfConnectionsWithParticipants.length === 0) { -======= + if (listOfConnectionsWithRemoteParticipants.length === 0) { ->>>>>>> livekit return of(new Epoch(new ConnectionManagerData(), epoch)); } diff --git a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts index f5929ff9..d26bac37 100644 --- a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts +++ b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts @@ -91,7 +91,7 @@ test("should signal participant not yet connected to livekit", () => { }), ); - const { matrixLivekitMembers$ } = createMatrixLivekitMembers$({ + const matrixLivekitMembers$ = createMatrixLivekitMembers$({ scope: testScope, membershipsWithTransport$: testScope.behavior(membershipsWithTransport$), connectionManager: { @@ -99,21 +99,24 @@ test("should signal participant not yet connected to livekit", () => { } as unknown as IConnectionManager, }); - expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe("a", { - a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => { - expect(data.length).toEqual(1); - expectObservable(data[0].membership$).toBe("a", { - a: bobMembership, - }); - expectObservable(data[0].participant.value$).toBe("a", { - a: null, - }); - expectObservable(data[0].connection$).toBe("a", { - a: null, - }); - return true; - }), - }); + expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe( + "a", + { + a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => { + expect(data.length).toEqual(1); + expectObservable(data[0].membership$).toBe("a", { + a: bobMembership, + }); + expectObservable(data[0].participant.value$).toBe("a", { + a: null, + }); + expectObservable(data[0].connection$).toBe("a", { + a: null, + }); + return true; + }), + }, + ); }); }); @@ -171,7 +174,7 @@ test("should signal participant on a connection that is publishing", () => { }), ); - const { matrixLivekitMembers$ } = createMatrixLivekitMembers$({ + const matrixLivekitMembers$ = createMatrixLivekitMembers$({ scope: testScope, membershipsWithTransport$: testScope.behavior(membershipsWithTransport$), connectionManager: { @@ -179,25 +182,28 @@ test("should signal participant on a connection that is publishing", () => { } as unknown as IConnectionManager, }); - expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe("a", { - a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => { - expect(data.length).toEqual(1); - expectObservable(data[0].membership$).toBe("a", { - a: bobMembership, - }); - expectObservable(data[0].participant.value$).toBe("a", { - a: expect.toSatisfy((participant) => { - expect(participant).toBeDefined(); - expect(participant!.identity).toEqual(bobParticipantId); - return true; - }), - }); - expectObservable(data[0].connection$).toBe("a", { - a: connection, - }); - return true; - }), - }); + expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe( + "a", + { + a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => { + expect(data.length).toEqual(1); + expectObservable(data[0].membership$).toBe("a", { + a: bobMembership, + }); + expectObservable(data[0].participant.value$).toBe("a", { + a: expect.toSatisfy((participant) => { + expect(participant).toBeDefined(); + expect(participant!.identity).toEqual(bobParticipantId); + return true; + }), + }); + expectObservable(data[0].connection$).toBe("a", { + a: connection, + }); + return true; + }), + }, + ); }); }); @@ -222,7 +228,7 @@ test("should signal participant on a connection that is not publishing", () => { }), ); - const { matrixLivekitMembers$ } = createMatrixLivekitMembers$({ + const matrixLivekitMembers$ = createMatrixLivekitMembers$({ scope: testScope, membershipsWithTransport$: testScope.behavior(membershipsWithTransport$), connectionManager: { @@ -230,21 +236,24 @@ test("should signal participant on a connection that is not publishing", () => { } as unknown as IConnectionManager, }); - expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe("a", { - a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => { - expect(data.length).toEqual(1); - expectObservable(data[0].membership$).toBe("a", { - a: bobMembership, - }); - expectObservable(data[0].participant.value$).toBe("a", { - a: null, - }); - expectObservable(data[0].connection$).toBe("a", { - a: connection, - }); - return true; - }), - }); + expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe( + "a", + { + a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => { + expect(data.length).toEqual(1); + expectObservable(data[0].membership$).toBe("a", { + a: bobMembership, + }); + expectObservable(data[0].participant.value$).toBe("a", { + a: null, + }); + expectObservable(data[0].connection$).toBe("a", { + a: connection, + }); + return true; + }), + }, + ); }); }); @@ -283,7 +292,7 @@ describe("Publication edge case", () => { }), ); - const { matrixLivekitMembers$ } = createMatrixLivekitMembers$({ + const matrixLivekitMembers$ = createMatrixLivekitMembers$({ scope: testScope, membershipsWithTransport$: testScope.behavior( membershipsWithTransport$, @@ -349,7 +358,7 @@ describe("Publication edge case", () => { }), ); - const { matrixLivekitMembers$ } = createMatrixLivekitMembers$({ + const matrixLivekitMembers$ = createMatrixLivekitMembers$({ scope: testScope, membershipsWithTransport$: testScope.behavior( membershipsWithTransport$, diff --git a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts index a77037dd..6501adb4 100644 --- a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts +++ b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts @@ -85,7 +85,7 @@ export function createMatrixLivekitMembers$({ * Stream of all the call members and their associated livekit data (if available). */ - const matrixLivekitMembers$ = scope.behavior( + return scope.behavior( combineLatest([ membershipsWithTransport$, connectionManager.connectionManagerData$, @@ -142,11 +142,6 @@ export function createMatrixLivekitMembers$({ ), ), ); - return { - matrixLivekitMembers$, - // TODO add only publishing participants... maybe. disucss at least - // scope.behavior(matrixLivekitMembers$.pipe(map((items) => items.value.map((i)=>{ i.})))) - }; } // TODO add back in the callviewmodel pauseWhen(this.pretendToBeDisconnected$) diff --git a/src/state/CallViewModel/remoteMembers/integration.test.ts b/src/state/CallViewModel/remoteMembers/integration.test.ts index 1093d721..6108c7bc 100644 --- a/src/state/CallViewModel/remoteMembers/integration.test.ts +++ b/src/state/CallViewModel/remoteMembers/integration.test.ts @@ -124,14 +124,14 @@ test("bob, carl, then bob joining no tracks yet", () => { logger: logger, }); - const { matrixLivekitMembers$ } = createMatrixLivekitMembers$({ + const matrixLivekitMembers$ = createMatrixLivekitMembers$({ scope: testScope, membershipsWithTransport$: membershipsAndTransports.membershipsWithTransport$, connectionManager, }); - expectObservable(matrixLivekitItems$).toBe(vMarble, { + expectObservable(matrixLivekitMembers$).toBe(vMarble, { a: expect.toSatisfy((e: Epoch) => { const items = e.value; expect(items.length).toBe(1); diff --git a/yarn.lock b/yarn.lock index c220c9ce..d9202de2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11735,6 +11735,15 @@ __metadata: languageName: node linkType: hard +"pkg-dir@npm:^5.0.0": + version: 5.0.0 + resolution: "pkg-dir@npm:5.0.0" + dependencies: + find-up: "npm:^5.0.0" + checksum: 10c0/793a496d685dc55bbbdbbb22d884535c3b29241e48e3e8d37e448113a71b9e42f5481a61fdc672d7322de12fbb2c584dd3a68bf89b18fffce5c48a390f911bc5 + languageName: node + linkType: hard + "playwright-core@npm:1.57.0": version: 1.57.0 resolution: "playwright-core@npm:1.57.0"