diff --git a/locales/en/app.json b/locales/en/app.json index 2c6801bc..2b9eae5a 100644 --- a/locales/en/app.json +++ b/locales/en/app.json @@ -74,16 +74,16 @@ "matrix_id": "Matrix ID: {{id}}", "matrixRTCMode": { "Comptibility": { - "label": "Compatibility: state events & multi SFU", - "description": "Compatible with homeservers that do not support sticky events (but all other EC clients are v0.17.0 or later)" + "description": "Compatible with homeservers that do not support sticky events (but all other EC clients are v0.17.0 or later)", + "label": "Compatibility: state events & multi SFU" }, "Legacy": { - "label": "Legacy: state events & oldest membership SFU", - "description": "Compatible with old versions of EC that do not support multi SFU" + "description": "Compatible with old versions of EC that do not support multi SFU", + "label": "Legacy: state events & oldest membership SFU" }, "Matrix_2_0": { - "label": "Matrix 2.0: sticky events & multi SFU", - "description": "Compatible only with homservers supporting sticky events and all EC clients v0.17.0 or later" + "description": "Compatible only with homservers supporting sticky events and all EC clients v0.17.0 or later", + "label": "Matrix 2.0: sticky events & multi SFU" } }, "mute_all_audio": "Mute all audio (participants, reactions, join sounds)", diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index 76eeaeac..916d8e93 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -103,7 +103,10 @@ import { } from "../SessionBehaviors.ts"; import { ECConnectionFactory } from "./remoteMembers/ConnectionFactory.ts"; import { createConnectionManager$ } from "./remoteMembers/ConnectionManager.ts"; -import { createMatrixLivekitMembers$ } from "./remoteMembers/MatrixLivekitMembers.ts"; +import { + createMatrixLivekitMembers$, + type MatrixLivekitMember, +} from "./remoteMembers/MatrixLivekitMembers.ts"; import { createCallNotificationLifecycle$, createReceivedDecline$, @@ -269,6 +272,38 @@ export class CallViewModel { options: this.connectOptions$, }); + private localRtcMembership$ = this.scope.behavior( + this.memberships$.pipe( + map( + (memberships) => + memberships.value.find( + (membership) => + membership.userId === this.userId && + membership.deviceId === this.deviceId, + ) ?? null, + ), + ), + ); + private localMatrixLivekitMemberUninitialized = { + membership$: this.localRtcMembership$, + participant$: this.localMembership.participant$, + connection$: this.localMembership.connection$, + userId: this.userId, + }; + + private localMatrixLivekitMember$: Behavior = + this.scope.behavior( + this.localRtcMembership$.pipe( + switchMap((membership) => { + if (!membership) return of(null); + return of( + // casting is save here since we know that localRtcMembership$ is !== null since we reached this case. + this.localMatrixLivekitMemberUninitialized as MatrixLivekitMember, + ); + }), + ), + ); + // ------------------------------------------------------------------------ private callLifecycle = createCallNotificationLifecycle$({ @@ -283,6 +318,7 @@ export class CallViewModel { localUser: { userId: this.userId, deviceId: this.deviceId }, }); public autoLeave$ = this.callLifecycle.autoLeave$; + // ------------------------------------------------------------------------ /** @@ -377,12 +413,10 @@ export class CallViewModel { ), ); - private roomMembers$ = createRoomMembers$(this.scope, this.matrixRoom); - private matrixMemberMetadataStore = createMatrixMemberMetadata$( this.scope, this.scope.behavior(this.memberships$.pipe(map((mems) => mems.value))), - this.roomMembers$, + createRoomMembers$(this.scope, this.matrixRoom), ); /** @@ -390,22 +424,55 @@ export class CallViewModel { */ // TODO this also needs the local participant to be added. private readonly userMedia$ = this.scope.behavior( - combineLatest([this.matrixLivekitMembers$, duplicateTiles.value$]).pipe( + combineLatest([ + this.localMatrixLivekitMember$, + this.matrixLivekitMembers$, + duplicateTiles.value$, + ]).pipe( // Generate a collection of MediaItems from the list of expected (whether // present or missing) LiveKit participants. generateItems( - function* ([{ value: matrixLivekitMembers }, duplicateTiles]) { + function* ([ + localMatrixLivekitMember, + { value: matrixLivekitMembers }, + duplicateTiles, + ]) { + // add local member if available + if (localMatrixLivekitMember) { + const { + userId, + participant$, + connection$, + // membership$, + } = localMatrixLivekitMember; + const participantId = participant$.value?.identity; // should be membership$.value.membershipID which is not optional + // const participantId = membership$.value.membershipID; + if (participantId) { + for (let dup = 0; dup < 1 + duplicateTiles; dup++) { + yield { + keys: [dup, participantId, userId, participant$, connection$], + data: undefined, + }; + } + } + } + // add remote members that are available for (const { - participantId, userId, participant$, connection$, - } of matrixLivekitMembers) - for (let dup = 0; dup < 1 + duplicateTiles; dup++) + // membership$ + } of matrixLivekitMembers) { + const participantId = participant$.value?.identity; + // const participantId = membership$.value?.identity; + if (!participantId) continue; + for (let dup = 0; dup < 1 + duplicateTiles; dup++) { yield { keys: [dup, participantId, userId, participant$, connection$], data: undefined, }; + } + } }, ( scope, diff --git a/src/state/CallViewModel/localMember/LocalMembership.ts b/src/state/CallViewModel/localMember/LocalMembership.ts index df4d3b6b..86112f4e 100644 --- a/src/state/CallViewModel/localMember/LocalMembership.ts +++ b/src/state/CallViewModel/localMember/LocalMembership.ts @@ -10,6 +10,7 @@ import { type E2EEOptions, type Participant, ParticipantEvent, + type LocalParticipant, } from "livekit-client"; import { observeParticipantEvents } from "@livekit/components-core"; import { @@ -54,6 +55,7 @@ import { getUrlParams } from "../../../UrlParams.ts"; import { PosthogAnalytics } from "../../../analytics/PosthogAnalytics.ts"; import { MatrixRTCMode } from "../../../settings/settings.ts"; import { Config } from "../../../config/Config.ts"; +import { type Connection } from "../remoteMembers/Connection.ts"; export enum LivekitState { Uninitialized = "uninitialized", @@ -82,8 +84,8 @@ type LocalMemberMatrixState = | { state: MatrixState.Disconnected }; export interface LocalMemberConnectionState { - livekit$: BehaviorSubject; - matrix$: BehaviorSubject; + livekit$: Behavior; + matrix$: Behavior; } /* @@ -145,7 +147,8 @@ export const createLocalMembership$ = ({ // Use null here since behavior cannot be initialised with undefined. sharingScreen$: Behavior; toggleScreenSharing: (() => void) | null; - + participant$: Behavior; + connection$: Behavior; // deprecated fields /** @deprecated use state instead*/ homeserverConnected$: Behavior; @@ -317,6 +320,7 @@ export const createLocalMembership$ = ({ state.livekit$.next({ state: LivekitState.Error, error }); }); }); + combineLatest([localTransport$, connectRequested$]).subscribe( ([transport, connectRequested]) => { if ( @@ -515,6 +519,9 @@ export const createLocalMembership$ = ({ alternativeScreenshareToggle, ); + const participant$ = scope.behavior( + connection$.pipe(map((c) => c?.livekitRoom.localParticipant ?? null)), + ); return { startTracks, requestConnect, @@ -526,6 +533,8 @@ export const createLocalMembership$ = ({ configError$, sharingScreen$, toggleScreenSharing, + participant$, + connection$, }; }; diff --git a/src/state/CallViewModel/remoteMembers/Connection.test.ts b/src/state/CallViewModel/remoteMembers/Connection.test.ts index 01a73301..eb9ae89d 100644 --- a/src/state/CallViewModel/remoteMembers/Connection.test.ts +++ b/src/state/CallViewModel/remoteMembers/Connection.test.ts @@ -389,15 +389,17 @@ describe("Publishing participants observations", () => { const bobIsAPublisher = Promise.withResolvers(); const danIsAPublisher = Promise.withResolvers(); const observedPublishers: PublishingParticipant[][] = []; - const s = connection.participants$.subscribe((publishers) => { - observedPublishers.push(publishers); - if (publishers.some((p) => p.identity === "@bob:example.org:DEV111")) { - bobIsAPublisher.resolve(); - } - if (publishers.some((p) => p.identity === "@dan:example.org:DEV333")) { - danIsAPublisher.resolve(); - } - }); + const s = connection.remoteParticipantsWithTracks$.subscribe( + (publishers) => { + observedPublishers.push(publishers); + if (publishers.some((p) => p.identity === "@bob:example.org:DEV111")) { + bobIsAPublisher.resolve(); + } + if (publishers.some((p) => p.identity === "@dan:example.org:DEV333")) { + danIsAPublisher.resolve(); + } + }, + ); onTestFinished(() => s.unsubscribe()); // The publishingParticipants$ observable is derived from the current members of the // livekitRoom and the rtc membership in order to publish the members that are publishing @@ -513,9 +515,11 @@ describe("Publishing participants observations", () => { const connection = setupRemoteConnection(); let observedPublishers: PublishingParticipant[][] = []; - const s = connection.participants$.subscribe((publishers) => { - observedPublishers.push(publishers); - }); + const s = connection.remoteParticipantsWithTracks$.subscribe( + (publishers) => { + observedPublishers.push(publishers); + }, + ); onTestFinished(() => s.unsubscribe()); let participants: RemoteParticipant[] = [ diff --git a/src/state/CallViewModel/remoteMembers/Connection.ts b/src/state/CallViewModel/remoteMembers/Connection.ts index 60251541..fa66183a 100644 --- a/src/state/CallViewModel/remoteMembers/Connection.ts +++ b/src/state/CallViewModel/remoteMembers/Connection.ts @@ -179,12 +179,13 @@ export class Connection { } /** - * An observable of the participants that are publishing on this connection. + * An observable of the participants that are publishing on this connection. (Excluding our local participant) * This is derived from `participantsIncludingSubscribers$` and `remoteTransports$`. * It filters the participants to only those that are associated with a membership that claims to publish on this connection. */ - - public readonly participants$: Behavior; + public readonly remoteParticipantsWithTracks$: Behavior< + PublishingParticipant[] + >; /** * The media transport to connect to. @@ -211,7 +212,9 @@ export class Connection { this.transport = transport; this.client = client; - this.participants$ = scope.behavior( + // REMOTE participants with track!!! + // this.remoteParticipantsWithTracks$ + this.remoteParticipantsWithTracks$ = scope.behavior( // only tracks remote participants connectedParticipantsObserver(this.livekitRoom, { additionalRoomEvents: [ @@ -219,10 +222,11 @@ export class Connection { RoomEvent.TrackUnpublished, ], }).pipe( - map((participants) => [ - this.livekitRoom.localParticipant, - ...participants, - ]), + map((participants) => { + return participants.filter( + (participant) => participant.getTrackPublications().length > 0, + ); + }), ), [], ); diff --git a/src/state/CallViewModel/remoteMembers/ConnectionManager.ts b/src/state/CallViewModel/remoteMembers/ConnectionManager.ts index 73ca3d16..32d42d75 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionManager.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionManager.ts @@ -55,8 +55,8 @@ export class ConnectionManagerData { public getConnectionForTransport( transport: LivekitTransport, - ): Connection | undefined { - return this.store.get(this.getKey(transport))?.[0]; + ): Connection | null { + return this.store.get(this.getKey(transport))?.[0] ?? null; } public getParticipantForTransport( @@ -181,7 +181,7 @@ export function createConnectionManager$({ // Map the connections to list of {connection, participants}[] const listOfConnectionsWithPublishingParticipants = connections.value.map((connection) => { - return connection.participants$.pipe( + return connection.remoteParticipantsWithTracks$.pipe( map((participants) => ({ connection, participants, diff --git a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts index 326eb0f6..3b31cd33 100644 --- a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts +++ b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts @@ -30,13 +30,14 @@ const logger = rootLogger.getChild("MatrixLivekitMembers"); * or if it has no livekit transport at all. */ export interface MatrixLivekitMember { - participantId: string; - userId: string; membership$: Behavior; participant$: Behavior< LocalLivekitParticipant | RemoteLivekitParticipant | null >; - connection$: Behavior; + connection$: Behavior; + // participantId: string; We do not want a participantId here since it will be generated by the jwt + // TODO decide if we can also drop the userId. Its in the matrix membership anyways. + userId: string; } interface Props { @@ -96,7 +97,7 @@ export function createMatrixLivekitMembers$({ participants.find((p) => p.identity == participantId) ?? null; const connection = transport ? managerData.getConnectionForTransport(transport) - : undefined; + : null; yield { keys: [participantId, membership.userId],