diff --git a/.github/workflows/publish-embedded-packages.yaml b/.github/workflows/publish-embedded-packages.yaml index 256e440e..546191ab 100644 --- a/.github/workflows/publish-embedded-packages.yaml +++ b/.github/workflows/publish-embedded-packages.yaml @@ -85,7 +85,7 @@ jobs: run: find ${{ env.FILENAME_PREFIX }} -type f -print0 | sort -z | xargs -0 sha256sum | tee ${{ env.FILENAME_PREFIX }}.sha256 - name: Upload if: ${{ needs.versioning.outputs.DRY_RUN == 'false' }} - uses: softprops/action-gh-release@aec2ec56f94eb8180ceec724245f64ef008b89f5 # v2 + uses: softprops/action-gh-release@6da8fa9354ddfdc4aeace5fc48d7f679b5214090 # v2 with: files: | ${{ env.FILENAME_PREFIX }}.tar.gz @@ -100,7 +100,7 @@ jobs: ARTIFACT_VERSION: ${{ steps.artifact_version.outputs.ARTIFACT_VERSION }} permissions: contents: read - id-token: write # required for the provenance flag on npm publish + id-token: write # Allow npm to authenticate as a trusted publisher steps: - name: Checkout uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 @@ -126,8 +126,6 @@ jobs: npm version ${{ needs.versioning.outputs.PREFIXED_VERSION }} --no-git-tag-version echo "ARTIFACT_VERSION=$(jq '.version' --raw-output package.json)" >> "$GITHUB_ENV" npm publish --provenance --access public --tag ${{ needs.versioning.outputs.TAG }} ${{ needs.versioning.outputs.DRY_RUN == 'true' && '--dry-run' || '' }} - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_RELEASE_TOKEN }} - id: artifact_version name: Output artifact version @@ -264,7 +262,7 @@ jobs: echo "iOS: ${{ needs.publish_ios.outputs.ARTIFACT_VERSION }}" - name: Add release notes if: ${{ needs.versioning.outputs.DRY_RUN == 'false' }} - uses: softprops/action-gh-release@aec2ec56f94eb8180ceec724245f64ef008b89f5 # v2 + uses: softprops/action-gh-release@6da8fa9354ddfdc4aeace5fc48d7f679b5214090 # v2 with: append_body: true body: | diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 5b4b7936..34835635 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -42,7 +42,7 @@ jobs: - name: Create Checksum run: find ${{ env.FILENAME_PREFIX }} -type f -print0 | sort -z | xargs -0 sha256sum | tee ${{ env.FILENAME_PREFIX }}.sha256 - name: Upload - uses: softprops/action-gh-release@aec2ec56f94eb8180ceec724245f64ef008b89f5 # v2 + uses: softprops/action-gh-release@6da8fa9354ddfdc4aeace5fc48d7f679b5214090 # v2 with: files: | ${{ env.FILENAME_PREFIX }}.tar.gz @@ -68,7 +68,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Add release note - uses: softprops/action-gh-release@aec2ec56f94eb8180ceec724245f64ef008b89f5 # v2 + uses: softprops/action-gh-release@6da8fa9354ddfdc4aeace5fc48d7f679b5214090 # v2 with: append_body: true body: | diff --git a/.node-version b/.node-version index 2bd5a0a9..a45fd52c 100644 --- a/.node-version +++ b/.node-version @@ -1 +1 @@ -22 +24 diff --git a/config/config_netlify_preview.json b/config/config_netlify_preview.json index cf0148e9..313f0d02 100644 --- a/config/config_netlify_preview.json +++ b/config/config_netlify_preview.json @@ -18,7 +18,7 @@ "api_host": "https://posthog-element-call.element.io" }, "rageshake": { - "submit_url": "https://element.io/bugreports/submit" + "submit_url": "https://rageshakes.element.io/api/submit" }, "sentry": { "environment": "netlify-pr-preview", diff --git a/docs/self-hosting.md b/docs/self-hosting.md index 7127abee..d15f2910 100644 --- a/docs/self-hosting.md +++ b/docs/self-hosting.md @@ -136,8 +136,8 @@ handle @jwt_service { reverse_proxy http://[::1]:8080 { header_up Host {host} header_up X-Forwarded-Server {host} - header_up X-Real-IP {remote_addr} - header_up X-Forwarded-For {remote_addr} + header_up X-Real-IP {remote_host} + header_up X-Forwarded-For {remote_host} } } @@ -146,8 +146,8 @@ handle { reverse_proxy http://localhost:7880 { header_up Host {host} header_up X-Forwarded-Server {host} - header_up X-Real-IP {remote_addr} - header_up X-Forwarded-For {remote_addr} + header_up X-Real-IP {remote_host} + header_up X-Forwarded-For {remote_host} } } ``` diff --git a/docs/url-params.md b/docs/url-params.md index b2af8416..a474daed 100644 --- a/docs/url-params.md +++ b/docs/url-params.md @@ -96,6 +96,6 @@ These parameters are only supported in the [embedded](./embedded-standalone.md) | -------------------- | -------------------------------------------------------------------------- | -------- | ---------------------------------------------------------------------------------------------------------------------------------- | | `posthogApiHost` | Posthog server URL | No | e.g. `https://posthog-element-call.element.io`. Only supported in embedded package. In full package the value from config is used. | | `posthogApiKey` | Posthog project API key | No | Only supported in embedded package. In full package the value from config is used. | -| `rageshakeSubmitUrl` | Rageshake server URL endpoint | No | e.g. `https://element.io/bugreports/submit`. In full package the value from config is used. | +| `rageshakeSubmitUrl` | Rageshake server URL endpoint | No | e.g. `https://rageshakes.element.io/api/submit`. In full package the value from config is used. | | `sentryDsn` | Sentry [DSN](https://docs.sentry.io/concepts/key-terms/dsn-explainer/) | No | In full package the value from config is used. | | `sentryEnvironment` | Sentry [environment](https://docs.sentry.io/concepts/key-terms/key-terms/) | No | In full package the value from config is used. | diff --git a/embedded/android/gradle/libs.versions.toml b/embedded/android/gradle/libs.versions.toml index 9982f14d..8ec7801a 100644 --- a/embedded/android/gradle/libs.versions.toml +++ b/embedded/android/gradle/libs.versions.toml @@ -2,7 +2,7 @@ # https://docs.gradle.org/current/userguide/platforms.html#sub::toml-dependencies-format [versions] -android_gradle_plugin = "8.11.1" +android_gradle_plugin = "8.13.0" [libraries] android_gradle_plugin = { module = "com.android.tools.build:gradle", version.ref = "android_gradle_plugin" } diff --git a/locales/en/app.json b/locales/en/app.json index 11267439..ff26fbe3 100644 --- a/locales/en/app.json +++ b/locales/en/app.json @@ -72,12 +72,22 @@ "livekit_server_info": "LiveKit Server Info", "livekit_sfu": "LiveKit SFU: {{url}}", "matrix_id": "Matrix ID: {{id}}", - "multi_sfu": "Multi-SFU media transport", - "mute_all_audio": "Mute all audio (participants, reactions, join sounds)", - "prefer_sticky_events": { - "description": "Improves reliability of calls (requires homeserver support)", - "label": "Prefer sticky events" + "matrixRTCMode": { + "Comptibility": { + "description": "Compatible with homeservers that do not support sticky events (but all other EC clients are v0.17.0 or later)", + "label": "Compatibility: state events & multi SFU" + }, + "Legacy": { + "description": "Compatible with old versions of EC that do not support multi SFU", + "label": "Legacy: state events & oldest membership SFU" + }, + "Matrix_2_0": { + "description": "Compatible only with homservers supporting sticky events and all EC clients v0.17.0 or later", + "label": "Matrix 2.0: sticky events & multi SFU" + }, + "title": "MatrixRTC mode" }, + "mute_all_audio": "Mute all audio (participants, reactions, join sounds)", "show_connection_stats": "Show connection statistics", "url_params": "URL parameters" }, diff --git a/package.json b/package.json index 35468c21..62ea9f4f 100644 --- a/package.json +++ b/package.json @@ -71,7 +71,7 @@ "@types/grecaptcha": "^3.0.9", "@types/jsdom": "^21.1.7", "@types/lodash-es": "^4.17.12", - "@types/node": "^22.0.0", + "@types/node": "^24.0.0", "@types/pako": "^2.0.3", "@types/qrcode": "^1.5.5", "@types/react": "^19.0.0", diff --git a/src/button/ReactionToggleButton.test.tsx b/src/button/ReactionToggleButton.test.tsx index c7ac5aa0..f6b7a2ea 100644 --- a/src/button/ReactionToggleButton.test.tsx +++ b/src/button/ReactionToggleButton.test.tsx @@ -13,7 +13,7 @@ import { type ReactNode } from "react"; import { ReactionToggleButton } from "./ReactionToggleButton"; import { ElementCallReactionEventType } from "../reactions"; -import { type CallViewModel } from "../state/CallViewModel"; +import { type CallViewModel } from "../state/CallViewModel/CallViewModel"; import { getBasicCallViewModelEnvironment } from "../utils/test-viewmodel"; import { alice, local, localRtcMember } from "../utils/test-fixtures"; import { type MockRTCSession } from "../utils/test"; diff --git a/src/button/ReactionToggleButton.tsx b/src/button/ReactionToggleButton.tsx index 69673293..0c722baf 100644 --- a/src/button/ReactionToggleButton.tsx +++ b/src/button/ReactionToggleButton.tsx @@ -33,7 +33,7 @@ import { ReactionsRowSize, } from "../reactions"; import { Modal } from "../Modal"; -import { type CallViewModel } from "../state/CallViewModel"; +import { type CallViewModel } from "../state/CallViewModel/CallViewModel"; import { useBehavior } from "../useBehavior"; interface InnerButtonProps extends ComponentPropsWithoutRef<"button"> { diff --git a/src/livekit/TrackProcessorContext.tsx b/src/livekit/TrackProcessorContext.tsx index bb13d5dc..02888466 100644 --- a/src/livekit/TrackProcessorContext.tsx +++ b/src/livekit/TrackProcessorContext.tsx @@ -28,6 +28,7 @@ import { } from "../settings/settings"; import { BlurBackgroundTransformer } from "./BlurBackgroundTransformer"; import { type Behavior } from "../state/Behavior"; +import { type ObservableScope } from "../state/ObservableScope"; //TODO-MULTI-SFU: This is not yet fully there. // it is a combination of exposing observable and react hooks. @@ -63,13 +64,17 @@ export function useTrackProcessorObservable$(): Observable { return state$; } +/** + * Updates your video tracks to always use the given processor. + */ export const trackProcessorSync = ( + scope: ObservableScope, videoTrack$: Behavior, processor$: Behavior, ): void => { - // TODO-MULTI-SFU: Bind to an ObservableScope to avoid leaking resources. - combineLatest([videoTrack$, processor$]).subscribe( - ([videoTrack, processorState]) => { + combineLatest([videoTrack$, processor$]) + .pipe(scope.bind()) + .subscribe(([videoTrack, processorState]) => { if (!processorState) return; if (!videoTrack) return; const { processor } = processorState; @@ -79,8 +84,7 @@ export const trackProcessorSync = ( if (!processor && videoTrack.getProcessor()) { void videoTrack.stopProcessor(); } - }, - ); + }); }; export const useTrackProcessorSync = ( diff --git a/src/livekit/openIDSFU.ts b/src/livekit/openIDSFU.ts index 70d1786d..073f6c75 100644 --- a/src/livekit/openIDSFU.ts +++ b/src/livekit/openIDSFU.ts @@ -25,7 +25,7 @@ export type OpenIDClientParts = Pick< export async function getSFUConfigWithOpenID( client: OpenIDClientParts, serviceUrl: string, - livekitAlias: string, + matrixRoomId: string, ): Promise { let openIdToken: IOpenIDToken; try { @@ -43,7 +43,7 @@ export async function getSFUConfigWithOpenID( const sfuConfig = await getLiveKitJWT( client, serviceUrl, - livekitAlias, + matrixRoomId, openIdToken, ); logger.info(`Got JWT from call's active focus URL.`); diff --git a/src/main.tsx b/src/main.tsx index e41aaff8..946e0238 100644 --- a/src/main.tsx +++ b/src/main.tsx @@ -62,7 +62,7 @@ Initializer.initBeforeReact() .then(() => { root.render( - , + , ); }) diff --git a/src/reactions/ReactionsReader.ts b/src/reactions/ReactionsReader.ts index c1f78b51..74b47c77 100644 --- a/src/reactions/ReactionsReader.ts +++ b/src/reactions/ReactionsReader.ts @@ -266,6 +266,7 @@ export class ReactionsReader { ); return; } + // TODO refactor to use memer id `membershipEvent.membershipID` (needs to happen in combination with other memberId refactors) const identifier = `${membershipEvent.userId}:${membershipEvent.deviceId}`; if (!content.emoji) { diff --git a/src/reactions/useReactionsSender.tsx b/src/reactions/useReactionsSender.tsx index ec29c2af..afb9b789 100644 --- a/src/reactions/useReactionsSender.tsx +++ b/src/reactions/useReactionsSender.tsx @@ -20,7 +20,7 @@ import { logger } from "matrix-js-sdk/lib/logger"; import { useMatrixRTCSessionMemberships } from "../useMatrixRTCSessionMemberships"; import { useClientState } from "../ClientContext"; import { ElementCallReactionEventType, type ReactionOption } from "."; -import { type CallViewModel } from "../state/CallViewModel"; +import { type CallViewModel } from "../state/CallViewModel/CallViewModel"; import { useBehavior } from "../useBehavior"; interface ReactionsSenderContextType { diff --git a/src/room/CallEventAudioRenderer.test.tsx b/src/room/CallEventAudioRenderer.test.tsx index e49c7011..733346eb 100644 --- a/src/room/CallEventAudioRenderer.test.tsx +++ b/src/room/CallEventAudioRenderer.test.tsx @@ -38,7 +38,7 @@ import { local, localRtcMember, } from "../utils/test-fixtures"; -import { MAX_PARTICIPANT_COUNT_FOR_SOUND } from "../state/CallViewModel"; +import { MAX_PARTICIPANT_COUNT_FOR_SOUND } from "../state/CallViewModel/CallViewModel"; vitest.mock("livekit-client/e2ee-worker?worker"); vitest.mock("../useAudioContext"); diff --git a/src/room/CallEventAudioRenderer.tsx b/src/room/CallEventAudioRenderer.tsx index 23997c37..d33f3b84 100644 --- a/src/room/CallEventAudioRenderer.tsx +++ b/src/room/CallEventAudioRenderer.tsx @@ -7,7 +7,7 @@ Please see LICENSE in the repository root for full details. import { type ReactNode, useEffect } from "react"; -import { type CallViewModel } from "../state/CallViewModel"; +import { type CallViewModel } from "../state/CallViewModel/CallViewModel"; import joinCallSoundMp3 from "../sound/join_call.mp3"; import joinCallSoundOgg from "../sound/join_call.ogg"; import leftCallSoundMp3 from "../sound/left_call.mp3"; diff --git a/src/room/GroupCallView.test.tsx b/src/room/GroupCallView.test.tsx index 4f95a327..44b272ba 100644 --- a/src/room/GroupCallView.test.tsx +++ b/src/room/GroupCallView.test.tsx @@ -78,13 +78,13 @@ const leaveRTCSession = vi.hoisted(() => ), ); -vi.mock("../rtcSessionHelpers", async (importOriginal) => { - // TODO: perhaps there is a more elegant way to manage the type import here? - // eslint-disable-next-line @typescript-eslint/consistent-type-imports - const orig = await importOriginal(); - // TODO: leaveRTCSession no longer exists! Tests need adapting. - return { ...orig, enterRTCSession, leaveRTCSession }; -}); +// vi.mock("../rtcSessionHelpers", async (importOriginal) => { +// // TODO: perhaps there is a more elegant way to manage the type import here? +// // eslint-disable-next-line @typescript-eslint/consistent-type-imports +// const orig = await importOriginal(); +// // TODO: leaveRTCSession no longer exists! Tests need adapting. +// return { ...orig, enterRTCSession, leaveRTCSession }; +// }); let playSound: MockedFunction< NonNullable>["playSound"] @@ -346,6 +346,7 @@ test.skip("GroupCallView leaves the session when an error occurs", async () => { test.skip("GroupCallView shows errors that occur during joining", async () => { const user = userEvent.setup(); + // This should not mock this error that deep. it should only mock the CallViewModel. enterRTCSession.mockRejectedValue(new MatrixRTCTransportMissingError("")); onTestFinished(() => { enterRTCSession.mockReset(); diff --git a/src/room/InCallView.test.tsx b/src/room/InCallView.test.tsx index d388ebc3..a137074b 100644 --- a/src/room/InCallView.test.tsx +++ b/src/room/InCallView.test.tsx @@ -43,9 +43,6 @@ import { LivekitRoomAudioRenderer } from "../livekit/MatrixAudioRenderer"; import { MediaDevicesContext } from "../MediaDevicesContext"; import { HeaderStyle } from "../UrlParams"; -// vi.hoisted(() => { -// localStorage = {} as unknown as Storage; -// }); vi.hoisted( () => (global.ImageData = class MockImageData { @@ -109,6 +106,7 @@ function createInCallView(): RenderResult & { getUserId: () => localRtcMember.userId, getDeviceId: () => localRtcMember.deviceId, getRoom: (rId) => (rId === roomId ? room : null), + getDomain: () => "example.com", } as Partial as MatrixClient; const room = mockMatrixRoom({ relations: { @@ -119,7 +117,8 @@ function createInCallView(): RenderResult & { } as unknown as RelationsContainer, client, roomId, - getMember: (userId) => roomMembers.get(userId) ?? null, + // getMember: (userId) => roomMembers.get(userId) ?? null, + getMembers: () => Array.from(roomMembers.values()), getMxcAvatarUrl: () => null, hasEncryptionStateEvent: vi.fn().mockReturnValue(true), getCanonicalAlias: () => null, diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index ac0b5025..28293d29 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -58,7 +58,11 @@ import { type MuteStates } from "../state/MuteStates"; import { type MatrixInfo } from "./VideoPreview"; import { InviteButton } from "../button/InviteButton"; import { LayoutToggle } from "./LayoutToggle"; -import { CallViewModel, type GridMode } from "../state/CallViewModel"; +import { + type CallViewModel, + createCallViewModel$, + type GridMode, +} from "../state/CallViewModel/CallViewModel.ts"; import { Grid, type TileProps } from "../grid/Grid"; import { useInitial } from "../useInitial"; import { SpotlightTile } from "../tile/SpotlightTile"; @@ -117,17 +121,17 @@ export interface ActiveCallProps } export const ActiveCall: FC = (props) => { - const mediaDevices = useMediaDevices(); const [vm, setVm] = useState(null); - const { autoLeaveWhenOthersLeft, waitForCallPickup, sendNotificationType } = - useUrlParams(); - + const urlParams = useUrlParams(); + const mediaDevices = useMediaDevices(); const trackProcessorState$ = useTrackProcessorObservable$(); useEffect(() => { const scope = new ObservableScope(); const reactionsReader = new ReactionsReader(scope, props.rtcSession); - const vm = new CallViewModel( + const { autoLeaveWhenOthersLeft, waitForCallPickup, sendNotificationType } = + urlParams; + const vm = createCallViewModel$( scope, props.rtcSession, props.matrixRoom, @@ -140,7 +144,7 @@ export const ActiveCall: FC = (props) => { }, reactionsReader.raisedHands$, reactionsReader.reactions$, - trackProcessorState$, + scope.behavior(trackProcessorState$), ); setVm(vm); @@ -151,13 +155,11 @@ export const ActiveCall: FC = (props) => { }, [ props.rtcSession, props.matrixRoom, - mediaDevices, props.muteStates, props.e2eeSystem, - autoLeaveWhenOthersLeft, - sendNotificationType, - waitForCallPickup, props.onLeft, + urlParams, + mediaDevices, trackProcessorState$, ]); @@ -249,7 +251,6 @@ export const InCallView: FC = ({ () => void toggleRaisedHand(), ); - const allLivekitRooms = useBehavior(vm.allLivekitRooms$); const audioParticipants = useBehavior(vm.audioParticipants$); const participantCount = useBehavior(vm.participantCount$); const reconnecting = useBehavior(vm.reconnecting$); @@ -264,6 +265,7 @@ export const InCallView: FC = ({ const audioOutputSwitcher = useBehavior(vm.audioOutputSwitcher$); const sharingScreen = useBehavior(vm.sharingScreen$); + const ringOverlay = useBehavior(vm.ringOverlay$); const fatalCallError = useBehavior(vm.configError$); // Stop the rendering and throw for the error boundary if (fatalCallError) throw fatalCallError; @@ -300,47 +302,26 @@ export const InCallView: FC = ({ // Waiting UI overlay const waitingOverlay: JSX.Element | null = useMemo(() => { - // No overlay if not in ringing state - if (callPickupState !== "ringing") return null; - - // Use room state for other participants data (the one that we likely want to reach) - // TODO: this screams it wants to be a behavior in the vm. - const roomOthers = [ - ...matrixRoom.getMembersWithMembership("join"), - ...matrixRoom.getMembersWithMembership("invite"), - ].filter((m) => m.userId !== client.getUserId()); - // Yield if there are not other members in the room. - if (roomOthers.length === 0) return null; - - const otherMember = roomOthers.length > 0 ? roomOthers[0] : undefined; - const isOneOnOne = roomOthers.length === 1 && otherMember; - const text = isOneOnOne - ? `Waiting for ${otherMember.name ?? otherMember.userId} to join…` - : "Waiting for other participants…"; - const avatarMxc = isOneOnOne - ? (otherMember.getMxcAvatarUrl?.() ?? undefined) - : (matrixRoom.getMxcAvatarUrl() ?? undefined); - - return ( + return ringOverlay ? (
- {text} + {ringOverlay.text}
- ); - }, [callPickupState, client, matrixRoom]); + ) : null; + }, [ringOverlay]); // Ideally we could detect taps by listening for click events and checking // that the pointerType of the event is "touch", but this isn't yet supported @@ -821,7 +802,7 @@ export const InCallView: FC = ({ key={url} url={url} livekitRoom={livekitRoom} - validIdentities={participants.map((p) => p.identity)} + validIdentities={participants} muted={muteAllAudio} /> ))} @@ -843,7 +824,8 @@ export const InCallView: FC = ({ onDismiss={closeSettings} tab={settingsTab} onTabChange={setSettingsTab} - livekitRooms={allLivekitRooms} + // TODO expose correct data to setttings modal + livekitRooms={[]} /> )} diff --git a/src/room/ReactionAudioRenderer.test.tsx b/src/room/ReactionAudioRenderer.test.tsx index 83188be7..988d43a6 100644 --- a/src/room/ReactionAudioRenderer.test.tsx +++ b/src/room/ReactionAudioRenderer.test.tsx @@ -10,8 +10,9 @@ import { afterAll, afterEach, beforeEach, + describe, expect, - test, + it, vitest, type MockedFunction, type Mock, @@ -27,7 +28,7 @@ import { import { useAudioContext } from "../useAudioContext"; import { GenericReaction, ReactionSet } from "../reactions"; import { prefetchSounds } from "../soundUtils"; -import { type CallViewModel } from "../state/CallViewModel"; +import { type CallViewModel } from "../state/CallViewModel/CallViewModel"; import { getBasicCallViewModelEnvironment } from "../utils/test-viewmodel"; import { alice, @@ -49,122 +50,125 @@ vitest.mock("livekit-client/e2ee-worker?worker"); vitest.mock("../useAudioContext"); vitest.mock("../soundUtils"); -afterEach(() => { - vitest.resetAllMocks(); - playReactionsSoundSetting.setValue(playReactionsSoundSetting.defaultValue); - soundEffectVolumeSetting.setValue(soundEffectVolumeSetting.defaultValue); -}); - -afterAll(() => { - vitest.restoreAllMocks(); -}); - let playSound: Mock< NonNullable>["playSound"] >; -beforeEach(() => { - (prefetchSounds as MockedFunction).mockResolvedValue({ - sound: new ArrayBuffer(0), +describe("ReactionAudioRenderer", () => { + afterEach(() => { + playReactionsSoundSetting.setValue(playReactionsSoundSetting.defaultValue); + soundEffectVolumeSetting.setValue(soundEffectVolumeSetting.defaultValue); }); - playSound = vitest.fn(); - (useAudioContext as MockedFunction).mockReturnValue({ - playSound, - playSoundLooping: vitest.fn(), - soundDuration: {}, - }); -}); - -test("preloads all audio elements", () => { - const { vm } = getBasicCallViewModelEnvironment([local, alice]); - playReactionsSoundSetting.setValue(true); - render(); - expect(prefetchSounds).toHaveBeenCalledOnce(); -}); - -test("will play an audio sound when there is a reaction", () => { - const { vm, reactionsSubject$ } = getBasicCallViewModelEnvironment([ - local, - alice, - ]); - playReactionsSoundSetting.setValue(true); - render(); - - // Find the first reaction with a sound effect - const chosenReaction = ReactionSet.find((r) => !!r.sound); - if (!chosenReaction) { - throw Error( - "No reactions have sounds configured, this test cannot succeed", + beforeEach(() => { + (prefetchSounds as MockedFunction).mockResolvedValue( + { + sound: new ArrayBuffer(0), + }, ); - } - act(() => { - reactionsSubject$.next({ - [aliceRtcMember.deviceId]: { - reactionOption: chosenReaction, - expireAfter: new Date(0), + playSound = vitest.fn(); + (useAudioContext as MockedFunction).mockReturnValue( + { + playSound, + playSoundLooping: vitest.fn(), + soundDuration: {}, }, - }); - }); - expect(playSound).toHaveBeenCalledWith(chosenReaction.name); -}); - -test("will play the generic audio sound when there is soundless reaction", () => { - const { vm, reactionsSubject$ } = getBasicCallViewModelEnvironment([ - local, - alice, - ]); - playReactionsSoundSetting.setValue(true); - render(); - - // Find the first reaction with a sound effect - const chosenReaction = ReactionSet.find((r) => !r.sound); - if (!chosenReaction) { - throw Error( - "No reactions have sounds configured, this test cannot succeed", ); - } - act(() => { - reactionsSubject$.next({ - [aliceRtcMember.deviceId]: { - reactionOption: chosenReaction, - expireAfter: new Date(0), - }, - }); }); - expect(playSound).toHaveBeenCalledWith(GenericReaction.name); -}); - -test("will play multiple audio sounds when there are multiple different reactions", () => { - const { vm, reactionsSubject$ } = getBasicCallViewModelEnvironment([ - local, - alice, - ]); - playReactionsSoundSetting.setValue(true); - render(); - - // Find the first reaction with a sound effect - const [reaction1, reaction2] = ReactionSet.filter((r) => !!r.sound); - if (!reaction1 || !reaction2) { - throw Error( - "No reactions have sounds configured, this test cannot succeed", - ); - } - act(() => { - reactionsSubject$.next({ - [aliceRtcMember.deviceId]: { - reactionOption: reaction1, - expireAfter: new Date(0), - }, - [bobRtcMember.deviceId]: { - reactionOption: reaction2, - expireAfter: new Date(0), - }, - [localRtcMember.deviceId]: { - reactionOption: reaction1, - expireAfter: new Date(0), - }, - }); + afterAll(() => { + vitest.restoreAllMocks(); + }); + + it("preloads all audio elements", () => { + const { vm } = getBasicCallViewModelEnvironment([local, alice]); + playReactionsSoundSetting.setValue(true); + render(); + expect(prefetchSounds).toHaveBeenCalledOnce(); + }); + + it("will play an audio sound when there is a reaction", () => { + const { vm, reactionsSubject$ } = getBasicCallViewModelEnvironment([ + local, + alice, + ]); + playReactionsSoundSetting.setValue(true); + render(); + + // Find the first reaction with a sound effect + const chosenReaction = ReactionSet.find((r) => !!r.sound); + if (!chosenReaction) { + throw Error( + "No reactions have sounds configured, this test cannot succeed", + ); + } + act(() => { + reactionsSubject$.next({ + [aliceRtcMember.deviceId]: { + reactionOption: chosenReaction, + expireAfter: new Date(0), + }, + }); + }); + expect(playSound).toHaveBeenCalledWith(chosenReaction.name); + }); + + it("will play the generic audio sound when there is soundless reaction", () => { + const { vm, reactionsSubject$ } = getBasicCallViewModelEnvironment([ + local, + alice, + ]); + playReactionsSoundSetting.setValue(true); + render(); + + // Find the first reaction with a sound effect + const chosenReaction = ReactionSet.find((r) => !r.sound); + if (!chosenReaction) { + throw Error( + "No reactions have sounds configured, this test cannot succeed", + ); + } + act(() => { + reactionsSubject$.next({ + [aliceRtcMember.deviceId]: { + reactionOption: chosenReaction, + expireAfter: new Date(0), + }, + }); + }); + expect(playSound).toHaveBeenCalledWith(GenericReaction.name); + }); + + it("will play multiple audio sounds when there are multiple different reactions", () => { + const { vm, reactionsSubject$ } = getBasicCallViewModelEnvironment([ + local, + alice, + ]); + playReactionsSoundSetting.setValue(true); + render(); + + // Find the first reaction with a sound effect + const [reaction1, reaction2] = ReactionSet.filter((r) => !!r.sound); + if (!reaction1 || !reaction2) { + throw Error( + "No reactions have sounds configured, this test cannot succeed", + ); + } + act(() => { + reactionsSubject$.next({ + [aliceRtcMember.deviceId]: { + reactionOption: reaction1, + expireAfter: new Date(0), + }, + [bobRtcMember.deviceId]: { + reactionOption: reaction2, + expireAfter: new Date(0), + }, + [localRtcMember.deviceId]: { + reactionOption: reaction1, + expireAfter: new Date(0), + }, + }); + }); + expect(playSound).toHaveBeenCalledWith(reaction1.name); + expect(playSound).toHaveBeenCalledWith(reaction2.name); }); - expect(playSound).toHaveBeenCalledWith(reaction1.name); - expect(playSound).toHaveBeenCalledWith(reaction2.name); }); diff --git a/src/room/ReactionAudioRenderer.tsx b/src/room/ReactionAudioRenderer.tsx index 2b95acb9..06170d19 100644 --- a/src/room/ReactionAudioRenderer.tsx +++ b/src/room/ReactionAudioRenderer.tsx @@ -12,7 +12,7 @@ import { GenericReaction, ReactionSet } from "../reactions"; import { useAudioContext } from "../useAudioContext"; import { prefetchSounds } from "../soundUtils"; import { useLatest } from "../useLatest"; -import { type CallViewModel } from "../state/CallViewModel"; +import { type CallViewModel } from "../state/CallViewModel/CallViewModel"; const soundMap = Object.fromEntries([ ...ReactionSet.filter((v) => v.sound !== undefined).map((v) => [ diff --git a/src/room/ReactionsOverlay.tsx b/src/room/ReactionsOverlay.tsx index f3dff848..e7c097d2 100644 --- a/src/room/ReactionsOverlay.tsx +++ b/src/room/ReactionsOverlay.tsx @@ -8,7 +8,7 @@ Please see LICENSE in the repository root for full details. import { type ReactNode } from "react"; import styles from "./ReactionsOverlay.module.css"; -import { type CallViewModel } from "../state/CallViewModel"; +import { type CallViewModel } from "../state/CallViewModel/CallViewModel"; import { useBehavior } from "../useBehavior"; export function ReactionsOverlay({ vm }: { vm: CallViewModel }): ReactNode { diff --git a/src/room/__snapshots__/InCallView.test.tsx.snap b/src/room/__snapshots__/InCallView.test.tsx.snap index 661d13ad..c610b1fb 100644 --- a/src/room/__snapshots__/InCallView.test.tsx.snap +++ b/src/room/__snapshots__/InCallView.test.tsx.snap @@ -83,9 +83,6 @@ exports[`InCallView > rendering > renders 1`] = ` class="nav rightNav" /> -
- mocked: MatrixAudioRenderer -
diff --git a/src/rtcSessionHelpers.ts b/src/rtcSessionHelpers.ts deleted file mode 100644 index fadc7b37..00000000 --- a/src/rtcSessionHelpers.ts +++ /dev/null @@ -1,161 +0,0 @@ -/* -Copyright 2023, 2024 New Vector Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { - type MatrixRTCSession, - isLivekitTransportConfig, - type LivekitTransportConfig, - type LivekitTransport, -} from "matrix-js-sdk/lib/matrixrtc"; -import { logger } from "matrix-js-sdk/lib/logger"; -import { AutoDiscovery } from "matrix-js-sdk/lib/autodiscovery"; - -import { PosthogAnalytics } from "./analytics/PosthogAnalytics"; -import { Config } from "./config/Config"; -import { ElementWidgetActions, widget } from "./widget"; -import { MatrixRTCTransportMissingError } from "./utils/errors"; -import { getUrlParams } from "./UrlParams"; -import { getSFUConfigWithOpenID } from "./livekit/openIDSFU.ts"; - -const FOCI_WK_KEY = "org.matrix.msc4143.rtc_foci"; - -export function getLivekitAlias(rtcSession: MatrixRTCSession): string { - // For now we assume everything is a room-scoped call - return rtcSession.room.roomId; -} - -async function makeTransportInternal( - rtcSession: MatrixRTCSession, -): Promise { - logger.log("Searching for a preferred transport"); - const livekitAlias = getLivekitAlias(rtcSession); - - // TODO-MULTI-SFU: Either remove this dev tool or make it more official - const urlFromStorage = - localStorage.getItem("robin-matrixrtc-auth") ?? - localStorage.getItem("timo-focus-url"); - if (urlFromStorage !== null) { - const transportFromStorage: LivekitTransport = { - type: "livekit", - livekit_service_url: urlFromStorage, - livekit_alias: livekitAlias, - }; - logger.log( - "Using LiveKit transport from local storage: ", - transportFromStorage, - ); - return transportFromStorage; - } - - // Prioritize the .well-known/matrix/client, if available, over the configured SFU - const domain = rtcSession.room.client.getDomain(); - if (domain) { - // we use AutoDiscovery instead of relying on the MatrixClient having already - // been fully configured and started - const wellKnownFoci = (await AutoDiscovery.getRawClientConfig(domain))?.[ - FOCI_WK_KEY - ]; - if (Array.isArray(wellKnownFoci)) { - const transport: LivekitTransportConfig | undefined = wellKnownFoci.find( - (f) => f && isLivekitTransportConfig(f), - ); - if (transport !== undefined) { - logger.log("Using LiveKit transport from .well-known: ", transport); - return { ...transport, livekit_alias: livekitAlias }; - } - } - } - - const urlFromConf = Config.get().livekit?.livekit_service_url; - if (urlFromConf) { - const transportFromConf: LivekitTransport = { - type: "livekit", - livekit_service_url: urlFromConf, - livekit_alias: livekitAlias, - }; - logger.log("Using LiveKit transport from config: ", transportFromConf); - return transportFromConf; - } - - throw new MatrixRTCTransportMissingError(domain ?? ""); -} - -export async function makeTransport( - rtcSession: MatrixRTCSession, -): Promise { - const transport = await makeTransportInternal(rtcSession); - // this will call the jwt/sfu/get endpoint to pre create the livekit room. - await getSFUConfigWithOpenID( - rtcSession.room.client, - transport.livekit_service_url, - transport.livekit_alias, - ); - return transport; -} - -export interface EnterRTCSessionOptions { - encryptMedia: boolean; - /** EXPERIMENTAL: If true, will use the multi-sfu codepath where each member connects to its SFU instead of everyone connecting to an elected on. */ - useMultiSfu: boolean; - preferStickyEvents: boolean; -} - -/** - * TODO! document this function properly - * @param rtcSession - * @param transport - * @param options - */ -export async function enterRTCSession( - rtcSession: MatrixRTCSession, - transport: LivekitTransport, - { encryptMedia, useMultiSfu, preferStickyEvents }: EnterRTCSessionOptions, -): Promise { - PosthogAnalytics.instance.eventCallEnded.cacheStartCall(new Date()); - PosthogAnalytics.instance.eventCallStarted.track(rtcSession.room.roomId); - - // This must be called before we start trying to join the call, as we need to - // have started tracking by the time calls start getting created. - // groupCallOTelMembership?.onJoinCall(); - - const { features, matrix_rtc_session: matrixRtcSessionConfig } = Config.get(); - const useDeviceSessionMemberEvents = - features?.feature_use_device_session_member_events; - const { sendNotificationType: notificationType, callIntent } = getUrlParams(); - // Multi-sfu does not need a preferred foci list. just the focus that is actually used. - rtcSession.joinRoomSession( - useMultiSfu ? [] : [transport], - useMultiSfu ? transport : undefined, - { - notificationType, - callIntent, - manageMediaKeys: encryptMedia, - ...(useDeviceSessionMemberEvents !== undefined && { - useLegacyMemberEvents: !useDeviceSessionMemberEvents, - }), - delayedLeaveEventRestartMs: - matrixRtcSessionConfig?.delayed_leave_event_restart_ms, - delayedLeaveEventDelayMs: - matrixRtcSessionConfig?.delayed_leave_event_delay_ms, - delayedLeaveEventRestartLocalTimeoutMs: - matrixRtcSessionConfig?.delayed_leave_event_restart_local_timeout_ms, - networkErrorRetryMs: matrixRtcSessionConfig?.network_error_retry_ms, - makeKeyDelay: matrixRtcSessionConfig?.wait_for_key_rotation_ms, - membershipEventExpiryMs: - matrixRtcSessionConfig?.membership_event_expiry_ms, - useExperimentalToDeviceTransport: true, - unstableSendStickyEvents: preferStickyEvents, - }, - ); - if (widget) { - try { - await widget.api.transport.send(ElementWidgetActions.JoinCall, {}); - } catch (e) { - logger.error("Failed to send join action", e); - } - } -} diff --git a/src/settings/DeveloperSettingsTab.tsx b/src/settings/DeveloperSettingsTab.tsx index 08c22557..60d9028d 100644 --- a/src/settings/DeveloperSettingsTab.tsx +++ b/src/settings/DeveloperSettingsTab.tsx @@ -12,6 +12,7 @@ import { useEffect, useMemo, useState, + useId, } from "react"; import { useTranslation } from "react-i18next"; import { @@ -19,6 +20,14 @@ import { type MatrixClient, } from "matrix-js-sdk"; import { logger } from "matrix-js-sdk/lib/logger"; +import { + Root as Form, + Heading, + HelpMessage, + InlineField, + Label, + RadioControl, +} from "@vector-im/compound-web"; import { FieldRow, InputField } from "../input/Input"; import { @@ -26,10 +35,10 @@ import { duplicateTiles as duplicateTilesSetting, debugTileLayout as debugTileLayoutSetting, showConnectionStats as showConnectionStatsSetting, - multiSfu as multiSfuSetting, muteAllAudio as muteAllAudioSetting, alwaysShowIphoneEarpiece as alwaysShowIphoneEarpieceSetting, - preferStickyEvents as preferStickyEventsSetting, + matrixRTCMode as matrixRTCModeSetting, + MatrixRTCMode, } from "./settings"; import type { Room as LivekitRoom } from "livekit-client"; import styles from "./DeveloperSettingsTab.module.css"; @@ -59,8 +68,13 @@ export const DeveloperSettingsTab: FC = ({ client, livekitRooms }) => { }); }, [client]); - const [preferStickyEvents, setPreferStickyEvents] = useSetting( - preferStickyEventsSetting, + const [matrixRTCMode, setMatrixRTCMode] = useSetting(matrixRTCModeSetting); + const matrixRTCModeRadioGroup = useId(); + const onMatrixRTCModeChange = useCallback( + (e: ChangeEvent) => { + setMatrixRTCMode(e.target.value as MatrixRTCMode); + }, + [setMatrixRTCMode], ); const [showConnectionStats, setShowConnectionStats] = useSetting( @@ -71,8 +85,6 @@ export const DeveloperSettingsTab: FC = ({ client, livekitRooms }) => { alwaysShowIphoneEarpieceSetting, ); - const [multiSfu, setMultiSfu] = useSetting(multiSfuSetting); - const [muteAllAudio, setMuteAllAudio] = useSetting(muteAllAudioSetting); const urlParams = useUrlParams(); @@ -89,7 +101,7 @@ export const DeveloperSettingsTab: FC = ({ client, livekitRooms }) => { }, [livekitRooms]); return ( - <> +

{t("developer_mode.hostname", { hostname: window.location.hostname || "unknown", @@ -146,22 +158,6 @@ export const DeveloperSettingsTab: FC = ({ client, livekitRooms }) => { } /> - - ): void => { - setPreferStickyEvents(event.target.checked); - }, - [setPreferStickyEvents], - )} - /> - = ({ client, livekitRooms }) => { )} /> - - ): void => { - setMultiSfu(event.target.checked); - }, - [setMultiSfu], - )} - /> - = ({ client, livekitRooms }) => { )} />{" "} + + {t("developer_mode.matrixRTCMode.title")} + + + } + > + + + {t("developer_mode.matrixRTCMode.Legacy.description")} + + + + } + > + + + {t("developer_mode.matrixRTCMode.Comptibility.description")} + + + + } + > + + + {t("developer_mode.matrixRTCMode.Matrix_2_0.description")} + + {livekitRooms?.map((livekitRoom) => ( <>

@@ -244,6 +273,6 @@ export const DeveloperSettingsTab: FC = ({ client, livekitRooms }) => { 
{JSON.stringify(import.meta.env, null, 2)}

{t("developer_mode.url_params")}

{JSON.stringify(urlParams, null, 2)}
- + ); }; diff --git a/src/settings/settings.ts b/src/settings/settings.ts index b58db983..5309ecf8 100644 --- a/src/settings/settings.ts +++ b/src/settings/settings.ts @@ -83,11 +83,6 @@ export const showConnectionStats = new Setting( false, ); -export const preferStickyEvents = new Setting( - "prefer-sticky-events", - false, -); - export const audioInput = new Setting( "audio-input", undefined, @@ -120,8 +115,6 @@ export const soundEffectVolume = new Setting( 0.5, ); -export const multiSfu = new Setting("multi-sfu", false); - export const muteAllAudio = new Setting("mute-all-audio", false); export const alwaysShowSelf = new Setting("always-show-self", true); @@ -130,3 +123,14 @@ export const alwaysShowIphoneEarpiece = new Setting( "always-show-iphone-earpiece", false, ); + +export enum MatrixRTCMode { + Legacy = "legacy", + Compatibil = "compatibil", + Matrix_2_0 = "matrix_2_0", +} + +export const matrixRTCMode = new Setting( + "matrix-rtc-mode", + MatrixRTCMode.Legacy, +); diff --git a/src/state/Async.ts b/src/state/Async.ts deleted file mode 100644 index 61871f78..00000000 --- a/src/state/Async.ts +++ /dev/null @@ -1,53 +0,0 @@ -/* -Copyright 2025 New Vector Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { catchError, from, map, type Observable, of, startWith } from "rxjs"; - -/** - * Data that may need to be loaded asynchronously. - * - * This type is for when you need to represent the current state of an operation - * involving Promises as **immutable data**. See the async$ function below. - */ -export type Async = - | { state: "loading" } - | { state: "error"; value: Error } - | { state: "ready"; value: A }; - -export const loading: Async = { state: "loading" }; -export function error(value: Error): Async { - return { state: "error", value }; -} - -export function ready(value: A): Async { - return { state: "ready", value }; -} - -/** - * Turn a Promise into an Observable async value. The Observable will have the - * value "loading" while the Promise is pending, "ready" when the Promise - * resolves, and "error" when the Promise rejects. - */ -export function async$(promise: Promise): Observable> { - return from(promise).pipe( - map(ready), - startWith(loading), - catchError((e: unknown) => - of(error((e as Error) ?? new Error("Unknown error"))), - ), - ); -} - -/** - * If the async value is ready, apply the given function to the inner value. - */ -export function mapAsync( - async: Async, - project: (value: A) => B, -): Async { - return async.state === "ready" ? ready(project(async.value)) : async; -} diff --git a/src/state/CallViewModel.test.ts b/src/state/CallViewModel.test.ts deleted file mode 100644 index fec6b8cf..00000000 --- a/src/state/CallViewModel.test.ts +++ /dev/null @@ -1,1875 +0,0 @@ -/* -Copyright 2024 New Vector Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { test, vi, onTestFinished, it, describe, expect } from "vitest"; -import EventEmitter from "events"; -import { - BehaviorSubject, - combineLatest, - debounceTime, - distinctUntilChanged, - map, - NEVER, - type Observable, - of, - switchMap, -} from "rxjs"; -import { - ClientEvent, - SyncState, - type MatrixClient, - RoomEvent as MatrixRoomEvent, - MatrixEvent, - type IRoomTimelineData, - EventType, - type IEvent, -} from "matrix-js-sdk"; -import { - ConnectionState, - type LocalParticipant, - type LocalTrackPublication, - type Participant, - ParticipantEvent, - type RemoteParticipant, - type Room as LivekitRoom, -} from "livekit-client"; -import * as ComponentsCore from "@livekit/components-core"; -import { - Status, - type CallMembership, - type IRTCNotificationContent, - type ICallNotifyContent, - MatrixRTCSessionEvent, - type LivekitTransport, -} from "matrix-js-sdk/lib/matrixrtc"; -import { deepCompare } from "matrix-js-sdk/lib/utils"; -import { AutoDiscovery } from "matrix-js-sdk/lib/autodiscovery"; - -import { CallViewModel, type CallViewModelOptions } from "./CallViewModel"; -import { type Layout } from "./layout-types"; -import { - mockLocalParticipant, - mockMatrixRoom, - mockMatrixRoomMember, - mockRemoteParticipant, - withTestScheduler, - mockRtcMembership, - MockRTCSession, - mockMediaDevices, - mockMuteStates, - mockConfig, - testScope, - mockLivekitRoom, - exampleTransport, -} from "../utils/test"; -import { E2eeType } from "../e2ee/e2eeType"; -import type { RaisedHandInfo, ReactionInfo } from "../reactions"; -import { - alice, - aliceDoppelganger, - aliceDoppelgangerId, - aliceDoppelgangerRtcMember, - aliceId, - aliceParticipant, - aliceRtcMember, - bob, - bobId, - bobRtcMember, - bobZeroWidthSpace, - bobZeroWidthSpaceId, - bobZeroWidthSpaceRtcMember, - daveRTL, - daveRTLId, - daveRTLRtcMember, - local, - localId, - localRtcMember, - localRtcMemberDevice2, -} from "../utils/test-fixtures"; -import { MediaDevices } from "./MediaDevices"; -import { getValue } from "../utils/observable"; -import { type Behavior, constant } from "./Behavior"; -import type { ProcessorState } from "../livekit/TrackProcessorContext.tsx"; -import { - type ElementCallError, - MatrixRTCTransportMissingError, -} from "../utils/errors.ts"; - -vi.mock("rxjs", async (importOriginal) => ({ - ...(await importOriginal()), - // Disable interval Observables for the following tests since the test - // scheduler will loop on them forever and never call the test 'done' - interval: (): Observable => NEVER, -})); - -vi.mock("@livekit/components-core"); -vi.mock("livekit-client/e2ee-worker?worker"); - -vi.mock("../e2ee/matrixKeyProvider"); - -const getUrlParams = vi.hoisted(() => vi.fn(() => ({}))); -vi.mock("../UrlParams", () => ({ getUrlParams })); - -vi.mock("../rtcSessionHelpers", async (importOriginal) => ({ - ...(await importOriginal()), - makeTransport: async (): Promise => - Promise.resolve(exampleTransport), -})); - -const yesNo = { - y: true, - n: false, -}; - -const daveRtcMember = mockRtcMembership("@dave:example.org", "DDDD"); - -const carol = local; -const carolId = localId; -const dave = mockMatrixRoomMember(daveRtcMember, { rawDisplayName: "Dave" }); - -const daveId = `${dave.userId}:${daveRtcMember.deviceId}`; - -const localParticipant = mockLocalParticipant({ identity: "" }); -const aliceSharingScreen = mockRemoteParticipant({ - identity: aliceId, - isScreenShareEnabled: true, -}); -const bobParticipant = mockRemoteParticipant({ identity: bobId }); -const bobSharingScreen = mockRemoteParticipant({ - identity: bobId, - isScreenShareEnabled: true, -}); -const daveParticipant = mockRemoteParticipant({ identity: daveId }); - -const roomMembers = new Map( - [alice, aliceDoppelganger, bob, bobZeroWidthSpace, carol, dave, daveRTL].map( - (p) => [p.userId, p], - ), -); - -export interface GridLayoutSummary { - type: "grid"; - spotlight?: string[]; - grid: string[]; -} - -export interface SpotlightLandscapeLayoutSummary { - type: "spotlight-landscape"; - spotlight: string[]; - grid: string[]; -} - -export interface SpotlightPortraitLayoutSummary { - type: "spotlight-portrait"; - spotlight: string[]; - grid: string[]; -} - -export interface SpotlightExpandedLayoutSummary { - type: "spotlight-expanded"; - spotlight: string[]; - pip?: string; -} - -export interface OneOnOneLayoutSummary { - type: "one-on-one"; - local: string; - remote: string; -} - -export interface PipLayoutSummary { - type: "pip"; - spotlight: string[]; -} - -export type LayoutSummary = - | GridLayoutSummary - | SpotlightLandscapeLayoutSummary - | SpotlightPortraitLayoutSummary - | SpotlightExpandedLayoutSummary - | OneOnOneLayoutSummary - | PipLayoutSummary; - -function summarizeLayout$(l$: Observable): Observable { - return l$.pipe( - switchMap((l) => { - switch (l.type) { - case "grid": - return combineLatest( - [ - l.spotlight?.media$ ?? constant(undefined), - ...l.grid.map((vm) => vm.media$), - ], - (spotlight, ...grid) => ({ - type: l.type, - spotlight: spotlight?.map((vm) => vm.id), - grid: grid.map((vm) => vm.id), - }), - ); - case "spotlight-landscape": - case "spotlight-portrait": - return combineLatest( - [l.spotlight.media$, ...l.grid.map((vm) => vm.media$)], - (spotlight, ...grid) => ({ - type: l.type, - spotlight: spotlight.map((vm) => vm.id), - grid: grid.map((vm) => vm.id), - }), - ); - case "spotlight-expanded": - return combineLatest( - [l.spotlight.media$, l.pip?.media$ ?? constant(undefined)], - (spotlight, pip) => ({ - type: l.type, - spotlight: spotlight.map((vm) => vm.id), - pip: pip?.id, - }), - ); - case "one-on-one": - return combineLatest( - [l.local.media$, l.remote.media$], - (local, remote) => ({ - type: l.type, - local: local.id, - remote: remote.id, - }), - ); - case "pip": - return l.spotlight.media$.pipe( - map((spotlight) => ({ - type: l.type, - spotlight: spotlight.map((vm) => vm.id), - })), - ); - } - }), - // Sometimes there can be multiple (synchronous) updates per frame. We only - // care about the most recent value for each time step, so discard these - // extra values. - debounceTime(0), - distinctUntilChanged(deepCompare), - ); -} - -function mockRingEvent( - eventId: string, - lifetimeMs: number | undefined, - sender = local.userId, -): { event_id: string } & IRTCNotificationContent { - return { - event_id: eventId, - ...(lifetimeMs === undefined ? {} : { lifetime: lifetimeMs }), - notification_type: "ring", - sender, - } as unknown as { event_id: string } & IRTCNotificationContent; -} - -// The app doesn't really care about the content of these legacy events, we just -// need a value to fill in for them when emitting notifications -const mockLegacyRingEvent = {} as { event_id: string } & ICallNotifyContent; - -interface CallViewModelInputs { - remoteParticipants$: Behavior; - rtcMembers$: Behavior[]>; - livekitConnectionState$: Behavior; - speaking: Map>; - mediaDevices: MediaDevices; - initialSyncState: SyncState; -} - -export function withCallViewModel( - { - remoteParticipants$ = constant([]), - rtcMembers$ = constant([localRtcMember]), - livekitConnectionState$: connectionState$ = constant( - ConnectionState.Connected, - ), - speaking = new Map(), - mediaDevices = mockMediaDevices({}), - initialSyncState = SyncState.Syncing, - }: Partial, - continuation: ( - vm: CallViewModel, - rtcSession: MockRTCSession, - subjects: { raisedHands$: BehaviorSubject> }, - setSyncState: (value: SyncState) => void, - ) => void, - options: CallViewModelOptions = { - encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, - autoLeaveWhenOthersLeft: false, - }, -): void { - let syncState = initialSyncState; - const setSyncState = (value: SyncState): void => { - const prev = syncState; - syncState = value; - room.client.emit(ClientEvent.Sync, value, prev); - }; - const room = mockMatrixRoom({ - client: new (class extends EventEmitter { - public getUserId(): string | undefined { - return localRtcMember.userId; - } - public getDeviceId(): string { - return localRtcMember.deviceId; - } - public getSyncState(): SyncState { - return syncState; - } - })() as Partial as MatrixClient, - getMember: (userId) => roomMembers.get(userId) ?? null, - }); - const rtcSession = new MockRTCSession(room, []).withMemberships(rtcMembers$); - const participantsSpy = vi - .spyOn(ComponentsCore, "connectedParticipantsObserver") - .mockReturnValue(remoteParticipants$); - const mediaSpy = vi - .spyOn(ComponentsCore, "observeParticipantMedia") - .mockImplementation((p) => - of({ participant: p } as Partial< - ComponentsCore.ParticipantMedia - > as ComponentsCore.ParticipantMedia), - ); - const eventsSpy = vi - .spyOn(ComponentsCore, "observeParticipantEvents") - .mockImplementation((p, ...eventTypes) => { - if (eventTypes.includes(ParticipantEvent.IsSpeakingChanged)) { - return (speaking.get(p) ?? of(false)).pipe( - map((s) => ({ ...p, isSpeaking: s }) as Participant), - ); - } else { - return of(p); - } - }); - - const roomEventSelectorSpy = vi - .spyOn(ComponentsCore, "roomEventSelector") - .mockImplementation((_room, _eventType) => of()); - const muteStates = mockMuteStates(); - const raisedHands$ = new BehaviorSubject>({}); - const reactions$ = new BehaviorSubject>({}); - - const vm = new CallViewModel( - testScope(), - rtcSession.asMockedSession(), - room, - mediaDevices, - muteStates, - { - ...options, - livekitRoomFactory: (): LivekitRoom => - mockLivekitRoom({ - localParticipant, - disconnect: async () => Promise.resolve(), - setE2EEEnabled: async () => Promise.resolve(), - }), - connectionState$, - }, - raisedHands$, - reactions$, - new BehaviorSubject({ - processor: undefined, - supported: undefined, - }), - ); - - onTestFinished(() => { - participantsSpy.mockRestore(); - mediaSpy.mockRestore(); - eventsSpy.mockRestore(); - roomEventSelectorSpy.mockRestore(); - }); - - continuation(vm, rtcSession, { raisedHands$: raisedHands$ }, setSyncState); -} - -// TODO: Restore this test. It requires makeTransport to not be mocked, unlike -// the rest of the tests in this file… what do we do? -test.skip("test missing RTC config error", async () => { - const rtcMemberships$ = new BehaviorSubject([]); - const emitter = new EventEmitter(); - const client = vi.mocked({ - on: emitter.on.bind(emitter), - off: emitter.off.bind(emitter), - getSyncState: vi.fn().mockReturnValue(SyncState.Syncing), - getUserId: vi.fn().mockReturnValue("@user:localhost"), - getUser: vi.fn().mockReturnValue(null), - getDeviceId: vi.fn().mockReturnValue("DEVICE"), - credentials: { - userId: "@user:localhost", - }, - getCrypto: vi.fn().mockReturnValue(undefined), - getDomain: vi.fn().mockReturnValue("example.org"), - } as unknown as MatrixClient); - - const matrixRoom = mockMatrixRoom({ - roomId: "!myRoomId:example.com", - client, - getMember: vi.fn().mockReturnValue(undefined), - }); - - const fakeRtcSession = new MockRTCSession(matrixRoom).withMemberships( - rtcMemberships$, - ); - - mockConfig({}); - vi.spyOn(AutoDiscovery, "getRawClientConfig").mockResolvedValue({}); - - const callVM = new CallViewModel( - testScope(), - fakeRtcSession.asMockedSession(), - matrixRoom, - mockMediaDevices({}), - mockMuteStates(), - { - encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, - autoLeaveWhenOthersLeft: false, - livekitRoomFactory: (): LivekitRoom => - mockLivekitRoom({ - localParticipant, - disconnect: async () => Promise.resolve(), - setE2EEEnabled: async () => Promise.resolve(), - }), - }, - new BehaviorSubject({} as Record), - new BehaviorSubject({} as Record), - of({ processor: undefined, supported: false }), - ); - - const failPromise = Promise.withResolvers(); - callVM.configError$.subscribe((error) => { - if (error) { - failPromise.resolve(error); - } - }); - - const error = await failPromise.promise; - expect(error).toBeInstanceOf(MatrixRTCTransportMissingError); -}); - -test("participants are retained during a focus switch", () => { - withTestScheduler(({ behavior, expectObservable }) => { - // Participants disappear on frame 2 and come back on frame 3 - const participantInputMarbles = "a-ba"; - // Start switching focus on frame 1 and reconnect on frame 3 - const connectionInputMarbles = " cs-c"; - // The visible participants should remain the same throughout the switch - const expectedLayoutMarbles = " a"; - - withCallViewModel( - { - remoteParticipants$: behavior(participantInputMarbles, { - a: [aliceParticipant, bobParticipant], - b: [], - }), - rtcMembers$: constant([localRtcMember, aliceRtcMember, bobRtcMember]), - livekitConnectionState$: behavior(connectionInputMarbles, { - c: ConnectionState.Connected, - s: ConnectionState.Connecting, - }), - }, - (vm) => { - expectObservable(summarizeLayout$(vm.layout$)).toBe( - expectedLayoutMarbles, - { - a: { - type: "grid", - spotlight: undefined, - grid: [`${localId}:0`, `${aliceId}:0`, `${bobId}:0`], - }, - }, - ); - }, - ); - }); -}); - -test("screen sharing activates spotlight layout", () => { - withTestScheduler(({ behavior, schedule, expectObservable }) => { - // Start with no screen shares, then have Alice and Bob share their screens, - // then return to no screen shares, then have just Alice share for a bit - const participantInputMarbles = " abcda-ba"; - // While there are no screen shares, switch to spotlight manually, and then - // switch back to grid at the end - const modeInputMarbles = " -----s--g"; - // We should automatically enter spotlight for the first round of screen - // sharing, then return to grid, then manually go into spotlight, and - // remain in spotlight until we manually go back to grid - const expectedLayoutMarbles = " abcdaefeg"; - const expectedShowSpeakingMarbles = "y----nyny"; - withCallViewModel( - { - remoteParticipants$: behavior(participantInputMarbles, { - a: [aliceParticipant, bobParticipant], - b: [aliceSharingScreen, bobParticipant], - c: [aliceSharingScreen, bobSharingScreen], - d: [aliceParticipant, bobSharingScreen], - }), - rtcMembers$: constant([localRtcMember, aliceRtcMember, bobRtcMember]), - }, - (vm) => { - schedule(modeInputMarbles, { - s: () => vm.setGridMode("spotlight"), - g: () => vm.setGridMode("grid"), - }); - - expectObservable(summarizeLayout$(vm.layout$)).toBe( - expectedLayoutMarbles, - { - a: { - type: "grid", - spotlight: undefined, - grid: [`${localId}:0`, `${aliceId}:0`, `${bobId}:0`], - }, - b: { - type: "spotlight-landscape", - spotlight: [`${aliceId}:0:screen-share`], - grid: [`${localId}:0`, `${aliceId}:0`, `${bobId}:0`], - }, - c: { - type: "spotlight-landscape", - spotlight: [ - `${aliceId}:0:screen-share`, - `${bobId}:0:screen-share`, - ], - grid: [`${localId}:0`, `${aliceId}:0`, `${bobId}:0`], - }, - d: { - type: "spotlight-landscape", - spotlight: [`${bobId}:0:screen-share`], - grid: [`${localId}:0`, `${aliceId}:0`, `${bobId}:0`], - }, - e: { - type: "spotlight-landscape", - spotlight: [`${aliceId}:0`], - grid: [`${localId}:0`, `${bobId}:0`], - }, - f: { - type: "spotlight-landscape", - spotlight: [`${aliceId}:0:screen-share`], - grid: [`${localId}:0`, `${bobId}:0`, `${aliceId}:0`], - }, - g: { - type: "grid", - spotlight: undefined, - grid: [`${localId}:0`, `${bobId}:0`, `${aliceId}:0`], - }, - }, - ); - expectObservable(vm.showSpeakingIndicators$).toBe( - expectedShowSpeakingMarbles, - yesNo, - ); - }, - ); - }); -}); - -test("participants stay in the same order unless to appear/disappear", () => { - withTestScheduler(({ behavior, schedule, expectObservable }) => { - const visibilityInputMarbles = "a"; - // First Bob speaks, then Dave, then Alice - const aSpeakingInputMarbles = " n- 1998ms - 1999ms y"; - const bSpeakingInputMarbles = " ny 1998ms n 1999ms -"; - const dSpeakingInputMarbles = " n- 1998ms y 1999ms n"; - // Nothing should change when Bob speaks, because Bob is already on screen. - // When Dave speaks he should switch with Alice because she's the one who - // hasn't spoken at all. Then when Alice speaks, she should return to her - // place at the top. - const expectedLayoutMarbles = " a 1999ms b 1999ms a 57999ms c 1999ms a"; - - withCallViewModel( - { - remoteParticipants$: constant([ - aliceParticipant, - bobParticipant, - daveParticipant, - ]), - rtcMembers$: constant([ - localRtcMember, - aliceRtcMember, - bobRtcMember, - daveRtcMember, - ]), - speaking: new Map([ - [aliceParticipant, behavior(aSpeakingInputMarbles, yesNo)], - [bobParticipant, behavior(bSpeakingInputMarbles, yesNo)], - [daveParticipant, behavior(dSpeakingInputMarbles, yesNo)], - ]), - }, - (vm) => { - schedule(visibilityInputMarbles, { - a: () => { - // We imagine that only three tiles (the first three) will be visible - // on screen at a time - vm.layout$.subscribe((layout) => { - if (layout.type === "grid") layout.setVisibleTiles(3); - }); - }, - }); - - expectObservable(summarizeLayout$(vm.layout$)).toBe( - expectedLayoutMarbles, - { - a: { - type: "grid", - spotlight: undefined, - grid: [ - `${localId}:0`, - `${aliceId}:0`, - `${bobId}:0`, - `${daveId}:0`, - ], - }, - b: { - type: "grid", - spotlight: undefined, - grid: [ - `${localId}:0`, - `${daveId}:0`, - `${bobId}:0`, - `${aliceId}:0`, - ], - }, - c: { - type: "grid", - spotlight: undefined, - grid: [ - `${localId}:0`, - `${aliceId}:0`, - `${daveId}:0`, - `${bobId}:0`, - ], - }, - }, - ); - }, - ); - }); -}); - -test("participants adjust order when space becomes constrained", () => { - withTestScheduler(({ behavior, schedule, expectObservable }) => { - // Start with all tiles on screen then shrink to 3 - const visibilityInputMarbles = "a-b"; - // Bob and Dave speak - const bSpeakingInputMarbles = " ny"; - const dSpeakingInputMarbles = " ny"; - // Nothing should change when Bob or Dave initially speak, because they are - // on screen. When the screen becomes smaller Alice should move off screen - // to make way for the speakers (specifically, she should swap with Dave). - const expectedLayoutMarbles = " a-b"; - - withCallViewModel( - { - remoteParticipants$: constant([ - aliceParticipant, - bobParticipant, - daveParticipant, - ]), - rtcMembers$: constant([ - localRtcMember, - aliceRtcMember, - bobRtcMember, - daveRtcMember, - ]), - speaking: new Map([ - [bobParticipant, behavior(bSpeakingInputMarbles, yesNo)], - [daveParticipant, behavior(dSpeakingInputMarbles, yesNo)], - ]), - }, - (vm) => { - let setVisibleTiles: ((value: number) => void) | null = null; - vm.layout$.subscribe((layout) => { - if (layout.type === "grid") setVisibleTiles = layout.setVisibleTiles; - }); - schedule(visibilityInputMarbles, { - a: () => setVisibleTiles!(Infinity), - b: () => setVisibleTiles!(3), - }); - - expectObservable(summarizeLayout$(vm.layout$)).toBe( - expectedLayoutMarbles, - { - a: { - type: "grid", - spotlight: undefined, - grid: [ - `${localId}:0`, - `${aliceId}:0`, - `${bobId}:0`, - `${daveId}:0`, - ], - }, - b: { - type: "grid", - spotlight: undefined, - grid: [ - `${localId}:0`, - `${daveId}:0`, - `${bobId}:0`, - `${aliceId}:0`, - ], - }, - }, - ); - }, - ); - }); -}); - -test("spotlight speakers swap places", () => { - withTestScheduler(({ behavior, schedule, expectObservable }) => { - // Go immediately into spotlight mode for the test - const modeInputMarbles = " s"; - // First Bob speaks, then Dave, then Alice - const aSpeakingInputMarbles = "n--y"; - const bSpeakingInputMarbles = "nyn"; - const dSpeakingInputMarbles = "n-yn"; - // Alice should start in the spotlight, then Bob, then Dave, then Alice - // again. However, the positions of Dave and Bob in the grid should be - // reversed by the end because they've been swapped in and out of the - // spotlight. - const expectedLayoutMarbles = "abcd"; - - withCallViewModel( - { - remoteParticipants$: constant([ - aliceParticipant, - bobParticipant, - daveParticipant, - ]), - rtcMembers$: constant([ - localRtcMember, - aliceRtcMember, - bobRtcMember, - daveRtcMember, - ]), - speaking: new Map([ - [aliceParticipant, behavior(aSpeakingInputMarbles, yesNo)], - [bobParticipant, behavior(bSpeakingInputMarbles, yesNo)], - [daveParticipant, behavior(dSpeakingInputMarbles, yesNo)], - ]), - }, - (vm) => { - schedule(modeInputMarbles, { s: () => vm.setGridMode("spotlight") }); - - expectObservable(summarizeLayout$(vm.layout$)).toBe( - expectedLayoutMarbles, - { - a: { - type: "spotlight-landscape", - spotlight: [`${aliceId}:0`], - grid: [`${localId}:0`, `${bobId}:0`, `${daveId}:0`], - }, - b: { - type: "spotlight-landscape", - spotlight: [`${bobId}:0`], - grid: [`${localId}:0`, `${aliceId}:0`, `${daveId}:0`], - }, - c: { - type: "spotlight-landscape", - spotlight: [`${daveId}:0`], - grid: [`${localId}:0`, `${aliceId}:0`, `${bobId}:0`], - }, - d: { - type: "spotlight-landscape", - spotlight: [`${aliceId}:0`], - grid: [`${localId}:0`, `${daveId}:0`, `${bobId}:0`], - }, - }, - ); - - // While we expect the media on tiles to change, layout$ itself should - // *never* meaningfully change. That is, we expect there to be no layout - // shifts as the spotlight speaker changes; instead, the same tiles - // should be reused for the whole duration of the test and simply have - // their media swapped out. This is meaningful for keeping the interface - // not too visually distracting during back-and-forth conversations, - // while still animating tiles to express people joining, leaving, etc. - expectObservable( - vm.layout$.pipe( - distinctUntilChanged(deepCompare), - debounceTime(0), - map(() => "x"), - ), - ).toBe("x"); // Expect just one emission - }, - ); - }); -}); - -test("layout enters picture-in-picture mode when requested", () => { - withTestScheduler(({ schedule, expectObservable }) => { - // Enable then disable picture-in-picture - const pipControlInputMarbles = "-ed"; - // Should go into picture-in-picture layout then back to grid - const expectedLayoutMarbles = " aba"; - - withCallViewModel( - { - remoteParticipants$: constant([aliceParticipant, bobParticipant]), - rtcMembers$: constant([localRtcMember, aliceRtcMember, bobRtcMember]), - }, - (vm) => { - schedule(pipControlInputMarbles, { - e: () => window.controls.enablePip(), - d: () => window.controls.disablePip(), - }); - - expectObservable(summarizeLayout$(vm.layout$)).toBe( - expectedLayoutMarbles, - { - a: { - type: "grid", - spotlight: undefined, - grid: [`${localId}:0`, `${aliceId}:0`, `${bobId}:0`], - }, - b: { - type: "pip", - spotlight: [`${aliceId}:0`], - }, - }, - ); - }, - ); - }); -}); - -test("PiP tile in expanded spotlight layout switches speakers without layout shifts", () => { - withTestScheduler(({ behavior, schedule, expectObservable }) => { - // Switch to spotlight immediately - const modeInputMarbles = " s"; - // And expand the spotlight immediately - const expandInputMarbles = " a"; - // First Bob speaks, then Dave, then Bob again - const bSpeakingInputMarbles = "n-yn--yn"; - const dSpeakingInputMarbles = "n---yn"; - // Should show Alice (presenter) in the PiP, then Bob, then Dave, then Bob - // again - const expectedLayoutMarbles = "a-b-c-b"; - - withCallViewModel( - { - remoteParticipants$: constant([ - aliceSharingScreen, - bobParticipant, - daveParticipant, - ]), - rtcMembers$: constant([ - localRtcMember, - aliceRtcMember, - bobRtcMember, - daveRtcMember, - ]), - speaking: new Map([ - [bobParticipant, behavior(bSpeakingInputMarbles, yesNo)], - [daveParticipant, behavior(dSpeakingInputMarbles, yesNo)], - ]), - }, - (vm) => { - schedule(modeInputMarbles, { - s: () => vm.setGridMode("spotlight"), - }); - schedule(expandInputMarbles, { - a: () => vm.toggleSpotlightExpanded$.value!(), - }); - - expectObservable(summarizeLayout$(vm.layout$)).toBe( - expectedLayoutMarbles, - { - a: { - type: "spotlight-expanded", - spotlight: [`${aliceId}:0:screen-share`], - pip: `${aliceId}:0`, - }, - b: { - type: "spotlight-expanded", - spotlight: [`${aliceId}:0:screen-share`], - pip: `${bobId}:0`, - }, - c: { - type: "spotlight-expanded", - spotlight: [`${aliceId}:0:screen-share`], - pip: `${daveId}:0`, - }, - }, - ); - - // While we expect the media on the PiP tile to change, layout$ itself - // should *never* meaningfully change. That is, we expect the same PiP - // tile to exist throughout the test and just have its media swapped out - // when the speaker changes, rather than for tiles to animate in/out. - // This is meaningful for keeping the interface not too visually - // distracting during back-and-forth conversations. - expectObservable( - vm.layout$.pipe( - distinctUntilChanged(deepCompare), - debounceTime(0), - map(() => "x"), - ), - ).toBe("x"); // Expect just one emission - }, - ); - }); -}); - -test("spotlight remembers whether it's expanded", () => { - withTestScheduler(({ schedule, expectObservable }) => { - // Start in spotlight mode, then switch to grid and back to spotlight a - // couple times - const modeInputMarbles = " s-gs-gs"; - // Expand and collapse the spotlight - const expandInputMarbles = " -a--a"; - // Spotlight should stay expanded during the first mode switch, and stay - // collapsed during the second mode switch - const expectedLayoutMarbles = "abcbada"; - - withCallViewModel( - { - remoteParticipants$: constant([aliceParticipant, bobParticipant]), - rtcMembers$: constant([localRtcMember, aliceRtcMember, bobRtcMember]), - }, - (vm) => { - schedule(modeInputMarbles, { - s: () => vm.setGridMode("spotlight"), - g: () => vm.setGridMode("grid"), - }); - schedule(expandInputMarbles, { - a: () => vm.toggleSpotlightExpanded$.value!(), - }); - - expectObservable(summarizeLayout$(vm.layout$)).toBe( - expectedLayoutMarbles, - { - a: { - type: "spotlight-landscape", - spotlight: [`${aliceId}:0`], - grid: [`${localId}:0`, `${bobId}:0`], - }, - b: { - type: "spotlight-expanded", - spotlight: [`${aliceId}:0`], - pip: `${localId}:0`, - }, - c: { - type: "grid", - spotlight: undefined, - grid: [`${localId}:0`, `${aliceId}:0`, `${bobId}:0`], - }, - d: { - type: "grid", - spotlight: undefined, - grid: [`${localId}:0`, `${bobId}:0`, `${aliceId}:0`], - }, - }, - ); - }, - ); - }); -}); - -test("participants must have a MatrixRTCSession to be visible", () => { - withTestScheduler(({ behavior, expectObservable }) => { - // iterate through a number of combinations of participants and MatrixRTC memberships - // Bob never has an MatrixRTC membership - const scenarioInputMarbles = " abcdec"; - // Bob should never be visible - const expectedLayoutMarbles = "a-bc-b"; - - withCallViewModel( - { - remoteParticipants$: behavior(scenarioInputMarbles, { - a: [], - b: [bobParticipant], - c: [aliceParticipant, bobParticipant], - d: [aliceParticipant, daveParticipant, bobParticipant], - e: [aliceParticipant, daveParticipant, bobSharingScreen], - }), - rtcMembers$: behavior(scenarioInputMarbles, { - a: [localRtcMember], - b: [localRtcMember], - c: [localRtcMember, aliceRtcMember], - d: [localRtcMember, aliceRtcMember, daveRtcMember], - e: [localRtcMember, aliceRtcMember, daveRtcMember], - }), - }, - (vm) => { - vm.setGridMode("grid"); - expectObservable(summarizeLayout$(vm.layout$)).toBe( - expectedLayoutMarbles, - { - a: { - type: "grid", - spotlight: undefined, - grid: [`${localId}:0`], - }, - b: { - type: "one-on-one", - local: `${localId}:0`, - remote: `${aliceId}:0`, - }, - c: { - type: "grid", - spotlight: undefined, - grid: [`${localId}:0`, `${aliceId}:0`, `${daveId}:0`], - }, - }, - ); - }, - ); - }); -}); - -it("should show at least one tile per MatrixRTCSession", () => { - withTestScheduler(({ behavior, expectObservable }) => { - // iterate through some combinations of MatrixRTC memberships - const scenarioInputMarbles = " abcd"; - // There should always be one tile for each MatrixRTCSession - const expectedLayoutMarbles = "abcd"; - - withCallViewModel( - { - rtcMembers$: behavior(scenarioInputMarbles, { - a: [localRtcMember], - b: [localRtcMember, aliceRtcMember], - c: [localRtcMember, aliceRtcMember, daveRtcMember], - d: [localRtcMember, daveRtcMember], - }), - }, - (vm) => { - vm.setGridMode("grid"); - expectObservable(summarizeLayout$(vm.layout$)).toBe( - expectedLayoutMarbles, - { - a: { - type: "grid", - spotlight: undefined, - grid: [`${localId}:0`], - }, - b: { - type: "one-on-one", - local: `${localId}:0`, - remote: `${aliceId}:0`, - }, - c: { - type: "grid", - spotlight: undefined, - grid: [`${localId}:0`, `${aliceId}:0`, `${daveId}:0`], - }, - d: { - type: "one-on-one", - local: `${localId}:0`, - remote: `${daveId}:0`, - }, - }, - ); - }, - ); - }); -}); - -test("should disambiguate users with the same displayname", () => { - withTestScheduler(({ behavior, expectObservable }) => { - const scenarioInputMarbles = "abcde"; - const expectedLayoutMarbles = "abcde"; - - withCallViewModel( - { - rtcMembers$: behavior(scenarioInputMarbles, { - a: [localRtcMember], - b: [localRtcMember, aliceRtcMember], - c: [localRtcMember, aliceRtcMember, aliceDoppelgangerRtcMember], - d: [ - localRtcMember, - aliceRtcMember, - aliceDoppelgangerRtcMember, - bobRtcMember, - ], - e: [localRtcMember, aliceDoppelgangerRtcMember, bobRtcMember], - }), - }, - (vm) => { - expectObservable(vm.memberDisplaynames$).toBe(expectedLayoutMarbles, { - // Carol has no displayname - So userId is used. - a: new Map([[carolId, carol.userId]]), - b: new Map([ - [carolId, carol.userId], - [aliceId, alice.rawDisplayName], - ]), - // The second alice joins. - c: new Map([ - [carolId, carol.userId], - [aliceId, "Alice (@alice:example.org)"], - [aliceDoppelgangerId, "Alice (@alice2:example.org)"], - ]), - // Bob also joins - d: new Map([ - [carolId, carol.userId], - [aliceId, "Alice (@alice:example.org)"], - [aliceDoppelgangerId, "Alice (@alice2:example.org)"], - [bobId, bob.rawDisplayName], - ]), - // Alice leaves, and the displayname should reset. - e: new Map([ - [carolId, carol.userId], - [aliceDoppelgangerId, "Alice"], - [bobId, bob.rawDisplayName], - ]), - }); - }, - ); - }); -}); - -test("should disambiguate users with invisible characters", () => { - withTestScheduler(({ behavior, expectObservable }) => { - const scenarioInputMarbles = "ab"; - const expectedLayoutMarbles = "ab"; - - withCallViewModel( - { - rtcMembers$: behavior(scenarioInputMarbles, { - a: [localRtcMember], - b: [localRtcMember, bobRtcMember, bobZeroWidthSpaceRtcMember], - }), - }, - (vm) => { - expectObservable(vm.memberDisplaynames$).toBe(expectedLayoutMarbles, { - // Carol has no displayname - So userId is used. - a: new Map([[carolId, carol.userId]]), - // Both Bobs join, and should handle zero width hacks. - b: new Map([ - [carolId, carol.userId], - [bobId, `Bob (${bob.userId})`], - [ - bobZeroWidthSpaceId, - `${bobZeroWidthSpace.rawDisplayName} (${bobZeroWidthSpace.userId})`, - ], - ]), - }); - }, - ); - }); -}); - -test("should strip RTL characters from displayname", () => { - withTestScheduler(({ behavior, expectObservable }) => { - const scenarioInputMarbles = "ab"; - const expectedLayoutMarbles = "ab"; - - withCallViewModel( - { - rtcMembers$: behavior(scenarioInputMarbles, { - a: [localRtcMember], - b: [localRtcMember, daveRtcMember, daveRTLRtcMember], - }), - }, - (vm) => { - expectObservable(vm.memberDisplaynames$).toBe(expectedLayoutMarbles, { - // Carol has no displayname - So userId is used. - a: new Map([[carolId, carol.userId]]), - // Both Dave's join. Since after stripping - b: new Map([ - [carolId, carol.userId], - // Not disambiguated - [daveId, "Dave"], - // This one is, since it's using RTL. - [daveRTLId, `evaD (${daveRTL.userId})`], - ]), - }); - }, - ); - }); -}); - -it("should rank raised hands above video feeds and below speakers and presenters", () => { - withTestScheduler(({ schedule, expectObservable }) => { - // There should always be one tile for each MatrixRTCSession - const expectedLayoutMarbles = "ab"; - - withCallViewModel( - { - remoteParticipants$: constant([aliceParticipant, bobParticipant]), - rtcMembers$: constant([localRtcMember, aliceRtcMember, bobRtcMember]), - }, - (vm, _rtcSession, { raisedHands$ }) => { - schedule("ab", { - a: () => { - // We imagine that only two tiles (the first two) will be visible on screen at a time - vm.layout$.subscribe((layout) => { - if (layout.type === "grid") { - layout.setVisibleTiles(2); - } - }); - }, - b: () => { - raisedHands$.next({ - [`${bobRtcMember.userId}:${bobRtcMember.deviceId}`]: { - time: new Date(), - reactionEventId: "", - membershipEventId: "", - }, - }); - }, - }); - expectObservable(summarizeLayout$(vm.layout$)).toBe( - expectedLayoutMarbles, - { - a: { - type: "grid", - spotlight: undefined, - grid: [ - `${localId}:0`, - "@alice:example.org:AAAA:0", - "@bob:example.org:BBBB:0", - ], - }, - b: { - type: "grid", - spotlight: undefined, - grid: [ - `${localId}:0`, - // Bob shifts up! - "@bob:example.org:BBBB:0", - "@alice:example.org:AAAA:0", - ], - }, - }, - ); - }, - ); - }); -}); - -function nooneEverThere$( - behavior: (marbles: string, values: Record) => Behavior, -): Behavior { - return behavior("a-b-c-d", { - a: [], // Start empty - b: [], // Alice joins - c: [], // Alice still there - d: [], // Alice leaves - }); -} - -function participantJoinLeave$( - behavior: ( - marbles: string, - values: Record, - ) => Behavior, -): Behavior { - return behavior("a-b-c-d", { - a: [], // Start empty - b: [aliceParticipant], // Alice joins - c: [aliceParticipant], // Alice still there - d: [], // Alice leaves - }); -} - -function rtcMemberJoinLeave$( - behavior: ( - marbles: string, - values: Record, - ) => Behavior, -): Behavior { - return behavior("a-b-c-d", { - a: [localRtcMember], // Start empty - b: [localRtcMember, aliceRtcMember], // Alice joins - c: [localRtcMember, aliceRtcMember], // Alice still there - d: [localRtcMember], // Alice leaves - }); -} - -test("autoLeave$ emits only when autoLeaveWhenOthersLeft option is enabled", () => { - withTestScheduler(({ behavior, expectObservable }) => { - withCallViewModel( - { - remoteParticipants$: participantJoinLeave$(behavior), - rtcMembers$: rtcMemberJoinLeave$(behavior), - }, - (vm) => { - expectObservable(vm.autoLeave$).toBe("------a", { - a: "allOthersLeft", - }); - }, - { - autoLeaveWhenOthersLeft: true, - encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, - }, - ); - }); -}); - -test("autoLeave$ never emits autoLeaveWhenOthersLeft option is enabled but no-one is there", () => { - withTestScheduler(({ behavior, expectObservable }) => { - withCallViewModel( - { - remoteParticipants$: nooneEverThere$(behavior), - rtcMembers$: nooneEverThere$(behavior), - }, - (vm) => { - expectObservable(vm.autoLeave$).toBe("-"); - }, - { - autoLeaveWhenOthersLeft: true, - encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, - }, - ); - }); -}); - -test("autoLeave$ doesn't emit when autoLeaveWhenOthersLeft option is disabled and all others left", () => { - withTestScheduler(({ behavior, expectObservable }) => { - withCallViewModel( - { - remoteParticipants$: participantJoinLeave$(behavior), - rtcMembers$: rtcMemberJoinLeave$(behavior), - }, - (vm) => { - expectObservable(vm.autoLeave$).toBe("-"); - }, - { - autoLeaveWhenOthersLeft: false, - encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, - }, - ); - }); -}); - -test("autoLeave$ emits when autoLeaveWhenOthersLeft option is enabled and all others left", () => { - withTestScheduler(({ behavior, expectObservable }) => { - withCallViewModel( - { - remoteParticipants$: behavior("a-b-c-d", { - a: [], // Alone - b: [aliceParticipant], // Alice joins - c: [aliceParticipant], - d: [], // Local joins with a second device - }), - rtcMembers$: behavior("a-b-c-d", { - a: [localRtcMember], // Start empty - b: [localRtcMember, aliceRtcMember], // Alice joins - c: [localRtcMember, aliceRtcMember, localRtcMemberDevice2], // Alice still there - d: [localRtcMember, localRtcMemberDevice2], // The second Alice leaves - }), - }, - (vm) => { - expectObservable(vm.autoLeave$).toBe("------a", { - a: "allOthersLeft", - }); - }, - { - autoLeaveWhenOthersLeft: true, - encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, - }, - ); - }); -}); - -describe("waitForCallPickup$", () => { - test("unknown -> ringing -> timeout when notified and nobody joins", () => { - withTestScheduler(({ schedule, expectObservable }) => { - // No one ever joins (only local user) - withCallViewModel( - { remoteParticipants$: constant([]) }, - (vm, rtcSession) => { - // Fire a call notification at 10ms with lifetime 30ms - schedule(" 10ms r", { - r: () => { - rtcSession.emit( - MatrixRTCSessionEvent.DidSendCallNotification, - mockRingEvent("$notif1", 30), - mockLegacyRingEvent, - ); - }, - }); - - expectObservable(vm.callPickupState$).toBe("a 9ms b 29ms c", { - a: "unknown", - b: "ringing", - c: "timeout", - }); - }, - { - waitForCallPickup: true, - encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, - }, - ); - }); - }); - - test("regression test: does stop ringing in case livekitConnectionState$ emits after didSendCallNotification$ has already emitted", () => { - withTestScheduler(({ schedule, expectObservable, behavior }) => { - withCallViewModel( - { - livekitConnectionState$: behavior("d 9ms c", { - d: ConnectionState.Disconnected, - c: ConnectionState.Connected, - }), - }, - (vm, rtcSession) => { - // Fire a call notification IMMEDIATELY (its important for this test, that this happens before the livekitConnectionState$ emits) - schedule("n", { - n: () => { - rtcSession.emit( - MatrixRTCSessionEvent.DidSendCallNotification, - mockRingEvent("$notif1", 30), - mockLegacyRingEvent, - ); - }, - }); - - expectObservable(vm.callPickupState$).toBe("a 9ms b 19ms c", { - a: "unknown", - b: "ringing", - c: "timeout", - }); - }, - { - waitForCallPickup: true, - encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, - }, - ); - }); - }); - - test("ringing -> success if someone joins before timeout", () => { - withTestScheduler(({ behavior, schedule, expectObservable }) => { - // Someone joins at 20ms (both LiveKit participant and MatrixRTC member) - withCallViewModel( - { - remoteParticipants$: behavior("a 19ms b", { - a: [], - b: [aliceParticipant], - }), - rtcMembers$: behavior("a 19ms b", { - a: [localRtcMember], - b: [localRtcMember, aliceRtcMember], - }), - }, - (vm, rtcSession) => { - // Notify at 5ms so we enter ringing, then success at 20ms - schedule(" 5ms r", { - r: () => { - rtcSession.emit( - MatrixRTCSessionEvent.DidSendCallNotification, - mockRingEvent("$notif2", 100), - mockLegacyRingEvent, - ); - }, - }); - - expectObservable(vm.callPickupState$).toBe("a 4ms b 14ms c", { - a: "unknown", - b: "ringing", - c: "success", - }); - }, - { - waitForCallPickup: true, - encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, - }, - ); - }); - }); - - test("ringing -> unknown if we get disconnected", () => { - withTestScheduler(({ behavior, schedule, expectObservable }) => { - const connectionState$ = new BehaviorSubject(ConnectionState.Connected); - // Someone joins at 20ms (both LiveKit participant and MatrixRTC member) - withCallViewModel( - { - remoteParticipants$: behavior("a 19ms b", { - a: [], - b: [aliceParticipant], - }), - rtcMembers$: behavior("a 19ms b", { - a: [localRtcMember], - b: [localRtcMember, aliceRtcMember], - }), - livekitConnectionState$: connectionState$, - }, - (vm, rtcSession) => { - // Notify at 5ms so we enter ringing, then get disconnected 5ms later - schedule(" 5ms r 5ms d", { - r: () => { - rtcSession.emit( - MatrixRTCSessionEvent.DidSendCallNotification, - mockRingEvent("$notif2", 100), - mockLegacyRingEvent, - ); - }, - d: () => { - connectionState$.next(ConnectionState.Disconnected); - }, - }); - - expectObservable(vm.callPickupState$).toBe("a 4ms b 5ms c", { - a: "unknown", - b: "ringing", - c: "unknown", - }); - }, - { - waitForCallPickup: true, - encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, - }, - ); - }); - }); - - test("success when someone joins before we notify", () => { - withTestScheduler(({ behavior, schedule, expectObservable }) => { - // Join at 10ms, notify later at 20ms (state should stay success) - withCallViewModel( - { - remoteParticipants$: behavior("a 9ms b", { - a: [], - b: [aliceParticipant], - }), - rtcMembers$: behavior("a 9ms b", { - a: [localRtcMember], - b: [localRtcMember, aliceRtcMember], - }), - }, - (vm, rtcSession) => { - schedule(" 20ms r", { - r: () => { - rtcSession.emit( - MatrixRTCSessionEvent.DidSendCallNotification, - mockRingEvent("$notif3", 50), - mockLegacyRingEvent, - ); - }, - }); - expectObservable(vm.callPickupState$).toBe("a 9ms b", { - a: "unknown", - b: "success", - }); - }, - { - waitForCallPickup: true, - encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, - }, - ); - }); - }); - - test("notify without lifetime -> immediate timeout", () => { - withTestScheduler(({ schedule, expectObservable }) => { - withCallViewModel( - {}, - (vm, rtcSession) => { - schedule(" 10ms r", { - r: () => { - rtcSession.emit( - MatrixRTCSessionEvent.DidSendCallNotification, - mockRingEvent("$notif4", undefined), - mockLegacyRingEvent, - ); - }, - }); - expectObservable(vm.callPickupState$).toBe("a 9ms b", { - a: "unknown", - b: "timeout", - }); - }, - { - waitForCallPickup: true, - encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, - }, - ); - }); - }); - - test("stays null when waitForCallPickup=false", () => { - withTestScheduler(({ behavior, schedule, expectObservable }) => { - withCallViewModel( - { - remoteParticipants$: behavior("a--b", { - a: [], - b: [aliceParticipant], - }), - rtcMembers$: behavior("a--b", { - a: [localRtcMember], - b: [localRtcMember, aliceRtcMember], - }), - }, - (vm, rtcSession) => { - schedule(" 5ms r", { - r: () => { - rtcSession.emit( - MatrixRTCSessionEvent.DidSendCallNotification, - mockRingEvent("$notif5", 30), - mockLegacyRingEvent, - ); - }, - }); - expectObservable(vm.callPickupState$).toBe("(n)", { - n: null, - }); - }, - { - waitForCallPickup: false, - encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, - }, - ); - }); - }); - - test("decline before timeout window ends -> decline", () => { - withTestScheduler(({ schedule, expectObservable }) => { - withCallViewModel( - {}, - (vm, rtcSession) => { - // Notify at 10ms with 50ms lifetime, decline at 40ms with matching id - schedule(" 10ms r 29ms d", { - r: () => { - rtcSession.emit( - MatrixRTCSessionEvent.DidSendCallNotification, - mockRingEvent("$decl1", 50), - mockLegacyRingEvent, - ); - }, - d: () => { - // Emit decline timeline event with id matching the notification - rtcSession.room.emit( - MatrixRoomEvent.Timeline, - new MatrixEvent({ - type: EventType.RTCDecline, - content: { - "m.relates_to": { - rel_type: "m.reference", - event_id: "$decl1", - }, - }, - }), - rtcSession.room, - undefined, - false, - {} as IRoomTimelineData, - ); - }, - }); - expectObservable(vm.callPickupState$).toBe("a 9ms b 29ms e", { - a: "unknown", - b: "ringing", - e: "decline", - }); - }, - { - waitForCallPickup: true, - encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, - }, - ); - }); - }); - - test("decline after timeout window ends -> stays timeout", () => { - withTestScheduler(({ schedule, expectObservable }) => { - withCallViewModel( - {}, - (vm, rtcSession) => { - // Notify at 10ms with 20ms lifetime (timeout at 30ms), decline at 40ms - schedule(" 10ms r 20ms t 10ms d", { - r: () => { - rtcSession.emit( - MatrixRTCSessionEvent.DidSendCallNotification, - mockRingEvent("$decl2", 20), - mockLegacyRingEvent, - ); - }, - t: () => {}, - d: () => { - rtcSession.room.emit( - MatrixRoomEvent.Timeline, - new MatrixEvent({ event_id: "$decl2", type: "m.rtc.decline" }), - rtcSession.room, - undefined, - false, - {} as IRoomTimelineData, - ); - }, - }); - expectObservable(vm.callPickupState$).toBe("a 9ms b 19ms c", { - a: "unknown", - b: "ringing", - c: "timeout", - }); - }, - { - waitForCallPickup: true, - encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, - }, - ); - }); - }); - - function testStaysRinging(declineEvent: Partial): void { - withTestScheduler(({ schedule, expectObservable }) => { - withCallViewModel( - {}, - (vm, rtcSession) => { - // Notify at 10ms with id A, decline arrives at 20ms with id B - schedule(" 10ms r 10ms d", { - r: () => { - rtcSession.emit( - MatrixRTCSessionEvent.DidSendCallNotification, - mockRingEvent("$right", 50), - mockLegacyRingEvent, - ); - }, - d: () => { - rtcSession.room.emit( - MatrixRoomEvent.Timeline, - new MatrixEvent(declineEvent), - rtcSession.room, - undefined, - false, - {} as IRoomTimelineData, - ); - }, - }); - // We assert up to 21ms to see the ringing at 10ms and no change at 20ms - expectObservable(vm.callPickupState$, "21ms !").toBe("a 9ms b", { - a: "unknown", - b: "ringing", - }); - }, - { - waitForCallPickup: true, - encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, - }, - ); - }); - } - - test("decline with wrong id is ignored (stays ringing)", () => { - testStaysRinging({ - event_id: "$wrong", - type: "m.rtc.decline", - sender: local.userId, - }); - }); - - test("decline with sender being the local user is ignored (stays ringing)", () => { - testStaysRinging({ - event_id: "$right", - type: "m.rtc.decline", - sender: alice.userId, - }); - }); -}); - -test("audio output changes when toggling earpiece mode", () => { - withTestScheduler(({ schedule, expectObservable }) => { - getUrlParams.mockReturnValue({ controlledAudioDevices: true }); - vi.mocked(ComponentsCore.createMediaDeviceObserver).mockReturnValue(of([])); - - const devices = new MediaDevices(testScope()); - - window.controls.setAvailableAudioDevices([ - { id: "speaker", name: "Speaker", isSpeaker: true }, - { id: "earpiece", name: "Handset", isEarpiece: true }, - { id: "headphones", name: "Headphones" }, - ]); - window.controls.setAudioDevice("headphones"); - - const toggleInputMarbles = " -aaa"; - const expectedEarpieceModeMarbles = "n-yn"; - const expectedTargetStateMarbles = " sese"; - - withCallViewModel({ mediaDevices: devices }, (vm) => { - schedule(toggleInputMarbles, { - a: () => getValue(vm.audioOutputSwitcher$)?.switch(), - }); - expectObservable(vm.earpieceMode$).toBe( - expectedEarpieceModeMarbles, - yesNo, - ); - expectObservable( - vm.audioOutputSwitcher$.pipe(map((switcher) => switcher?.targetOutput)), - ).toBe(expectedTargetStateMarbles, { s: "speaker", e: "earpiece" }); - }); - }); -}); - -test("media tracks are paused while reconnecting to MatrixRTC", () => { - withTestScheduler(({ schedule, expectObservable }) => { - const trackRunning$ = new BehaviorSubject(true); - const originalPublications = localParticipant.trackPublications; - localParticipant.trackPublications = new Map([ - [ - "video", - { - track: new (class { - public get isUpstreamPaused(): boolean { - return !trackRunning$.value; - } - public async pauseUpstream(): Promise { - trackRunning$.next(false); - return Promise.resolve(); - } - public async resumeUpstream(): Promise { - trackRunning$.next(true); - return Promise.resolve(); - } - })(), - } as unknown as LocalTrackPublication, - ], - ]); - onTestFinished(() => { - localParticipant.trackPublications = originalPublications; - }); - - // There are three indicators that the client might be disconnected from - // MatrixRTC: whether the sync loop is connected, whether the membership is - // present in local room state, and whether the membership manager thinks - // we've hit the timeout for the delayed leave event. Let's test all - // combinations of these conditions. - const syncingMarbles = " nyny----n--y"; - const membershipStatusMarbles = " y---ny-n-yn-y"; - const probablyLeftMarbles = " n-----y-ny---n"; - const expectedReconnectingMarbles = "n-ynyny------n"; - const expectedTrackRunningMarbles = "nynynyn------y"; - - withCallViewModel( - { initialSyncState: SyncState.Reconnecting }, - (vm, rtcSession, _subjects, setSyncState) => { - schedule(syncingMarbles, { - y: () => setSyncState(SyncState.Syncing), - n: () => setSyncState(SyncState.Reconnecting), - }); - schedule(membershipStatusMarbles, { - y: () => { - rtcSession.membershipStatus = Status.Connected; - }, - n: () => { - rtcSession.membershipStatus = Status.Reconnecting; - }, - }); - schedule(probablyLeftMarbles, { - y: () => { - rtcSession.probablyLeft = true; - }, - n: () => { - rtcSession.probablyLeft = false; - }, - }); - expectObservable(vm.reconnecting$).toBe( - expectedReconnectingMarbles, - yesNo, - ); - expectObservable(trackRunning$).toBe( - expectedTrackRunningMarbles, - yesNo, - ); - }, - ); - }); -}); diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts deleted file mode 100644 index d7735b26..00000000 --- a/src/state/CallViewModel.ts +++ /dev/null @@ -1,1959 +0,0 @@ -/* -Copyright 2023, 2024, 2025 New Vector Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { - type BaseKeyProvider, - ConnectionState, - type E2EEOptions, - ExternalE2EEKeyProvider, - type LocalParticipant, - RemoteParticipant, - type Room as LivekitRoom, - type RoomOptions, -} from "livekit-client"; -import E2EEWorker from "livekit-client/e2ee-worker?worker"; -import { - ClientEvent, - type EventTimelineSetHandlerMap, - EventType, - type Room as MatrixRoom, - RoomEvent, - type RoomMember, - RoomStateEvent, - SyncState, -} from "matrix-js-sdk"; -import { deepCompare } from "matrix-js-sdk/lib/utils"; -import { - BehaviorSubject, - combineLatest, - concat, - distinctUntilChanged, - EMPTY, - endWith, - filter, - from, - fromEvent, - ignoreElements, - map, - merge, - NEVER, - type Observable, - of, - pairwise, - race, - repeat, - scan, - skip, - skipWhile, - startWith, - Subject, - switchAll, - switchMap, - switchScan, - take, - takeUntil, - takeWhile, - tap, - throttleTime, - timer, -} from "rxjs"; -import { logger } from "matrix-js-sdk/lib/logger"; -import { - type CallMembership, - isLivekitTransport, - type LivekitTransport, - type MatrixRTCSession, - MatrixRTCSessionEvent, - type MatrixRTCSessionEventHandlerMap, - MembershipManagerEvent, - Status, -} from "matrix-js-sdk/lib/matrixrtc"; -import { type IWidgetApiRequest } from "matrix-widget-api"; - -import { - LocalUserMediaViewModel, - type MediaViewModel, - type RemoteUserMediaViewModel, - ScreenShareViewModel, - type UserMediaViewModel, -} from "./MediaViewModel"; -import { - accumulate, - and$, - generateKeyed$, - pauseWhen, -} from "../utils/observable"; -import { - duplicateTiles, - multiSfu, - playReactionsSound, - preferStickyEvents, - showReactions, -} from "../settings/settings"; -import { isFirefox } from "../Platform"; -import { setPipEnabled$ } from "../controls"; -import { TileStore } from "./TileStore"; -import { gridLikeLayout } from "./GridLikeLayout"; -import { spotlightExpandedLayout } from "./SpotlightExpandedLayout"; -import { oneOnOneLayout } from "./OneOnOneLayout"; -import { pipLayout } from "./PipLayout"; -import { type EncryptionSystem } from "../e2ee/sharedKeyManagement"; -import { - type RaisedHandInfo, - type ReactionInfo, - type ReactionOption, -} from "../reactions"; -import { shallowEquals } from "../utils/array"; -import { calculateDisplayName, shouldDisambiguate } from "../utils/displayname"; -import { type MediaDevices } from "./MediaDevices"; -import { type Behavior, constant } from "./Behavior"; -import { - enterRTCSession, - getLivekitAlias, - makeTransport, -} from "../rtcSessionHelpers"; -import { E2eeType } from "../e2ee/e2eeType"; -import { MatrixKeyProvider } from "../e2ee/matrixKeyProvider"; -import { type Connection, RemoteConnection } from "./Connection"; -import { type MuteStates } from "./MuteStates"; -import { getUrlParams } from "../UrlParams"; -import { type ProcessorState } from "../livekit/TrackProcessorContext"; -import { ElementWidgetActions, widget } from "../widget"; -import { PublishConnection } from "./PublishConnection.ts"; -import { type Async, async$, mapAsync, ready } from "./Async"; -import { sharingScreen$, UserMedia } from "./UserMedia.ts"; -import { ScreenShare } from "./ScreenShare.ts"; -import { - type GridLayoutMedia, - type Layout, - type LayoutMedia, - type OneOnOneLayoutMedia, - type SpotlightExpandedLayoutMedia, - type SpotlightLandscapeLayoutMedia, - type SpotlightPortraitLayoutMedia, -} from "./layout-types.ts"; -import { ElementCallError, UnknownCallError } from "../utils/errors.ts"; -import { ObservableScope } from "./ObservableScope.ts"; - -export interface CallViewModelOptions { - encryptionSystem: EncryptionSystem; - autoLeaveWhenOthersLeft?: boolean; - /** - * If the call is started in a way where we want it to behave like a telephone usecase - * If we sent a notification event, we want the ui to show a ringing state - */ - waitForCallPickup?: boolean; - /** Optional factory to create LiveKit rooms, mainly for testing purposes. */ - livekitRoomFactory?: (options?: RoomOptions) => LivekitRoom; - /** Optional behavior overriding the local connection state, mainly for testing purposes. */ - connectionState$?: Behavior; -} - -// Do not play any sounds if the participant count has exceeded this -// number. -export const MAX_PARTICIPANT_COUNT_FOR_SOUND = 8; -export const THROTTLE_SOUND_EFFECT_MS = 500; - -// This is the number of participants that we think constitutes a "small" call -// on mobile. No spotlight tile should be shown below this threshold. -const smallMobileCallThreshold = 3; - -// How long the footer should be shown for when hovering over or interacting -// with the interface -const showFooterMs = 4000; - -export type GridMode = "grid" | "spotlight"; - -export type WindowMode = "normal" | "narrow" | "flat" | "pip"; - -interface LayoutScanState { - layout: Layout | null; - tiles: TileStore; -} - -type MediaItem = UserMedia | ScreenShare; - -/** - * A view model providing all the application logic needed to show the in-call - * UI (may eventually be expanded to cover the lobby and feedback screens in the - * future). - */ -// Throughout this class and related code we must distinguish between MatrixRTC -// state and LiveKit state. We use the common terminology of room "members", RTC -// "memberships", and LiveKit "participants". -export class CallViewModel { - private readonly urlParams = getUrlParams(); - - private readonly livekitAlias = getLivekitAlias(this.matrixRTCSession); - - private readonly livekitE2EEKeyProvider = getE2eeKeyProvider( - this.options.encryptionSystem, - this.matrixRTCSession, - ); - private readonly e2eeLivekitOptions = (): E2EEOptions | undefined => - this.livekitE2EEKeyProvider - ? { - keyProvider: this.livekitE2EEKeyProvider, - worker: new E2EEWorker(), - } - : undefined; - - private readonly _configError$ = new BehaviorSubject( - null, - ); - - /** - * If there is a configuration error with the call (e.g. misconfigured E2EE). - * This is a fatal error that prevents the call from being created/joined. - * Should render a blocking error screen. - */ - public get configError$(): Behavior { - return this._configError$; - } - - private readonly join$ = new Subject(); - - public join(): void { - this.join$.next(); - } - - // This is functionally the same Observable as leave$, except here it's - // hoisted to the top of the class. This enables the cyclic dependency between - // leave$ -> autoLeave$ -> callPickupState$ -> livekitConnectionState$ -> - // localConnection$ -> transports$ -> joined$ -> leave$. - private readonly leaveHoisted$ = new Subject< - "user" | "timeout" | "decline" | "allOthersLeft" - >(); - - /** - * Whether we are joined to the call. This reflects our local state rather - * than whether all connections are truly up and running. - */ - private readonly joined$ = this.scope.behavior( - this.join$.pipe( - map(() => true), - // Using takeUntil with the repeat operator is perfectly valid. - // eslint-disable-next-line rxjs/no-unsafe-takeuntil - takeUntil(this.leaveHoisted$), - endWith(false), - repeat(), - startWith(false), - ), - ); - - /** - * The MatrixRTC session participants. - */ - // Note that MatrixRTCSession already filters the call memberships by users - // that are joined to the room; we don't need to perform extra filtering here. - private readonly memberships$ = this.scope.behavior( - fromEvent( - this.matrixRTCSession, - MatrixRTCSessionEvent.MembershipsChanged, - ).pipe( - startWith(null), - map(() => this.matrixRTCSession.memberships), - ), - ); - - /** - * The transport that we would personally prefer to publish on (if not for the - * transport preferences of others, perhaps). - */ - private readonly preferredTransport$ = this.scope.behavior( - async$(makeTransport(this.matrixRTCSession)), - ); - - /** - * Lists the transports used by ourselves, plus all other MatrixRTC session - * members. For completeness this also lists the preferred transport and - * whether we are in multi-SFU mode or sticky events mode (because - * advertisedTransport$ wants to read them at the same time, and bundling data - * together when it might change together is what you have to do in RxJS to - * avoid reading inconsistent state or observing too many changes.) - */ - // TODO-MULTI-SFU find a better name for this. with the addition of sticky events it's no longer just about transports. - private readonly transports$: Behavior<{ - local: Async; - remote: { membership: CallMembership; transport: LivekitTransport }[]; - preferred: Async; - multiSfu: boolean; - preferStickyEvents: boolean; - } | null> = this.scope.behavior( - this.joined$.pipe( - switchMap((joined) => - joined - ? combineLatest( - [ - this.preferredTransport$, - this.memberships$, - multiSfu.value$, - preferStickyEvents.value$, - ], - (preferred, memberships, preferMultiSfu, preferStickyEvents) => { - // Multi-SFU must be implicitly enabled when using sticky events - const multiSfu = preferStickyEvents || preferMultiSfu; - - const oldestMembership = - this.matrixRTCSession.getOldestMembership(); - const remote = memberships.flatMap((m) => { - if (m.userId === this.userId && m.deviceId === this.deviceId) - return []; - const t = m.getTransport(oldestMembership ?? m); - return t && isLivekitTransport(t) - ? [{ membership: m, transport: t }] - : []; - }); - - let local = preferred; - if (!multiSfu) { - const oldest = this.matrixRTCSession.getOldestMembership(); - if (oldest !== undefined) { - const selection = oldest.getTransport(oldest); - // TODO selection can be null if no transport is configured should we report an error? - if (selection && isLivekitTransport(selection)) - local = ready(selection); - } - } - - if (local.state === "error") { - this._configError$.next( - local.value instanceof ElementCallError - ? local.value - : new UnknownCallError(local.value), - ); - } - - return { - local, - remote, - preferred, - multiSfu, - preferStickyEvents, - }; - }, - ) - : of(null), - ), - ), - ); - - /** - * Lists the transports used by each MatrixRTC session member other than - * ourselves. - */ - private readonly remoteTransports$ = this.scope.behavior( - this.transports$.pipe(map((transports) => transports?.remote ?? [])), - ); - - /** - * The transport over which we should be actively publishing our media. - * null when not joined. - */ - private readonly localTransport$: Behavior | null> = - this.scope.behavior( - this.transports$.pipe( - map((transports) => transports?.local ?? null), - distinctUntilChanged | null>(deepCompare), - ), - ); - - /** - * The transport we should advertise in our MatrixRTC membership (plus whether - * it is a multi-SFU transport and whether we should use sticky events). - */ - private readonly advertisedTransport$: Behavior<{ - multiSfu: boolean; - preferStickyEvents: boolean; - transport: LivekitTransport; - } | null> = this.scope.behavior( - this.transports$.pipe( - map((transports) => - transports?.local.state === "ready" && - transports.preferred.state === "ready" - ? { - multiSfu: transports.multiSfu, - preferStickyEvents: transports.preferStickyEvents, - // In non-multi-SFU mode we should always advertise the preferred - // SFU to minimize the number of membership updates - transport: transports.multiSfu - ? transports.local.value - : transports.preferred.value, - } - : null, - ), - distinctUntilChanged<{ - multiSfu: boolean; - preferStickyEvents: boolean; - transport: LivekitTransport; - } | null>(deepCompare), - ), - ); - - /** - * The local connection over which we will publish our media. It could - * possibly also have some remote users' media available on it. - * null when not joined. - */ - private readonly localConnection$: Behavior | null> = - this.scope.behavior( - generateKeyed$< - Async | null, - PublishConnection, - Async | null - >( - this.localTransport$, - (transport, createOrGet) => - transport && - mapAsync(transport, (transport) => - createOrGet( - // Stable key that uniquely idenifies the transport - JSON.stringify({ - url: transport.livekit_service_url, - alias: transport.livekit_alias, - }), - (scope) => - new PublishConnection( - { - transport, - client: this.matrixRoom.client, - scope, - remoteTransports$: this.remoteTransports$, - livekitRoomFactory: this.options.livekitRoomFactory, - }, - this.mediaDevices, - this.muteStates, - this.e2eeLivekitOptions(), - this.scope.behavior(this.trackProcessorState$), - ), - ), - ), - ), - ); - - public readonly livekitConnectionState$ = - // TODO: This options.connectionState$ behavior is a small hack inserted - // here to facilitate testing. This would likely be better served by - // breaking CallViewModel down into more naturally testable components. - this.options.connectionState$ ?? - this.scope.behavior( - this.localConnection$.pipe( - switchMap((c) => - c?.state === "ready" - ? // TODO mapping to ConnectionState for compatibility, but we should use the full state? - c.value.transportState$.pipe( - switchMap((s) => { - if (s.state === "ConnectedToLkRoom") - return s.connectionState$; - return of(ConnectionState.Disconnected); - }), - ) - : of(ConnectionState.Disconnected), - ), - ), - ); - - /** - * Connections for each transport in use by one or more session members that - * is *distinct* from the local transport. - */ - private readonly remoteConnections$ = this.scope.behavior( - generateKeyed$( - this.transports$, - (transports, createOrGet) => { - const connections: Connection[] = []; - - // Until the local transport becomes ready we have no idea which - // transports will actually need a dedicated remote connection - if (transports?.local.state === "ready") { - // TODO: Handle custom transport.livekit_alias values here - const localServiceUrl = transports.local.value.livekit_service_url; - const remoteServiceUrls = new Set( - transports.remote.map( - ({ transport }) => transport.livekit_service_url, - ), - ); - remoteServiceUrls.delete(localServiceUrl); - - for (const remoteServiceUrl of remoteServiceUrls) - connections.push( - createOrGet( - remoteServiceUrl, - (scope) => - new RemoteConnection( - { - transport: { - type: "livekit", - livekit_service_url: remoteServiceUrl, - livekit_alias: this.livekitAlias, - }, - client: this.matrixRoom.client, - scope, - remoteTransports$: this.remoteTransports$, - livekitRoomFactory: this.options.livekitRoomFactory, - }, - this.e2eeLivekitOptions(), - ), - ), - ); - } - - return connections; - }, - ), - ); - - /** - * A list of the connections that should be active at any given time. - */ - private readonly connections$ = this.scope.behavior( - combineLatest( - [this.localConnection$, this.remoteConnections$], - (local, remote) => [ - ...(local?.state === "ready" ? [local.value] : []), - ...remote.values(), - ], - ), - ); - - /** - * Emits with connections whenever they should be started or stopped. - */ - private readonly connectionInstructions$ = this.connections$.pipe( - pairwise(), - map(([prev, next]) => { - const start = new Set(next.values()); - for (const connection of prev) start.delete(connection); - const stop = new Set(prev.values()); - for (const connection of next) stop.delete(connection); - - return { start, stop }; - }), - ); - - public readonly allLivekitRooms$ = this.scope.behavior( - this.connections$.pipe( - map((connections) => - [...connections.values()].map((c) => ({ - room: c.livekitRoom, - url: c.transport.livekit_service_url, - isLocal: c instanceof PublishConnection, - })), - ), - ), - ); - - private readonly userId = this.matrixRoom.client.getUserId()!; - private readonly deviceId = this.matrixRoom.client.getDeviceId()!; - - /** - * Whether we are connected to the MatrixRTC session. - */ - private readonly matrixConnected$ = this.scope.behavior( - // To consider ourselves connected to MatrixRTC, we check the following: - and$( - // The client is connected to the sync loop - ( - fromEvent(this.matrixRoom.client, ClientEvent.Sync) as Observable< - [SyncState] - > - ).pipe( - startWith([this.matrixRoom.client.getSyncState()]), - map(([state]) => state === SyncState.Syncing), - ), - // Room state observed by session says we're connected - fromEvent( - this.matrixRTCSession, - MembershipManagerEvent.StatusChanged, - ).pipe( - startWith(null), - map(() => this.matrixRTCSession.membershipStatus === Status.Connected), - ), - // Also watch out for warnings that we've likely hit a timeout and our - // delayed leave event is being sent (this condition is here because it - // provides an earlier warning than the sync loop timeout, and we wouldn't - // see the actual leave event until we reconnect to the sync loop) - fromEvent( - this.matrixRTCSession, - MembershipManagerEvent.ProbablyLeft, - ).pipe( - startWith(null), - map(() => this.matrixRTCSession.probablyLeft !== true), - ), - ), - ); - - /** - * Whether we are "fully" connected to the call. Accounts for both the - * connection to the MatrixRTC session and the LiveKit publish connection. - */ - private readonly connected$ = this.scope.behavior( - and$( - this.matrixConnected$, - this.livekitConnectionState$.pipe( - map((state) => state === ConnectionState.Connected), - ), - ), - ); - - /** - * Whether we should tell the user that we're reconnecting to the call. - */ - public readonly reconnecting$ = this.scope.behavior( - this.connected$.pipe( - // We are reconnecting if we previously had some successful initial - // connection but are now disconnected - scan( - ({ connectedPreviously }, connectedNow) => ({ - connectedPreviously: connectedPreviously || connectedNow, - reconnecting: connectedPreviously && !connectedNow, - }), - { connectedPreviously: false, reconnecting: false }, - ), - map(({ reconnecting }) => reconnecting), - ), - ); - - /** - * Whether various media/event sources should pretend to be disconnected from - * all network input, even if their connection still technically works. - */ - // We do this when the app is in the 'reconnecting' state, because it might be - // that the LiveKit connection is still functional while the homeserver is - // down, for example, and we want to avoid making people worry that the app is - // in a split-brained state. - private readonly pretendToBeDisconnected$ = this.reconnecting$; - - /** - * Lists, for each LiveKit room, the LiveKit participants whose media should - * be presented. - */ - private readonly participantsByRoom$ = this.scope.behavior< - { - livekitRoom: LivekitRoom; - url: string; // Included for use as a React key - participants: { - id: string; - participant: LocalParticipant | RemoteParticipant | undefined; - member: RoomMember; - }[]; - }[] - >( - // TODO: Move this logic into Connection/PublishConnection if possible - this.localConnection$ - .pipe( - switchMap((localConnection) => { - if (localConnection?.state !== "ready") return []; - const memberError = (): never => { - throw new Error("No room member for call membership"); - }; - const localParticipant = { - id: `${this.userId}:${this.deviceId}`, - participant: localConnection.value.livekitRoom.localParticipant, - member: - this.matrixRoom.getMember(this.userId ?? "") ?? memberError(), - }; - - return this.remoteConnections$.pipe( - switchMap((remoteConnections) => - combineLatest( - [localConnection.value, ...remoteConnections].map((c) => - c.publishingParticipants$.pipe( - map((ps) => { - const participants: { - id: string; - participant: - | LocalParticipant - | RemoteParticipant - | undefined; - member: RoomMember; - }[] = ps.map(({ participant, membership }) => ({ - id: `${membership.userId}:${membership.deviceId}`, - participant, - member: - getRoomMemberFromRtcMember( - membership, - this.matrixRoom, - )?.member ?? memberError(), - })); - if (c === localConnection.value) - participants.push(localParticipant); - - return { - livekitRoom: c.livekitRoom, - url: c.transport.livekit_service_url, - participants, - }; - }), - ), - ), - ), - ), - ); - }), - ) - .pipe(startWith([]), pauseWhen(this.pretendToBeDisconnected$)), - ); - - /** - * Lists, for each LiveKit room, the LiveKit participants whose audio should - * be rendered. - */ - // (This is effectively just participantsByRoom$ with a stricter type) - public readonly audioParticipants$ = this.scope.behavior( - this.participantsByRoom$.pipe( - map((data) => - data.map(({ livekitRoom, url, participants }) => ({ - livekitRoom, - url, - participants: participants.flatMap(({ participant }) => - participant instanceof RemoteParticipant ? [participant] : [], - ), - })), - ), - ), - ); - - /** - * Displaynames for each member of the call. This will disambiguate - * any displaynames that clashes with another member. Only members - * joined to the call are considered here. - */ - // It turns out that doing the disambiguation above is rather expensive on Safari (10x slower - // than on Chrome/Firefox). This means it is important that we multicast the result so that we - // don't do this work more times than we need to. This is achieved by converting to a behavior: - public readonly memberDisplaynames$ = this.scope.behavior( - combineLatest( - [ - // Handle call membership changes - this.memberships$, - // Additionally handle display name changes (implicitly reacting to them) - fromEvent(this.matrixRoom, RoomStateEvent.Members).pipe( - startWith(null), - ), - // TODO: do we need: pauseWhen(this.pretendToBeDisconnected$), - ], - (memberships, _displaynames) => { - const displaynameMap = new Map([ - [ - `${this.userId}:${this.deviceId}`, - this.matrixRoom.getMember(this.userId)?.rawDisplayName ?? - this.userId, - ], - ]); - const room = this.matrixRoom; - - // We only consider RTC members for disambiguation as they are the only visible members. - for (const rtcMember of memberships) { - const matrixIdentifier = `${rtcMember.userId}:${rtcMember.deviceId}`; - const { member } = getRoomMemberFromRtcMember(rtcMember, room); - if (!member) { - logger.error( - "Could not find member for media id:", - matrixIdentifier, - ); - continue; - } - const disambiguate = shouldDisambiguate(member, memberships, room); - displaynameMap.set( - matrixIdentifier, - calculateDisplayName(member, disambiguate), - ); - } - return displaynameMap; - }, - ), - ); - - public readonly handsRaised$ = this.scope.behavior( - this.handsRaisedSubject$.pipe(pauseWhen(this.pretendToBeDisconnected$)), - ); - - public readonly reactions$ = this.scope.behavior( - this.reactionsSubject$.pipe( - map((v) => - Object.fromEntries( - Object.entries(v).map(([a, { reactionOption }]) => [ - a, - reactionOption, - ]), - ), - ), - pauseWhen(this.pretendToBeDisconnected$), - ), - ); - - /** - * List of MediaItems that we want to have tiles for. - */ - private readonly mediaItems$ = this.scope.behavior( - generateKeyed$< - [typeof this.participantsByRoom$.value, number], - MediaItem, - MediaItem[] - >( - // Generate a collection of MediaItems from the list of expected (whether - // present or missing) LiveKit participants. - combineLatest([this.participantsByRoom$, duplicateTiles.value$]), - ([participantsByRoom, duplicateTiles], createOrGet) => { - const items: MediaItem[] = []; - - for (const { livekitRoom, participants, url } of participantsByRoom) { - for (const { id, participant, member } of participants) { - for (let i = 0; i < 1 + duplicateTiles; i++) { - const mediaId = `${id}:${i}`; - const item = createOrGet( - mediaId, - (scope) => - // We create UserMedia with or without a participant. - // This will be the initial value of a BehaviourSubject. - // Once a participant appears we will update the BehaviourSubject. (see below) - new UserMedia( - scope, - mediaId, - member, - participant, - this.options.encryptionSystem, - livekitRoom, - url, - this.mediaDevices, - this.pretendToBeDisconnected$, - this.memberDisplaynames$.pipe( - map((m) => m.get(id) ?? "[👻]"), - ), - this.handsRaised$.pipe(map((v) => v[id]?.time ?? null)), - this.reactions$.pipe(map((v) => v[id] ?? undefined)), - ), - ); - items.push(item); - (item as UserMedia).updateParticipant(participant); - - if (participant?.isScreenShareEnabled) { - const screenShareId = `${mediaId}:screen-share`; - items.push( - createOrGet( - screenShareId, - (scope) => - new ScreenShare( - scope, - screenShareId, - member, - participant, - this.options.encryptionSystem, - livekitRoom, - url, - this.pretendToBeDisconnected$, - this.memberDisplaynames$.pipe( - map((m) => m.get(id) ?? "[👻]"), - ), - ), - ), - ); - } - } - } - } - - return items; - }, - ), - ); - - /** - * List of MediaItems that we want to display, that are of type UserMedia - */ - private readonly userMedia$ = this.scope.behavior( - this.mediaItems$.pipe( - map((mediaItems) => - mediaItems.filter((m): m is UserMedia => m instanceof UserMedia), - ), - ), - ); - - public readonly joinSoundEffect$ = this.userMedia$.pipe( - pairwise(), - filter( - ([prev, current]) => - current.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && - current.length > prev.length, - ), - map(() => {}), - throttleTime(THROTTLE_SOUND_EFFECT_MS), - ); - - /** - * The number of participants currently in the call. - * - * - Each participant has a corresponding MatrixRTC membership state event - * - There can be multiple participants for one Matrix user if they join from - * multiple devices. - */ - public readonly participantCount$ = this.scope.behavior( - this.memberships$.pipe(map((ms) => ms.length)), - ); - - private readonly allOthersLeft$ = this.memberships$.pipe( - pairwise(), - filter( - ([prev, current]) => - current.every((m) => m.userId === this.userId) && - prev.some((m) => m.userId !== this.userId), - ), - map(() => {}), - ); - - private readonly didSendCallNotification$ = fromEvent( - this.matrixRTCSession, - MatrixRTCSessionEvent.DidSendCallNotification, - ) as Observable< - Parameters< - MatrixRTCSessionEventHandlerMap[MatrixRTCSessionEvent.DidSendCallNotification] - > - >; - - /** - * Whenever the RTC session tells us that it intends to ring the remote - * participant's devices, this emits an Observable tracking the current state of - * that ringing process. - */ - // This is a behavior since we need to store the latest state for when we subscribe to this after `didSendCallNotification$` - // has already emitted but we still need the latest observable with a timeout timer that only gets created on after receiving `notificationEvent`. - // A behavior will emit the latest observable with the running timer to new subscribers. - // see also: callPickupState$ and in particular the line: `return this.ring$.pipe(mergeAll());` here we otherwise might get an EMPTY observable if - // `ring$` would not be a behavior. - private readonly ring$: Behavior<"ringing" | "timeout" | "decline" | null> = - this.scope.behavior( - this.didSendCallNotification$.pipe( - filter( - ([notificationEvent]) => - notificationEvent.notification_type === "ring", - ), - switchMap(([notificationEvent]) => { - const lifetimeMs = notificationEvent?.lifetime ?? 0; - return concat( - lifetimeMs === 0 - ? // If no lifetime, skip the ring state - of(null) - : // Ring until lifetime ms have passed - timer(lifetimeMs).pipe( - ignoreElements(), - startWith("ringing" as const), - ), - // The notification lifetime has timed out, meaning ringing has likely - // stopped on all receiving clients. - of("timeout" as const), - // This makes sure we will not drop into the `endWith("decline" as const)` state - NEVER, - ).pipe( - takeUntil( - ( - fromEvent(this.matrixRoom, RoomEvent.Timeline) as Observable< - Parameters - > - ).pipe( - filter( - ([event]) => - event.getType() === EventType.RTCDecline && - event.getRelation()?.rel_type === "m.reference" && - event.getRelation()?.event_id === - notificationEvent.event_id && - event.getSender() !== this.userId, - ), - ), - ), - endWith("decline" as const), - ); - }), - ), - null, - ); - - /** - * Whether some Matrix user other than ourself is joined to the call. - */ - private readonly someoneElseJoined$ = this.memberships$.pipe( - map((ms) => ms.some((m) => m.userId !== this.userId)), - ) as Behavior; - - /** - * The current call pickup state of the call. - * - "unknown": The client has not yet sent the notification event. We don't know if it will because it first needs to send its own membership. - * Then we can conclude if we were the first one to join or not. - * This may also be set if we are disconnected. - * - "ringing": The call is ringing on other devices in this room (This client should give audiovisual feedback that this is happening). - * - "timeout": No-one picked up in the defined time this call should be ringing on others devices. - * The call failed. If desired this can be used as a trigger to exit the call. - * - "success": Someone else joined. The call is in a normal state. No audiovisual feedback. - * - null: EC is configured to never show any waiting for answer state. - */ - public readonly callPickupState$: Behavior< - "unknown" | "ringing" | "timeout" | "decline" | "success" | null - > = this.options.waitForCallPickup - ? this.scope.behavior< - "unknown" | "ringing" | "timeout" | "decline" | "success" - >( - combineLatest( - [this.livekitConnectionState$, this.someoneElseJoined$, this.ring$], - (livekitConnectionState, someoneElseJoined, ring) => { - if (livekitConnectionState === ConnectionState.Disconnected) { - // Do not ring until we're connected. - return "unknown" as const; - } else if (someoneElseJoined) { - return "success" as const; - } - // Show the ringing state of the most recent ringing attempt. - // as long as we have not yet sent an RTC notification event, ring will be null -> callPickupState$ = unknown. - return ring ?? ("unknown" as const); - }, - ), - ) - : constant(null); - - public readonly leaveSoundEffect$ = combineLatest([ - this.callPickupState$, - this.userMedia$, - ]).pipe( - // Until the call is successful, do not play a leave sound. - // If callPickupState$ is null, then we always play the sound as it will not conflict with a decline sound. - skipWhile(([c]) => c !== null && c !== "success"), - map(([, userMedia]) => userMedia), - pairwise(), - filter( - ([prev, current]) => - current.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && - current.length < prev.length, - ), - map(() => {}), - throttleTime(THROTTLE_SOUND_EFFECT_MS), - ); - - // Public for testing - public readonly autoLeave$ = merge( - this.options.autoLeaveWhenOthersLeft - ? this.allOthersLeft$.pipe(map(() => "allOthersLeft" as const)) - : NEVER, - this.callPickupState$.pipe( - filter((state) => state === "timeout" || state === "decline"), - ), - ); - - private readonly userHangup$ = new Subject(); - public hangup(): void { - this.userHangup$.next(); - } - - private readonly widgetHangup$ = - widget === null - ? NEVER - : ( - fromEvent( - widget.lazyActions, - ElementWidgetActions.HangupCall, - ) as Observable> - ).pipe( - tap((ev) => { - widget!.api.transport.reply(ev.detail, {}); - }), - ); - - public readonly leave$: Observable< - "user" | "timeout" | "decline" | "allOthersLeft" - > = merge( - this.autoLeave$, - merge(this.userHangup$, this.widgetHangup$).pipe( - map(() => "user" as const), - ), - ).pipe( - this.scope.share, - tap((reason) => this.leaveHoisted$.next(reason)), - ); - - /** - * List of MediaItems that we want to display, that are of type ScreenShare - */ - private readonly screenShares$ = this.scope.behavior( - this.mediaItems$.pipe( - map((mediaItems) => - mediaItems.filter((m): m is ScreenShare => m instanceof ScreenShare), - ), - ), - ); - - private readonly spotlightSpeaker$ = - this.scope.behavior( - this.userMedia$.pipe( - switchMap((mediaItems) => - mediaItems.length === 0 - ? of([]) - : combineLatest( - mediaItems.map((m) => - m.vm.speaking$.pipe(map((s) => [m, s] as const)), - ), - ), - ), - scan<(readonly [UserMedia, boolean])[], UserMedia | undefined, null>( - (prev, mediaItems) => { - // Only remote users that are still in the call should be sticky - const [stickyMedia, stickySpeaking] = - (!prev?.vm.local && mediaItems.find(([m]) => m === prev)) || []; - // Decide who to spotlight: - // If the previous speaker is still speaking, stick with them rather - // than switching eagerly to someone else - return stickySpeaking - ? stickyMedia! - : // Otherwise, select any remote user who is speaking - (mediaItems.find(([m, s]) => !m.vm.local && s)?.[0] ?? - // Otherwise, stick with the person who was last speaking - stickyMedia ?? - // Otherwise, spotlight an arbitrary remote user - mediaItems.find(([m]) => !m.vm.local)?.[0] ?? - // Otherwise, spotlight the local user - mediaItems.find(([m]) => m.vm.local)?.[0]); - }, - null, - ), - map((speaker) => speaker?.vm ?? null), - ), - ); - - private readonly grid$ = this.scope.behavior( - this.userMedia$.pipe( - switchMap((mediaItems) => { - const bins = mediaItems.map((m) => - m.bin$.pipe(map((bin) => [m, bin] as const)), - ); - // Sort the media by bin order and generate a tile for each one - return bins.length === 0 - ? of([]) - : combineLatest(bins, (...bins) => - bins.sort(([, bin1], [, bin2]) => bin1 - bin2).map(([m]) => m.vm), - ); - }), - distinctUntilChanged(shallowEquals), - ), - ); - - private readonly spotlight$ = this.scope.behavior( - this.screenShares$.pipe( - switchMap((screenShares) => { - if (screenShares.length > 0) { - return of(screenShares.map((m) => m.vm)); - } - - return this.spotlightSpeaker$.pipe( - map((speaker) => (speaker ? [speaker] : [])), - ); - }), - distinctUntilChanged(shallowEquals), - ), - ); - - private readonly pip$ = this.scope.behavior( - combineLatest([ - this.screenShares$, - this.spotlightSpeaker$, - this.mediaItems$, - ]).pipe( - switchMap(([screenShares, spotlight, mediaItems]) => { - if (screenShares.length > 0) { - return this.spotlightSpeaker$; - } - if (!spotlight || spotlight.local) { - return of(null); - } - - const localUserMedia = mediaItems.find( - (m) => m.vm instanceof LocalUserMediaViewModel, - ) as UserMedia | undefined; - - const localUserMediaViewModel = localUserMedia?.vm as - | LocalUserMediaViewModel - | undefined; - - if (!localUserMediaViewModel) { - return of(null); - } - return localUserMediaViewModel.alwaysShow$.pipe( - map((alwaysShow) => { - if (alwaysShow) { - return localUserMediaViewModel; - } - - return null; - }), - ); - }), - ), - ); - - private readonly hasRemoteScreenShares$: Observable = - this.spotlight$.pipe( - map((spotlight) => - spotlight.some((vm) => !vm.local && vm instanceof ScreenShareViewModel), - ), - distinctUntilChanged(), - ); - - private readonly pipEnabled$ = this.scope.behavior(setPipEnabled$, false); - - private readonly naturalWindowMode$ = this.scope.behavior( - fromEvent(window, "resize").pipe( - startWith(null), - map(() => { - const height = window.innerHeight; - const width = window.innerWidth; - if (height <= 400 && width <= 340) return "pip"; - // Our layouts for flat windows are better at adapting to a small width - // than our layouts for narrow windows are at adapting to a small height, - // so we give "flat" precedence here - if (height <= 600) return "flat"; - if (width <= 600) return "narrow"; - return "normal"; - }), - ), - ); - - /** - * The general shape of the window. - */ - public readonly windowMode$ = this.scope.behavior( - this.pipEnabled$.pipe( - switchMap((pip) => - pip ? of("pip") : this.naturalWindowMode$, - ), - ), - ); - - private readonly spotlightExpandedToggle$ = new Subject(); - public readonly spotlightExpanded$ = this.scope.behavior( - this.spotlightExpandedToggle$.pipe( - accumulate(false, (expanded) => !expanded), - ), - ); - - private readonly gridModeUserSelection$ = new Subject(); - /** - * The layout mode of the media tile grid. - */ - public readonly gridMode$ = - // If the user hasn't selected spotlight and somebody starts screen sharing, - // automatically switch to spotlight mode and reset when screen sharing ends - this.scope.behavior( - this.gridModeUserSelection$.pipe( - startWith(null), - switchMap((userSelection) => - (userSelection === "spotlight" - ? EMPTY - : combineLatest([ - this.hasRemoteScreenShares$, - this.windowMode$, - ]).pipe( - skip(userSelection === null ? 0 : 1), - map( - ([hasScreenShares, windowMode]): GridMode => - hasScreenShares || windowMode === "flat" - ? "spotlight" - : "grid", - ), - ) - ).pipe(startWith(userSelection ?? "grid")), - ), - ), - ); - - public setGridMode(value: GridMode): void { - this.gridModeUserSelection$.next(value); - } - - private readonly gridLayoutMedia$: Observable = - combineLatest([this.grid$, this.spotlight$], (grid, spotlight) => ({ - type: "grid", - spotlight: spotlight.some((vm) => vm instanceof ScreenShareViewModel) - ? spotlight - : undefined, - grid, - })); - - private readonly spotlightLandscapeLayoutMedia$: Observable = - combineLatest([this.grid$, this.spotlight$], (grid, spotlight) => ({ - type: "spotlight-landscape", - spotlight, - grid, - })); - - private readonly spotlightPortraitLayoutMedia$: Observable = - combineLatest([this.grid$, this.spotlight$], (grid, spotlight) => ({ - type: "spotlight-portrait", - spotlight, - grid, - })); - - private readonly spotlightExpandedLayoutMedia$: Observable = - combineLatest([this.spotlight$, this.pip$], (spotlight, pip) => ({ - type: "spotlight-expanded", - spotlight, - pip: pip ?? undefined, - })); - - private readonly oneOnOneLayoutMedia$: Observable = - this.mediaItems$.pipe( - map((mediaItems) => { - if (mediaItems.length !== 2) return null; - const local = mediaItems.find((vm) => vm.vm.local)?.vm as - | LocalUserMediaViewModel - | undefined; - const remote = mediaItems.find((vm) => !vm.vm.local)?.vm as - | RemoteUserMediaViewModel - | undefined; - // There might not be a remote tile if there are screen shares, or if - // only the local user is in the call and they're using the duplicate - // tiles option - if (!remote || !local) return null; - - return { type: "one-on-one", local, remote }; - }), - ); - - private readonly pipLayoutMedia$: Observable = - this.spotlight$.pipe(map((spotlight) => ({ type: "pip", spotlight }))); - - /** - * The media to be used to produce a layout. - */ - private readonly layoutMedia$ = this.scope.behavior( - this.windowMode$.pipe( - switchMap((windowMode) => { - switch (windowMode) { - case "normal": - return this.gridMode$.pipe( - switchMap((gridMode) => { - switch (gridMode) { - case "grid": - return this.oneOnOneLayoutMedia$.pipe( - switchMap((oneOnOne) => - oneOnOne === null - ? this.gridLayoutMedia$ - : of(oneOnOne), - ), - ); - case "spotlight": - return this.spotlightExpanded$.pipe( - switchMap((expanded) => - expanded - ? this.spotlightExpandedLayoutMedia$ - : this.spotlightLandscapeLayoutMedia$, - ), - ); - } - }), - ); - case "narrow": - return this.oneOnOneLayoutMedia$.pipe( - switchMap((oneOnOne) => - oneOnOne === null - ? combineLatest( - [this.grid$, this.spotlight$], - (grid, spotlight) => - grid.length > smallMobileCallThreshold || - spotlight.some( - (vm) => vm instanceof ScreenShareViewModel, - ) - ? this.spotlightPortraitLayoutMedia$ - : this.gridLayoutMedia$, - ).pipe(switchAll()) - : // The expanded spotlight layout makes for a better one-on-one - // experience in narrow windows - this.spotlightExpandedLayoutMedia$, - ), - ); - case "flat": - return this.gridMode$.pipe( - switchMap((gridMode) => { - switch (gridMode) { - case "grid": - // Yes, grid mode actually gets you a "spotlight" layout in - // this window mode. - return this.spotlightLandscapeLayoutMedia$; - case "spotlight": - return this.spotlightExpandedLayoutMedia$; - } - }), - ); - case "pip": - return this.pipLayoutMedia$; - } - }), - ), - ); - - // There is a cyclical dependency here: the layout algorithms want to know - // which tiles are on screen, but to know which tiles are on screen we have to - // first render a layout. To deal with this we assume initially that no tiles - // are visible, and loop the data back into the layouts with a Subject. - private readonly visibleTiles$ = new Subject(); - private readonly setVisibleTiles = (value: number): void => - this.visibleTiles$.next(value); - - private readonly layoutInternals$ = this.scope.behavior< - LayoutScanState & { layout: Layout } - >( - combineLatest([ - this.layoutMedia$, - this.visibleTiles$.pipe(startWith(0), distinctUntilChanged()), - ]).pipe( - scan< - [LayoutMedia, number], - LayoutScanState & { layout: Layout }, - LayoutScanState - >( - ({ tiles: prevTiles }, [media, visibleTiles]) => { - let layout: Layout; - let newTiles: TileStore; - switch (media.type) { - case "grid": - case "spotlight-landscape": - case "spotlight-portrait": - [layout, newTiles] = gridLikeLayout( - media, - visibleTiles, - this.setVisibleTiles, - prevTiles, - ); - break; - case "spotlight-expanded": - [layout, newTiles] = spotlightExpandedLayout(media, prevTiles); - break; - case "one-on-one": - [layout, newTiles] = oneOnOneLayout(media, prevTiles); - break; - case "pip": - [layout, newTiles] = pipLayout(media, prevTiles); - break; - } - - return { layout, tiles: newTiles }; - }, - { layout: null, tiles: TileStore.empty() }, - ), - ), - ); - - /** - * The layout of tiles in the call interface. - */ - public readonly layout$ = this.scope.behavior( - this.layoutInternals$.pipe(map(({ layout }) => layout)), - ); - - /** - * The current generation of the tile store, exposed for debugging purposes. - */ - public readonly tileStoreGeneration$ = this.scope.behavior( - this.layoutInternals$.pipe(map(({ tiles }) => tiles.generation)), - ); - - public showSpotlightIndicators$ = this.scope.behavior( - this.layout$.pipe(map((l) => l.type !== "grid")), - ); - - public showSpeakingIndicators$ = this.scope.behavior( - this.layout$.pipe( - switchMap((l) => { - switch (l.type) { - case "spotlight-landscape": - case "spotlight-portrait": - // If the spotlight is showing the active speaker, we can do without - // speaking indicators as they're a redundant visual cue. But if - // screen sharing feeds are in the spotlight we still need them. - return l.spotlight.media$.pipe( - map((models: MediaViewModel[]) => - models.some((m) => m instanceof ScreenShareViewModel), - ), - ); - // In expanded spotlight layout, the active speaker is always shown in - // the picture-in-picture tile so there is no need for speaking - // indicators. And in one-on-one layout there's no question as to who is - // speaking. - case "spotlight-expanded": - case "one-on-one": - return of(false); - default: - return of(true); - } - }), - ), - ); - - public readonly toggleSpotlightExpanded$ = this.scope.behavior< - (() => void) | null - >( - this.windowMode$.pipe( - switchMap((mode) => - mode === "normal" - ? this.layout$.pipe( - map( - (l) => - l.type === "spotlight-landscape" || - l.type === "spotlight-expanded", - ), - ) - : of(false), - ), - distinctUntilChanged(), - map((enabled) => - enabled ? (): void => this.spotlightExpandedToggle$.next() : null, - ), - ), - ); - - private readonly screenTap$ = new Subject(); - private readonly controlsTap$ = new Subject(); - private readonly screenHover$ = new Subject(); - private readonly screenUnhover$ = new Subject(); - - /** - * Callback for when the user taps the call view. - */ - public tapScreen(): void { - this.screenTap$.next(); - } - - /** - * Callback for when the user taps the call's controls. - */ - public tapControls(): void { - this.controlsTap$.next(); - } - - /** - * Callback for when the user hovers over the call view. - */ - public hoverScreen(): void { - this.screenHover$.next(); - } - - /** - * Callback for when the user stops hovering over the call view. - */ - public unhoverScreen(): void { - this.screenUnhover$.next(); - } - - public readonly showHeader$ = this.scope.behavior( - this.windowMode$.pipe(map((mode) => mode !== "pip" && mode !== "flat")), - ); - - public readonly showFooter$ = this.scope.behavior( - this.windowMode$.pipe( - switchMap((mode) => { - switch (mode) { - case "pip": - return of(false); - case "normal": - case "narrow": - return of(true); - case "flat": - // Sadly Firefox has some layering glitches that prevent the footer - // from appearing properly. They happen less often if we never hide - // the footer. - if (isFirefox()) return of(true); - // Show/hide the footer in response to interactions - return merge( - this.screenTap$.pipe(map(() => "tap screen" as const)), - this.controlsTap$.pipe(map(() => "tap controls" as const)), - this.screenHover$.pipe(map(() => "hover" as const)), - ).pipe( - switchScan((state, interaction) => { - switch (interaction) { - case "tap screen": - return state - ? // Toggle visibility on tap - of(false) - : // Hide after a timeout - timer(showFooterMs).pipe( - map(() => false), - startWith(true), - ); - case "tap controls": - // The user is interacting with things, so reset the timeout - return timer(showFooterMs).pipe( - map(() => false), - startWith(true), - ); - case "hover": - // Show on hover and hide after a timeout - return race( - timer(showFooterMs), - this.screenUnhover$.pipe(take(1)), - ).pipe( - map(() => false), - startWith(true), - ); - } - }, false), - startWith(false), - ); - } - }), - ), - ); - - /** - * Whether audio is currently being output through the earpiece. - */ - public readonly earpieceMode$ = this.scope.behavior( - combineLatest( - [ - this.mediaDevices.audioOutput.available$, - this.mediaDevices.audioOutput.selected$, - ], - (available, selected) => - selected !== undefined && - available.get(selected.id)?.type === "earpiece", - ), - ); - - /** - * Callback to toggle between the earpiece and the loudspeaker. - * - * This will be `null` in case the target does not exist in the list - * of available audio outputs. - */ - public readonly audioOutputSwitcher$ = this.scope.behavior<{ - targetOutput: "earpiece" | "speaker"; - switch: () => void; - } | null>( - combineLatest( - [ - this.mediaDevices.audioOutput.available$, - this.mediaDevices.audioOutput.selected$, - ], - (available, selected) => { - const selectionType = selected && available.get(selected.id)?.type; - - // If we are in any output mode other than speaker switch to speaker. - const newSelectionType: "earpiece" | "speaker" = - selectionType === "speaker" ? "earpiece" : "speaker"; - const newSelection = [...available].find( - ([, d]) => d.type === newSelectionType, - ); - if (newSelection === undefined) return null; - - const [id] = newSelection; - return { - targetOutput: newSelectionType, - switch: (): void => this.mediaDevices.audioOutput.select(id), - }; - }, - ), - ); - - /** - * Emits an array of reactions that should be visible on the screen. - */ - public readonly visibleReactions$ = this.scope.behavior( - showReactions.value$.pipe( - switchMap((show) => (show ? this.reactions$ : of({}))), - scan< - Record, - { sender: string; emoji: string; startX: number }[] - >((acc, latest) => { - const newSet: { sender: string; emoji: string; startX: number }[] = []; - for (const [sender, reaction] of Object.entries(latest)) { - const startX = - acc.find((v) => v.sender === sender && v.emoji)?.startX ?? - Math.ceil(Math.random() * 80) + 10; - newSet.push({ sender, emoji: reaction.emoji, startX }); - } - return newSet; - }, []), - ), - ); - - /** - * Emits an array of reactions that should be played. - */ - public readonly audibleReactions$ = playReactionsSound.value$.pipe( - switchMap((show) => - show ? this.reactions$ : of>({}), - ), - map((reactions) => Object.values(reactions).map((v) => v.name)), - scan( - (acc, latest) => { - return { - playing: latest.filter( - (v) => acc.playing.includes(v) || acc.newSounds.includes(v), - ), - newSounds: latest.filter( - (v) => !acc.playing.includes(v) && !acc.newSounds.includes(v), - ), - }; - }, - { playing: [], newSounds: [] }, - ), - map((v) => v.newSounds), - ); - - /** - * Emits an event every time a new hand is raised in - * the call. - */ - public readonly newHandRaised$ = this.handsRaised$.pipe( - map((v) => Object.keys(v).length), - scan( - (acc, newValue) => ({ - value: newValue, - playSounds: newValue > acc.value, - }), - { value: 0, playSounds: false }, - ), - filter((v) => v.playSounds), - ); - - /** - * Emits an event every time a new screenshare is started in - * the call. - */ - public readonly newScreenShare$ = this.screenShares$.pipe( - map((v) => v.length), - scan( - (acc, newValue) => ({ - value: newValue, - playSounds: newValue > acc.value, - }), - { value: 0, playSounds: false }, - ), - filter((v) => v.playSounds), - ); - - /** - * Whether we are sharing our screen. - */ - public readonly sharingScreen$ = this.scope.behavior( - from(this.localConnection$).pipe( - switchMap((c) => - c?.state === "ready" - ? sharingScreen$(c.value.livekitRoom.localParticipant) - : of(false), - ), - ), - ); - - /** - * Callback for toggling screen sharing. If null, screen sharing is not - * available. - */ - public readonly toggleScreenSharing = - "getDisplayMedia" in (navigator.mediaDevices ?? {}) && - !this.urlParams.hideScreensharing - ? (): void => - // Once a connection is ready... - void this.localConnection$ - .pipe( - takeWhile((c) => c !== null && c.state !== "error"), - switchMap((c) => (c.state === "ready" ? of(c.value) : NEVER)), - take(1), - this.scope.bind(), - ) - // ...toggle screen sharing. - .subscribe( - (c) => - void c.livekitRoom.localParticipant - .setScreenShareEnabled(!this.sharingScreen$.value, { - audio: true, - selfBrowserSurface: "include", - surfaceSwitching: "include", - systemAudio: "include", - }) - .catch(logger.error), - ) - : null; - - public constructor( - private readonly scope: ObservableScope, - // A call is permanently tied to a single Matrix room - private readonly matrixRTCSession: MatrixRTCSession, - private readonly matrixRoom: MatrixRoom, - private readonly mediaDevices: MediaDevices, - private readonly muteStates: MuteStates, - private readonly options: CallViewModelOptions, - private readonly handsRaisedSubject$: Observable< - Record - >, - private readonly reactionsSubject$: Observable< - Record - >, - private readonly trackProcessorState$: Observable, - ) { - // Start and stop local and remote connections as needed - this.connectionInstructions$ - .pipe(this.scope.bind()) - .subscribe(({ start, stop }) => { - for (const c of stop) { - logger.info(`Disconnecting from ${c.transport.livekit_service_url}`); - c.stop().catch((err) => { - // TODO: better error handling - logger.error( - `Fail to stop connection to ${c.transport.livekit_service_url}`, - err, - ); - }); - } - for (const c of start) { - c.start().then( - () => - logger.info(`Connected to ${c.transport.livekit_service_url}`), - (e) => { - // We only want to report fatal errors `_configError$` for the publish connection. - // If there is an error with another connection, it will not terminate the call and will be displayed - // on eacn tile. - if ( - c instanceof PublishConnection && - e instanceof ElementCallError - ) { - this._configError$.next(e); - } - logger.error( - `Failed to start connection to ${c.transport.livekit_service_url}`, - e, - ); - }, - ); - } - }); - - // Start and stop session membership as needed - this.scope.reconcile(this.advertisedTransport$, async (advertised) => { - if (advertised !== null) { - try { - this._configError$.next(null); - await enterRTCSession(this.matrixRTCSession, advertised.transport, { - encryptMedia: this.options.encryptionSystem.kind !== E2eeType.NONE, - useMultiSfu: advertised.multiSfu, - preferStickyEvents: advertised.preferStickyEvents, - }); - } catch (e) { - logger.error("Error entering RTC session", e); - } - - // Update our member event when our mute state changes. - const intentScope = new ObservableScope(); - intentScope.reconcile( - this.muteStates.video.enabled$, - async (videoEnabled) => - this.matrixRTCSession.updateCallIntent( - videoEnabled ? "video" : "audio", - ), - ); - - return async (): Promise => { - intentScope.end(); - // Only sends Matrix leave event. The LiveKit session will disconnect - // as soon as either the stopConnection$ handler above gets to it or - // the view model is destroyed. - try { - await this.matrixRTCSession.leaveRoomSession(); - } catch (e) { - logger.error("Error leaving RTC session", e); - } - try { - await widget?.api.transport.send( - ElementWidgetActions.HangupCall, - {}, - ); - } catch (e) { - logger.error("Failed to send hangup action", e); - } - }; - } - }); - - // Pause upstream of all local media tracks when we're disconnected from - // MatrixRTC, because it can be an unpleasant surprise for the app to say - // 'reconnecting' and yet still be transmitting your media to others. - // We use matrixConnected$ rather than reconnecting$ because we want to - // pause tracks during the initial joining sequence too until we're sure - // that our own media is displayed on screen. - combineLatest([this.localConnection$, this.matrixConnected$]) - .pipe(this.scope.bind()) - .subscribe(([connection, connected]) => { - if (connection?.state !== "ready") return; - const publications = - connection.value.livekitRoom.localParticipant.trackPublications.values(); - if (connected) { - for (const p of publications) { - if (p.track?.isUpstreamPaused === true) { - const kind = p.track.kind; - logger.log( - `Resuming ${kind} track (MatrixRTC connection present)`, - ); - p.track - .resumeUpstream() - .catch((e) => - logger.error( - `Failed to resume ${kind} track after MatrixRTC reconnection`, - e, - ), - ); - } - } - } else { - for (const p of publications) { - if (p.track?.isUpstreamPaused === false) { - const kind = p.track.kind; - logger.log( - `Pausing ${kind} track (uncertain MatrixRTC connection)`, - ); - p.track - .pauseUpstream() - .catch((e) => - logger.error( - `Failed to pause ${kind} track after entering uncertain MatrixRTC connection`, - e, - ), - ); - } - } - } - }); - - // Join automatically - this.join(); // TODO-MULTI-SFU: Use this view model for the lobby as well, and only call this once 'join' is clicked? - } -} - -// TODO-MULTI-SFU // Setup and update the keyProvider which was create by `createRoom` was a thing before. Now we never update if the E2EEsystem changes -// do we need this? - -function getE2eeKeyProvider( - e2eeSystem: EncryptionSystem, - rtcSession: MatrixRTCSession, -): BaseKeyProvider | undefined { - if (e2eeSystem.kind === E2eeType.NONE) return undefined; - - if (e2eeSystem.kind === E2eeType.PER_PARTICIPANT) { - const keyProvider = new MatrixKeyProvider(); - keyProvider.setRTCSession(rtcSession); - return keyProvider; - } else if (e2eeSystem.kind === E2eeType.SHARED_KEY && e2eeSystem.secret) { - const keyProvider = new ExternalE2EEKeyProvider(); - keyProvider - .setKey(e2eeSystem.secret) - .catch((e) => logger.error("Failed to set shared key for E2EE", e)); - return keyProvider; - } -} - -function getRoomMemberFromRtcMember( - rtcMember: CallMembership, - room: MatrixRoom, -): { id: string; member: RoomMember | undefined } { - return { - id: rtcMember.userId + ":" + rtcMember.deviceId, - member: room.getMember(rtcMember.userId) ?? undefined, - }; -} diff --git a/src/state/CallViewModel/CallNotificationLifecycle.test.ts b/src/state/CallViewModel/CallNotificationLifecycle.test.ts new file mode 100644 index 00000000..236c126a --- /dev/null +++ b/src/state/CallViewModel/CallNotificationLifecycle.test.ts @@ -0,0 +1,354 @@ +/* +Copyright 2025 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + type ICallNotifyContent, + type IRTCNotificationContent, +} from "matrix-js-sdk/lib/matrixrtc"; +import { describe, it } from "vitest"; +import { + EventType, + type IEvent, + type IRoomTimelineData, + MatrixEvent, + type Room, +} from "matrix-js-sdk"; + +import { withTestScheduler } from "../../utils/test"; +import { + aliceRtcMember, + local, + localRtcMember, +} from "../../utils/test-fixtures"; +import { + createCallNotificationLifecycle$, + type Props as CallNotificationLifecycleProps, +} from "./CallNotificationLifecycle"; +import { trackEpoch } from "../ObservableScope"; + +const mockLegacyRingEvent = {} as { event_id: string } & ICallNotifyContent; +function mockRingEvent( + eventId: string, + lifetimeMs: number | undefined, + sender = local.userId, +): { event_id: string } & IRTCNotificationContent { + return { + event_id: eventId, + ...(lifetimeMs === undefined ? {} : { lifetime: lifetimeMs }), + notification_type: "ring", + sender, + } as unknown as { event_id: string } & IRTCNotificationContent; +} + +describe("waitForCallPickup$", () => { + it("unknown -> ringing -> timeout when notified and nobody joins", () => { + withTestScheduler(({ scope, expectObservable, behavior, hot }) => { + // No one ever joins (only local user) + const props: CallNotificationLifecycleProps = { + scope, + memberships$: scope.behavior( + behavior("a", { a: [] }).pipe(trackEpoch()), + ), + sentCallNotification$: hot("10ms a", { + a: [mockRingEvent("$notif1", 30), mockLegacyRingEvent], + }), + receivedDecline$: hot(""), + options: { + waitForCallPickup: true, + autoLeaveWhenOthersLeft: false, + }, + localUser: localRtcMember, + }; + + const lifecycle = createCallNotificationLifecycle$(props); + + expectObservable(lifecycle.callPickupState$).toBe("a 9ms b 29ms c", { + a: "unknown", + b: "ringing", + c: "timeout", + }); + }); + }); + + it("ringing -> success if someone joins before timeout is reached", () => { + withTestScheduler(({ scope, hot, behavior, expectObservable }) => { + // Someone joins at 20ms (both LiveKit participant and MatrixRTC member) + const props: CallNotificationLifecycleProps = { + scope, + memberships$: scope.behavior( + behavior("a 19ms b", { + a: [localRtcMember], + b: [localRtcMember, aliceRtcMember], + }).pipe(trackEpoch()), + ), + sentCallNotification$: hot("5ms a", { + a: [mockRingEvent("$notif2", 100), mockLegacyRingEvent], + }), + receivedDecline$: hot(""), + options: { + waitForCallPickup: true, + autoLeaveWhenOthersLeft: false, + }, + localUser: localRtcMember, + }; + const lifecycle = createCallNotificationLifecycle$(props); + expectObservable(lifecycle.callPickupState$).toBe("a 4ms b 14ms c", { + a: "unknown", + b: "ringing", + c: "success", + }); + }); + }); + it("success when someone joins before we notify", () => { + withTestScheduler(({ scope, hot, behavior, expectObservable }) => { + // Someone joins at 20ms (both LiveKit participant and MatrixRTC member) + const props: CallNotificationLifecycleProps = { + scope, + memberships$: scope.behavior( + behavior("a 9ms b", { + a: [localRtcMember], + b: [localRtcMember, aliceRtcMember], + }).pipe(trackEpoch()), + ), + sentCallNotification$: hot("20ms a", { + a: [mockRingEvent("$notif2", 50), mockLegacyRingEvent], + }), + receivedDecline$: hot(""), + options: { + waitForCallPickup: true, + autoLeaveWhenOthersLeft: false, + }, + localUser: localRtcMember, + }; + const lifecycle = createCallNotificationLifecycle$(props); + expectObservable(lifecycle.callPickupState$).toBe("a 9ms b", { + a: "unknown", + b: "success", + }); + }); + }); + it("notify without lifetime -> immediate timeout", () => { + withTestScheduler(({ scope, hot, behavior, expectObservable }) => { + // Someone joins at 20ms (both LiveKit participant and MatrixRTC member) + const props: CallNotificationLifecycleProps = { + scope, + memberships$: scope.behavior( + behavior("a", { + a: [localRtcMember], + }).pipe(trackEpoch()), + ), + sentCallNotification$: hot("10ms a", { + a: [mockRingEvent("$notif2", undefined), mockLegacyRingEvent], + }), + receivedDecline$: hot(""), + options: { + waitForCallPickup: true, + autoLeaveWhenOthersLeft: false, + }, + localUser: localRtcMember, + }; + const lifecycle = createCallNotificationLifecycle$(props); + expectObservable(lifecycle.callPickupState$).toBe("a 9ms b", { + a: "unknown", + b: "timeout", + }); + }); + }); + + it("stays null when waitForCallPickup=false", () => { + withTestScheduler(({ scope, hot, behavior, expectObservable }) => { + // Someone joins at 20ms (both LiveKit participant and MatrixRTC member) + const validProps: CallNotificationLifecycleProps = { + scope, + memberships$: scope.behavior( + behavior("a--b", { + a: [localRtcMember], + b: [localRtcMember, aliceRtcMember], + }).pipe(trackEpoch()), + ), + sentCallNotification$: hot("10ms a", { + a: [mockRingEvent("$notif5", 30), mockLegacyRingEvent], + }), + receivedDecline$: hot(""), + options: { + waitForCallPickup: true, + autoLeaveWhenOthersLeft: false, + }, + localUser: localRtcMember, + }; + const propsDeactivated = { + ...validProps, + options: { + ...validProps.options, + waitForCallPickup: false, + }, + }; + const lifecycle = createCallNotificationLifecycle$(propsDeactivated); + expectObservable(lifecycle.callPickupState$).toBe("n", { + n: null, + }); + const lifecycleReference = createCallNotificationLifecycle$(validProps); + expectObservable(lifecycleReference.callPickupState$).toBe("u--s", { + u: "unknown", + s: "success", + }); + }); + }); + + it("decline before timeout window ends -> decline", () => { + withTestScheduler(({ scope, hot, behavior, expectObservable }) => { + // Someone joins at 20ms (both LiveKit participant and MatrixRTC member) + const props: CallNotificationLifecycleProps = { + scope, + memberships$: scope.behavior( + behavior("a", { + a: [localRtcMember], + }).pipe(trackEpoch()), + ), + sentCallNotification$: hot("10ms a", { + a: [mockRingEvent("$decl1", 50), mockLegacyRingEvent], + }), + receivedDecline$: hot("40ms d", { + d: [ + new MatrixEvent({ + type: EventType.RTCDecline, + content: { + "m.relates_to": { + rel_type: "m.reference", + event_id: "$decl1", + }, + }, + }), + {} as Room, + undefined, + false, + {} as IRoomTimelineData, + ], + }), + options: { + waitForCallPickup: true, + autoLeaveWhenOthersLeft: false, + }, + localUser: localRtcMember, + }; + const lifecycle = createCallNotificationLifecycle$(props); + expectObservable(lifecycle.callPickupState$).toBe("a 9ms b 29ms e", { + a: "unknown", + b: "ringing", + e: "decline", + }); + }); + }); + it("decline after timeout window ends -> stays timeout", () => { + withTestScheduler(({ scope, hot, behavior, expectObservable }) => { + // Someone joins at 20ms (both LiveKit participant and MatrixRTC member) + const props: CallNotificationLifecycleProps = { + scope, + memberships$: scope.behavior( + behavior("a", { + a: [localRtcMember], + }).pipe(trackEpoch()), + ), + sentCallNotification$: hot("10ms a", { + a: [mockRingEvent("$decl", 20), mockLegacyRingEvent], + }), + receivedDecline$: hot("40ms d", { + d: [ + new MatrixEvent({ + type: EventType.RTCDecline, + content: { + "m.relates_to": { + rel_type: "m.reference", + event_id: "$decl", + }, + }, + }), + {} as Room, + undefined, + false, + {} as IRoomTimelineData, + ], + }), + options: { + waitForCallPickup: true, + autoLeaveWhenOthersLeft: false, + }, + localUser: localRtcMember, + }; + const lifecycle = createCallNotificationLifecycle$(props); + expectObservable(lifecycle.callPickupState$, "50ms !").toBe( + "a 9ms b 19ms e", + { + a: "unknown", + b: "ringing", + e: "timeout", + }, + ); + }); + }); + // + function testStaysRinging( + declineEvent: Partial, + expectDecline: boolean, + ): void { + withTestScheduler(({ scope, hot, behavior, expectObservable }) => { + // Someone joins at 20ms (both LiveKit participant and MatrixRTC member) + const props: CallNotificationLifecycleProps = { + scope, + memberships$: scope.behavior( + behavior("a", { + a: [localRtcMember], + }).pipe(trackEpoch()), + ), + sentCallNotification$: hot("10ms a", { + a: [mockRingEvent("$right", 50), mockLegacyRingEvent], + }), + receivedDecline$: hot("20ms d", { + d: [ + new MatrixEvent(declineEvent), + {} as Room, + undefined, + false, + {} as IRoomTimelineData, + ], + }), + options: { + waitForCallPickup: true, + autoLeaveWhenOthersLeft: false, + }, + localUser: localRtcMember, + }; + const lifecycle = createCallNotificationLifecycle$(props); + const marbles = expectDecline ? "a 9ms b 9ms d" : "a 9ms b"; + expectObservable(lifecycle.callPickupState$, "21ms !").toBe(marbles, { + a: "unknown", + b: "ringing", + d: "decline", + }); + }); + } + const reference = (refId?: string, sender?: string): Partial => ({ + event_id: "$decline", + type: EventType.RTCDecline, + sender: sender ?? "@other:example.org", + content: { + "m.relates_to": { + rel_type: "m.reference", + event_id: refId ?? "$right", + }, + }, + }); + it("decline reference works", () => { + testStaysRinging(reference(), true); + }); + it("decline with wrong id is ignored (stays ringing)", () => { + testStaysRinging(reference("$wrong"), false); + }); + it("decline with wrong id is ignored (stays ringing)", () => { + testStaysRinging(reference(undefined, local.userId), false); + }); +}); diff --git a/src/state/CallViewModel/CallNotificationLifecycle.ts b/src/state/CallViewModel/CallNotificationLifecycle.ts new file mode 100644 index 00000000..2a0bf2f1 --- /dev/null +++ b/src/state/CallViewModel/CallNotificationLifecycle.ts @@ -0,0 +1,211 @@ +/* +Copyright 2025 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + type CallMembership, + type MatrixRTCSession, + MatrixRTCSessionEvent, + type MatrixRTCSessionEventHandlerMap, +} from "matrix-js-sdk/lib/matrixrtc"; +import { + combineLatest, + concat, + endWith, + filter, + fromEvent, + ignoreElements, + map, + merge, + NEVER, + type Observable, + of, + pairwise, + startWith, + switchMap, + takeUntil, + timer, +} from "rxjs"; +import { + type EventTimelineSetHandlerMap, + EventType, + type Room as MatrixRoom, + RoomEvent, +} from "matrix-js-sdk"; + +import { type Behavior } from "../Behavior"; +import { type Epoch, mapEpoch, type ObservableScope } from "../ObservableScope"; +export type AutoLeaveReason = "allOthersLeft" | "timeout" | "decline"; +export type CallPickupState = + | "unknown" + | "ringing" + | "timeout" + | "decline" + | "success" + | null; +export type CallNotificationWrapper = Parameters< + MatrixRTCSessionEventHandlerMap[MatrixRTCSessionEvent.DidSendCallNotification] +>; +export function createSentCallNotification$( + scope: ObservableScope, + matrixRTCSession: MatrixRTCSession, +): Behavior { + const sentCallNotification$ = scope.behavior( + fromEvent(matrixRTCSession, MatrixRTCSessionEvent.DidSendCallNotification), + null, + ) as Behavior; + return sentCallNotification$; +} + +export function createReceivedDecline$( + matrixRoom: MatrixRoom, +): Observable> { + return ( + fromEvent(matrixRoom, RoomEvent.Timeline) as Observable< + Parameters + > + ).pipe(filter(([event]) => event.getType() === EventType.RTCDecline)); +} + +export interface Props { + scope: ObservableScope; + memberships$: Behavior>; + sentCallNotification$: Observable; + receivedDecline$: Observable< + Parameters + >; + options: { waitForCallPickup?: boolean; autoLeaveWhenOthersLeft?: boolean }; + localUser: { deviceId: string; userId: string }; +} +/** + * @returns {callPickupState$, autoLeave$} + * `callPickupState$` The current call pickup state of the call. + * - "unknown": The client has not yet sent the notification event. We don't know if it will because it first needs to send its own membership. + * Then we can conclude if we were the first one to join or not. + * This may also be set if we are disconnected. + * - "ringing": The call is ringing on other devices in this room (This client should give audiovisual feedback that this is happening). + * - "timeout": No-one picked up in the defined time this call should be ringing on others devices. + * The call failed. If desired this can be used as a trigger to exit the call. + * - "success": Someone else joined. The call is in a normal state. No audiovisual feedback. + * - null: EC is configured to never show any waiting for answer state. + * + * `autoLeave$` An observable that emits (null) when the call should be automatically left. + * - if options.autoLeaveWhenOthersLeft is set to true it emits when all others left. + * - if options.waitForCallPickup is set to true it emits if noone picked up the ring or if the ring got declined. + * - if options.autoLeaveWhenOthersLeft && options.waitForCallPickup is false it will never emit. + * + */ +export function createCallNotificationLifecycle$({ + scope, + memberships$, + sentCallNotification$, + receivedDecline$, + options, + localUser, +}: Props): { + callPickupState$: Behavior; + autoLeave$: Observable; +} { + const allOthersLeft$ = memberships$.pipe( + pairwise(), + filter( + ([{ value: prev }, { value: current }]) => + current.every((m) => m.userId === localUser.userId) && + prev.some((m) => m.userId !== localUser.userId), + ), + map(() => {}), + ); + + /** + * Whether some Matrix user other than ourself is joined to the call. + */ + const someoneElseJoined$ = memberships$.pipe( + mapEpoch((ms) => ms.some((m) => m.userId !== localUser.userId)), + ) as Behavior>; + + /** + * Whenever the RTC session tells us that it intends to ring the remote + * participant's devices, this emits an Observable tracking the current state of + * that ringing process. + */ + // This is a behavior since we need to store the latest state for when we subscribe to this after `didSendCallNotification$` + // has already emitted but we still need the latest observable with a timeout timer that only gets created on after receiving `notificationEvent`. + // A behavior will emit the latest observable with the running timer to new subscribers. + // see also: callPickupState$ and in particular the line: `return this.ring$.pipe(mergeAll());` here we otherwise might get an EMPTY observable if + // `ring$` would not be a behavior. + const remoteRingState$: Behavior<"ringing" | "timeout" | "decline" | null> = + scope.behavior( + sentCallNotification$.pipe( + filter( + (newAndLegacyEvents) => + // only care about new events (legacy do not have decline pattern) + newAndLegacyEvents?.[0].notification_type === "ring", + ), + map((e) => e as CallNotificationWrapper), + switchMap(([notificationEvent]) => { + const lifetimeMs = notificationEvent?.lifetime ?? 0; + return concat( + lifetimeMs === 0 + ? // If no lifetime, skip the ring state + of(null) + : // Ring until lifetime ms have passed + timer(lifetimeMs).pipe( + ignoreElements(), + startWith("ringing" as const), + ), + // The notification lifetime has timed out, meaning ringing has likely + // stopped on all receiving clients. + of("timeout" as const), + // This makes sure we will not drop into the `endWith("decline" as const)` state + NEVER, + ).pipe( + takeUntil( + receivedDecline$.pipe( + filter( + ([event]) => + event.getRelation()?.rel_type === "m.reference" && + event.getRelation()?.event_id === + notificationEvent.event_id && + event.getSender() !== localUser.userId && + callPickupState$.value !== "timeout", + ), + ), + ), + endWith("decline" as const), + ); + }), + ), + null, + ); + + const callPickupState$ = scope.behavior( + options.waitForCallPickup === true + ? combineLatest( + [someoneElseJoined$, remoteRingState$], + (someoneElseJoined, ring) => { + if (someoneElseJoined.value === true) { + return "success" as const; + } + // Show the ringing state of the most recent ringing attempt. + // as long as we have not yet sent an RTC notification event or noone else joined, + // ring will be null -> callPickupState$ = unknown. + return ring ?? ("unknown" as const); + }, + ) + : NEVER, + null, + ); + + const autoLeave$ = merge( + options.autoLeaveWhenOthersLeft === true + ? allOthersLeft$.pipe(map(() => "allOthersLeft" as const)) + : NEVER, + callPickupState$.pipe( + filter((state) => state === "timeout" || state === "decline"), + ), + ); + return { autoLeave$, callPickupState$ }; +} diff --git a/src/state/CallViewModel/CallViewModel.test.ts b/src/state/CallViewModel/CallViewModel.test.ts new file mode 100644 index 00000000..3a621f33 --- /dev/null +++ b/src/state/CallViewModel/CallViewModel.test.ts @@ -0,0 +1,1313 @@ +/* +Copyright 2025 Element Creations Ltd. +Copyright 2024 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { test, vi, onTestFinished, it, describe, expect } from "vitest"; +import EventEmitter from "events"; +import { + BehaviorSubject, + combineLatest, + debounceTime, + distinctUntilChanged, + map, + NEVER, + type Observable, + of, + switchMap, +} from "rxjs"; +import { SyncState, type MatrixClient } from "matrix-js-sdk"; +import { + ConnectionState, + type LocalTrackPublication, + type RemoteParticipant, + type Room as LivekitRoom, +} from "livekit-client"; +import * as ComponentsCore from "@livekit/components-core"; +import { + Status, + type CallMembership, + type IRTCNotificationContent, + type ICallNotifyContent, + MatrixRTCSessionEvent, + type LivekitTransport, +} from "matrix-js-sdk/lib/matrixrtc"; +import { deepCompare } from "matrix-js-sdk/lib/utils"; +import { AutoDiscovery } from "matrix-js-sdk/lib/autodiscovery"; + +import { createCallViewModel$ } from "./CallViewModel"; +import { type Layout } from "../layout-types.ts"; +import { + mockLocalParticipant, + mockMatrixRoom, + mockMatrixRoomMember, + mockRemoteParticipant, + withTestScheduler, + mockRtcMembership, + MockRTCSession, + mockMediaDevices, + mockMuteStates, + mockConfig, + testScope, + mockLivekitRoom, + exampleTransport, +} from "../../utils/test.ts"; +import { E2eeType } from "../../e2ee/e2eeType.ts"; +import type { RaisedHandInfo, ReactionInfo } from "../../reactions/index.ts"; +import { + aliceId, + aliceParticipant, + aliceRtcMember, + bobId, + bobRtcMember, + local, + localId, + localRtcMember, + localRtcMemberDevice2, +} from "../../utils/test-fixtures.ts"; +import { MediaDevices } from "../MediaDevices.ts"; +import { getValue } from "../../utils/observable.ts"; +import { type Behavior, constant } from "../Behavior.ts"; +import { + type ElementCallError, + MatrixRTCTransportMissingError, +} from "../../utils/errors.ts"; +import { withCallViewModel } from "./CallViewModelTestUtils.ts"; + +vi.mock("rxjs", async (importOriginal) => ({ + ...(await importOriginal()), + // Disable interval Observables for the following tests since the test + // scheduler will loop on them forever and never call the test 'done' + interval: (): Observable => NEVER, +})); + +vi.mock("@livekit/components-core"); +vi.mock("livekit-client/e2ee-worker?worker"); + +vi.mock("../e2ee/matrixKeyProvider"); + +const getUrlParams = vi.hoisted(() => vi.fn(() => ({}))); +vi.mock("../UrlParams", () => ({ getUrlParams })); + +vi.mock("../rtcSessionHelpers", async (importOriginal) => ({ + ...(await importOriginal()), + makeTransport: async (): Promise => + Promise.resolve(exampleTransport), +})); + +const yesNo = { + y: true, + n: false, +}; + +const daveRtcMember = mockRtcMembership("@dave:example.org", "DDDD"); + +// const carol = local; + +const dave = mockMatrixRoomMember(daveRtcMember, { rawDisplayName: "Dave" }); + +const daveId = `${dave.userId}:${daveRtcMember.deviceId}`; + +const localParticipant = mockLocalParticipant({ identity: "" }); +const aliceSharingScreen = mockRemoteParticipant({ + identity: aliceId, + isScreenShareEnabled: true, +}); +const bobParticipant = mockRemoteParticipant({ identity: bobId }); +const bobSharingScreen = mockRemoteParticipant({ + identity: bobId, + isScreenShareEnabled: true, +}); +const daveParticipant = mockRemoteParticipant({ identity: daveId }); + +export interface GridLayoutSummary { + type: "grid"; + spotlight?: string[]; + grid: string[]; +} + +export interface SpotlightLandscapeLayoutSummary { + type: "spotlight-landscape"; + spotlight: string[]; + grid: string[]; +} + +export interface SpotlightPortraitLayoutSummary { + type: "spotlight-portrait"; + spotlight: string[]; + grid: string[]; +} + +export interface SpotlightExpandedLayoutSummary { + type: "spotlight-expanded"; + spotlight: string[]; + pip?: string; +} + +export interface OneOnOneLayoutSummary { + type: "one-on-one"; + local: string; + remote: string; +} + +export interface PipLayoutSummary { + type: "pip"; + spotlight: string[]; +} + +export type LayoutSummary = + | GridLayoutSummary + | SpotlightLandscapeLayoutSummary + | SpotlightPortraitLayoutSummary + | SpotlightExpandedLayoutSummary + | OneOnOneLayoutSummary + | PipLayoutSummary; + +function summarizeLayout$(l$: Observable): Observable { + return l$.pipe( + switchMap((l) => { + switch (l.type) { + case "grid": + return combineLatest( + [ + l.spotlight?.media$ ?? constant(undefined), + ...l.grid.map((vm) => vm.media$), + ], + (spotlight, ...grid) => ({ + type: l.type, + spotlight: spotlight?.map((vm) => vm.id), + grid: grid.map((vm) => vm.id), + }), + ); + case "spotlight-landscape": + case "spotlight-portrait": + return combineLatest( + [l.spotlight.media$, ...l.grid.map((vm) => vm.media$)], + (spotlight, ...grid) => ({ + type: l.type, + spotlight: spotlight.map((vm) => vm.id), + grid: grid.map((vm) => vm.id), + }), + ); + case "spotlight-expanded": + return combineLatest( + [l.spotlight.media$, l.pip?.media$ ?? constant(undefined)], + (spotlight, pip) => ({ + type: l.type, + spotlight: spotlight.map((vm) => vm.id), + pip: pip?.id, + }), + ); + case "one-on-one": + return combineLatest( + [l.local.media$, l.remote.media$], + (local, remote) => ({ + type: l.type, + local: local.id, + remote: remote.id, + }), + ); + case "pip": + return l.spotlight.media$.pipe( + map((spotlight) => ({ + type: l.type, + spotlight: spotlight.map((vm) => vm.id), + })), + ); + } + }), + // Sometimes there can be multiple (synchronous) updates per frame. We only + // care about the most recent value for each time step, so discard these + // extra values. + debounceTime(0), + distinctUntilChanged(deepCompare), + ); +} + +function mockRingEvent( + eventId: string, + lifetimeMs: number | undefined, + sender = local.userId, +): { event_id: string } & IRTCNotificationContent { + return { + event_id: eventId, + ...(lifetimeMs === undefined ? {} : { lifetime: lifetimeMs }), + notification_type: "ring", + sender, + } as unknown as { event_id: string } & IRTCNotificationContent; +} + +// The app doesn't really care about the content of these legacy events, we just +// need a value to fill in for them when emitting notifications +const mockLegacyRingEvent = {} as { event_id: string } & ICallNotifyContent; + +describe("CallViewModel", () => { + // TODO: Restore this test. It requires makeTransport to not be mocked, unlike + // the rest of the tests in this file… what do we do? + it.skip("test missing RTC config error", async () => { + const rtcMemberships$ = new BehaviorSubject([]); + const emitter = new EventEmitter(); + const client = vi.mocked({ + on: emitter.on.bind(emitter), + off: emitter.off.bind(emitter), + getSyncState: vi.fn().mockReturnValue(SyncState.Syncing), + getUserId: vi.fn().mockReturnValue("@user:localhost"), + getUser: vi.fn().mockReturnValue(null), + getDeviceId: vi.fn().mockReturnValue("DEVICE"), + credentials: { + userId: "@user:localhost", + }, + getCrypto: vi.fn().mockReturnValue(undefined), + getDomain: vi.fn().mockReturnValue("example.org"), + } as unknown as MatrixClient); + + const matrixRoom = mockMatrixRoom({ + roomId: "!myRoomId:example.com", + client, + getMember: vi.fn().mockReturnValue(undefined), + }); + + const fakeRtcSession = new MockRTCSession(matrixRoom).withMemberships( + rtcMemberships$, + ); + + mockConfig({}); + + vi.spyOn(AutoDiscovery, "getRawClientConfig").mockResolvedValue({}); + + const callVM = createCallViewModel$( + testScope(), + fakeRtcSession.asMockedSession(), + matrixRoom, + mockMediaDevices({}), + mockMuteStates(), + { + encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + autoLeaveWhenOthersLeft: false, + livekitRoomFactory: (): LivekitRoom => + mockLivekitRoom({ + localParticipant, + disconnect: async () => Promise.resolve(), + setE2EEEnabled: async () => Promise.resolve(), + }), + }, + new BehaviorSubject({} as Record), + new BehaviorSubject({} as Record), + constant({ processor: undefined, supported: false }), + ); + + const failPromise = Promise.withResolvers(); + callVM.configError$.subscribe((error) => { + if (error) { + failPromise.resolve(error); + } + }); + + const error = await failPromise.promise; + expect(error).toBeInstanceOf(MatrixRTCTransportMissingError); + }); + + test("participants are retained during a focus switch", () => { + withTestScheduler(({ behavior, expectObservable }) => { + // Participants disappear on frame 2 and come back on frame 3 + const participantInputMarbles = "a-ba"; + // Start switching focus on frame 1 and reconnect on frame 3 + const connectionInputMarbles = " cs-c"; + // The visible participants should remain the same throughout the switch + const expectedLayoutMarbles = " a"; + + withCallViewModel( + { + remoteParticipants$: behavior(participantInputMarbles, { + a: [aliceParticipant, bobParticipant], + b: [], + }), + rtcMembers$: constant([localRtcMember, aliceRtcMember, bobRtcMember]), + livekitConnectionState$: behavior(connectionInputMarbles, { + c: ConnectionState.Connected, + s: ConnectionState.Connecting, + }), + }, + (vm) => { + expectObservable(summarizeLayout$(vm.layout$)).toBe( + expectedLayoutMarbles, + { + a: { + type: "grid", + spotlight: undefined, + grid: [`${localId}:0`, `${aliceId}:0`, `${bobId}:0`], + }, + }, + ); + }, + ); + }); + }); + + it.skip("screen sharing activates spotlight layout", () => { + withTestScheduler(({ behavior, schedule, expectObservable }) => { + // Start with no screen shares, then have Alice and Bob share their screens, + // then return to no screen shares, then have just Alice share for a bit + const participantInputMarbles = " abcda-ba"; + // While there are no screen shares, switch to spotlight manually, and then + // switch back to grid at the end + const modeInputMarbles = " -----s--g"; + // We should automatically enter spotlight for the first round of screen + // sharing, then return to grid, then manually go into spotlight, and + // remain in spotlight until we manually go back to grid + const expectedLayoutMarbles = " abcdaefeg"; + const expectedShowSpeakingMarbles = "y----nyny"; + withCallViewModel( + { + remoteParticipants$: behavior(participantInputMarbles, { + a: [aliceParticipant, bobParticipant], + b: [aliceSharingScreen, bobParticipant], + c: [aliceSharingScreen, bobSharingScreen], + d: [aliceParticipant, bobSharingScreen], + }), + rtcMembers$: constant([localRtcMember, aliceRtcMember, bobRtcMember]), + }, + (vm) => { + schedule(modeInputMarbles, { + s: () => vm.setGridMode("spotlight"), + g: () => vm.setGridMode("grid"), + }); + + expectObservable(summarizeLayout$(vm.layout$)).toBe( + expectedLayoutMarbles, + { + a: { + type: "grid", + spotlight: undefined, + grid: [`${localId}:0`, `${aliceId}:0`, `${bobId}:0`], + }, + b: { + type: "spotlight-landscape", + spotlight: [`${aliceId}:0:screen-share`], + grid: [`${localId}:0`, `${aliceId}:0`, `${bobId}:0`], + }, + c: { + type: "spotlight-landscape", + spotlight: [ + `${aliceId}:0:screen-share`, + `${bobId}:0:screen-share`, + ], + grid: [`${localId}:0`, `${aliceId}:0`, `${bobId}:0`], + }, + d: { + type: "spotlight-landscape", + spotlight: [`${bobId}:0:screen-share`], + grid: [`${localId}:0`, `${aliceId}:0`, `${bobId}:0`], + }, + e: { + type: "spotlight-landscape", + spotlight: [`${aliceId}:0`], + grid: [`${localId}:0`, `${bobId}:0`], + }, + f: { + type: "spotlight-landscape", + spotlight: [`${aliceId}:0:screen-share`], + grid: [`${localId}:0`, `${bobId}:0`, `${aliceId}:0`], + }, + g: { + type: "grid", + spotlight: undefined, + grid: [`${localId}:0`, `${bobId}:0`, `${aliceId}:0`], + }, + }, + ); + expectObservable(vm.showSpeakingIndicators$).toBe( + expectedShowSpeakingMarbles, + yesNo, + ); + }, + ); + }); + }); + + test("participants stay in the same order unless to appear/disappear", () => { + withTestScheduler(({ behavior, schedule, expectObservable }) => { + const visibilityInputMarbles = "a"; + // First Bob speaks, then Dave, then Alice + const aSpeakingInputMarbles = " n- 1998ms - 1999ms y"; + const bSpeakingInputMarbles = " ny 1998ms n 1999ms -"; + const dSpeakingInputMarbles = " n- 1998ms y 1999ms n"; + // Nothing should change when Bob speaks, because Bob is already on screen. + // When Dave speaks he should switch with Alice because she's the one who + // hasn't spoken at all. Then when Alice speaks, she should return to her + // place at the top. + const expectedLayoutMarbles = " a 1999ms b 1999ms a 57999ms c 1999ms a"; + + withCallViewModel( + { + remoteParticipants$: constant([ + aliceParticipant, + bobParticipant, + daveParticipant, + ]), + rtcMembers$: constant([ + localRtcMember, + aliceRtcMember, + bobRtcMember, + daveRtcMember, + ]), + speaking: new Map([ + [aliceParticipant, behavior(aSpeakingInputMarbles, yesNo)], + [bobParticipant, behavior(bSpeakingInputMarbles, yesNo)], + [daveParticipant, behavior(dSpeakingInputMarbles, yesNo)], + ]), + }, + (vm) => { + schedule(visibilityInputMarbles, { + a: () => { + // We imagine that only three tiles (the first three) will be visible + // on screen at a time + vm.layout$.subscribe((layout) => { + if (layout.type === "grid") layout.setVisibleTiles(3); + }); + }, + }); + + expectObservable(summarizeLayout$(vm.layout$)).toBe( + expectedLayoutMarbles, + { + a: { + type: "grid", + spotlight: undefined, + grid: [ + `${localId}:0`, + `${aliceId}:0`, + `${bobId}:0`, + `${daveId}:0`, + ], + }, + b: { + type: "grid", + spotlight: undefined, + grid: [ + `${localId}:0`, + `${daveId}:0`, + `${bobId}:0`, + `${aliceId}:0`, + ], + }, + c: { + type: "grid", + spotlight: undefined, + grid: [ + `${localId}:0`, + `${aliceId}:0`, + `${daveId}:0`, + `${bobId}:0`, + ], + }, + }, + ); + }, + ); + }); + }); + + test("participants adjust order when space becomes constrained", () => { + withTestScheduler(({ behavior, schedule, expectObservable }) => { + // Start with all tiles on screen then shrink to 3 + const visibilityInputMarbles = "a-b"; + // Bob and Dave speak + const bSpeakingInputMarbles = " ny"; + const dSpeakingInputMarbles = " ny"; + // Nothing should change when Bob or Dave initially speak, because they are + // on screen. When the screen becomes smaller Alice should move off screen + // to make way for the speakers (specifically, she should swap with Dave). + const expectedLayoutMarbles = " a-b"; + + withCallViewModel( + { + remoteParticipants$: constant([ + aliceParticipant, + bobParticipant, + daveParticipant, + ]), + rtcMembers$: constant([ + localRtcMember, + aliceRtcMember, + bobRtcMember, + daveRtcMember, + ]), + speaking: new Map([ + [bobParticipant, behavior(bSpeakingInputMarbles, yesNo)], + [daveParticipant, behavior(dSpeakingInputMarbles, yesNo)], + ]), + }, + (vm) => { + let setVisibleTiles: ((value: number) => void) | null = null; + vm.layout$.subscribe((layout) => { + if (layout.type === "grid") + setVisibleTiles = layout.setVisibleTiles; + }); + schedule(visibilityInputMarbles, { + a: () => setVisibleTiles!(Infinity), + b: () => setVisibleTiles!(3), + }); + + expectObservable(summarizeLayout$(vm.layout$)).toBe( + expectedLayoutMarbles, + { + a: { + type: "grid", + spotlight: undefined, + grid: [ + `${localId}:0`, + `${aliceId}:0`, + `${bobId}:0`, + `${daveId}:0`, + ], + }, + b: { + type: "grid", + spotlight: undefined, + grid: [ + `${localId}:0`, + `${daveId}:0`, + `${bobId}:0`, + `${aliceId}:0`, + ], + }, + }, + ); + }, + ); + }); + }); + + test("spotlight speakers swap places", () => { + withTestScheduler(({ behavior, schedule, expectObservable }) => { + // Go immediately into spotlight mode for the test + const modeInputMarbles = " s"; + // First Bob speaks, then Dave, then Alice + const aSpeakingInputMarbles = "n--y"; + const bSpeakingInputMarbles = "nyn"; + const dSpeakingInputMarbles = "n-yn"; + // Alice should start in the spotlight, then Bob, then Dave, then Alice + // again. However, the positions of Dave and Bob in the grid should be + // reversed by the end because they've been swapped in and out of the + // spotlight. + const expectedLayoutMarbles = "abcd"; + + withCallViewModel( + { + remoteParticipants$: constant([ + aliceParticipant, + bobParticipant, + daveParticipant, + ]), + rtcMembers$: constant([ + localRtcMember, + aliceRtcMember, + bobRtcMember, + daveRtcMember, + ]), + speaking: new Map([ + [aliceParticipant, behavior(aSpeakingInputMarbles, yesNo)], + [bobParticipant, behavior(bSpeakingInputMarbles, yesNo)], + [daveParticipant, behavior(dSpeakingInputMarbles, yesNo)], + ]), + }, + (vm) => { + schedule(modeInputMarbles, { s: () => vm.setGridMode("spotlight") }); + + expectObservable(summarizeLayout$(vm.layout$)).toBe( + expectedLayoutMarbles, + { + a: { + type: "spotlight-landscape", + spotlight: [`${aliceId}:0`], + grid: [`${localId}:0`, `${bobId}:0`, `${daveId}:0`], + }, + b: { + type: "spotlight-landscape", + spotlight: [`${bobId}:0`], + grid: [`${localId}:0`, `${aliceId}:0`, `${daveId}:0`], + }, + c: { + type: "spotlight-landscape", + spotlight: [`${daveId}:0`], + grid: [`${localId}:0`, `${aliceId}:0`, `${bobId}:0`], + }, + d: { + type: "spotlight-landscape", + spotlight: [`${aliceId}:0`], + grid: [`${localId}:0`, `${daveId}:0`, `${bobId}:0`], + }, + }, + ); + + // While we expect the media on tiles to change, layout$ itself should + // *never* meaningfully change. That is, we expect there to be no layout + // shifts as the spotlight speaker changes; instead, the same tiles + // should be reused for the whole duration of the test and simply have + // their media swapped out. This is meaningful for keeping the interface + // not too visually distracting during back-and-forth conversations, + // while still animating tiles to express people joining, leaving, etc. + expectObservable( + vm.layout$.pipe( + distinctUntilChanged(deepCompare), + debounceTime(0), + map(() => "x"), + ), + ).toBe("x"); // Expect just one emission + }, + ); + }); + }); + + test("layout enters picture-in-picture mode when requested", () => { + withTestScheduler(({ schedule, expectObservable }) => { + // Enable then disable picture-in-picture + const pipControlInputMarbles = "-ed"; + // Should go into picture-in-picture layout then back to grid + const expectedLayoutMarbles = " aba"; + + withCallViewModel( + { + remoteParticipants$: constant([aliceParticipant, bobParticipant]), + rtcMembers$: constant([localRtcMember, aliceRtcMember, bobRtcMember]), + }, + (vm) => { + schedule(pipControlInputMarbles, { + e: () => window.controls.enablePip(), + d: () => window.controls.disablePip(), + }); + + expectObservable(summarizeLayout$(vm.layout$)).toBe( + expectedLayoutMarbles, + { + a: { + type: "grid", + spotlight: undefined, + grid: [`${localId}:0`, `${aliceId}:0`, `${bobId}:0`], + }, + b: { + type: "pip", + spotlight: [`${aliceId}:0`], + }, + }, + ); + }, + ); + }); + }); + + test("PiP tile in expanded spotlight layout switches speakers without layout shifts", () => { + withTestScheduler(({ behavior, schedule, expectObservable }) => { + // Switch to spotlight immediately + const modeInputMarbles = " s"; + // And expand the spotlight immediately + const expandInputMarbles = " a"; + // First Bob speaks, then Dave, then Bob again + const bSpeakingInputMarbles = "n-yn--yn"; + const dSpeakingInputMarbles = "n---yn"; + // Should show Alice (presenter) in the PiP, then Bob, then Dave, then Bob + // again + const expectedLayoutMarbles = "a-b-c-b"; + + withCallViewModel( + { + remoteParticipants$: constant([ + aliceSharingScreen, + bobParticipant, + daveParticipant, + ]), + rtcMembers$: constant([ + localRtcMember, + aliceRtcMember, + bobRtcMember, + daveRtcMember, + ]), + speaking: new Map([ + [bobParticipant, behavior(bSpeakingInputMarbles, yesNo)], + [daveParticipant, behavior(dSpeakingInputMarbles, yesNo)], + ]), + }, + (vm) => { + schedule(modeInputMarbles, { + s: () => vm.setGridMode("spotlight"), + }); + schedule(expandInputMarbles, { + a: () => vm.toggleSpotlightExpanded$.value!(), + }); + + expectObservable(summarizeLayout$(vm.layout$)).toBe( + expectedLayoutMarbles, + { + a: { + type: "spotlight-expanded", + spotlight: [`${aliceId}:0:screen-share`], + pip: `${aliceId}:0`, + }, + b: { + type: "spotlight-expanded", + spotlight: [`${aliceId}:0:screen-share`], + pip: `${bobId}:0`, + }, + c: { + type: "spotlight-expanded", + spotlight: [`${aliceId}:0:screen-share`], + pip: `${daveId}:0`, + }, + }, + ); + + // While we expect the media on the PiP tile to change, layout$ itself + // should *never* meaningfully change. That is, we expect the same PiP + // tile to exist throughout the test and just have its media swapped out + // when the speaker changes, rather than for tiles to animate in/out. + // This is meaningful for keeping the interface not too visually + // distracting during back-and-forth conversations. + expectObservable( + vm.layout$.pipe( + distinctUntilChanged(deepCompare), + debounceTime(0), + map(() => "x"), + ), + ).toBe("x"); // Expect just one emission + }, + ); + }); + }); + + test("spotlight remembers whether it's expanded", () => { + withTestScheduler(({ schedule, expectObservable }) => { + // Start in spotlight mode, then switch to grid and back to spotlight a + // couple times + const modeInputMarbles = " s-gs-gs"; + // Expand and collapse the spotlight + const expandInputMarbles = " -a--a"; + // Spotlight should stay expanded during the first mode switch, and stay + // collapsed during the second mode switch + const expectedLayoutMarbles = "abcbada"; + + withCallViewModel( + { + remoteParticipants$: constant([aliceParticipant, bobParticipant]), + rtcMembers$: constant([localRtcMember, aliceRtcMember, bobRtcMember]), + }, + (vm) => { + schedule(modeInputMarbles, { + s: () => vm.setGridMode("spotlight"), + g: () => vm.setGridMode("grid"), + }); + schedule(expandInputMarbles, { + a: () => vm.toggleSpotlightExpanded$.value!(), + }); + + expectObservable(summarizeLayout$(vm.layout$)).toBe( + expectedLayoutMarbles, + { + a: { + type: "spotlight-landscape", + spotlight: [`${aliceId}:0`], + grid: [`${localId}:0`, `${bobId}:0`], + }, + b: { + type: "spotlight-expanded", + spotlight: [`${aliceId}:0`], + pip: `${localId}:0`, + }, + c: { + type: "grid", + spotlight: undefined, + grid: [`${localId}:0`, `${aliceId}:0`, `${bobId}:0`], + }, + d: { + type: "grid", + spotlight: undefined, + grid: [`${localId}:0`, `${bobId}:0`, `${aliceId}:0`], + }, + }, + ); + }, + ); + }); + }); + + test("participants must have a MatrixRTCSession to be visible", () => { + withTestScheduler(({ behavior, expectObservable }) => { + // iterate through a number of combinations of participants and MatrixRTC memberships + // Bob never has an MatrixRTC membership + const scenarioInputMarbles = " abcdec"; + // Bob should never be visible + const expectedLayoutMarbles = "a-bc-b"; + + withCallViewModel( + { + remoteParticipants$: behavior(scenarioInputMarbles, { + a: [], + b: [bobParticipant], + c: [aliceParticipant, bobParticipant], + d: [aliceParticipant, daveParticipant, bobParticipant], + e: [aliceParticipant, daveParticipant, bobSharingScreen], + }), + rtcMembers$: behavior(scenarioInputMarbles, { + a: [localRtcMember], + b: [localRtcMember], + c: [localRtcMember, aliceRtcMember], + d: [localRtcMember, aliceRtcMember, daveRtcMember], + e: [localRtcMember, aliceRtcMember, daveRtcMember], + }), + }, + (vm) => { + vm.setGridMode("grid"); + expectObservable(summarizeLayout$(vm.layout$)).toBe( + expectedLayoutMarbles, + { + a: { + type: "grid", + spotlight: undefined, + grid: [`${localId}:0`], + }, + b: { + type: "one-on-one", + local: `${localId}:0`, + remote: `${aliceId}:0`, + }, + c: { + type: "grid", + spotlight: undefined, + grid: [`${localId}:0`, `${aliceId}:0`, `${daveId}:0`], + }, + }, + ); + }, + ); + }); + }); + + it("should show at least one tile per MatrixRTCSession", () => { + withTestScheduler(({ behavior, expectObservable }) => { + // iterate through some combinations of MatrixRTC memberships + const scenarioInputMarbles = " abcd"; + // There should always be one tile for each MatrixRTCSession + const expectedLayoutMarbles = "abcd"; + + withCallViewModel( + { + rtcMembers$: behavior(scenarioInputMarbles, { + a: [localRtcMember], + b: [localRtcMember, aliceRtcMember], + c: [localRtcMember, aliceRtcMember, daveRtcMember], + d: [localRtcMember, daveRtcMember], + }), + }, + (vm) => { + vm.setGridMode("grid"); + expectObservable(summarizeLayout$(vm.layout$)).toBe( + expectedLayoutMarbles, + { + a: { + type: "grid", + spotlight: undefined, + grid: [`${localId}:0`], + }, + b: { + type: "one-on-one", + local: `${localId}:0`, + remote: `${aliceId}:0`, + }, + c: { + type: "grid", + spotlight: undefined, + grid: [`${localId}:0`, `${aliceId}:0`, `${daveId}:0`], + }, + d: { + type: "one-on-one", + local: `${localId}:0`, + remote: `${daveId}:0`, + }, + }, + ); + }, + ); + }); + }); + + it("should rank raised hands above video feeds and below speakers and presenters", () => { + withTestScheduler(({ schedule, expectObservable }) => { + // There should always be one tile for each MatrixRTCSession + const expectedLayoutMarbles = "ab"; + + withCallViewModel( + { + remoteParticipants$: constant([aliceParticipant, bobParticipant]), + rtcMembers$: constant([localRtcMember, aliceRtcMember, bobRtcMember]), + }, + (vm, _rtcSession, { raisedHands$ }) => { + schedule("ab", { + a: () => { + // We imagine that only two tiles (the first two) will be visible on screen at a time + vm.layout$.subscribe((layout) => { + if (layout.type === "grid") { + layout.setVisibleTiles(2); + } + }); + }, + b: () => { + raisedHands$.next({ + [`${bobRtcMember.userId}:${bobRtcMember.deviceId}`]: { + time: new Date(), + reactionEventId: "", + membershipEventId: "", + }, + }); + }, + }); + expectObservable(summarizeLayout$(vm.layout$)).toBe( + expectedLayoutMarbles, + { + a: { + type: "grid", + spotlight: undefined, + grid: [ + `${localId}:0`, + "@alice:example.org:AAAA:0", + "@bob:example.org:BBBB:0", + ], + }, + b: { + type: "grid", + spotlight: undefined, + grid: [ + `${localId}:0`, + // Bob shifts up! + "@bob:example.org:BBBB:0", + "@alice:example.org:AAAA:0", + ], + }, + }, + ); + }, + ); + }); + }); + + function nooneEverThere$( + behavior: (marbles: string, values: Record) => Behavior, + ): Behavior { + return behavior("a-b-c-d", { + a: [], // Start empty + b: [], // Alice joins + c: [], // Alice still there + d: [], // Alice leaves + }); + } + + function participantJoinLeave$( + behavior: ( + marbles: string, + values: Record, + ) => Behavior, + ): Behavior { + return behavior("a-b-c-d", { + a: [], // Start empty + b: [aliceParticipant], // Alice joins + c: [aliceParticipant], // Alice still there + d: [], // Alice leaves + }); + } + + function rtcMemberJoinLeave$( + behavior: ( + marbles: string, + values: Record, + ) => Behavior, + ): Behavior { + return behavior("a-b-c-d", { + a: [localRtcMember], // Start empty + b: [localRtcMember, aliceRtcMember], // Alice joins + c: [localRtcMember, aliceRtcMember], // Alice still there + d: [localRtcMember], // Alice leaves + }); + } + + test("autoLeave$ emits only when autoLeaveWhenOthersLeft option is enabled", () => { + withTestScheduler(({ behavior, expectObservable }) => { + withCallViewModel( + { + remoteParticipants$: participantJoinLeave$(behavior), + rtcMembers$: rtcMemberJoinLeave$(behavior), + }, + (vm) => { + expectObservable(vm.autoLeave$).toBe("------a", { + a: "allOthersLeft", + }); + }, + { + autoLeaveWhenOthersLeft: true, + encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + }, + ); + }); + }); + + test("autoLeave$ never emits autoLeaveWhenOthersLeft option is enabled but no-one is there", () => { + withTestScheduler(({ behavior, expectObservable }) => { + withCallViewModel( + { + remoteParticipants$: nooneEverThere$(behavior), + rtcMembers$: nooneEverThere$(behavior), + }, + (vm) => { + expectObservable(vm.autoLeave$).toBe("-"); + }, + { + autoLeaveWhenOthersLeft: true, + encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + }, + ); + }); + }); + + test("autoLeave$ doesn't emit when autoLeaveWhenOthersLeft option is disabled and all others left", () => { + withTestScheduler(({ behavior, expectObservable }) => { + withCallViewModel( + { + remoteParticipants$: participantJoinLeave$(behavior), + rtcMembers$: rtcMemberJoinLeave$(behavior), + }, + (vm) => { + expectObservable(vm.autoLeave$).toBe("-"); + }, + { + autoLeaveWhenOthersLeft: false, + encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + }, + ); + }); + }); + + test("autoLeave$ emits when autoLeaveWhenOthersLeft option is enabled and all others left", () => { + withTestScheduler(({ behavior, expectObservable }) => { + withCallViewModel( + { + remoteParticipants$: behavior("a-b-c-d", { + a: [], // Alone + b: [aliceParticipant], // Alice joins + c: [aliceParticipant], + d: [], // Local joins with a second device + }), + rtcMembers$: behavior("a-b-c-d", { + a: [localRtcMember], // Start empty + b: [localRtcMember, aliceRtcMember], // Alice joins + c: [localRtcMember, aliceRtcMember, localRtcMemberDevice2], // Alice still there + d: [localRtcMember, localRtcMemberDevice2], // The second Alice leaves + }), + }, + (vm) => { + expectObservable(vm.autoLeave$).toBe("------a", { + a: "allOthersLeft", + }); + }, + { + autoLeaveWhenOthersLeft: true, + encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + }, + ); + }); + }); + + describe("waitForCallPickup$", () => { + it.skip("regression test: does stop ringing in case livekitConnectionState$ emits after didSendCallNotification$ has already emitted", () => { + withTestScheduler(({ schedule, expectObservable, behavior }) => { + withCallViewModel( + { + livekitConnectionState$: behavior("d 9ms c", { + d: ConnectionState.Disconnected, + c: ConnectionState.Connected, + }), + }, + (vm, rtcSession) => { + // Fire a call notification IMMEDIATELY (its important for this test, that this happens before the livekitConnectionState$ emits) + schedule("n", { + n: () => { + rtcSession.emit( + MatrixRTCSessionEvent.DidSendCallNotification, + mockRingEvent("$notif1", 30), + mockLegacyRingEvent, + ); + }, + }); + + expectObservable(vm.callPickupState$).toBe("a 9ms b 19ms c", { + a: "unknown", + b: "ringing", + c: "timeout", + }); + }, + { + waitForCallPickup: true, + encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + }, + ); + }); + }); + + it.skip("ringing -> unknown if we get disconnected", () => { + withTestScheduler(({ behavior, schedule, expectObservable }) => { + const connectionState$ = new BehaviorSubject(ConnectionState.Connected); + // Someone joins at 20ms (both LiveKit participant and MatrixRTC member) + withCallViewModel( + { + remoteParticipants$: behavior("a 19ms b", { + a: [], + b: [aliceParticipant], + }), + rtcMembers$: behavior("a 19ms b", { + a: [localRtcMember], + b: [localRtcMember, aliceRtcMember], + }), + livekitConnectionState$: connectionState$, + }, + (vm, rtcSession) => { + // Notify at 5ms so we enter ringing, then get disconnected 5ms later + schedule(" 5ms r 5ms d", { + r: () => { + rtcSession.emit( + MatrixRTCSessionEvent.DidSendCallNotification, + mockRingEvent("$notif2", 100), + mockLegacyRingEvent, + ); + }, + d: () => { + connectionState$.next(ConnectionState.Disconnected); + }, + }); + + expectObservable(vm.callPickupState$).toBe("a 4ms b 5ms c", { + a: "unknown", + b: "ringing", + c: "unknown", + }); + }, + { + waitForCallPickup: true, + encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + }, + ); + }); + }); + }); + + it.skip("audio output changes when toggling earpiece mode", () => { + withTestScheduler(({ schedule, expectObservable }) => { + getUrlParams.mockReturnValue({ controlledAudioDevices: true }); + vi.mocked(ComponentsCore.createMediaDeviceObserver).mockReturnValue( + of([]), + ); + + const devices = new MediaDevices(testScope()); + + window.controls.setAvailableAudioDevices([ + { id: "speaker", name: "Speaker", isSpeaker: true }, + { id: "earpiece", name: "Handset", isEarpiece: true }, + { id: "headphones", name: "Headphones" }, + ]); + window.controls.setAudioDevice("headphones"); + + const toggleInputMarbles = " -aaa"; + const expectedEarpieceModeMarbles = "n-yn"; + const expectedTargetStateMarbles = " sese"; + + withCallViewModel({ mediaDevices: devices }, (vm) => { + schedule(toggleInputMarbles, { + a: () => getValue(vm.audioOutputSwitcher$)?.switch(), + }); + expectObservable(vm.earpieceMode$).toBe( + expectedEarpieceModeMarbles, + yesNo, + ); + expectObservable( + vm.audioOutputSwitcher$.pipe( + map((switcher) => switcher?.targetOutput), + ), + ).toBe(expectedTargetStateMarbles, { s: "speaker", e: "earpiece" }); + }); + }); + }); + + it.skip("media tracks are paused while reconnecting to MatrixRTC", () => { + withTestScheduler(({ schedule, expectObservable }) => { + const trackRunning$ = new BehaviorSubject(true); + const originalPublications = localParticipant.trackPublications; + localParticipant.trackPublications = new Map([ + [ + "video", + { + track: new (class { + public get isUpstreamPaused(): boolean { + return !trackRunning$.value; + } + public async pauseUpstream(): Promise { + trackRunning$.next(false); + return Promise.resolve(); + } + public async resumeUpstream(): Promise { + trackRunning$.next(true); + return Promise.resolve(); + } + })(), + } as unknown as LocalTrackPublication, + ], + ]); + onTestFinished(() => { + localParticipant.trackPublications = originalPublications; + }); + + // There are three indicators that the client might be disconnected from + // MatrixRTC: whether the sync loop is connected, whether the membership is + // present in local room state, and whether the membership manager thinks + // we've hit the timeout for the delayed leave event. Let's test all + // combinations of these conditions. + const syncingMarbles = " nyny----n--y"; + const membershipStatusMarbles = " y---ny-n-yn-y"; + const probablyLeftMarbles = " n-----y-ny---n"; + const expectedReconnectingMarbles = "n-ynyny------n"; + const expectedTrackRunningMarbles = "nynynyn------y"; + + withCallViewModel( + { initialSyncState: SyncState.Reconnecting }, + (vm, rtcSession, _subjects, setSyncState) => { + schedule(syncingMarbles, { + y: () => setSyncState(SyncState.Syncing), + n: () => setSyncState(SyncState.Reconnecting), + }); + schedule(membershipStatusMarbles, { + y: () => { + rtcSession.membershipStatus = Status.Connected; + }, + n: () => { + rtcSession.membershipStatus = Status.Reconnecting; + }, + }); + schedule(probablyLeftMarbles, { + y: () => { + rtcSession.probablyLeft = true; + }, + n: () => { + rtcSession.probablyLeft = false; + }, + }); + expectObservable(vm.reconnecting$).toBe( + expectedReconnectingMarbles, + yesNo, + ); + expectObservable(trackRunning$).toBe( + expectedTrackRunningMarbles, + yesNo, + ); + }, + ); + }); + }); +}); diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts new file mode 100644 index 00000000..eb270641 --- /dev/null +++ b/src/state/CallViewModel/CallViewModel.ts @@ -0,0 +1,1495 @@ +/* +Copyright 2025 Element Creations Ltd. +Copyright 2023, 2024, 2025 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + type BaseKeyProvider, + type ConnectionState, + ExternalE2EEKeyProvider, + type Room as LivekitRoom, + type RoomOptions, +} from "livekit-client"; +import { type Room as MatrixRoom } from "matrix-js-sdk"; +import { + combineLatest, + distinctUntilChanged, + EMPTY, + filter, + fromEvent, + map, + merge, + NEVER, + type Observable, + of, + pairwise, + race, + scan, + skip, + skipWhile, + startWith, + Subject, + switchAll, + switchMap, + switchScan, + take, + tap, + throttleTime, + timer, +} from "rxjs"; +import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; +import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc"; +import { type IWidgetApiRequest } from "matrix-widget-api"; + +import { + LocalUserMediaViewModel, + type MediaViewModel, + type RemoteUserMediaViewModel, + ScreenShareViewModel, + type UserMediaViewModel, +} from "../MediaViewModel"; +import { accumulate, generateItems, pauseWhen } from "../../utils/observable"; +import { + duplicateTiles, + MatrixRTCMode, + matrixRTCMode, + playReactionsSound, + showReactions, +} from "../../settings/settings"; +import { isFirefox } from "../../Platform"; +import { setPipEnabled$ } from "../../controls"; +import { TileStore } from "../TileStore"; +import { gridLikeLayout } from "../GridLikeLayout"; +import { spotlightExpandedLayout } from "../SpotlightExpandedLayout"; +import { oneOnOneLayout } from "../OneOnOneLayout"; +import { pipLayout } from "../PipLayout"; +import { type EncryptionSystem } from "../../e2ee/sharedKeyManagement"; +import { + type RaisedHandInfo, + type ReactionInfo, + type ReactionOption, +} from "../../reactions"; +import { shallowEquals } from "../../utils/array"; +import { type MediaDevices } from "../MediaDevices"; +import { type Behavior } from "../Behavior"; +import { E2eeType } from "../../e2ee/e2eeType"; +import { MatrixKeyProvider } from "../../e2ee/matrixKeyProvider"; +import { type MuteStates } from "../MuteStates"; +import { getUrlParams } from "../../UrlParams"; +import { type ProcessorState } from "../../livekit/TrackProcessorContext"; +import { ElementWidgetActions, widget } from "../../widget"; +import { UserMedia } from "../UserMedia.ts"; +import { ScreenShare } from "../ScreenShare.ts"; +import { + type GridLayoutMedia, + type Layout, + type LayoutMedia, + type OneOnOneLayoutMedia, + type SpotlightExpandedLayoutMedia, + type SpotlightLandscapeLayoutMedia, + type SpotlightPortraitLayoutMedia, +} from "../layout-types.ts"; +import { type ElementCallError } from "../../utils/errors.ts"; +import { type ObservableScope } from "../ObservableScope.ts"; +import { + createLocalMembership$, + type LocalMemberConnectionState, +} from "./localMember/LocalMembership.ts"; +import { createLocalTransport$ } from "./localMember/LocalTransport.ts"; +import { + createMemberships$, + membershipsAndTransports$, +} from "../SessionBehaviors.ts"; +import { ECConnectionFactory } from "./remoteMembers/ConnectionFactory.ts"; +import { createConnectionManager$ } from "./remoteMembers/ConnectionManager.ts"; +import { + createMatrixLivekitMembers$, + type MatrixLivekitMember, +} from "./remoteMembers/MatrixLivekitMembers.ts"; +import { + type AutoLeaveReason, + createCallNotificationLifecycle$, + createReceivedDecline$, + createSentCallNotification$, +} from "./CallNotificationLifecycle.ts"; +import { + createDMMember$, + createMatrixMemberMetadata$, + createRoomMembers$, +} from "./remoteMembers/MatrixMemberMetadata.ts"; + +const logger = rootLogger.getChild("[CallViewModel]"); +//TODO +// Larger rename +// member,membership -> rtcMember +// participant -> livekitParticipant +// matrixLivekitItem -> callMember +// js-sdk +// callMembership -> rtcMembership +export interface CallViewModelOptions { + encryptionSystem: EncryptionSystem; + autoLeaveWhenOthersLeft?: boolean; + /** + * If the call is started in a way where we want it to behave like a telephone usecase + * If we sent a notification event, we want the ui to show a ringing state + */ + waitForCallPickup?: boolean; + /** Optional factory to create LiveKit rooms, mainly for testing purposes. */ + livekitRoomFactory?: (options?: RoomOptions) => LivekitRoom; + /** Optional behavior overriding the local connection state, mainly for testing purposes. */ + connectionState$?: Behavior; +} + +// Do not play any sounds if the participant count has exceeded this +// number. +export const MAX_PARTICIPANT_COUNT_FOR_SOUND = 8; +export const THROTTLE_SOUND_EFFECT_MS = 500; + +// This is the number of participants that we think constitutes a "small" call +// on mobile. No spotlight tile should be shown below this threshold. +const smallMobileCallThreshold = 3; + +// How long the footer should be shown for when hovering over or interacting +// with the interface +const showFooterMs = 4000; + +export type GridMode = "grid" | "spotlight"; + +export type WindowMode = "normal" | "narrow" | "flat" | "pip"; + +interface LayoutScanState { + layout: Layout | null; + tiles: TileStore; +} + +type MediaItem = UserMedia | ScreenShare; +type AudioLivekitItem = { + livekitRoom: LivekitRoom; + participants: string[]; + url: string; +}; + +/** + * The return of createCallViewModel$ + * this interface represents the root source of data for the call view. + * They are a list of observables and objects containing observables to allow for a very granular update mechanism. + * + * This allows to have one huge call view model that represents the entire view without a unnecessary amount of updates. + * + * (Mocking this interface should allow building a full view in all states.) + */ +export interface CallViewModel { + // lifecycle + autoLeave$: Observable; + // TODO if we are in "unknown" state we need a loading rendering (or empty screen) + // Otherwise it looks like we already connected and only than the ringing starts which is weird. + callPickupState$: Behavior< + "unknown" | "ringing" | "timeout" | "decline" | "success" | null + >; + leave$: Observable<"user" | AutoLeaveReason>; + /** Call to initiate hangup. Use in conbination with connectino state track the async hangup process. */ + hangup: () => void; + + // joining + join: () => LocalMemberConnectionState; + + // screen sharing + /** + * Callback to toggle screen sharing. If null, screen sharing is not possible. + */ + toggleScreenSharing: (() => void) | null; + /** + * Whether we are sharing our screen. + */ + sharingScreen$: Behavior; + + // UI interactions + /** + * Callback for when the user taps the call view. + */ + tapScreen: () => void; + /** + * Callback for when the user taps the call's controls. + */ + tapControls: () => void; + /** + * Callback for when the user hovers over the call view. + */ + hoverScreen: () => void; + /** + * Callback for when the user stops hovering over the call view. + */ + unhoverScreen: () => void; + + // errors + /** + * If there is a configuration error with the call (e.g. misconfigured E2EE). + * This is a fatal error that prevents the call from being created/joined. + * Should render a blocking error screen. + */ + configError$: Behavior; + + // participants and counts + /** + * The number of participants currently in the call. + * + * - Each participant has a corresponding MatrixRTC membership state event + * - There can be multiple participants for one Matrix user if they join from + * multiple devices. + */ + participantCount$: Behavior; + /** Participants sorted by livekit room so they can be used in the audio rendering */ + audioParticipants$: Behavior; + /** List of participants raising their hand */ + handsRaised$: Behavior>; + /** List of reactions. Keys are: membership.membershipId (currently predefined as: `${membershipEvent.userId}:${membershipEvent.deviceId}`)*/ + reactions$: Behavior>; + + ringOverlay$: Behavior; + // sounds and events + joinSoundEffect$: Observable; + leaveSoundEffect$: Observable; + /** + * Emits an event every time a new hand is raised in + * the call. + */ + newHandRaised$: Observable<{ value: number; playSounds: boolean }>; + /** + * Emits an event every time a new screenshare is started in + * the call. + */ + newScreenShare$: Observable<{ value: number; playSounds: boolean }>; + /** + * Emits an array of reactions that should be played. + */ + audibleReactions$: Observable; + /** + * Emits an array of reactions that should be visible on the screen. + */ + // DISCUSSION move this into a reaction file + visibleReactions$: Behavior< + { sender: string; emoji: string; startX: number }[] + >; + + // window/layout + /** + * The general shape of the window. + */ + windowMode$: Behavior; + spotlightExpanded$: Behavior; + toggleSpotlightExpanded$: Behavior<(() => void) | null>; + gridMode$: Behavior; + setGridMode: (value: GridMode) => void; + + // media view models and layout + grid$: Behavior; + spotlight$: Behavior; + pip$: Behavior; + /** + * The layout of tiles in the call interface. + */ + layout$: Behavior; + /** + * The current generation of the tile store, exposed for debugging purposes. + */ + tileStoreGeneration$: Behavior; + showSpotlightIndicators$: Behavior; + showSpeakingIndicators$: Behavior; + + // header/footer visibility + showHeader$: Behavior; + showFooter$: Behavior; + + // audio routing + /** + * Whether audio is currently being output through the earpiece. + */ + earpieceMode$: Behavior; + /** + * Callback to toggle between the earpiece and the loudspeaker. + * + * This will be `null` in case the target does not exist in the list + * of available audio outputs. + */ + audioOutputSwitcher$: Behavior<{ + targetOutput: "earpiece" | "speaker"; + switch: () => void; + } | null>; + + // connection state + /** + * Whether various media/event sources should pretend to be disconnected from + * all network input, even if their connection still technically works. + */ + // We do this when the app is in the 'reconnecting' state, because it might be + // that the LiveKit connection is still functional while the homeserver is + // down, for example, and we want to avoid making people worry that the app is + // in a split-brained state. + // DISCUSSION own membership manager ALSO this probably can be simplifis + reconnecting$: Behavior; +} +/** + * A view model providing all the application logic needed to show the in-call + * UI (may eventually be expanded to cover the lobby and feedback screens in the + * future). + */ +// Throughout this class and related code we must distinguish between MatrixRTC +// state and LiveKit state. We use the common terminology of room "members", RTC +// "memberships", and LiveKit "participants". +export function createCallViewModel$( + scope: ObservableScope, + // A call is permanently tied to a single Matrix room + matrixRTCSession: MatrixRTCSession, + matrixRoom: MatrixRoom, + mediaDevices: MediaDevices, + muteStates: MuteStates, + options: CallViewModelOptions, + handsRaisedSubject$: Observable>, + reactionsSubject$: Observable>, + trackProcessorState$: Behavior, +): CallViewModel { + const userId = matrixRoom.client.getUserId()!; + const deviceId = matrixRoom.client.getDeviceId()!; + + const livekitKeyProvider = getE2eeKeyProvider( + options.encryptionSystem, + matrixRTCSession, + ); + + // Each hbar seperates a block of input variables required for the CallViewModel to function. + // The outputs of this block is written under the hbar. + // + // For mocking purposes it is recommended to only mock the functions creating those outputs. + // All other fields are just temp computations for the mentioned output. + // The class does not need anything except the values underneath the bar. + // The creations of the values under the bar are all tested independently and testing the callViewModel Should + // not test their creation. Call view model only needs: + // - memberships$ via createMemberships$ + // - localMembership via createLocalMembership$ + // - callLifecycle via createCallNotificationLifecycle$ + // - matrixMemberMetadataStore via createMatrixMemberMetadata$ + + // ------------------------------------------------------------------------ + // memberships$ + const memberships$ = createMemberships$(scope, matrixRTCSession); + + // ------------------------------------------------------------------------ + // matrixLivekitMembers$ AND localMembership + + const membershipsAndTransports = membershipsAndTransports$( + scope, + memberships$, + ); + + const localTransport$ = createLocalTransport$({ + scope: scope, + memberships$: memberships$, + client: matrixRoom.client, + roomId: matrixRoom.roomId, + useOldestMember$: scope.behavior( + matrixRTCMode.value$.pipe(map((v) => v === MatrixRTCMode.Legacy)), + ), + }); + + const connectionFactory = new ECConnectionFactory( + matrixRoom.client, + mediaDevices, + trackProcessorState$, + livekitKeyProvider, + getUrlParams().controlledAudioDevices, + options.livekitRoomFactory, + ); + + const connectionManager = createConnectionManager$({ + scope: scope, + connectionFactory: connectionFactory, + inputTransports$: scope.behavior( + combineLatest( + [localTransport$, membershipsAndTransports.transports$], + (localTransport, transports) => { + const localTransportAsArray = localTransport ? [localTransport] : []; + return transports.mapInner((transports) => [ + ...localTransportAsArray, + ...transports, + ]); + }, + ), + ), + logger: logger, + }); + + const matrixLivekitMembers$ = createMatrixLivekitMembers$({ + scope: scope, + membershipsWithTransport$: + membershipsAndTransports.membershipsWithTransport$, + connectionManager: connectionManager, + }); + + const connectOptions$ = scope.behavior( + matrixRTCMode.value$.pipe( + map((mode) => ({ + encryptMedia: livekitKeyProvider !== undefined, + // TODO. This might need to get called again on each change of matrixRTCMode... + matrixRTCMode: mode, + })), + ), + ); + + const localMembership = createLocalMembership$({ + scope: scope, + muteStates: muteStates, + mediaDevices: mediaDevices, + connectionManager: connectionManager, + matrixRTCSession: matrixRTCSession, + matrixRoom: matrixRoom, + localTransport$: localTransport$, + trackProcessorState$: trackProcessorState$, + widget, + options: connectOptions$, + logger: logger.getChild(`[${Date.now()}]`), + }); + + const localRtcMembership$ = scope.behavior( + memberships$.pipe( + map( + (memberships) => + memberships.value.find( + (membership) => + membership.userId === userId && membership.deviceId === deviceId, + ) ?? null, + ), + ), + ); + + const localMatrixLivekitMemberUninitialized = { + membership$: localRtcMembership$, + participant$: localMembership.participant$, + connection$: localMembership.connection$, + userId: userId, + }; + + const localMatrixLivekitMember$: Behavior = + scope.behavior( + localRtcMembership$.pipe( + switchMap((membership) => { + if (!membership) return of(null); + return of( + // casting is save here since we know that localRtcMembership$ is !== null since we reached this case. + localMatrixLivekitMemberUninitialized as MatrixLivekitMember, + ); + }), + ), + ); + + // ------------------------------------------------------------------------ + // callLifecycle + + // TODO if we are in "unknown" state we need a loading rendering (or empty screen) + // Otherwise it looks like we already connected and only than the ringing starts which is weird. + const { callPickupState$, autoLeave$ } = createCallNotificationLifecycle$({ + scope: scope, + memberships$: memberships$, + sentCallNotification$: createSentCallNotification$(scope, matrixRTCSession), + receivedDecline$: createReceivedDecline$(matrixRoom), + options: options, + localUser: { userId: userId, deviceId: deviceId }, + }); + + // ------------------------------------------------------------------------ + // matrixMemberMetadataStore + + const matrixRoomMembers$ = createRoomMembers$(scope, matrixRoom); + const matrixMemberMetadataStore = createMatrixMemberMetadata$( + scope, + scope.behavior(memberships$.pipe(map((mems) => mems.value))), + matrixRoomMembers$, + ); + + const dmMember$ = createDMMember$(scope, matrixRoomMembers$, matrixRoom); + const noUserToCallInRoom$ = scope.behavior( + matrixRoomMembers$.pipe( + map( + (roomMembersMap) => + roomMembersMap.size === 1 && roomMembersMap.get(userId) !== undefined, + ), + ), + ); + + const ringOverlay$ = scope.behavior( + combineLatest([noUserToCallInRoom$, dmMember$, callPickupState$]).pipe( + map(([noUserToCallInRoom, dmMember, callPickupState]) => { + // No overlay if not in ringing state + if (callPickupState !== "ringing" || noUserToCallInRoom) return null; + + const name = dmMember ? dmMember.rawDisplayName : matrixRoom.name; + const id = dmMember ? dmMember.userId : matrixRoom.roomId; + const text = dmMember + ? `Waiting for ${name} to join…` + : "Waiting for other participants…"; + const avatarMxc = dmMember + ? (dmMember.getMxcAvatarUrl?.() ?? undefined) + : (matrixRoom.getMxcAvatarUrl() ?? undefined); + return { + name: name ?? id, + idForAvatar: id, + text, + avatarMxc, + }; + }), + ), + ); + + // CODESMELL? + // This is functionally the same Observable as leave$, except here it's + // hoisted to the top of the class. This enables the cyclic dependency between + // leave$ -> autoLeave$ -> callPickupState$ -> livekitConnectionState$ -> + // localConnection$ -> transports$ -> joined$ -> leave$. + const leaveHoisted$ = new Subject< + "user" | "timeout" | "decline" | "allOthersLeft" + >(); + + /** + * Whether various media/event sources should pretend to be disconnected from + * all network input, even if their connection still technically works. + */ + // We do this when the app is in the 'reconnecting' state, because it might be + // that the LiveKit connection is still functional while the homeserver is + // down, for example, and we want to avoid making people worry that the app is + // in a split-brained state. + // DISCUSSION own membership manager ALSO this probably can be simplifis + const reconnecting$ = localMembership.reconnecting$; + const pretendToBeDisconnected$ = reconnecting$; + + const audioParticipants$ = scope.behavior( + matrixLivekitMembers$.pipe( + switchMap((membersWithEpoch) => { + const members = membersWithEpoch.value; + const a$ = combineLatest( + members.map((member) => + combineLatest([member.connection$, member.participant$]).pipe( + map(([connection, participant]) => { + // do not render audio for local participant + if (!connection || !participant || participant.isLocal) + return null; + const livekitRoom = connection.livekitRoom; + const url = connection.transport.livekit_service_url; + + return { + url, + livekitRoom, + participant: participant.identity, + }; + }), + ), + ), + ); + return a$; + }), + map((members) => + members.reduce((acc, curr) => { + if (!curr) return acc; + + const existing = acc.find((item) => item.url === curr.url); + if (existing) { + existing.participants.push(curr.participant); + } else { + acc.push({ + livekitRoom: curr.livekitRoom, + participants: [curr.participant], + url: curr.url, + }); + } + return acc; + }, []), + ), + ), + [], + ); + + const handsRaised$ = scope.behavior( + handsRaisedSubject$.pipe(pauseWhen(pretendToBeDisconnected$)), + ); + + const reactions$ = scope.behavior( + reactionsSubject$.pipe( + map((v) => + Object.fromEntries( + Object.entries(v).map(([a, { reactionOption }]) => [ + a, + reactionOption, + ]), + ), + ), + pauseWhen(pretendToBeDisconnected$), + ), + ); + + /** + * List of user media (camera feeds) that we want tiles for. + */ + const userMedia$ = scope.behavior( + combineLatest([ + localMatrixLivekitMember$, + matrixLivekitMembers$, + duplicateTiles.value$, + ]).pipe( + // Generate a collection of MediaItems from the list of expected (whether + // present or missing) LiveKit participants. + generateItems( + function* ([ + localMatrixLivekitMember, + { value: matrixLivekitMembers }, + duplicateTiles, + ]) { + let localParticipantId = undefined; + // add local member if available + if (localMatrixLivekitMember) { + const { userId, participant$, connection$, membership$ } = + localMatrixLivekitMember; + localParticipantId = `${userId}:${membership$.value.deviceId}`; // should be membership$.value.membershipID which is not optional + // const participantId = membership$.value.membershipID; + if (localParticipantId) { + for (let dup = 0; dup < 1 + duplicateTiles; dup++) { + yield { + keys: [ + dup, + localParticipantId, + userId, + participant$, + connection$, + ], + data: undefined, + }; + } + } + } + // add remote members that are available + for (const { + userId, + participant$, + connection$, + membership$, + } of matrixLivekitMembers) { + const participantId = `${userId}:${membership$.value.deviceId}`; + if (participantId === localParticipantId) continue; + // const participantId = membership$.value?.identity; + for (let dup = 0; dup < 1 + duplicateTiles; dup++) { + yield { + keys: [dup, participantId, userId, participant$, connection$], + data: undefined, + }; + } + } + }, + ( + scope, + _data$, + dup, + participantId, + userId, + participant$, + connection$, + ) => { + const livekitRoom$ = scope.behavior( + connection$.pipe(map((c) => c?.livekitRoom)), + ); + const focusUrl$ = scope.behavior( + connection$.pipe(map((c) => c?.transport.livekit_service_url)), + ); + const displayName$ = scope.behavior( + matrixMemberMetadataStore + .createDisplayNameBehavior$(userId) + .pipe(map((name) => name ?? userId)), + ); + + return new UserMedia( + scope, + `${participantId}:${dup}`, + userId, + participant$, + options.encryptionSystem, + livekitRoom$, + focusUrl$, + mediaDevices, + pretendToBeDisconnected$, + displayName$, + matrixMemberMetadataStore.createAvatarUrlBehavior$(userId), + handsRaised$.pipe(map((v) => v[participantId]?.time ?? null)), + reactions$.pipe(map((v) => v[participantId] ?? undefined)), + ); + }, + ), + ), + ); + + /** + * List of all media items (user media and screen share media) that we want + * tiles for. + */ + const mediaItems$ = scope.behavior( + userMedia$.pipe( + switchMap((userMedia) => + userMedia.length === 0 + ? of([]) + : combineLatest( + userMedia.map((m) => m.screenShares$), + (...screenShares) => [...userMedia, ...screenShares.flat(1)], + ), + ), + ), + ); + + /** + * List of MediaItems that we want to display, that are of type ScreenShare + */ + const screenShares$ = scope.behavior( + mediaItems$.pipe( + map((mediaItems) => + mediaItems.filter((m): m is ScreenShare => m instanceof ScreenShare), + ), + ), + ); + + const joinSoundEffect$ = userMedia$.pipe( + pairwise(), + filter( + ([prev, current]) => + current.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && + current.length > prev.length, + ), + map(() => {}), + throttleTime(THROTTLE_SOUND_EFFECT_MS), + ); + + /** + * The number of participants currently in the call. + * + * - Each participant has a corresponding MatrixRTC membership state event + * - There can be multiple participants for one Matrix user if they join from + * multiple devices. + */ + const participantCount$ = scope.behavior( + matrixLivekitMembers$.pipe(map((ms) => ms.value.length)), + ); + + const leaveSoundEffect$ = combineLatest([callPickupState$, userMedia$]).pipe( + // Until the call is successful, do not play a leave sound. + // If callPickupState$ is null, then we always play the sound as it will not conflict with a decline sound. + skipWhile(([c]) => c !== null && c !== "success"), + map(([, userMedia]) => userMedia), + pairwise(), + filter( + ([prev, current]) => + current.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && + current.length < prev.length, + ), + map(() => {}), + throttleTime(THROTTLE_SOUND_EFFECT_MS), + ); + + const userHangup$ = new Subject(); + + const widgetHangup$ = + widget === null + ? NEVER + : ( + fromEvent( + widget.lazyActions, + ElementWidgetActions.HangupCall, + ) as Observable> + ).pipe( + tap((ev) => { + widget!.api.transport.reply(ev.detail, {}); + }), + ); + + const leave$: Observable<"user" | "timeout" | "decline" | "allOthersLeft"> = + merge( + autoLeave$, + merge(userHangup$, widgetHangup$).pipe(map(() => "user" as const)), + ).pipe( + scope.share, + tap((reason) => leaveHoisted$.next(reason)), + ); + + const spotlightSpeaker$ = scope.behavior( + userMedia$.pipe( + switchMap((mediaItems) => + mediaItems.length === 0 + ? of([]) + : combineLatest( + mediaItems.map((m) => + m.vm.speaking$.pipe(map((s) => [m, s] as const)), + ), + ), + ), + scan<(readonly [UserMedia, boolean])[], UserMedia | undefined, null>( + (prev, mediaItems) => { + // Only remote users that are still in the call should be sticky + const [stickyMedia, stickySpeaking] = + (!prev?.vm.local && mediaItems.find(([m]) => m === prev)) || []; + // Decide who to spotlight: + // If the previous speaker is still speaking, stick with them rather + // than switching eagerly to someone else + return stickySpeaking + ? stickyMedia! + : // Otherwise, select any remote user who is speaking + (mediaItems.find(([m, s]) => !m.vm.local && s)?.[0] ?? + // Otherwise, stick with the person who was last speaking + stickyMedia ?? + // Otherwise, spotlight an arbitrary remote user + mediaItems.find(([m]) => !m.vm.local)?.[0] ?? + // Otherwise, spotlight the local user + mediaItems.find(([m]) => m.vm.local)?.[0]); + }, + null, + ), + map((speaker) => speaker?.vm ?? null), + ), + ); + + const grid$ = scope.behavior( + userMedia$.pipe( + switchMap((mediaItems) => { + const bins = mediaItems.map((m) => + m.bin$.pipe(map((bin) => [m, bin] as const)), + ); + // Sort the media by bin order and generate a tile for each one + return bins.length === 0 + ? of([]) + : combineLatest(bins, (...bins) => + bins.sort(([, bin1], [, bin2]) => bin1 - bin2).map(([m]) => m.vm), + ); + }), + distinctUntilChanged(shallowEquals), + ), + ); + + const spotlight$ = scope.behavior( + screenShares$.pipe( + switchMap((screenShares) => { + if (screenShares.length > 0) { + return of(screenShares.map((m) => m.vm)); + } + + return spotlightSpeaker$.pipe( + map((speaker) => (speaker ? [speaker] : [])), + ); + }), + distinctUntilChanged(shallowEquals), + ), + ); + + const pip$ = scope.behavior( + combineLatest([ + // TODO This also needs epoch logic to dedupe the screenshares and mediaItems emits + screenShares$, + spotlightSpeaker$, + mediaItems$, + ]).pipe( + switchMap(([screenShares, spotlight, mediaItems]) => { + if (screenShares.length > 0) { + return spotlightSpeaker$; + } + if (!spotlight || spotlight.local) { + return of(null); + } + + const localUserMedia = mediaItems.find( + (m) => m.vm instanceof LocalUserMediaViewModel, + ) as UserMedia | undefined; + + const localUserMediaViewModel = localUserMedia?.vm as + | LocalUserMediaViewModel + | undefined; + + if (!localUserMediaViewModel) { + return of(null); + } + return localUserMediaViewModel.alwaysShow$.pipe( + map((alwaysShow) => { + if (alwaysShow) { + return localUserMediaViewModel; + } + + return null; + }), + ); + }), + ), + ); + + const hasRemoteScreenShares$: Observable = spotlight$.pipe( + map((spotlight) => + spotlight.some((vm) => !vm.local && vm instanceof ScreenShareViewModel), + ), + distinctUntilChanged(), + ); + + const pipEnabled$ = scope.behavior(setPipEnabled$, false); + + const naturalWindowMode$ = scope.behavior( + fromEvent(window, "resize").pipe( + map(() => { + const height = window.innerHeight; + const width = window.innerWidth; + if (height <= 400 && width <= 340) return "pip"; + // Our layouts for flat windows are better at adapting to a small width + // than our layouts for narrow windows are at adapting to a small height, + // so we give "flat" precedence here + if (height <= 600) return "flat"; + if (width <= 600) return "narrow"; + return "normal"; + }), + ), + "normal", + ); + + /** + * The general shape of the window. + */ + const windowMode$ = scope.behavior( + pipEnabled$.pipe( + switchMap((pip) => (pip ? of("pip") : naturalWindowMode$)), + ), + ); + + const spotlightExpandedToggle$ = new Subject(); + const spotlightExpanded$ = scope.behavior( + spotlightExpandedToggle$.pipe(accumulate(false, (expanded) => !expanded)), + ); + + const gridModeUserSelection$ = new Subject(); + /** + * The layout mode of the media tile grid. + */ + const gridMode$ = + // If the user hasn't selected spotlight and somebody starts screen sharing, + // automatically switch to spotlight mode and reset when screen sharing ends + scope.behavior( + gridModeUserSelection$.pipe( + switchMap((userSelection) => + (userSelection === "spotlight" + ? EMPTY + : combineLatest([hasRemoteScreenShares$, windowMode$]).pipe( + skip(userSelection === null ? 0 : 1), + map( + ([hasScreenShares, windowMode]): GridMode => + hasScreenShares || windowMode === "flat" + ? "spotlight" + : "grid", + ), + ) + ).pipe(startWith(userSelection ?? "grid")), + ), + ), + "grid", + ); + + const setGridMode = (value: GridMode): void => { + gridModeUserSelection$.next(value); + }; + + const gridLayoutMedia$: Observable = combineLatest( + [grid$, spotlight$], + (grid, spotlight) => ({ + type: "grid", + spotlight: spotlight.some((vm) => vm instanceof ScreenShareViewModel) + ? spotlight + : undefined, + grid, + }), + ); + + const spotlightLandscapeLayoutMedia$: Observable = + combineLatest([grid$, spotlight$], (grid, spotlight) => ({ + type: "spotlight-landscape", + spotlight, + grid, + })); + + const spotlightPortraitLayoutMedia$: Observable = + combineLatest([grid$, spotlight$], (grid, spotlight) => ({ + type: "spotlight-portrait", + spotlight, + grid, + })); + + const spotlightExpandedLayoutMedia$: Observable = + combineLatest([spotlight$, pip$], (spotlight, pip) => ({ + type: "spotlight-expanded", + spotlight, + pip: pip ?? undefined, + })); + + const oneOnOneLayoutMedia$: Observable = + mediaItems$.pipe( + map((mediaItems) => { + if (mediaItems.length !== 2) return null; + const local = mediaItems.find((vm) => vm.vm.local)?.vm as + | LocalUserMediaViewModel + | undefined; + const remote = mediaItems.find((vm) => !vm.vm.local)?.vm as + | RemoteUserMediaViewModel + | undefined; + // There might not be a remote tile if there are screen shares, or if + // only the local user is in the call and they're using the duplicate + // tiles option + if (!remote || !local) return null; + + return { type: "one-on-one", local, remote }; + }), + ); + + const pipLayoutMedia$: Observable = spotlight$.pipe( + map((spotlight) => ({ type: "pip", spotlight })), + ); + + /** + * The media to be used to produce a layout. + */ + const layoutMedia$ = scope.behavior( + windowMode$.pipe( + switchMap((windowMode) => { + switch (windowMode) { + case "normal": + return gridMode$.pipe( + switchMap((gridMode) => { + switch (gridMode) { + case "grid": + return oneOnOneLayoutMedia$.pipe( + switchMap((oneOnOne) => + oneOnOne === null ? gridLayoutMedia$ : of(oneOnOne), + ), + ); + case "spotlight": + return spotlightExpanded$.pipe( + switchMap((expanded) => + expanded + ? spotlightExpandedLayoutMedia$ + : spotlightLandscapeLayoutMedia$, + ), + ); + } + }), + ); + case "narrow": + return oneOnOneLayoutMedia$.pipe( + switchMap((oneOnOne) => + oneOnOne === null + ? combineLatest([grid$, spotlight$], (grid, spotlight) => + grid.length > smallMobileCallThreshold || + spotlight.some((vm) => vm instanceof ScreenShareViewModel) + ? spotlightPortraitLayoutMedia$ + : gridLayoutMedia$, + ).pipe(switchAll()) + : // The expanded spotlight layout makes for a better one-on-one + // experience in narrow windows + spotlightExpandedLayoutMedia$, + ), + ); + case "flat": + return gridMode$.pipe( + switchMap((gridMode) => { + switch (gridMode) { + case "grid": + // Yes, grid mode actually gets you a "spotlight" layout in + // this window mode. + return spotlightLandscapeLayoutMedia$; + case "spotlight": + return spotlightExpandedLayoutMedia$; + } + }), + ); + case "pip": + return pipLayoutMedia$; + } + }), + ), + ); + + // There is a cyclical dependency here: the layout algorithms want to know + // which tiles are on screen, but to know which tiles are on screen we have to + // first render a layout. To deal with this we assume initially that no tiles + // are visible, and loop the data back into the layouts with a Subject. + const visibleTiles$ = new Subject(); + const setVisibleTiles = (value: number): void => visibleTiles$.next(value); + + const layoutInternals$ = scope.behavior( + combineLatest([ + layoutMedia$, + visibleTiles$.pipe(startWith(0), distinctUntilChanged()), + ]).pipe( + scan< + [LayoutMedia, number], + LayoutScanState & { layout: Layout }, + LayoutScanState + >( + ({ tiles: prevTiles }, [media, visibleTiles]) => { + let layout: Layout; + let newTiles: TileStore; + switch (media.type) { + case "grid": + case "spotlight-landscape": + case "spotlight-portrait": + [layout, newTiles] = gridLikeLayout( + media, + visibleTiles, + setVisibleTiles, + prevTiles, + ); + break; + case "spotlight-expanded": + [layout, newTiles] = spotlightExpandedLayout(media, prevTiles); + break; + case "one-on-one": + [layout, newTiles] = oneOnOneLayout(media, prevTiles); + break; + case "pip": + [layout, newTiles] = pipLayout(media, prevTiles); + break; + } + + return { layout, tiles: newTiles }; + }, + { layout: null, tiles: TileStore.empty() }, + ), + ), + ); + + /** + * The layout of tiles in the call interface. + */ + const layout$ = scope.behavior( + layoutInternals$.pipe(map(({ layout }) => layout)), + ); + + /** + * The current generation of the tile store, exposed for debugging purposes. + */ + const tileStoreGeneration$ = scope.behavior( + layoutInternals$.pipe(map(({ tiles }) => tiles.generation)), + ); + + const showSpotlightIndicators$ = scope.behavior( + layout$.pipe(map((l) => l.type !== "grid")), + ); + + const showSpeakingIndicators$ = scope.behavior( + layout$.pipe( + switchMap((l) => { + switch (l.type) { + case "spotlight-landscape": + case "spotlight-portrait": + // If the spotlight is showing the active speaker, we can do without + // speaking indicators as they're a redundant visual cue. But if + // screen sharing feeds are in the spotlight we still need them. + return l.spotlight.media$.pipe( + map((models: MediaViewModel[]) => + models.some((m) => m instanceof ScreenShareViewModel), + ), + ); + // In expanded spotlight layout, the active speaker is always shown in + // the picture-in-picture tile so there is no need for speaking + // indicators. And in one-on-one layout there's no question as to who is + // speaking. + case "spotlight-expanded": + case "one-on-one": + return of(false); + default: + return of(true); + } + }), + ), + ); + + const toggleSpotlightExpanded$ = scope.behavior<(() => void) | null>( + windowMode$.pipe( + switchMap((mode) => + mode === "normal" + ? layout$.pipe( + map( + (l) => + l.type === "spotlight-landscape" || + l.type === "spotlight-expanded", + ), + ) + : of(false), + ), + distinctUntilChanged(), + map((enabled) => + enabled ? (): void => spotlightExpandedToggle$.next() : null, + ), + ), + ); + + const screenTap$ = new Subject(); + const controlsTap$ = new Subject(); + const screenHover$ = new Subject(); + const screenUnhover$ = new Subject(); + + const showHeader$ = scope.behavior( + windowMode$.pipe(map((mode) => mode !== "pip" && mode !== "flat")), + ); + + const showFooter$ = scope.behavior( + windowMode$.pipe( + switchMap((mode) => { + switch (mode) { + case "pip": + return of(false); + case "normal": + case "narrow": + return of(true); + case "flat": + // Sadly Firefox has some layering glitches that prevent the footer + // from appearing properly. They happen less often if we never hide + // the footer. + if (isFirefox()) return of(true); + // Show/hide the footer in response to interactions + return merge( + screenTap$.pipe(map(() => "tap screen" as const)), + controlsTap$.pipe(map(() => "tap controls" as const)), + screenHover$.pipe(map(() => "hover" as const)), + ).pipe( + switchScan((state, interaction) => { + switch (interaction) { + case "tap screen": + return state + ? // Toggle visibility on tap + of(false) + : // Hide after a timeout + timer(showFooterMs).pipe( + map(() => false), + startWith(true), + ); + case "tap controls": + // The user is interacting with things, so reset the timeout + return timer(showFooterMs).pipe( + map(() => false), + startWith(true), + ); + case "hover": + // Show on hover and hide after a timeout + return race( + timer(showFooterMs), + screenUnhover$.pipe(take(1)), + ).pipe( + map(() => false), + startWith(true), + ); + } + }, false), + startWith(false), + ); + } + }), + ), + ); + + /** + * Whether audio is currently being output through the earpiece. + */ + const earpieceMode$ = scope.behavior( + combineLatest( + [mediaDevices.audioOutput.available$, mediaDevices.audioOutput.selected$], + (available, selected) => + selected !== undefined && + available.get(selected.id)?.type === "earpiece", + ), + ); + + /** + * Callback to toggle between the earpiece and the loudspeaker. + * + * This will be `null` in case the target does not exist in the list + * of available audio outputs. + */ + const audioOutputSwitcher$ = scope.behavior<{ + targetOutput: "earpiece" | "speaker"; + switch: () => void; + } | null>( + combineLatest( + [mediaDevices.audioOutput.available$, mediaDevices.audioOutput.selected$], + (available, selected) => { + const selectionType = selected && available.get(selected.id)?.type; + + // If we are in any output mode other than speaker switch to speaker. + const newSelectionType: "earpiece" | "speaker" = + selectionType === "speaker" ? "earpiece" : "speaker"; + const newSelection = [...available].find( + ([, d]) => d.type === newSelectionType, + ); + if (newSelection === undefined) return null; + + const [id] = newSelection; + return { + targetOutput: newSelectionType, + switch: (): void => mediaDevices.audioOutput.select(id), + }; + }, + ), + ); + + /** + * Emits an array of reactions that should be visible on the screen. + */ + // DISCUSSION move this into a reaction file + // const {visibleReactions$, audibleReactions$} = reactionsObservables$(showReactionSetting$, ) + const visibleReactions$ = scope.behavior( + showReactions.value$.pipe( + switchMap((show) => (show ? reactions$ : of({}))), + scan< + Record, + { sender: string; emoji: string; startX: number }[] + >((acc, latest) => { + const newSet: { sender: string; emoji: string; startX: number }[] = []; + for (const [sender, reaction] of Object.entries(latest)) { + const startX = + acc.find((v) => v.sender === sender && v.emoji)?.startX ?? + Math.ceil(Math.random() * 80) + 10; + newSet.push({ sender, emoji: reaction.emoji, startX }); + } + return newSet; + }, []), + ), + ); + + /** + * Emits an array of reactions that should be played. + */ + const audibleReactions$ = playReactionsSound.value$.pipe( + switchMap((show) => + show ? reactions$ : of>({}), + ), + map((reactions) => Object.values(reactions).map((v) => v.name)), + scan( + (acc, latest) => { + return { + playing: latest.filter( + (v) => acc.playing.includes(v) || acc.newSounds.includes(v), + ), + newSounds: latest.filter( + (v) => !acc.playing.includes(v) && !acc.newSounds.includes(v), + ), + }; + }, + { playing: [], newSounds: [] }, + ), + map((v) => v.newSounds), + ); + + const newHandRaised$ = handsRaised$.pipe( + map((v) => Object.keys(v).length), + scan( + (acc, newValue) => ({ + value: newValue, + playSounds: newValue > acc.value, + }), + { value: 0, playSounds: false }, + ), + filter((v) => v.playSounds), + ); + + const newScreenShare$ = screenShares$.pipe( + map((v) => v.length), + scan( + (acc, newValue) => ({ + value: newValue, + playSounds: newValue > acc.value, + }), + { value: 0, playSounds: false }, + ), + filter((v) => v.playSounds), + ); + + /** + * Whether we are sharing our screen. + */ + // reassigned here to make it publicly accessible + const sharingScreen$ = localMembership.sharingScreen$; + + /** + * Callback to toggle screen sharing. If null, screen sharing is not possible. + */ + // reassigned here to make it publicly accessible + const toggleScreenSharing = localMembership.toggleScreenSharing; + + const join = localMembership.requestConnect; + // TODO-MULTI-SFU: Use this view model for the lobby as well, and only call this once 'join' is clicked? + join(); + return { + autoLeave$: autoLeave$, + callPickupState$: callPickupState$, + ringOverlay$: ringOverlay$, + leave$: leave$, + hangup: (): void => userHangup$.next(), + join: join, + toggleScreenSharing: toggleScreenSharing, + sharingScreen$: sharingScreen$, + + tapScreen: (): void => screenTap$.next(), + tapControls: (): void => controlsTap$.next(), + hoverScreen: (): void => screenHover$.next(), + unhoverScreen: (): void => screenUnhover$.next(), + + configError$: localMembership.configError$, + participantCount$: participantCount$, + audioParticipants$: audioParticipants$, + + handsRaised$: handsRaised$, + reactions$: reactions$, + joinSoundEffect$: joinSoundEffect$, + leaveSoundEffect$: leaveSoundEffect$, + newHandRaised$: newHandRaised$, + newScreenShare$: newScreenShare$, + audibleReactions$: audibleReactions$, + visibleReactions$: visibleReactions$, + + windowMode$: windowMode$, + spotlightExpanded$: spotlightExpanded$, + toggleSpotlightExpanded$: toggleSpotlightExpanded$, + gridMode$: gridMode$, + setGridMode: setGridMode, + grid$: grid$, + spotlight$: spotlight$, + pip$: pip$, + layout$: layout$, + tileStoreGeneration$: tileStoreGeneration$, + showSpotlightIndicators$: showSpotlightIndicators$, + showSpeakingIndicators$: showSpeakingIndicators$, + showHeader$: showHeader$, + showFooter$: showFooter$, + earpieceMode$: earpieceMode$, + audioOutputSwitcher$: audioOutputSwitcher$, + reconnecting$: reconnecting$, + }; +} + +function getE2eeKeyProvider( + e2eeSystem: EncryptionSystem, + rtcSession: MatrixRTCSession, +): BaseKeyProvider | undefined { + if (e2eeSystem.kind === E2eeType.NONE) return undefined; + + if (e2eeSystem.kind === E2eeType.PER_PARTICIPANT) { + const keyProvider = new MatrixKeyProvider(); + keyProvider.setRTCSession(rtcSession); + return keyProvider; + } else if (e2eeSystem.kind === E2eeType.SHARED_KEY && e2eeSystem.secret) { + const keyProvider = new ExternalE2EEKeyProvider(); + keyProvider + .setKey(e2eeSystem.secret) + .catch((e) => logger.error("Failed to set shared key for E2EE", e)); + return keyProvider; + } +} diff --git a/src/state/CallViewModel/CallViewModelTestUtils.ts b/src/state/CallViewModel/CallViewModelTestUtils.ts new file mode 100644 index 00000000..f86921c5 --- /dev/null +++ b/src/state/CallViewModel/CallViewModelTestUtils.ts @@ -0,0 +1,193 @@ +/* +Copyright 2025 Element Corp. +Copyright 2024 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + ConnectionState, + type LocalParticipant, + type Participant, + ParticipantEvent, + type RemoteParticipant, + type Room as LivekitRoom, +} from "livekit-client"; +import { SyncState } from "matrix-js-sdk/lib/sync"; +import { BehaviorSubject, type Observable, map, of } from "rxjs"; +import { onTestFinished, vi } from "vitest"; +import { ClientEvent, type MatrixClient } from "matrix-js-sdk"; +import EventEmitter from "events"; +import * as ComponentsCore from "@livekit/components-core"; + +import type { CallMembership } from "matrix-js-sdk/lib/matrixrtc"; +import { E2eeType } from "../../e2ee/e2eeType"; +import { type RaisedHandInfo, type ReactionInfo } from "../../reactions"; +import { + type CallViewModel, + createCallViewModel$, + type CallViewModelOptions, +} from "./CallViewModel"; +import { + mockConfig, + mockLivekitRoom, + mockLocalParticipant, + mockMatrixRoom, + mockMatrixRoomMember, + mockMediaDevices, + mockMuteStates, + MockRTCSession, + testScope, +} from "../../utils/test"; +import { + alice, + aliceDoppelganger, + bob, + bobZeroWidthSpace, + daveRTL, + daveRTLRtcMember, + local, + localRtcMember, +} from "../../utils/test-fixtures"; +import { type Behavior, constant } from "../Behavior"; +import { type ProcessorState } from "../../livekit/TrackProcessorContext"; +import { type MediaDevices } from "../MediaDevices"; + +mockConfig({ + livekit: { livekit_service_url: "http://my-default-service-url.com" }, +}); + +const carol = local; + +const dave = mockMatrixRoomMember(daveRTLRtcMember, { rawDisplayName: "Dave" }); + +const roomMembers = new Map( + [alice, aliceDoppelganger, bob, bobZeroWidthSpace, carol, dave, daveRTL].map( + (p) => [p.userId, p], + ), +); + +export interface CallViewModelInputs { + remoteParticipants$: Behavior; + rtcMembers$: Behavior[]>; + livekitConnectionState$: Behavior; + speaking: Map>; + mediaDevices: MediaDevices; + initialSyncState: SyncState; +} + +const localParticipant = mockLocalParticipant({ identity: "" }); + +export function withCallViewModel( + { + remoteParticipants$ = constant([]), + rtcMembers$ = constant([localRtcMember]), + livekitConnectionState$: connectionState$ = constant( + ConnectionState.Connected, + ), + speaking = new Map(), + mediaDevices = mockMediaDevices({}), + initialSyncState = SyncState.Syncing, + }: Partial = {}, + continuation: ( + vm: CallViewModel, + rtcSession: MockRTCSession, + subjects: { raisedHands$: BehaviorSubject> }, + setSyncState: (value: SyncState) => void, + ) => void, + options: CallViewModelOptions = { + encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + autoLeaveWhenOthersLeft: false, + }, +): void { + let syncState = initialSyncState; + const setSyncState = (value: SyncState): void => { + const prev = syncState; + syncState = value; + room.client.emit(ClientEvent.Sync, value, prev); + }; + const room = mockMatrixRoom({ + client: new (class extends EventEmitter { + public getUserId(): string | undefined { + return localRtcMember.userId; + } + + public getDeviceId(): string { + return localRtcMember.deviceId; + } + + public getDomain(): string { + return "example.com"; + } + + public getSyncState(): SyncState { + return syncState; + } + })() as Partial as MatrixClient, + getMembers: () => Array.from(roomMembers.values()), + getMembersWithMembership: () => Array.from(roomMembers.values()), + }); + const rtcSession = new MockRTCSession(room, []).withMemberships(rtcMembers$); + const participantsSpy = vi + .spyOn(ComponentsCore, "connectedParticipantsObserver") + .mockReturnValue(remoteParticipants$); + const mediaSpy = vi + .spyOn(ComponentsCore, "observeParticipantMedia") + .mockImplementation((p) => + of({ participant: p } as Partial< + ComponentsCore.ParticipantMedia + > as ComponentsCore.ParticipantMedia), + ); + const eventsSpy = vi + .spyOn(ComponentsCore, "observeParticipantEvents") + .mockImplementation((p, ...eventTypes) => { + if (eventTypes.includes(ParticipantEvent.IsSpeakingChanged)) { + return (speaking.get(p) ?? of(false)).pipe( + map((s): Participant => ({ ...p, isSpeaking: s }) as Participant), + ); + } else { + return of(p); + } + }); + + const roomEventSelectorSpy = vi + .spyOn(ComponentsCore, "roomEventSelector") + .mockImplementation((_room, _eventType) => of()); + const muteStates = mockMuteStates(); + const raisedHands$ = new BehaviorSubject>({}); + const reactions$ = new BehaviorSubject>({}); + + const vm = createCallViewModel$( + testScope(), + rtcSession.asMockedSession(), + room, + mediaDevices, + muteStates, + { + ...options, + livekitRoomFactory: (): LivekitRoom => + mockLivekitRoom({ + localParticipant, + disconnect: async () => Promise.resolve(), + setE2EEEnabled: async () => Promise.resolve(), + }), + connectionState$, + }, + raisedHands$, + reactions$, + new BehaviorSubject({ + processor: undefined, + supported: undefined, + }), + ); + + onTestFinished(() => { + participantsSpy.mockRestore(); + mediaSpy.mockRestore(); + eventsSpy.mockRestore(); + roomEventSelectorSpy.mockRestore(); + }); + + continuation(vm, rtcSession, { raisedHands$: raisedHands$ }, setSyncState); +} diff --git a/src/rtcSessionHelpers.test.ts b/src/state/CallViewModel/localMember/LocalMembership.test.ts similarity index 89% rename from src/rtcSessionHelpers.test.ts rename to src/state/CallViewModel/localMember/LocalMembership.test.ts index 8aca40f5..716740d3 100644 --- a/src/rtcSessionHelpers.test.ts +++ b/src/state/CallViewModel/localMember/LocalMembership.test.ts @@ -1,4 +1,5 @@ /* +Copyright 2025 Element Creations Ltd. Copyright 2024 New Vector Ltd. SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial @@ -10,16 +11,16 @@ import { expect, test, vi } from "vitest"; import { AutoDiscovery } from "matrix-js-sdk/lib/autodiscovery"; import EventEmitter from "events"; -import { enterRTCSession } from "../src/rtcSessionHelpers"; -import { mockConfig } from "./utils/test"; +import { MatrixRTCMode } from "../../../settings/settings"; +import { mockConfig } from "../../../utils/test"; +import { enterRTCSession } from "./LocalMembership"; -const USE_MUTI_SFU = false; +const MATRIX_RTC_MODE = MatrixRTCMode.Legacy; const getUrlParams = vi.hoisted(() => vi.fn(() => ({}))); -vi.mock("./UrlParams", () => ({ getUrlParams })); +vi.mock("../../../UrlParams", () => ({ getUrlParams })); -const actualWidget = await vi.hoisted(async () => vi.importActual("./widget")); -vi.mock("./widget", () => ({ - ...actualWidget, +vi.mock("../../../widget", async (importOriginal) => ({ + ...(await importOriginal()), widget: { api: { setAlwaysOnScreen: (): void => {}, @@ -94,8 +95,7 @@ test("It joins the correct Session", async () => { }, { encryptMedia: true, - useMultiSfu: USE_MUTI_SFU, - preferStickyEvents: false, + matrixRTCMode: MATRIX_RTC_MODE, }, ); @@ -153,8 +153,7 @@ test("It should not fail with configuration error if homeserver config has livek }, { encryptMedia: true, - useMultiSfu: USE_MUTI_SFU, - preferStickyEvents: false, + matrixRTCMode: MATRIX_RTC_MODE, }, ); }); diff --git a/src/state/CallViewModel/localMember/LocalMembership.ts b/src/state/CallViewModel/localMember/LocalMembership.ts new file mode 100644 index 00000000..68b34d94 --- /dev/null +++ b/src/state/CallViewModel/localMember/LocalMembership.ts @@ -0,0 +1,633 @@ +/* +Copyright 2025 Element Creations Ltd. + +SPDX-License-IdFentifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + type LocalTrack, + type Participant, + ParticipantEvent, + type LocalParticipant, +} from "livekit-client"; +import { observeParticipantEvents } from "@livekit/components-core"; +import { + type LivekitTransport, + type MatrixRTCSession, + MembershipManagerEvent, + Status, +} from "matrix-js-sdk/lib/matrixrtc"; +import { ClientEvent, SyncState, type Room as MatrixRoom } from "matrix-js-sdk"; +import { + BehaviorSubject, + combineLatest, + distinctUntilChanged, + fromEvent, + map, + type Observable, + of, + scan, + startWith, + switchMap, + tap, +} from "rxjs"; +import { type Logger } from "matrix-js-sdk/lib/logger"; + +import { type Behavior } from "../../Behavior"; +import { type IConnectionManager } from "../remoteMembers/ConnectionManager"; +import { ObservableScope } from "../../ObservableScope"; +import { Publisher } from "./Publisher"; +import { type MuteStates } from "../../MuteStates"; +import { type ProcessorState } from "../../../livekit/TrackProcessorContext"; +import { type MediaDevices } from "../../MediaDevices"; +import { and$ } from "../../../utils/observable"; +import { ElementCallError, UnknownCallError } from "../../../utils/errors"; +import { + ElementWidgetActions, + widget, + type WidgetHelpers, +} from "../../../widget"; +import { areLivekitTransportsEqual } from "../remoteMembers/MatrixLivekitMembers"; +import { getUrlParams } from "../../../UrlParams.ts"; +import { PosthogAnalytics } from "../../../analytics/PosthogAnalytics.ts"; +import { MatrixRTCMode } from "../../../settings/settings.ts"; +import { Config } from "../../../config/Config.ts"; +import { + type Connection, + type ConnectionState, +} from "../remoteMembers/Connection.ts"; + +export enum LivekitState { + Uninitialized = "uninitialized", + Connecting = "connecting", + Connected = "connected", + Error = "error", + Disconnected = "disconnected", + Disconnecting = "disconnecting", +} + +type LocalMemberLivekitState = + | { state: LivekitState.Error; error: string } + | { state: LivekitState.Connected } + | { state: LivekitState.Connecting } + | { state: LivekitState.Uninitialized } + | { state: LivekitState.Disconnected } + | { state: LivekitState.Disconnecting }; + +export enum MatrixState { + Connected = "connected", + Disconnected = "disconnected", + Connecting = "connecting", +} + +type LocalMemberMatrixState = + | { state: MatrixState.Connected } + | { state: MatrixState.Connecting } + | { state: MatrixState.Disconnected }; + +export interface LocalMemberConnectionState { + livekit$: Behavior; + matrix$: Behavior; +} + +/* + * - get well known + * - get oldest membership + * - get transport to use + * - get openId + jwt token + * - wait for createTrack() call + * - create tracks + * - wait for join() call + * - Publisher.publishTracks() + * - send join state/sticky event + */ +interface Props { + options: Behavior; + scope: ObservableScope; + mediaDevices: MediaDevices; + muteStates: MuteStates; + connectionManager: IConnectionManager; + matrixRTCSession: MatrixRTCSession; + matrixRoom: MatrixRoom; + localTransport$: Behavior; + trackProcessorState$: Behavior; + widget: WidgetHelpers | null; + logger: Logger; +} + +/** + * This class is responsible for managing the own membership in a room. + * We want + * - a publisher + * - + * @param param0 + * @returns + * - publisher: The handle to create tracks and publish them to the room. + * - connected$: the current connection state. Including matrix server and livekit server connection. (only considering the livekit server we are using for our own media publication) + * - transport$: the transport object the ownMembership$ ended up using. + * - connectionState: the current connection state. Including matrix server and livekit server connection. + * - sharingScreen$: Whether we are sharing our screen. `undefined` if we cannot share the screen. + */ +export const createLocalMembership$ = ({ + scope, + options, + muteStates, + mediaDevices, + connectionManager, + matrixRTCSession, + localTransport$, + matrixRoom, + trackProcessorState$, + widget, + logger: parentLogger, +}: Props): { + // publisher: Publisher + requestConnect: () => LocalMemberConnectionState; + startTracks: () => Behavior; + requestDisconnect: () => Observable | null; + connectionState: LocalMemberConnectionState; + sharingScreen$: Behavior; + /** + * Callback to toggle screen sharing. If null, screen sharing is not possible. + */ + toggleScreenSharing: (() => void) | null; + participant$: Behavior; + connection$: Behavior; + // deprecated fields + /** @deprecated use state instead*/ + homeserverConnected$: Behavior; + /** @deprecated use state instead*/ + connected$: Behavior; + // this needs to be discussed + /** @deprecated use state instead*/ + reconnecting$: Behavior; + // also needs to be disccues + /** @deprecated use state instead*/ + configError$: Behavior; +} => { + const logger = parentLogger.getChild("[LocalMembership]"); + logger.debug(`Creating local membership..`); + const state = { + livekit$: new BehaviorSubject({ + state: LivekitState.Uninitialized, + }), + matrix$: new BehaviorSubject({ + state: MatrixState.Disconnected, + }), + }; + + // This should be used in a combineLatest with publisher$ to connect. + // to make it possible to call startTracks before the preferredTransport$ has resolved. + const trackStartRequested$ = new BehaviorSubject(false); + + // This should be used in a combineLatest with publisher$ to connect. + // to make it possible to call startTracks before the preferredTransport$ has resolved. + const connectRequested$ = new BehaviorSubject(false); + + // This should be used in a combineLatest with publisher$ to connect. + const tracks$ = new BehaviorSubject([]); + + // Drop Epoch data here since we will not combine this anymore + const localConnection$ = scope.behavior( + combineLatest([connectionManager.connections$, localTransport$]).pipe( + map(([connections, localTransport]) => { + if (localTransport === null) { + return null; + } + return ( + connections.value.find((connection) => + areLivekitTransportsEqual(connection.transport, localTransport), + ) ?? null + ); + }), + tap((connection) => { + logger.info( + `Local connection updated: ${connection?.transport?.livekit_service_url}`, + ); + }), + ), + ); + /** + * Whether we are connected to the MatrixRTC session. + */ + const homeserverConnected$ = scope.behavior( + // To consider ourselves connected to MatrixRTC, we check the following: + and$( + // The client is connected to the sync loop + ( + fromEvent(matrixRoom.client, ClientEvent.Sync) as Observable< + [SyncState] + > + ).pipe( + startWith([matrixRoom.client.getSyncState()]), + map(([state]) => state === SyncState.Syncing), + ), + // Room state observed by session says we're connected + fromEvent(matrixRTCSession, MembershipManagerEvent.StatusChanged).pipe( + startWith(null), + map(() => matrixRTCSession.membershipStatus === Status.Connected), + ), + // Also watch out for warnings that we've likely hit a timeout and our + // delayed leave event is being sent (this condition is here because it + // provides an earlier warning than the sync loop timeout, and we wouldn't + // see the actual leave event until we reconnect to the sync loop) + fromEvent(matrixRTCSession, MembershipManagerEvent.ProbablyLeft).pipe( + startWith(null), + map(() => matrixRTCSession.probablyLeft !== true), + ), + ).pipe( + tap((connected) => { + logger.info(`Homeserver connected update: ${connected}`); + }), + ), + ); + + // /** + // * Whether we are "fully" connected to the call. Accounts for both the + // * connection to the MatrixRTC session and the LiveKit publish connection. + // */ + // // TODO use this in combination with the MemberState. + const connected$ = scope.behavior( + and$( + homeserverConnected$, + localConnection$.pipe( + switchMap((c) => + c + ? c.state$.pipe(map((state) => state.state === "ConnectedToLkRoom")) + : of(false), + ), + ), + ), + ); + + const publisher$ = new BehaviorSubject(null); + localConnection$.pipe(scope.bind()).subscribe((connection) => { + if (connection !== null && publisher$.value === null) { + // TODO looks strange to not change publisher if connection changes. + publisher$.next( + new Publisher( + scope, + connection, + mediaDevices, + muteStates, + trackProcessorState$, + ), + ); + } + }); + + combineLatest([publisher$, trackStartRequested$]).subscribe( + ([publisher, shouldStartTracks]) => { + if (publisher && shouldStartTracks) { + publisher + .createAndSetupTracks() + .then((tracks) => { + tracks$.next(tracks); + }) + .catch((error) => { + logger.error("Error creating tracks:", error); + }); + } + }, + ); + + // MATRIX RELATED + + // /** + // * Whether we should tell the user that we're reconnecting to the call. + // */ + // DISCUSSION is there a better way to do this? + // sth that is more deriectly implied from the membership manager of the js sdk. (fromEvent(matrixRTCSession, Reconnecting)) ??? or similar + const reconnecting$ = scope.behavior( + connected$.pipe( + // We are reconnecting if we previously had some successful initial + // connection but are now disconnected + scan( + ({ connectedPreviously }, connectedNow) => ({ + connectedPreviously: connectedPreviously || connectedNow, + reconnecting: connectedPreviously && !connectedNow, + }), + { connectedPreviously: false, reconnecting: false }, + ), + map(({ reconnecting }) => reconnecting), + ), + ); + + const startTracks = (): Behavior => { + trackStartRequested$.next(true); + return tracks$; + }; + + combineLatest([publisher$, tracks$]).subscribe(([publisher, tracks]) => { + if ( + tracks.length === 0 || + // change this to !== Publishing + state.livekit$.value.state !== LivekitState.Uninitialized + ) { + return; + } + state.livekit$.next({ state: LivekitState.Connecting }); + publisher + ?.startPublishing() + .then(() => { + state.livekit$.next({ state: LivekitState.Connected }); + }) + .catch((error) => { + state.livekit$.next({ state: LivekitState.Error, error }); + }); + }); + + combineLatest([localTransport$, connectRequested$]).subscribe( + // TODO reconnect when transport changes => create test. + ([transport, connectRequested]) => { + if ( + transport === null || + !connectRequested || + state.matrix$.value.state !== MatrixState.Disconnected + ) { + logger.info( + "Not yet connecting because: ", + "transport === null:", + transport === null, + "!connectRequested:", + !connectRequested, + "state.matrix$.value.state !== MatrixState.Disconnected:", + state.matrix$.value.state !== MatrixState.Disconnected, + ); + return; + } + state.matrix$.next({ state: MatrixState.Connecting }); + logger.info("Matrix State connecting"); + enterRTCSession(matrixRTCSession, transport, options.value).catch( + (error) => { + logger.error(error); + }, + ); + }, + ); + + const requestConnect = (): LocalMemberConnectionState => { + trackStartRequested$.next(true); + connectRequested$.next(true); + + return state; + }; + + const requestDisconnect = (): Behavior | null => { + if (state.livekit$.value.state !== LivekitState.Connected) return null; + state.livekit$.next({ state: LivekitState.Disconnecting }); + combineLatest([publisher$, tracks$], (publisher, tracks) => { + publisher + ?.stopPublishing() + .then(() => { + tracks.forEach((track) => track.stop()); + state.livekit$.next({ state: LivekitState.Disconnected }); + }) + .catch((error) => { + state.livekit$.next({ state: LivekitState.Error, error }); + }); + }); + + return state.livekit$; + }; + + // Pause upstream of all local media tracks when we're disconnected from + // MatrixRTC, because it can be an unpleasant surprise for the app to say + // 'reconnecting' and yet still be transmitting your media to others. + // We use matrixConnected$ rather than reconnecting$ because we want to + // pause tracks during the initial joining sequence too until we're sure + // that our own media is displayed on screen. + combineLatest([localConnection$, homeserverConnected$]) + .pipe(scope.bind()) + .subscribe(([connection, connected]) => { + if (connection?.state$.value.state !== "ConnectedToLkRoom") return; + const publications = + connection.livekitRoom.localParticipant.trackPublications.values(); + if (connected) { + for (const p of publications) { + if (p.track?.isUpstreamPaused === true) { + const kind = p.track.kind; + logger.info( + `Resuming ${kind} track (MatrixRTC connection present)`, + ); + p.track + .resumeUpstream() + .catch((e) => + logger.error( + `Failed to resume ${kind} track after MatrixRTC reconnection`, + e, + ), + ); + } + } + } else { + for (const p of publications) { + if (p.track?.isUpstreamPaused === false) { + const kind = p.track.kind; + logger.info( + `Pausing ${kind} track (uncertain MatrixRTC connection)`, + ); + p.track + .pauseUpstream() + .catch((e) => + logger.error( + `Failed to pause ${kind} track after entering uncertain MatrixRTC connection`, + e, + ), + ); + } + } + } + }); + + const configError$ = new BehaviorSubject(null); + // TODO I do not fully understand what this does. + // Is it needed? + // Is this at the right place? + // Can this be simplified? + // Start and stop session membership as needed + scope.reconcile(localTransport$, async (advertised) => { + if (advertised !== null && advertised !== undefined) { + try { + await enterRTCSession(matrixRTCSession, advertised, options.value); + configError$.next(null); + } catch (e) { + logger.error("Error entering RTC session", e); + } + + // Update our member event when our mute state changes. + const intentScope = new ObservableScope(); + intentScope.reconcile(muteStates.video.enabled$, async (videoEnabled) => + matrixRTCSession.updateCallIntent(videoEnabled ? "video" : "audio"), + ); + + return async (): Promise => { + intentScope.end(); + // Only sends Matrix leave event. The LiveKit session will disconnect + // as soon as either the stopConnection$ handler above gets to it or + // the view model is destroyed. + try { + await matrixRTCSession.leaveRoomSession(); + } catch (e) { + logger.error("Error leaving RTC session", e); + } + try { + await widget?.api.transport.send(ElementWidgetActions.HangupCall, {}); + } catch (e) { + logger.error("Failed to send hangup action", e); + } + }; + } + }); + + localConnection$ + .pipe( + distinctUntilChanged(), + switchMap((c) => + c === null ? of({ state: "Initialized" } as ConnectionState) : c.state$, + ), + map((s) => { + logger.trace(`Local connection state update: ${s.state}`); + if (s.state == "FailedToStart") { + return s.error instanceof ElementCallError + ? s.error + : new UnknownCallError(s.error); + } else { + return null; + } + }), + scope.bind(), + ) + .subscribe((fatalError) => { + configError$.next(fatalError); + }); + + /** + * Whether the user is currently sharing their screen. + */ + const sharingScreen$ = scope.behavior( + localConnection$.pipe( + switchMap((c) => + c === null + ? of(false) + : observeSharingScreen$(c.livekitRoom.localParticipant), + ), + ), + ); + + const toggleScreenSharing = + "getDisplayMedia" in (navigator.mediaDevices ?? {}) && + !getUrlParams().hideScreensharing + ? (): void => + // If a connection is ready, toggle screen sharing. + // We deliberately do nothing in the case of a null connection because + // it looks nice for the call control buttons to all become available + // at once upon joining the call, rather than introducing a disabled + // state. The user can just click again. + // We also allow screen sharing to be toggled even if the connection + // is still initializing or publishing tracks, because there's no + // technical reason to disallow this. LiveKit will publish if it can. + void localConnection$.value?.livekitRoom.localParticipant + .setScreenShareEnabled(!sharingScreen$.value, { + audio: true, + selfBrowserSurface: "include", + surfaceSwitching: "include", + systemAudio: "include", + }) + .catch(logger.error) + : null; + + const participant$ = scope.behavior( + localConnection$.pipe(map((c) => c?.livekitRoom.localParticipant ?? null)), + ); + return { + startTracks, + requestConnect, + requestDisconnect, + connectionState: state, + homeserverConnected$, + connected$, + reconnecting$, + configError$, + sharingScreen$, + toggleScreenSharing, + participant$, + connection$: localConnection$, + }; +}; + +export function observeSharingScreen$(p: Participant): Observable { + return observeParticipantEvents( + p, + ParticipantEvent.TrackPublished, + ParticipantEvent.TrackUnpublished, + ParticipantEvent.LocalTrackPublished, + ParticipantEvent.LocalTrackUnpublished, + ).pipe(map((p) => p.isScreenShareEnabled)); +} + +interface EnterRTCSessionOptions { + encryptMedia: boolean; + matrixRTCMode: MatrixRTCMode; +} + +/** + * Does the necessary steps to enter the RTC session on the matrix side: + * - Preparing the membership info (FOCUS to use, options) + * - Sends the matrix event to join the call, and starts the membership manager: + * - Delay events management + * - Handles retries (fails only after several attempts) + * + * @param rtcSession + * @param transport + * @param options + * @throws If the widget could not send ElementWidgetActions.JoinCall action. + */ +// Exported for unit testing +export async function enterRTCSession( + rtcSession: MatrixRTCSession, + transport: LivekitTransport, + { encryptMedia, matrixRTCMode }: EnterRTCSessionOptions, +): Promise { + PosthogAnalytics.instance.eventCallEnded.cacheStartCall(new Date()); + PosthogAnalytics.instance.eventCallStarted.track(rtcSession.room.roomId); + + // This must be called before we start trying to join the call, as we need to + // have started tracking by the time calls start getting created. + // groupCallOTelMembership?.onJoinCall(); + + const { features, matrix_rtc_session: matrixRtcSessionConfig } = Config.get(); + const useDeviceSessionMemberEvents = + features?.feature_use_device_session_member_events; + const { sendNotificationType: notificationType, callIntent } = getUrlParams(); + const multiSFU = matrixRTCMode !== MatrixRTCMode.Legacy; + // Multi-sfu does not need a preferred foci list. just the focus that is actually used. + rtcSession.joinRoomSession( + multiSFU ? [] : [transport], + multiSFU ? transport : undefined, + { + notificationType, + callIntent, + manageMediaKeys: encryptMedia, + ...(useDeviceSessionMemberEvents !== undefined && { + useLegacyMemberEvents: !useDeviceSessionMemberEvents, + }), + delayedLeaveEventRestartMs: + matrixRtcSessionConfig?.delayed_leave_event_restart_ms, + delayedLeaveEventDelayMs: + matrixRtcSessionConfig?.delayed_leave_event_delay_ms, + delayedLeaveEventRestartLocalTimeoutMs: + matrixRtcSessionConfig?.delayed_leave_event_restart_local_timeout_ms, + networkErrorRetryMs: matrixRtcSessionConfig?.network_error_retry_ms, + makeKeyDelay: matrixRtcSessionConfig?.wait_for_key_rotation_ms, + membershipEventExpiryMs: + matrixRtcSessionConfig?.membership_event_expiry_ms, + useExperimentalToDeviceTransport: true, + unstableSendStickyEvents: matrixRTCMode === MatrixRTCMode.Matrix_2_0, + }, + ); + if (widget) { + await widget.api.transport.send(ElementWidgetActions.JoinCall, {}); + } +} diff --git a/src/state/CallViewModel/localMember/LocalTransport.ts b/src/state/CallViewModel/localMember/LocalTransport.ts new file mode 100644 index 00000000..6bb31e57 --- /dev/null +++ b/src/state/CallViewModel/localMember/LocalTransport.ts @@ -0,0 +1,179 @@ +/* +Copyright 2025 Element Creations Ltd. + +SPDX-License-IdFentifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + type CallMembership, + isLivekitTransport, + type LivekitTransportConfig, + type LivekitTransport, + isLivekitTransportConfig, +} from "matrix-js-sdk/lib/matrixrtc"; +import { type MatrixClient } from "matrix-js-sdk"; +import { combineLatest, distinctUntilChanged, first, from, map } from "rxjs"; +import { logger } from "matrix-js-sdk/lib/logger"; +import { AutoDiscovery } from "matrix-js-sdk/lib/autodiscovery"; + +import { type Behavior } from "../../Behavior.ts"; +import { type Epoch, type ObservableScope } from "../../ObservableScope.ts"; +import { Config } from "../../../config/Config.ts"; +import { MatrixRTCTransportMissingError } from "../../../utils/errors.ts"; +import { getSFUConfigWithOpenID } from "../../../livekit/openIDSFU.ts"; +import { areLivekitTransportsEqual } from "../remoteMembers/MatrixLivekitMembers.ts"; + +/* + * - get well known + * - get oldest membership + * - get transport to use + * - get openId + jwt token + * - wait for createTrack() call + * - create tracks + * - wait for join() call + * - Publisher.publishTracks() + * - send join state/sticky event + */ +interface Props { + scope: ObservableScope; + memberships$: Behavior>; + client: MatrixClient; + roomId: string; + useOldestMember$: Behavior; +} + +/** + * This class is responsible for managing the local transport. + * "Which transport is the local member going to use" + * + * @prop useOldestMember Whether to use the same transport as the oldest member. + * This will only update once the first oldest member appears. Will not recompute if the oldest member leaves. + */ +export const createLocalTransport$ = ({ + scope, + memberships$, + client, + roomId, + useOldestMember$, +}: Props): Behavior => { + /** + * The transport over which we should be actively publishing our media. + * undefined when not joined. + */ + const oldestMemberTransport$ = scope.behavior( + memberships$.pipe( + map( + (memberships) => + memberships.value[0]?.getTransport(memberships.value[0]) ?? null, + ), + first((t) => t != null && isLivekitTransport(t)), + ), + null, + ); + + /** + * The transport that we would personally prefer to publish on (if not for the + * transport preferences of others, perhaps). + */ + const preferredTransport$: Behavior = scope.behavior( + from(makeTransport(client, roomId)), + null, + ); + + /** + * The transport we should advertise in our MatrixRTC membership. + */ + const advertisedTransport$ = scope.behavior( + combineLatest([ + useOldestMember$, + oldestMemberTransport$, + preferredTransport$, + ]).pipe( + map(([useOldestMember, oldestMemberTransport, preferredTransport]) => + useOldestMember + ? (oldestMemberTransport ?? preferredTransport) + : preferredTransport, + ), + distinctUntilChanged(areLivekitTransportsEqual), + ), + ); + return advertisedTransport$; +}; + +const FOCI_WK_KEY = "org.matrix.msc4143.rtc_foci"; + +async function makeTransportInternal( + client: MatrixClient, + roomId: string, +): Promise { + logger.log("Searching for a preferred transport"); + //TODO refactor this to use the jwt service returned alias. + const livekitAlias = roomId; + // TODO-MULTI-SFU: Either remove this dev tool or make it more official + const urlFromStorage = + localStorage.getItem("robin-matrixrtc-auth") ?? + localStorage.getItem("timo-focus-url"); + if (urlFromStorage !== null) { + const transportFromStorage: LivekitTransport = { + type: "livekit", + livekit_service_url: urlFromStorage, + livekit_alias: livekitAlias, + }; + logger.log( + "Using LiveKit transport from local storage: ", + transportFromStorage, + ); + return transportFromStorage; + } + + // Prioritize the .well-known/matrix/client, if available, over the configured SFU + const domain = client.getDomain(); + if (domain) { + // we use AutoDiscovery instead of relying on the MatrixClient having already + // been fully configured and started + const wellKnownFoci = (await AutoDiscovery.getRawClientConfig(domain))?.[ + FOCI_WK_KEY + ]; + if (Array.isArray(wellKnownFoci)) { + const transport: LivekitTransportConfig | undefined = wellKnownFoci.find( + (f) => f && isLivekitTransportConfig(f), + ); + if (transport !== undefined) { + logger.log("Using LiveKit transport from .well-known: ", transport); + return { ...transport, livekit_alias: livekitAlias }; + } + } + } + + const urlFromConf = Config.get().livekit?.livekit_service_url; + if (urlFromConf) { + const transportFromConf: LivekitTransport = { + type: "livekit", + livekit_service_url: urlFromConf, + livekit_alias: livekitAlias, + }; + logger.log("Using LiveKit transport from config: ", transportFromConf); + return transportFromConf; + } + + throw new MatrixRTCTransportMissingError(domain ?? ""); +} + +async function makeTransport( + client: MatrixClient, + roomId: string, +): Promise { + const transport = await makeTransportInternal(client, roomId); + // this will call the jwt/sfu/get endpoint to pre create the livekit room. + try { + await getSFUConfigWithOpenID( + client, + transport.livekit_service_url, + transport.livekit_alias, + ); + } catch (e) { + logger.warn(`Failed to get SFU config for transport: ${e}`); + } + return transport; +} diff --git a/src/state/PublishConnection.ts b/src/state/CallViewModel/localMember/Publisher.ts similarity index 52% rename from src/state/PublishConnection.ts rename to src/state/CallViewModel/localMember/Publisher.ts index cfbcba90..11f35424 100644 --- a/src/state/PublishConnection.ts +++ b/src/state/CallViewModel/localMember/Publisher.ts @@ -1,16 +1,17 @@ /* +Copyright 2025 Element Creations Ltd. Copyright 2025 New Vector Ltd. SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ import { - ConnectionState, - type E2EEOptions, LocalVideoTrack, - Room as LivekitRoom, - type RoomOptions, + type Room as LivekitRoom, Track, + type LocalTrack, + type LocalTrackPublication, + ConnectionState as LivekitConnectionState, } from "livekit-client"; import { map, @@ -19,65 +20,52 @@ import { type Subscription, switchMap, } from "rxjs"; -import { logger } from "matrix-js-sdk/lib/logger"; +import { type Logger } from "matrix-js-sdk/lib/logger"; -import type { Behavior } from "./Behavior.ts"; -import type { MediaDevices, SelectedDevice } from "./MediaDevices.ts"; -import type { MuteStates } from "./MuteStates.ts"; +import type { Behavior } from "../../Behavior.ts"; +import type { MediaDevices, SelectedDevice } from "../../MediaDevices.ts"; +import type { MuteStates } from "../../MuteStates.ts"; import { type ProcessorState, trackProcessorSync, -} from "../livekit/TrackProcessorContext.tsx"; -import { getUrlParams } from "../UrlParams.ts"; -import { defaultLiveKitOptions } from "../livekit/options.ts"; -import { getValue } from "../utils/observable.ts"; -import { observeTrackReference$ } from "./MediaViewModel.ts"; -import { Connection, type ConnectionOpts } from "./Connection.ts"; -import { type ObservableScope } from "./ObservableScope.ts"; +} from "../../../livekit/TrackProcessorContext.tsx"; +import { getUrlParams } from "../../../UrlParams.ts"; +import { observeTrackReference$ } from "../../MediaViewModel.ts"; +import { type Connection } from "../remoteMembers/Connection.ts"; +import { type ObservableScope } from "../../ObservableScope.ts"; /** - * A connection to the local LiveKit room, the one the user is publishing to. - * This connection will publish the local user's audio and video tracks. + * A wrapper for a Connection object. + * This wrapper will manage the connection used to publish to the LiveKit room. + * The Publisher is also responsible for creating the media tracks. */ -export class PublishConnection extends Connection { - private readonly scope: ObservableScope; - +export class Publisher { + public tracks: LocalTrack[] = []; /** - * Creates a new PublishConnection. - * @param args - The connection options. {@link ConnectionOpts} + * Creates a new Publisher. + * @param scope - The observable scope to use for managing the publisher. + * @param connection - The connection to use for publishing. * @param devices - The media devices to use for audio and video input. * @param muteStates - The mute states for audio and video. * @param e2eeLivekitOptions - The E2EE options to use for the LiveKit room. Use to share the same key provider across connections!. * @param trackerProcessorState$ - The processor state for the video track processor (e.g. background blur). */ public constructor( - args: ConnectionOpts, + private scope: ObservableScope, + private connection: Connection, devices: MediaDevices, private readonly muteStates: MuteStates, - e2eeLivekitOptions: E2EEOptions | undefined, trackerProcessorState$: Behavior, + private logger?: Logger, ) { - const { scope } = args; - logger.info("[PublishConnection] Create LiveKit room"); + this.logger?.info("[PublishConnection] Create LiveKit room"); const { controlledAudioDevices } = getUrlParams(); - const factory = - args.livekitRoomFactory ?? - ((options: RoomOptions): LivekitRoom => new LivekitRoom(options)); - const room = factory( - generateRoomOption( - devices, - trackerProcessorState$.value, - controlledAudioDevices, - e2eeLivekitOptions, - ), - ); - room.setE2EEEnabled(e2eeLivekitOptions !== undefined)?.catch((e) => { - logger.error("Failed to set E2EE enabled on room", e); - }); + const room = connection.livekitRoom; - super(room, args); - this.scope = scope; + room.setE2EEEnabled(room.options.e2ee !== undefined)?.catch((e: Error) => { + this.logger?.error("Failed to set E2EE enabled on room", e); + }); // Setup track processor syncing (blur) this.observeTrackProcessors(scope, room, trackerProcessorState$); @@ -85,61 +73,118 @@ export class PublishConnection extends Connection { this.observeMediaDevices(scope, devices, controlledAudioDevices); this.workaroundRestartAudioInputTrackChrome(devices, scope); + this.scope.onEnd(() => { + this.logger?.info( + "[PublishConnection] Scope ended -> stop publishing all tracks", + ); + void this.stopPublishing(); + }); } /** * Start the connection to LiveKit and publish local tracks. * * This will: - * 1. Request an OpenId token `request_token` (allows matrix users to verify their identity with a third-party service.) - * 2. Use this token to request the SFU config to the MatrixRtc authentication service. - * 3. Connect to the configured LiveKit room. - * 4. Create local audio and video tracks based on the current mute states and publish them to the room. + * wait for the connection to be ready. + // * 1. Request an OpenId token `request_token` (allows matrix users to verify their identity with a third-party service.) + // * 2. Use this token to request the SFU config to the MatrixRtc authentication service. + // * 3. Connect to the configured LiveKit room. + // * 4. Create local audio and video tracks based on the current mute states and publish them to the room. * * @throws {InsufficientCapacityError} if the LiveKit server indicates that it has insufficient capacity to accept the connection. * @throws {SFURoomCreationRestrictedError} if the LiveKit server indicates that the room does not exist and cannot be created. */ - public async start(): Promise { - this.stopped = false; - + public async createAndSetupTracks(): Promise { + const lkRoom = this.connection.livekitRoom; // Observe mute state changes and update LiveKit microphone/camera states accordingly this.observeMuteStates(this.scope); + // TODO: This should be an autostarted connection no need to start here. just check the connection state. // TODO: This will fetch the JWT token. Perhaps we could keep it preloaded // instead? This optimization would only be safe for a publish connection, // because we don't want to leak the user's intent to perhaps join a call to // remote servers before they actually commit to it. - await super.start(); - - if (this.stopped) return; - + // const { promise, resolve, reject } = Promise.withResolvers(); + // const sub = this.connection.state$.subscribe((s) => { + // if (s.state === "FailedToStart") { + // reject(new Error("Disconnected from LiveKit server")); + // } else if (s.state === "ConnectedToLkRoom") { + // resolve(); + // } + // }); + // try { + // await promise; + // } catch (e) { + // throw e; + // } finally { + // sub.unsubscribe(); + // } // TODO-MULTI-SFU: Prepublish a microphone track const audio = this.muteStates.audio.enabled$.value; const video = this.muteStates.video.enabled$.value; // createTracks throws if called with audio=false and video=false if (audio || video) { // TODO this can still throw errors? It will also prompt for permissions if not already granted - const tracks = await this.livekitRoom.localParticipant.createTracks({ - audio, - video, - }); - if (this.stopped) return; - for (const track of tracks) { - // TODO: handle errors? Needs the signaling connection to be up, but it has some retries internally - // with a timeout. - await this.livekitRoom.localParticipant.publishTrack(track); - if (this.stopped) return; - // TODO: check if the connection is still active? and break the loop if not? - } + this.tracks = + (await lkRoom.localParticipant + .createTracks({ + audio, + video, + }) + .catch((error) => { + this.logger?.error("Failed to create tracks", error); + })) ?? []; } + return this.tracks; } - public async stop(): Promise { + public async startPublishing(): Promise { + const lkRoom = this.connection.livekitRoom; + const { promise, resolve, reject } = Promise.withResolvers(); + const sub = this.connection.state$.subscribe((s) => { + switch (s.state) { + case "ConnectedToLkRoom": + resolve(); + break; + case "FailedToStart": + reject(new Error("Failed to connect to LiveKit server")); + break; + default: + this.logger?.info("waiting for connection: ", s.state); + } + }); + try { + await promise; + } catch (e) { + throw e; + } finally { + sub.unsubscribe(); + } + for (const track of this.tracks) { + // TODO: handle errors? Needs the signaling connection to be up, but it has some retries internally + // with a timeout. + await lkRoom.localParticipant.publishTrack(track).catch((error) => { + this.logger?.error("Failed to publish track", error); + }); + + // TODO: check if the connection is still active? and break the loop if not? + } + return this.tracks; + } + + public async stopPublishing(): Promise { // TODO-MULTI-SFU: Move these calls back to ObservableScope.onEnd once scope // actually has the right lifetime this.muteStates.audio.unsetHandler(); this.muteStates.video.unsetHandler(); - await super.stop(); + + const localParticipant = this.connection.livekitRoom.localParticipant; + const tracks: LocalTrack[] = []; + const addToTracksIfDefined = (p: LocalTrackPublication): void => { + if (p.track !== undefined) tracks.push(p.track); + }; + localParticipant.trackPublications.forEach(addToTracksIfDefined); + await localParticipant.unpublishTracks(tracks); } /// Private methods @@ -156,15 +201,16 @@ export class PublishConnection extends Connection { devices: MediaDevices, scope: ObservableScope, ): void { + const lkRoom = this.connection.livekitRoom; devices.audioInput.selected$ .pipe( switchMap((device) => device?.hardwareDeviceChange$ ?? NEVER), scope.bind(), ) .subscribe(() => { - if (this.livekitRoom.state != ConnectionState.Connected) return; + if (lkRoom.state != LivekitConnectionState.Connected) return; const activeMicTrack = Array.from( - this.livekitRoom.localParticipant.audioTrackPublications.values(), + lkRoom.localParticipant.audioTrackPublications.values(), ).find((d) => d.source === Track.Source.Microphone)?.track; if ( @@ -179,11 +225,11 @@ export class PublishConnection extends Connection { // getUserMedia() call with deviceId: default to get the *new* default device. // Note that room.switchActiveDevice() won't work: Livekit will ignore it because // the deviceId hasn't changed (was & still is default). - this.livekitRoom.localParticipant + lkRoom.localParticipant .getTrackPublication(Track.Source.Microphone) ?.audioTrack?.restartTrack() .catch((e) => { - logger.error(`Failed to restart audio device track`, e); + this.logger?.error(`Failed to restart audio device track`, e); }); } }); @@ -195,27 +241,31 @@ export class PublishConnection extends Connection { devices: MediaDevices, controlledAudioDevices: boolean, ): void { + const lkRoom = this.connection.livekitRoom; const syncDevice = ( kind: MediaDeviceKind, selected$: Observable, ): Subscription => selected$.pipe(scope.bind()).subscribe((device) => { - if (this.livekitRoom.state != ConnectionState.Connected) return; + if (lkRoom.state != LivekitConnectionState.Connected) return; // if (this.connectionState$.value !== ConnectionState.Connected) return; - logger.info( + this.logger?.info( "[LivekitRoom] syncDevice room.getActiveDevice(kind) !== d.id :", - this.livekitRoom.getActiveDevice(kind), + lkRoom.getActiveDevice(kind), " !== ", device?.id, ); if ( device !== undefined && - this.livekitRoom.getActiveDevice(kind) !== device.id + lkRoom.getActiveDevice(kind) !== device.id ) { - this.livekitRoom + lkRoom .switchActiveDevice(kind, device.id) - .catch((e) => - logger.error(`Failed to sync ${kind} device with LiveKit`, e), + .catch((e: Error) => + this.logger?.error( + `Failed to sync ${kind} device with LiveKit`, + e, + ), ); } }); @@ -232,21 +282,28 @@ export class PublishConnection extends Connection { * @private */ private observeMuteStates(scope: ObservableScope): void { + const lkRoom = this.connection.livekitRoom; this.muteStates.audio.setHandler(async (desired) => { try { - await this.livekitRoom.localParticipant.setMicrophoneEnabled(desired); + await lkRoom.localParticipant.setMicrophoneEnabled(desired); } catch (e) { - logger.error("Failed to update LiveKit audio input mute state", e); + this.logger?.error( + "Failed to update LiveKit audio input mute state", + e, + ); } - return this.livekitRoom.localParticipant.isMicrophoneEnabled; + return lkRoom.localParticipant.isMicrophoneEnabled; }); this.muteStates.video.setHandler(async (desired) => { try { - await this.livekitRoom.localParticipant.setCameraEnabled(desired); + await lkRoom.localParticipant.setCameraEnabled(desired); } catch (e) { - logger.error("Failed to update LiveKit video input mute state", e); + this.logger?.error( + "Failed to update LiveKit video input mute state", + e, + ); } - return this.livekitRoom.localParticipant.isCameraEnabled; + return lkRoom.localParticipant.isCameraEnabled; }); } @@ -262,37 +319,8 @@ export class PublishConnection extends Connection { return track instanceof LocalVideoTrack ? track : null; }), ), + null, ); - trackProcessorSync(track$, trackerProcessorState$); + trackProcessorSync(scope, track$, trackerProcessorState$); } } - -// Generate the initial LiveKit RoomOptions based on the current media devices and processor state. -function generateRoomOption( - devices: MediaDevices, - processorState: ProcessorState, - controlledAudioDevices: boolean, - e2eeLivekitOptions: E2EEOptions | undefined, -): RoomOptions { - return { - ...defaultLiveKitOptions, - videoCaptureDefaults: { - ...defaultLiveKitOptions.videoCaptureDefaults, - deviceId: devices.videoInput.selected$.value?.id, - processor: processorState.processor, - }, - audioCaptureDefaults: { - ...defaultLiveKitOptions.audioCaptureDefaults, - deviceId: devices.audioInput.selected$.value?.id, - }, - audioOutput: { - // When using controlled audio devices, we don't want to set the - // deviceId here, because it will be set by the native app. - // (also the id does not need to match a browser device id) - deviceId: controlledAudioDevices - ? undefined - : getValue(devices.audioOutput.selected$)?.id, - }, - e2ee: e2eeLivekitOptions, - }; -} diff --git a/src/state/Connection.test.ts b/src/state/CallViewModel/remoteMembers/Connection.test.ts similarity index 52% rename from src/state/Connection.test.ts rename to src/state/CallViewModel/remoteMembers/Connection.test.ts index b5389db4..3f58bcf6 100644 --- a/src/state/Connection.test.ts +++ b/src/state/CallViewModel/remoteMembers/Connection.test.ts @@ -1,4 +1,5 @@ /* +Copyright 2025 Element Creations Ltd. Copyright 2025 New Vector Ltd. SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial @@ -10,41 +11,32 @@ import { describe, expect, it, - type Mock, type MockedObject, onTestFinished, vi, } from "vitest"; -import { BehaviorSubject, of } from "rxjs"; import { - ConnectionState, type LocalParticipant, type RemoteParticipant, type Room as LivekitRoom, RoomEvent, - type RoomOptions, + ConnectionState as LivekitConnectionState, } from "livekit-client"; import fetchMock from "fetch-mock"; import EventEmitter from "events"; import { type IOpenIDToken } from "matrix-js-sdk"; +import { logger } from "matrix-js-sdk/lib/logger"; -import type { - CallMembership, - LivekitTransport, -} from "matrix-js-sdk/lib/matrixrtc"; +import type { LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; import { + Connection, type ConnectionOpts, - type TransportState, + type ConnectionState, type PublishingParticipant, - RemoteConnection, } from "./Connection.ts"; -import { ObservableScope } from "./ObservableScope.ts"; -import { type OpenIDClientParts } from "../livekit/openIDSFU.ts"; -import { FailToGetOpenIdToken } from "../utils/errors.ts"; -import { PublishConnection } from "./PublishConnection.ts"; -import { mockMediaDevices, mockMuteStates } from "../utils/test.ts"; -import type { ProcessorState } from "../livekit/TrackProcessorContext.tsx"; -import { type MuteStates } from "./MuteStates.ts"; +import { ObservableScope } from "../../ObservableScope.ts"; +import { type OpenIDClientParts } from "../../../livekit/openIDSFU.ts"; +import { FailToGetOpenIdToken } from "../../../utils/errors.ts"; let testScope: ObservableScope; @@ -56,9 +48,9 @@ let localParticipantEventEmiter: EventEmitter; let fakeLocalParticipant: MockedObject; let fakeRoomEventEmiter: EventEmitter; -let fakeMembershipsFocusMap$: BehaviorSubject< - { membership: CallMembership; transport: LivekitTransport }[] ->; +// let fakeMembershipsFocusMap$: BehaviorSubject< +// { membership: CallMembership; transport: LivekitTransport }[] +// >; const livekitFocus: LivekitTransport = { livekit_alias: "!roomID:example.org", @@ -77,9 +69,6 @@ function setupTest(): void { }), getDeviceId: vi.fn().mockReturnValue("ABCDEF"), } as unknown as OpenIDClientParts); - fakeMembershipsFocusMap$ = new BehaviorSubject< - { membership: CallMembership; transport: LivekitTransport }[] - >([]); localParticipantEventEmiter = new EventEmitter(); @@ -106,7 +95,7 @@ function setupTest(): void { disconnect: vi.fn(), remoteParticipants: new Map(), localParticipant: fakeLocalParticipant, - state: ConnectionState.Disconnected, + state: LivekitConnectionState.Disconnected, on: fakeRoomEventEmiter.on.bind(fakeRoomEventEmiter), off: fakeRoomEventEmiter.off.bind(fakeRoomEventEmiter), addListener: fakeRoomEventEmiter.addListener.bind(fakeRoomEventEmiter), @@ -118,11 +107,10 @@ function setupTest(): void { } as unknown as LivekitRoom); } -function setupRemoteConnection(): RemoteConnection { +function setupRemoteConnection(): Connection { const opts: ConnectionOpts = { client: client, transport: livekitFocus, - remoteTransports$: fakeMembershipsFocusMap$, scope: testScope, livekitRoomFactory: () => fakeLivekitRoom, }; @@ -139,7 +127,7 @@ function setupRemoteConnection(): RemoteConnection { fakeLivekitRoom.connect.mockResolvedValue(undefined); - return new RemoteConnection(opts, undefined); + return new Connection(opts, logger); } afterEach(() => { @@ -155,13 +143,12 @@ describe("Start connection states", () => { const opts: ConnectionOpts = { client: client, transport: livekitFocus, - remoteTransports$: fakeMembershipsFocusMap$, scope: testScope, livekitRoomFactory: () => fakeLivekitRoom, }; - const connection = new RemoteConnection(opts, undefined); + const connection = new Connection(opts, logger); - expect(connection.transportState$.getValue().state).toEqual("Initialized"); + expect(connection.state$.getValue().state).toEqual("Initialized"); }); it("fail to getOpenId token then error state", async () => { @@ -171,15 +158,14 @@ describe("Start connection states", () => { const opts: ConnectionOpts = { client: client, transport: livekitFocus, - remoteTransports$: fakeMembershipsFocusMap$, scope: testScope, livekitRoomFactory: () => fakeLivekitRoom, }; - const connection = new RemoteConnection(opts, undefined); + const connection = new Connection(opts, logger); - const capturedStates: TransportState[] = []; - const s = connection.transportState$.subscribe((value) => { + const capturedStates: ConnectionState[] = []; + const s = connection.state$.subscribe((value) => { capturedStates.push(value); }); onTestFinished(() => s.unsubscribe()); @@ -224,15 +210,14 @@ describe("Start connection states", () => { const opts: ConnectionOpts = { client: client, transport: livekitFocus, - remoteTransports$: fakeMembershipsFocusMap$, scope: testScope, livekitRoomFactory: () => fakeLivekitRoom, }; - const connection = new RemoteConnection(opts, undefined); + const connection = new Connection(opts, logger); - const capturedStates: TransportState[] = []; - const s = connection.transportState$.subscribe((value) => { + const capturedStates: ConnectionState[] = []; + const s = connection.state$.subscribe((value) => { capturedStates.push(value); }); onTestFinished(() => s.unsubscribe()); @@ -281,15 +266,14 @@ describe("Start connection states", () => { const opts: ConnectionOpts = { client: client, transport: livekitFocus, - remoteTransports$: fakeMembershipsFocusMap$, scope: testScope, livekitRoomFactory: () => fakeLivekitRoom, }; - const connection = new RemoteConnection(opts, undefined); + const connection = new Connection(opts, logger); - const capturedStates: TransportState[] = []; - const s = connection.transportState$.subscribe((value) => { + const capturedStates: ConnectionState[] = []; + const s = connection.state$.subscribe((value) => { capturedStates.push(value); }); onTestFinished(() => s.unsubscribe()); @@ -345,8 +329,8 @@ describe("Start connection states", () => { const connection = setupRemoteConnection(); - const capturedStates: TransportState[] = []; - const s = connection.transportState$.subscribe((value) => { + const capturedStates: ConnectionState[] = []; + const s = connection.state$.subscribe((value) => { capturedStates.push(value); }); onTestFinished(() => s.unsubscribe()); @@ -379,21 +363,18 @@ describe("Start connection states", () => { }); }); -function fakeRemoteLivekitParticipant(id: string): RemoteParticipant { +function fakeRemoteLivekitParticipant( + id: string, + publications: number = 1, +): RemoteParticipant { return { identity: id, + getTrackPublications: () => Array(publications), } as unknown as RemoteParticipant; } -function fakeRtcMemberShip(userId: string, deviceId: string): CallMembership { - return { - userId, - deviceId, - } as unknown as CallMembership; -} - describe("Publishing participants observations", () => { - it("should emit the list of publishing participants", async () => { + it("should emit the list of publishing participants", () => { setupTest(); const connection = setupRemoteConnection(); @@ -401,135 +382,53 @@ describe("Publishing participants observations", () => { const bobIsAPublisher = Promise.withResolvers(); const danIsAPublisher = Promise.withResolvers(); const observedPublishers: PublishingParticipant[][] = []; - const s = connection.publishingParticipants$.subscribe((publishers) => { - observedPublishers.push(publishers); - if ( - publishers.some( - (p) => p.participant?.identity === "@bob:example.org:DEV111", - ) - ) { - bobIsAPublisher.resolve(); - } - if ( - publishers.some( - (p) => p.participant?.identity === "@dan:example.org:DEV333", - ) - ) { - danIsAPublisher.resolve(); - } - }); + const s = connection.remoteParticipantsWithTracks$.subscribe( + (publishers) => { + observedPublishers.push(publishers); + if (publishers.some((p) => p.identity === "@bob:example.org:DEV111")) { + bobIsAPublisher.resolve(); + } + if (publishers.some((p) => p.identity === "@dan:example.org:DEV333")) { + danIsAPublisher.resolve(); + } + }, + ); onTestFinished(() => s.unsubscribe()); // The publishingParticipants$ observable is derived from the current members of the // livekitRoom and the rtc membership in order to publish the members that are publishing // on this connection. let participants: RemoteParticipant[] = [ - fakeRemoteLivekitParticipant("@alice:example.org:DEV000"), - fakeRemoteLivekitParticipant("@bob:example.org:DEV111"), - fakeRemoteLivekitParticipant("@carol:example.org:DEV222"), - fakeRemoteLivekitParticipant("@dan:example.org:DEV333"), + fakeRemoteLivekitParticipant("@alice:example.org:DEV000", 0), + fakeRemoteLivekitParticipant("@bob:example.org:DEV111", 0), + fakeRemoteLivekitParticipant("@carol:example.org:DEV222", 0), + fakeRemoteLivekitParticipant("@dan:example.org:DEV333", 0), ]; // Let's simulate 3 members on the livekitRoom - vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get").mockReturnValue( - new Map(participants.map((p) => [p.identity, p])), + vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get").mockImplementation( + () => new Map(participants.map((p) => [p.identity, p])), ); - for (const participant of participants) { - fakeRoomEventEmiter.emit(RoomEvent.ParticipantConnected, participant); - } + participants.forEach((p) => + fakeRoomEventEmiter.emit(RoomEvent.ParticipantConnected, p), + ); // At this point there should be no publishers expect(observedPublishers.pop()!.length).toEqual(0); - const otherFocus: LivekitTransport = { - livekit_alias: "!roomID:example.org", - livekit_service_url: "https://other-matrix-rtc.example.org/livekit/jwt", - type: "livekit", - }; - - const rtcMemberships = [ - // Say bob is on the same focus - { - membership: fakeRtcMemberShip("@bob:example.org", "DEV111"), - transport: livekitFocus, - }, - // Alice and carol is on a different focus - { - membership: fakeRtcMemberShip("@alice:example.org", "DEV000"), - transport: otherFocus, - }, - { - membership: fakeRtcMemberShip("@carol:example.org", "DEV222"), - transport: otherFocus, - }, - // NO DAVE YET + participants = [ + fakeRemoteLivekitParticipant("@alice:example.org:DEV000", 1), + fakeRemoteLivekitParticipant("@bob:example.org:DEV111", 1), + fakeRemoteLivekitParticipant("@carol:example.org:DEV222", 1), + fakeRemoteLivekitParticipant("@dan:example.org:DEV333", 2), ]; - // signal this change in rtc memberships - fakeMembershipsFocusMap$.next(rtcMemberships); - - // We should have bob has a publisher now - await bobIsAPublisher.promise; - const publishers = observedPublishers.pop(); - expect(publishers?.length).toEqual(1); - expect(publishers?.[0].participant?.identity).toEqual( - "@bob:example.org:DEV111", + participants.forEach((p) => + fakeRoomEventEmiter.emit(RoomEvent.ParticipantConnected, p), ); - // Now let's make dan join the rtc memberships - rtcMemberships.push({ - membership: fakeRtcMemberShip("@dan:example.org", "DEV333"), - transport: livekitFocus, - }); - fakeMembershipsFocusMap$.next(rtcMemberships); - - // We should have bob and dan has publishers now - await danIsAPublisher.promise; - const twoPublishers = observedPublishers.pop(); - expect(twoPublishers?.length).toEqual(2); - expect( - twoPublishers?.some( - (p) => p.participant?.identity === "@bob:example.org:DEV111", - ), - ).toBeTruthy(); - expect( - twoPublishers?.some( - (p) => p.participant?.identity === "@dan:example.org:DEV333", - ), - ).toBeTruthy(); - - // Now let's make bob leave the livekit room - participants = participants.filter( - (p) => p.identity !== "@bob:example.org:DEV111", - ); - vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get").mockReturnValue( - new Map(participants.map((p) => [p.identity, p])), - ); - fakeRoomEventEmiter.emit( - RoomEvent.ParticipantDisconnected, - fakeRemoteLivekitParticipant("@bob:example.org:DEV111"), - ); - - const updatedPublishers = observedPublishers.pop(); - // Bob is not connected to the room but he is still in the rtc memberships declaring that - // he is using that focus to publish, so he should still appear as a publisher - expect(updatedPublishers?.length).toEqual(2); - const pp = updatedPublishers?.find( - (p) => p.membership.userId == "@bob:example.org", - ); - expect(pp).toBeDefined(); - expect(pp!.participant).not.toBeDefined(); - expect( - updatedPublishers?.some( - (p) => p.participant?.identity === "@dan:example.org:DEV333", - ), - ).toBeTruthy(); - // Now if bob is not in the rtc memberships, he should disappear - const noBob = rtcMemberships.filter( - ({ membership }) => membership.userId !== "@bob:example.org", - ); - fakeMembershipsFocusMap$.next(noBob); - expect(observedPublishers.pop()?.length).toEqual(1); + // At this point there should be no publishers + expect(observedPublishers.pop()!.length).toEqual(4); }); it("should be scoped to parent scope", (): void => { @@ -538,18 +437,20 @@ describe("Publishing participants observations", () => { const connection = setupRemoteConnection(); let observedPublishers: PublishingParticipant[][] = []; - const s = connection.publishingParticipants$.subscribe((publishers) => { - observedPublishers.push(publishers); - }); + const s = connection.remoteParticipantsWithTracks$.subscribe( + (publishers) => { + observedPublishers.push(publishers); + }, + ); onTestFinished(() => s.unsubscribe()); let participants: RemoteParticipant[] = [ - fakeRemoteLivekitParticipant("@bob:example.org:DEV111"), + fakeRemoteLivekitParticipant("@bob:example.org:DEV111", 0), ]; // Let's simulate 3 members on the livekitRoom - vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get").mockReturnValue( - new Map(participants.map((p) => [p.identity, p])), + vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get").mockImplementation( + () => new Map(participants.map((p) => [p.identity, p])), ); for (const participant of participants) { @@ -559,22 +460,16 @@ describe("Publishing participants observations", () => { // At this point there should be no publishers expect(observedPublishers.pop()!.length).toEqual(0); - const rtcMemberships = [ - // Say bob is on the same focus - { - membership: fakeRtcMemberShip("@bob:example.org", "DEV111"), - transport: livekitFocus, - }, - ]; - // signal this change in rtc memberships - fakeMembershipsFocusMap$.next(rtcMemberships); + participants = [fakeRemoteLivekitParticipant("@bob:example.org:DEV111", 1)]; + + for (const participant of participants) { + fakeRoomEventEmiter.emit(RoomEvent.ParticipantConnected, participant); + } // We should have bob has a publisher now const publishers = observedPublishers.pop(); expect(publishers?.length).toEqual(1); - expect(publishers?.[0].participant?.identity).toEqual( - "@bob:example.org:DEV111", - ); + expect(publishers?.[0]?.identity).toEqual("@bob:example.org:DEV111"); // end the parent scope testScope.end(); @@ -584,9 +479,7 @@ describe("Publishing participants observations", () => { participants = participants.filter( (p) => p.identity !== "@bob:example.org:DEV111", ); - vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get").mockReturnValue( - new Map(participants.map((p) => [p.identity, p])), - ); + fakeRoomEventEmiter.emit( RoomEvent.ParticipantDisconnected, fakeRemoteLivekitParticipant("@bob:example.org:DEV111"), @@ -596,108 +489,112 @@ describe("Publishing participants observations", () => { }); }); -describe("PublishConnection", () => { - // let fakeBlurProcessor: ProcessorWrapper; - let roomFactoryMock: Mock<() => LivekitRoom>; - let muteStates: MockedObject; +// +// NOT USED ANYMORE ? +// +// This setup look like sth for the Publisher. Not a connection. - function setUpPublishConnection(): void { - setupTest(); +// describe("PublishConnection", () => { +// // let fakeBlurProcessor: ProcessorWrapper; +// let roomFactoryMock: Mock<() => LivekitRoom>; +// let muteStates: MockedObject; - roomFactoryMock = vi.fn().mockReturnValue(fakeLivekitRoom); +// function setUpPublishConnection(): void { +// setupTest(); - muteStates = mockMuteStates(); +// roomFactoryMock = vi.fn().mockReturnValue(fakeLivekitRoom); - // fakeBlurProcessor = vi.mocked>({ - // name: "BackgroundBlur", - // restart: vi.fn().mockResolvedValue(undefined), - // setOptions: vi.fn().mockResolvedValue(undefined), - // getOptions: vi.fn().mockReturnValue({ strength: 0.5 }), - // isRunning: vi.fn().mockReturnValue(false) - // }); - } +// muteStates = mockMuteStates(); - describe("Livekit room creation", () => { - function createSetup(): void { - setUpPublishConnection(); +// // fakeBlurProcessor = vi.mocked>({ +// // name: "BackgroundBlur", +// // restart: vi.fn().mockResolvedValue(undefined), +// // setOptions: vi.fn().mockResolvedValue(undefined), +// // getOptions: vi.fn().mockReturnValue({ strength: 0.5 }), +// // isRunning: vi.fn().mockReturnValue(false) +// // }); +// } - const fakeTrackProcessorSubject$ = new BehaviorSubject({ - supported: true, - processor: undefined, - }); +// describe("Livekit room creation", () => { +// function createSetup(): void { +// setUpPublishConnection(); - const opts: ConnectionOpts = { - client: client, - transport: livekitFocus, - remoteTransports$: fakeMembershipsFocusMap$, - scope: testScope, - livekitRoomFactory: roomFactoryMock, - }; +// const fakeTrackProcessorSubject$ = new BehaviorSubject({ +// supported: true, +// processor: undefined, +// }); - const audioInput = { - available$: of(new Map([["mic1", { id: "mic1" }]])), - selected$: new BehaviorSubject({ id: "mic1" }), - select(): void {}, - }; +// const opts: ConnectionOpts = { +// client: client, +// transport: livekitFocus, +// scope: testScope, +// livekitRoomFactory: roomFactoryMock, +// }; - const videoInput = { - available$: of(new Map([["cam1", { id: "cam1" }]])), - selected$: new BehaviorSubject({ id: "cam1" }), - select(): void {}, - }; +// const audioInput = { +// available$: of(new Map([["mic1", { id: "mic1" }]])), +// selected$: new BehaviorSubject({ id: "mic1" }), +// select(): void {}, +// }; - const audioOutput = { - available$: of(new Map([["speaker", { id: "speaker" }]])), - selected$: new BehaviorSubject({ id: "speaker" }), - select(): void {}, - }; +// const videoInput = { +// available$: of(new Map([["cam1", { id: "cam1" }]])), +// selected$: new BehaviorSubject({ id: "cam1" }), +// select(): void {}, +// }; - // TODO understand what is wrong with our mocking that requires ts-expect-error - const fakeDevices = mockMediaDevices({ - // @ts-expect-error Mocking only - audioInput, - // @ts-expect-error Mocking only - videoInput, - // @ts-expect-error Mocking only - audioOutput, - }); +// const audioOutput = { +// available$: of(new Map([["speaker", { id: "speaker" }]])), +// selected$: new BehaviorSubject({ id: "speaker" }), +// select(): void {}, +// }; - new PublishConnection( - opts, - fakeDevices, - muteStates, - undefined, - fakeTrackProcessorSubject$, - ); - } +// // TODO understand what is wrong with our mocking that requires ts-expect-error +// const fakeDevices = mockMediaDevices({ +// // @ts-expect-error Mocking only +// audioInput, +// // @ts-expect-error Mocking only +// videoInput, +// // @ts-expect-error Mocking only +// audioOutput, +// }); - it("should create room with proper initial audio and video settings", () => { - createSetup(); +// new Connection( +// opts, +// fakeDevices, +// muteStates, +// undefined, +// fakeTrackProcessorSubject$, +// ); +// } - expect(roomFactoryMock).toHaveBeenCalled(); +// it("should create room with proper initial audio and video settings", () => { +// createSetup(); - const lastCallArgs = - roomFactoryMock.mock.calls[roomFactoryMock.mock.calls.length - 1]; +// expect(roomFactoryMock).toHaveBeenCalled(); - const roomOptions = lastCallArgs.pop() as unknown as RoomOptions; - expect(roomOptions).toBeDefined(); +// const lastCallArgs = +// roomFactoryMock.mock.calls[roomFactoryMock.mock.calls.length - 1]; - expect(roomOptions!.videoCaptureDefaults?.deviceId).toEqual("cam1"); - expect(roomOptions!.audioCaptureDefaults?.deviceId).toEqual("mic1"); - expect(roomOptions!.audioOutput?.deviceId).toEqual("speaker"); - }); +// const roomOptions = lastCallArgs.pop() as unknown as RoomOptions; +// expect(roomOptions).toBeDefined(); - it("respect controlledAudioDevices", () => { - // TODO: Refactor the code to make it testable. - // The UrlParams module is a singleton has a cache and is very hard to test. - // This breaks other tests as well if not handled properly. - // vi.mock(import("./../UrlParams"), () => { - // return { - // getUrlParams: vi.fn().mockReturnValue({ - // controlledAudioDevices: true - // }) - // }; - // }); - }); - }); -}); +// expect(roomOptions!.videoCaptureDefaults?.deviceId).toEqual("cam1"); +// expect(roomOptions!.audioCaptureDefaults?.deviceId).toEqual("mic1"); +// expect(roomOptions!.audioOutput?.deviceId).toEqual("speaker"); +// }); + +// it("respect controlledAudioDevices", () => { +// // TODO: Refactor the code to make it testable. +// // The UrlParams module is a singleton has a cache and is very hard to test. +// // This breaks other tests as well if not handled properly. +// // vi.mock(import("./../UrlParams"), () => { +// // return { +// // getUrlParams: vi.fn().mockReturnValue({ +// // controlledAudioDevices: true +// // }) +// // }; +// // }); +// }); +// }); +// }); diff --git a/src/state/Connection.ts b/src/state/CallViewModel/remoteMembers/Connection.ts similarity index 60% rename from src/state/Connection.ts rename to src/state/CallViewModel/remoteMembers/Connection.ts index 005c1359..c17fae2b 100644 --- a/src/state/Connection.ts +++ b/src/state/CallViewModel/remoteMembers/Connection.ts @@ -1,4 +1,5 @@ /* +Copyright 2025 Element Creations Ltd. Copyright 2025 New Vector Ltd. SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial @@ -11,31 +12,29 @@ import { } from "@livekit/components-core"; import { ConnectionError, - type ConnectionState, - type E2EEOptions, + type ConnectionState as LivekitConenctionState, + type Room as LivekitRoom, + type LocalParticipant, type RemoteParticipant, - Room as LivekitRoom, - type RoomOptions, + RoomEvent, } from "livekit-client"; -import { - type CallMembership, - type LivekitTransport, -} from "matrix-js-sdk/lib/matrixrtc"; -import { logger } from "matrix-js-sdk/lib/logger"; -import { BehaviorSubject, combineLatest, type Observable } from "rxjs"; +import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; +import { BehaviorSubject, map, type Observable } from "rxjs"; +import { type Logger } from "matrix-js-sdk/lib/logger"; import { getSFUConfigWithOpenID, type OpenIDClientParts, type SFUConfig, -} from "../livekit/openIDSFU"; -import { type Behavior } from "./Behavior"; -import { type ObservableScope } from "./ObservableScope"; -import { defaultLiveKitOptions } from "../livekit/options"; +} from "../../../livekit/openIDSFU.ts"; +import { type Behavior } from "../../Behavior.ts"; +import { type ObservableScope } from "../../ObservableScope.ts"; import { InsufficientCapacityError, SFURoomCreationRestrictedError, -} from "../utils/errors.ts"; +} from "../../../utils/errors.ts"; + +export type PublishingParticipant = LocalParticipant | RemoteParticipant; export interface ConnectionOpts { /** The media transport to connect to. */ @@ -44,16 +43,12 @@ export interface ConnectionOpts { client: OpenIDClientParts; /** The observable scope to use for this connection. */ scope: ObservableScope; - /** An observable of the current RTC call memberships and their associated transports. */ - remoteTransports$: Behavior< - { membership: CallMembership; transport: LivekitTransport }[] - >; /** Optional factory to create the LiveKit room, mainly for testing purposes. */ - livekitRoomFactory?: (options?: RoomOptions) => LivekitRoom; + livekitRoomFactory: () => LivekitRoom; } -export type TransportState = +export type ConnectionState = | { state: "Initialized" } | { state: "FetchingConfig"; transport: LivekitTransport } | { state: "ConnectingToLkRoom"; transport: LivekitTransport } @@ -61,26 +56,11 @@ export type TransportState = | { state: "FailedToStart"; error: Error; transport: LivekitTransport } | { state: "ConnectedToLkRoom"; - connectionState$: Observable; + livekitConnectionState$: Observable; transport: LivekitTransport; } | { state: "Stopped"; transport: LivekitTransport }; -/** - * Represents participant publishing or expected to publish on the connection. - * It is paired with its associated rtc membership. - */ -export type PublishingParticipant = { - /** - * The LiveKit participant publishing on this connection, or undefined if the participant is not currently (yet) connected to the livekit room. - */ - participant: RemoteParticipant | undefined; - /** - * The rtc call membership associated with this participant. - */ - membership: CallMembership; -}; - /** * A connection to a Matrix RTC LiveKit backend. * @@ -88,15 +68,14 @@ export type PublishingParticipant = { */ export class Connection { // Private Behavior - private readonly _transportState$ = new BehaviorSubject({ + private readonly _state$ = new BehaviorSubject({ state: "Initialized", }); /** * The current state of the connection to the media transport. */ - public readonly transportState$: Behavior = - this._transportState$; + public readonly state$: Behavior = this._state$; /** * Whether the connection has been stopped. @@ -112,13 +91,18 @@ export class Connection { * 2. Use this token to request the SFU config to the MatrixRtc authentication service. * 3. Connect to the configured LiveKit room. * + * The errors are also represented as a state in the `state$` observable. + * It is safe to ignore those errors and handle them accordingly via the `state$` observable. * @throws {InsufficientCapacityError} if the LiveKit server indicates that it has insufficient capacity to accept the connection. * @throws {SFURoomCreationRestrictedError} if the LiveKit server indicates that the room does not exist and cannot be created. */ + // TODO dont make this throw and instead store a connection error state in this class? + // TODO consider an autostart pattern... public async start(): Promise { + this.logger.debug("Starting Connection"); this.stopped = false; try { - this._transportState$.next({ + this._state$.next({ state: "FetchingConfig", transport: this.transport, }); @@ -126,7 +110,7 @@ export class Connection { // If we were stopped while fetching the config, don't proceed to connect if (this.stopped) return; - this._transportState$.next({ + this._state$.next({ state: "ConnectingToLkRoom", transport: this.transport, }); @@ -157,13 +141,14 @@ export class Connection { // If we were stopped while connecting, don't proceed to update state. if (this.stopped) return; - this._transportState$.next({ + this._state$.next({ state: "ConnectedToLkRoom", transport: this.transport, - connectionState$: connectionStateObserver(this.livekitRoom), + livekitConnectionState$: connectionStateObserver(this.livekitRoom), }); } catch (error) { - this._transportState$.next({ + this.logger.debug(`Failed to connect to LiveKit room: ${error}`); + this._state$.next({ state: "FailedToStart", error: error instanceof Error ? error : new Error(`${error}`), transport: this.transport, @@ -179,6 +164,7 @@ export class Connection { this.transport.livekit_alias, ); } + /** * Stops the connection. * @@ -186,9 +172,12 @@ export class Connection { * If the connection is already stopped, this is a no-op. */ public async stop(): Promise { + this.logger.debug( + `Stopping connection to ${this.transport.livekit_service_url}`, + ); if (this.stopped) return; await this.livekitRoom.disconnect(); - this._transportState$.next({ + this._state$.next({ state: "Stopped", transport: this.transport, }); @@ -196,11 +185,13 @@ export class Connection { } /** - * An observable of the participants that are publishing on this connection. + * An observable of the participants that are publishing on this connection. (Excluding our local participant) * This is derived from `participantsIncludingSubscribers$` and `remoteTransports$`. * It filters the participants to only those that are associated with a membership that claims to publish on this connection. */ - public readonly publishingParticipants$: Behavior; + public readonly remoteParticipantsWithTracks$: Behavior< + PublishingParticipant[] + >; /** * The media transport to connect to. @@ -208,79 +199,50 @@ export class Connection { public readonly transport: LivekitTransport; private readonly client: OpenIDClientParts; + public readonly livekitRoom: LivekitRoom; + + private readonly logger: Logger; + /** * Creates a new connection to a matrix RTC LiveKit backend. * - * @param livekitRoom - LiveKit room instance to use. * @param opts - Connection options {@link ConnectionOpts}. * + * @param logger */ - protected constructor( - public readonly livekitRoom: LivekitRoom, - opts: ConnectionOpts, - ) { - logger.log( + public constructor(opts: ConnectionOpts, logger: Logger) { + this.logger = logger.getChild("[Connection]"); + this.logger.info( `[Connection] Creating new connection to ${opts.transport.livekit_service_url} ${opts.transport.livekit_alias}`, ); - const { transport, client, scope, remoteTransports$ } = opts; + const { transport, client, scope } = opts; + this.livekitRoom = opts.livekitRoomFactory(); this.transport = transport; this.client = client; - const participantsIncludingSubscribers$ = scope.behavior( - connectedParticipantsObserver(this.livekitRoom), - [], - ); - - this.publishingParticipants$ = scope.behavior( - combineLatest( - [participantsIncludingSubscribers$, remoteTransports$], - (participants, remoteTransports) => - remoteTransports - // Find all members that claim to publish on this connection - .flatMap(({ membership, transport }) => - transport.livekit_service_url === - this.transport.livekit_service_url - ? [membership] - : [], - ) - // Pair with their associated LiveKit participant (if any) - .map((membership) => { - const id = `${membership.userId}:${membership.deviceId}`; - const participant = participants.find((p) => p.identity === id); - return { participant, membership }; - }), + // REMOTE participants with track!!! + // this.remoteParticipantsWithTracks$ + this.remoteParticipantsWithTracks$ = scope.behavior( + // only tracks remote participants + connectedParticipantsObserver(this.livekitRoom, { + additionalRoomEvents: [ + RoomEvent.TrackPublished, + RoomEvent.TrackUnpublished, + ], + }).pipe( + map((participants) => { + return participants.filter( + (participant) => participant.getTrackPublications().length > 0, + ); + }), ), [], ); - scope.onEnd(() => void this.stop()); - } -} - -/** - * A remote connection to the Matrix RTC LiveKit backend. - * - * This connection is used for subscribing to remote participants. - * It does not publish any local tracks. - */ -export class RemoteConnection extends Connection { - /** - * Creates a new remote connection to a matrix RTC LiveKit backend. - * @param opts - * @param sharedE2eeOption - The shared E2EE options to use for the connection. - */ - public constructor( - opts: ConnectionOpts, - sharedE2eeOption: E2EEOptions | undefined, - ) { - const factory = - opts.livekitRoomFactory ?? - ((options: RoomOptions): LivekitRoom => new LivekitRoom(options)); - const livekitRoom = factory({ - ...defaultLiveKitOptions, - e2ee: sharedE2eeOption, + scope.onEnd(() => { + this.logger.info(`Connection scope ended, stopping connection`); + void this.stop(); }); - super(livekitRoom, opts); } } diff --git a/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts b/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts new file mode 100644 index 00000000..9f448cd9 --- /dev/null +++ b/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts @@ -0,0 +1,121 @@ +/* +Copyright 2025 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; +import { + type E2EEOptions, + Room as LivekitRoom, + type RoomOptions, + type BaseKeyProvider, +} from "livekit-client"; +import { type Logger } from "matrix-js-sdk/lib/logger"; +import E2EEWorker from "livekit-client/e2ee-worker?worker"; + +import { type ObservableScope } from "../../ObservableScope.ts"; +import { Connection } from "./Connection.ts"; +import type { OpenIDClientParts } from "../../../livekit/openIDSFU.ts"; +import type { MediaDevices } from "../../MediaDevices.ts"; +import type { Behavior } from "../../Behavior.ts"; +import type { ProcessorState } from "../../../livekit/TrackProcessorContext.tsx"; +import { defaultLiveKitOptions } from "../../../livekit/options.ts"; + +export interface ConnectionFactory { + createConnection( + transport: LivekitTransport, + scope: ObservableScope, + logger: Logger, + ): Connection; +} + +export class ECConnectionFactory implements ConnectionFactory { + private readonly livekitRoomFactory: () => LivekitRoom; + + /** + * Creates a ConnectionFactory for LiveKit connections. + * + * @param client - The OpenID client parts for authentication, needed to get openID and JWT tokens. + * @param devices - Used for video/audio out/in capture options. + * @param processorState$ - Effects like background blur (only for publishing connection?) + * @param e2eeLivekitOptions - The E2EE options to use for the LiveKit Room. + * @param controlledAudioDevices - Option to indicate whether audio output device is controlled externally (native mobile app). + * @param livekitRoomFactory - Optional factory function (for testing) to create LivekitRoom instances. If not provided, a default factory is used. + */ + public constructor( + private client: OpenIDClientParts, + private devices: MediaDevices, + private processorState$: Behavior, + livekitKeyProvider: BaseKeyProvider | undefined, + private controlledAudioDevices: boolean, + livekitRoomFactory?: () => LivekitRoom, + ) { + const defaultFactory = (): LivekitRoom => + new LivekitRoom( + generateRoomOption( + this.devices, + this.processorState$.value, + livekitKeyProvider && { + keyProvider: livekitKeyProvider, + // It's important that every room use a separate E2EE worker. + // They get confused if given streams from multiple rooms. + worker: new E2EEWorker(), + }, + this.controlledAudioDevices, + ), + ); + this.livekitRoomFactory = livekitRoomFactory ?? defaultFactory; + } + + public createConnection( + transport: LivekitTransport, + scope: ObservableScope, + logger: Logger, + ): Connection { + return new Connection( + { + transport, + client: this.client, + scope: scope, + livekitRoomFactory: this.livekitRoomFactory, + }, + logger, + ); + } +} + +/** + * Generate the initial LiveKit RoomOptions based on the current media devices and processor state. + */ +function generateRoomOption( + devices: MediaDevices, + processorState: ProcessorState, + e2eeLivekitOptions: E2EEOptions | undefined, + controlledAudioDevices: boolean, +): RoomOptions { + return { + ...defaultLiveKitOptions, + videoCaptureDefaults: { + ...defaultLiveKitOptions.videoCaptureDefaults, + deviceId: devices.videoInput.selected$.value?.id, + processor: processorState.processor, + }, + audioCaptureDefaults: { + ...defaultLiveKitOptions.audioCaptureDefaults, + deviceId: devices.audioInput.selected$.value?.id, + }, + audioOutput: { + // When using controlled audio devices, we don't want to set the + // deviceId here, because it will be set by the native app. + // (also the id does not need to match a browser device id) + deviceId: controlledAudioDevices + ? undefined + : devices.audioOutput.selected$.value?.id, + }, + e2ee: e2eeLivekitOptions, + // TODO test and consider this: + // webAudioMix: true, + }; +} diff --git a/src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts b/src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts new file mode 100644 index 00000000..5887442c --- /dev/null +++ b/src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts @@ -0,0 +1,332 @@ +/* +Copyright 2025 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; +import { BehaviorSubject } from "rxjs"; +import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; +import { type Participant as LivekitParticipant } from "livekit-client"; +import { logger } from "matrix-js-sdk/lib/logger"; + +import { Epoch, ObservableScope } from "../../ObservableScope.ts"; +import { + createConnectionManager$, + type ConnectionManagerData, +} from "./ConnectionManager.ts"; +import { type ConnectionFactory } from "./ConnectionFactory.ts"; +import { type Connection } from "./Connection.ts"; +import { withTestScheduler } from "../../../utils/test.ts"; +import { areLivekitTransportsEqual } from "./MatrixLivekitMembers.ts"; +import { type Behavior } from "../../Behavior.ts"; + +// Some test constants + +const TRANSPORT_1: LivekitTransport = { + type: "livekit", + livekit_service_url: "https://lk.example.org", + livekit_alias: "!alias:example.org", +}; + +const TRANSPORT_2: LivekitTransport = { + type: "livekit", + livekit_service_url: "https://lk.sample.com", + livekit_alias: "!alias:sample.com", +}; + +let fakeConnectionFactory: ConnectionFactory; +let testScope: ObservableScope; + +// Can be useful to track all created connections in tests, even the disposed ones +let allCreatedConnections: Connection[]; + +beforeEach(() => { + testScope = new ObservableScope(); + allCreatedConnections = []; + fakeConnectionFactory = {} as unknown as ConnectionFactory; + vi.mocked(fakeConnectionFactory).createConnection = vi + .fn() + .mockImplementation( + (transport: LivekitTransport, scope: ObservableScope) => { + const mockConnection = { + transport, + remoteParticipantsWithTracks$: new BehaviorSubject([]), + } as unknown as Connection; + vi.mocked(mockConnection).start = vi.fn(); + vi.mocked(mockConnection).stop = vi.fn(); + // Tie the connection's lifecycle to the scope to test scope lifecycle management + scope.onEnd(() => { + void mockConnection.stop(); + }); + allCreatedConnections.push(mockConnection); + return mockConnection; + }, + ); +}); + +afterEach(() => { + testScope.end(); +}); + +describe("connections$ stream", () => { + test("Should create and start new connections for each transports", () => { + withTestScheduler(({ behavior, expectObservable }) => { + const { connections$ } = createConnectionManager$({ + scope: testScope, + connectionFactory: fakeConnectionFactory, + inputTransports$: behavior("a", { + a: new Epoch([TRANSPORT_1, TRANSPORT_2], 0), + }), + logger: logger, + }); + + expectObservable(connections$).toBe("a", { + a: expect.toSatisfy((e: Epoch) => { + const connections = e.value; + expect(connections.length).toBe(2); + + expect( + vi.mocked(fakeConnectionFactory).createConnection, + ).toHaveBeenCalledTimes(2); + + const conn1 = connections.find((c) => + areLivekitTransportsEqual(c.transport, TRANSPORT_1), + ); + expect(conn1).toBeDefined(); + expect(conn1!.start).toHaveBeenCalled(); + + const conn2 = connections.find((c) => + areLivekitTransportsEqual(c.transport, TRANSPORT_2), + ); + expect(conn2).toBeDefined(); + expect(conn2!.start).toHaveBeenCalled(); + return true; + }), + }); + }); + }); + + test("Should start connection only once", () => { + withTestScheduler(({ behavior, expectObservable }) => { + const { connections$ } = createConnectionManager$({ + scope: testScope, + connectionFactory: fakeConnectionFactory, + inputTransports$: behavior("abcdef", { + a: new Epoch([TRANSPORT_1], 0), + b: new Epoch([TRANSPORT_1], 1), + c: new Epoch([TRANSPORT_1], 2), + d: new Epoch([TRANSPORT_1], 3), + e: new Epoch([TRANSPORT_1], 4), + f: new Epoch([TRANSPORT_1, TRANSPORT_2], 5), + }), + logger: logger, + }); + + expectObservable(connections$).toBe("xxxxxa", { + x: expect.anything(), + a: expect.toSatisfy((e: Epoch) => { + const connections = e.value; + + expect(connections.length).toBe(2); + expect( + vi.mocked(fakeConnectionFactory).createConnection, + ).toHaveBeenCalledTimes(2); + + const conn2 = connections.find((c) => + areLivekitTransportsEqual(c.transport, TRANSPORT_2), + ); + expect(conn2).toBeDefined(); + + const conn1 = connections.find((c) => + areLivekitTransportsEqual(c.transport, TRANSPORT_1), + ); + expect(conn1).toBeDefined(); + expect(conn1!.start).toHaveBeenCalledOnce(); + + return true; + }), + }); + }); + }); + + test("Should cleanup connections when not needed anymore", () => { + withTestScheduler(({ behavior, expectObservable }) => { + const { connections$ } = createConnectionManager$({ + scope: testScope, + connectionFactory: fakeConnectionFactory, + inputTransports$: behavior("abc", { + a: new Epoch([TRANSPORT_1], 0), + b: new Epoch([TRANSPORT_1, TRANSPORT_2], 1), + c: new Epoch([TRANSPORT_1], 2), + }), + logger: logger, + }); + + expectObservable(connections$).toBe("xab", { + x: expect.anything(), + a: expect.toSatisfy((e: Epoch) => { + const connections = e.value; + expect(connections.length).toBe(2); + return true; + }), + b: expect.toSatisfy((e: Epoch) => { + const connections = e.value; + + expect(connections.length).toBe(1); + // The second connection should have been stopped has it is no longer needed. + const connection2 = allCreatedConnections.find((c) => + areLivekitTransportsEqual(c.transport, TRANSPORT_2), + ); + expect(connection2).toBeDefined(); + expect(connection2!.stop).toHaveBeenCalled(); + + // The first connection should still be active + const conn1 = connections[0]; + expect(conn1.stop).not.toHaveBeenCalledOnce(); + + return true; + }), + }); + }); + }); +}); + +describe("connectionManagerData$ stream", () => { + // Used in test to control fake connections' remoteParticipantsWithTracks$ streams + let fakePublishingParticipantsStreams: Map< + string, + Behavior + >; + + function keyForTransport(transport: LivekitTransport): string { + return `${transport.livekit_service_url}|${transport.livekit_alias}`; + } + + beforeEach(() => { + fakePublishingParticipantsStreams = new Map(); + + function getPublishingParticipantsFor( + transport: LivekitTransport, + ): Behavior { + return ( + fakePublishingParticipantsStreams.get(keyForTransport(transport)) ?? + new BehaviorSubject([]) + ); + } + + // need a more advanced fake connection factory + vi.mocked(fakeConnectionFactory).createConnection = vi + .fn() + .mockImplementation( + (transport: LivekitTransport, scope: ObservableScope) => { + const fakePublishingParticipants$ = new BehaviorSubject< + LivekitParticipant[] + >([]); + const mockConnection = { + transport, + remoteParticipantsWithTracks$: + getPublishingParticipantsFor(transport), + } as unknown as Connection; + vi.mocked(mockConnection).start = vi.fn(); + vi.mocked(mockConnection).stop = vi.fn(); + // Tie the connection's lifecycle to the scope to test scope lifecycle management + scope.onEnd(() => { + void mockConnection.stop(); + }); + + fakePublishingParticipantsStreams.set( + keyForTransport(transport), + fakePublishingParticipants$, + ); + return mockConnection; + }, + ); + }); + + test("Should report connections with the publishing participants", () => { + withTestScheduler(({ expectObservable, schedule, behavior }) => { + // Setup the fake participants streams behavior + // ============================== + fakePublishingParticipantsStreams.set( + keyForTransport(TRANSPORT_1), + behavior("oa-b", { + o: [], + a: [{ identity: "user1A" } as LivekitParticipant], + b: [ + { identity: "user1A" } as LivekitParticipant, + { identity: "user1B" } as LivekitParticipant, + ], + }), + ); + + fakePublishingParticipantsStreams.set( + keyForTransport(TRANSPORT_2), + behavior("o-a", { + o: [], + a: [{ identity: "user2A" } as LivekitParticipant], + }), + ); + // ============================== + + const { connectionManagerData$ } = createConnectionManager$({ + scope: testScope, + connectionFactory: fakeConnectionFactory, + inputTransports$: behavior("a", { + a: new Epoch([TRANSPORT_1, TRANSPORT_2], 0), + }), + logger, + }); + + expectObservable(connectionManagerData$).toBe("abcd", { + a: expect.toSatisfy((e) => { + const data: ConnectionManagerData = e.value; + expect(data.getConnections().length).toBe(2); + expect(data.getParticipantForTransport(TRANSPORT_1).length).toBe(0); + expect(data.getParticipantForTransport(TRANSPORT_2).length).toBe(0); + return true; + }), + b: expect.toSatisfy((e) => { + const data: ConnectionManagerData = e.value; + expect(data.getConnections().length).toBe(2); + expect(data.getParticipantForTransport(TRANSPORT_1).length).toBe(1); + expect(data.getParticipantForTransport(TRANSPORT_2).length).toBe(0); + expect(data.getParticipantForTransport(TRANSPORT_1)[0].identity).toBe( + "user1A", + ); + return true; + }), + c: expect.toSatisfy((e) => { + const data: ConnectionManagerData = e.value; + expect(data.getConnections().length).toBe(2); + expect(data.getParticipantForTransport(TRANSPORT_1).length).toBe(1); + expect(data.getParticipantForTransport(TRANSPORT_2).length).toBe(1); + expect(data.getParticipantForTransport(TRANSPORT_1)[0].identity).toBe( + "user1A", + ); + expect(data.getParticipantForTransport(TRANSPORT_2)[0].identity).toBe( + "user2A", + ); + return true; + }), + d: expect.toSatisfy((e) => { + const data: ConnectionManagerData = e.value; + expect(data.getConnections().length).toBe(2); + expect(data.getParticipantForTransport(TRANSPORT_1).length).toBe(2); + expect(data.getParticipantForTransport(TRANSPORT_2).length).toBe(1); + expect(data.getParticipantForTransport(TRANSPORT_1)[0].identity).toBe( + "user1A", + ); + expect(data.getParticipantForTransport(TRANSPORT_1)[1].identity).toBe( + "user1B", + ); + expect(data.getParticipantForTransport(TRANSPORT_2)[0].identity).toBe( + "user2A", + ); + return true; + }), + }); + }); + }); +}); diff --git a/src/state/CallViewModel/remoteMembers/ConnectionManager.ts b/src/state/CallViewModel/remoteMembers/ConnectionManager.ts new file mode 100644 index 00000000..f284c9e3 --- /dev/null +++ b/src/state/CallViewModel/remoteMembers/ConnectionManager.ts @@ -0,0 +1,231 @@ +/* +Copyright 2025 Element Creations Ltd. +Copyright 2025 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + type LivekitTransport, + type ParticipantId, +} from "matrix-js-sdk/lib/matrixrtc"; +import { BehaviorSubject, combineLatest, map, of, switchMap, tap } from "rxjs"; +import { type Logger } from "matrix-js-sdk/lib/logger"; +import { type LocalParticipant, type RemoteParticipant } from "livekit-client"; + +import { type Behavior } from "../../Behavior.ts"; +import { type Connection } from "./Connection.ts"; +import { Epoch, type ObservableScope } from "../../ObservableScope.ts"; +import { generateItemsWithEpoch } from "../../../utils/observable.ts"; +import { areLivekitTransportsEqual } from "./MatrixLivekitMembers.ts"; +import { type ConnectionFactory } from "./ConnectionFactory.ts"; + +export class ConnectionManagerData { + private readonly store: Map< + string, + [Connection, (LocalParticipant | RemoteParticipant)[]] + > = new Map(); + + public constructor() {} + + public add( + connection: Connection, + participants: (LocalParticipant | RemoteParticipant)[], + ): void { + const key = this.getKey(connection.transport); + const existing = this.store.get(key); + if (!existing) { + this.store.set(key, [connection, participants]); + } else { + existing[1].push(...participants); + } + } + + private getKey(transport: LivekitTransport): string { + return transport.livekit_service_url + "|" + transport.livekit_alias; + } + + public getConnections(): Connection[] { + return Array.from(this.store.values()).map(([connection]) => connection); + } + + public getConnectionForTransport( + transport: LivekitTransport, + ): Connection | null { + return this.store.get(this.getKey(transport))?.[0] ?? null; + } + + public getParticipantForTransport( + transport: LivekitTransport, + ): (LocalParticipant | RemoteParticipant)[] { + const key = transport.livekit_service_url + "|" + transport.livekit_alias; + const existing = this.store.get(key); + if (existing) { + return existing[1]; + } + return []; + } + /** + * Get all connections where the given participant is publishing. + * In theory, there could be several connections where the same participant is publishing but with + * only well behaving clients a participant should only be publishing on a single connection. + * @param participantId + */ + public getConnectionsForParticipant( + participantId: ParticipantId, + ): Connection[] { + const connections: Connection[] = []; + for (const [connection, participants] of this.store.values()) { + if (participants.some((p) => p.identity === participantId)) { + connections.push(connection); + } + } + return connections; + } +} +interface Props { + scope: ObservableScope; + connectionFactory: ConnectionFactory; + inputTransports$: Behavior>; + logger: Logger; +} +// TODO - write test for scopes (do we really need to bind scope) +export interface IConnectionManager { + transports$: Behavior>; + connectionManagerData$: Behavior>; + connections$: Behavior>; +} +/** + * Crete a `ConnectionManager` + * @param scope the observable scope used by this object. + * @param connectionFactory used to create new connections. + * @param _transportsSubscriptions$ A list of Behaviors each containing a LIST of LivekitTransport. + * Each of these behaviors can be interpreted as subscribed list of transports. + * + * Using `registerTransports` independent external modules can control what connections + * are created by the ConnectionManager. + * + * The connection manager will remove all duplicate transports in each subscibed list. + * + * See `unregisterAllTransports` and `unregisterTransport` for details on how to unsubscribe. + */ +export function createConnectionManager$({ + scope, + connectionFactory, + inputTransports$, + logger: parentLogger, +}: Props): IConnectionManager { + const logger = parentLogger.getChild("[ConnectionManager]"); + + const running$ = new BehaviorSubject(true); + scope.onEnd(() => running$.next(false)); + // TODO logger: only construct one logger from the client and make it compatible via a EC specific sing + + /** + * All transports currently managed by the ConnectionManager. + * + * This list does not include duplicate transports. + * + * It is build based on the list of subscribed transports (`transportsSubscriptions$`). + * externally this is modified via `registerTransports()`. + */ + const transports$ = scope.behavior( + combineLatest([running$, inputTransports$]).pipe( + map(([running, transports]) => + transports.mapInner((transport) => (running ? transport : [])), + ), + map((transports) => transports.mapInner(removeDuplicateTransports)), + tap(({ value: transports }) => { + logger.trace( + `Managing transports: ${transports.map((t) => t.livekit_service_url).join(", ")}`, + ); + }), + ), + ); + + /** + * Connections for each transport in use by one or more session members. + */ + const connections$ = scope.behavior( + transports$.pipe( + generateItemsWithEpoch( + function* (transports) { + for (const transport of transports) + yield { + keys: [transport.livekit_service_url, transport.livekit_alias], + data: undefined, + }; + }, + (scope, _data$, serviceUrl, alias) => { + logger.debug(`Creating connection to ${serviceUrl} (${alias})`); + const connection = connectionFactory.createConnection( + { + type: "livekit", + livekit_service_url: serviceUrl, + livekit_alias: alias, + }, + scope, + logger, + ); + // Start the connection immediately + // Use connection state to track connection progress + void connection.start(); + // TODO subscribe to connection state to retry or log issues? + return connection; + }, + ), + ), + ); + + const connectionManagerData$ = scope.behavior( + connections$.pipe( + switchMap((connections) => { + const epoch = connections.epoch; + + // Map the connections to list of {connection, participants}[] + const listOfConnectionsWithPublishingParticipants = + connections.value.map((connection) => { + return connection.remoteParticipantsWithTracks$.pipe( + map((participants) => ({ + connection, + participants, + })), + ); + }); + + // probably not required + if (listOfConnectionsWithPublishingParticipants.length === 0) { + return of(new Epoch(new ConnectionManagerData(), epoch)); + } + + // combineLatest the several streams into a single stream with the ConnectionManagerData + return combineLatest(listOfConnectionsWithPublishingParticipants).pipe( + map( + (lists) => + new Epoch( + lists.reduce((data, { connection, participants }) => { + data.add(connection, participants); + return data; + }, new ConnectionManagerData()), + epoch, + ), + ), + ); + }), + ), + new Epoch(new ConnectionManagerData()), + ); + + return { transports$, connectionManagerData$, connections$ }; +} + +function removeDuplicateTransports( + transports: LivekitTransport[], +): LivekitTransport[] { + return transports.reduce((acc, transport) => { + if (!acc.some((t) => areLivekitTransportsEqual(t, transport))) + acc.push(transport); + return acc; + }, [] as LivekitTransport[]); +} diff --git a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts new file mode 100644 index 00000000..e675f723 --- /dev/null +++ b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts @@ -0,0 +1,384 @@ +/* +Copyright 2025 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { describe, test, expect, beforeEach, afterEach } from "vitest"; +import { + type CallMembership, + type LivekitTransport, +} from "matrix-js-sdk/lib/matrixrtc"; +import { getParticipantId } from "matrix-js-sdk/lib/matrixrtc/utils"; +import { combineLatest, map, type Observable } from "rxjs"; + +import { type IConnectionManager } from "./ConnectionManager.ts"; +import { + type MatrixLivekitMember, + createMatrixLivekitMembers$, +} from "./MatrixLivekitMembers.ts"; +import { + Epoch, + mapEpoch, + ObservableScope, + trackEpoch, +} from "../../ObservableScope.ts"; +import { ConnectionManagerData } from "./ConnectionManager.ts"; +import { + mockCallMembership, + mockRemoteParticipant, + withTestScheduler, +} from "../../../utils/test.ts"; +import { type Connection } from "./Connection.ts"; + +let testScope: ObservableScope; + +const transportA: LivekitTransport = { + type: "livekit", + livekit_service_url: "https://lk.example.org", + livekit_alias: "!alias:example.org", +}; + +const transportB: LivekitTransport = { + type: "livekit", + livekit_service_url: "https://lk.sample.com", + livekit_alias: "!alias:sample.com", +}; + +const bobMembership = mockCallMembership( + "@bob:example.org", + "DEV000", + transportA, +); +const carlMembership = mockCallMembership( + "@carl:sample.com", + "DEV111", + transportB, +); + +beforeEach(() => { + testScope = new ObservableScope(); +}); + +afterEach(() => { + testScope.end(); +}); + +function epochMeWith$( + source$: Observable>, + me$: Observable, +): Observable> { + return combineLatest([source$, me$]).pipe( + map(([ep, cd]) => { + return new Epoch(cd, ep.epoch); + }), + ); +} + +test("should signal participant not yet connected to livekit", () => { + withTestScheduler(({ behavior, expectObservable }) => { + const { memberships$, membershipsWithTransport$ } = fromMemberships$( + behavior("a", { + a: [bobMembership], + }), + ); + + const connectionManagerData$ = epochMeWith$( + memberships$, + behavior("a", { + a: new ConnectionManagerData(), + }), + ); + + const matrixLivekitMember$ = createMatrixLivekitMembers$({ + scope: testScope, + membershipsWithTransport$: testScope.behavior(membershipsWithTransport$), + connectionManager: { + connectionManagerData$: connectionManagerData$, + } as unknown as IConnectionManager, + }); + + expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe("a", { + a: expect.toSatisfy((data: MatrixLivekitMember[]) => { + expect(data.length).toEqual(1); + expectObservable(data[0].membership$).toBe("a", { + a: bobMembership, + }); + expectObservable(data[0].participant$).toBe("a", { + a: null, + }); + expectObservable(data[0].connection$).toBe("a", { + a: null, + }); + return true; + }), + }); + }); +}); + +// Helper to create epoch'ed memberships$ and membershipsWithTransport$ from memberships observable. +function fromMemberships$(m$: Observable): { + memberships$: Observable>; + membershipsWithTransport$: Observable< + Epoch<{ membership: CallMembership; transport?: LivekitTransport }[]> + >; +} { + const memberships$ = m$.pipe(trackEpoch()); + const membershipsWithTransport$ = memberships$.pipe( + mapEpoch((members) => { + return members.map((m) => { + const tr = m.getTransport(m); + return { + membership: m, + transport: + tr?.type === "livekit" ? (tr as LivekitTransport) : undefined, + }; + }); + }), + ); + return { + memberships$, + membershipsWithTransport$, + }; +} + +test("should signal participant on a connection that is publishing", () => { + withTestScheduler(({ behavior, expectObservable }) => { + const bobParticipantId = getParticipantId( + bobMembership.userId, + bobMembership.deviceId, + ); + + const { memberships$, membershipsWithTransport$ } = fromMemberships$( + behavior("a", { + a: [bobMembership], + }), + ); + + const connection = { + transport: bobMembership.getTransport(bobMembership), + } as unknown as Connection; + const dataWithPublisher = new ConnectionManagerData(); + dataWithPublisher.add(connection, [ + mockRemoteParticipant({ identity: bobParticipantId }), + ]); + + const connectionManagerData$ = epochMeWith$( + memberships$, + behavior("a", { + a: dataWithPublisher, + }), + ); + + const matrixLivekitMember$ = createMatrixLivekitMembers$({ + scope: testScope, + membershipsWithTransport$: testScope.behavior(membershipsWithTransport$), + connectionManager: { + connectionManagerData$: connectionManagerData$, + } as unknown as IConnectionManager, + }); + + expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe("a", { + a: expect.toSatisfy((data: MatrixLivekitMember[]) => { + expect(data.length).toEqual(1); + expectObservable(data[0].membership$).toBe("a", { + a: bobMembership, + }); + expectObservable(data[0].participant$).toBe("a", { + a: expect.toSatisfy((participant) => { + expect(participant).toBeDefined(); + expect(participant!.identity).toEqual(bobParticipantId); + return true; + }), + }); + expectObservable(data[0].connection$).toBe("a", { + a: connection, + }); + return true; + }), + }); + }); +}); + +test("should signal participant on a connection that is not publishing", () => { + withTestScheduler(({ behavior, expectObservable }) => { + const { memberships$, membershipsWithTransport$ } = fromMemberships$( + behavior("a", { + a: [bobMembership], + }), + ); + + const connection = { + transport: bobMembership.getTransport(bobMembership), + } as unknown as Connection; + const dataWithPublisher = new ConnectionManagerData(); + dataWithPublisher.add(connection, []); + + const connectionManagerData$ = epochMeWith$( + memberships$, + behavior("a", { + a: dataWithPublisher, + }), + ); + + const matrixLivekitMember$ = createMatrixLivekitMembers$({ + scope: testScope, + membershipsWithTransport$: testScope.behavior(membershipsWithTransport$), + connectionManager: { + connectionManagerData$: connectionManagerData$, + } as unknown as IConnectionManager, + }); + + expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe("a", { + a: expect.toSatisfy((data: MatrixLivekitMember[]) => { + expect(data.length).toEqual(1); + expectObservable(data[0].membership$).toBe("a", { + a: bobMembership, + }); + expectObservable(data[0].participant$).toBe("a", { + a: null, + }); + expectObservable(data[0].connection$).toBe("a", { + a: connection, + }); + return true; + }), + }); + }); +}); + +describe("Publication edge case", () => { + test("bob is publishing in several connections", () => { + withTestScheduler(({ behavior, expectObservable }) => { + const { memberships$, membershipsWithTransport$ } = fromMemberships$( + behavior("a", { + a: [bobMembership, carlMembership], + }), + ); + + const connectionWithPublisher = new ConnectionManagerData(); + const bobParticipantId = getParticipantId( + bobMembership.userId, + bobMembership.deviceId, + ); + const connectionA = { + transport: transportA, + } as unknown as Connection; + const connectionB = { + transport: transportB, + } as unknown as Connection; + + connectionWithPublisher.add(connectionA, [ + mockRemoteParticipant({ identity: bobParticipantId }), + ]); + connectionWithPublisher.add(connectionB, [ + mockRemoteParticipant({ identity: bobParticipantId }), + ]); + + const connectionManagerData$ = epochMeWith$( + memberships$, + behavior("a", { + a: connectionWithPublisher, + }), + ); + + const matrixLivekitMember$ = createMatrixLivekitMembers$({ + scope: testScope, + membershipsWithTransport$: testScope.behavior( + membershipsWithTransport$, + ), + connectionManager: { + connectionManagerData$: connectionManagerData$, + } as unknown as IConnectionManager, + }); + + expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe( + "a", + { + a: expect.toSatisfy((data: MatrixLivekitMember[]) => { + expect(data.length).toEqual(2); + expectObservable(data[0].membership$).toBe("a", { + a: bobMembership, + }); + expectObservable(data[0].connection$).toBe("a", { + // The real connection should be from transportA as per the membership + a: connectionA, + }); + expectObservable(data[0].participant$).toBe("a", { + a: expect.toSatisfy((participant) => { + expect(participant).toBeDefined(); + expect(participant!.identity).toEqual(bobParticipantId); + return true; + }), + }); + return true; + }), + }, + ); + }); + }); + + test("bob is publishing in the wrong connection", () => { + withTestScheduler(({ behavior, expectObservable }) => { + const { memberships$, membershipsWithTransport$ } = fromMemberships$( + behavior("a", { + a: [bobMembership, carlMembership], + }), + ); + + const connectionWithPublisher = new ConnectionManagerData(); + const bobParticipantId = getParticipantId( + bobMembership.userId, + bobMembership.deviceId, + ); + const connectionA = { transport: transportA } as unknown as Connection; + const connectionB = { transport: transportB } as unknown as Connection; + + // Bob is not publishing on A + connectionWithPublisher.add(connectionA, []); + // Bob is publishing on B but his membership says A + connectionWithPublisher.add(connectionB, [ + mockRemoteParticipant({ identity: bobParticipantId }), + ]); + + const connectionManagerData$ = epochMeWith$( + memberships$, + behavior("a", { + a: connectionWithPublisher, + }), + ); + + const matrixLivekitMember$ = createMatrixLivekitMembers$({ + scope: testScope, + membershipsWithTransport$: testScope.behavior( + membershipsWithTransport$, + ), + connectionManager: { + connectionManagerData$: connectionManagerData$, + } as unknown as IConnectionManager, + }); + + expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe( + "a", + { + a: expect.toSatisfy((data: MatrixLivekitMember[]) => { + expect(data.length).toEqual(2); + expectObservable(data[0].membership$).toBe("a", { + a: bobMembership, + }); + expectObservable(data[0].connection$).toBe("a", { + // The real connection should be from transportA as per the membership + a: connectionA, + }); + expectObservable(data[0].participant$).toBe("a", { + // No participant as Bob is not publishing on his membership transport + a: null, + }); + return true; + }), + }, + ); + }); + }); +}); diff --git a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts new file mode 100644 index 00000000..2f152630 --- /dev/null +++ b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts @@ -0,0 +1,138 @@ +/* +Copyright 2025 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + type LocalParticipant as LocalLivekitParticipant, + type RemoteParticipant as RemoteLivekitParticipant, +} from "livekit-client"; +import { + type LivekitTransport, + type CallMembership, +} from "matrix-js-sdk/lib/matrixrtc"; +import { combineLatest, filter, map } from "rxjs"; +import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; + +import { type Behavior } from "../../Behavior"; +import { type IConnectionManager } from "./ConnectionManager"; +import { Epoch, type ObservableScope } from "../../ObservableScope"; +import { type Connection } from "./Connection"; +import { generateItemsWithEpoch } from "../../../utils/observable"; + +const logger = rootLogger.getChild("[MatrixLivekitMembers]"); + +/** + * Represents a Matrix call member and their associated LiveKit participation. + * `livekitParticipant` can be undefined if the member is not yet connected to the livekit room + * or if it has no livekit transport at all. + */ +export interface MatrixLivekitMember { + membership$: Behavior; + participant$: Behavior< + LocalLivekitParticipant | RemoteLivekitParticipant | null + >; + connection$: Behavior; + // participantId: string; We do not want a participantId here since it will be generated by the jwt + // TODO decide if we can also drop the userId. Its in the matrix membership anyways. + userId: string; +} + +interface Props { + scope: ObservableScope; + membershipsWithTransport$: Behavior< + Epoch<{ membership: CallMembership; transport?: LivekitTransport }[]> + >; + connectionManager: IConnectionManager; +} +/** + * Combines MatrixRTC and Livekit worlds. + * + * It has a small public interface: + * - in (via constructor): + * - an observable of CallMembership[] to track the call members (The matrix side) + * - a `ConnectionManager` for the lk rooms (The livekit side) + * - out (via public Observable): + * - `remoteMatrixLivekitMember` an observable of MatrixLivekitMember[] to track the remote members and associated livekit data. + */ +export function createMatrixLivekitMembers$({ + scope, + membershipsWithTransport$, + connectionManager, +}: Props): Behavior> { + /** + * Stream of all the call members and their associated livekit data (if available). + */ + + return scope.behavior( + combineLatest([ + membershipsWithTransport$, + connectionManager.connectionManagerData$, + ]).pipe( + filter((values) => + values.every((value) => value.epoch === values[0].epoch), + ), + map( + ([ + { value: membershipsWithTransports, epoch }, + { value: managerData }, + ]) => + new Epoch([membershipsWithTransports, managerData] as const, epoch), + ), + generateItemsWithEpoch( + // Generator function. + // creates an array of `{key, data}[]` + // Each change in the keys (new key, missing key) will result in a call to the factory function. + function* ([membershipsWithTransports, managerData]) { + for (const { membership, transport } of membershipsWithTransports) { + // TODO! cannot use membership.membershipID yet, Currently its hardcoded by the jwt service to + const participantId = /*membership.membershipID*/ `${membership.userId}:${membership.deviceId}`; + + const participants = transport + ? managerData.getParticipantForTransport(transport) + : []; + const participant = + participants.find((p) => p.identity == participantId) ?? null; + const connection = transport + ? managerData.getConnectionForTransport(transport) + : null; + + yield { + keys: [participantId, membership.userId], + data: { membership, participant, connection }, + }; + } + }, + // Each update where the key of the generator array do not change will result in updates to the `data$` observable in the factory. + (scope, data$, participantId, userId) => { + logger.debug( + `Updating data$ for participantId: ${participantId}, userId: ${userId}`, + ); + // will only get called once per `participantId, userId` pair. + // updates to data$ and as a result to displayName$ and mxcAvatarUrl$ are more frequent. + return { + participantId, + userId, + ...scope.splitBehavior(data$), + }; + }, + ), + ), + ); +} + +// TODO add back in the callviewmodel pauseWhen(this.pretendToBeDisconnected$) + +// TODO add this to the JS-SDK +export function areLivekitTransportsEqual( + t1: LivekitTransport | null, + t2: LivekitTransport | null, +): boolean { + if (t1 && t2) return t1.livekit_service_url === t2.livekit_service_url; + // In case we have different lk rooms in the same SFU (depends on the livekit authorization service) + // It is only needed in case the livekit authorization service is not behaving as expected (or custom implementation) + if (!t1 && !t2) return true; + return false; +} diff --git a/src/state/CallViewModel/remoteMembers/MatrixMemberMetadata.test.ts b/src/state/CallViewModel/remoteMembers/MatrixMemberMetadata.test.ts new file mode 100644 index 00000000..6f392351 --- /dev/null +++ b/src/state/CallViewModel/remoteMembers/MatrixMemberMetadata.test.ts @@ -0,0 +1,613 @@ +/* +Copyright 2025 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { afterEach, beforeEach, describe, vi } from "vitest"; +import { + type MatrixEvent, + type RoomMember, + type RoomState, + RoomStateEvent, +} from "matrix-js-sdk"; +import EventEmitter from "events"; +import { it } from "vitest"; + +import { ObservableScope } from "../../ObservableScope.ts"; +import type { Room as MatrixRoom } from "matrix-js-sdk/lib/models/room"; +import { + mockCallMembership, + mockMatrixRoomMember, + withTestScheduler, +} from "../../../utils/test.ts"; +import { + createMatrixMemberMetadata$, + createRoomMembers$, +} from "./MatrixMemberMetadata.ts"; +let testScope: ObservableScope; +let mockMatrixRoom: MatrixRoom; + +describe("MatrixMemberMetadata", () => { + /* + * To be populated in the test setup. + * Maps userId to a partial/mock RoomMember object. + */ + let fakeMembersMap: Map>; + + beforeEach(() => { + testScope = new ObservableScope(); + fakeMembersMap = new Map>(); + + const roomEmitter = new EventEmitter(); + mockMatrixRoom = { + on: roomEmitter.on.bind(roomEmitter), + off: roomEmitter.off.bind(roomEmitter), + emit: roomEmitter.emit.bind(roomEmitter), + // addListener: roomEmitter.addListener.bind(roomEmitter), + // removeListener: roomEmitter.removeListener.bind(roomEmitter), + getMember: vi.fn().mockImplementation((userId: string) => { + const member = fakeMembersMap.get(userId); + if (member) { + return member as RoomMember; + } + return null; + }), + getMembers: vi.fn().mockImplementation(() => { + const members = Array.from(fakeMembersMap.values()); + return members; + }), + getMembersWithMembership: vi.fn().mockImplementation(() => { + const members = Array.from(fakeMembersMap.values()); + return members; + }), + } as unknown as MatrixRoom; + }); + + function fakeMemberWith(data: Partial): void { + const userId = data.userId || "@alice:example.com"; + const member: Partial = { + userId: userId, + rawDisplayName: data.rawDisplayName ?? userId, + getMxcAvatarUrl: + data.getMxcAvatarUrl || + vi.fn().mockImplementation(() => { + return `mxc://example.com/${userId}`; + }), + ...data, + } as unknown as RoomMember; + fakeMembersMap.set(userId, member); + } + + afterEach(() => { + fakeMembersMap.clear(); + }); + + describe("displayname", () => { + function updateDisplayName( + userId: `@${string}:${string}`, + newDisplayName: string, + ): void { + const member = fakeMembersMap.get(userId); + if (member) { + member.rawDisplayName = newDisplayName; + // Emit the event to notify listeners + mockMatrixRoom.emit( + RoomStateEvent.Members, + {} as unknown as MatrixEvent, + {} as unknown as RoomState, + member as RoomMember, + ); + } else { + throw new Error(`No member found with userId: ${userId}`); + } + } + + it("should show our own user if present in rtc session and room", () => { + withTestScheduler(({ behavior, expectObservable }) => { + fakeMemberWith({ + userId: "@local:example.com", + rawDisplayName: "it's a me", + }); + const memberships$ = behavior("a", { + a: [mockCallMembership("@local:example.com", "DEVICE1")], + }); + const metadataStore = createMatrixMemberMetadata$( + testScope, + memberships$, + createRoomMembers$(testScope, mockMatrixRoom), + ); + const dn$ = + metadataStore.createDisplayNameBehavior$("@local:example.com"); + + expectObservable(dn$).toBe("a", { + a: "it's a me", + }); + expectObservable(metadataStore.displaynameMap$).toBe("a", { + a: new Map([["@local:example.com", "it's a me"]]), + }); + }); + }); + + function setUpBasicRoom(): void { + fakeMemberWith({ + userId: "@local:example.com", + rawDisplayName: "it's a me", + }); + fakeMemberWith({ userId: "@alice:example.com", rawDisplayName: "Alice" }); + fakeMemberWith({ userId: "@bob:example.com", rawDisplayName: "Bob" }); + fakeMemberWith({ userId: "@carl:example.com", rawDisplayName: "Carl" }); + fakeMemberWith({ userId: "@evil:example.com", rawDisplayName: "Carl" }); + fakeMemberWith({ userId: "@bob:foo.bar", rawDisplayName: "Bob" }); + fakeMemberWith({ userId: "@no-name:foo.bar" }); + } + + it("should get displayName for users", () => { + setUpBasicRoom(); + + withTestScheduler(({ behavior, expectObservable }) => { + const memberships$ = behavior("a", { + a: [ + mockCallMembership("@alice:example.com", "DEVICE1"), + mockCallMembership("@bob:example.com", "DEVICE1"), + ], + }); + const metadataStore = createMatrixMemberMetadata$( + testScope, + memberships$, + createRoomMembers$(testScope, mockMatrixRoom), + ); + const aliceDispName$ = + metadataStore.createDisplayNameBehavior$("@alice:example.com"); + + expectObservable(aliceDispName$).toBe("a", { + a: "Alice", + }); + + expectObservable(metadataStore.displaynameMap$).toBe("a", { + a: new Map([ + ["@alice:example.com", "Alice"], + ["@bob:example.com", "Bob"], + ]), + }); + }); + }); + + it("should use userId if no display name", () => { + withTestScheduler(({ behavior, expectObservable }) => { + setUpBasicRoom(); + + const memberships$ = behavior("a", { + a: [mockCallMembership("@no-name:foo.bar", "D000")], + }); + const metadataStore = createMatrixMemberMetadata$( + testScope, + memberships$, + createRoomMembers$(testScope, mockMatrixRoom), + ); + + expectObservable(metadataStore.displaynameMap$).toBe("a", { + a: new Map([ + ["@no-name:foo.bar", "@no-name:foo.bar"], + ]), + }); + }); + }); + + it("should disambiguate users with same display name", () => { + withTestScheduler(({ behavior, expectObservable }) => { + setUpBasicRoom(); + + const memberships$ = behavior("a", { + a: [ + mockCallMembership("@bob:example.com", "DEVICE1"), + mockCallMembership("@bob:example.com", "DEVICE2"), + mockCallMembership("@bob:foo.bar", "BOB000"), + mockCallMembership("@carl:example.com", "C000"), + mockCallMembership("@evil:example.com", "E000"), + ], + }); + + const metadataStore = createMatrixMemberMetadata$( + testScope, + memberships$, + createRoomMembers$(testScope, mockMatrixRoom), + ); + + expectObservable(metadataStore.displaynameMap$).toBe("a", { + a: new Map([ + // ["@local:example.com", "it's a me"], + ["@bob:example.com", "Bob (@bob:example.com)"], + ["@bob:example.com", "Bob (@bob:example.com)"], + ["@bob:foo.bar", "Bob (@bob:foo.bar)"], + ["@carl:example.com", "Carl (@carl:example.com)"], + ["@evil:example.com", "Carl (@evil:example.com)"], + ]), + }); + }); + }); + + it("should start to disambiguate reactivly when needed", () => { + withTestScheduler(({ behavior, expectObservable }) => { + setUpBasicRoom(); + + const memberships$ = behavior("ab", { + a: [mockCallMembership("@bob:example.com", "DEVICE1")], + b: [ + mockCallMembership("@bob:example.com", "DEVICE1"), + mockCallMembership("@bob:foo.bar", "BOB000"), + ], + }); + + const metadataStore = createMatrixMemberMetadata$( + testScope, + memberships$, + createRoomMembers$(testScope, mockMatrixRoom), + ); + + expectObservable(metadataStore.displaynameMap$).toBe("ab", { + a: new Map([["@bob:example.com", "Bob"]]), + b: new Map([ + ["@bob:example.com", "Bob (@bob:example.com)"], + ["@bob:foo.bar", "Bob (@bob:foo.bar)"], + ]), + }); + }); + }); + + it("should keep disambiguated name when other leave", () => { + withTestScheduler(({ behavior, expectObservable }) => { + setUpBasicRoom(); + + const memberships$ = behavior("ab", { + a: [ + mockCallMembership("@bob:example.com", "DEVICE1"), + mockCallMembership("@bob:foo.bar", "BOB000"), + ], + b: [mockCallMembership("@bob:example.com", "DEVICE1")], + }); + + const metadataStore = createMatrixMemberMetadata$( + testScope, + memberships$, + createRoomMembers$(testScope, mockMatrixRoom), + ); + + expectObservable(metadataStore.displaynameMap$).toBe("ab", { + a: new Map([ + ["@bob:example.com", "Bob (@bob:example.com)"], + ["@bob:foo.bar", "Bob (@bob:foo.bar)"], + ]), + b: new Map([ + ["@bob:example.com", "Bob (@bob:example.com)"], + ]), + }); + }); + }); + + it("should disambiguate on name change", () => { + withTestScheduler(({ behavior, schedule, expectObservable }) => { + setUpBasicRoom(); + + const memberships$ = behavior("a", { + a: [ + mockCallMembership("@bob:example.com", "B000"), + mockCallMembership("@carl:example.com", "C000"), + ], + }); + const metadataStore = createMatrixMemberMetadata$( + testScope, + memberships$, + createRoomMembers$(testScope, mockMatrixRoom), + ); + + schedule("-a", { + a: () => { + updateDisplayName("@carl:example.com", "Bob"); + }, + }); + + expectObservable(metadataStore.displaynameMap$).toBe("ab", { + a: new Map([ + ["@bob:example.com", "Bob"], + ["@carl:example.com", "Carl"], + ]), + b: new Map([ + ["@bob:example.com", "Bob (@bob:example.com)"], + ["@carl:example.com", "Bob (@carl:example.com)"], + ]), + }); + }); + }); + + it("should track individual member id with createDisplayNameBehavior", () => { + withTestScheduler(({ behavior, schedule, expectObservable }) => { + setUpBasicRoom(); + const BOB = "@bob:example.com"; + const CARL = "@carl:example.com"; + // for this test we build a mock environment that does all possible changes: + // - memberships join/leave + // - room join/leave + // - disambiguate + const memberships$ = behavior("ab-d", { + a: [mockCallMembership(CARL, "C000")], + b: [ + mockCallMembership(CARL, "C000"), + // bob joins + mockCallMembership(BOB, "B000"), + ], + // c carl gets renamed to BOB + d: [ + // carl leaves + mockCallMembership(BOB, "B000"), + ], + }); + schedule("--a-", { + a: () => { + // carl renames + updateDisplayName(CARL, "Bob"); + }, + }); + + const metadataStore = createMatrixMemberMetadata$( + testScope, + memberships$, + createRoomMembers$(testScope, mockMatrixRoom), + ); + + const bob$ = metadataStore.createDisplayNameBehavior$(BOB); + const carl$ = metadataStore.createDisplayNameBehavior$(CARL); + + expectObservable(bob$).toBe("abc-", { + a: undefined, + b: "Bob", + c: "Bob (@bob:example.com)", + // bob stays disambiguate even though carl left + // d: "Bob (@bob:example.com)", + }); + + expectObservable(carl$).toBe("a-cd", { + a: "Carl", + // b: "Carl", + // carl gets renamed and disambiguate + c: "Bob (@carl:example.com)", + d: undefined, + }); + }); + }); + + it("should disambiguate users with invisible characters", () => { + withTestScheduler(({ behavior, expectObservable }) => { + const bobRtcMember = mockCallMembership("@bob:example.org", "BBBB"); + const bobZeroWidthSpaceRtcMember = mockCallMembership( + "@bob2:example.org", + "BBBB", + ); + const bob = mockMatrixRoomMember(bobRtcMember, { + rawDisplayName: "Bob", + }); + const bobZeroWidthSpace = mockMatrixRoomMember( + bobZeroWidthSpaceRtcMember, + { + rawDisplayName: "Bo\u200bb", + }, + ); + fakeMemberWith(bob); + fakeMemberWith(bobZeroWidthSpace); + fakeMemberWith({ userId: "@carol:example.org" }); + const memberships$ = behavior("ab", { + a: [mockCallMembership("@carol:example.org", "1111"), bobRtcMember], + b: [ + mockCallMembership("@carol:example.org", "1111"), + bobRtcMember, + bobZeroWidthSpaceRtcMember, + ], + }); + + const metadataStore = createMatrixMemberMetadata$( + testScope, + memberships$, + createRoomMembers$(testScope, mockMatrixRoom), + ); + + const bob$ = + metadataStore.createDisplayNameBehavior$("@bob:example.org"); + const bob2$ = + metadataStore.createDisplayNameBehavior$("@bob2:example.org"); + const carol$ = + metadataStore.createDisplayNameBehavior$("@carol:example.org"); + expectObservable(bob$).toBe("ab", { + a: "Bob", + b: "Bob (@bob:example.org)", + }); + expectObservable(bob2$).toBe("ab", { + a: undefined, + b: "Bo\u200bb (@bob2:example.org)", + }); + expectObservable(carol$).toBe("a-", { + a: "@carol:example.org", + }); + + expectObservable(metadataStore.displaynameMap$).toBe("ab", { + // Carol has no displayname - So userId is used. + a: new Map([ + ["@carol:example.org", "@carol:example.org"], + ["@bob:example.org", "Bob"], + ]), + // Other Bob joins, and should handle zero width hacks. + b: new Map([ + ["@carol:example.org", "@carol:example.org"], + [bobRtcMember.userId, `Bob (@bob:example.org)`], + [ + bobZeroWidthSpace.userId, + `${bobZeroWidthSpace.rawDisplayName} (${bobZeroWidthSpace.userId})`, + ], + ]), + }); + }); + }); + + it("should strip RTL characters from displayname", () => { + withTestScheduler(({ behavior, expectObservable }) => { + const daveRtcMember = mockCallMembership("@dave:example.org", "DDDD"); + const daveRTLRtcMember = mockCallMembership( + "@dave2:example.org", + "DDDD", + ); + const dave = mockMatrixRoomMember(daveRtcMember, { + rawDisplayName: "Dave", + }); + const daveRTL = mockMatrixRoomMember(daveRTLRtcMember, { + rawDisplayName: "\u202eevaD", + }); + + fakeMemberWith({ userId: "@carol:example.org" }); + fakeMemberWith(daveRTL); + fakeMemberWith(dave); + const memberships$ = behavior("ab", { + a: [mockCallMembership("@carol:example.org", "DDDD")], + b: [ + mockCallMembership("@carol:example.org", "DDDD"), + daveRtcMember, + daveRTLRtcMember, + ], + }); + + const metadataStore = createMatrixMemberMetadata$( + testScope, + memberships$, + createRoomMembers$(testScope, mockMatrixRoom), + ); + + expectObservable(metadataStore.displaynameMap$).toBe("ab", { + // Carol has no displayname - So userId is used. + a: new Map([["@carol:example.org", "@carol:example.org"]]), + // Both Dave's join. Since after stripping + b: new Map([ + ["@carol:example.org", "@carol:example.org"], + // Not disambiguated + ["@dave:example.org", "Dave"], + // This one is, since it's using RTL. + ["@dave2:example.org", "evaD (@dave2:example.org)"], + ]), + }); + }); + }); + }); + + describe("avatarUrl", () => { + function updateAvatarUrl( + userId: `@${string}:${string}`, + avatarUrl: string, + ): void { + const member = fakeMembersMap.get(userId); + if (member) { + member.getMxcAvatarUrl = vi.fn().mockReturnValue(avatarUrl); + // Emit the event to notify listeners + mockMatrixRoom.emit( + RoomStateEvent.Members, + {} as unknown as MatrixEvent, + {} as unknown as RoomState, + member as RoomMember, + ); + } else { + throw new Error(`No member found with userId: ${userId}`); + } + } + + it("should use avatar url from room members", () => { + withTestScheduler(({ behavior, expectObservable }) => { + fakeMemberWith({ + userId: "@local:example.com", + }); + fakeMemberWith({ + userId: "@alice:example.com", + getMxcAvatarUrl: vi.fn().mockReturnValue("mxc://custom.url/avatar"), + }); + const memberships$ = behavior("a", { + a: [ + mockCallMembership("@local:example.com", "DEVICE1"), + mockCallMembership("@alice:example.com", "DEVICE1"), + ], + }); + const metadataStore = createMatrixMemberMetadata$( + testScope, + memberships$, + createRoomMembers$(testScope, mockMatrixRoom), + ); + const local$ = + metadataStore.createAvatarUrlBehavior$("@local:example.com"); + + const alice$ = + metadataStore.createAvatarUrlBehavior$("@alice:example.com"); + + expectObservable(local$).toBe("a", { + a: "mxc://example.com/@local:example.com", + }); + expectObservable(alice$).toBe("a", { + a: "mxc://custom.url/avatar", + }); + expectObservable(metadataStore.avatarMap$).toBe("a", { + a: new Map([ + ["@local:example.com", "mxc://example.com/@local:example.com"], + ["@alice:example.com", "mxc://custom.url/avatar"], + ]), + }); + }); + }); + + it("should update on avatar change and user join/leave", () => { + withTestScheduler(({ behavior, schedule, expectObservable }) => { + fakeMemberWith({ userId: "@carl:example.com" }); + fakeMemberWith({ userId: "@bob:example.com" }); + const memberships$ = behavior("ab-d", { + a: [mockCallMembership("@bob:example.com", "B000")], + b: [ + mockCallMembership("@bob:example.com", "B000"), + mockCallMembership("@carl:example.com", "C000"), + ], + d: [mockCallMembership("@carl:example.com", "C000")], + }); + + const metadataStore = createMatrixMemberMetadata$( + testScope, + memberships$, + createRoomMembers$(testScope, mockMatrixRoom), + ); + + schedule("--c-", { + c: () => { + updateAvatarUrl( + "@carl:example.com", + "mxc://updated.me/updatedAvatar", + ); + }, + }); + + const bob$ = metadataStore.createAvatarUrlBehavior$("@bob:example.com"); + const carl$ = + metadataStore.createAvatarUrlBehavior$("@carl:example.com"); + expectObservable(bob$).toBe("a---", { + a: "mxc://example.com/@bob:example.com", + }); + expectObservable(carl$).toBe("a-c-", { + a: "mxc://example.com/@carl:example.com", + + c: "mxc://updated.me/updatedAvatar", + }); + expectObservable(metadataStore.avatarMap$).toBe("a-c-", { + a: new Map([ + ["@bob:example.com", "mxc://example.com/@bob:example.com"], + ["@carl:example.com", "mxc://example.com/@carl:example.com"], + ]), + // expect an update once we update the avatar URL + c: new Map([ + ["@bob:example.com", "mxc://example.com/@bob:example.com"], + ["@carl:example.com", "mxc://updated.me/updatedAvatar"], + ]), + }); + }); + }); + }); +}); diff --git a/src/state/CallViewModel/remoteMembers/MatrixMemberMetadata.ts b/src/state/CallViewModel/remoteMembers/MatrixMemberMetadata.ts new file mode 100644 index 00000000..c1a7a499 --- /dev/null +++ b/src/state/CallViewModel/remoteMembers/MatrixMemberMetadata.ts @@ -0,0 +1,180 @@ +/* +Copyright 2025 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { type RoomMember, RoomStateEvent } from "matrix-js-sdk"; +import { combineLatest, fromEvent, map } from "rxjs"; +import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc"; +import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; +import { + KnownMembership, + type Room as MatrixRoom, +} from "matrix-js-sdk/lib/matrix"; +// eslint-disable-next-line rxjs/no-internal + +import { type ObservableScope } from "../../ObservableScope"; +import { + calculateDisplayName, + shouldDisambiguate, +} from "../../../utils/displayname"; +import { type Behavior } from "../../Behavior"; + +const logger = rootLogger.getChild("[MatrixMemberMetadata]"); + +export type RoomMemberMap = Map< + string, + Pick +>; +export function roomToMembersMap(matrixRoom: MatrixRoom): RoomMemberMap { + const members = matrixRoom + .getMembersWithMembership(KnownMembership.Join) + .concat(matrixRoom.getMembersWithMembership(KnownMembership.Invite)); + return members.reduce((acc, member) => { + acc.set(member.userId, { + userId: member.userId, + getMxcAvatarUrl: member.getMxcAvatarUrl.bind(member), + rawDisplayName: member.rawDisplayName, + }); + return acc; + }, new Map()); +} + +export function createRoomMembers$( + scope: ObservableScope, + matrixRoom: MatrixRoom, +): Behavior { + return scope.behavior( + fromEvent(matrixRoom, RoomStateEvent.Members).pipe( + map(() => roomToMembersMap(matrixRoom)), + ), + roomToMembersMap(matrixRoom), + ); +} + +/** + * creates the member that this DM is with in case it is a DM (two members) otherwise null + */ +export function createDMMember$( + scope: ObservableScope, + roomMembers$: Behavior, + matrixRoom: MatrixRoom, +): Behavior | null> { + // We cannot use the normal direct check from matrix since we do not have access to the account data. + // use primitive member count === 2 check instead. + return scope.behavior( + roomMembers$.pipe( + map((membersMap) => { + // primitive appraoch do to no access to account data. + const isDM = membersMap.size === 2; + if (!isDM) return null; + return matrixRoom.getMember(matrixRoom.guessDMUserId()); + }), + ), + ); +} + +/** + * Displayname for each member of the call. This will disambiguate + * any displayname that clashes with another member. Only members + * joined to the call are considered here. + * + * @returns Map uses the Matrix user ID as the key. + */ +// don't do this work more times than we need to. This is achieved by converting to a behavior: +export const memberDisplaynames$ = ( + scope: ObservableScope, + memberships$: Behavior[]>, + roomMembers$: Behavior, +): Behavior> => { + // This map tracks userIds that at some point needed disambiguation. + // This is a memory leak bound to the number of participants. + // A call application will always increase the memory if there have been more members in a call. + // Its capped by room member participants. + const shouldDisambiguateTrackerMap = new Set(); + return scope.behavior( + combineLatest([ + // Handle call membership changes + memberships$, + // Additionally handle display name changes (implicitly reacting to them) + roomMembers$, + // TODO: do we need: pauseWhen(this.pretendToBeDisconnected$), + ]).pipe( + map(([memberships, roomMembers]) => { + const displaynameMap = new Map(); + // We only consider RTC members for disambiguation as they are the only visible members. + for (const rtcMember of memberships) { + const member = roomMembers.get(rtcMember.userId); + if (!member) { + logger.error(`Could not find member for user ${rtcMember.userId}`); + continue; + } + const disambiguateComputed = shouldDisambiguate( + member, + memberships, + roomMembers, + ); + + const disambiguate = + shouldDisambiguateTrackerMap.has(rtcMember.userId) || + disambiguateComputed; + if (disambiguate) shouldDisambiguateTrackerMap.add(rtcMember.userId); + displaynameMap.set( + rtcMember.userId, + calculateDisplayName(member, disambiguate), + ); + } + return displaynameMap; + }), + ), + ); +}; + +export const createMatrixMemberMetadata$ = ( + scope: ObservableScope, + memberships$: Behavior[]>, + roomMembers$: Behavior, +): { + createDisplayNameBehavior$: (userId: string) => Behavior; + createAvatarUrlBehavior$: (userId: string) => Behavior; + displaynameMap$: Behavior>; + avatarMap$: Behavior>; +} => { + const displaynameMap$ = memberDisplaynames$( + scope, + memberships$, + roomMembers$, + ); + const avatarMap$ = scope.behavior( + roomMembers$.pipe( + map((roomMembers) => + Array.from(roomMembers.keys()).reduce((acc, key) => { + acc.set(key, roomMembers.get(key)?.getMxcAvatarUrl()); + return acc; + }, new Map()), + ), + ), + ); + return { + createDisplayNameBehavior$: (userId: string) => + scope.behavior( + displaynameMap$.pipe( + map((displaynameMap) => displaynameMap.get(userId)), + ), + ), + createAvatarUrlBehavior$: (userId: string) => + scope.behavior( + roomMembers$.pipe( + map((roomMembers) => roomMembers.get(userId)?.getMxcAvatarUrl()), + ), + ), + // mostly for testing purposes + displaynameMap$, + avatarMap$, + }; +}; diff --git a/src/state/CallViewModel/remoteMembers/integration.test.ts b/src/state/CallViewModel/remoteMembers/integration.test.ts new file mode 100644 index 00000000..e3aa6be8 --- /dev/null +++ b/src/state/CallViewModel/remoteMembers/integration.test.ts @@ -0,0 +1,228 @@ +/* +Copyright 2025 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { test, vi, expect, beforeEach, afterEach } from "vitest"; +import { BehaviorSubject } from "rxjs"; +import { type Room as LivekitRoom } from "livekit-client"; +import EventEmitter from "events"; +import fetchMock from "fetch-mock"; +import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; +import { logger } from "matrix-js-sdk/lib/logger"; + +import { + type Epoch, + ObservableScope, + trackEpoch, +} from "../../ObservableScope.ts"; +import { ECConnectionFactory } from "./ConnectionFactory.ts"; +import { type OpenIDClientParts } from "../../../livekit/openIDSFU.ts"; +import { + mockCallMembership, + mockMediaDevices, + withTestScheduler, +} from "../../../utils/test.ts"; +import { type ProcessorState } from "../../../livekit/TrackProcessorContext.tsx"; +import { + areLivekitTransportsEqual, + createMatrixLivekitMembers$, + type MatrixLivekitMember, +} from "./MatrixLivekitMembers.ts"; +import { createConnectionManager$ } from "./ConnectionManager.ts"; +import { membershipsAndTransports$ } from "../../SessionBehaviors.ts"; + +// Test the integration of ConnectionManager and MatrixLivekitMerger + +let testScope: ObservableScope; +let ecConnectionFactory: ECConnectionFactory; +let mockClient: OpenIDClientParts; +let lkRoomFactory: () => LivekitRoom; + +const createdMockLivekitRooms: Map = new Map(); + +beforeEach(() => { + testScope = new ObservableScope(); + mockClient = { + getOpenIdToken: vi.fn().mockReturnValue(""), + getDeviceId: vi.fn().mockReturnValue("DEV000"), + }; + + lkRoomFactory = vi.fn().mockImplementation(() => { + const emitter = new EventEmitter(); + const base = { + on: emitter.on.bind(emitter), + off: emitter.off.bind(emitter), + emit: emitter.emit.bind(emitter), + disconnect: vi.fn(), + remoteParticipants: new Map(), + } as unknown as LivekitRoom; + + vi.mocked(base).connect = vi.fn().mockImplementation(({ url }) => { + createdMockLivekitRooms.set(url, base); + }); + return base; + }); + + ecConnectionFactory = new ECConnectionFactory( + mockClient, + mockMediaDevices({}), + new BehaviorSubject({ + supported: true, + processor: undefined, + }), + undefined, + false, + lkRoomFactory, + ); + + //TODO a bit annoying to have to do a http mock? + fetchMock.post(`path:/sfu/get`, (url) => { + const domain = new URL(url).hostname; // Extract the domain from the URL + return { + status: 200, + body: { + url: `wss://${domain}/livekit/sfu`, + jwt: "ATOKEN", + }, + }; + }); +}); + +afterEach(() => { + testScope.end(); + fetchMock.reset(); +}); + +test("bob, carl, then bob joining no tracks yet", () => { + withTestScheduler(({ expectObservable, behavior, scope }) => { + const bobMembership = mockCallMembership("@bob:example.com", "BDEV000"); + const carlMembership = mockCallMembership("@carl:example.com", "CDEV000"); + const daveMembership = mockCallMembership("@dave:foo.bar", "DDEV000"); + + const eMarble = "abc"; + const vMarble = "abc"; + const memberships$ = scope.behavior( + behavior(eMarble, { + a: [bobMembership], + b: [bobMembership, carlMembership], + c: [bobMembership, carlMembership, daveMembership], + }).pipe(trackEpoch()), + ); + + const membershipsAndTransports = membershipsAndTransports$( + testScope, + memberships$, + ); + + const connectionManager = createConnectionManager$({ + scope: testScope, + connectionFactory: ecConnectionFactory, + inputTransports$: membershipsAndTransports.transports$, + logger: logger, + }); + + const matrixLivekitItems$ = createMatrixLivekitMembers$({ + scope: testScope, + membershipsWithTransport$: + membershipsAndTransports.membershipsWithTransport$, + connectionManager, + }); + + expectObservable(matrixLivekitItems$).toBe(vMarble, { + a: expect.toSatisfy((e: Epoch) => { + const items = e.value; + expect(items.length).toBe(1); + const item = items[0]!; + expectObservable(item.membership$).toBe("a", { + a: bobMembership, + }); + expectObservable(item.connection$).toBe("a", { + a: expect.toSatisfy((co) => + areLivekitTransportsEqual( + co.transport, + bobMembership.transports[0]! as LivekitTransport, + ), + ), + }); + expectObservable(item.participant$).toBe("a", { + a: null, + }); + return true; + }), + b: expect.toSatisfy((e: Epoch) => { + const items = e.value; + expect(items.length).toBe(2); + + { + const item = items[0]!; + expectObservable(item.membership$).toBe("a", { + a: bobMembership, + }); + expectObservable(item.participant$).toBe("a", { + a: null, + }); + } + + { + const item = items[1]!; + + expectObservable(item.membership$).toBe("a", { + a: carlMembership, + }); + expectObservable(item.participant$).toBe("a", { + a: null, + }); + expectObservable(item.connection$).toBe("a", { + a: expect.toSatisfy((connection) => { + expect( + areLivekitTransportsEqual( + connection.transport, + carlMembership.transports[0]! as LivekitTransport, + ), + ).toBe(true); + return true; + }), + }); + } + return true; + }), + c: expect.toSatisfy((e: Epoch) => { + const items = e.value; + expect(items.length).toBe(3); + + expectObservable(items[0].membership$).toBe("a", { + a: bobMembership, + }); + expectObservable(items[1].membership$).toBe("b", { + a: carlMembership, + }); + + { + const item = items[2]!; + expectObservable(item.membership$).toBe("a", { + a: daveMembership, + }); + expectObservable(item.connection$).toBe("a", { + a: expect.toSatisfy((connection) => { + expect( + areLivekitTransportsEqual( + connection.transport, + daveMembership.transports[0]! as LivekitTransport, + ), + ).toBe(true); + return true; + }), + }); + expectObservable(item.participant$).toBe("a", { + a: null, + }); + } + return true; + }), + x: expect.anything(), + }); + }); +}); diff --git a/src/state/CallViewModelWidget.test.ts b/src/state/CallViewModelWidget.test.ts index 045e2472..afcf69ba 100644 --- a/src/state/CallViewModelWidget.test.ts +++ b/src/state/CallViewModelWidget.test.ts @@ -5,15 +5,18 @@ Copyright 2025 Element Creations Ltd. Please see LICENSE in the repository root for full details. */ -import { test, vi, expect } from "vitest"; +import { it, vi, expect } from "vitest"; import EventEmitter from "events"; +// import * as ComponentsCore from "@livekit/components-core"; +import { withCallViewModel } from "./CallViewModel/CallViewModelTestUtils.ts"; +import { type CallViewModel } from "./CallViewModel/CallViewModel.ts"; import { constant } from "./Behavior.ts"; -import { withCallViewModel } from "./CallViewModel.test.ts"; import { aliceParticipant, localRtcMember } from "../utils/test-fixtures.ts"; import { ElementWidgetActions, widget } from "../widget.ts"; import { E2eeType } from "../e2ee/e2eeType.ts"; -import { type CallViewModel } from "./CallViewModel.ts"; + +vi.mock("@livekit/components-core", { spy: true }); vi.mock("../widget", () => ({ ElementWidgetActions: { @@ -31,7 +34,7 @@ vi.mock("../widget", () => ({ }, })); -test("expect leave when ElementWidgetActions.HangupCall is called", async () => { +it("expect leave when ElementWidgetActions.HangupCall is called", async () => { const pr = Promise.withResolvers(); withCallViewModel( { diff --git a/src/state/MediaViewModel.ts b/src/state/MediaViewModel.ts index 0b79183e..74e64b93 100644 --- a/src/state/MediaViewModel.ts +++ b/src/state/MediaViewModel.ts @@ -27,7 +27,6 @@ import { RoomEvent as LivekitRoomEvent, RemoteTrack, } from "livekit-client"; -import { type RoomMember } from "matrix-js-sdk"; import { logger } from "matrix-js-sdk/lib/logger"; import { BehaviorSubject, @@ -44,6 +43,7 @@ import { startWith, switchMap, throttleTime, + distinctUntilChanged, } from "rxjs"; import { alwaysShowSelf } from "../settings/settings"; @@ -180,29 +180,35 @@ function observeRemoteTrackReceivingOkay$( } function encryptionErrorObservable$( - room: LivekitRoom, + room$: Behavior, participant: Participant, encryptionSystem: EncryptionSystem, criteria: string, ): Observable { - return roomEventSelector(room, LivekitRoomEvent.EncryptionError).pipe( - map((e) => { - const [err] = e; - if (encryptionSystem.kind === E2eeType.PER_PARTICIPANT) { - return ( - // Ideally we would pull the participant identity from the field on the error. - // However, it gets lost in the serialization process between workers. - // So, instead we do a string match - (err?.message.includes(participant.identity) && - err?.message.includes(criteria)) ?? - false - ); - } else if (encryptionSystem.kind === E2eeType.SHARED_KEY) { - return !!err?.message.includes(criteria); - } + return room$.pipe( + switchMap((room) => { + if (room === undefined) return of(false); + return roomEventSelector(room, LivekitRoomEvent.EncryptionError).pipe( + map((e) => { + const [err] = e; + if (encryptionSystem.kind === E2eeType.PER_PARTICIPANT) { + return ( + // Ideally we would pull the participant identity from the field on the error. + // However, it gets lost in the serialization process between workers. + // So, instead we do a string match + (err?.message.includes(participant.identity) && + err?.message.includes(criteria)) ?? + false + ); + } else if (encryptionSystem.kind === E2eeType.SHARED_KEY) { + return !!err?.message.includes(criteria); + } - return false; + return false; + }), + ); }), + distinctUntilChanged(), throttleTime(1000), // Throttle to avoid spamming the UI startWith(false), ); @@ -220,7 +226,7 @@ abstract class BaseMediaViewModel { /** * The LiveKit video track for this media. */ - public readonly video$: Behavior; + public readonly video$: Behavior; /** * Whether there should be a warning that this media is unencrypted. */ @@ -235,12 +241,10 @@ abstract class BaseMediaViewModel { private observeTrackReference$( source: Track.Source, - ): Behavior { + ): Behavior { return this.scope.behavior( this.participant$.pipe( - switchMap((p) => - p === undefined ? of(undefined) : observeTrackReference$(p, source), - ), + switchMap((p) => (!p ? of(null) : observeTrackReference$(p, source))), ), ); } @@ -252,23 +256,22 @@ abstract class BaseMediaViewModel { */ public readonly id: string, /** - * The Matrix room member to which this media belongs. + * The Matrix user to which this media belongs. */ - // TODO: Fully separate the data layer from the UI layer by keeping the - // member object internal - public readonly member: RoomMember, + public readonly userId: string, // We don't necessarily have a participant if a user connects via MatrixRTC but not (yet) through // livekit. protected readonly participant$: Observable< - LocalParticipant | RemoteParticipant | undefined + LocalParticipant | RemoteParticipant | null >, encryptionSystem: EncryptionSystem, audioSource: AudioSource, videoSource: VideoSource, - livekitRoom: LivekitRoom, - public readonly focusURL: string, + livekitRoom$: Behavior, + public readonly focusUrl$: Behavior, public readonly displayName$: Behavior, + public readonly mxcAvatarUrl$: Behavior, ) { const audio$ = this.observeTrackReference$(audioSource); this.video$ = this.observeTrackReference$(videoSource); @@ -296,13 +299,13 @@ abstract class BaseMediaViewModel { } else if (encryptionSystem.kind === E2eeType.PER_PARTICIPANT) { return combineLatest([ encryptionErrorObservable$( - livekitRoom, + livekitRoom$, participant, encryptionSystem, "MissingKey", ), encryptionErrorObservable$( - livekitRoom, + livekitRoom$, participant, encryptionSystem, "InvalidKey", @@ -322,7 +325,7 @@ abstract class BaseMediaViewModel { } else { return combineLatest([ encryptionErrorObservable$( - livekitRoom, + livekitRoom$, participant, encryptionSystem, "InvalidKey", @@ -404,26 +407,28 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel { public constructor( scope: ObservableScope, id: string, - member: RoomMember, - participant$: Observable, + userId: string, + participant$: Observable, encryptionSystem: EncryptionSystem, - livekitRoom: LivekitRoom, - focusUrl: string, + livekitRoom$: Behavior, + focusUrl$: Behavior, displayName$: Behavior, + mxcAvatarUrl$: Behavior, public readonly handRaised$: Behavior, public readonly reaction$: Behavior, ) { super( scope, id, - member, + userId, participant$, encryptionSystem, Track.Source.Microphone, Track.Source.Camera, - livekitRoom, - focusUrl, + livekitRoom$, + focusUrl$, displayName$, + mxcAvatarUrl$, ); const media$ = this.scope.behavior( @@ -540,25 +545,27 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel { public constructor( scope: ObservableScope, id: string, - member: RoomMember, - participant$: Behavior, + userId: string, + participant$: Behavior, encryptionSystem: EncryptionSystem, - livekitRoom: LivekitRoom, - focusURL: string, + livekitRoom$: Behavior, + focusUrl$: Behavior, private readonly mediaDevices: MediaDevices, displayName$: Behavior, + mxcAvatarUrl$: Behavior, handRaised$: Behavior, reaction$: Behavior, ) { super( scope, id, - member, + userId, participant$, encryptionSystem, - livekitRoom, - focusURL, + livekitRoom$, + focusUrl$, displayName$, + mxcAvatarUrl$, handRaised$, reaction$, ); @@ -650,25 +657,27 @@ export class RemoteUserMediaViewModel extends BaseUserMediaViewModel { public constructor( scope: ObservableScope, id: string, - member: RoomMember, - participant$: Observable, + userId: string, + participant$: Observable, encryptionSystem: EncryptionSystem, - livekitRoom: LivekitRoom, - focusUrl: string, + livekitRoom$: Behavior, + focusUrl$: Behavior, private readonly pretendToBeDisconnected$: Behavior, - displayname$: Behavior, + displayName$: Behavior, + mxcAvatarUrl$: Behavior, handRaised$: Behavior, reaction$: Behavior, ) { super( scope, id, - member, + userId, participant$, encryptionSystem, - livekitRoom, - focusUrl, - displayname$, + livekitRoom$, + focusUrl$, + displayName$, + mxcAvatarUrl$, handRaised$, reaction$, ); @@ -749,26 +758,28 @@ export class ScreenShareViewModel extends BaseMediaViewModel { public constructor( scope: ObservableScope, id: string, - member: RoomMember, + userId: string, participant$: Observable, encryptionSystem: EncryptionSystem, - livekitRoom: LivekitRoom, - focusUrl: string, + livekitRoom$: Behavior, + focusUrl$: Behavior, private readonly pretendToBeDisconnected$: Behavior, - displayname$: Behavior, + displayName$: Behavior, + mxcAvatarUrl$: Behavior, public readonly local: boolean, ) { super( scope, id, - member, + userId, participant$, encryptionSystem, Track.Source.ScreenShareAudio, Track.Source.ScreenShare, - livekitRoom, - focusUrl, - displayname$, + livekitRoom$, + focusUrl$, + displayName$, + mxcAvatarUrl$, ); } } diff --git a/src/state/ObservableScope.test.ts b/src/state/ObservableScope.test.ts new file mode 100644 index 00000000..99f2b424 --- /dev/null +++ b/src/state/ObservableScope.test.ts @@ -0,0 +1,104 @@ +/* +Copyright 2025 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { describe, expect, it } from "vitest"; +import { BehaviorSubject, combineLatest, Subject } from "rxjs"; +import { logger } from "matrix-js-sdk/lib/logger"; + +import { + Epoch, + mapEpoch, + ObservableScope, + trackEpoch, +} from "./ObservableScope"; +import { withTestScheduler } from "../utils/test"; + +describe("Epoch", () => { + it("should map the value correctly", () => { + const epoch = new Epoch(1); + const mappedEpoch = epoch.mapInner((v) => v + 1); + expect(mappedEpoch.value).toBe(2); + expect(mappedEpoch.epoch).toBe(0); + }); + + it("should be tracked from an observable", () => { + withTestScheduler(({ expectObservable, behavior }) => { + const observable$ = behavior("abc", { + a: 1, + b: 2, + c: 3, + }); + const epochObservable$ = observable$.pipe(trackEpoch()); + expectObservable(epochObservable$).toBe("abc", { + a: expect.toSatisfy((e) => e.epoch === 0 && e.value === 1), + b: expect.toSatisfy((e) => e.epoch === 1 && e.value === 2), + c: expect.toSatisfy((e) => e.epoch === 2 && e.value === 3), + }); + }); + }); + + it("can be mapped without loosing epoch information", () => { + withTestScheduler(({ expectObservable, behavior }) => { + const observable$ = behavior("abc", { + a: "A", + b: "B", + c: "C", + }); + const epochObservable$ = observable$.pipe(trackEpoch()); + const derivedEpoch$ = epochObservable$.pipe( + mapEpoch((e) => e + "-mapped"), + ); + + expectObservable(derivedEpoch$).toBe("abc", { + a: new Epoch("A-mapped", 0), + b: new Epoch("B-mapped", 1), + c: new Epoch("C-mapped", 2), + }); + }); + }); + + it("diamonds emits in a predictable order", () => { + const sb$ = new BehaviorSubject("initial"); + const root$ = sb$.pipe(trackEpoch()); + const derivedA$ = root$.pipe(mapEpoch((e) => e + "-A")); + const derivedB$ = root$.pipe(mapEpoch((e) => e + "-B")); + combineLatest([root$, derivedB$, derivedA$]).subscribe( + ([root, derivedA, derivedB]) => { + logger.log( + "combined" + + root.epoch + + root.value + + "\n" + + derivedA.epoch + + derivedA.value + + "\n" + + derivedB.epoch + + derivedB.value, + ); + }, + ); + sb$.next("updated"); + sb$.next("ANOTERUPDATE"); + }); + + it("behavior test", () => { + const scope = new ObservableScope(); + const s$ = new Subject(); + const behavior$ = scope.behavior(s$, 0); + behavior$.subscribe((value) => { + logger.log(`Received value: ${value}`); + }); + s$.next(1); + s$.next(2); + s$.next(3); + s$.next(3); + s$.next(3); + s$.next(3); + s$.next(3); + s$.complete(); + }); +}); diff --git a/src/state/ObservableScope.ts b/src/state/ObservableScope.ts index 879445e6..27f501c7 100644 --- a/src/state/ObservableScope.ts +++ b/src/state/ObservableScope.ts @@ -12,7 +12,9 @@ import { EMPTY, endWith, filter, + map, type Observable, + type OperatorFunction, share, take, takeUntil, @@ -22,6 +24,10 @@ import { type Behavior } from "./Behavior"; type MonoTypeOperator = (o: Observable) => Observable; +type SplitBehavior = keyof T extends string | number + ? { [K in keyof T as `${K}$`]: Behavior } + : never; + const nothing = Symbol("nothing"); /** @@ -145,9 +151,132 @@ export class ObservableScope { })(); }); } + + /** + * Splits a Behavior of objects with static properties into an object with + * Behavior properties. + * + * For example, splitting a `Behavior<{ name: string; age: number }>` results + * in an object of type `{ name$: Behavior; age$: Behavior }`. + */ + public splitBehavior( + input$: Behavior, + ): SplitBehavior { + return Object.fromEntries( + Object.keys(input$.value).map((key) => [ + `${key}$`, + this.behavior(input$.pipe(map((input) => input[key as keyof T]))), + ]), + ) as SplitBehavior; + } } /** * The global scope, a scope which never ends. */ export const globalScope = new ObservableScope(); + +/** + * `Epoch`'s can be used to create `Behavior`s and `Observable`s which derivitives can be merged + * with `combinedLatest` without duplicated emissions. + * + * This is useful in the following example: + * ``` + * const rootObs$ = of("red","green","blue"); + * const derivedObs$ = rootObs$.pipe( + * map((v)=> {red:"fire", green:"grass", blue:"water"}[v]) + * ); + * const otherDerivedObs$ = rootObs$.pipe( + * map((v)=> {red:"tomatoes", green:"leaves", blue:"sky"}[v]) + * ); + * const mergedObs$ = combineLatest([rootObs$, derivedObs$, otherDerivedObs$]).pipe( + * map(([color, a,b]) => color + " like " + a + " and " + b) + * ); + * + * ``` + * will result in 6 emissions with mismatching items like "red like fire and leaves" + * + * # Use Epoch + * ``` + * const ancestorObs$ = of(1,2,3).pipe(trackEpoch()); + * const derivedObs$ = ancestorObs$.pipe( + * mapEpoch((v)=> "this number: " + v) + * ); + * const otherDerivedObs$ = ancestorObs$.pipe( + * mapEpoch((v)=> "multiplied by: " + v) + * ); + * const mergedObs$ = combineLatest([derivedObs$, otherDerivedObs$]).pipe( + * filter((values) => values.every((v) => v.epoch === values[0].v)), + * map(([color, a, b]) => color + " like " + a + " and " + b) + * ); + * + * ``` + * will result in 3 emissions all matching (e.g. "blue like water and sky") + */ +export class Epoch { + public readonly epoch: number; + public readonly value: T; + + public constructor(value: T, epoch?: number) { + this.value = value; + this.epoch = epoch ?? 0; + } + /** + * Maps the value inside the epoch to a new value while keeping the epoch number. + * # usage + * ``` + * const myEpoch$ = myObservable$.pipe( + * map(trackEpoch()), + * // this is the preferred way using mapEpoch + * mapEpoch((v)=> v+1) + * // This is how inner map can be used: + * map((epoch) => epoch.innerMap((v)=> v+1)) + * // It is equivalent to: + * map((epoch) => new Epoch(epoch.value + 1, epoch.epoch)) + * ) + * ``` + * See also `Epoch` + */ + public mapInner(map: (value: T) => U): Epoch { + return new Epoch(map(this.value), this.epoch); + } +} + +/** + * A `pipe` compatible map oparator that keeps the epoch in tact but allows mapping the value. + * # usage + * ``` + * const myEpoch$ = myObservable$.pipe( + * map(trackEpoch()), + * // this is the preferred way using mapEpoch + * mapEpoch((v)=> v+1) + * // This is how inner map can be used: + * map((epoch) => epoch.innerMap((v)=> v+1)) + * // It is equivalent to: + * map((epoch) => new Epoch(epoch.value + 1, epoch.epoch)) + * ) + * ``` + * See also `Epoch` + */ +export function mapEpoch( + mapFn: (value: T) => U, +): OperatorFunction, Epoch> { + return map((e) => e.mapInner(mapFn)); +} + +/** + * # usage + * ``` + * const myEpoch$ = myObservable$.pipe( + * map(trackEpoch()), + * map((epoch) => epoch.innerMap((v)=> v+1)) + * ) + * const derived = myEpoch$.pipe( + * mapEpoch((v)=>v^2) + * ) + * ``` + * See also `Epoch` + */ +export function trackEpoch(): OperatorFunction> { + return map>((value, number) => new Epoch(value, number)); +} diff --git a/src/state/ScreenShare.ts b/src/state/ScreenShare.ts index 9803a5f4..0a241cdf 100644 --- a/src/state/ScreenShare.ts +++ b/src/state/ScreenShare.ts @@ -4,7 +4,7 @@ Copyright 2025 New Vector Ltd. SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { of, type Observable } from "rxjs"; +import { of } from "rxjs"; import { type LocalParticipant, type RemoteParticipant, @@ -13,7 +13,6 @@ import { import { type ObservableScope } from "./ObservableScope.ts"; import { ScreenShareViewModel } from "./MediaViewModel.ts"; -import type { RoomMember } from "matrix-js-sdk"; import type { EncryptionSystem } from "../e2ee/sharedKeyManagement.ts"; import type { Behavior } from "./Behavior.ts"; @@ -28,24 +27,26 @@ export class ScreenShare { public constructor( private readonly scope: ObservableScope, id: string, - member: RoomMember, + userId: string, participant: LocalParticipant | RemoteParticipant, encryptionSystem: EncryptionSystem, - livekitRoom: LivekitRoom, - focusUrl: string, + livekitRoom$: Behavior, + focusUrl$: Behavior, pretendToBeDisconnected$: Behavior, - displayName$: Observable, + displayName$: Behavior, + mxcAvatarUrl$: Behavior, ) { this.vm = new ScreenShareViewModel( this.scope, id, - member, + userId, of(participant), encryptionSystem, - livekitRoom, - focusUrl, + livekitRoom$, + focusUrl$, pretendToBeDisconnected$, - this.scope.behavior(displayName$), + displayName$, + mxcAvatarUrl$, participant.isLocal, ); } diff --git a/src/state/SessionBehaviors.ts b/src/state/SessionBehaviors.ts new file mode 100644 index 00000000..e174a1cc --- /dev/null +++ b/src/state/SessionBehaviors.ts @@ -0,0 +1,81 @@ +/* +Copyright 2025 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + type CallMembership, + isLivekitTransport, + type LivekitTransport, + type MatrixRTCSession, + MatrixRTCSessionEvent, +} from "matrix-js-sdk/lib/matrixrtc"; +import { fromEvent } from "rxjs"; + +import { + Epoch, + mapEpoch, + trackEpoch, + type ObservableScope, +} from "./ObservableScope"; +import { type Behavior } from "./Behavior"; + +export const membershipsAndTransports$ = ( + scope: ObservableScope, + memberships$: Behavior>, +): { + membershipsWithTransport$: Behavior< + Epoch<{ membership: CallMembership; transport?: LivekitTransport }[]> + >; + transports$: Behavior>; +} => { + /** + * Lists the transports used by ourselves, plus all other MatrixRTC session + * members. For completeness this also lists the preferred transport and + * whether we are in multi-SFU mode or sticky events mode (because + * advertisedTransport$ wants to read them at the same time, and bundling data + * together when it might change together is what you have to do in RxJS to + * avoid reading inconsistent state or observing too many changes.) + */ + const membershipsWithTransport$ = scope.behavior( + memberships$.pipe( + mapEpoch((memberships) => { + return memberships.map((membership) => { + const oldestMembership = memberships[0] ?? membership; + const transport = membership.getTransport(oldestMembership); + return { + membership, + transport: isLivekitTransport(transport) ? transport : undefined, + }; + }); + }), + ), + ); + + const transports$ = scope.behavior( + membershipsWithTransport$.pipe( + mapEpoch((mts) => mts.flatMap(({ transport: t }) => (t ? [t] : []))), + ), + ); + + return { + membershipsWithTransport$, + transports$, + }; +}; + +export const createMemberships$ = ( + scope: ObservableScope, + matrixRTCSession: MatrixRTCSession, +): Behavior> => { + return scope.behavior( + fromEvent( + matrixRTCSession, + MatrixRTCSessionEvent.MembershipsChanged, + (_, memberships: CallMembership[]) => memberships, + ).pipe(trackEpoch()), + new Epoch(matrixRTCSession.memberships), + ); +}; diff --git a/src/state/TileStore.ts b/src/state/TileStore.ts index f6db0930..7b95bd8e 100644 --- a/src/state/TileStore.ts +++ b/src/state/TileStore.ts @@ -14,7 +14,7 @@ import { fillGaps } from "../utils/iter"; import { debugTileLayout } from "../settings/settings"; function debugEntries(entries: GridTileData[]): string[] { - return entries.map((e) => e.media.member?.rawDisplayName ?? "[👻]"); + return entries.map((e) => e.media.displayName$.value); } let DEBUG_ENABLED = false; @@ -156,7 +156,7 @@ export class TileStoreBuilder { public registerSpotlight(media: MediaViewModel[], maximised: boolean): void { if (DEBUG_ENABLED) logger.debug( - `[TileStore, ${this.generation}] register spotlight: ${media.map((m) => m.member?.rawDisplayName ?? "[👻]")}`, + `[TileStore, ${this.generation}] register spotlight: ${media.map((m) => m.displayName$.value)}`, ); if (this.spotlight !== null) throw new Error("Spotlight already set"); @@ -180,7 +180,7 @@ export class TileStoreBuilder { public registerGridTile(media: UserMediaViewModel): void { if (DEBUG_ENABLED) logger.debug( - `[TileStore, ${this.generation}] register grid tile: ${media.member?.rawDisplayName ?? "[👻]"}`, + `[TileStore, ${this.generation}] register grid tile: ${media.displayName$.value}`, ); if (this.spotlight !== null) { @@ -263,7 +263,7 @@ export class TileStoreBuilder { public registerPipTile(media: UserMediaViewModel): void { if (DEBUG_ENABLED) logger.debug( - `[TileStore, ${this.generation}] register PiP tile: ${media.member?.rawDisplayName ?? "[👻]"}`, + `[TileStore, ${this.generation}] register PiP tile: ${media.displayName$.value}`, ); // If there is a single grid tile that we can reuse diff --git a/src/state/UserMedia.ts b/src/state/UserMedia.ts index 65bd4e92..38f22122 100644 --- a/src/state/UserMedia.ts +++ b/src/state/UserMedia.ts @@ -5,17 +5,9 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { - BehaviorSubject, - combineLatest, - map, - type Observable, - of, - switchMap, -} from "rxjs"; +import { combineLatest, map, type Observable, of, switchMap } from "rxjs"; import { type LocalParticipant, - type Participant, ParticipantEvent, type RemoteParticipant, type Room as LivekitRoom, @@ -29,11 +21,12 @@ import { type UserMediaViewModel, } from "./MediaViewModel.ts"; import type { Behavior } from "./Behavior.ts"; -import type { RoomMember } from "matrix-js-sdk"; import type { EncryptionSystem } from "../e2ee/sharedKeyManagement.ts"; import type { MediaDevices } from "./MediaDevices.ts"; import type { ReactionOption } from "../reactions"; import { observeSpeaker$ } from "./observeSpeaker.ts"; +import { generateItems } from "../utils/observable.ts"; +import { ScreenShare } from "./ScreenShare.ts"; /** * Sorting bins defining the order in which media tiles appear in the layout. @@ -72,35 +65,35 @@ enum SortingBin { /** * A user media item to be presented in a tile. This is a thin wrapper around * UserMediaViewModel which additionally determines the media item's sorting bin - * for inclusion in the call layout. + * for inclusion in the call layout and tracks associated screen shares. */ export class UserMedia { - private readonly participant$ = new BehaviorSubject(this.initialParticipant); - public readonly vm: UserMediaViewModel = this.participant$.value?.isLocal ? new LocalUserMediaViewModel( this.scope, this.id, - this.member, - this.participant$ as Behavior, + this.userId, + this.participant$ as Behavior, this.encryptionSystem, - this.livekitRoom, - this.focusURL, + this.livekitRoom$, + this.focusUrl$, this.mediaDevices, - this.scope.behavior(this.displayname$), + this.displayName$, + this.mxcAvatarUrl$, this.scope.behavior(this.handRaised$), this.scope.behavior(this.reaction$), ) : new RemoteUserMediaViewModel( this.scope, this.id, - this.member, - this.participant$ as Observable, + this.userId, + this.participant$ as Behavior, this.encryptionSystem, - this.livekitRoom, - this.focusURL, + this.livekitRoom$, + this.focusUrl$, this.pretendToBeDisconnected$, - this.scope.behavior(this.displayname$), + this.displayName$, + this.mxcAvatarUrl$, this.scope.behavior(this.handRaised$), this.scope.behavior(this.reaction$), ); @@ -109,12 +102,55 @@ export class UserMedia { observeSpeaker$(this.vm.speaking$), ); - private readonly presenter$ = this.scope.behavior( + /** + * All screen share media associated with this user media. + */ + public readonly screenShares$ = this.scope.behavior( this.participant$.pipe( - switchMap((p) => (p === undefined ? of(false) : sharingScreen$(p))), + switchMap((p) => + p === null + ? of([]) + : observeParticipantEvents( + p, + ParticipantEvent.TrackPublished, + ParticipantEvent.TrackUnpublished, + ParticipantEvent.LocalTrackPublished, + ParticipantEvent.LocalTrackUnpublished, + ).pipe( + // Technically more than one screen share might be possible... our + // MediaViewModels don't support it though since they look for a unique + // track for the given source. So generateItems here is a bit overkill. + generateItems( + function* (p) { + if (p.isScreenShareEnabled) + yield { + keys: ["screen-share"], + data: undefined, + }; + }, + (scope, _data$, key) => + new ScreenShare( + scope, + `${this.id}:${key}`, + this.userId, + p, + this.encryptionSystem, + this.livekitRoom$, + this.focusUrl$, + this.pretendToBeDisconnected$, + this.displayName$, + this.mxcAvatarUrl$, + ), + ), + ), + ), ), ); + private readonly presenter$ = this.scope.behavior( + this.screenShares$.pipe(map((screenShares) => screenShares.length > 0)), + ); + /** * Which sorting bin the media item should be placed in. */ @@ -147,37 +183,18 @@ export class UserMedia { public constructor( private readonly scope: ObservableScope, public readonly id: string, - private readonly member: RoomMember, - private readonly initialParticipant: - | LocalParticipant - | RemoteParticipant - | undefined, + private readonly userId: string, + private readonly participant$: Behavior< + LocalParticipant | RemoteParticipant | null + >, private readonly encryptionSystem: EncryptionSystem, - private readonly livekitRoom: LivekitRoom, - private readonly focusURL: string, + private readonly livekitRoom$: Behavior, + private readonly focusUrl$: Behavior, private readonly mediaDevices: MediaDevices, private readonly pretendToBeDisconnected$: Behavior, - private readonly displayname$: Observable, + private readonly displayName$: Behavior, + private readonly mxcAvatarUrl$: Behavior, private readonly handRaised$: Observable, private readonly reaction$: Observable, ) {} - - public updateParticipant( - newParticipant: LocalParticipant | RemoteParticipant | undefined, - ): void { - if (this.participant$.value !== newParticipant) { - // Update the BehaviourSubject in the UserMedia. - this.participant$.next(newParticipant); - } - } -} - -export function sharingScreen$(p: Participant): Observable { - return observeParticipantEvents( - p, - ParticipantEvent.TrackPublished, - ParticipantEvent.TrackUnpublished, - ParticipantEvent.LocalTrackPublished, - ParticipantEvent.LocalTrackUnpublished, - ).pipe(map((p) => p.isScreenShareEnabled)); } diff --git a/src/tile/GridTile.test.tsx b/src/tile/GridTile.test.tsx index dd0bc9d6..e3172a22 100644 --- a/src/tile/GridTile.test.tsx +++ b/src/tile/GridTile.test.tsx @@ -15,7 +15,7 @@ import { GridTile } from "./GridTile"; import { mockRtcMembership, createRemoteMedia } from "../utils/test"; import { GridTileViewModel } from "../state/TileViewModel"; import { ReactionsSenderProvider } from "../reactions/useReactionsSender"; -import type { CallViewModel } from "../state/CallViewModel"; +import type { CallViewModel } from "../state/CallViewModel/CallViewModel"; import { constant } from "../state/Behavior"; global.IntersectionObserver = class MockIntersectionObserver { diff --git a/src/tile/GridTile.tsx b/src/tile/GridTile.tsx index 421cefda..57409869 100644 --- a/src/tile/GridTile.tsx +++ b/src/tile/GridTile.tsx @@ -58,7 +58,9 @@ interface TileProps { style?: ComponentProps["style"]; targetWidth: number; targetHeight: number; + focusUrl: string | undefined; displayName: string; + mxcAvatarUrl: string | undefined; showSpeakingIndicators: boolean; focusable: boolean; } @@ -81,7 +83,9 @@ const UserMediaTile: FC = ({ menuStart, menuEnd, className, + focusUrl, displayName, + mxcAvatarUrl, focusable, ...props }) => { @@ -144,8 +148,8 @@ const UserMediaTile: FC = ({ const tile = ( = ({ /> } displayName={displayName} + mxcAvatarUrl={mxcAvatarUrl} focusable={focusable} primaryButton={ primaryButton ?? ( @@ -190,7 +195,7 @@ const UserMediaTile: FC = ({ currentReaction={reaction ?? undefined} raisedHandOnClick={raisedHandOnClick} localParticipant={vm.local} - focusUrl={vm.focusURL} + focusUrl={focusUrl} audioStreamStats={audioStreamStats} videoStreamStats={videoStreamStats} {...props} @@ -359,7 +364,9 @@ export const GridTile: FC = ({ const ourRef = useRef(null); const ref = useMergedRefs(ourRef, theirRef); const media = useBehavior(vm.media$); + const focusUrl = useBehavior(media.focusUrl$); const displayName = useBehavior(media.displayName$); + const mxcAvatarUrl = useBehavior(media.mxcAvatarUrl$); if (media instanceof LocalUserMediaViewModel) { return ( @@ -367,7 +374,9 @@ export const GridTile: FC = ({ ref={ref} vm={media} onOpenProfile={onOpenProfile} + focusUrl={focusUrl} displayName={displayName} + mxcAvatarUrl={mxcAvatarUrl} {...props} /> ); @@ -376,7 +385,9 @@ export const GridTile: FC = ({ ); diff --git a/src/tile/MediaView.test.tsx b/src/tile/MediaView.test.tsx index c26a4d5f..c8ffbefd 100644 --- a/src/tile/MediaView.test.tsx +++ b/src/tile/MediaView.test.tsx @@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { describe, expect, it, test, vi } from "vitest"; +import { describe, expect, it, test } from "vitest"; import { render, screen } from "@testing-library/react"; import { axe } from "vitest-axe"; import { TooltipProvider } from "@vector-im/compound-web"; @@ -16,7 +16,6 @@ import { import { LocalTrackPublication, Track } from "livekit-client"; import { TrackInfo } from "@livekit/protocol"; import { type ComponentProps } from "react"; -import { type RoomMember } from "matrix-js-sdk"; import { MediaView } from "./MediaView"; import { EncryptionStatus } from "../state/MediaViewModel"; @@ -46,10 +45,8 @@ describe("MediaView", () => { mirror: false, unencryptedWarning: false, video: trackReference, - member: vi.mocked({ - userId: "@alice:example.com", - getMxcAvatarUrl: vi.fn().mockReturnValue(undefined), - } as unknown as RoomMember), + userId: "@alice:example.com", + mxcAvatarUrl: undefined, localParticipant: false, focusable: true, }; diff --git a/src/tile/MediaView.tsx b/src/tile/MediaView.tsx index 149b4177..e8a30cd4 100644 --- a/src/tile/MediaView.tsx +++ b/src/tile/MediaView.tsx @@ -7,7 +7,6 @@ Please see LICENSE in the repository root for full details. import { type TrackReferenceOrPlaceholder } from "@livekit/components-core"; import { animated } from "@react-spring/web"; -import { type RoomMember } from "matrix-js-sdk"; import { type FC, type ComponentProps, type ReactNode } from "react"; import { useTranslation } from "react-i18next"; import classNames from "classnames"; @@ -32,12 +31,13 @@ interface Props extends ComponentProps { video: TrackReferenceOrPlaceholder | undefined; videoFit: "cover" | "contain"; mirror: boolean; - member: RoomMember; + userId: string; videoEnabled: boolean; unencryptedWarning: boolean; encryptionStatus: EncryptionStatus; nameTagLeadingIcon?: ReactNode; displayName: string; + mxcAvatarUrl: string | undefined; focusable: boolean; primaryButton?: ReactNode; raisedHandTime?: Date; @@ -59,11 +59,12 @@ export const MediaView: FC = ({ video, videoFit, mirror, - member, + userId, videoEnabled, unencryptedWarning, nameTagLeadingIcon, displayName, + mxcAvatarUrl, focusable, primaryButton, encryptionStatus, @@ -94,10 +95,10 @@ export const MediaView: FC = ({ >
diff --git a/src/tile/SpotlightTile.tsx b/src/tile/SpotlightTile.tsx index 6034c846..48dd0f8c 100644 --- a/src/tile/SpotlightTile.tsx +++ b/src/tile/SpotlightTile.tsx @@ -27,7 +27,6 @@ import { useObservableRef } from "observable-hooks"; import { useTranslation } from "react-i18next"; import classNames from "classnames"; import { type TrackReferenceOrPlaceholder } from "@livekit/components-core"; -import { type RoomMember } from "matrix-js-sdk"; import FullScreenMaximiseIcon from "../icons/FullScreenMaximise.svg?react"; import FullScreenMinimiseIcon from "../icons/FullScreenMinimise.svg?react"; @@ -55,10 +54,12 @@ interface SpotlightItemBaseProps { targetHeight: number; video: TrackReferenceOrPlaceholder | undefined; videoEnabled: boolean; - member: RoomMember; + userId: string; unencryptedWarning: boolean; encryptionStatus: EncryptionStatus; + focusUrl: string | undefined; displayName: string; + mxcAvatarUrl: string | undefined; focusable: boolean; "aria-hidden"?: boolean; localParticipant: boolean; @@ -78,7 +79,7 @@ const SpotlightLocalUserMediaItem: FC = ({ ...props }) => { const mirror = useBehavior(vm.mirror$); - return ; + return ; }; SpotlightLocalUserMediaItem.displayName = "SpotlightLocalUserMediaItem"; @@ -134,7 +135,9 @@ const SpotlightItem: FC = ({ }) => { const ourRef = useRef(null); const ref = useMergedRefs(ourRef, theirRef); + const focusUrl = useBehavior(vm.focusUrl$); const displayName = useBehavior(vm.displayName$); + const mxcAvatarUrl = useBehavior(vm.mxcAvatarUrl$); const video = useBehavior(vm.video$); const videoEnabled = useBehavior(vm.videoEnabled$); const unencryptedWarning = useBehavior(vm.unencryptedWarning$); @@ -161,11 +164,13 @@ const SpotlightItem: FC = ({ className: classNames(styles.item, { [styles.snap]: snap }), targetWidth, targetHeight, - video, + video: video ?? undefined, videoEnabled, - member: vm.member, + userId: vm.userId, unencryptedWarning, + focusUrl, displayName, + mxcAvatarUrl, focusable, encryptionStatus, "aria-hidden": ariaHidden, diff --git a/src/utils/displayname-integration.test.ts b/src/utils/displayname-integration.test.ts index 5ba42e70..12d65176 100644 --- a/src/utils/displayname-integration.test.ts +++ b/src/utils/displayname-integration.test.ts @@ -6,10 +6,10 @@ Please see LICENSE in the repository root for full details. */ import { afterEach, beforeAll, describe, expect, test, vi } from "vitest"; +import { type RoomMember } from "matrix-js-sdk"; import { shouldDisambiguate } from "./displayname"; import { alice } from "./test-fixtures"; -import { mockMatrixRoom } from "./test"; // Ideally these tests would be in ./displayname.test.ts but I can't figure out how to // just spy on the removeHiddenChars() function without impacting the other tests. @@ -29,7 +29,7 @@ describe("shouldDisambiguate", () => { }); test("should only call removeHiddenChars once for a single displayname", () => { - const room = mockMatrixRoom({}); + const room: Map> = new Map([]); shouldDisambiguate(alice, [], room); expect(jsUtils.removeHiddenChars).toHaveBeenCalledTimes(1); for (let i = 0; i < 10; i++) { diff --git a/src/utils/displayname.test.ts b/src/utils/displayname.test.ts index f28a3e55..fa732f19 100644 --- a/src/utils/displayname.test.ts +++ b/src/utils/displayname.test.ts @@ -20,62 +20,70 @@ import { daveRTL, } from "./test-fixtures"; import { mockMatrixRoom } from "./test"; +import { roomToMembersMap } from "../state/CallViewModel/remoteMembers/MatrixMemberMetadata"; describe("shouldDisambiguate", () => { test("should not disambiguate a solo member", () => { - const room = mockMatrixRoom({}); - expect(shouldDisambiguate(alice, [], room)).toEqual(false); + const room = mockMatrixRoom({ + getMembersWithMembership: () => [], + }); + expect(shouldDisambiguate(alice, [], roomToMembersMap(room))).toEqual( + false, + ); }); test("should not disambiguate a member with an empty displayname", () => { const room = mockMatrixRoom({ - getMember: (u) => - [alice, aliceDoppelganger].find((m) => m.userId === u) ?? null, + getMembersWithMembership: () => [alice, aliceDoppelganger], }); expect( shouldDisambiguate( { rawDisplayName: "", userId: alice.userId }, [aliceRtcMember, aliceDoppelgangerRtcMember], - room, + roomToMembersMap(room), ), ).toEqual(false); }); test("should disambiguate a member with RTL characters", () => { - const room = mockMatrixRoom({}); - expect(shouldDisambiguate(daveRTL, [], room)).toEqual(true); + const room = mockMatrixRoom({ getMembersWithMembership: () => [] }); + expect(shouldDisambiguate(daveRTL, [], roomToMembersMap(room))).toEqual( + true, + ); }); test("should disambiguate a member with a matching displayname", () => { const room = mockMatrixRoom({ - getMember: (u) => - [alice, aliceDoppelganger].find((m) => m.userId === u) ?? null, + getMembersWithMembership: () => [alice, aliceDoppelganger], }); expect( shouldDisambiguate( alice, [aliceRtcMember, aliceDoppelgangerRtcMember], - room, + roomToMembersMap(room), ), ).toEqual(true); expect( shouldDisambiguate( aliceDoppelganger, [aliceRtcMember, aliceDoppelgangerRtcMember], - room, + roomToMembersMap(room), ), ).toEqual(true); }); test("should disambiguate a member with a matching displayname with hidden spaces", () => { const room = mockMatrixRoom({ - getMember: (u) => - [bob, bobZeroWidthSpace].find((m) => m.userId === u) ?? null, + getMembersWithMembership: () => [bob, bobZeroWidthSpace], }); expect( - shouldDisambiguate(bob, [bobRtcMember, bobZeroWidthSpaceRtcMember], room), + shouldDisambiguate( + bob, + [bobRtcMember, bobZeroWidthSpaceRtcMember], + roomToMembersMap(room), + ), ).toEqual(true); expect( shouldDisambiguate( bobZeroWidthSpace, [bobRtcMember, bobZeroWidthSpaceRtcMember], - room, + roomToMembersMap(room), ), ).toEqual(true); }); @@ -83,11 +91,14 @@ describe("shouldDisambiguate", () => { "should disambiguate a member with a displayname containing a mxid-like string '%s'", (rawDisplayName) => { const room = mockMatrixRoom({ - getMember: (u) => - [alice, aliceDoppelganger].find((m) => m.userId === u) ?? null, + getMembersWithMembership: () => [alice, aliceDoppelganger], }); expect( - shouldDisambiguate({ rawDisplayName, userId: alice.userId }, [], room), + shouldDisambiguate( + { rawDisplayName, userId: alice.userId }, + [], + roomToMembersMap(room), + ), ).toEqual(true); }, ); diff --git a/src/utils/displayname.ts b/src/utils/displayname.ts index 8e989d3b..5ab5de9b 100644 --- a/src/utils/displayname.ts +++ b/src/utils/displayname.ts @@ -10,7 +10,7 @@ import { removeHiddenChars as removeHiddenCharsUncached, } from "matrix-js-sdk/lib/utils"; -import type { Room } from "matrix-js-sdk"; +import type { RoomMember } from "matrix-js-sdk"; import type { CallMembership } from "matrix-js-sdk/lib/matrixrtc"; // Calling removeHiddenChars() can be slow on Safari, so we cache the results. @@ -40,8 +40,8 @@ function removeHiddenChars(str: string): string { // Borrowed from https://github.com/matrix-org/matrix-js-sdk/blob/f10deb5ef2e8f061ff005af0476034382ea128ca/src/models/room-member.ts#L409 export function shouldDisambiguate( member: { rawDisplayName?: string; userId: string }, - memberships: CallMembership[], - room: Room, + memberships: Pick[], + roomMembers: Map>, ): boolean { const { rawDisplayName: displayName, userId } = member; if (!displayName || displayName === userId) return false; @@ -65,7 +65,7 @@ export function shouldDisambiguate( // displayname, after hidden character removal. return ( memberships - .map((m) => m.userId && room.getMember(m.userId)) + .map((m) => m.userId && roomMembers.get(m.userId)) // NOTE: We *should* have a room member for everyone. .filter((m) => !!m) .filter((m) => m.userId !== userId) diff --git a/src/utils/observable.test.ts b/src/utils/observable.test.ts index e039c846..d1034e7b 100644 --- a/src/utils/observable.test.ts +++ b/src/utils/observable.test.ts @@ -9,7 +9,7 @@ import { test } from "vitest"; import { Subject } from "rxjs"; import { withTestScheduler } from "./test"; -import { generateKeyed$, pauseWhen } from "./observable"; +import { generateItems, pauseWhen } from "./observable"; test("pauseWhen", () => { withTestScheduler(({ behavior, expectObservable }) => { @@ -24,7 +24,7 @@ test("pauseWhen", () => { }); }); -test("generateKeyed$ has the right output and ends scopes at the right times", () => { +test("generateItems", () => { const scope1$ = new Subject(); const scope2$ = new Subject(); const scope3$ = new Subject(); @@ -44,18 +44,27 @@ test("generateKeyed$ has the right output and ends scopes at the right times", ( const scope4Marbles = " ----yn"; expectObservable( - generateKeyed$(hot(inputMarbles), (input, createOrGet) => { - for (let i = 1; i <= +input; i++) { - createOrGet(i.toString(), (scope) => { + hot(inputMarbles).pipe( + generateItems( + function* (input) { + for (let i = 1; i <= +input; i++) { + yield { keys: [i], data: undefined }; + } + }, + (scope, data$, i) => { scopeSubjects[i - 1].next("y"); scope.onEnd(() => scopeSubjects[i - 1].next("n")); return i.toString(); - }); - } - return "abcd"[+input - 1]; - }), + }, + ), + ), subscriptionMarbles, - ).toBe(outputMarbles); + ).toBe(outputMarbles, { + a: ["1"], + b: ["1", "2"], + c: ["1", "2", "3"], + d: ["1", "2", "3", "4"], + }); expectObservable(scope1$).toBe(scope1Marbles); expectObservable(scope2$).toBe(scope2Marbles); diff --git a/src/utils/observable.ts b/src/utils/observable.ts index eb817991..053921cd 100644 --- a/src/utils/observable.ts +++ b/src/utils/observable.ts @@ -20,10 +20,12 @@ import { takeWhile, tap, withLatestFrom, + BehaviorSubject, + type OperatorFunction, } from "rxjs"; import { type Behavior } from "../state/Behavior"; -import { ObservableScope } from "../state/ObservableScope"; +import { Epoch, ObservableScope } from "../state/ObservableScope"; const nothing = Symbol("nothing"); @@ -119,70 +121,156 @@ export function pauseWhen(pause$: Behavior) { ); } +interface ItemHandle { + scope: ObservableScope; + data$: BehaviorSubject; + item: Item; +} + /** - * Maps a changing input value to an output value consisting of items that have - * automatically generated ObservableScopes tied to a key. Items will be - * automatically created when their key is requested for the first time, reused - * when the same key is requested at a later time, and destroyed (have their - * scope ended) when the key is no longer requested. + * Maps a changing input value to a collection of items that each capture some + * dynamic data and are tied to a key. Items will be automatically created when + * their key is requested for the first time, reused when the same key is + * requested at a later time, and destroyed (have their scope ended) when the + * key is no longer requested. * * @param input$ The input value to be mapped. - * @param project A function mapping input values to output values. This - * function receives an additional callback `createOrGet` which can be used - * within the function body to request that an item be generated for a certain - * key. The caller provides a factory which will be used to create the item if - * it is being requested for the first time. Otherwise, the item previously - * existing under that key will be returned. + * @param generator A generator function yielding a tuple of keys and the + * currently associated data for each item that it wants to exist. + * @param factory A function constructing an individual item, given the item's key, + * dynamic data, and an automatically managed ObservableScope for the item. */ -export function generateKeyed$( - input$: Observable, - project: ( - input: In, - createOrGet: ( - key: string, - factory: (scope: ObservableScope) => Item, - ) => Item, - ) => Out, -): Observable { - return input$.pipe( - // Keep track of the existing items over time, so we can reuse them - scan< - In, - { - items: Map; - output: Out; - }, - { items: Map } - >( - (state, data) => { - const nextItems = new Map< - string, - { item: Item; scope: ObservableScope } - >(); +export function generateItems< + Input, + Keys extends [unknown, ...unknown[]], + Data, + Item, +>( + generator: ( + input: Input, + ) => Generator<{ keys: readonly [...Keys]; data: Data }, void, void>, + factory: ( + scope: ObservableScope, + data$: Behavior, + ...keys: Keys + ) => Item, +): OperatorFunction { + return generateItemsInternal(generator, factory, (items) => items); +} - const output = project(data, (key, factory) => { - let item = state.items.get(key); - if (item === undefined) { - // First time requesting the key; create the item - const scope = new ObservableScope(); - item = { item: factory(scope), scope }; - } - nextItems.set(key, item); - return item.item; - }); - - // Destroy all items that are no longer being requested - for (const [key, { scope }] of state.items) - if (!nextItems.has(key)) scope.end(); - - return { items: nextItems, output }; - }, - { items: new Map() }, - ), - finalizeValue((state) => { - // Destroy all remaining items when no longer subscribed - for (const { scope } of state.items.values()) scope.end(); - }), - map(({ output }) => output), +/** + * Same as generateItems, but preserves epoch data. + */ +export function generateItemsWithEpoch< + Input, + Keys extends [unknown, ...unknown[]], + Data, + Item, +>( + generator: ( + input: Input, + ) => Generator<{ keys: readonly [...Keys]; data: Data }, void, void>, + factory: ( + scope: ObservableScope, + data$: Behavior, + ...keys: Keys + ) => Item, +): OperatorFunction, Epoch> { + return generateItemsInternal( + function* (input) { + yield* generator(input.value); + }, + factory, + (items, input) => new Epoch(items, input.epoch), ); } + +function generateItemsInternal< + Input, + Keys extends [unknown, ...unknown[]], + Data, + Item, + Output, +>( + generator: ( + input: Input, + ) => Generator<{ keys: readonly [...Keys]; data: Data }, void, void>, + factory: ( + scope: ObservableScope, + data$: Behavior, + ...keys: Keys + ) => Item, + project: (items: Item[], input: Input) => Output, +): OperatorFunction { + /* eslint-disable @typescript-eslint/no-explicit-any */ + return (input$) => + input$.pipe( + // Keep track of the existing items over time, so they can persist + scan< + Input, + { + map: Map; + items: Set>; + input: Input; + }, + { map: Map; items: Set> } + >( + ({ map: prevMap, items: prevItems }, input) => { + const nextMap = new Map(); + const nextItems = new Set>(); + + for (const { keys, data } of generator(input)) { + // Disable type checks for a second to grab the item out of a nested map + let i: any = prevMap; + for (const key of keys) i = i?.get(key); + let item = i as ItemHandle | undefined; + + if (item === undefined) { + // First time requesting the key; create the item + const scope = new ObservableScope(); + const data$ = new BehaviorSubject(data); + item = { scope, data$, item: factory(scope, data$, ...keys) }; + } else { + item.data$.next(data); + } + + // Likewise, disable type checks to insert the item in the nested map + let m: Map = nextMap; + for (let i = 0; i < keys.length - 1; i++) { + let inner = m.get(keys[i]); + if (inner === undefined) { + inner = new Map(); + m.set(keys[i], inner); + } + m = inner; + } + const finalKey = keys[keys.length - 1]; + if (m.has(finalKey)) + throw new Error( + `Keys must be unique (tried to generate multiple items for key ${keys})`, + ); + m.set(keys[keys.length - 1], item); + nextItems.add(item); + } + + // Destroy all items that are no longer being requested + for (const item of prevItems) + if (!nextItems.has(item)) item.scope.end(); + + return { map: nextMap, items: nextItems, input }; + }, + { map: new Map(), items: new Set() }, + ), + finalizeValue(({ items }) => { + // Destroy all remaining items when no longer subscribed + for (const { scope } of items) scope.end(); + }), + map(({ items, input }) => + project( + [...items].map(({ item }) => item), + input, + ), + ), + ); + /* eslint-enable @typescript-eslint/no-explicit-any */ +} diff --git a/src/utils/test-fixtures.ts b/src/utils/test-fixtures.ts index 9d93267e..4cf330b7 100644 --- a/src/utils/test-fixtures.ts +++ b/src/utils/test-fixtures.ts @@ -11,9 +11,9 @@ import { mockRemoteParticipant, } from "./test"; -export const localRtcMember = mockRtcMembership("@carol:example.org", "1111"); +export const localRtcMember = mockRtcMembership("@local:example.org", "1111"); export const localRtcMemberDevice2 = mockRtcMembership( - "@carol:example.org", + "@local:example.org", "2222", ); export const local = mockMatrixRoomMember(localRtcMember); @@ -37,7 +37,6 @@ export const aliceDoppelganger = mockMatrixRoomMember( rawDisplayName: "Alice", }, ); -export const aliceDoppelgangerId = `${aliceDoppelganger.userId}:${aliceDoppelgangerRtcMember.deviceId}`; export const bobRtcMember = mockRtcMembership("@bob:example.org", "BBBB"); export const bob = mockMatrixRoomMember(bobRtcMember, { @@ -55,10 +54,8 @@ export const bobZeroWidthSpace = mockMatrixRoomMember( rawDisplayName: "Bo\u200bb", }, ); -export const bobZeroWidthSpaceId = `${bobZeroWidthSpace.userId}:${bobZeroWidthSpaceRtcMember.deviceId}`; export const daveRTLRtcMember = mockRtcMembership("@dave2:example.org", "DDDD"); export const daveRTL = mockMatrixRoomMember(daveRTLRtcMember, { rawDisplayName: "\u202eevaD", }); -export const daveRTLId = `${daveRTL.userId}:${daveRTLRtcMember.deviceId}`; diff --git a/src/utils/test-viewmodel.ts b/src/utils/test-viewmodel.ts index 5cd64eb3..98c45d86 100644 --- a/src/utils/test-viewmodel.ts +++ b/src/utils/test-viewmodel.ts @@ -6,7 +6,7 @@ Please see LICENSE in the repository root for full details. */ import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc"; -import { BehaviorSubject, of } from "rxjs"; +import { BehaviorSubject } from "rxjs"; import { vitest } from "vitest"; import { type RelationsContainer } from "matrix-js-sdk/lib/models/relations-container"; import EventEmitter from "events"; @@ -20,10 +20,12 @@ import { ConnectionState, type Room as LivekitRoom } from "livekit-client"; import { E2eeType } from "../e2ee/e2eeType"; import { - CallViewModel, + type CallViewModel, + createCallViewModel$, type CallViewModelOptions, -} from "../state/CallViewModel"; +} from "../state/CallViewModel/CallViewModel"; import { + mockConfig, mockLivekitRoom, mockLocalParticipant, mockMatrixRoom, @@ -36,6 +38,8 @@ import { aliceRtcMember, localRtcMember } from "./test-fixtures"; import { type RaisedHandInfo, type ReactionInfo } from "../reactions"; import { constant } from "../state/Behavior"; +mockConfig({ livekit: { livekit_service_url: "https://example.com" } }); + export function getBasicRTCSession( members: RoomMember[], initialRtcMemberships: CallMembership[] = [localRtcMember, aliceRtcMember], @@ -57,6 +61,7 @@ export function getBasicRTCSession( getUserId: () => localRtcMember.userId, getDeviceId: () => localRtcMember.deviceId, getSyncState: () => SyncState.Syncing, + getDomain: () => null, sendEvent: vitest.fn().mockResolvedValue({ event_id: "$fake:event" }), redactEvent: vitest.fn().mockResolvedValue({ event_id: "$fake:event" }), decryptEventIfNeeded: vitest.fn().mockResolvedValue(undefined), @@ -78,6 +83,9 @@ export function getBasicRTCSession( ), } as Partial as MatrixClient, getMember: (userId) => matrixRoomMembers.get(userId) ?? null, + getMembers: () => Array.from(matrixRoomMembers.values()), + getMembersWithMembership: () => Array.from(matrixRoomMembers.values()), + guessDMUserId: vitest.fn(), roomId: matrixRoomId, on: vitest .fn() @@ -138,7 +146,7 @@ export function getBasicCallViewModelEnvironment( // const remoteParticipants$ = of([aliceParticipant]); - const vm = new CallViewModel( + const vm = createCallViewModel$( testScope(), rtcSession.asMockedSession(), matrixRoom, @@ -158,7 +166,7 @@ export function getBasicCallViewModelEnvironment( }, handRaisedSubject$, reactionsSubject$, - of({ processor: undefined, supported: false }), + constant({ processor: undefined, supported: false }), ); return { vm, diff --git a/src/utils/test.ts b/src/utils/test.ts index db85da4a..4fec433c 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -6,25 +6,32 @@ Please see LICENSE in the repository root for full details. */ import { map, type Observable, of, type SchedulerLike } from "rxjs"; import { type RunHelpers, TestScheduler } from "rxjs/testing"; -import { expect, type MockedObject, onTestFinished, vi, vitest } from "vitest"; import { - type RoomMember, - type Room as MatrixRoom, + expect, + type MockedObject, + type MockInstance, + onTestFinished, + vi, + vitest, +} from "vitest"; +import { MatrixEvent, + type Room as MatrixRoom, type Room, + type RoomMember, TypedEventEmitter, } from "matrix-js-sdk"; import { CallMembership, - type Transport, + type LivekitFocusSelection, + type LivekitTransport, + type MatrixRTCSession, MatrixRTCSessionEvent, type MatrixRTCSessionEventHandlerMap, MembershipManagerEvent, type SessionMembershipData, Status, - type LivekitFocusSelection, - type MatrixRTCSession, - type LivekitTransport, + type Transport, } from "matrix-js-sdk/lib/matrixrtc"; import { type MembershipManagerEventHandlerMap } from "matrix-js-sdk/lib/matrixrtc/IMembershipManager"; import { @@ -78,11 +85,11 @@ export interface OurRunHelpers extends RunHelpers { * diagram. */ schedule: (marbles: string, actions: Record void>) => void; - behavior( + behavior: ( marbles: string, values?: { [marble: string]: T }, error?: unknown, - ): Behavior; + ) => Behavior; scope: ObservableScope; } @@ -106,7 +113,7 @@ export function withTestScheduler( continuation: (helpers: OurRunHelpers) => void, ): void { const scheduler = new TestScheduler((actual, expected) => { - expect(actual).deep.equals(expected); + expect(actual).toStrictEqual(expected); }); const scope = new ObservableScope(); // we set the test scheduler as a global so that you can watch it in a debugger @@ -187,6 +194,29 @@ export const exampleTransport: LivekitTransport = { livekit_alias: "!alias:example.org", }; +export function mockCallMembership( + userId: string, + deviceId: string, + transport?: Transport, +): CallMembership { + const t = transport ?? transportForUser(userId); + return { + userId: userId, + deviceId: deviceId, + getTransport: vi.fn().mockReturnValue(t), + transports: [t], + } as unknown as CallMembership; +} + +function transportForUser(userId: string): Transport { + const domain = userId.split(":")[1]; + return { + type: "livekit", + livekit_service_url: `https://lk.${domain}`, + livekit_alias: `!alias:${domain}`, + }; +} + export function mockRtcMembership( user: string | RoomMember, deviceId: string, @@ -246,6 +276,7 @@ export function mockLivekitRoom( }: { remoteParticipants$?: Observable } = {}, ): LivekitRoom { const livekitRoom = { + options: {}, ...mockEmitter(), ...room, } as Partial as LivekitRoom; @@ -268,6 +299,7 @@ export function mockLocalParticipant( return { isLocal: true, trackPublications: new Map(), + unpublishTracks: async () => Promise.resolve(), getTrackPublication: () => ({}) as Partial as LocalTrackPublication, ...mockEmitter(), @@ -281,18 +313,20 @@ export function createLocalMedia( localParticipant: LocalParticipant, mediaDevices: MediaDevices, ): LocalUserMediaViewModel { + const member = mockMatrixRoomMember(localRtcMember, roomMember); return new LocalUserMediaViewModel( testScope(), "local", - mockMatrixRoomMember(localRtcMember, roomMember), + member.userId, constant(localParticipant), { kind: E2eeType.PER_PARTICIPANT, }, - mockLivekitRoom({ localParticipant }), - "https://rtc-example.org", + constant(mockLivekitRoom({ localParticipant })), + constant("https://rtc-example.org"), mediaDevices, - constant(roomMember.rawDisplayName ?? "nodisplayname"), + constant(member.rawDisplayName ?? "nodisplayname"), + constant(member.getMxcAvatarUrl()), constant(null), constant(null), ); @@ -306,6 +340,8 @@ export function mockRemoteParticipant( setVolume() {}, getTrackPublication: () => ({}) as Partial as RemoteTrackPublication, + // this will only get used for `getTrackPublications().length` + getTrackPublications: () => [0], ...mockEmitter(), ...participant, } as RemoteParticipant; @@ -316,31 +352,38 @@ export function createRemoteMedia( roomMember: Partial, participant: Partial, ): RemoteUserMediaViewModel { + const member = mockMatrixRoomMember(localRtcMember, roomMember); const remoteParticipant = mockRemoteParticipant(participant); return new RemoteUserMediaViewModel( testScope(), "remote", - mockMatrixRoomMember(localRtcMember, roomMember), + member.userId, of(remoteParticipant), { kind: E2eeType.PER_PARTICIPANT, }, - mockLivekitRoom({}, { remoteParticipants$: of([remoteParticipant]) }), - "https://rtc-example.org", + constant( + mockLivekitRoom({}, { remoteParticipants$: of([remoteParticipant]) }), + ), + constant("https://rtc-example.org"), constant(false), - constant(roomMember.rawDisplayName ?? "nodisplayname"), + constant(member.rawDisplayName ?? "nodisplayname"), + constant(member.getMxcAvatarUrl()), constant(null), constant(null), ); } -export function mockConfig(config: Partial = {}): void { - vi.spyOn(Config, "get").mockReturnValue({ +export function mockConfig( + config: Partial = {}, +): MockInstance<() => ResolvedConfigOptions> { + const spy = vi.spyOn(Config, "get").mockReturnValue({ ...DEFAULT_CONFIG, ...config, }); // simulate loading the config vi.spyOn(Config, "init").mockResolvedValue(void 0); + return spy; } export class MockRTCSession extends TypedEventEmitter< diff --git a/tsconfig.json b/tsconfig.json index 41ca7f8b..e864ecfc 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -50,5 +50,6 @@ "plugins": [{ "name": "typescript-eslint-language-service" }] }, - "include": ["./src/**/*.ts", "./src/**/*.tsx", "./playwright/**/*.ts"] + "include": ["./src/**/*.ts", "./src/**/*.tsx", "./playwright/**/*.ts"], + "exclude": ["**.test.ts"] } diff --git a/yarn.lock b/yarn.lock index e78dbbf2..97ca1985 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2731,37 +2731,39 @@ __metadata: languageName: node linkType: hard -"@livekit/components-core@npm:0.12.10, @livekit/components-core@npm:^0.12.0": - version: 0.12.10 - resolution: "@livekit/components-core@npm:0.12.10" +"@livekit/components-core@npm:0.12.11, @livekit/components-core@npm:^0.12.0": + version: 0.12.11 + resolution: "@livekit/components-core@npm:0.12.11" dependencies: "@floating-ui/dom": "npm:1.6.13" loglevel: "npm:1.9.1" rxjs: "npm:7.8.2" peerDependencies: - livekit-client: ^2.13.3 + livekit-client: ^2.15.14 tslib: ^2.6.2 - checksum: 10c0/bfd84fb950f72dd037bd5329658c1e750a8ac6b8f2953ea673e3e944b8ea9d412ef9c98eb8b690052323e03c675964b162aacb00e60530cdc5187f77d21979bd + checksum: 10c0/9c2ac3d30bb8cc9067ae0b2049784f81e90e57df9eabf7edbaf3c8ceb65a63f644a4e6abeb6cc38d3ebe52663d8dbb88535e01a965011f365d5ae1f3daf86052 languageName: node linkType: hard "@livekit/components-react@npm:^2.0.0": - version: 2.9.15 - resolution: "@livekit/components-react@npm:2.9.15" + version: 2.9.16 + resolution: "@livekit/components-react@npm:2.9.16" dependencies: - "@livekit/components-core": "npm:0.12.10" + "@livekit/components-core": "npm:0.12.11" clsx: "npm:2.1.1" + events: "npm:^3.3.0" + jose: "npm:^6.0.12" usehooks-ts: "npm:3.1.1" peerDependencies: "@livekit/krisp-noise-filter": ^0.2.12 || ^0.3.0 - livekit-client: ^2.13.3 + livekit-client: ^2.15.14 react: ">=18" react-dom: ">=18" tslib: ^2.6.2 peerDependenciesMeta: "@livekit/krisp-noise-filter": optional: true - checksum: 10c0/58a93d85c3b8267d0afd00eceb4f34992ce66124f93450e828a78dd825ecc20e254c3123ed22ec33061a3728f50f4f020ff896769fb3a0ff79656ed8cf452a2b + checksum: 10c0/4ba4ff473c5a29d3107412733a6676a3b708d70684ed463e9b34cda26abb3d2f317c2828a52e730837b756de9df3fc248260d6f390aedebfb6ec96ef63c7b151 languageName: node linkType: hard @@ -5247,12 +5249,12 @@ __metadata: languageName: node linkType: hard -"@types/node@npm:^22.0.0": - version: 22.17.0 - resolution: "@types/node@npm:22.17.0" +"@types/node@npm:^24.0.0": + version: 24.10.0 + resolution: "@types/node@npm:24.10.0" dependencies: - undici-types: "npm:~6.21.0" - checksum: 10c0/e1c603b660d3de3243dfc02ded5d40623ff3f36315ffbdd8cdc81bc2c5a8da172035879d437b72e9fa61ca01827f28e9c2b0c32898f411a8e9ba0a5efac0b4ca + undici-types: "npm:~7.16.0" + checksum: 10c0/f82ed7194e16f5590ef7afdc20c6d09068c76d50278b485ede8f0c5749683536e3064ffa8def8db76915196afb3724b854aa5723c64d6571b890b14492943b46 languageName: node linkType: hard @@ -7507,7 +7509,7 @@ __metadata: "@types/grecaptcha": "npm:^3.0.9" "@types/jsdom": "npm:^21.1.7" "@types/lodash-es": "npm:^4.17.12" - "@types/node": "npm:^22.0.0" + "@types/node": "npm:^24.0.0" "@types/pako": "npm:^2.0.3" "@types/qrcode": "npm:^1.5.5" "@types/react": "npm:^19.0.0" @@ -9828,6 +9830,13 @@ __metadata: languageName: node linkType: hard +"jose@npm:^6.0.12": + version: 6.1.2 + resolution: "jose@npm:6.1.2" + checksum: 10c0/55f79426f43e652ed6d5de938d50f66bb0a10dcae078db81a23f8d3303e889ce226f000e815f3211f9956bb84badce10da892d130d40fe2eca658045a6f1778e + languageName: node + linkType: hard + "jose@npm:^6.1.0": version: 6.1.0 resolution: "jose@npm:6.1.0" @@ -10110,8 +10119,8 @@ __metadata: linkType: hard "livekit-client@npm:^2.13.0": - version: 2.15.13 - resolution: "livekit-client@npm:2.15.13" + version: 2.16.0 + resolution: "livekit-client@npm:2.16.0" dependencies: "@livekit/mutex": "npm:1.1.1" "@livekit/protocol": "npm:1.42.2" @@ -10125,7 +10134,7 @@ __metadata: webrtc-adapter: "npm:^9.0.1" peerDependencies: "@types/dom-mediacapture-record": ^1 - checksum: 10c0/5a061df9000461a6d40ef8aa1e72e8aedc640181cc57fe6f2c48c5c7f90ce96a735b125aede377fc43f4692a685e098f17eeae0f42c5b2fed473305867bf2789 + checksum: 10c0/5d03adc5d09efde343ab894db397529dff26117598e773b23a5df90a4fb166bde12c6bb1f2cfd1d28dbaf93fe9f275026d7abb75f2ffd2ba816393a2d58e6c7e languageName: node linkType: hard @@ -13603,10 +13612,10 @@ __metadata: languageName: node linkType: hard -"undici-types@npm:~6.21.0": - version: 6.21.0 - resolution: "undici-types@npm:6.21.0" - checksum: 10c0/c01ed51829b10aa72fc3ce64b747f8e74ae9b60eafa19a7b46ef624403508a54c526ffab06a14a26b3120d055e1104d7abe7c9017e83ced038ea5cf52f8d5e04 +"undici-types@npm:~7.16.0": + version: 7.16.0 + resolution: "undici-types@npm:7.16.0" + checksum: 10c0/3033e2f2b5c9f1504bdc5934646cb54e37ecaca0f9249c983f7b1fc2e87c6d18399ebb05dc7fd5419e02b2e915f734d872a65da2e3eeed1813951c427d33cc9a languageName: node linkType: hard