From 7c5336fc40f832632cf058a56b7a42ca2833c03b Mon Sep 17 00:00:00 2001 From: Robin Date: Thu, 12 Jun 2025 19:15:43 -0400 Subject: [PATCH 01/24] Remove the switch camera button --- src/button/Button.tsx | 18 ------- src/room/InCallView.module.css | 1 - src/room/InCallView.tsx | 12 ----- src/room/LobbyView.tsx | 14 +---- src/room/useSwitchCamera.ts | 93 ---------------------------------- src/state/CallViewModel.ts | 14 ----- 6 files changed, 1 insertion(+), 151 deletions(-) delete mode 100644 src/room/useSwitchCamera.ts diff --git a/src/button/Button.tsx b/src/button/Button.tsx index 4caa9c4f..c11c92dd 100644 --- a/src/button/Button.tsx +++ b/src/button/Button.tsx @@ -16,7 +16,6 @@ import { EndCallIcon, ShareScreenSolidIcon, SettingsSolidIcon, - SwitchCameraSolidIcon, } from "@vector-im/compound-design-tokens/assets/web/icons"; import styles from "./Button.module.css"; @@ -67,23 +66,6 @@ export const VideoButton: FC = ({ muted, ...props }) => { ); }; -export const SwitchCameraButton: FC> = ( - props, -) => { - const { t } = useTranslation(); - - return ( - - - - ); -}; - interface ShareScreenButtonProps extends ComponentPropsWithoutRef<"button"> { enabled: boolean; } diff --git a/src/room/InCallView.module.css b/src/room/InCallView.module.css index 24dfbe5c..41d48db1 100644 --- a/src/room/InCallView.module.css +++ b/src/room/InCallView.module.css @@ -115,7 +115,6 @@ Please see LICENSE in the repository root for full details. @media (max-width: 340px) { .invite, - .switchCamera, .shareScreen { display: none; } diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 452e8572..53fc0667 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -44,7 +44,6 @@ import { ShareScreenButton, SettingsButton, ReactionToggleButton, - SwitchCameraButton, } from "../button"; import { Header, LeftNav, RightNav, RoomHeaderInfo } from "../Header"; import { type HeaderStyle, useUrlParams } from "../UrlParams"; @@ -94,7 +93,6 @@ import { useReactionsSender, } from "../reactions/useReactionsSender"; import { ReactionsAudioRenderer } from "./ReactionAudioRenderer"; -import { useSwitchCamera } from "./useSwitchCamera"; import { ReactionsOverlay } from "./ReactionsOverlay"; import { CallEventAudioRenderer } from "./CallEventAudioRenderer"; import { @@ -311,7 +309,6 @@ export const InCallView: FC = ({ const showFooter = useObservableEagerState(vm.showFooter$); const earpieceMode = useObservableEagerState(vm.earpieceMode$); const audioOutputSwitcher = useObservableEagerState(vm.audioOutputSwitcher$); - const switchCamera = useSwitchCamera(vm.localVideo$); // Ideally we could detect taps by listening for click events and checking // that the pointerType of the event is "touch", but this isn't yet supported @@ -672,15 +669,6 @@ export const InCallView: FC = ({ data-testid="incall_videomute" />, ); - if (switchCamera !== null) - buttons.push( - , - ); if (canScreenshare && !hideScreensharing) { buttons.push( = ({ }, [devices, videoInputId, videoTrack]); useTrackProcessorSync(videoTrack); - const showSwitchCamera = useShowSwitchCamera( - useObservable( - (inputs$) => inputs$.pipe(map(([video]) => video)), - [videoTrack], - ), - ); // TODO: Unify this component with InCallView, so we can get slick joining // animations and don't have to feel bad about reusing its CSS @@ -257,9 +248,6 @@ export const LobbyView: FC = ({ onClick={onVideoPress} disabled={muteStates.video.setEnabled === null} /> - {showSwitchCamera && ( - - )} {!confineToRoom && } diff --git a/src/room/useSwitchCamera.ts b/src/room/useSwitchCamera.ts deleted file mode 100644 index 975776ae..00000000 --- a/src/room/useSwitchCamera.ts +++ /dev/null @@ -1,93 +0,0 @@ -/* -Copyright 2024 New Vector Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { - fromEvent, - map, - merge, - type Observable, - of, - startWith, - switchMap, -} from "rxjs"; -import { - facingModeFromLocalTrack, - type LocalVideoTrack, - TrackEvent, -} from "livekit-client"; -import { useObservable, useObservableEagerState } from "observable-hooks"; -import { logger } from "matrix-js-sdk/lib/logger"; - -import { useMediaDevices } from "../MediaDevicesContext"; -import { platform } from "../Platform"; -import { useLatest } from "../useLatest"; - -/** - * Determines whether the user should be shown a button to switch their camera, - * producing a callback if so. - */ -export function useSwitchCamera( - video$: Observable, -): (() => void) | null { - const mediaDevices = useMediaDevices(); - const setVideoInput = useLatest(mediaDevices.videoInput.select); - - // Produce an observable like the input 'video' observable, except make it - // emit whenever the track is muted or the device changes - const videoTrack$: Observable = useObservable( - (inputs$) => - inputs$.pipe( - switchMap(([video$]) => video$), - switchMap((video) => { - if (video === null) return of(null); - return merge( - fromEvent(video, TrackEvent.Restarted).pipe( - startWith(null), - map(() => video), - ), - fromEvent(video, TrackEvent.Muted).pipe(map(() => null)), - ); - }), - ), - [video$], - ); - - const switchCamera$: Observable<(() => void) | null> = useObservable( - (inputs$) => - platform === "desktop" - ? of(null) - : inputs$.pipe( - switchMap(([track$]) => track$), - map((track) => { - if (track === null) return null; - const facingMode = facingModeFromLocalTrack(track).facingMode; - // If the camera isn't front or back-facing, don't provide a switch - // camera shortcut at all - if (facingMode !== "user" && facingMode !== "environment") - return null; - // Restart the track with a camera facing the opposite direction - return (): void => - void track - .restartTrack({ - facingMode: facingMode === "user" ? "environment" : "user", - }) - .then(() => { - // Inform the MediaDeviceContext which camera was chosen - const deviceId = - track.mediaStreamTrack.getSettings().deviceId; - if (deviceId !== undefined) setVideoInput.current(deviceId); - }) - .catch((e) => - logger.error("Failed to switch camera", facingMode, e), - ); - }), - ), - [videoTrack$], - ); - - return useObservableEagerState(switchCamera$); -} diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index fc1222c4..ae7b78c1 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -13,10 +13,8 @@ import { import { type Room as LivekitRoom, type LocalParticipant, - LocalVideoTrack, ParticipantEvent, type RemoteParticipant, - Track, } from "livekit-client"; import { RoomStateEvent, type Room, type RoomMember } from "matrix-js-sdk"; import { @@ -60,7 +58,6 @@ import { import { LocalUserMediaViewModel, type MediaViewModel, - observeTrackReference$, RemoteUserMediaViewModel, ScreenShareViewModel, type UserMediaViewModel, @@ -382,17 +379,6 @@ function getRoomMemberFromRtcMember( // TODO: Move wayyyy more business logic from the call and lobby views into here export class CallViewModel extends ViewModel { - public readonly localVideo$: Observable = - observeTrackReference$( - of(this.livekitRoom.localParticipant), - Track.Source.Camera, - ).pipe( - map((trackRef) => { - const track = trackRef?.publication?.track; - return track instanceof LocalVideoTrack ? track : null; - }), - ); - /** * The raw list of RemoteParticipants as reported by LiveKit */ From 0c194617a30fbdf707fd3661ec39a674129b333a Mon Sep 17 00:00:00 2001 From: Robin Date: Thu, 12 Jun 2025 19:16:37 -0400 Subject: [PATCH 02/24] Add camera switching to the media view model --- src/state/CallViewModel.ts | 4 ++ src/state/MediaViewModel.test.ts | 119 +++++++++++++++++++++++++++++-- src/state/MediaViewModel.ts | 75 ++++++++++++++++--- src/tile/SpotlightTile.test.tsx | 4 ++ src/utils/test.ts | 4 +- 5 files changed, 187 insertions(+), 19 deletions(-) diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index ae7b78c1..9dd2c3f0 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -255,6 +255,7 @@ class UserMedia { participant: LocalParticipant | RemoteParticipant | undefined, encryptionSystem: EncryptionSystem, livekitRoom: LivekitRoom, + mediaDevices: MediaDevices, displayname$: Observable, handRaised$: Observable, reaction$: Observable, @@ -268,6 +269,7 @@ class UserMedia { this.participant$.asObservable() as Observable, encryptionSystem, livekitRoom, + mediaDevices, displayname$, handRaised$, reaction$, @@ -565,6 +567,7 @@ export class CallViewModel extends ViewModel { participant, this.encryptionSystem, this.livekitRoom, + this.mediaDevices, this.memberDisplaynames$.pipe( map((m) => m.get(matrixIdentifier) ?? "[👻]"), ), @@ -629,6 +632,7 @@ export class CallViewModel extends ViewModel { participant, this.encryptionSystem, this.livekitRoom, + this.mediaDevices, this.memberDisplaynames$.pipe( map((m) => m.get(participant.identity) ?? "[👻]"), ), diff --git a/src/state/MediaViewModel.test.ts b/src/state/MediaViewModel.test.ts index 601133ac..8de550d4 100644 --- a/src/state/MediaViewModel.test.ts +++ b/src/state/MediaViewModel.test.ts @@ -5,14 +5,40 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { expect, test, vi } from "vitest"; +import { expect, onTestFinished, test, vi } from "vitest"; +import { of } from "rxjs"; +import { + type LocalTrackPublication, + LocalVideoTrack, + TrackEvent, +} from "livekit-client"; +import { waitFor } from "@testing-library/dom"; import { + mockLocalParticipant, + mockMediaDevices, mockRtcMembership, withLocalMedia, withRemoteMedia, withTestScheduler, } from "../utils/test"; +import { getValue } from "../utils/observable"; + +global.MediaStreamTrack = class {} as unknown as { + new (): MediaStreamTrack; + prototype: MediaStreamTrack; +}; +global.MediaStream = class {} as unknown as { + new (): MediaStream; + prototype: MediaStream; +}; + +const platformMock = vi.hoisted(() => vi.fn(() => "desktop")); +vi.mock("../Platform", () => ({ + get platform(): string { + return platformMock(); + }, +})); const rtcMembership = mockRtcMembership("@alice:example.org", "AAAA"); @@ -79,17 +105,23 @@ test("toggle fit/contain for a participant's video", async () => { }); test("local media remembers whether it should always be shown", async () => { - await withLocalMedia(rtcMembership, {}, (vm) => - withTestScheduler(({ expectObservable, schedule }) => { - schedule("-a|", { a: () => vm.setAlwaysShow(false) }); - expectObservable(vm.alwaysShow$).toBe("ab", { a: true, b: false }); - }), + await withLocalMedia( + rtcMembership, + {}, + mockLocalParticipant({}), + mockMediaDevices({}), + (vm) => + withTestScheduler(({ expectObservable, schedule }) => { + schedule("-a|", { a: () => vm.setAlwaysShow(false) }); + expectObservable(vm.alwaysShow$).toBe("ab", { a: true, b: false }); + }), ); // Next local media should start out *not* always shown await withLocalMedia( rtcMembership, - {}, + mockLocalParticipant({}), + mockMediaDevices({}), (vm) => withTestScheduler(({ expectObservable, schedule }) => { schedule("-a|", { a: () => vm.setAlwaysShow(true) }); @@ -97,3 +129,76 @@ test("local media remembers whether it should always be shown", async () => { }), ); }); + +test("switch cameras", async () => { + // Camera switching is only available on mobile + platformMock.mockReturnValue("android"); + onTestFinished(() => void platformMock.mockReset()); + + // Construct a mock video track which knows how to be restarted + const track = new LocalVideoTrack({ + getConstraints() {}, + addEventListener() {}, + removeEventListener() {}, + } as unknown as MediaStreamTrack); + + let deviceId = "front camera"; + const restartTrack = vi.fn(async ({ facingMode }) => { + deviceId = facingMode === "user" ? "front camera" : "back camera"; + track.emit(TrackEvent.Restarted); + return Promise.resolve(); + }); + track.restartTrack = restartTrack; + + Object.defineProperty(track, "mediaStreamTrack", { + get() { + return { + label: "Video", + getSettings: (): object => ({ + deviceId, + facingMode: deviceId === "front camera" ? "user" : "environment", + }), + }; + }, + }); + + const selectVideoInput = vi.fn(); + + await withLocalMedia( + rtcMembership, + {}, + mockLocalParticipant({ + getTrackPublication() { + return { track } as unknown as LocalTrackPublication; + }, + }), + mockMediaDevices({ + videoInput: { + available$: of(new Map()), + selected$: of(undefined), + select: selectVideoInput, + }, + }), + async (vm) => { + // Switch to back camera + getValue(vm.switchCamera$)!(); + expect(restartTrack).toHaveBeenCalledTimes(1); + expect(restartTrack).toHaveBeenCalledWith({ facingMode: "environment" }); + await waitFor(() => { + expect(selectVideoInput).toHaveBeenCalledTimes(1); + expect(selectVideoInput).toHaveBeenCalledWith("back camera"); + }); + expect(deviceId).toBe("back camera"); + + // Switch to front camera + getValue(vm.switchCamera$)!(); + expect(restartTrack).toHaveBeenCalledTimes(2); + expect(restartTrack).toHaveBeenLastCalledWith({ facingMode: "user" }); + await waitFor(() => { + expect(selectVideoInput).toHaveBeenCalledTimes(2); + expect(selectVideoInput).toHaveBeenLastCalledWith("front camera"); + }); + expect(deviceId).toBe("front camera"); + }, + ); +}); diff --git a/src/state/MediaViewModel.ts b/src/state/MediaViewModel.ts index 424d003e..920b6ef3 100644 --- a/src/state/MediaViewModel.ts +++ b/src/state/MediaViewModel.ts @@ -16,6 +16,7 @@ import { import { type LocalParticipant, LocalTrack, + LocalVideoTrack, type Participant, ParticipantEvent, type RemoteParticipant, @@ -27,6 +28,7 @@ import { RemoteTrack, } from "livekit-client"; import { type RoomMember } from "matrix-js-sdk"; +import { logger } from "matrix-js-sdk/lib/logger"; import { BehaviorSubject, type Observable, @@ -51,6 +53,8 @@ import { accumulate } from "../utils/observable"; import { type EncryptionSystem } from "../e2ee/sharedKeyManagement"; import { E2eeType } from "../e2ee/e2eeType"; import { type ReactionOption } from "../reactions"; +import { platform } from "../Platform"; +import { type MediaDevices } from "./MediaDevices"; export function observeTrackReference$( participant$: Observable, @@ -433,20 +437,35 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel { * The local participant's user media. */ export class LocalUserMediaViewModel extends BaseUserMediaViewModel { + /** + * The local video track as an observable that emits whenever the track + * changes, the camera is switched, or the track is muted. + */ + private readonly videoTrack$: Observable = + this.video$.pipe( + switchMap((v) => { + const track = v?.publication?.track; + if (!(track instanceof LocalVideoTrack)) return of(null); + return merge( + // Watch for track restarts because they indicate a camera switch + fromEvent(track, TrackEvent.Restarted).pipe( + startWith(null), + map(() => track), + ), + fromEvent(track, TrackEvent.Muted).pipe(map(() => null)), + ); + }), + ); + /** * Whether the video should be mirrored. */ - public readonly mirror$ = this.video$.pipe( - switchMap((v) => { - const track = v?.publication?.track; - if (!(track instanceof LocalTrack)) return of(false); - // Watch for track restarts, because they indicate a camera switch - return fromEvent(track, TrackEvent.Restarted).pipe( - startWith(null), - // Mirror only front-facing cameras (those that face the user) - map(() => facingModeFromLocalTrack(track).facingMode === "user"), - ); - }), + public readonly mirror$ = this.videoTrack$.pipe( + // Mirror only front-facing cameras (those that face the user) + map( + (track) => + track !== null && facingModeFromLocalTrack(track).facingMode === "user", + ), this.scope.state(), ); @@ -457,12 +476,46 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel { public readonly alwaysShow$ = alwaysShowSelf.value$; public readonly setAlwaysShow = alwaysShowSelf.setValue; + /** + * Callback for switching between the front and back cameras. + */ + public readonly switchCamera$: Observable<(() => void) | null> = + platform === "desktop" + ? of(null) + : this.videoTrack$.pipe( + map((track) => { + if (track === null) return null; + const facingMode = facingModeFromLocalTrack(track).facingMode; + // If the camera isn't front or back-facing, don't provide a switch + // camera shortcut at all + if (facingMode !== "user" && facingMode !== "environment") + return null; + // Restart the track with a camera facing the opposite direction + return (): void => + void track + .restartTrack({ + facingMode: facingMode === "user" ? "environment" : "user", + }) + .then(() => { + // Inform the MediaDevices which camera was chosen + const deviceId = + track.mediaStreamTrack.getSettings().deviceId; + if (deviceId !== undefined) + this.mediaDevices.videoInput.select(deviceId); + }) + .catch((e) => + logger.error("Failed to switch camera", facingMode, e), + ); + }), + ); + public constructor( id: string, member: RoomMember | undefined, participant$: Observable, encryptionSystem: EncryptionSystem, livekitRoom: LivekitRoom, + private readonly mediaDevices: MediaDevices, displayname$: Observable, handRaised$: Observable, reaction$: Observable, diff --git a/src/tile/SpotlightTile.test.tsx b/src/tile/SpotlightTile.test.tsx index 1f3b7a86..3e039bca 100644 --- a/src/tile/SpotlightTile.test.tsx +++ b/src/tile/SpotlightTile.test.tsx @@ -13,6 +13,8 @@ import { of } from "rxjs"; import { SpotlightTile } from "./SpotlightTile"; import { + mockLocalParticipant, + mockMediaDevices, mockRtcMembership, withLocalMedia, withRemoteMedia, @@ -39,6 +41,8 @@ test("SpotlightTile is accessible", async () => { rawDisplayName: "Bob", getMxcAvatarUrl: () => "mxc://dlskf", }, + mockLocalParticipant({}), + mockMediaDevices({}), async (vm2) => { const user = userEvent.setup(); const toggleExpanded = vi.fn(); diff --git a/src/utils/test.ts b/src/utils/test.ts index 8f8b19a3..a2234f57 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -205,9 +205,10 @@ export function mockLocalParticipant( export async function withLocalMedia( localRtcMember: CallMembership, roomMember: Partial, + localParticipant: LocalParticipant, + mediaDevices: MediaDevices, continuation: (vm: LocalUserMediaViewModel) => void | Promise, ): Promise { - const localParticipant = mockLocalParticipant({}); const vm = new LocalUserMediaViewModel( "local", mockMatrixRoomMember(localRtcMember, roomMember), @@ -216,6 +217,7 @@ export async function withLocalMedia( kind: E2eeType.PER_PARTICIPANT, }, mockLivekitRoom({ localParticipant }), + mediaDevices, of(roomMember.rawDisplayName ?? "nodisplayname"), of(null), of(null), From f53558cb819ec2cd578f142587f6ebc324e7e966 Mon Sep 17 00:00:00 2001 From: Robin Date: Thu, 10 Jul 2025 15:26:44 -0400 Subject: [PATCH 03/24] Fix bug in drag handler which breaks buttons inside tiles --- src/grid/TileWrapper.tsx | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/grid/TileWrapper.tsx b/src/grid/TileWrapper.tsx index 9e58fd7c..1bed08da 100644 --- a/src/grid/TileWrapper.tsx +++ b/src/grid/TileWrapper.tsx @@ -61,7 +61,12 @@ const TileWrapper_ = memo( useDrag((state) => onDrag?.current!(id, state), { target: ref, filterTaps: true, - preventScroll: true, + // Previous designs, which allowed tiles to be dragged and dropped around + // the scrolling grid, required us to set preventScroll to true here. But + // our designs no longer call for this, and meanwhile there's a bug in + // use-gesture that causes filterTaps + preventScroll to break buttons + // within tiles (like the 'switch camera' button) on mobile. + // https://github.com/pmndrs/use-gesture/issues/593 }); return ( From 31bb46485fa63f837528ca9ac437bd61d2f69e31 Mon Sep 17 00:00:00 2001 From: Robin Date: Thu, 12 Jun 2025 21:33:04 -0400 Subject: [PATCH 04/24] Put a switch camera button on the local user's tile --- src/tile/GridTile.module.css | 22 +++++++++++++++++ src/tile/GridTile.tsx | 46 ++++++++++++++++++++++++----------- src/tile/MediaView.module.css | 3 ++- 3 files changed, 56 insertions(+), 15 deletions(-) diff --git a/src/tile/GridTile.module.css b/src/tile/GridTile.module.css index 867c1cef..ee605e46 100644 --- a/src/tile/GridTile.module.css +++ b/src/tile/GridTile.module.css @@ -83,3 +83,25 @@ borders don't support gradients */ .volumeSlider { width: 100%; } + +.tile .switchCamera { + opacity: 1; + background: var(--cpd-color-bg-action-secondary-rest); + border: 1px solid var(--cpd-color-border-interactive-secondary); +} + +.tile .switchCamera > svg { + color: var(--cpd-color-icon-primary); +} + +@media (hover) { + .tile .switchCamera:hover { + background: var(--cpd-color-bg-subtle-secondary); + border-color: var(--cpd-color-border-interactive-hovered); + } +} + +.tile .switchCamera:active { + background: var(--cpd-color-bg-subtle-primary); + border-color: var(--cpd-color-border-interactive-hovered); +} diff --git a/src/tile/GridTile.tsx b/src/tile/GridTile.tsx index 1e363b18..78534dae 100644 --- a/src/tile/GridTile.tsx +++ b/src/tile/GridTile.tsx @@ -28,6 +28,7 @@ import { UserProfileIcon, ExpandIcon, VolumeOffSolidIcon, + SwitchCameraSolidIcon, } from "@vector-im/compound-design-tokens/assets/web/icons"; import { ContextMenu, @@ -64,6 +65,7 @@ interface UserMediaTileProps extends TileProps { vm: UserMediaViewModel; mirror: boolean; locallyMuted: boolean; + primaryButton?: ReactNode; menuStart?: ReactNode; menuEnd?: ReactNode; } @@ -73,6 +75,7 @@ const UserMediaTile: FC = ({ vm, showSpeakingIndicators, locallyMuted, + primaryButton, menuStart, menuEnd, className, @@ -159,20 +162,22 @@ const UserMediaTile: FC = ({ } displayName={displayName} primaryButton={ - - - - } - side="left" - align="start" - > - {menu} - + primaryButton ?? ( + + + + } + side="left" + align="start" + > + {menu} + + ) } raisedHandTime={handRaised ?? undefined} currentReaction={reaction ?? undefined} @@ -207,6 +212,8 @@ const LocalUserMediaTile: FC = ({ const { t } = useTranslation(); const mirror = useObservableEagerState(vm.mirror$); const alwaysShow = useObservableEagerState(vm.alwaysShow$); + const switchCamera = useObservableEagerState(vm.switchCamera$); + const latestAlwaysShow = useLatest(alwaysShow); const onSelectAlwaysShow = useCallback( (e: Event) => { @@ -222,6 +229,17 @@ const LocalUserMediaTile: FC = ({ vm={vm} locallyMuted={false} mirror={mirror} + primaryButton={ + switchCamera === null ? undefined : ( + + ) + } menuStart={ button:active { - background: var(--cpd-color-bg-action-primary-pressed) !important; + background: var(--cpd-color-bg-action-primary-pressed); } .fg > button[data-state="open"] { From dc789e63f2d7cd693c1b728b109dfd14eab94449 Mon Sep 17 00:00:00 2001 From: Robin Date: Fri, 15 Aug 2025 18:32:37 +0200 Subject: [PATCH 05/24] Avoid using the deprecated 'room' field on MatrixRTCSession --- src/room/GroupCallView.tsx | 1 + src/room/InCallView.test.tsx | 1 + src/room/InCallView.tsx | 16 ++++++++++------ src/state/CallViewModel.test.ts | 5 +++-- src/state/CallViewModel.ts | 17 +++++++++++------ src/utils/test-viewmodel.ts | 11 +++++++---- 6 files changed, 33 insertions(+), 18 deletions(-) diff --git a/src/room/GroupCallView.tsx b/src/room/GroupCallView.tsx index 76352523..ea57bd10 100644 --- a/src/room/GroupCallView.tsx +++ b/src/room/GroupCallView.tsx @@ -452,6 +452,7 @@ export const GroupCallView: FC = ({ client={client} matrixInfo={matrixInfo} rtcSession={rtcSession as MatrixRTCSession} + matrixRoom={room} participantCount={participantCount} onLeave={onLeave} header={header} diff --git a/src/room/InCallView.test.tsx b/src/room/InCallView.test.tsx index b88aaad7..ec057e94 100644 --- a/src/room/InCallView.test.tsx +++ b/src/room/InCallView.test.tsx @@ -175,6 +175,7 @@ function createInCallView(): RenderResult & { kind: E2eeType.NONE, }, }} + matrixRoom={room} livekitRoom={livekitRoom} participantCount={0} onLeave={function (): void { diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 5aa270d2..2061289a 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -7,8 +7,8 @@ Please see LICENSE in the repository root for full details. import { RoomContext, useLocalParticipant } from "@livekit/components-react"; import { IconButton, Text, Tooltip } from "@vector-im/compound-web"; -import { ConnectionState, type Room } from "livekit-client"; -import { type MatrixClient } from "matrix-js-sdk"; +import { ConnectionState, type Room as LivekitRoom } from "livekit-client"; +import { type MatrixClient, type Room as MatrixRoom } from "matrix-js-sdk"; import { type FC, type PointerEvent, @@ -166,6 +166,7 @@ export const ActiveCall: FC = (props) => { const reactionsReader = new ReactionsReader(props.rtcSession); const vm = new CallViewModel( props.rtcSession, + props.matrixRoom, livekitRoom, mediaDevices, { @@ -184,6 +185,7 @@ export const ActiveCall: FC = (props) => { } }, [ props.rtcSession, + props.matrixRoom, livekitRoom, mediaDevices, props.e2eeSystem, @@ -212,7 +214,8 @@ export interface InCallViewProps { vm: CallViewModel; matrixInfo: MatrixInfo; rtcSession: MatrixRTCSession; - livekitRoom: Room; + matrixRoom: MatrixRoom; + livekitRoom: LivekitRoom; muteStates: MuteStates; participantCount: number; /** Function to call when the user explicitly ends the call */ @@ -228,6 +231,7 @@ export const InCallView: FC = ({ vm, matrixInfo, rtcSession, + matrixRoom, livekitRoom, muteStates, participantCount, @@ -272,7 +276,7 @@ export const InCallView: FC = ({ const [useExperimentalToDeviceTransport] = useSetting( useExperimentalToDeviceTransportSetting, ); - const encryptionSystem = useRoomEncryptionSystem(rtcSession.room.roomId); + const encryptionSystem = useRoomEncryptionSystem(matrixRoom.roomId); const memberships = useMatrixRTCSessionMemberships(rtcSession); const showToDeviceEncryption = useMemo( @@ -642,7 +646,7 @@ export const InCallView: FC = ({ }; const rageshakeRequestModalProps = useRageshakeRequestModal( - rtcSession.room.roomId, + matrixRoom.roomId, ); const toggleScreensharing = useCallback(() => { @@ -800,7 +804,7 @@ export const InCallView: FC = ({ of()); - const liveKitRoom = mockLivekitRoom( + const livekitRoom = mockLivekitRoom( { localParticipant }, { remoteParticipants$ }, ); @@ -288,7 +288,8 @@ function withCallViewModel( const vm = new CallViewModel( rtcSession as unknown as MatrixRTCSession, - liveKitRoom, + room, + livekitRoom, mediaDevices, options, connectionState$, diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 70183a37..80076cb2 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -18,7 +18,11 @@ import { type RemoteParticipant, Track, } from "livekit-client"; -import { RoomStateEvent, type Room, type RoomMember } from "matrix-js-sdk"; +import { + RoomStateEvent, + type Room as MatrixRoom, + type RoomMember, +} from "matrix-js-sdk"; import { BehaviorSubject, EMPTY, @@ -368,7 +372,7 @@ type MediaItem = UserMedia | ScreenShare; function getRoomMemberFromRtcMember( rtcMember: CallMembership, - room: Room, + room: MatrixRoom, ): { id: string; member: RoomMember | undefined } { // WARN! This is not exactly the sender but the user defined in the state key. // This will be available once we change to the new "member as object" format in the MatrixRTC object. @@ -481,7 +485,7 @@ export class CallViewModel extends ViewModel { // Handle call membership changes. fromEvent(this.matrixRTCSession, MatrixRTCSessionEvent.MembershipsChanged), // Handle room membership changes (and displayname updates) - fromEvent(this.matrixRTCSession.room, RoomStateEvent.Members), + fromEvent(this.matrixRoom, RoomStateEvent.Members), ).pipe( startWith(this.matrixRTCSession.memberships), map(() => { @@ -497,7 +501,7 @@ export class CallViewModel extends ViewModel { public readonly memberDisplaynames$ = this.memberships$.pipe( map((memberships) => { const displaynameMap = new Map(); - const { room } = this.matrixRTCSession; + const room = this.matrixRoom; // We only consider RTC members for disambiguation as they are the only visible members. for (const rtcMember of memberships) { @@ -565,7 +569,7 @@ export class CallViewModel extends ViewModel { ) => { const newItems = new Map( function* (this: CallViewModel): Iterable<[string, MediaItem]> { - const room = this.matrixRTCSession.room; + const room = this.matrixRoom; // m.rtc.members are the basis for calculating what is visible in the call for (const rtcMember of this.matrixRTCSession.memberships) { const { member, id: livekitParticipantId } = @@ -783,7 +787,7 @@ export class CallViewModel extends ViewModel { public readonly allOthersLeft$ = this.matrixUserChanges$.pipe( map(({ userIds, leftUserIds }) => { - const userId = this.matrixRTCSession.room.client.getUserId(); + const userId = this.matrixRoom.client.getUserId(); if (!userId) { logger.warn("Could access client.getUserId to compute allOthersLeft"); return false; @@ -1485,6 +1489,7 @@ export class CallViewModel extends ViewModel { public constructor( // A call is permanently tied to a single Matrix room and LiveKit room private readonly matrixRTCSession: MatrixRTCSession, + private readonly matrixRoom: MatrixRoom, private readonly livekitRoom: LivekitRoom, private readonly mediaDevices: MediaDevices, private readonly options: CallViewModelOptions, diff --git a/src/utils/test-viewmodel.ts b/src/utils/test-viewmodel.ts index 4781bf3d..7978bd96 100644 --- a/src/utils/test-viewmodel.ts +++ b/src/utils/test-viewmodel.ts @@ -15,7 +15,7 @@ import { vitest } from "vitest"; import { type RelationsContainer } from "matrix-js-sdk/lib/models/relations-container"; import EventEmitter from "events"; -import type { RoomMember, MatrixClient } from "matrix-js-sdk"; +import type { RoomMember, MatrixClient, Room } from "matrix-js-sdk"; import { E2eeType } from "../e2ee/e2eeType"; import { CallViewModel } from "../state/CallViewModel"; import { @@ -37,6 +37,7 @@ export function getBasicRTCSession( initialRemoteRtcMemberships: CallMembership[] = [aliceRtcMember], ): { rtcSession: MockRTCSession; + matrixRoom: Room; remoteRtcMemberships$: BehaviorSubject; } { const matrixRoomId = "!myRoomId:example.com"; @@ -102,6 +103,7 @@ export function getBasicRTCSession( return { rtcSession, + matrixRoom, remoteRtcMemberships$, }; } @@ -122,7 +124,7 @@ export function getBasicCallViewModelEnvironment( handRaisedSubject$: BehaviorSubject>; reactionsSubject$: BehaviorSubject>; } { - const { rtcSession, remoteRtcMemberships$ } = getBasicRTCSession( + const { rtcSession, matrixRoom, remoteRtcMemberships$ } = getBasicRTCSession( members, initialRemoteRtcMemberships, ); @@ -130,13 +132,14 @@ export function getBasicCallViewModelEnvironment( const reactionsSubject$ = new BehaviorSubject({}); const remoteParticipants$ = of([aliceParticipant]); - const liveKitRoom = mockLivekitRoom( + const livekitRoom = mockLivekitRoom( { localParticipant }, { remoteParticipants$ }, ); const vm = new CallViewModel( rtcSession as unknown as MatrixRTCSession, - liveKitRoom, + matrixRoom, + livekitRoom, mockMediaDevices({}), { encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, From f08ae36f9e2218f815acd7cd538112c18a7ed448 Mon Sep 17 00:00:00 2001 From: Robin Date: Fri, 15 Aug 2025 18:38:52 +0200 Subject: [PATCH 06/24] Pause media tracks and show a message when reconnecting to MatrixRTC --- locales/en/app.json | 1 + src/Toast.tsx | 53 +++++++----- src/room/InCallView.tsx | 14 +++- .../__snapshots__/InCallView.test.tsx.snap | 12 +-- src/state/CallViewModel.ts | 82 +++++++++++++++++-- src/utils/test.ts | 1 + 6 files changed, 127 insertions(+), 36 deletions(-) diff --git a/locales/en/app.json b/locales/en/app.json index d375b629..007e372a 100644 --- a/locales/en/app.json +++ b/locales/en/app.json @@ -55,6 +55,7 @@ "profile": "Profile", "reaction": "Reaction", "reactions": "Reactions", + "reconnecting": "Reconnecting…", "settings": "Settings", "unencrypted": "Not encrypted", "username": "Username", diff --git a/src/Toast.tsx b/src/Toast.tsx index ada5b29c..105572c8 100644 --- a/src/Toast.tsx +++ b/src/Toast.tsx @@ -45,6 +45,12 @@ interface Props { * A supporting icon to display within the toast. */ Icon?: ComponentType>; + /** + * Whether the toast should be portaled into the root of the document (rather + * than rendered in-place within the component tree). + * @default true + */ + portal?: boolean; } /** @@ -56,6 +62,7 @@ export const Toast: FC = ({ autoDismiss, children, Icon, + portal = true, }) => { const onOpenChange = useCallback( (open: boolean) => { @@ -71,29 +78,33 @@ export const Toast: FC = ({ } }, [open, autoDismiss, onDismiss]); + const content = ( + <> + + + + + + {children} + + + {Icon && } + + + + ); + return ( - - - - - - - {children} - - - {Icon && } - - - + {portal ? {content} : content} ); }; diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 2061289a..f9bd681c 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -113,6 +113,7 @@ import { useMediaDevices } from "../MediaDevicesContext.ts"; import { EarpieceOverlay } from "./EarpieceOverlay.tsx"; import { useAppBarHidden, useAppBarSecondaryButton } from "../AppBar.tsx"; import { useBehavior } from "../useBehavior.ts"; +import { Toast } from "../Toast.tsx"; const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {}); @@ -313,6 +314,7 @@ export const InCallView: FC = ({ () => void toggleRaisedHand(), ); + const reconnecting = useBehavior(vm.reconnecting$); const windowMode = useBehavior(vm.windowMode$); const layout = useBehavior(vm.layout$); const tileStoreGeneration = useBehavior(vm.tileStoreGeneration$); @@ -766,6 +768,9 @@ export const InCallView: FC = ({ ); + // The reconnecting toast cannot be dismissed + const onDismissReconnectingToast = useCallback(() => {}, []); + return (
= ({ {renderContent()} + + {t("common.reconnecting")} + diff --git a/src/room/__snapshots__/InCallView.test.tsx.snap b/src/room/__snapshots__/InCallView.test.tsx.snap index b45a30ad..7d6ab966 100644 --- a/src/room/__snapshots__/InCallView.test.tsx.snap +++ b/src/room/__snapshots__/InCallView.test.tsx.snap @@ -256,7 +256,7 @@ exports[`InCallView > rendering > renders 1`] = ` >