diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index ae7b78c1..9dd2c3f0 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -255,6 +255,7 @@ class UserMedia { participant: LocalParticipant | RemoteParticipant | undefined, encryptionSystem: EncryptionSystem, livekitRoom: LivekitRoom, + mediaDevices: MediaDevices, displayname$: Observable, handRaised$: Observable, reaction$: Observable, @@ -268,6 +269,7 @@ class UserMedia { this.participant$.asObservable() as Observable, encryptionSystem, livekitRoom, + mediaDevices, displayname$, handRaised$, reaction$, @@ -565,6 +567,7 @@ export class CallViewModel extends ViewModel { participant, this.encryptionSystem, this.livekitRoom, + this.mediaDevices, this.memberDisplaynames$.pipe( map((m) => m.get(matrixIdentifier) ?? "[👻]"), ), @@ -629,6 +632,7 @@ export class CallViewModel extends ViewModel { participant, this.encryptionSystem, this.livekitRoom, + this.mediaDevices, this.memberDisplaynames$.pipe( map((m) => m.get(participant.identity) ?? "[👻]"), ), diff --git a/src/state/MediaViewModel.test.ts b/src/state/MediaViewModel.test.ts index 601133ac..8de550d4 100644 --- a/src/state/MediaViewModel.test.ts +++ b/src/state/MediaViewModel.test.ts @@ -5,14 +5,40 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { expect, test, vi } from "vitest"; +import { expect, onTestFinished, test, vi } from "vitest"; +import { of } from "rxjs"; +import { + type LocalTrackPublication, + LocalVideoTrack, + TrackEvent, +} from "livekit-client"; +import { waitFor } from "@testing-library/dom"; import { + mockLocalParticipant, + mockMediaDevices, mockRtcMembership, withLocalMedia, withRemoteMedia, withTestScheduler, } from "../utils/test"; +import { getValue } from "../utils/observable"; + +global.MediaStreamTrack = class {} as unknown as { + new (): MediaStreamTrack; + prototype: MediaStreamTrack; +}; +global.MediaStream = class {} as unknown as { + new (): MediaStream; + prototype: MediaStream; +}; + +const platformMock = vi.hoisted(() => vi.fn(() => "desktop")); +vi.mock("../Platform", () => ({ + get platform(): string { + return platformMock(); + }, +})); const rtcMembership = mockRtcMembership("@alice:example.org", "AAAA"); @@ -79,17 +105,23 @@ test("toggle fit/contain for a participant's video", async () => { }); test("local media remembers whether it should always be shown", async () => { - await withLocalMedia(rtcMembership, {}, (vm) => - withTestScheduler(({ expectObservable, schedule }) => { - schedule("-a|", { a: () => vm.setAlwaysShow(false) }); - expectObservable(vm.alwaysShow$).toBe("ab", { a: true, b: false }); - }), + await withLocalMedia( + rtcMembership, + {}, + mockLocalParticipant({}), + mockMediaDevices({}), + (vm) => + withTestScheduler(({ expectObservable, schedule }) => { + schedule("-a|", { a: () => vm.setAlwaysShow(false) }); + expectObservable(vm.alwaysShow$).toBe("ab", { a: true, b: false }); + }), ); // Next local media should start out *not* always shown await withLocalMedia( rtcMembership, - {}, + mockLocalParticipant({}), + mockMediaDevices({}), (vm) => withTestScheduler(({ expectObservable, schedule }) => { schedule("-a|", { a: () => vm.setAlwaysShow(true) }); @@ -97,3 +129,76 @@ test("local media remembers whether it should always be shown", async () => { }), ); }); + +test("switch cameras", async () => { + // Camera switching is only available on mobile + platformMock.mockReturnValue("android"); + onTestFinished(() => void platformMock.mockReset()); + + // Construct a mock video track which knows how to be restarted + const track = new LocalVideoTrack({ + getConstraints() {}, + addEventListener() {}, + removeEventListener() {}, + } as unknown as MediaStreamTrack); + + let deviceId = "front camera"; + const restartTrack = vi.fn(async ({ facingMode }) => { + deviceId = facingMode === "user" ? "front camera" : "back camera"; + track.emit(TrackEvent.Restarted); + return Promise.resolve(); + }); + track.restartTrack = restartTrack; + + Object.defineProperty(track, "mediaStreamTrack", { + get() { + return { + label: "Video", + getSettings: (): object => ({ + deviceId, + facingMode: deviceId === "front camera" ? "user" : "environment", + }), + }; + }, + }); + + const selectVideoInput = vi.fn(); + + await withLocalMedia( + rtcMembership, + {}, + mockLocalParticipant({ + getTrackPublication() { + return { track } as unknown as LocalTrackPublication; + }, + }), + mockMediaDevices({ + videoInput: { + available$: of(new Map()), + selected$: of(undefined), + select: selectVideoInput, + }, + }), + async (vm) => { + // Switch to back camera + getValue(vm.switchCamera$)!(); + expect(restartTrack).toHaveBeenCalledTimes(1); + expect(restartTrack).toHaveBeenCalledWith({ facingMode: "environment" }); + await waitFor(() => { + expect(selectVideoInput).toHaveBeenCalledTimes(1); + expect(selectVideoInput).toHaveBeenCalledWith("back camera"); + }); + expect(deviceId).toBe("back camera"); + + // Switch to front camera + getValue(vm.switchCamera$)!(); + expect(restartTrack).toHaveBeenCalledTimes(2); + expect(restartTrack).toHaveBeenLastCalledWith({ facingMode: "user" }); + await waitFor(() => { + expect(selectVideoInput).toHaveBeenCalledTimes(2); + expect(selectVideoInput).toHaveBeenLastCalledWith("front camera"); + }); + expect(deviceId).toBe("front camera"); + }, + ); +}); diff --git a/src/state/MediaViewModel.ts b/src/state/MediaViewModel.ts index 424d003e..920b6ef3 100644 --- a/src/state/MediaViewModel.ts +++ b/src/state/MediaViewModel.ts @@ -16,6 +16,7 @@ import { import { type LocalParticipant, LocalTrack, + LocalVideoTrack, type Participant, ParticipantEvent, type RemoteParticipant, @@ -27,6 +28,7 @@ import { RemoteTrack, } from "livekit-client"; import { type RoomMember } from "matrix-js-sdk"; +import { logger } from "matrix-js-sdk/lib/logger"; import { BehaviorSubject, type Observable, @@ -51,6 +53,8 @@ import { accumulate } from "../utils/observable"; import { type EncryptionSystem } from "../e2ee/sharedKeyManagement"; import { E2eeType } from "../e2ee/e2eeType"; import { type ReactionOption } from "../reactions"; +import { platform } from "../Platform"; +import { type MediaDevices } from "./MediaDevices"; export function observeTrackReference$( participant$: Observable, @@ -433,20 +437,35 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel { * The local participant's user media. */ export class LocalUserMediaViewModel extends BaseUserMediaViewModel { + /** + * The local video track as an observable that emits whenever the track + * changes, the camera is switched, or the track is muted. + */ + private readonly videoTrack$: Observable = + this.video$.pipe( + switchMap((v) => { + const track = v?.publication?.track; + if (!(track instanceof LocalVideoTrack)) return of(null); + return merge( + // Watch for track restarts because they indicate a camera switch + fromEvent(track, TrackEvent.Restarted).pipe( + startWith(null), + map(() => track), + ), + fromEvent(track, TrackEvent.Muted).pipe(map(() => null)), + ); + }), + ); + /** * Whether the video should be mirrored. */ - public readonly mirror$ = this.video$.pipe( - switchMap((v) => { - const track = v?.publication?.track; - if (!(track instanceof LocalTrack)) return of(false); - // Watch for track restarts, because they indicate a camera switch - return fromEvent(track, TrackEvent.Restarted).pipe( - startWith(null), - // Mirror only front-facing cameras (those that face the user) - map(() => facingModeFromLocalTrack(track).facingMode === "user"), - ); - }), + public readonly mirror$ = this.videoTrack$.pipe( + // Mirror only front-facing cameras (those that face the user) + map( + (track) => + track !== null && facingModeFromLocalTrack(track).facingMode === "user", + ), this.scope.state(), ); @@ -457,12 +476,46 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel { public readonly alwaysShow$ = alwaysShowSelf.value$; public readonly setAlwaysShow = alwaysShowSelf.setValue; + /** + * Callback for switching between the front and back cameras. + */ + public readonly switchCamera$: Observable<(() => void) | null> = + platform === "desktop" + ? of(null) + : this.videoTrack$.pipe( + map((track) => { + if (track === null) return null; + const facingMode = facingModeFromLocalTrack(track).facingMode; + // If the camera isn't front or back-facing, don't provide a switch + // camera shortcut at all + if (facingMode !== "user" && facingMode !== "environment") + return null; + // Restart the track with a camera facing the opposite direction + return (): void => + void track + .restartTrack({ + facingMode: facingMode === "user" ? "environment" : "user", + }) + .then(() => { + // Inform the MediaDevices which camera was chosen + const deviceId = + track.mediaStreamTrack.getSettings().deviceId; + if (deviceId !== undefined) + this.mediaDevices.videoInput.select(deviceId); + }) + .catch((e) => + logger.error("Failed to switch camera", facingMode, e), + ); + }), + ); + public constructor( id: string, member: RoomMember | undefined, participant$: Observable, encryptionSystem: EncryptionSystem, livekitRoom: LivekitRoom, + private readonly mediaDevices: MediaDevices, displayname$: Observable, handRaised$: Observable, reaction$: Observable, diff --git a/src/tile/SpotlightTile.test.tsx b/src/tile/SpotlightTile.test.tsx index 1f3b7a86..3e039bca 100644 --- a/src/tile/SpotlightTile.test.tsx +++ b/src/tile/SpotlightTile.test.tsx @@ -13,6 +13,8 @@ import { of } from "rxjs"; import { SpotlightTile } from "./SpotlightTile"; import { + mockLocalParticipant, + mockMediaDevices, mockRtcMembership, withLocalMedia, withRemoteMedia, @@ -39,6 +41,8 @@ test("SpotlightTile is accessible", async () => { rawDisplayName: "Bob", getMxcAvatarUrl: () => "mxc://dlskf", }, + mockLocalParticipant({}), + mockMediaDevices({}), async (vm2) => { const user = userEvent.setup(); const toggleExpanded = vi.fn(); diff --git a/src/utils/test.ts b/src/utils/test.ts index 8f8b19a3..a2234f57 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -205,9 +205,10 @@ export function mockLocalParticipant( export async function withLocalMedia( localRtcMember: CallMembership, roomMember: Partial, + localParticipant: LocalParticipant, + mediaDevices: MediaDevices, continuation: (vm: LocalUserMediaViewModel) => void | Promise, ): Promise { - const localParticipant = mockLocalParticipant({}); const vm = new LocalUserMediaViewModel( "local", mockMatrixRoomMember(localRtcMember, roomMember), @@ -216,6 +217,7 @@ export async function withLocalMedia( kind: E2eeType.PER_PARTICIPANT, }, mockLivekitRoom({ localParticipant }), + mediaDevices, of(roomMember.rawDisplayName ?? "nodisplayname"), of(null), of(null),