diff --git a/dev-backend-docker-compose.yml b/dev-backend-docker-compose.yml index 50498c7a..c7591847 100644 --- a/dev-backend-docker-compose.yml +++ b/dev-backend-docker-compose.yml @@ -47,7 +47,7 @@ services: - ecbackend livekit: - image: livekit/livekit-server:latest + image: livekit/livekit-server:v1.9.4 pull_policy: always hostname: livekit-sfu command: --dev --config /etc/livekit.yaml @@ -67,7 +67,7 @@ services: - ecbackend livekit-1: - image: livekit/livekit-server:latest + image: livekit/livekit-server:v1.9.4 pull_policy: always hostname: livekit-sfu-1 command: --dev --config /etc/livekit.yaml diff --git a/playwright.config.ts b/playwright.config.ts index 7a8ee530..391e746f 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -38,6 +38,7 @@ export default defineConfig({ projects: [ { name: "chromium", + testIgnore: "**/mobile/**", use: { ...devices["Desktop Chrome"], permissions: [ @@ -56,9 +57,9 @@ export default defineConfig({ }, }, }, - { name: "firefox", + testIgnore: "**/mobile/**", use: { ...devices["Desktop Firefox"], ignoreHTTPSErrors: true, @@ -70,6 +71,27 @@ export default defineConfig({ }, }, }, + { + name: "mobile", + testMatch: "**/mobile/**", + use: { + ...devices["Pixel 7"], + ignoreHTTPSErrors: true, + permissions: [ + "clipboard-write", + "clipboard-read", + "microphone", + "camera", + ], + launchOptions: { + args: [ + "--use-fake-ui-for-media-stream", + "--use-fake-device-for-media-stream", + "--mute-audio", + ], + }, + }, + }, // No safari for now, until I find a solution to fix `Not allowed to request resource` due to calling // clear http to the homeserver diff --git a/playwright/fixtures/fixture-mobile-create.ts b/playwright/fixtures/fixture-mobile-create.ts new file mode 100644 index 00000000..3920c978 --- /dev/null +++ b/playwright/fixtures/fixture-mobile-create.ts @@ -0,0 +1,73 @@ +/* +Copyright 2025 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { type Browser, type Page, test, expect } from "@playwright/test"; + +export interface MobileCreateFixtures { + asMobile: { + creatorPage: Page; + inviteLink: string; + }; +} + +export const mobileTest = test.extend({ + asMobile: async ({ browser }, pUse) => { + const fixtures = await createCallAndInvite(browser); + await pUse({ + creatorPage: fixtures.page, + inviteLink: fixtures.inviteLink, + }); + }, +}); + +/** + * Create a call and generate an invite link + */ +async function createCallAndInvite( + browser: Browser, +): Promise<{ page: Page; inviteLink: string }> { + const creatorContext = await browser.newContext({ reducedMotion: "reduce" }); + const creatorPage = await creatorContext.newPage(); + + await creatorPage.goto("/"); + + // ======== + // ARRANGE: The first user creates a call as guest, join it, then click the invite button to copy the invite link + // ======== + await creatorPage.getByTestId("home_callName").click(); + await creatorPage.getByTestId("home_callName").fill("Welcome"); + await creatorPage.getByTestId("home_displayName").click(); + await creatorPage.getByTestId("home_displayName").fill("Inviter"); + await creatorPage.getByTestId("home_go").click(); + await expect(creatorPage.locator("video")).toBeVisible(); + + await creatorPage + .getByRole("button", { name: "Continue in browser" }) + .click(); + // join + await creatorPage.getByTestId("lobby_joinCall").click(); + + // Get the invite link + await creatorPage.getByRole("button", { name: "Invite" }).click(); + await expect( + creatorPage.getByRole("heading", { name: "Invite to this call" }), + ).toBeVisible(); + await expect(creatorPage.getByRole("img", { name: "QR Code" })).toBeVisible(); + await expect(creatorPage.getByTestId("modal_inviteLink")).toBeVisible(); + await expect(creatorPage.getByTestId("modal_inviteLink")).toBeVisible(); + await creatorPage.getByTestId("modal_inviteLink").click(); + + const inviteLink = (await creatorPage.evaluate( + "navigator.clipboard.readText()", + )) as string; + expect(inviteLink).toContain("room/#/"); + + return { + page: creatorPage, + inviteLink, + }; +} diff --git a/playwright/mobile/create-call-mobile.spec.ts b/playwright/mobile/create-call-mobile.spec.ts new file mode 100644 index 00000000..141ffaae --- /dev/null +++ b/playwright/mobile/create-call-mobile.spec.ts @@ -0,0 +1,115 @@ +/* +Copyright 2025 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { expect, test } from "@playwright/test"; + +import { mobileTest } from "../fixtures/fixture-mobile-create.ts"; + +test("@mobile Start a new call then leave and show the feedback screen", async ({ + page, +}) => { + await page.goto("/"); + + await page.getByTestId("home_callName").click(); + await page.getByTestId("home_callName").fill("HelloCall"); + await page.getByTestId("home_displayName").click(); + await page.getByTestId("home_displayName").fill("John Doe"); + await page.getByTestId("home_go").click(); + + // await page.pause(); + await expect(page.locator("video")).toBeVisible(); + await expect(page.getByTestId("lobby_joinCall")).toBeVisible(); + + await page.getByRole("button", { name: "Continue in browser" }).click(); + // Join the call + await page.getByTestId("lobby_joinCall").click(); + + // Ensure that the call is connected + await page + .locator("div") + .filter({ hasText: /^HelloCall$/ }) + .click(); + // Check the number of participants + await expect(page.locator("div").filter({ hasText: /^1$/ })).toBeVisible(); + // The tooltip with the name should be visible + await expect(page.getByTestId("name_tag")).toContainText("John Doe"); + + // leave the call + await page.getByTestId("incall_leave").click(); + await expect(page.getByRole("heading")).toContainText( + "John Doe, your call has ended. How did it go?", + ); + await expect(page.getByRole("main")).toContainText( + "Why not finish by setting up a password to keep your account?", + ); + + await expect( + page.getByRole("link", { name: "Not now, return to home screen" }), + ).toBeVisible(); +}); + +mobileTest( + "Test earpiece overlay in controlledAudioDevices mode", + async ({ asMobile, browser }) => { + test.slow(); // Triples the timeout + const { creatorPage, inviteLink } = asMobile; + + // ======== + // ACT: The other user use the invite link to join the call as a guest + // ======== + const guestInviteeContext = await browser.newContext({ + reducedMotion: "reduce", + }); + const guestPage = await guestInviteeContext.newPage(); + await guestPage.goto(inviteLink + "&controlledAudioDevices=true"); + + await guestPage + .getByRole("button", { name: "Continue in browser" }) + .click(); + + await guestPage.getByTestId("joincall_displayName").fill("Invitee"); + await expect(guestPage.getByTestId("joincall_joincall")).toBeVisible(); + await guestPage.getByTestId("joincall_joincall").click(); + await guestPage.getByTestId("lobby_joinCall").click(); + + // ======== + // ASSERT: check that there are two members in the call + // ======== + + // There should be two participants now + await expect( + guestPage.getByTestId("roomHeader_participants_count"), + ).toContainText("2"); + expect(await guestPage.getByTestId("videoTile").count()).toBe(2); + + // Same in creator page + await expect( + creatorPage.getByTestId("roomHeader_participants_count"), + ).toContainText("2"); + expect(await creatorPage.getByTestId("videoTile").count()).toBe(2); + + // TEST: control audio devices from the invitee page + + await guestPage.evaluate(() => { + window.controls.setAvailableAudioDevices([ + { id: "speaker", name: "Speaker", isSpeaker: true }, + { id: "earpiece", name: "Handset", isEarpiece: true }, + { id: "headphones", name: "Headphones" }, + ]); + window.controls.setAudioDevice("earpiece"); + }); + await expect( + guestPage.getByRole("heading", { name: "Handset Mode" }), + ).toBeVisible(); + await expect( + guestPage.getByRole("button", { name: "Back to Speaker Mode" }), + ).toBeVisible(); + + // Should auto-mute the video when earpiece is selected + await expect(guestPage.getByTestId("incall_videomute")).toBeDisabled(); + }, +); diff --git a/src/UrlParams.test.ts b/src/UrlParams.test.ts index cd8fc6d5..faba394f 100644 --- a/src/UrlParams.test.ts +++ b/src/UrlParams.test.ts @@ -332,6 +332,42 @@ describe("UrlParams", () => { expect(computeUrlParams("?intent=join_existing").skipLobby).toBe(false); }); }); + + describe("noiseSuppression", () => { + it("defaults to true", () => { + expect(computeUrlParams().noiseSuppression).toBe(true); + }); + + it("is parsed", () => { + expect( + computeUrlParams("?intent=start_call&noiseSuppression=true") + .noiseSuppression, + ).toBe(true); + expect( + computeUrlParams("?intent=start_call&noiseSuppression&bar=foo") + .noiseSuppression, + ).toBe(true); + expect(computeUrlParams("?noiseSuppression=false").noiseSuppression).toBe( + false, + ); + }); + }); + + describe("echoCancellation", () => { + it("defaults to true", () => { + expect(computeUrlParams().echoCancellation).toBe(true); + }); + + it("is parsed", () => { + expect(computeUrlParams("?echoCancellation=true").echoCancellation).toBe( + true, + ); + expect(computeUrlParams("?echoCancellation=false").echoCancellation).toBe( + false, + ); + }); + }); + describe("header", () => { it("uses header if provided", () => { expect(computeUrlParams("?header=app_bar&hideHeader=true").header).toBe( diff --git a/src/UrlParams.ts b/src/UrlParams.ts index 4eb69298..f78841fb 100644 --- a/src/UrlParams.ts +++ b/src/UrlParams.ts @@ -233,6 +233,17 @@ export interface UrlConfiguration { */ waitForCallPickup: boolean; + /** + * Whether to enable echo cancellation for audio capture. + * Defaults to true. + */ + echoCancellation?: boolean; + /** + * Whether to enable noise suppression for audio capture. + * Defaults to true. + */ + noiseSuppression?: boolean; + callIntent?: RTCCallIntent; } interface IntentAndPlatformDerivedConfiguration { @@ -525,6 +536,8 @@ export const computeUrlParams = (search = "", hash = ""): UrlParams => { ]), waitForCallPickup: parser.getFlag("waitForCallPickup"), autoLeaveWhenOthersLeft: parser.getFlag("autoLeave"), + noiseSuppression: parser.getFlagParam("noiseSuppression", true), + echoCancellation: parser.getFlagParam("echoCancellation", true), }; // Log the final configuration for debugging purposes. diff --git a/src/analytics/PosthogAnalytics.ts b/src/analytics/PosthogAnalytics.ts index 8b2aa91d..46223afe 100644 --- a/src/analytics/PosthogAnalytics.ts +++ b/src/analytics/PosthogAnalytics.ts @@ -247,9 +247,8 @@ export class PosthogAnalytics { // wins, and the first writer will send tracking with an ID that doesn't match the one on the server // until the next time account data is refreshed and this function is called (most likely on next // page load). This will happen pretty infrequently, so we can tolerate the possibility. - const accountDataAnalyticsId = analyticsIdGenerator(); - await this.setAccountAnalyticsId(accountDataAnalyticsId); - analyticsID = await this.hashedEcAnalyticsId(accountDataAnalyticsId); + analyticsID = analyticsIdGenerator(); + await this.setAccountAnalyticsId(analyticsID); } } catch (e) { // The above could fail due to network requests, but not essential to starting the application, @@ -270,37 +269,14 @@ export class PosthogAnalytics { private async getAnalyticsId(): Promise { const client: MatrixClient = window.matrixclient; - let accountAnalyticsId: string | null; if (widget) { - accountAnalyticsId = getUrlParams().posthogUserId; + return getUrlParams().posthogUserId; } else { const accountData = await client.getAccountDataFromServer( PosthogAnalytics.ANALYTICS_EVENT_TYPE, ); - accountAnalyticsId = accountData?.id ?? null; + return accountData?.id ?? null; } - if (accountAnalyticsId) { - // we dont just use the element web analytics ID because that would allow to associate - // users between the two posthog instances. By using a hash from the username and the element web analytics id - // it is not possible to conclude the element web posthog user id from the element call user id and vice versa. - return await this.hashedEcAnalyticsId(accountAnalyticsId); - } - return null; - } - - private async hashedEcAnalyticsId( - accountAnalyticsId: string, - ): Promise { - const client: MatrixClient = window.matrixclient; - const posthogIdMaterial = "ec" + accountAnalyticsId + client.getUserId(); - const bufferForPosthogId = await crypto.subtle.digest( - "sha-256", - new TextEncoder().encode(posthogIdMaterial), - ); - const view = new Int32Array(bufferForPosthogId); - return Array.from(view) - .map((b) => Math.abs(b).toString(16).padStart(2, "0")) - .join(""); } private async setAccountAnalyticsId(analyticsID: string): Promise { diff --git a/src/room/EarpieceOverlay.module.css b/src/room/EarpieceOverlay.module.css index d0757cdb..e53a1974 100644 --- a/src/room/EarpieceOverlay.module.css +++ b/src/room/EarpieceOverlay.module.css @@ -7,34 +7,17 @@ align-items: center; justify-content: center; gap: var(--cpd-space-2x); -} - -@keyframes fade-in { - from { - opacity: 0; - } - to { - opacity: 1; - } + transition: opacity 200ms; } .overlay[data-show="true"] { - animation: fade-in 200ms; -} - -@keyframes fade-out { - from { - opacity: 1; - } - to { - opacity: 0; - display: none; - } + opacity: 1; } .overlay[data-show="false"] { - animation: fade-out 130ms forwards; + opacity: 0; pointer-events: none; + transition-duration: 130ms; } .overlay::before { diff --git a/src/state/CallViewModel/CallViewModel.test.ts b/src/state/CallViewModel/CallViewModel.test.ts index ef59270f..2e5b5700 100644 --- a/src/state/CallViewModel/CallViewModel.test.ts +++ b/src/state/CallViewModel/CallViewModel.test.ts @@ -502,6 +502,48 @@ describe("CallViewModel", () => { }); }); + test("layout reacts to window size", () => { + withTestScheduler(({ behavior, schedule, expectObservable }) => { + const windowSizeInputMarbles = "abc"; + const expectedLayoutMarbles = " abc"; + withCallViewModel( + { + remoteParticipants$: constant([aliceParticipant]), + rtcMembers$: constant([localRtcMember, aliceRtcMember]), + windowSize$: behavior(windowSizeInputMarbles, { + a: { width: 300, height: 600 }, // Start very narrow, like a phone + b: { width: 1000, height: 800 }, // Go to normal desktop window size + c: { width: 200, height: 180 }, // Go to PiP size + }), + }, + (vm) => { + expectObservable(summarizeLayout$(vm.layout$)).toBe( + expectedLayoutMarbles, + { + a: { + // This is the expected one-on-one layout for a narrow window + type: "spotlight-expanded", + spotlight: [`${aliceId}:0`], + pip: `${localId}:0`, + }, + b: { + // In a larger window, expect the normal one-on-one layout + type: "one-on-one", + local: `${localId}:0`, + remote: `${aliceId}:0`, + }, + c: { + // In a PiP-sized window, we of course expect a PiP layout + type: "pip", + spotlight: [`${aliceId}:0`], + }, + }, + ); + }, + ); + }); + }); + test("spotlight speakers swap places", () => { withTestScheduler(({ behavior, schedule, expectObservable }) => { // Go immediately into spotlight mode for the test diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index 4fb1c35a..d65ab138 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -16,7 +16,6 @@ import { } from "livekit-client"; import { type Room as MatrixRoom } from "matrix-js-sdk"; import { - BehaviorSubject, combineLatest, distinctUntilChanged, filter, @@ -128,6 +127,7 @@ import { } from "./remoteMembers/MatrixMemberMetadata.ts"; import { Publisher } from "./localMember/Publisher.ts"; import { type Connection } from "./remoteMembers/Connection.ts"; +import { createLayoutModeSwitch } from "./LayoutSwitch.ts"; const logger = rootLogger.getChild("[CallViewModel]"); //TODO @@ -149,6 +149,8 @@ export interface CallViewModelOptions { livekitRoomFactory?: (options?: RoomOptions) => LivekitRoom; /** Optional behavior overriding the local connection state, mainly for testing purposes. */ connectionState$?: Behavior; + /** Optional behavior overriding the computed window size, mainly for testing purposes. */ + windowSize$?: Behavior<{ width: number; height: number }>; } // Do not play any sounds if the participant count has exceeded this @@ -368,6 +370,7 @@ export interface CallViewModel { */ connectionState: LocalMemberConnectionState; } + /** * A view model providing all the application logic needed to show the in-call * UI (may eventually be expanded to cover the lobby and feedback screens in the @@ -438,6 +441,8 @@ export function createCallViewModel$( livekitKeyProvider, getUrlParams().controlledAudioDevices, options.livekitRoomFactory, + getUrlParams().echoCancellation, + getUrlParams().noiseSuppression, ); const connectionManager = createConnectionManager$({ @@ -972,11 +977,19 @@ export function createCallViewModel$( const pipEnabled$ = scope.behavior(setPipEnabled$, false); + const windowSize$ = + options.windowSize$ ?? + scope.behavior<{ width: number; height: number }>( + fromEvent(window, "resize").pipe( + startWith(null), + map(() => ({ width: window.innerWidth, height: window.innerHeight })), + ), + ); + + // A guess at what the window's mode should be based on its size and shape. const naturalWindowMode$ = scope.behavior( - fromEvent(window, "resize").pipe( - map(() => { - const height = window.innerHeight; - const width = window.innerWidth; + windowSize$.pipe( + map(({ width, height }) => { if (height <= 400 && width <= 340) return "pip"; // Our layouts for flat windows are better at adapting to a small width // than our layouts for narrow windows are at adapting to a small height, @@ -986,7 +999,6 @@ export function createCallViewModel$( return "normal"; }), ), - "normal", ); /** @@ -1003,49 +1015,11 @@ export function createCallViewModel$( spotlightExpandedToggle$.pipe(accumulate(false, (expanded) => !expanded)), ); - const gridModeUserSelection$ = new BehaviorSubject("grid"); - - // Callback to set the grid mode desired by the user. - // Notice that this is only a preference, the actual grid mode can be overridden - // if there is a remote screen share active. - const setGridMode = (value: GridMode): void => { - gridModeUserSelection$.next(value); - }; - /** - * The layout mode of the media tile grid. - */ - const gridMode$ = - // If the user hasn't selected spotlight and somebody starts screen sharing, - // automatically switch to spotlight mode and reset when screen sharing ends - scope.behavior( - gridModeUserSelection$.pipe( - switchMap((userSelection): Observable => { - if (userSelection === "spotlight") { - // If already in spotlight mode, stay there - return of("spotlight"); - } else { - // Otherwise, check if there is a remote screen share active - // as this could force us into spotlight mode. - return combineLatest([hasRemoteScreenShares$, windowMode$]).pipe( - map(([hasScreenShares, windowMode]): GridMode => { - const isFlatMode = windowMode === "flat"; - if (hasScreenShares || isFlatMode) { - logger.debug( - `Forcing spotlight mode, hasScreenShares=${hasScreenShares} windowMode=${windowMode}`, - ); - // override to spotlight mode - return "spotlight"; - } else { - // respect user choice - return "grid"; - } - }), - ); - } - }), - ), - "grid", - ); + const { setGridMode, gridMode$ } = createLayoutModeSwitch( + scope, + windowMode$, + hasRemoteScreenShares$, + ); const gridLayoutMedia$: Observable = combineLatest( [grid$, spotlight$], diff --git a/src/state/CallViewModel/CallViewModelTestUtils.ts b/src/state/CallViewModel/CallViewModelTestUtils.ts index f86921c5..f80b4bcb 100644 --- a/src/state/CallViewModel/CallViewModelTestUtils.ts +++ b/src/state/CallViewModel/CallViewModelTestUtils.ts @@ -75,6 +75,7 @@ export interface CallViewModelInputs { speaking: Map>; mediaDevices: MediaDevices; initialSyncState: SyncState; + windowSize$: Behavior<{ width: number; height: number }>; } const localParticipant = mockLocalParticipant({ identity: "" }); @@ -89,6 +90,7 @@ export function withCallViewModel( speaking = new Map(), mediaDevices = mockMediaDevices({}), initialSyncState = SyncState.Syncing, + windowSize$ = constant({ width: 1000, height: 800 }), }: Partial = {}, continuation: ( vm: CallViewModel, @@ -173,6 +175,7 @@ export function withCallViewModel( setE2EEEnabled: async () => Promise.resolve(), }), connectionState$, + windowSize$, }, raisedHands$, reactions$, diff --git a/src/state/CallViewModel/LayoutSwitch.test.ts b/src/state/CallViewModel/LayoutSwitch.test.ts new file mode 100644 index 00000000..ae5a3896 --- /dev/null +++ b/src/state/CallViewModel/LayoutSwitch.test.ts @@ -0,0 +1,202 @@ +/* +Copyright 2025 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { afterEach, beforeEach, describe, expect, test } from "vitest"; +import { firstValueFrom, of } from "rxjs"; + +import { createLayoutModeSwitch } from "./LayoutSwitch"; +import { ObservableScope } from "../ObservableScope"; +import { constant } from "../Behavior"; +import { withTestScheduler } from "../../utils/test"; + +let scope: ObservableScope; +beforeEach(() => { + scope = new ObservableScope(); +}); +afterEach(() => { + scope.end(); +}); + +describe("Default mode", () => { + test("Should be in grid layout by default", async () => { + const { gridMode$ } = createLayoutModeSwitch( + scope, + constant("normal"), + of(false), + ); + + const mode = await firstValueFrom(gridMode$); + expect(mode).toBe("grid"); + }); + + test("Should switch to spotlight mode when window mode is flat", async () => { + const { gridMode$ } = createLayoutModeSwitch( + scope, + constant("flat"), + of(false), + ); + + const mode = await firstValueFrom(gridMode$); + expect(mode).toBe("spotlight"); + }); +}); + +test("Should allow switching modes manually", () => { + withTestScheduler(({ cold, behavior, expectObservable, schedule }): void => { + const { gridMode$, setGridMode } = createLayoutModeSwitch( + scope, + behavior("n", { n: "normal" }), + cold("f", { f: false, t: true }), + ); + + schedule("--sgs", { + s: () => setGridMode("spotlight"), + g: () => setGridMode("grid"), + }); + + expectObservable(gridMode$).toBe("g-sgs", { + g: "grid", + s: "spotlight", + }); + }); +}); + +test("Should switch to spotlight mode when there is a remote screen share", () => { + withTestScheduler(({ cold, behavior, expectObservable }): void => { + const shareMarble = "f--t"; + const gridsMarble = "g--s"; + const { gridMode$ } = createLayoutModeSwitch( + scope, + behavior("n", { n: "normal" }), + cold(shareMarble, { f: false, t: true }), + ); + + expectObservable(gridMode$).toBe(gridsMarble, { + g: "grid", + s: "spotlight", + }); + }); +}); + +test("Can manually force grid when there is a screenshare", () => { + withTestScheduler(({ cold, behavior, expectObservable, schedule }): void => { + const { gridMode$, setGridMode } = createLayoutModeSwitch( + scope, + behavior("n", { n: "normal" }), + cold("-ft", { f: false, t: true }), + ); + + schedule("---g", { + g: () => setGridMode("grid"), + }); + + expectObservable(gridMode$).toBe("ggsg", { + g: "grid", + s: "spotlight", + }); + }); +}); + +test("Should auto-switch after manually selected grid", () => { + withTestScheduler(({ cold, behavior, expectObservable, schedule }): void => { + const { gridMode$, setGridMode } = createLayoutModeSwitch( + scope, + behavior("n", { n: "normal" }), + // Two screenshares will happen in sequence + cold("-ft-ft", { f: false, t: true }), + ); + + // There was a screen-share that forced spotlight, then + // the user manually switch back to grid + schedule("---g", { + g: () => setGridMode("grid"), + }); + + // If we did want to respect manual selection, the expectation would be: + // const expectation = "ggsg"; + const expectation = "ggsg-s"; + + expectObservable(gridMode$).toBe(expectation, { + g: "grid", + s: "spotlight", + }); + }); +}); + +test("Should switch back to grid mode when the remote screen share ends", () => { + withTestScheduler(({ cold, behavior, expectObservable }): void => { + const shareMarble = "f--t--f-"; + const gridsMarble = "g--s--g-"; + const { gridMode$ } = createLayoutModeSwitch( + scope, + behavior("n", { n: "normal" }), + cold(shareMarble, { f: false, t: true }), + ); + + expectObservable(gridMode$).toBe(gridsMarble, { + g: "grid", + s: "spotlight", + }); + }); +}); + +test("can auto-switch to spotlight again after first screen share ends", () => { + withTestScheduler(({ cold, behavior, expectObservable }): void => { + const shareMarble = "ftft"; + const gridsMarble = "gsgs"; + const { gridMode$ } = createLayoutModeSwitch( + scope, + behavior("n", { n: "normal" }), + cold(shareMarble, { f: false, t: true }), + ); + + expectObservable(gridMode$).toBe(gridsMarble, { + g: "grid", + s: "spotlight", + }); + }); +}); + +test("can switch manually to grid after screen share while manually in spotlight", () => { + withTestScheduler(({ cold, behavior, schedule, expectObservable }): void => { + // Initially, no one is sharing. Then the user manually switches to + // spotlight. After a screen share starts, the user manually switches to + // grid. + const shareMarbles = " f-t-"; + const setModeMarbles = "-s-g"; + const expectation = " gs-g"; + const { gridMode$, setGridMode } = createLayoutModeSwitch( + scope, + behavior("n", { n: "normal" }), + cold(shareMarbles, { f: false, t: true }), + ); + schedule(setModeMarbles, { + g: () => setGridMode("grid"), + s: () => setGridMode("spotlight"), + }); + + expectObservable(gridMode$).toBe(expectation, { + g: "grid", + s: "spotlight", + }); + }); +}); + +test("Should auto-switch to spotlight when in flat window mode", () => { + withTestScheduler(({ cold, behavior, expectObservable }): void => { + const { gridMode$ } = createLayoutModeSwitch( + scope, + behavior("naf", { n: "normal", a: "narrow", f: "flat" }), + cold("f", { f: false, t: true }), + ); + + expectObservable(gridMode$).toBe("g-s-", { + g: "grid", + s: "spotlight", + }); + }); +}); diff --git a/src/state/CallViewModel/LayoutSwitch.ts b/src/state/CallViewModel/LayoutSwitch.ts new file mode 100644 index 00000000..3ad93204 --- /dev/null +++ b/src/state/CallViewModel/LayoutSwitch.ts @@ -0,0 +1,130 @@ +/* +Copyright 2025 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + BehaviorSubject, + combineLatest, + map, + type Observable, + scan, +} from "rxjs"; +import { logger } from "matrix-js-sdk/lib/logger"; + +import { type GridMode, type WindowMode } from "./CallViewModel.ts"; +import { type Behavior } from "../Behavior.ts"; +import { type ObservableScope } from "../ObservableScope.ts"; + +/** + * Creates a layout mode switch that allows switching between grid and spotlight modes. + * The actual layout mode can be overridden to spotlight mode if there is a remote screen share active + * or if the window mode is flat. + * + * @param scope - The observable scope to manage subscriptions. + * @param windowMode$ - The current window mode observable. + * @param hasRemoteScreenShares$ - An observable indicating if there are remote screen shares active. + */ +export function createLayoutModeSwitch( + scope: ObservableScope, + windowMode$: Behavior, + hasRemoteScreenShares$: Observable, +): { + gridMode$: Behavior; + setGridMode: (value: GridMode) => void; +} { + const gridModeUserSelection$ = new BehaviorSubject("grid"); + + // Callback to set the grid mode desired by the user. + // Notice that this is only a preference, the actual grid mode can be overridden + // if there is a remote screen share active. + const setGridMode = (value: GridMode): void => { + gridModeUserSelection$.next(value); + }; + /** + * The layout mode of the media tile grid. + */ + const gridMode$ = + // If the user hasn't selected spotlight and somebody starts screen sharing, + // automatically switch to spotlight mode and reset when screen sharing ends + scope.behavior( + combineLatest([ + gridModeUserSelection$, + hasRemoteScreenShares$, + windowMode$, + ]).pipe( + // Scan to keep track if we have auto-switched already or not. + // To allow the user to override the auto-switch by selecting grid mode again. + scan< + [GridMode, boolean, WindowMode], + { + mode: GridMode; + /** Remember if the change was user driven or not */ + hasAutoSwitched: boolean; + /** To know if it is new screen share or an already handled */ + hasScreenShares: boolean; + } + >( + (prev, [userSelection, hasScreenShares, windowMode]) => { + const isFlatMode = windowMode === "flat"; + + // Always force spotlight in flat mode, grid layout is not supported + // in that mode. + // TODO: strange that we do that for flat mode but not for other modes? + // TODO: Why is this not handled in layoutMedia$ like other window modes? + if (isFlatMode) { + logger.debug(`Forcing spotlight mode, windowMode=${windowMode}`); + return { + mode: "spotlight", + hasAutoSwitched: prev.hasAutoSwitched, + hasScreenShares, + }; + } + + // User explicitly chose spotlight. + // Respect that choice. + if (userSelection === "spotlight") { + return { + mode: "spotlight", + hasAutoSwitched: prev.hasAutoSwitched, + hasScreenShares, + }; + } + + // User has chosen grid mode. If a screen share starts, we will + // auto-switch to spotlight mode for better experience. + // But we only do it once, if the user switches back to grid mode, + // we respect that choice until they explicitly change it again. + const isNewShare = hasScreenShares && !prev.hasScreenShares; + if (isNewShare && !prev.hasAutoSwitched) { + return { + mode: "spotlight", + hasAutoSwitched: true, + hasScreenShares: true, + }; + } + + // Respect user's grid choice + // XXX If we want to forbid switching automatically again after we can + // return hasAutoSwitched: acc.hasAutoSwitched here instead of setting to false. + return { + mode: "grid", + hasAutoSwitched: false, + hasScreenShares, + }; + }, + // initial value + { mode: "grid", hasAutoSwitched: false, hasScreenShares: false }, + ), + map(({ mode }) => mode), + ), + "grid", + ); + + return { + gridMode$, + setGridMode, + }; +} diff --git a/src/state/CallViewModel/localMember/LocalMembership.ts b/src/state/CallViewModel/localMember/LocalMembership.ts index 5782fa47..f959b822 100644 --- a/src/state/CallViewModel/localMember/LocalMembership.ts +++ b/src/state/CallViewModel/localMember/LocalMembership.ts @@ -327,12 +327,14 @@ export const createLocalMembership$ = ({ // - overwrite current publisher scope.reconcile(localConnection$, async (connection) => { if (connection !== null) { - publisher$.next(createPublisherFactory(connection)); + const publisher = createPublisherFactory(connection); + publisher$.next(publisher); + // Clean-up callback + return Promise.resolve(async (): Promise => { + await publisher.stopPublishing(); + publisher.stopTracks(); + }); } - return Promise.resolve(async (): Promise => { - await publisher$?.value?.stopPublishing(); - publisher$?.value?.stopTracks(); - }); }); // Use reconcile here to not run concurrent createAndSetupTracks calls diff --git a/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts b/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts index 4d4a23cb..48e5b8d8 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionFactory.ts @@ -7,10 +7,11 @@ Please see LICENSE in the repository root for full details. import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; import { - type E2EEOptions, Room as LivekitRoom, type RoomOptions, type BaseKeyProvider, + type E2EEManagerOptions, + type BaseE2EEManager, } from "livekit-client"; import { type Logger } from "matrix-js-sdk/lib/logger"; // imported as inline to support worker when loaded from a cdn (cross domain) @@ -42,8 +43,10 @@ export class ECConnectionFactory implements ConnectionFactory { * @param client - The OpenID client parts for authentication, needed to get openID and JWT tokens. * @param devices - Used for video/audio out/in capture options. * @param processorState$ - Effects like background blur (only for publishing connection?) - * @param e2eeLivekitOptions - The E2EE options to use for the LiveKit Room. + * @param livekitKeyProvider * @param controlledAudioDevices - Option to indicate whether audio output device is controlled externally (native mobile app). + * @param echoCancellation - Whether to enable echo cancellation for audio capture. + * @param noiseSuppression - Whether to enable noise suppression for audio capture. * @param livekitRoomFactory - Optional factory function (for testing) to create LivekitRoom instances. If not provided, a default factory is used. */ public constructor( @@ -53,20 +56,24 @@ export class ECConnectionFactory implements ConnectionFactory { livekitKeyProvider: BaseKeyProvider | undefined, private controlledAudioDevices: boolean, livekitRoomFactory?: () => LivekitRoom, + echoCancellation: boolean = true, + noiseSuppression: boolean = true, ) { const defaultFactory = (): LivekitRoom => new LivekitRoom( - generateRoomOption( - this.devices, - this.processorState$.value, - livekitKeyProvider && { + generateRoomOption({ + devices: this.devices, + processorState: this.processorState$.value, + e2eeLivekitOptions: livekitKeyProvider && { keyProvider: livekitKeyProvider, // It's important that every room use a separate E2EE worker. // They get confused if given streams from multiple rooms. worker: new E2EEWorker(), }, - this.controlledAudioDevices, - ), + controlledAudioDevices: this.controlledAudioDevices, + echoCancellation, + noiseSuppression, + }), ); this.livekitRoomFactory = livekitRoomFactory ?? defaultFactory; } @@ -91,12 +98,24 @@ export class ECConnectionFactory implements ConnectionFactory { /** * Generate the initial LiveKit RoomOptions based on the current media devices and processor state. */ -function generateRoomOption( - devices: MediaDevices, - processorState: ProcessorState, - e2eeLivekitOptions: E2EEOptions | undefined, - controlledAudioDevices: boolean, -): RoomOptions { +function generateRoomOption({ + devices, + processorState, + e2eeLivekitOptions, + controlledAudioDevices, + echoCancellation, + noiseSuppression, +}: { + devices: MediaDevices; + processorState: ProcessorState; + e2eeLivekitOptions: + | E2EEManagerOptions + | { e2eeManager: BaseE2EEManager } + | undefined; + controlledAudioDevices: boolean; + echoCancellation: boolean; + noiseSuppression: boolean; +}): RoomOptions { return { ...defaultLiveKitOptions, videoCaptureDefaults: { @@ -107,6 +126,8 @@ function generateRoomOption( audioCaptureDefaults: { ...defaultLiveKitOptions.audioCaptureDefaults, deviceId: devices.audioInput.selected$.value?.id, + echoCancellation, + noiseSuppression, }, audioOutput: { // When using controlled audio devices, we don't want to set the diff --git a/src/state/CallViewModel/remoteMembers/ConnectionManager.ts b/src/state/CallViewModel/remoteMembers/ConnectionManager.ts index cd94a657..088e085a 100644 --- a/src/state/CallViewModel/remoteMembers/ConnectionManager.ts +++ b/src/state/CallViewModel/remoteMembers/ConnectionManager.ts @@ -10,7 +10,7 @@ import { type LivekitTransport, type ParticipantId, } from "matrix-js-sdk/lib/matrixrtc"; -import { BehaviorSubject, combineLatest, map, of, switchMap, tap } from "rxjs"; +import { combineLatest, map, of, switchMap, tap } from "rxjs"; import { type Logger } from "matrix-js-sdk/lib/logger"; import { type LocalParticipant, type RemoteParticipant } from "livekit-client"; @@ -123,9 +123,6 @@ export function createConnectionManager$({ logger: parentLogger, }: Props): IConnectionManager { const logger = parentLogger.getChild("[ConnectionManager]"); - - const running$ = new BehaviorSubject(true); - scope.onEnd(() => running$.next(false)); // TODO logger: only construct one logger from the client and make it compatible via a EC specific sing /** @@ -137,10 +134,7 @@ export function createConnectionManager$({ * externally this is modified via `registerTransports()`. */ const transports$ = scope.behavior( - combineLatest([running$, inputTransports$]).pipe( - map(([running, transports]) => - transports.mapInner((transport) => (running ? transport : [])), - ), + inputTransports$.pipe( map((transports) => transports.mapInner(removeDuplicateTransports)), tap(({ value: transports }) => { logger.trace( diff --git a/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts b/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts new file mode 100644 index 00000000..0c439a6b --- /dev/null +++ b/src/state/CallViewModel/remoteMembers/ECConnectionFactory.test.ts @@ -0,0 +1,133 @@ +/* +Copyright 2025 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; +import { Room as LivekitRoom } from "livekit-client"; +import { BehaviorSubject } from "rxjs"; +import fetchMock from "fetch-mock"; +import { logger } from "matrix-js-sdk/lib/logger"; +import EventEmitter from "events"; + +import { ObservableScope } from "../../ObservableScope.ts"; +import { ECConnectionFactory } from "./ConnectionFactory.ts"; +import type { OpenIDClientParts } from "../../../livekit/openIDSFU.ts"; +import { exampleTransport, mockMediaDevices } from "../../../utils/test.ts"; +import type { ProcessorState } from "../../../livekit/TrackProcessorContext.tsx"; +import { constant } from "../../Behavior"; + +// At the top of your test file, after imports +vi.mock("livekit-client", async (importOriginal) => { + return { + ...(await importOriginal()), + Room: vi.fn().mockImplementation(function (this: LivekitRoom, options) { + const emitter = new EventEmitter(); + return { + on: emitter.on.bind(emitter), + off: emitter.off.bind(emitter), + emit: emitter.emit.bind(emitter), + disconnect: vi.fn(), + remoteParticipants: new Map(), + } as unknown as LivekitRoom; + }), + }; +}); + +let testScope: ObservableScope; +let mockClient: OpenIDClientParts; + +beforeEach(() => { + testScope = new ObservableScope(); + mockClient = { + getOpenIdToken: vi.fn().mockReturnValue(""), + getDeviceId: vi.fn().mockReturnValue("DEV000"), + }; +}); + +describe("ECConnectionFactory - Audio inputs options", () => { + test.each([ + { echo: true, noise: true }, + { echo: true, noise: false }, + { echo: false, noise: true }, + { echo: false, noise: false }, + ])( + "it sets echoCancellation=$echo and noiseSuppression=$noise based on constructor parameters", + ({ echo, noise }) => { + // test("it sets echoCancellation and noiseSuppression based on constructor parameters", () => { + const RoomConstructor = vi.mocked(LivekitRoom); + + const ecConnectionFactory = new ECConnectionFactory( + mockClient, + mockMediaDevices({}), + new BehaviorSubject({ + supported: true, + processor: undefined, + }), + undefined, + false, + undefined, + echo, + noise, + ); + ecConnectionFactory.createConnection(exampleTransport, testScope, logger); + + // Check if Room was constructed with expected options + expect(RoomConstructor).toHaveBeenCalledWith( + expect.objectContaining({ + audioCaptureDefaults: expect.objectContaining({ + echoCancellation: echo, + noiseSuppression: noise, + }), + }), + ); + }, + ); +}); + +describe("ECConnectionFactory - ControlledAudioDevice", () => { + test.each([{ controlled: true }, { controlled: false }])( + "it sets controlledAudioDevice=$controlled then uses deviceId accordingly", + ({ controlled }) => { + // test("it sets echoCancellation and noiseSuppression based on constructor parameters", () => { + const RoomConstructor = vi.mocked(LivekitRoom); + + const ecConnectionFactory = new ECConnectionFactory( + mockClient, + mockMediaDevices({ + audioOutput: { + available$: constant(new Map()), + selected$: constant({ id: "DEV00", virtualEarpiece: false }), + select: () => {}, + }, + }), + new BehaviorSubject({ + supported: true, + processor: undefined, + }), + undefined, + controlled, + undefined, + false, + false, + ); + ecConnectionFactory.createConnection(exampleTransport, testScope, logger); + + // Check if Room was constructed with expected options + expect(RoomConstructor).toHaveBeenCalledWith( + expect.objectContaining({ + audioOutput: expect.objectContaining({ + deviceId: controlled ? undefined : "DEV00", + }), + }), + ); + }, + ); +}); + +afterEach(() => { + testScope.end(); + fetchMock.reset(); +}); diff --git a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts index 72e2883a..e4238a4e 100644 --- a/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts +++ b/src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts @@ -108,7 +108,7 @@ export function createMatrixLivekitMembers$({ // Each update where the key of the generator array do not change will result in updates to the `data$` observable in the factory. (scope, data$, participantId, userId) => { logger.debug( - `Updating data$ for participantId: ${participantId}, userId: ${userId}`, + `Generating member for participantId: ${participantId}, userId: ${userId}`, ); // will only get called once per `participantId, userId` pair. // updates to data$ and as a result to displayName$ and mxcAvatarUrl$ are more frequent. diff --git a/src/state/MuteStates.test.ts b/src/state/MuteStates.test.ts new file mode 100644 index 00000000..f2a6e35f --- /dev/null +++ b/src/state/MuteStates.test.ts @@ -0,0 +1,212 @@ +/* +Copyright 2025 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; +import { BehaviorSubject } from "rxjs"; +import { logger } from "matrix-js-sdk/lib/logger"; + +import { MuteStates, MuteState } from "./MuteStates"; +import { + type AudioOutputDeviceLabel, + type DeviceLabel, + type MediaDevice, + type SelectedAudioOutputDevice, + type SelectedDevice, +} from "./MediaDevices"; +import { constant } from "./Behavior"; +import { ObservableScope } from "./ObservableScope"; +import { flushPromises, mockMediaDevices } from "../utils/test"; + +const getUrlParams = vi.hoisted(() => vi.fn(() => ({}))); +vi.mock("../UrlParams", () => ({ getUrlParams })); + +let testScope: ObservableScope; + +beforeEach(() => { + testScope = new ObservableScope(); +}); + +afterEach(() => { + testScope.end(); +}); + +describe("MuteState", () => { + test("should automatically mute if force mute is set", async () => { + const forceMute$ = new BehaviorSubject(false); + + const deviceStub = { + available$: constant( + new Map([ + ["fbac11", { type: "name", name: "HD Camera" }], + ]), + ), + selected$: constant({ id: "fbac11" }), + select(): void {}, + } as unknown as MediaDevice; + + const muteState = new MuteState( + testScope, + deviceStub, + constant(true), + true, + forceMute$, + ); + let lastEnabled: boolean = false; + muteState.enabled$.subscribe((enabled) => { + lastEnabled = enabled; + }); + let setEnabled: ((enabled: boolean) => void) | null = null; + muteState.setEnabled$.subscribe((setter) => { + setEnabled = setter; + }); + + await flushPromises(); + + setEnabled!(true); + await flushPromises(); + expect(lastEnabled).toBe(true); + + // Now force mute + forceMute$.next(true); + await flushPromises(); + // Should automatically mute + expect(lastEnabled).toBe(false); + + // Try to unmute can not work + expect(setEnabled).toBeNull(); + + // Disable force mute + forceMute$.next(false); + await flushPromises(); + + // TODO I'd expect it to go back to previous state (enabled) + // but actually it goes back to the initial state from construction (disabled) + // Should go back to previous state (enabled) + // Skip for now + // expect(lastEnabled).toBe(true); + + // But yet it can be unmuted now + expect(setEnabled).not.toBeNull(); + + setEnabled!(true); + await flushPromises(); + expect(lastEnabled).toBe(true); + }); +}); + +describe("MuteStates", () => { + function aAudioOutputDevices(): MediaDevice< + AudioOutputDeviceLabel, + SelectedAudioOutputDevice + > { + const selected$ = new BehaviorSubject< + SelectedAudioOutputDevice | undefined + >({ + id: "default", + virtualEarpiece: false, + }); + return { + available$: constant( + new Map([ + ["default", { type: "speaker" }], + ["0000", { type: "speaker" }], + ["1111", { type: "earpiece" }], + ["222", { type: "name", name: "Bluetooth Speaker" }], + ]), + ), + selected$, + select(id: string): void { + if (!this.available$.getValue().has(id)) { + logger.warn(`Attempted to select unknown device id: ${id}`); + return; + } + selected$.next({ + id, + /** For test purposes we ignore this */ + virtualEarpiece: false, + }); + }, + }; + } + + function aVideoInput(): MediaDevice { + const selected$ = new BehaviorSubject( + undefined, + ); + return { + available$: constant( + new Map([ + ["0000", { type: "name", name: "HD Camera" }], + ["1111", { type: "name", name: "WebCam Pro" }], + ]), + ), + selected$, + select(id: string): void { + if (!this.available$.getValue().has(id)) { + logger.warn(`Attempted to select unknown device id: ${id}`); + return; + } + selected$.next({ id }); + }, + }; + } + + test("should mute camera when in earpiece mode", async () => { + const audioOutputDevice = aAudioOutputDevices(); + + const mediaDevices = mockMediaDevices({ + audioOutput: audioOutputDevice, + videoInput: aVideoInput(), + // other devices are not relevant for this test + }); + const muteStates = new MuteStates( + testScope, + mediaDevices, + // consider joined + constant(true), + ); + + let latestSyncedState: boolean | null = null; + muteStates.video.setHandler(async (enabled: boolean): Promise => { + logger.info(`Video mute state set to: ${enabled}`); + latestSyncedState = enabled; + return Promise.resolve(enabled); + }); + + let lastVideoEnabled: boolean = false; + muteStates.video.enabled$.subscribe((enabled) => { + lastVideoEnabled = enabled; + }); + + expect(muteStates.video.setEnabled$.value).toBeDefined(); + muteStates.video.setEnabled$.value?.(true); + await flushPromises(); + + expect(lastVideoEnabled).toBe(true); + + // Select earpiece audio output + audioOutputDevice.select("1111"); + await flushPromises(); + // Video should be automatically muted + expect(lastVideoEnabled).toBe(false); + expect(latestSyncedState).toBe(false); + + // Try to switch to speaker + audioOutputDevice.select("0000"); + await flushPromises(); + // TODO I'd expect it to go back to previous state (enabled)?? + // But maybe not? If you move the phone away from your ear you may not want it + // to automatically enable video? + expect(lastVideoEnabled).toBe(false); + + // But yet it can be unmuted now + expect(muteStates.video.setEnabled$.value).toBeDefined(); + muteStates.video.setEnabled$.value?.(true); + await flushPromises(); + expect(lastVideoEnabled).toBe(true); + }); +}); diff --git a/src/state/MuteStates.ts b/src/state/MuteStates.ts index 50be5e05..632e0426 100644 --- a/src/state/MuteStates.ts +++ b/src/state/MuteStates.ts @@ -27,7 +27,7 @@ import { ElementWidgetActions, widget } from "../widget"; import { Config } from "../config/Config"; import { getUrlParams } from "../UrlParams"; import { type ObservableScope } from "./ObservableScope"; -import { type Behavior } from "./Behavior"; +import { type Behavior, constant } from "./Behavior"; interface MuteStateData { enabled$: Observable; @@ -38,31 +38,58 @@ interface MuteStateData { export type Handler = (desired: boolean) => Promise; const defaultHandler: Handler = async (desired) => Promise.resolve(desired); -class MuteState { +/** + * Internal class - exported only for testing purposes. + * Do not use directly outside of tests. + */ +export class MuteState { + // TODO: rewrite this to explain behavior, it is not understandable, and cannot add logging private readonly enabledByDefault$ = this.enabledByConfig && !getUrlParams().skipLobby ? this.joined$.pipe(map((isJoined) => !isJoined)) : of(false); private readonly handler$ = new BehaviorSubject(defaultHandler); + public setHandler(handler: Handler): void { if (this.handler$.value !== defaultHandler) throw new Error("Multiple mute state handlers are not supported"); this.handler$.next(handler); } + public unsetHandler(): void { this.handler$.next(defaultHandler); } + private readonly canControlDevices$ = combineLatest([ + this.device.available$, + this.forceMute$, + ]).pipe( + map(([available, forceMute]) => { + return !forceMute && available.size > 0; + }), + ); + private readonly data$ = this.scope.behavior( - this.device.available$.pipe( - map((available) => available.size > 0), + this.canControlDevices$.pipe( distinctUntilChanged(), withLatestFrom( this.enabledByDefault$, - (devicesConnected, enabledByDefault) => { - if (!devicesConnected) + (canControlDevices, enabledByDefault) => { + logger.info( + `MuteState: canControlDevices: ${canControlDevices}, enabled by default: ${enabledByDefault}`, + ); + if (!canControlDevices) { + logger.info( + `MuteState: devices connected: ${canControlDevices}, disabling`, + ); + // We need to sync the mute state with the handler + // to ensure nothing is beeing published. + this.handler$.value(false).catch((err) => { + logger.error("MuteState-disable: handler error", err); + }); return { enabled$: of(false), set: null, toggle: null }; + } // Assume the default value only once devices are actually connected let enabled = enabledByDefault; @@ -135,21 +162,45 @@ class MuteState { private readonly device: MediaDevice, private readonly joined$: Observable, private readonly enabledByConfig: boolean, + /** + * An optional observable which, when it emits `true`, will force the mute. + * Used for video to stop camera when earpiece mode is on. + * @private + */ + private readonly forceMute$: Observable, ) {} } export class MuteStates { + /** + * True if the selected audio output device is an earpiece. + * Used to force-disable video when on earpiece. + */ + private readonly isEarpiece$ = combineLatest( + this.mediaDevices.audioOutput.available$, + this.mediaDevices.audioOutput.selected$, + ).pipe( + map(([available, selected]) => { + if (!selected?.id) return false; + const device = available.get(selected.id); + logger.info(`MuteStates: selected audio output device:`, device); + return device?.type === "earpiece"; + }), + ); + public readonly audio = new MuteState( this.scope, this.mediaDevices.audioInput, this.joined$, Config.get().media_devices.enable_audio, + constant(false), ); public readonly video = new MuteState( this.scope, this.mediaDevices.videoInput, this.joined$, Config.get().media_devices.enable_video, + this.isEarpiece$, ); public constructor(