From fc39e82666182e273a40a7c3bc0a10b7bdf9267c Mon Sep 17 00:00:00 2001 From: Valere Date: Thu, 27 Nov 2025 15:52:25 +0100 Subject: [PATCH 1/3] Fix: Camera is not muted when the earpiece mode is enabled --- src/state/MuteStates.test.ts | 190 +++++++++++++++++++++++++++++++++++ src/state/MuteStates.ts | 58 ++++++++++- 2 files changed, 243 insertions(+), 5 deletions(-) create mode 100644 src/state/MuteStates.test.ts diff --git a/src/state/MuteStates.test.ts b/src/state/MuteStates.test.ts new file mode 100644 index 00000000..7b02d190 --- /dev/null +++ b/src/state/MuteStates.test.ts @@ -0,0 +1,190 @@ +/* +Copyright 2025 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; +import { BehaviorSubject } from "rxjs"; +import { logger } from "matrix-js-sdk/lib/logger"; + +import { MuteStates, MuteState } from "./MuteStates"; +import { + type AudioOutputDeviceLabel, + type DeviceLabel, + type MediaDevice, + type SelectedAudioOutputDevice, + type SelectedDevice, +} from "./MediaDevices"; +import { constant } from "./Behavior"; +import { ObservableScope } from "./ObservableScope"; +import { flushPromises, mockMediaDevices } from "../utils/test"; + +const getUrlParams = vi.hoisted(() => vi.fn(() => ({}))); +vi.mock("../UrlParams", () => ({ getUrlParams })); + +let testScope: ObservableScope; + +beforeEach(() => { + testScope = new ObservableScope(); +}); + +afterEach(() => { + testScope.end(); +}); + +describe("MuteState", () => { + test("should automatically mute if force mute is set", async () => { + const forceMute$ = new BehaviorSubject(false); + + const deviceStub = { + available$: constant( + new Map([ + ["fbac11", { type: "name", name: "HD Camera" }], + ]), + ), + selected$: constant({ id: "fbac11" }), + select(): void {}, + } as unknown as MediaDevice; + + const muteState = new MuteState( + testScope, + deviceStub, + constant(true), + true, + forceMute$, + ); + let lastEnabled: boolean = false; + muteState.enabled$.subscribe((enabled) => { + lastEnabled = enabled; + }); + let setEnabled: ((enabled: boolean) => void) | null = null; + muteState.setEnabled$.subscribe((setter) => { + setEnabled = setter; + }); + + await flushPromises(); + + setEnabled!(true); + await flushPromises(); + expect(lastEnabled).toBe(true); + + // Now force mute + forceMute$.next(true); + await flushPromises(); + // Should automatically mute + expect(lastEnabled).toBe(false); + + // Try to unmute can not work + expect(setEnabled).toBeNull(); + + // Disable force mute + forceMute$.next(false); + await flushPromises(); + + // TODO I'd expect it to go back to previous state (enabled) + // but actually it goes back to the initial state from construction (disabled) + // Should go back to previous state (enabled) + // Skip for now + // expect(lastEnabled).toBe(true); + + // But yet it can be unmuted now + expect(setEnabled).not.toBeNull(); + + setEnabled!(true); + await flushPromises(); + expect(lastEnabled).toBe(true); + }); +}); + +describe("MuteStates", () => { + function aAudioOutputDevices(): MediaDevice< + AudioOutputDeviceLabel, + SelectedAudioOutputDevice + > { + const selected$ = new BehaviorSubject< + SelectedAudioOutputDevice | undefined + >({ + id: "default", + virtualEarpiece: false, + }); + return { + available$: constant( + new Map([ + ["default", { type: "speaker" }], + ["0000", { type: "speaker" }], + ["1111", { type: "earpiece" }], + ["222", { type: "name", name: "Bluetooth Speaker" }], + ]), + ), + selected$, + select(id: string): void { + if (!this.available$.getValue().has(id)) { + logger.warn(`Attempted to select unknown device id: ${id}`); + return; + } + selected$.next({ + id, + /** For test purposes we ignore this */ + virtualEarpiece: false, + }); + }, + }; + } + + function aVideoInput(): MediaDevice { + const selected$ = new BehaviorSubject( + undefined, + ); + return { + available$: constant( + new Map([ + ["0000", { type: "name", name: "HD Camera" }], + ["1111", { type: "name", name: "WebCam Pro" }], + ]), + ), + selected$, + select(id: string): void { + if (!this.available$.getValue().has(id)) { + logger.warn(`Attempted to select unknown device id: ${id}`); + return; + } + selected$.next({ id }); + }, + }; + } + + test("should mute camera when in earpiece mode", async () => { + const audioOutputDevice = aAudioOutputDevices(); + + const mediaDevices = mockMediaDevices({ + audioOutput: audioOutputDevice, + videoInput: aVideoInput(), + // other devices are not relevant for this test + }); + const muteStates = new MuteStates( + testScope, + mediaDevices, + // consider joined + constant(true), + ); + + let lastVideoEnabled: boolean = false; + muteStates.video.enabled$.subscribe((enabled) => { + lastVideoEnabled = enabled; + }); + + expect(muteStates.video.setEnabled$.value).toBeDefined(); + muteStates.video.setEnabled$.value?.(true); + await flushPromises(); + + expect(lastVideoEnabled).toBe(true); + + // Select earpiece audio output + audioOutputDevice.select("1111"); + await flushPromises(); + // Video should be automatically muted + expect(lastVideoEnabled).toBe(false); + }); +}); diff --git a/src/state/MuteStates.ts b/src/state/MuteStates.ts index 50be5e05..777e3aa4 100644 --- a/src/state/MuteStates.ts +++ b/src/state/MuteStates.ts @@ -27,7 +27,7 @@ import { ElementWidgetActions, widget } from "../widget"; import { Config } from "../config/Config"; import { getUrlParams } from "../UrlParams"; import { type ObservableScope } from "./ObservableScope"; -import { type Behavior } from "./Behavior"; +import { type Behavior, constant } from "./Behavior"; interface MuteStateData { enabled$: Observable; @@ -38,31 +38,55 @@ interface MuteStateData { export type Handler = (desired: boolean) => Promise; const defaultHandler: Handler = async (desired) => Promise.resolve(desired); -class MuteState { +/** + * Internal class - exported only for testing purposes. + * Do not use directly outside of tests. + */ +export class MuteState { + // TODO: rewrite this to explain behavior, it is not understandable, and cannot add logging private readonly enabledByDefault$ = this.enabledByConfig && !getUrlParams().skipLobby ? this.joined$.pipe(map((isJoined) => !isJoined)) : of(false); private readonly handler$ = new BehaviorSubject(defaultHandler); + public setHandler(handler: Handler): void { if (this.handler$.value !== defaultHandler) throw new Error("Multiple mute state handlers are not supported"); this.handler$.next(handler); } + public unsetHandler(): void { this.handler$.next(defaultHandler); } + private readonly devicesConnected$ = combineLatest([ + this.device.available$, + this.forceMute$, + ]).pipe( + map(([available, forceMute]) => { + return !forceMute && available.size > 0; + }), + ); + private readonly data$ = this.scope.behavior( - this.device.available$.pipe( - map((available) => available.size > 0), + this.devicesConnected$.pipe( + // this.device.available$.pipe( + // map((available) => available.size > 0), distinctUntilChanged(), withLatestFrom( this.enabledByDefault$, (devicesConnected, enabledByDefault) => { - if (!devicesConnected) + logger.info( + `MuteState: devices connected: ${devicesConnected}, enabled by default: ${enabledByDefault}`, + ); + if (!devicesConnected) { + logger.info( + `MuteState: devices connected: ${devicesConnected}, disabling`, + ); return { enabled$: of(false), set: null, toggle: null }; + } // Assume the default value only once devices are actually connected let enabled = enabledByDefault; @@ -135,21 +159,45 @@ class MuteState { private readonly device: MediaDevice, private readonly joined$: Observable, private readonly enabledByConfig: boolean, + /** + * An optional observable which, when it emits `true`, will force the mute. + * Used for video to stop camera when earpiece mode is on. + * @private + */ + private readonly forceMute$: Observable, ) {} } export class MuteStates { + /** + * True if the selected audio output device is an earpiece. + * Used to force-disable video when on earpiece. + */ + private readonly isEarpiece$ = combineLatest( + this.mediaDevices.audioOutput.available$, + this.mediaDevices.audioOutput.selected$, + ).pipe( + map(([available, selected]) => { + if (!selected?.id) return false; + const device = available.get(selected.id); + logger.info(`MuteStates: selected audio output device:`, device); + return device?.type === "earpiece"; + }), + ); + public readonly audio = new MuteState( this.scope, this.mediaDevices.audioInput, this.joined$, Config.get().media_devices.enable_audio, + constant(false), ); public readonly video = new MuteState( this.scope, this.mediaDevices.videoInput, this.joined$, Config.get().media_devices.enable_video, + this.isEarpiece$, ); public constructor( From 149f3d02ae583d90c5cd458c7d290106d5aba1c5 Mon Sep 17 00:00:00 2001 From: Valere Date: Thu, 27 Nov 2025 18:47:33 +0100 Subject: [PATCH 2/3] fix: The force mute state was not synced to the handler --- src/state/MuteStates.test.ts | 22 ++++++++++++++++++++++ src/state/MuteStates.ts | 7 +++++-- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/src/state/MuteStates.test.ts b/src/state/MuteStates.test.ts index 7b02d190..f2a6e35f 100644 --- a/src/state/MuteStates.test.ts +++ b/src/state/MuteStates.test.ts @@ -170,6 +170,13 @@ describe("MuteStates", () => { constant(true), ); + let latestSyncedState: boolean | null = null; + muteStates.video.setHandler(async (enabled: boolean): Promise => { + logger.info(`Video mute state set to: ${enabled}`); + latestSyncedState = enabled; + return Promise.resolve(enabled); + }); + let lastVideoEnabled: boolean = false; muteStates.video.enabled$.subscribe((enabled) => { lastVideoEnabled = enabled; @@ -186,5 +193,20 @@ describe("MuteStates", () => { await flushPromises(); // Video should be automatically muted expect(lastVideoEnabled).toBe(false); + expect(latestSyncedState).toBe(false); + + // Try to switch to speaker + audioOutputDevice.select("0000"); + await flushPromises(); + // TODO I'd expect it to go back to previous state (enabled)?? + // But maybe not? If you move the phone away from your ear you may not want it + // to automatically enable video? + expect(lastVideoEnabled).toBe(false); + + // But yet it can be unmuted now + expect(muteStates.video.setEnabled$.value).toBeDefined(); + muteStates.video.setEnabled$.value?.(true); + await flushPromises(); + expect(lastVideoEnabled).toBe(true); }); }); diff --git a/src/state/MuteStates.ts b/src/state/MuteStates.ts index 777e3aa4..f1d61db5 100644 --- a/src/state/MuteStates.ts +++ b/src/state/MuteStates.ts @@ -72,8 +72,6 @@ export class MuteState { private readonly data$ = this.scope.behavior( this.devicesConnected$.pipe( - // this.device.available$.pipe( - // map((available) => available.size > 0), distinctUntilChanged(), withLatestFrom( this.enabledByDefault$, @@ -85,6 +83,11 @@ export class MuteState { logger.info( `MuteState: devices connected: ${devicesConnected}, disabling`, ); + // We need to sync the mute state with the handler + // to ensure nothing is beeing published. + this.handler$.value(false).catch((err) => { + logger.error("MuteState-disable: handler error", err); + }); return { enabled$: of(false), set: null, toggle: null }; } From 44980a2744e5f5b198c83e5de91f7d03aba0ab93 Mon Sep 17 00:00:00 2001 From: Valere Date: Tue, 2 Dec 2025 16:39:26 +0100 Subject: [PATCH 3/3] review: rename `deviceConnected` to `canControlDevices` --- src/state/MuteStates.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/state/MuteStates.ts b/src/state/MuteStates.ts index f1d61db5..632e0426 100644 --- a/src/state/MuteStates.ts +++ b/src/state/MuteStates.ts @@ -61,7 +61,7 @@ export class MuteState { this.handler$.next(defaultHandler); } - private readonly devicesConnected$ = combineLatest([ + private readonly canControlDevices$ = combineLatest([ this.device.available$, this.forceMute$, ]).pipe( @@ -71,17 +71,17 @@ export class MuteState { ); private readonly data$ = this.scope.behavior( - this.devicesConnected$.pipe( + this.canControlDevices$.pipe( distinctUntilChanged(), withLatestFrom( this.enabledByDefault$, - (devicesConnected, enabledByDefault) => { + (canControlDevices, enabledByDefault) => { logger.info( - `MuteState: devices connected: ${devicesConnected}, enabled by default: ${enabledByDefault}`, + `MuteState: canControlDevices: ${canControlDevices}, enabled by default: ${enabledByDefault}`, ); - if (!devicesConnected) { + if (!canControlDevices) { logger.info( - `MuteState: devices connected: ${devicesConnected}, disabling`, + `MuteState: devices connected: ${canControlDevices}, disabling`, ); // We need to sync the mute state with the handler // to ensure nothing is beeing published.