/* Copyright 2023-2025 New Vector Ltd. SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ import { type FC, createContext, useCallback, useContext, useEffect, useMemo, useRef, useState, type JSX, } from "react"; import { createMediaDeviceObserver } from "@livekit/components-core"; import { combineLatest, distinctUntilChanged, map, startWith } from "rxjs"; import { useObservable, useObservableEagerState } from "observable-hooks"; import { logger } from "matrix-js-sdk/lib/logger"; import { useSetting, audioInput as audioInputSetting, audioOutput as audioOutputSetting, videoInput as videoInputSetting, alwaysShowIphoneEarpiece as alwaysShowIphoneEarpieceSetting, type Setting, } from "../settings/settings"; import { outputDevice$, availableOutputDevices$ } from "../controls"; import { useUrlParams } from "../UrlParams"; // This hardcoded id is used in EX ios! It can only be changed in coordination with // the ios swift team. export const EARPIECE_CONFIG_ID = "earpiece-id"; export type DeviceLabel = | { type: "name"; name: string } | { type: "number"; number: number } | { type: "earpiece" } | { type: "default"; name: string | null }; export interface MediaDeviceHandle { /** * A map from available device IDs to labels. */ available: Map; selectedId: string | undefined; /** * An additional device configuration that makes us use only one channel of the * output device and a reduced volume. */ useAsEarpiece: boolean | undefined; /** * The group ID of the selected device. */ // This is exposed sort of ad-hoc because it's only needed for knowing when to // restart the tracks of default input devices, and ideally this behavior // would be encapsulated somehow… selectedGroupId: string | undefined; select: (deviceId: string) => void; } interface InputDevices { audioInput: MediaDeviceHandle; videoInput: MediaDeviceHandle; startUsingDeviceNames: () => void; stopUsingDeviceNames: () => void; usingNames: boolean; } export interface MediaDevices extends Omit { audioOutput: MediaDeviceHandle; } /** * An observable that represents if we should display the devices menu for iOS. * This implies the following * - hide any input devices (they do not work anyhow on ios) * - Show a button to show the native output picker instead. * - Only show the earpiece toggle option if the earpiece is available: * `availableOutputDevices$.includes((d)=>d.forEarpiece)` */ export const iosDeviceMenu$ = alwaysShowIphoneEarpieceSetting.value$.pipe( map((v) => v || navigator.userAgent.includes("iPhone")), ); function useSelectedId( available: Map, preferredId: string | undefined, ): string | undefined { return useMemo(() => { if (available.size) { // If the preferred device is available, use it. Or if every available // device ID is falsy, the browser is probably just being paranoid about // fingerprinting and we should still try using the preferred device. // Worst case it is not available and the browser will gracefully fall // back to some other device for us when requesting the media stream. // Otherwise, select the first available device. return (preferredId !== undefined && available.has(preferredId)) || (available.size === 1 && available.has("")) ? preferredId : available.keys().next().value; } return undefined; }, [available, preferredId]); } /** * Hook to get access to a mediaDevice handle for a kind. This allows to list * the available devices, read and set the selected device. * @param kind Audio input, output or video output. * @param setting The setting this handle's selection should be synced with. * @param usingNames If the hook should query device names for the associated * list. * @returns A handle for the chosen kind. */ function useMediaDeviceHandle( kind: MediaDeviceKind, setting: Setting, usingNames: boolean, ): MediaDeviceHandle { const hasRequestedPermissions = useRef(false); const requestPermissions = usingNames || hasRequestedPermissions.current; // Make sure we don't needlessly reset to a device observer without names, // once permissions are already given hasRequestedPermissions.current ||= usingNames; // We use a bare device observer here rather than one of the fancy device // selection hooks from @livekit/components-react, because // useMediaDeviceSelect expects a room or track, which we don't have here, and // useMediaDevices provides no way to request device names. // Tragically, the only way to get device names out of LiveKit is to specify a // kind, which then results in multiple permissions requests. const deviceObserver$ = useMemo( () => createMediaDeviceObserver( kind, () => logger.error("Error creating MediaDeviceObserver"), requestPermissions, ).pipe( startWith([]), // This Observable emits new values whenever the browser fires a // MediaDevices 'devicechange' event. One would think, innocently, that // a 'devicechange' event means the devices have changed. But as of the // time of writing, we are seeing mobile Safari firing spurious // 'devicechange' events (where no change has actually occurred) when // we call MediaDevices.getUserMedia. So, filter by deep equality. distinctUntilChanged((prev, current) => { if (prev.length !== current.length) return true; if (prev.length === 0) return false; return !current.every( (d, i) => d.deviceId === prev[i].deviceId && d.label === prev[i].label, ); }), ), [kind, requestPermissions], ); const available = useObservableEagerState( useMemo( () => deviceObserver$.pipe( map((availableRaw) => { // Sometimes browsers (particularly Firefox) can return multiple device // entries for the exact same device ID; using a map deduplicates them let available = new Map( availableRaw.map((d, i) => [ d.deviceId, d.label ? { type: "name", name: d.label } : { type: "number", number: i + 1 }, ]), ); // Create a virtual default audio output for browsers that don't have one. // Its device ID must be the empty string because that's what setSinkId // recognizes. // We also create this if we do not have any available devices, so that // we can use the default or the earpiece. if ( kind === "audiooutput" && !available.has("") && !available.has("default") && available.size ) available = new Map([ ["", { type: "default", name: availableRaw[0]?.label || null }], ...available, ]); // Note: creating virtual default input devices would be another problem // entirely, because requesting a media stream from deviceId "" won't // automatically track the default device. return available; }), ), [deviceObserver$, kind], ), ); const [preferredId, select] = useSetting(setting); const selectedId = useSelectedId(available, preferredId); const selectedGroupId = useObservableEagerState( useMemo( () => deviceObserver$.pipe( map( (availableRaw) => availableRaw.find((d) => d.deviceId === selectedId)?.groupId, ), ), [deviceObserver$, selectedId], ), ); return useMemo( () => ({ available, selectedId, useAsEarpiece: false, selectedGroupId, select, }), [available, selectedId, selectedGroupId, select], ); } export const deviceStub: MediaDeviceHandle = { available: new Map(), selectedId: undefined, selectedGroupId: undefined, select: () => {}, useAsEarpiece: false, }; export const devicesStub: MediaDevices = { audioInput: deviceStub, audioOutput: deviceStub, videoInput: deviceStub, startUsingDeviceNames: () => {}, stopUsingDeviceNames: () => {}, }; export const MediaDevicesContext = createContext(devicesStub); function useInputDevices(): InputDevices { // Counts the number of callers currently using device names. const [numCallersUsingNames, setNumCallersUsingNames] = useState(0); const usingNames = numCallersUsingNames > 0; const audioInput = useMediaDeviceHandle( "audioinput", audioInputSetting, usingNames, ); const videoInput = useMediaDeviceHandle( "videoinput", videoInputSetting, usingNames, ); const startUsingDeviceNames = useCallback( () => setNumCallersUsingNames((n) => n + 1), [setNumCallersUsingNames], ); const stopUsingDeviceNames = useCallback( () => setNumCallersUsingNames((n) => n - 1), [setNumCallersUsingNames], ); return { audioInput, videoInput, startUsingDeviceNames, stopUsingDeviceNames, usingNames, }; } interface Props { children: JSX.Element; } export const MediaDevicesProvider: FC = ({ children }) => { const { audioInput, videoInput, startUsingDeviceNames, stopUsingDeviceNames, usingNames, } = useInputDevices(); const { controlledAudioDevices } = useUrlParams(); const webViewAudioOutput = useMediaDeviceHandle( "audiooutput", audioOutputSetting, usingNames, ); const controlledAudioOutput = useControlledOutput(); const context: MediaDevices = useMemo( () => ({ audioInput, audioOutput: controlledAudioDevices ? controlledAudioOutput : webViewAudioOutput, videoInput, startUsingDeviceNames, stopUsingDeviceNames, }), [ audioInput, controlledAudioDevices, controlledAudioOutput, webViewAudioOutput, videoInput, startUsingDeviceNames, stopUsingDeviceNames, ], ); return ( {children} ); }; function useControlledOutput(): MediaDeviceHandle { const { available } = useObservableEagerState( useObservable(() => { const outputDeviceData$ = availableOutputDevices$.pipe( map((devices) => { const deviceForEarpiece = devices.find((d) => d.forEarpiece); const deviceMapTuple: [string, DeviceLabel][] = devices.map( ({ id, name, isEarpiece, isSpeaker /*,isExternalHeadset*/ }) => { let deviceLabel: DeviceLabel = { type: "name", name }; // if (isExternalHeadset) // Do we want this? if (isEarpiece) deviceLabel = { type: "earpiece" }; if (isSpeaker) deviceLabel = { type: "default", name }; return [id, deviceLabel]; }, ); return { devicesMap: new Map(deviceMapTuple), deviceForEarpiece, }; }), ); return combineLatest( [outputDeviceData$, iosDeviceMenu$], ({ devicesMap, deviceForEarpiece }, iosShowEarpiece) => { let available = devicesMap; if (iosShowEarpiece && !!deviceForEarpiece) { available = new Map([ ...devicesMap.entries(), [EARPIECE_CONFIG_ID, { type: "earpiece" }], ]); } return { available, deviceForEarpiece }; }, ); }), ); const [preferredId, setPreferredId] = useSetting(audioOutputSetting); useEffect(() => { const subscription = outputDevice$.subscribe((id) => { if (id) setPreferredId(id); }); return (): void => subscription.unsubscribe(); }, [setPreferredId]); const selectedId = useSelectedId(available, preferredId); const [asEarpiece, setAsEarpiece] = useState(false); useEffect(() => { // Let the hosting application know which output device has been selected. // This information is probably only of interest if the earpiece mode has been // selected - for example, Element X iOS listens to this to determine whether it // should enable the proximity sensor. if (selectedId) { window.controls.onAudioDeviceSelect?.(selectedId); // Call deprecated method for backwards compatibility. window.controls.onOutputDeviceSelect?.(selectedId); } setAsEarpiece(selectedId === EARPIECE_CONFIG_ID); }, [selectedId]); return useMemo( () => ({ available: available, selectedId, selectedGroupId: undefined, select: setPreferredId, useAsEarpiece: asEarpiece, }), [available, selectedId, setPreferredId, asEarpiece], ); } export const useMediaDevices = (): MediaDevices => useContext(MediaDevicesContext); /** * React hook that requests for the media devices context to be populated with * real device names while this component is mounted. This is not done by * default because it may involve requesting additional permissions from the * user. */ export const useMediaDeviceNames = ( context: MediaDevices, enabled = true, ): void => useEffect(() => { if (enabled) { context.startUsingDeviceNames(); return context.stopUsingDeviceNames; } }, [context, enabled]); /** * A convenience hook to get the audio node configuration for the earpiece. * It will check the `useAsEarpiece` of the `audioOutput` device and return * the appropriate pan and volume values. * * @returns pan and volume values for the earpiece audio node configuration. */ export const useEarpieceAudioConfig = (): { pan: number; volume: number; } => { const { audioOutput } = useMediaDevices(); // We use only the right speaker (pan = 1) for the earpiece. // This mimics the behavior of the native earpiece speaker (only the top speaker on an iPhone) const pan = useMemo( () => (audioOutput.useAsEarpiece ? 1 : 0), [audioOutput.useAsEarpiece], ); // We also do lower the volume by a factor of 10 to optimize for the usecase where // a user is holding the phone to their ear. const volume = useMemo( () => (audioOutput.useAsEarpiece ? 0.1 : 1), [audioOutput.useAsEarpiece], ); return { pan, volume }; };