mirror of
https://github.com/vector-im/element-call.git
synced 2026-03-16 06:17:10 +00:00
580 lines
20 KiB
TypeScript
580 lines
20 KiB
TypeScript
/*
|
|
Copyright 2025 New Vector Ltd.
|
|
|
|
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
|
Please see LICENSE in the repository root for full details.
|
|
*/
|
|
|
|
import {
|
|
combineLatest,
|
|
filter,
|
|
map,
|
|
merge,
|
|
pairwise,
|
|
startWith,
|
|
Subject,
|
|
switchMap,
|
|
type Observable,
|
|
tap,
|
|
} from "rxjs";
|
|
import { createMediaDeviceObserver } from "@livekit/components-core";
|
|
import { type Logger, logger as rootLogger } from "matrix-js-sdk/lib/logger";
|
|
import { type RTCCallIntent } from "matrix-js-sdk/lib/matrixrtc";
|
|
|
|
import {
|
|
audioInput as audioInputSetting,
|
|
audioOutput as audioOutputSetting,
|
|
videoInput as videoInputSetting,
|
|
alwaysShowIphoneEarpiece as alwaysShowIphoneEarpieceSetting,
|
|
} from "../settings/settings";
|
|
import { type ObservableScope } from "./ObservableScope";
|
|
import {
|
|
outputDevice$ as controlledOutputSelection$,
|
|
availableOutputDevices$ as controlledAvailableOutputDevices$,
|
|
} from "../controls";
|
|
import { getUrlParams } from "../UrlParams";
|
|
import { platform } from "../Platform";
|
|
import { switchWhen } from "../utils/observable";
|
|
import { type Behavior, constant } from "./Behavior";
|
|
|
|
// This hardcoded id is used in EX ios! It can only be changed in coordination with
|
|
// the ios swift team.
|
|
const EARPIECE_CONFIG_ID = "earpiece-id";
|
|
|
|
export type DeviceLabel =
|
|
| { type: "name"; name: string }
|
|
| { type: "number"; number: number };
|
|
|
|
export type AudioOutputDeviceLabel =
|
|
| DeviceLabel
|
|
| { type: "speaker" }
|
|
| { type: "earpiece" }
|
|
| { type: "default"; name: string | null };
|
|
|
|
/**
|
|
* Base selected-device value shared by all media kinds.
|
|
*
|
|
* `id` is the effective device identifier used by browser media APIs.
|
|
*/
|
|
export interface SelectedDevice {
|
|
id: string;
|
|
}
|
|
|
|
/**
|
|
* Selected audio input value with audio-input-specific metadata.
|
|
*/
|
|
export interface SelectedAudioInputDevice extends SelectedDevice {
|
|
/**
|
|
* Emits whenever we think that this audio input device has logically changed
|
|
* to refer to a different hardware device.
|
|
*/
|
|
hardwareDeviceChange$: Observable<void>;
|
|
}
|
|
|
|
/**
|
|
* Selected audio output value with output-routing-specific metadata.
|
|
*/
|
|
export interface SelectedAudioOutputDevice extends SelectedDevice {
|
|
/**
|
|
* Whether this device is a "virtual earpiece" device. If so, we should output
|
|
* on a single channel of the device at a reduced volume.
|
|
*/
|
|
virtualEarpiece: boolean;
|
|
}
|
|
|
|
/**
|
|
* Common reactive contract for selectable input/output media devices (mic, speaker, camera).
|
|
*
|
|
* `Label` is the type used to represent a device in UI lists.
|
|
* `Selected` is the type used to represent the active selection for a device kind.
|
|
*/
|
|
export interface MediaDevice<Label, Selected> {
|
|
/**
|
|
* Reactive map of currently available devices keyed by device ID.
|
|
*
|
|
* `Label` defines the UI-facing label data structure for each device type.
|
|
*/
|
|
available$: Behavior<Map<string, Label>>;
|
|
|
|
/**
|
|
* The active device selection.
|
|
* Can be `undefined` when no device is yet selected.
|
|
*
|
|
* When defined, `Selected` contains the selected device ID plus any
|
|
* type-specific metadata.
|
|
*/
|
|
selected$: Behavior<Selected | undefined>;
|
|
|
|
/**
|
|
* Requests selection of a device by ID.
|
|
*
|
|
* Implementations typically persist this preference and let `selected$`
|
|
* converge to the effective device (which may differ if the requested ID is
|
|
* unavailable).
|
|
*/
|
|
select(id: string): void;
|
|
}
|
|
|
|
/**
|
|
* An observable that represents if we should display the devices menu for iOS.
|
|
*
|
|
* This implies the following
|
|
* - hide any input devices (they do not work anyhow on ios)
|
|
* - Show a button to show the native output picker instead.
|
|
* - Only show the earpiece toggle option if the earpiece is available:
|
|
* `availableOutputDevices$.includes((d)=>d.forEarpiece)`
|
|
*/
|
|
export const iosDeviceMenu$ =
|
|
platform === "ios" ? constant(true) : alwaysShowIphoneEarpieceSetting.value$;
|
|
|
|
function availableRawDevices$(
|
|
kind: MediaDeviceKind,
|
|
usingNames$: Behavior<boolean>,
|
|
scope: ObservableScope,
|
|
logger: Logger,
|
|
): Behavior<MediaDeviceInfo[]> {
|
|
const logError = (e: Error): void =>
|
|
logger.error("Error creating MediaDeviceObserver", e);
|
|
const devices$ = createMediaDeviceObserver(kind, logError, false);
|
|
const devicesWithNames$ = createMediaDeviceObserver(kind, logError, true);
|
|
|
|
return scope.behavior(
|
|
usingNames$.pipe(
|
|
switchMap((withNames) =>
|
|
withNames
|
|
? // It might be that there is already a media stream running somewhere,
|
|
// and so we can do without requesting a second one. Only switch to the
|
|
// device observer that explicitly requests the names if we see that
|
|
// names are in fact missing from the initial device enumeration.
|
|
devices$.pipe(
|
|
switchWhen(
|
|
(devices, i) => i === 0 && devices.every((d) => !d.label),
|
|
devicesWithNames$,
|
|
),
|
|
)
|
|
: devices$,
|
|
),
|
|
),
|
|
[],
|
|
);
|
|
}
|
|
|
|
function buildDeviceMap(
|
|
availableRaw: MediaDeviceInfo[],
|
|
): Map<string, DeviceLabel> {
|
|
return new Map<string, DeviceLabel>(
|
|
availableRaw.map((d, i) => [
|
|
d.deviceId,
|
|
d.label
|
|
? { type: "name", name: d.label }
|
|
: { type: "number", number: i + 1 },
|
|
]),
|
|
);
|
|
}
|
|
|
|
function selectDevice$<Label>(
|
|
available$: Observable<Map<string, Label>>,
|
|
preferredId$: Observable<string | undefined>,
|
|
defaultPicker: (available: Map<string, Label>) => string | undefined = (
|
|
available,
|
|
) => available.keys().next().value,
|
|
): Observable<string | undefined> {
|
|
return combineLatest([available$, preferredId$], (available, preferredId) => {
|
|
if (available.size) {
|
|
if (preferredId !== undefined && available.has(preferredId)) {
|
|
// If the preferred device is available, use it.
|
|
return preferredId;
|
|
} else if (available.size === 1 && available.has("")) {
|
|
// In some cases the enumerateDevices will list the devices with empty string details:
|
|
// `{deviceId:'', kind:'audiooutput|audioinput|videoinput', label:'', groupId:''}`
|
|
// This can happen when:
|
|
// 1. The user has not yet granted permissions to microphone/devices
|
|
// 2. The page is not running in a secure context (e.g. localhost or https)
|
|
// 3. In embedded WebViews, restrictions are often tighter, need active capture..
|
|
// 3. The browser is blocking access to device details for privacy reasons (?)
|
|
// This is most likely transitional, so keep the current device selected until we get a more accurate enumerateDevices.
|
|
return preferredId;
|
|
} else {
|
|
// No preferred, so pick a default.
|
|
return defaultPicker(available);
|
|
}
|
|
}
|
|
return undefined;
|
|
});
|
|
}
|
|
|
|
class AudioInput implements MediaDevice<DeviceLabel, SelectedAudioInputDevice> {
|
|
private logger = rootLogger.getChild("[MediaDevices AudioInput]");
|
|
|
|
private readonly availableRaw$: Behavior<MediaDeviceInfo[]> =
|
|
availableRawDevices$(
|
|
"audioinput",
|
|
this.usingNames$,
|
|
this.scope,
|
|
this.logger,
|
|
);
|
|
|
|
public readonly available$ = this.scope.behavior(
|
|
this.availableRaw$.pipe(map(buildDeviceMap)).pipe(
|
|
tap((available) => {
|
|
const availableIds = Array.from(available.entries())
|
|
.map(([id, label]) => `${id}:${JSON.stringify(label)}`)
|
|
.join(", ");
|
|
this.logger.info(
|
|
"[MediaDevices AudioInput] available devices:",
|
|
availableIds,
|
|
);
|
|
}),
|
|
),
|
|
);
|
|
|
|
public readonly selected$ = this.scope.behavior(
|
|
selectDevice$(this.available$, audioInputSetting.value$).pipe(
|
|
map((id) =>
|
|
id === undefined
|
|
? undefined
|
|
: {
|
|
id,
|
|
// We can identify when the hardware device has changed by watching for
|
|
// changes in the group ID
|
|
hardwareDeviceChange$: this.availableRaw$.pipe(
|
|
map(
|
|
(devices) => devices.find((d) => d.deviceId === id)?.groupId,
|
|
),
|
|
pairwise(),
|
|
filter(([before, after]) => before !== after),
|
|
map(() => undefined),
|
|
),
|
|
},
|
|
),
|
|
),
|
|
);
|
|
|
|
public select(id: string): void {
|
|
audioInputSetting.setValue(id);
|
|
}
|
|
|
|
public constructor(
|
|
private readonly usingNames$: Behavior<boolean>,
|
|
private readonly scope: ObservableScope,
|
|
private readonly initialCallIntent: RTCCallIntent | undefined = undefined,
|
|
private platform: "android" | "ios" | "desktop" | undefined = undefined,
|
|
) {
|
|
this.available$.subscribe((available) => {
|
|
this.logger.info("[audio-input] available devices:", available);
|
|
});
|
|
}
|
|
}
|
|
|
|
class AudioOutput implements MediaDevice<
|
|
AudioOutputDeviceLabel,
|
|
SelectedAudioOutputDevice
|
|
> {
|
|
private logger = rootLogger.getChild("[MediaDevices AudioOutput]");
|
|
public readonly available$ = this.scope.behavior(
|
|
availableRawDevices$(
|
|
"audiooutput",
|
|
this.usingNames$,
|
|
this.scope,
|
|
this.logger,
|
|
).pipe(
|
|
map((availableRaw) => {
|
|
let available: Map<string, AudioOutputDeviceLabel> =
|
|
buildDeviceMap(availableRaw);
|
|
// Create a virtual default audio output for browsers that don't have one.
|
|
// Its device ID must be the empty string because that's what setSinkId
|
|
// recognizes.
|
|
if (available.size && !available.has("") && !available.has("default"))
|
|
available.set("", {
|
|
type: "default",
|
|
name: availableRaw[0]?.label || null,
|
|
});
|
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
const isSafari = !!(window as any).GestureEvent; // non standard api only found on Safari. https://developer.mozilla.org/en-US/docs/Web/API/GestureEvent#browser_compatibility
|
|
if (isSafari) {
|
|
// set to empty map if we are on Safari, because it does not support setSinkId
|
|
available = new Map();
|
|
}
|
|
// Note: creating virtual default input devices would be another problem
|
|
// entirely, because requesting a media stream from deviceId "" won't
|
|
// automatically track the default device.
|
|
return available;
|
|
}),
|
|
),
|
|
);
|
|
|
|
public readonly selected$ = this.scope.behavior(
|
|
selectDevice$(this.available$, audioOutputSetting.value$).pipe(
|
|
map((id) => {
|
|
if (id === undefined) {
|
|
return undefined;
|
|
} else {
|
|
return {
|
|
id,
|
|
virtualEarpiece: false,
|
|
};
|
|
}
|
|
}),
|
|
),
|
|
);
|
|
public select(id: string): void {
|
|
audioOutputSetting.setValue(id);
|
|
}
|
|
|
|
public constructor(
|
|
private readonly usingNames$: Behavior<boolean>,
|
|
private readonly scope: ObservableScope,
|
|
) {
|
|
this.available$.subscribe((available) => {
|
|
this.logger.info("[audio-output] available devices:", available);
|
|
});
|
|
}
|
|
}
|
|
|
|
/**
|
|
* A special implementation of audio output that allows the hosting application
|
|
* to have more control over the device selection process. This is used when the
|
|
* `controlledAudioDevices` URL parameter is set, which is currently only true on mobile.
|
|
*/
|
|
class ControlledAudioOutput implements MediaDevice<
|
|
AudioOutputDeviceLabel,
|
|
SelectedAudioOutputDevice
|
|
> {
|
|
private logger = rootLogger.getChild("[MediaDevices ControlledAudioOutput]");
|
|
// We need to subscribe to the raw devices so that the OS does update the input
|
|
// back to what it was before. otherwise we will switch back to the default
|
|
// whenever we allocate a new stream.
|
|
public readonly availableRaw$ = availableRawDevices$(
|
|
"audiooutput",
|
|
this.usingNames$,
|
|
this.scope,
|
|
this.logger,
|
|
);
|
|
|
|
public readonly available$ = this.scope.behavior(
|
|
combineLatest(
|
|
[controlledAvailableOutputDevices$.pipe(startWith([])), iosDeviceMenu$],
|
|
(availableRaw, iosDeviceMenu) => {
|
|
const available = new Map<string, AudioOutputDeviceLabel>(
|
|
availableRaw.map(
|
|
({ id, name, isEarpiece, isSpeaker /*,isExternalHeadset*/ }) => {
|
|
let deviceLabel: AudioOutputDeviceLabel;
|
|
// if (isExternalHeadset) // Do we want this?
|
|
if (isEarpiece) deviceLabel = { type: "earpiece" };
|
|
else if (isSpeaker) deviceLabel = { type: "speaker" };
|
|
else deviceLabel = { type: "name", name };
|
|
return [id, deviceLabel];
|
|
},
|
|
),
|
|
);
|
|
const availableIds = Array.from(available.entries())
|
|
.map(([id, label]) => `${id}:${JSON.stringify(label)}`)
|
|
.join(", ");
|
|
this.logger.info("available devices:", availableIds);
|
|
|
|
// Create a virtual earpiece device in case a non-earpiece device is
|
|
// designated for this purpose
|
|
if (iosDeviceMenu && availableRaw.some((d) => d.forEarpiece)) {
|
|
this.logger.info(
|
|
`IOS Add virtual earpiece device with id ${EARPIECE_CONFIG_ID}`,
|
|
);
|
|
available.set(EARPIECE_CONFIG_ID, { type: "earpiece" });
|
|
}
|
|
|
|
return available;
|
|
},
|
|
),
|
|
);
|
|
|
|
private readonly deviceSelection$ = new Subject<string>();
|
|
|
|
public select(id: string): void {
|
|
this.logger.info(`select device: ${id}`);
|
|
this.deviceSelection$.next(id);
|
|
}
|
|
|
|
public readonly selected$ = this.scope.behavior(
|
|
combineLatest(
|
|
[
|
|
this.available$,
|
|
merge(
|
|
controlledOutputSelection$.pipe(startWith(undefined)),
|
|
this.deviceSelection$,
|
|
),
|
|
],
|
|
(available, preferredId) => {
|
|
this.logger.debug(
|
|
`selecting device: Preferred:${preferredId}: platform ${this.platform}, intent:${this.initialIntent}: Available: ${Array.from(available.keys()).join(",")}`,
|
|
);
|
|
let id: string | undefined = preferredId;
|
|
if (id === undefined) {
|
|
// No preferred device, so pick a default.
|
|
// The logic for the default is different based on the call type.
|
|
// For example for a voice call we want to default to the earpiece if it's available,
|
|
// but for a video call we want to default to the speaker.
|
|
// If the user is using a BT headset we want to default to that, as it's likely what they want to use for both video and voice calls.
|
|
// The audio routing / selection is also different on iOS and Android, so we need to take that into account as well.
|
|
if (this.platform == "android") {
|
|
this.logger.debug(
|
|
`Android routing logic intent: ${this.initialIntent} finding best default...`,
|
|
);
|
|
if (this.initialIntent === "audio") {
|
|
const systemProposed = available.keys().next().value;
|
|
// If no headset is connected, android will route to the speaker by default,
|
|
// but for a voice call we want to route to the earpiece instead,
|
|
// so override the system proposed routing in that case.
|
|
if (
|
|
systemProposed &&
|
|
available.get(systemProposed)?.type === "speaker"
|
|
) {
|
|
// search for the headset
|
|
const headsetEntry = Array.from(available.entries()).find(
|
|
([_, label]) => label.type === "earpiece",
|
|
);
|
|
if (headsetEntry) {
|
|
this.logger.debug(
|
|
`Android routing: Switch to earpiece instead of speaker for voice call`,
|
|
);
|
|
id = headsetEntry[0];
|
|
} else {
|
|
this.logger.debug(
|
|
`Android routing: no earpiece found, cannot switch, use system proposed routing`,
|
|
);
|
|
id = systemProposed;
|
|
}
|
|
} else {
|
|
this.logger.debug(
|
|
`Android routing: Use system proposed routing`,
|
|
);
|
|
id = systemProposed;
|
|
}
|
|
} else {
|
|
// Use the system best proposed best routing.
|
|
id = available.keys().next().value;
|
|
}
|
|
} else {
|
|
// Pick the first one, based on the assumption that the OS will have already routed to the "best" available output device.
|
|
id = available.keys().next().value;
|
|
}
|
|
}
|
|
|
|
if (id !== undefined) {
|
|
return { id, virtualEarpiece: id === EARPIECE_CONFIG_ID };
|
|
} else {
|
|
return undefined;
|
|
}
|
|
// id = preferredId ?? available.keys().next().value;
|
|
// return id === undefined
|
|
// ? undefined
|
|
// : { id, virtualEarpiece: id === EARPIECE_CONFIG_ID };
|
|
},
|
|
).pipe(
|
|
tap((selected) => {
|
|
this.logger.debug(`selected device: ${selected?.id}`);
|
|
}),
|
|
),
|
|
);
|
|
|
|
public constructor(
|
|
private readonly usingNames$: Behavior<boolean>,
|
|
private readonly scope: ObservableScope,
|
|
private initialIntent: RTCCallIntent | undefined = undefined,
|
|
private platform: "android" | "ios" | "desktop" | undefined = undefined,
|
|
) {
|
|
this.selected$.subscribe((device) => {
|
|
// Let the hosting application know which output device has been selected.
|
|
// This information is probably only of interest if the earpiece mode has
|
|
// been selected - for example, Element X iOS listens to this to determine
|
|
// whether it should enable the proximity sensor.
|
|
if (device !== undefined) {
|
|
this.logger.info("onAudioDeviceSelect called:", device);
|
|
window.controls.onAudioDeviceSelect?.(device.id);
|
|
// Also invoke the deprecated callback for backward compatibility
|
|
window.controls.onOutputDeviceSelect?.(device.id);
|
|
}
|
|
});
|
|
this.available$.subscribe((available) => {
|
|
this.logger.debug("available devices:", available);
|
|
});
|
|
this.availableRaw$.subscribe((availableRaw) => {
|
|
this.logger.debug("available raw devices:", availableRaw);
|
|
});
|
|
}
|
|
}
|
|
|
|
class VideoInput implements MediaDevice<DeviceLabel, SelectedDevice> {
|
|
private logger = rootLogger.getChild("[MediaDevices VideoInput]");
|
|
|
|
public readonly available$ = this.scope.behavior(
|
|
availableRawDevices$(
|
|
"videoinput",
|
|
this.usingNames$,
|
|
this.scope,
|
|
this.logger,
|
|
).pipe(map(buildDeviceMap)),
|
|
);
|
|
public readonly selected$ = this.scope.behavior(
|
|
selectDevice$(this.available$, videoInputSetting.value$).pipe(
|
|
map((id) => (id === undefined ? undefined : { id })),
|
|
),
|
|
);
|
|
public select(id: string): void {
|
|
videoInputSetting.setValue(id);
|
|
}
|
|
|
|
public constructor(
|
|
private readonly usingNames$: Behavior<boolean>,
|
|
private readonly scope: ObservableScope,
|
|
) {
|
|
// This also has the purpose of subscribing to the available devices
|
|
this.available$.subscribe((available) => {
|
|
this.logger.info("[video-input] available devices:", available);
|
|
});
|
|
}
|
|
}
|
|
|
|
export class MediaDevices {
|
|
private readonly deviceNamesRequest$ = new Subject<void>();
|
|
/**
|
|
* Requests that the media devices be populated with the names of each
|
|
* available device, rather than numbered identifiers. This may invoke a
|
|
* permissions pop-up, so it should only be called when there is a clear user
|
|
* intent to view the device list.
|
|
*/
|
|
public requestDeviceNames(): void {
|
|
this.deviceNamesRequest$.next();
|
|
}
|
|
|
|
// Start using device names as soon as requested. This will cause LiveKit to
|
|
// briefly request device permissions and acquire media streams for each
|
|
// device type while calling `enumerateDevices`, which is what browsers want
|
|
// you to do to receive device names in lieu of a more explicit permissions
|
|
// API. This flag never resets to false, because once permissions are granted
|
|
// the first time, the user won't be prompted again until reload of the page.
|
|
private readonly usingNames$ = this.scope.behavior(
|
|
this.deviceNamesRequest$.pipe(map(() => true)),
|
|
false,
|
|
);
|
|
public readonly audioInput: MediaDevice<
|
|
DeviceLabel,
|
|
SelectedAudioInputDevice
|
|
> = new AudioInput(this.usingNames$, this.scope);
|
|
|
|
public readonly audioOutput: MediaDevice<
|
|
AudioOutputDeviceLabel,
|
|
SelectedAudioOutputDevice
|
|
> = getUrlParams().controlledAudioDevices
|
|
? new ControlledAudioOutput(
|
|
this.usingNames$,
|
|
this.scope,
|
|
getUrlParams().callIntent,
|
|
platform,
|
|
)
|
|
: new AudioOutput(this.usingNames$, this.scope);
|
|
|
|
public readonly videoInput: MediaDevice<DeviceLabel, SelectedDevice> =
|
|
new VideoInput(this.usingNames$, this.scope);
|
|
|
|
public constructor(private readonly scope: ObservableScope) {}
|
|
}
|