for the browser developer tool experience/tidyness.
+
+ {tracks.map((trackRef) => (
+
+ ))}
+
+ );
+}
+
+interface StereoPanAudioTrackProps {
+ muted?: boolean;
+ audioContext?: AudioContext;
+ audioNodes: {
+ gain: GainNode;
+ pan: StereoPannerNode;
+ };
+}
+
+/**
+ * This wraps `livekit.AudioTrack` to allow adding audio nodes to a track.
+ * It main purpose is to remount the AudioTrack component when switching from
+ * audiooContext to normal audio playback.
+ * As of now the AudioTrack component does not support adding audio nodes while being mounted.
+ * @param param0
+ * @returns
+ */
+function AudioTrackWithAudioNodes({
+ trackRef,
+ muted,
+ audioContext,
+ audioNodes,
+ ...props
+}: StereoPanAudioTrackProps &
+ AudioTrackProps &
+ React.RefAttributes
): ReactNode {
+ // This is used to unmount/remount the AudioTrack component.
+ // Mounting needs to happen after the audioContext is set.
+ // (adding the audio context when already mounted did not work outside strict mode)
+ const [trackReady, setTrackReady] = useReactiveState(
+ () => false,
+ [audioContext || audioNodes],
+ );
+
+ useEffect(() => {
+ if (!trackRef || trackReady) return;
+ const track = trackRef.publication.track as RemoteAudioTrack;
+ track.setAudioContext(audioContext);
+ track.setWebAudioPlugins(
+ audioContext ? [audioNodes.gain, audioNodes.pan] : [],
+ );
+ setTrackReady(true);
+ }, [audioContext, audioNodes, setTrackReady, trackReady, trackRef]);
+
+ return (
+ trackReady &&
+ );
+}
diff --git a/src/livekit/MediaDevicesContext.tsx b/src/livekit/MediaDevicesContext.tsx
index c2fc63e5..3e30c04d 100644
--- a/src/livekit/MediaDevicesContext.tsx
+++ b/src/livekit/MediaDevicesContext.tsx
@@ -27,11 +27,15 @@ import {
audioOutput as audioOutputSetting,
videoInput as videoInputSetting,
type Setting,
+ alwaysShowIphoneEarpieceSetting,
} from "../settings/settings";
+export const EARPIECE_CONFIG_ID = "earpiece-id";
+
export type DeviceLabel =
| { type: "name"; name: string }
| { type: "number"; number: number }
+ | { type: "earpiece" }
| { type: "default"; name: string | null };
export interface MediaDevice {
@@ -40,6 +44,11 @@ export interface MediaDevice {
*/
available: Map;
selectedId: string | undefined;
+ /**
+ * An additional device configuration that makes us use only one channel of the
+ * output device and a reduced volume.
+ */
+ useAsEarpiece: boolean | undefined;
/**
* The group ID of the selected device.
*/
@@ -65,6 +74,7 @@ function useMediaDevice(
): MediaDevice {
// Make sure we don't needlessly reset to a device observer without names,
// once permissions are already given
+ const [alwaysShowIphoneEarpice] = useSetting(alwaysShowIphoneEarpieceSetting);
const hasRequestedPermissions = useRef(false);
const requestPermissions = usingNames || hasRequestedPermissions.current;
hasRequestedPermissions.current ||= usingNames;
@@ -102,27 +112,39 @@ function useMediaDevice(
// Create a virtual default audio output for browsers that don't have one.
// Its device ID must be the empty string because that's what setSinkId
// recognizes.
+ // We also create this if we do not have any available devices, so that
+ // we can use the default or the earpiece.
+ const showEarpiece =
+ navigator.userAgent.match("iPhone") || alwaysShowIphoneEarpice;
if (
kind === "audiooutput" &&
- available.size &&
!available.has("") &&
- !available.has("default")
+ !available.has("default") &&
+ (available.size || showEarpiece)
)
available = new Map([
["", { type: "default", name: availableRaw[0]?.label || null }],
...available,
]);
+ if (kind === "audiooutput" && showEarpiece)
+ // On IPhones we have to create a virtual earpiece device, because
+ // the earpiece is not available as a device ID.
+ available = new Map([
+ ...available,
+ [EARPIECE_CONFIG_ID, { type: "earpiece" }],
+ ]);
// Note: creating virtual default input devices would be another problem
// entirely, because requesting a media stream from deviceId "" won't
// automatically track the default device.
return available;
}),
),
- [kind, deviceObserver$],
+ [alwaysShowIphoneEarpice, deviceObserver$, kind],
),
);
- const [preferredId, select] = useSetting(setting);
+ const [preferredId, setPreferredId] = useSetting(setting);
+ const [asEarpice, setAsEarpiece] = useState(false);
const selectedId = useMemo(() => {
if (available.size) {
// If the preferred device is available, use it. Or if every available
@@ -138,6 +160,7 @@ function useMediaDevice(
}
return undefined;
}, [available, preferredId]);
+
const selectedGroupId = useObservableEagerState(
useMemo(
() =>
@@ -151,14 +174,27 @@ function useMediaDevice(
),
);
+ const select = useCallback(
+ (id: string) => {
+ if (id === EARPIECE_CONFIG_ID) {
+ setAsEarpiece(true);
+ } else {
+ setAsEarpiece(false);
+ setPreferredId(id);
+ }
+ },
+ [setPreferredId],
+ );
+
return useMemo(
() => ({
available,
selectedId,
+ useAsEarpiece: asEarpice,
selectedGroupId,
select,
}),
- [available, selectedId, selectedGroupId, select],
+ [available, selectedId, asEarpice, selectedGroupId, select],
);
}
@@ -167,6 +203,7 @@ export const deviceStub: MediaDevice = {
selectedId: undefined,
selectedGroupId: undefined,
select: () => {},
+ useAsEarpiece: false,
};
export const devicesStub: MediaDevices = {
audioInput: deviceStub,
@@ -255,3 +292,30 @@ export const useMediaDeviceNames = (
return context.stopUsingDeviceNames;
}
}, [context, enabled]);
+
+/**
+ * A convenience hook to get the audio node configuration for the earpiece.
+ * It will check the `useAsEarpiece` of the `audioOutput` device and return
+ * the appropriate pan and volume values.
+ *
+ * @returns pan and volume values for the earpiece audio node configuration.
+ */
+export const useEarpieceAudioConfig = (): {
+ pan: number;
+ volume: number;
+} => {
+ const { audioOutput } = useMediaDevices();
+ // We use only the right speaker (pan = 1) for the earpiece.
+ // This mimics the behavior of the native earpiece speaker (only the top speaker on an iPhone)
+ const pan = useMemo(
+ () => (audioOutput.useAsEarpiece ? 1 : 0),
+ [audioOutput.useAsEarpiece],
+ );
+ // We also do lower the volume by a factor of 10 to optimize for the usecase where
+ // a user is holding the phone to their ear.
+ const volume = useMemo(
+ () => (audioOutput.useAsEarpiece ? 0.1 : 1),
+ [audioOutput.useAsEarpiece],
+ );
+ return { pan, volume };
+};
diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx
index 768ddfdd..9fa15547 100644
--- a/src/room/InCallView.tsx
+++ b/src/room/InCallView.tsx
@@ -5,11 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
-import {
- RoomAudioRenderer,
- RoomContext,
- useLocalParticipant,
-} from "@livekit/components-react";
+import { RoomContext, useLocalParticipant } from "@livekit/components-react";
import { Text } from "@vector-im/compound-web";
import { ConnectionState, type Room } from "livekit-client";
import { type MatrixClient } from "matrix-js-sdk";
@@ -107,6 +103,7 @@ import {
import { ReactionsReader } from "../reactions/ReactionsReader";
import { ConnectionLostError } from "../utils/errors.ts";
import { useTypedEventEmitter } from "../useEvents.ts";
+import { MatrixAudioRenderer } from "../livekit/MatrixAudioRenderer.tsx";
const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {});
@@ -713,7 +710,10 @@ export const InCallView: FC = ({
)
}
-
+
{renderContent()}
diff --git a/src/settings/DeveloperSettingsTab.tsx b/src/settings/DeveloperSettingsTab.tsx
index fdeaa704..36df5c39 100644
--- a/src/settings/DeveloperSettingsTab.tsx
+++ b/src/settings/DeveloperSettingsTab.tsx
@@ -18,6 +18,7 @@ import {
useNewMembershipManager as useNewMembershipManagerSetting,
useExperimentalToDeviceTransport as useExperimentalToDeviceTransportSetting,
muteAllAudio as muteAllAudioSetting,
+ alwaysShowIphoneEarpiece as alwaysShowIphoneEarpieceSetting,
} from "./settings";
import type { MatrixClient } from "matrix-js-sdk";
import type { Room as LivekitRoom } from "livekit-client";
@@ -46,6 +47,9 @@ export const DeveloperSettingsTab: FC = ({ client, livekitRoom }) => {
useNewMembershipManagerSetting,
);
+ const [alwaysShowIphoneEarpiece, setAlwaysShowIphoneEarpiece] = useSetting(
+ alwaysShowIphoneEarpieceSetting,
+ );
const [
useExperimentalToDeviceTransport,
setUseExperimentalToDeviceTransport,
@@ -192,6 +196,20 @@ export const DeveloperSettingsTab: FC = ({ client, livekitRoom }) => {
[setMuteAllAudio],
)}
/>
+ {" "}
+
+ ): void => {
+ setAlwaysShowIphoneEarpiece(event.target.checked);
+ },
+ [setAlwaysShowIphoneEarpiece],
+ )}
+ />{" "}
{livekitRoom ? (
<>
diff --git a/src/settings/DeviceSelection.tsx b/src/settings/DeviceSelection.tsx
index 0bdabbe7..12e89f56 100644
--- a/src/settings/DeviceSelection.tsx
+++ b/src/settings/DeviceSelection.tsx
@@ -22,17 +22,20 @@ import {
} from "@vector-im/compound-web";
import { Trans, useTranslation } from "react-i18next";
-import { type MediaDevice } from "../livekit/MediaDevicesContext";
+import {
+ EARPIECE_CONFIG_ID,
+ type MediaDevice,
+} from "../livekit/MediaDevicesContext";
import styles from "./DeviceSelection.module.css";
interface Props {
- devices: MediaDevice;
+ device: MediaDevice;
title: string;
numberedLabel: (number: number) => string;
}
export const DeviceSelection: FC = ({
- devices,
+ device,
title,
numberedLabel,
}) => {
@@ -40,12 +43,13 @@ export const DeviceSelection: FC = ({
const groupId = useId();
const onChange = useCallback(
(e: ChangeEvent) => {
- devices.select(e.target.value);
+ device.select(e.target.value);
},
- [devices],
+ [device],
);
- if (devices.available.size == 0) return null;
+ // There is no need to show the menu if there is no choice that can be made.
+ if (device.available.size == 1) return null;
return (
@@ -60,7 +64,7 @@ export const DeviceSelection: FC
= ({
- {[...devices.available].map(([id, label]) => {
+ {[...device.available].map(([id, label]) => {
let labelText: ReactNode;
switch (label.type) {
case "name":
@@ -85,6 +89,16 @@ export const DeviceSelection: FC
= ({
);
break;
+ case "earpiece":
+ labelText = t("settings.devices.earpiece");
+ break;
+ }
+
+ let isSelected = false;
+ if (device.useAsEarpiece) {
+ isSelected = id === EARPIECE_CONFIG_ID;
+ } else {
+ isSelected = id === device.selectedId;
}
return (
@@ -93,7 +107,7 @@ export const DeviceSelection: FC = ({
name={groupId}
control={
diff --git a/src/settings/SettingsModal.tsx b/src/settings/SettingsModal.tsx
index b0a4b79e..1c97a87d 100644
--- a/src/settings/SettingsModal.tsx
+++ b/src/settings/SettingsModal.tsx
@@ -98,7 +98,6 @@ export const SettingsModal: FC = ({
useMediaDeviceNames(devices, open);
const [soundVolume, setSoundVolume] = useSetting(soundEffectVolumeSetting);
const [soundVolumeRaw, setSoundVolumeRaw] = useState(soundVolume);
-
const [showDeveloperSettingsTab] = useSetting(developerMode);
const { available: isRageshakeAvailable } = useSubmitRageshake();
@@ -110,17 +109,18 @@ export const SettingsModal: FC = ({
<>