Merge branch 'livekit' into hughns/settings-simplification

This commit is contained in:
Hugh Nimmo-Smith
2024-12-09 14:56:51 +00:00
28 changed files with 1019 additions and 473 deletions

View File

@@ -28,7 +28,7 @@ jobs:
uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4
with:
cache: "yarn"
node-version: "lts/*"
node-version-file: ".node-version"
- name: Install dependencies
run: "yarn install"
- name: Build

View File

@@ -12,7 +12,7 @@ jobs:
uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4
with:
cache: "yarn"
node-version: "lts/*"
node-version-file: ".node-version"
- name: Install dependencies
run: "yarn install"
- name: Prettier

View File

@@ -14,7 +14,7 @@ jobs:
uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4
with:
cache: "yarn"
node-version: "lts/*"
node-version-file: ".node-version"
- name: Install dependencies
run: "yarn install"
- name: Vitest

View File

@@ -18,7 +18,7 @@ jobs:
- uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4
with:
cache: "yarn"
node-version: "lts/*"
node-version-file: ".node-version"
- name: Install Deps
run: "yarn install --frozen-lockfile"

1
.node-version Normal file
View File

@@ -0,0 +1 @@
22

View File

@@ -17,5 +17,9 @@
},
"rageshake": {
"submit_url": "https://element.io/bugreports/submit"
},
"sentry": {
"environment": "netlify-pr-preview",
"DSN": "https://b1e328d49be3402ba96101338989fb35@sentry.tools.element.io/41"
}
}

View File

@@ -69,9 +69,10 @@
"video": "Video"
},
"developer_mode": {
"crypto_version": "Krypto-Version:{{version}}",
"crypto_version": "Krypto-Version: {{version}}",
"device_id": "Geräte-ID: {{id}}",
"duplicate_tiles_label": "Anzahl zusätzlicher Kachelkopien pro Teilnehmer",
"hostname": "Hostname: {{hostname}}",
"matrix_id": "Matrix-ID: {{id}}"
},
"disconnected_banner": "Die Verbindung zum Server wurde getrennt.",
@@ -146,7 +147,6 @@
"screenshare_button_label": "Bildschirm teilen",
"settings": {
"audio_tab": {
"effect_volume_description": "Lautstärke anpassen, mit der Reaktionen und Handmeldungen abgespielt werden",
"effect_volume_label": "Lautstärke der Soundeffekte"
},
"developer_settings_label": "Entwicklereinstellungen",
@@ -193,6 +193,7 @@
"expand": "Erweitern",
"mute_for_me": "Für mich stumm schalten",
"muted_for_me": "Für mich stumm geschaltet",
"volume": "Lautstärke"
"volume": "Lautstärke",
"waiting_for_media": "Warten auf Medien..."
}
}

View File

@@ -147,7 +147,7 @@
"screenshare_button_label": "Share screen",
"settings": {
"audio_tab": {
"effect_volume_description": "Adjust the volume at which reactions and hand raised effects play",
"effect_volume_description": "Volume for sound effects such as: joining or leaving a call, and reactions.",
"effect_volume_label": "Sound effect volume"
},
"developer_tab_title": "Developer",

195
locales/ro/app.json Normal file
View File

@@ -0,0 +1,195 @@
{
"a11y": {
"user_menu": "Meniul utilizatorului"
},
"action": {
"close": "Închide",
"copy_link": "Copiază linkul",
"edit": "Editare",
"go": "Du-te",
"invite": "Invită",
"lower_hand": "Mâna inferioară",
"no": "No",
"pick_reaction": "Alegeți reacția",
"raise_hand": "Ridicați mâna",
"register": "Inregistrare",
"remove": "elimina",
"show_less": "Arată mai puțin",
"show_more": "Arată mai mult",
"sign_in": "Autentificare",
"sign_out": "Sign out",
"submit": "Trimiteți",
"upload_file": "Încărcați fișierul"
},
"analytics_notice": "Prin participarea la această versiune beta, sunteți de acord cu colectarea de date anonime, pe care le folosim pentru a îmbunătăți produsul. Puteți găsi mai multe informații despre datele pe care le urmărim în Politica noastră de <2> confidențialitate </2> și Politica noastră <6> privind cookie-urile</6>.",
"app_selection_modal": {
"continue_in_browser": "Continuați în browser",
"open_in_app": "Deschideți în aplicație",
"text": "Sunteți gata să vă alăturați?",
"title": "Selectați aplicația"
},
"application_opened_another_tab": "Această aplicație a fost deschisă într-o altă filă.",
"browser_media_e2ee_unsupported": "Browserul dvs. web nu acceptă criptarea media end-to-end. Browserele acceptate sunt Chrome, Safari, Firefox > = 117",
"browser_media_e2ee_unsupported_heading": "Browser incompatibil",
"call_ended_view": {
"body": "Ai fost deconectat de la apel",
"create_account_button": "Creează cont",
"create_account_prompt": "<0>De ce să nu terminați prin configurarea unei parole pentru a vă păstra contul? </0><1>Veți putea să vă păstrați numele și să setați un avatar pentru a fi utilizat la apelurile viitoare </1>",
"feedback_done": "<0>Vă mulțumim pentru feedback! </0>",
"feedback_prompt": "<0>Ne-ar plăcea să auzim feedback-ul dvs., astfel încât să vă putem îmbunătăți experiența. </0>",
"headline": "{{displayName}}, apelul tău s-a încheiat.",
"not_now_button": "Nu acum, reveniți la ecranul de pornire",
"reconnect_button": "Reconecta",
"survey_prompt": "Cum a mers?"
},
"call_name": "Numele apelului",
"common": {
"analytics": "Analiză",
"audio": "Audio",
"avatar": "avatar",
"back": "Înapoi",
"camera": "Aparat foto",
"display_name": "Nume afișat",
"encrypted": "Criptat",
"error": "Eroare",
"home": "Acasa",
"loading": "Se încarcă...",
"microphone": "Microfon",
"next": "Urmator\n",
"options": "Opțiuni",
"password": "Parolă",
"preferences": "preferinte",
"profile": "Profil",
"reaction": "Reacție",
"reactions": "Reacții",
"settings": "Settings",
"something_went_wrong": "Ceva nu a mers bine",
"unencrypted": "Nu este criptat",
"username": "Nume utilizator",
"video": "Videoclip"
},
"developer_mode": {
"crypto_version": "Versiunea Crypto: {{version}}",
"device_id": "ID-ul dispozitivului: {{id}}",
"duplicate_tiles_label": "Numărul de exemplare suplimentare de cartonașe per participant",
"hostname": "Numele gazdei: {{hostname}}",
"matrix_id": "ID-ul matricei: {{id}}"
},
"disconnected_banner": "Conectivitatea la server a fost pierdută.",
"full_screen_view_description": "<0>Trimiterea jurnalelor de depanare ne va ajuta să urmărim problema. </0>",
"full_screen_view_h1": "<0>Hopa, ceva nu a mers bine. </0>",
"group_call_loader": {
"banned_body": "Ai fost interzis să ieși din cameră.",
"banned_heading": "Interzis",
"call_ended_body": "Ați fost eliminat din apel.",
"call_ended_heading": "Apel încheiat",
"failed_heading": "Nu s-a putut alătura",
"failed_text": "Apelul nu a fost găsit sau nu este accesibil.",
"knock_reject_body": "Cererea dvs. de a vă alătura a fost respinsă.",
"knock_reject_heading": "Acces refuzat",
"reason": "Motivul"
},
"hangup_button_label": "Încheiați apelul",
"header_label": "Element Call Home",
"header_participants_label": "Participanți",
"invite_modal": {
"link_copied_toast": "Link copiat în clipboard",
"title": "Invitați la acest apel"
},
"join_existing_call_modal": {
"join_button": "Da, alăturați-vă apelului",
"text": "Acest apel există deja, doriți să vă alăturați?",
"title": "Alăturați-vă apelului existent?"
},
"layout_grid_label": "GRILĂ",
"layout_spotlight_label": "Spotlight",
"lobby": {
"ask_to_join": "Solicitare de participare la apel",
"join_as_guest": "Alăturați-vă ca invitat",
"join_button": "Alăturați-vă apelului",
"leave_button": "Înapoi la cele mai recente",
"waiting_for_invite": "Solicitare trimisă! În așteptarea permisiunii de a participa..."
},
"log_in": "Autentificare",
"logging_in": "Autentificare...",
"login_auth_links": "<0>Creați un cont </0> sau <2> accesați ca invitat </2>",
"login_auth_links_prompt": "Nu sunteți încă înregistrat?",
"login_subheading": "Pentru a continua la Element",
"login_title": "Logare",
"microphone_off": "Microfon oprit",
"microphone_on": "Microfon pornit",
"mute_microphone_button_label": "Dezactivați microfonul",
"qr_code": "COD QR",
"rageshake_button_error_caption": "Încearcă din nou trimiterea jurnalelor",
"rageshake_request_modal": {
"body": "Un alt utilizator al acestui apel are o problemă. Pentru a diagnostica mai bine aceste probleme, am dori să colectăm un jurnal de depanare.",
"title": "Solicitare jurnal de depanare"
},
"rageshake_send_logs": "Trimiteți jurnale de depanare",
"rageshake_sending": "Trimiterea...",
"rageshake_sending_logs": "Trimiterea jurnalelor de depanare...",
"rageshake_sent": "Multumesc!",
"recaptcha_caption": "Acest site este protejat de reCAPTCHA și se aplică Politica de <2> confidențialitate Google </2> și <6> Termenii și condițiile. </6> <9></9>Făcând clic pe „Înregistrare”, sunteți de acord cu Acordul nostru de licențiere pentru utilizatorul <12> final (EULA) </12>",
"recaptcha_dismissed": "Recaptcha a fost respins",
"recaptcha_not_loaded": "Recaptcha nu a fost încărcat",
"register": {
"passwords_must_match": "Parolele trebuie să se potrivească",
"registering": "Înregistrare..."
},
"register_auth_links": "<0>Ai deja un cont? </0><1><0>Conectați-vă </0> sau <2> accesați ca invitat </2> </1>",
"register_confirm_password_label": "Confirmă Parola",
"register_heading": "Creează-ți contul",
"return_home_button": "Reveniți la ecranul de pornire",
"room_auth_view_continue_button": "Continuă",
"room_auth_view_eula_caption": "Făcând clic pe „Continuați”, sunteți de acord cu Acordul nostru de licențiere pentru utilizatorul <2> final (EULA) </2>",
"screenshare_button_label": "Partajare ecran",
"settings": {
"audio_tab": {
"effect_volume_description": "Reglați volumul la care reacționează reacțiile și efectele ridicate de mână",
"effect_volume_label": "Volumul efectului sonor"
},
"developer_settings_label": "Setări pentru dezvoltatori",
"developer_settings_label_description": "Expuneți setările dezvoltatorului în fereastra de setări.",
"developer_tab_title": "dezvoltator",
"feedback_tab_body": "Dacă întâmpinați probleme sau pur și simplu doriți să oferiți feedback, vă rugăm să ne trimiteți o scurtă descriere mai jos.",
"feedback_tab_description_label": "Feedback-ul tău",
"feedback_tab_h4": "Trimiteți Feedback",
"feedback_tab_send_logs_label": "Includeți jurnale de depanare",
"feedback_tab_thank_you": "Vă mulțumim, am primit feedback-ul dvs.!",
"feedback_tab_title": "Feedback",
"more_tab_title": "Mai mult",
"opt_in_description": "<0></0><1></1>Puteți retrage consimțământul debifând această casetă. Dacă sunteți în prezent la un apel, această setare va intra în vigoare la sfârșitul apelului.",
"preferences_tab": {
"reactions_play_sound_description": "Redați un efect sonor atunci când cineva trimite o reacție la un apel.",
"reactions_play_sound_label": "Redați sunete de reacție",
"reactions_show_description": "Afișați o animație atunci când cineva trimite o reacție.",
"reactions_show_label": "Afișați reacțiile",
"reactions_title": "Reacții"
},
"preferences_tab_body": "Aici puteți configura opțiuni suplimentare pentru o experiență îmbunătățită",
"preferences_tab_h4": "preferinte",
"preferences_tab_show_hand_raised_timer_description": "Afișați un cronometru atunci când un participant ridică mâna",
"preferences_tab_show_hand_raised_timer_label": "Afișați durata ridicării mâinii",
"speaker_device_selection_label": "vorbitor"
},
"start_new_call": "Începe un nou apel",
"start_video_button_label": "Începeți videoclipul",
"stop_screenshare_button_label": "Partajarea ecranului",
"stop_video_button_label": "Opriți videoclipul",
"submitting": "Trimiterea...",
"switch_camera": "Comutați camera",
"unauthenticated_view_body": "Nu sunteți încă înregistrat? <2>Creați un cont </2>",
"unauthenticated_view_eula_caption": "Făcând clic pe „Go”, sunteți de acord cu Acordul nostru de licențiere pentru utilizatorul <2> final (EULA) </2>",
"unauthenticated_view_login_button": "Conectați-vă la contul dvs.",
"unmute_microphone_button_label": "Anulează microfonul",
"version": "{{productName}}Versiune: {{version}}",
"video_tile": {
"always_show": "Arată întotdeauna",
"change_fit_contain": "Se potrivește cadrului",
"collapse": "colaps",
"expand": "Extindeți",
"mute_for_me": "Mute pentru mine",
"muted_for_me": "Dezactivat pentru mine",
"volume": "VOLUM"
}
}

156
src/Avatar.test.tsx Normal file
View File

@@ -0,0 +1,156 @@
/*
Copyright 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { afterEach, expect, test, vi } from "vitest";
import { render, screen } from "@testing-library/react";
import { MatrixClient } from "matrix-js-sdk/src/client";
import { FC, PropsWithChildren } from "react";
import { ClientContextProvider } from "./ClientContext";
import { Avatar } from "./Avatar";
import { mockMatrixRoomMember, mockRtcMembership } from "./utils/test";
const TestComponent: FC<
PropsWithChildren<{ client: MatrixClient; supportsThumbnails?: boolean }>
> = ({ client, children, supportsThumbnails }) => {
return (
<ClientContextProvider
value={{
state: "valid",
disconnected: false,
supportedFeatures: {
reactions: true,
thumbnails: supportsThumbnails ?? true,
},
setClient: vi.fn(),
authenticated: {
client,
isPasswordlessUser: true,
changePassword: vi.fn(),
logout: vi.fn(),
},
}}
>
{children}
</ClientContextProvider>
);
};
afterEach(() => {
vi.unstubAllGlobals();
});
test("should just render a placeholder when the user has no avatar", () => {
const client = vi.mocked<MatrixClient>({
getAccessToken: () => "my-access-token",
mxcUrlToHttp: () => vi.fn(),
} as unknown as MatrixClient);
vi.spyOn(client, "mxcUrlToHttp");
const member = mockMatrixRoomMember(
mockRtcMembership("@alice:example.org", "AAAA"),
{
getMxcAvatarUrl: () => undefined,
},
);
const displayName = "Alice";
render(
<TestComponent client={client}>
<Avatar
id={member.userId}
name={displayName}
size={96}
src={member.getMxcAvatarUrl()}
/>
</TestComponent>,
);
const element = screen.getByRole("img", { name: "@alice:example.org" });
expect(element.tagName).toEqual("SPAN");
expect(client.mxcUrlToHttp).toBeCalledTimes(0);
});
test("should just render a placeholder when thumbnails are not supported", () => {
const client = vi.mocked<MatrixClient>({
getAccessToken: () => "my-access-token",
mxcUrlToHttp: () => vi.fn(),
} as unknown as MatrixClient);
vi.spyOn(client, "mxcUrlToHttp");
const member = mockMatrixRoomMember(
mockRtcMembership("@alice:example.org", "AAAA"),
{
getMxcAvatarUrl: () => "mxc://example.org/alice-avatar",
},
);
const displayName = "Alice";
render(
<TestComponent client={client} supportsThumbnails={false}>
<Avatar
id={member.userId}
name={displayName}
size={96}
src={member.getMxcAvatarUrl()}
/>
</TestComponent>,
);
const element = screen.getByRole("img", { name: "@alice:example.org" });
expect(element.tagName).toEqual("SPAN");
expect(client.mxcUrlToHttp).toBeCalledTimes(0);
});
test("should attempt to fetch authenticated media", async () => {
const expectedAuthUrl = "http://example.org/media/alice-avatar";
const expectedObjectURL = "my-object-url";
const accessToken = "my-access-token";
const theBlob = new Blob([]);
// vitest doesn't have a implementation of create/revokeObjectURL, so we need
// to delete the property. It's a bit odd, but it works.
Reflect.deleteProperty(global.window.URL, "createObjectURL");
globalThis.URL.createObjectURL = vi.fn().mockReturnValue(expectedObjectURL);
Reflect.deleteProperty(global.window.URL, "revokeObjectURL");
globalThis.URL.revokeObjectURL = vi.fn();
const fetchFn = vi.fn().mockResolvedValue({
blob: async () => Promise.resolve(theBlob),
});
vi.stubGlobal("fetch", fetchFn);
const client = vi.mocked<MatrixClient>({
getAccessToken: () => accessToken,
mxcUrlToHttp: () => vi.fn(),
} as unknown as MatrixClient);
vi.spyOn(client, "mxcUrlToHttp").mockReturnValue(expectedAuthUrl);
const member = mockMatrixRoomMember(
mockRtcMembership("@alice:example.org", "AAAA"),
{
getMxcAvatarUrl: () => "mxc://example.org/alice-avatar",
},
);
const displayName = "Alice";
render(
<TestComponent client={client}>
<Avatar
id={member.userId}
name={displayName}
size={96}
src={member.getMxcAvatarUrl()}
/>
</TestComponent>,
);
// Fetch is asynchronous, so wait for this to resolve.
await vi.waitUntil(() =>
document.querySelector(`img[src='${expectedObjectURL}']`),
);
expect(client.mxcUrlToHttp).toBeCalledTimes(1);
expect(globalThis.fetch).toBeCalledWith(expectedAuthUrl, {
headers: { Authorization: `Bearer ${accessToken}` },
});
});

View File

@@ -5,11 +5,11 @@ SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { useMemo, FC, CSSProperties } from "react";
import { useMemo, FC, CSSProperties, useState, useEffect } from "react";
import { Avatar as CompoundAvatar } from "@vector-im/compound-web";
import { MatrixClient } from "matrix-js-sdk/src/client";
import { getAvatarUrl } from "./utils/matrix";
import { useClient } from "./ClientContext";
import { useClientState } from "./ClientContext";
export enum Size {
XS = "xs",
@@ -36,6 +36,28 @@ interface Props {
style?: CSSProperties;
}
export function getAvatarUrl(
client: MatrixClient,
mxcUrl: string | null,
avatarSize = 96,
): string | null {
const width = Math.floor(avatarSize * window.devicePixelRatio);
const height = Math.floor(avatarSize * window.devicePixelRatio);
// scale is more suitable for larger sizes
const resizeMethod = avatarSize <= 96 ? "crop" : "scale";
return mxcUrl
? client.mxcUrlToHttp(
mxcUrl,
width,
height,
resizeMethod,
false,
true,
true,
)
: null;
}
export const Avatar: FC<Props> = ({
className,
id,
@@ -45,7 +67,7 @@ export const Avatar: FC<Props> = ({
style,
...props
}) => {
const { client } = useClient();
const clientState = useClientState();
const sizePx = useMemo(
() =>
@@ -55,10 +77,50 @@ export const Avatar: FC<Props> = ({
[size],
);
const resolvedSrc = useMemo(() => {
if (!client || !src || !sizePx) return undefined;
return src.startsWith("mxc://") ? getAvatarUrl(client, src, sizePx) : src;
}, [client, src, sizePx]);
const [avatarUrl, setAvatarUrl] = useState<string | undefined>(undefined);
useEffect(() => {
if (clientState?.state !== "valid") {
return;
}
const { authenticated, supportedFeatures } = clientState;
const client = authenticated?.client;
if (!client || !src || !sizePx || !supportedFeatures.thumbnails) {
return;
}
const token = client.getAccessToken();
if (!token) {
return;
}
const resolveSrc = getAvatarUrl(client, src, sizePx);
if (!resolveSrc) {
setAvatarUrl(undefined);
return;
}
let objectUrl: string | undefined;
fetch(resolveSrc, {
headers: {
Authorization: `Bearer ${token}`,
},
})
.then(async (req) => req.blob())
.then((blob) => {
objectUrl = URL.createObjectURL(blob);
setAvatarUrl(objectUrl);
})
.catch((ex) => {
setAvatarUrl(undefined);
});
return (): void => {
if (objectUrl) {
URL.revokeObjectURL(objectUrl);
}
};
}, [clientState, src, sizePx]);
return (
<CompoundAvatar
@@ -66,7 +128,7 @@ export const Avatar: FC<Props> = ({
id={id}
name={name}
size={`${sizePx}px`}
src={resolvedSrc}
src={avatarUrl}
style={style}
{...props}
/>

View File

@@ -48,6 +48,7 @@ export type ValidClientState = {
disconnected: boolean;
supportedFeatures: {
reactions: boolean;
thumbnails: boolean;
};
setClient: (params?: SetClientParams) => void;
};
@@ -71,6 +72,8 @@ export type SetClientParams = {
const ClientContext = createContext<ClientState | undefined>(undefined);
export const ClientContextProvider = ClientContext.Provider;
export const useClientState = (): ClientState | undefined =>
useContext(ClientContext);
@@ -253,6 +256,7 @@ export const ClientProvider: FC<Props> = ({ children }) => {
const [isDisconnected, setIsDisconnected] = useState(false);
const [supportsReactions, setSupportsReactions] = useState(false);
const [supportsThumbnails, setSupportsThumbnails] = useState(false);
const state: ClientState | undefined = useMemo(() => {
if (alreadyOpenedErr) {
@@ -278,6 +282,7 @@ export const ClientProvider: FC<Props> = ({ children }) => {
disconnected: isDisconnected,
supportedFeatures: {
reactions: supportsReactions,
thumbnails: supportsThumbnails,
},
};
}, [
@@ -288,6 +293,7 @@ export const ClientProvider: FC<Props> = ({ children }) => {
setClient,
isDisconnected,
supportsReactions,
supportsThumbnails,
]);
const onSync = useCallback(
@@ -313,6 +319,8 @@ export const ClientProvider: FC<Props> = ({ children }) => {
}
if (initClientState.widgetApi) {
// There is currently no widget API for authenticated media thumbnails.
setSupportsThumbnails(false);
const reactSend = initClientState.widgetApi.hasCapability(
"org.matrix.msc2762.send.event:m.reaction",
);
@@ -334,6 +342,7 @@ export const ClientProvider: FC<Props> = ({ children }) => {
}
} else {
setSupportsReactions(true);
setSupportsThumbnails(true);
}
return (): void => {

View File

@@ -16,6 +16,9 @@ interface Props {
className?: string;
label: string;
value: number;
/**
* Event handler called when the value changes during an interaction.
*/
onValueChange: (value: number) => void;
/**
* Event handler called when the value changes at the end of an interaction.

View File

@@ -115,12 +115,12 @@ function useMediaDevice(
}, [available, preferredId, select, alwaysDefault]);
}
const deviceStub: MediaDevice = {
export const deviceStub: MediaDevice = {
available: [],
selectedId: undefined,
select: () => {},
};
const devicesStub: MediaDevices = {
export const devicesStub: MediaDevices = {
audioInput: deviceStub,
audioOutput: deviceStub,
videoInput: deviceStub,

View File

@@ -6,25 +6,30 @@ Please see LICENSE in the repository root for full details.
*/
import { render } from "@testing-library/react";
import { beforeEach, expect, test } from "vitest";
import {
afterAll,
beforeEach,
expect,
MockedFunction,
test,
vitest,
} from "vitest";
import { MatrixClient } from "matrix-js-sdk/src/client";
import { ConnectionState, Room } from "livekit-client";
import { ConnectionState } from "livekit-client";
import { BehaviorSubject, of } from "rxjs";
import { afterEach } from "node:test";
import { act } from "react";
import { act, ReactNode } from "react";
import {
CallMembership,
type MatrixRTCSession,
} from "matrix-js-sdk/src/matrixrtc";
import { RoomMember } from "matrix-js-sdk/src/matrix";
import { soundEffectVolumeSetting } from "../settings/settings";
import {
EmittableMockLivekitRoom,
mockLivekitRoom,
mockLocalParticipant,
mockMatrixRoom,
mockMatrixRoomMember,
mockMediaPlay,
mockRemoteParticipant,
mockRtcMembership,
MockRTCSession,
@@ -35,37 +40,69 @@ import {
CallEventAudioRenderer,
MAX_PARTICIPANT_COUNT_FOR_SOUND,
} from "./CallEventAudioRenderer";
import { useAudioContext } from "../useAudioContext";
import { TestReactionsWrapper } from "../utils/testReactions";
import { prefetchSounds } from "../soundUtils";
const localRtcMember = mockRtcMembership("@carol:example.org", "CCCC");
const local = mockMatrixRoomMember(localRtcMember);
const aliceRtcMember = mockRtcMembership("@alice:example.org", "AAAA");
const alice = mockMatrixRoomMember(aliceRtcMember);
const bobRtcMember = mockRtcMembership("@bob:example.org", "BBBB");
const bob = mockMatrixRoomMember(bobRtcMember);
const localParticipant = mockLocalParticipant({ identity: "" });
const aliceId = `${alice.userId}:${aliceRtcMember.deviceId}`;
const bobId = `${bob.userId}:${bobRtcMember.deviceId}`;
const aliceParticipant = mockRemoteParticipant({ identity: aliceId });
const bobParticipant = mockRemoteParticipant({ identity: bobId });
const originalPlayFn = window.HTMLMediaElement.prototype.play;
const enterSound = "http://localhost:3000/src/sound/join_call.ogg";
const leaveSound = "http://localhost:3000/src/sound/left_call.ogg";
beforeEach(() => {
soundEffectVolumeSetting.setValue(soundEffectVolumeSetting.defaultValue);
});
vitest.mock("../useAudioContext");
vitest.mock("../soundUtils");
afterEach(() => {
window.HTMLMediaElement.prototype.play = originalPlayFn;
vitest.resetAllMocks();
});
test("plays a sound when entering a call", () => {
const audioIsPlaying: string[] = mockMediaPlay();
const matrixRoomMembers = new Map(
[local, alice, bob].map((p) => [p.userId, p]),
afterAll(() => {
vitest.restoreAllMocks();
});
let playSound: MockedFunction<
NonNullable<ReturnType<typeof useAudioContext>>["playSound"]
>;
beforeEach(() => {
(prefetchSounds as MockedFunction<typeof prefetchSounds>).mockResolvedValue({
sound: new ArrayBuffer(0),
});
playSound = vitest.fn();
(useAudioContext as MockedFunction<typeof useAudioContext>).mockReturnValue({
playSound,
});
});
function TestComponent({
rtcSession,
vm,
}: {
rtcSession: MockRTCSession;
vm: CallViewModel;
}): ReactNode {
return (
<TestReactionsWrapper
rtcSession={rtcSession as unknown as MatrixRTCSession}
>
<CallEventAudioRenderer vm={vm} />
</TestReactionsWrapper>
);
}
function getMockEnv(
members: RoomMember[],
initialRemoteRtcMemberships: CallMembership[] = [aliceRtcMember],
): {
vm: CallViewModel;
session: MockRTCSession;
remoteRtcMemberships: BehaviorSubject<CallMembership[]>;
} {
const matrixRoomMembers = new Map(members.map((p) => [p.userId, p]));
const remoteParticipants = of([aliceParticipant]);
const liveKitRoom = mockLivekitRoom(
{ localParticipant },
@@ -75,221 +112,90 @@ test("plays a sound when entering a call", () => {
client: {
getUserId: () => localRtcMember.sender,
getDeviceId: () => localRtcMember.deviceId,
on: vitest.fn(),
off: vitest.fn(),
} as Partial<MatrixClient> as MatrixClient,
getMember: (userId) => matrixRoomMembers.get(userId) ?? null,
});
const session = new MockRTCSession(matrixRoom, localRtcMember, [
aliceRtcMember,
]) as unknown as MatrixRTCSession;
const vm = new CallViewModel(
session,
liveKitRoom,
{
kind: E2eeType.PER_PARTICIPANT,
},
of(ConnectionState.Connected),
const remoteRtcMemberships = new BehaviorSubject<CallMembership[]>(
initialRemoteRtcMemberships,
);
render(<CallEventAudioRenderer vm={vm} />);
expect(audioIsPlaying).toEqual([
// Joining the call
enterSound,
]);
});
test("plays no sound when muted", () => {
soundEffectVolumeSetting.setValue(0);
const audioIsPlaying: string[] = mockMediaPlay();
const matrixRoomMembers = new Map(
[local, alice, bob].map((p) => [p.userId, p]),
);
const remoteParticipants = of([aliceParticipant, bobParticipant]);
const liveKitRoom = mockLivekitRoom(
{ localParticipant },
{ remoteParticipants },
);
const matrixRoom = mockMatrixRoom({
client: {
getUserId: () => localRtcMember.sender,
getDeviceId: () => localRtcMember.deviceId,
} as Partial<MatrixClient> as MatrixClient,
getMember: (userId) => matrixRoomMembers.get(userId) ?? null,
});
const session = new MockRTCSession(matrixRoom, localRtcMember, [
aliceRtcMember,
]) as unknown as MatrixRTCSession;
const vm = new CallViewModel(
session,
liveKitRoom,
{
kind: E2eeType.PER_PARTICIPANT,
},
of(ConnectionState.Connected),
);
render(<CallEventAudioRenderer vm={vm} />);
// Play a sound when joining a call.
expect(audioIsPlaying).toHaveLength(0);
});
test("plays a sound when a user joins", () => {
const audioIsPlaying: string[] = mockMediaPlay();
const matrixRoomMembers = new Map([local, alice].map((p) => [p.userId, p]));
const remoteParticipants = new Map(
[aliceParticipant].map((p) => [p.identity, p]),
);
const liveKitRoom = new EmittableMockLivekitRoom({
localParticipant,
remoteParticipants,
});
const matrixRoom = mockMatrixRoom({
client: {
getUserId: () => localRtcMember.sender,
getDeviceId: () => localRtcMember.deviceId,
} as Partial<MatrixClient> as MatrixClient,
getMember: (userId) => matrixRoomMembers.get(userId) ?? null,
});
const remoteRtcMemberships = new BehaviorSubject<CallMembership[]>([
aliceRtcMember,
]);
// we give Bob an RTC session now, but no participant yet
const session = new MockRTCSession(
matrixRoom,
localRtcMember,
).withMemberships(
remoteRtcMemberships.asObservable(),
) as unknown as MatrixRTCSession;
).withMemberships(remoteRtcMemberships);
const vm = new CallViewModel(
session,
liveKitRoom as unknown as Room,
session as unknown as MatrixRTCSession,
liveKitRoom,
{
kind: E2eeType.PER_PARTICIPANT,
},
of(ConnectionState.Connected),
);
render(<CallEventAudioRenderer vm={vm} />);
return { vm, session, remoteRtcMemberships };
}
/**
* We don't want to play a sound when loading the call state
* because typically this occurs in two stages. We first join
* the call as a local participant and *then* the remote
* participants join from our perspective. We don't want to make
* a noise every time.
*/
test("plays one sound when entering a call", () => {
const { session, vm, remoteRtcMemberships } = getMockEnv([local, alice]);
render(<TestComponent rtcSession={session} vm={vm} />);
// Joining a call usually means remote participants are added later.
act(() => {
remoteRtcMemberships.next([aliceRtcMember, bobRtcMember]);
});
expect(playSound).toHaveBeenCalledOnce();
});
// TODO: Same test?
test("plays a sound when a user joins", () => {
const { session, vm, remoteRtcMemberships } = getMockEnv([local, alice]);
render(<TestComponent rtcSession={session} vm={vm} />);
act(() => {
remoteRtcMemberships.next([aliceRtcMember, bobRtcMember]);
});
// Play a sound when joining a call.
expect(audioIsPlaying).toEqual([
// Joining the call
enterSound,
// Bob joins
enterSound,
]);
expect(playSound).toBeCalledWith("join");
});
test("plays a sound when a user leaves", () => {
const audioIsPlaying: string[] = mockMediaPlay();
const matrixRoomMembers = new Map([local, alice].map((p) => [p.userId, p]));
const remoteParticipants = new Map(
[aliceParticipant].map((p) => [p.identity, p]),
);
const liveKitRoom = new EmittableMockLivekitRoom({
localParticipant,
remoteParticipants,
});
const matrixRoom = mockMatrixRoom({
client: {
getUserId: () => localRtcMember.sender,
getDeviceId: () => localRtcMember.deviceId,
} as Partial<MatrixClient> as MatrixClient,
getMember: (userId) => matrixRoomMembers.get(userId) ?? null,
});
const remoteRtcMemberships = new BehaviorSubject<CallMembership[]>([
aliceRtcMember,
]);
const session = new MockRTCSession(
matrixRoom,
localRtcMember,
).withMemberships(remoteRtcMemberships) as unknown as MatrixRTCSession;
const vm = new CallViewModel(
session,
liveKitRoom as unknown as Room,
{
kind: E2eeType.PER_PARTICIPANT,
},
of(ConnectionState.Connected),
);
render(<CallEventAudioRenderer vm={vm} />);
const { session, vm, remoteRtcMemberships } = getMockEnv([local, alice]);
render(<TestComponent rtcSession={session} vm={vm} />);
act(() => {
remoteRtcMemberships.next([]);
});
expect(audioIsPlaying).toEqual([
// Joining the call
enterSound,
// Alice leaves
leaveSound,
]);
expect(playSound).toBeCalledWith("left");
});
test("plays no sound when the session member count is larger than the max, until decreased", () => {
const audioIsPlaying: string[] = mockMediaPlay();
const matrixRoomMembers = new Map([local, alice].map((p) => [p.userId, p]));
const remoteParticipants = new Map(
[aliceParticipant].map((p) => [p.identity, p]),
);
test("plays no sound when the participant list is more than the maximum size", () => {
const mockRtcMemberships: CallMembership[] = [];
for (let i = 0; i < MAX_PARTICIPANT_COUNT_FOR_SOUND; i++) {
mockRtcMemberships.push(
mockRtcMembership(`@user${i}:example.org`, `DEVICE${i}`),
);
}
const remoteRtcMemberships = new BehaviorSubject<CallMembership[]>(
const { session, vm, remoteRtcMemberships } = getMockEnv(
[local, alice],
mockRtcMemberships,
);
const liveKitRoom = new EmittableMockLivekitRoom({
localParticipant,
remoteParticipants,
});
const matrixRoom = mockMatrixRoom({
client: {
getUserId: () => localRtcMember.sender,
getDeviceId: () => localRtcMember.deviceId,
} as Partial<MatrixClient> as MatrixClient,
getMember: (userId) => matrixRoomMembers.get(userId) ?? null,
});
const session = new MockRTCSession(
matrixRoom,
localRtcMember,
).withMemberships(remoteRtcMemberships) as unknown as MatrixRTCSession;
const vm = new CallViewModel(
session,
liveKitRoom as unknown as Room,
{
kind: E2eeType.PER_PARTICIPANT,
},
of(ConnectionState.Connected),
);
render(<CallEventAudioRenderer vm={vm} />);
expect(audioIsPlaying).toEqual([]);
// When the count drops to the max we should play the leave sound
render(<TestComponent rtcSession={session} vm={vm} />);
expect(playSound).not.toBeCalled();
act(() => {
remoteRtcMemberships.next(
mockRtcMemberships.slice(0, MAX_PARTICIPANT_COUNT_FOR_SOUND - 1),
);
});
expect(audioIsPlaying).toEqual([leaveSound]);
expect(playSound).toBeCalledWith("left");
});

View File

@@ -5,47 +5,76 @@ SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { ReactNode, useEffect, useRef } from "react";
import { filter } from "rxjs";
import { ReactNode, useDeferredValue, useEffect, useMemo } from "react";
import { filter, interval, throttle } from "rxjs";
import {
soundEffectVolumeSetting as effectSoundVolumeSetting,
useSetting,
} from "../settings/settings";
import { CallViewModel } from "../state/CallViewModel";
import enterCallSoundMp3 from "../sound/join_call.mp3";
import enterCallSoundOgg from "../sound/join_call.ogg";
import joinCallSoundMp3 from "../sound/join_call.mp3";
import joinCallSoundOgg from "../sound/join_call.ogg";
import leftCallSoundMp3 from "../sound/left_call.mp3";
import leftCallSoundOgg from "../sound/left_call.ogg";
import handSoundOgg from "../sound/raise_hand.ogg?url";
import handSoundMp3 from "../sound/raise_hand.mp3?url";
import { useAudioContext } from "../useAudioContext";
import { prefetchSounds } from "../soundUtils";
import { useReactions } from "../useReactions";
import { useLatest } from "../useLatest";
// Do not play any sounds if the participant count has exceeded this
// number.
export const MAX_PARTICIPANT_COUNT_FOR_SOUND = 8;
export const CONCURRENT_AUDIO_CHANNELS = 2;
export const THROTTLE_SOUND_EFFECT_MS = 500;
const sounds = prefetchSounds({
join: {
mp3: joinCallSoundMp3,
ogg: joinCallSoundOgg,
},
left: {
mp3: leftCallSoundMp3,
ogg: leftCallSoundOgg,
},
raiseHand: {
mp3: handSoundMp3,
ogg: handSoundOgg,
},
});
export function CallEventAudioRenderer({
vm,
}: {
vm: CallViewModel;
}): ReactNode {
const [effectSoundVolume] = useSetting(effectSoundVolumeSetting);
const callEntered = useRef<(HTMLAudioElement | null)[]>([]);
const callLeft = useRef<(HTMLAudioElement | null)[]>([]);
const audioEngineCtx = useAudioContext({
sounds,
latencyHint: "interactive",
});
const audioEngineRef = useLatest(audioEngineCtx);
const { raisedHands } = useReactions();
const raisedHandCount = useMemo(
() => Object.keys(raisedHands).length,
[raisedHands],
);
const previousRaisedHandCount = useDeferredValue(raisedHandCount);
useEffect(() => {
if (effectSoundVolume === 0) {
return;
if (audioEngineRef.current && previousRaisedHandCount < raisedHandCount) {
audioEngineRef.current.playSound("raiseHand");
}
}, [audioEngineRef, previousRaisedHandCount, raisedHandCount]);
useEffect(() => {
const joinSub = vm.memberChanges
.pipe(
filter(
({ joined, ids }) =>
ids.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && joined.length > 0,
),
throttle(() => interval(THROTTLE_SOUND_EFFECT_MS)),
)
.subscribe(({ joined }) => {
const availablePlayer = callEntered.current.find((v) => v?.paused);
void availablePlayer?.play();
.subscribe(() => {
audioEngineRef.current?.playSound("join");
});
const leftSub = vm.memberChanges
@@ -54,64 +83,17 @@ export function CallEventAudioRenderer({
({ ids, left }) =>
ids.length <= MAX_PARTICIPANT_COUNT_FOR_SOUND && left.length > 0,
),
throttle(() => interval(THROTTLE_SOUND_EFFECT_MS)),
)
.subscribe(() => {
const availablePlayer = callLeft.current.find((v) => v?.paused);
void availablePlayer?.play();
audioEngineRef.current?.playSound("left");
});
return (): void => {
joinSub.unsubscribe();
leftSub.unsubscribe();
};
}, [effectSoundVolume, callEntered, callLeft, vm]);
}, [audioEngineRef, vm]);
// Set volume.
useEffect(() => {
callEntered.current.forEach((a) => {
if (a) {
a.volume = effectSoundVolume;
}
});
callLeft.current.forEach((a) => {
if (a) {
a.volume = effectSoundVolume;
}
});
}, [callEntered, callLeft, effectSoundVolume]);
// Do not render any audio elements if playback is disabled. Will save
// audio file fetches.
if (effectSoundVolume === 0) {
return null;
}
return (
// Will play as soon as it's mounted, which is what we want as this will
// play when the call is entered.
<>
{Array.from({ length: CONCURRENT_AUDIO_CHANNELS }).map((_, index) => (
<audio
key={index}
ref={(r) => (callEntered.current[index] = r)}
preload="auto"
hidden
>
<source src={enterCallSoundOgg} type="audio/ogg; codecs=vorbis" />
<source src={enterCallSoundMp3} type="audio/mpeg" />
</audio>
))}
{Array.from({ length: CONCURRENT_AUDIO_CHANNELS }).map((_, index) => (
<audio
key={index}
ref={(r) => (callLeft.current[index] = r)}
preload="auto"
hidden
>
<source src={leftCallSoundOgg} type="audio/ogg; codecs=vorbis" />
<source src={leftCallSoundMp3} type="audio/mpeg" />
</audio>
))}
</>
);
return <></>;
}

View File

@@ -331,7 +331,7 @@ export const GroupCallView: FC<Props> = ({
<ActiveCall
client={client}
matrixInfo={matrixInfo}
rtcSession={rtcSession}
rtcSession={rtcSession as unknown as MatrixRTCSession}
participantCount={participantCount}
onLeave={onLeave}
hideHeader={hideHeader}

View File

@@ -19,7 +19,6 @@ import {
TouchEvent,
forwardRef,
useCallback,
useDeferredValue,
useEffect,
useMemo,
useRef,
@@ -81,11 +80,8 @@ import { makeSpotlightLandscapeLayout } from "../grid/SpotlightLandscapeLayout";
import { makeSpotlightPortraitLayout } from "../grid/SpotlightPortraitLayout";
import { GridTileViewModel, TileViewModel } from "../state/TileViewModel";
import { ReactionsProvider, useReactions } from "../useReactions";
import handSoundOgg from "../sound/raise_hand.ogg?url";
import handSoundMp3 from "../sound/raise_hand.mp3?url";
import { ReactionsAudioRenderer } from "./ReactionAudioRenderer";
import { useSwitchCamera } from "./useSwitchCamera";
import { soundEffectVolumeSetting, useSetting } from "../settings/settings";
import { ReactionsOverlay } from "./ReactionsOverlay";
import { CallEventAudioRenderer } from "./CallEventAudioRenderer";
@@ -178,14 +174,7 @@ export const InCallView: FC<InCallViewProps> = ({
connState,
onShareClick,
}) => {
const [soundEffectVolume] = useSetting(soundEffectVolumeSetting);
const { supportsReactions, raisedHands, sendReaction, toggleRaisedHand } =
useReactions();
const raisedHandCount = useMemo(
() => Object.keys(raisedHands).length,
[raisedHands],
);
const previousRaisedHandCount = useDeferredValue(raisedHandCount);
const { supportsReactions, sendReaction, toggleRaisedHand } = useReactions();
useWakeLock();
@@ -335,25 +324,6 @@ export const InCallView: FC<InCallViewProps> = ({
[vm],
);
// Play a sound when the raised hand count increases.
const handRaisePlayer = useRef<HTMLAudioElement>(null);
useEffect(() => {
if (!handRaisePlayer.current) {
return;
}
if (previousRaisedHandCount < raisedHandCount) {
handRaisePlayer.current.volume = soundEffectVolume;
handRaisePlayer.current.play().catch((ex) => {
logger.warn("Failed to play raise hand sound", ex);
});
}
}, [
raisedHandCount,
handRaisePlayer,
previousRaisedHandCount,
soundEffectVolume,
]);
useEffect(() => {
widget?.api.transport
.send(
@@ -667,10 +637,6 @@ export const InCallView: FC<InCallViewProps> = ({
<RoomAudioRenderer />
{renderContent()}
<CallEventAudioRenderer vm={vm} />
<audio ref={handRaisePlayer} preload="auto" hidden>
<source src={handSoundOgg} type="audio/ogg; codecs=vorbis" />
<source src={handSoundMp3} type="audio/mpeg" />
</audio>
<ReactionsAudioRenderer />
<ReactionsOverlay />
{footer}

View File

@@ -6,9 +6,18 @@ Please see LICENSE in the repository root for full details.
*/
import { render } from "@testing-library/react";
import { afterAll, expect, test } from "vitest";
import {
afterAll,
beforeEach,
expect,
test,
vitest,
MockedFunction,
Mock,
} from "vitest";
import { TooltipProvider } from "@vector-im/compound-web";
import { act, ReactNode } from "react";
import { afterEach } from "node:test";
import {
MockRoom,
@@ -16,12 +25,13 @@ import {
TestReactionsWrapper,
} from "../utils/testReactions";
import { ReactionsAudioRenderer } from "./ReactionAudioRenderer";
import { GenericReaction, ReactionSet } from "../reactions";
import {
playReactionsSound,
soundEffectVolumeSetting,
} from "../settings/settings";
import { mockMediaPlay } from "../utils/test";
import { useAudioContext } from "../useAudioContext";
import { GenericReaction, ReactionSet } from "../reactions";
import { prefetchSounds } from "../soundUtils";
const memberUserIdAlice = "@alice:example.org";
const memberUserIdBob = "@bob:example.org";
@@ -50,11 +60,31 @@ function TestComponent({
);
}
const originalPlayFn = window.HTMLMediaElement.prototype.play;
afterAll(() => {
vitest.mock("../useAudioContext");
vitest.mock("../soundUtils");
afterEach(() => {
vitest.resetAllMocks();
playReactionsSound.setValue(playReactionsSound.defaultValue);
soundEffectVolumeSetting.setValue(soundEffectVolumeSetting.defaultValue);
window.HTMLMediaElement.prototype.play = originalPlayFn;
});
afterAll(() => {
vitest.restoreAllMocks();
});
let playSound: Mock<
NonNullable<ReturnType<typeof useAudioContext>>["playSound"]
>;
beforeEach(() => {
(prefetchSounds as MockedFunction<typeof prefetchSounds>).mockResolvedValue({
sound: new ArrayBuffer(0),
});
playSound = vitest.fn();
(useAudioContext as MockedFunction<typeof useAudioContext>).mockReturnValue({
playSound,
});
});
test("preloads all audio elements", () => {
@@ -63,25 +93,11 @@ test("preloads all audio elements", () => {
new MockRoom(memberUserIdAlice),
membership,
);
const { container } = render(<TestComponent rtcSession={rtcSession} />);
expect(container.getElementsByTagName("audio")).toHaveLength(
// All reactions plus the generic sound
ReactionSet.filter((r) => r.sound).length + 1,
);
});
test("loads no audio elements when disabled in settings", () => {
playReactionsSound.setValue(false);
const rtcSession = new MockRTCSession(
new MockRoom(memberUserIdAlice),
membership,
);
const { container } = render(<TestComponent rtcSession={rtcSession} />);
expect(container.getElementsByTagName("audio")).toHaveLength(0);
render(<TestComponent rtcSession={rtcSession} />);
expect(prefetchSounds).toHaveBeenCalledOnce();
});
test("will play an audio sound when there is a reaction", () => {
const audioIsPlaying: string[] = mockMediaPlay();
playReactionsSound.setValue(true);
const room = new MockRoom(memberUserIdAlice);
const rtcSession = new MockRTCSession(room, membership);
@@ -97,12 +113,10 @@ test("will play an audio sound when there is a reaction", () => {
act(() => {
room.testSendReaction(memberEventAlice, chosenReaction, membership);
});
expect(audioIsPlaying).toHaveLength(1);
expect(audioIsPlaying[0]).toContain(chosenReaction.sound?.ogg);
expect(playSound).toHaveBeenCalledWith(chosenReaction.name);
});
test("will play the generic audio sound when there is soundless reaction", () => {
const audioIsPlaying: string[] = mockMediaPlay();
playReactionsSound.setValue(true);
const room = new MockRoom(memberUserIdAlice);
const rtcSession = new MockRTCSession(room, membership);
@@ -118,34 +132,10 @@ test("will play the generic audio sound when there is soundless reaction", () =>
act(() => {
room.testSendReaction(memberEventAlice, chosenReaction, membership);
});
expect(audioIsPlaying).toHaveLength(1);
expect(audioIsPlaying[0]).toContain(GenericReaction.sound?.ogg);
});
test("will play an audio sound with the correct volume", () => {
playReactionsSound.setValue(true);
soundEffectVolumeSetting.setValue(0.5);
const room = new MockRoom(memberUserIdAlice);
const rtcSession = new MockRTCSession(room, membership);
const { getByTestId } = render(<TestComponent rtcSession={rtcSession} />);
// Find the first reaction with a sound effect
const chosenReaction = ReactionSet.find((r) => !!r.sound);
if (!chosenReaction) {
throw Error(
"No reactions have sounds configured, this test cannot succeed",
);
}
act(() => {
room.testSendReaction(memberEventAlice, chosenReaction, membership);
});
expect((getByTestId(chosenReaction.name) as HTMLAudioElement).volume).toEqual(
0.5,
);
expect(playSound).toHaveBeenCalledWith(GenericReaction.name);
});
test("will play multiple audio sounds when there are multiple different reactions", () => {
const audioIsPlaying: string[] = mockMediaPlay();
playReactionsSound.setValue(true);
const room = new MockRoom(memberUserIdAlice);
@@ -164,7 +154,6 @@ test("will play multiple audio sounds when there are multiple different reaction
room.testSendReaction(memberEventBob, reaction2, membership);
room.testSendReaction(memberEventCharlie, reaction1, membership);
});
expect(audioIsPlaying).toHaveLength(2);
expect(audioIsPlaying[0]).toContain(reaction1.sound?.ogg);
expect(audioIsPlaying[1]).toContain(reaction2.sound?.ogg);
expect(playSound).toHaveBeenCalledWith(reaction1.name);
expect(playSound).toHaveBeenCalledWith(reaction2.name);
});

View File

@@ -5,70 +5,67 @@ SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { ReactNode, useEffect, useRef } from "react";
import { ReactNode, useDeferredValue, useEffect, useState } from "react";
import { useReactions } from "../useReactions";
import {
playReactionsSound,
soundEffectVolumeSetting as effectSoundVolumeSetting,
useSetting,
} from "../settings/settings";
import { playReactionsSound, useSetting } from "../settings/settings";
import { GenericReaction, ReactionSet } from "../reactions";
import { useAudioContext } from "../useAudioContext";
import { prefetchSounds } from "../soundUtils";
import { useLatest } from "../useLatest";
const soundMap = Object.fromEntries([
...ReactionSet.filter((v) => v.sound !== undefined).map((v) => [
v.name,
v.sound!,
]),
[GenericReaction.name, GenericReaction.sound],
]);
export function ReactionsAudioRenderer(): ReactNode {
const { reactions } = useReactions();
const [shouldPlay] = useSetting(playReactionsSound);
const [effectSoundVolume] = useSetting(effectSoundVolumeSetting);
const audioElements = useRef<Record<string, HTMLAudioElement | null>>({});
const [soundCache, setSoundCache] = useState<ReturnType<
typeof prefetchSounds
> | null>(null);
const audioEngineCtx = useAudioContext({
sounds: soundCache,
latencyHint: "interactive",
});
const audioEngineRef = useLatest(audioEngineCtx);
const oldReactions = useDeferredValue(reactions);
useEffect(() => {
if (!audioElements.current) {
if (!shouldPlay || soundCache) {
return;
}
// This is fine even if we load the component multiple times,
// as the browser's cache should ensure once the media is loaded
// once that future fetches come via the cache.
setSoundCache(prefetchSounds(soundMap));
}, [soundCache, shouldPlay]);
if (!shouldPlay) {
useEffect(() => {
if (!shouldPlay || !audioEngineRef.current) {
return;
}
const oldReactionSet = new Set(
Object.values(oldReactions).map((r) => r.name),
);
for (const reactionName of new Set(
Object.values(reactions).map((r) => r.name),
)) {
const audioElement =
audioElements.current[reactionName] ?? audioElements.current.generic;
if (audioElement?.paused) {
audioElement.volume = effectSoundVolume;
void audioElement.play();
if (oldReactionSet.has(reactionName)) {
// Don't replay old reactions
return;
}
if (soundMap[reactionName]) {
audioEngineRef.current.playSound(reactionName);
} else {
// Fallback sounds.
audioEngineRef.current.playSound("generic");
}
}
}, [audioElements, shouldPlay, reactions, effectSoundVolume]);
// Do not render any audio elements if playback is disabled. Will save
// audio file fetches.
if (!shouldPlay) {
return null;
}
// NOTE: We load all audio elements ahead of time to allow the cache
// to be populated, rather than risk a cache miss and have the audio
// be delayed.
return (
<>
{[GenericReaction, ...ReactionSet].map(
(r) =>
r.sound && (
<audio
ref={(el) => (audioElements.current[r.name] = el)}
data-testid={r.name}
key={r.name}
preload="auto"
hidden
>
<source src={r.sound.ogg} type="audio/ogg; codecs=vorbis" />
{r.sound.mp3 ? (
<source src={r.sound.mp3} type="audio/mpeg" />
) : null}
</audio>
),
)}
</>
);
}, [audioEngineRef, shouldPlay, oldReactions, reactions]);
return null;
}

View File

@@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { FC } from "react";
import { FC, useState } from "react";
import { useTranslation } from "react-i18next";
import { MatrixClient } from "matrix-js-sdk/src/matrix";
import { Root as Form } from "@vector-im/compound-web";
@@ -63,8 +63,8 @@ export const SettingsModal: FC<Props> = ({
const devices = useMediaDevices();
useMediaDeviceNames(devices, open);
const [soundVolume, setSoundVolume] = useSetting(soundEffectVolumeSetting);
const [soundVolumeRaw, setSoundVolumeRaw] = useState(soundVolume);
const [showDeveloperSettingsTab] = useSetting(developerSettingsTab);
@@ -89,8 +89,9 @@ export const SettingsModal: FC<Props> = ({
<p>{t("settings.audio_tab.effect_volume_description")}</p>
<Slider
label={t("video_tile.volume")}
value={soundVolume}
onValueChange={setSoundVolume}
value={soundVolumeRaw}
onValueChange={setSoundVolumeRaw}
onValueCommit={setSoundVolume}
min={0}
max={1}
step={0.01}

63
src/soundUtils.ts Normal file
View File

@@ -0,0 +1,63 @@
/*
Copyright 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { logger } from "matrix-js-sdk/src/logger";
type SoundDefinition = { mp3?: string; ogg: string };
export type PrefetchedSounds<S extends string> = Promise<
Record<S, ArrayBuffer>
>;
/**
* Determine the best format we can use to play our sounds
* through. We prefer ogg support if possible, but will fall
* back to MP3.
* @returns "ogg" if the browser is likely to support it, or "mp3" otherwise.
*/
function getPreferredAudioFormat(): "ogg" | "mp3" {
const a = document.createElement("audio");
if (a.canPlayType("audio/ogg") === "maybe") {
return "ogg";
}
// Otherwise just assume MP3, as that has a chance of being more widely supported.
return "mp3";
}
const preferredFormat = getPreferredAudioFormat();
/**
* Prefetch sounds to be used by the AudioContext. This can
* be called outside the scope of a component to ensure the
* sounds load ahead of time.
* @param sounds A set of sound files that may be played.
* @returns A map of sound files to buffers.
*/
export async function prefetchSounds<S extends string>(
sounds: Record<S, SoundDefinition>,
): PrefetchedSounds<S> {
const buffers: Record<string, ArrayBuffer> = {};
await Promise.all(
Object.entries(sounds).map(async ([name, file]) => {
const { mp3, ogg } = file as SoundDefinition;
// Use preferred format, fallback to ogg if no mp3 is provided.
// Load an audio file
const response = await fetch(
preferredFormat === "ogg" ? ogg : (mp3 ?? ogg),
);
if (!response.ok) {
// If the sound doesn't load, it's not the end of the world. We won't play
// the sound when requested, but it's better than failing the whole application.
logger.warn(`Could not load sound ${name}, response was not okay`);
return;
}
// Decode it
buffers[name] = await response.arrayBuffer();
}),
);
return buffers as Record<S, ArrayBuffer>;
}

View File

@@ -335,7 +335,7 @@ function findMatrixRoomMember(
// must be at least 3 parts because we know the first part is a userId which must necessarily contain a colon
if (parts.length < 3) {
logger.warn(
"Livekit participants ID doesn't look like a userId:deviceId combination",
`Livekit participants ID (${id}) doesn't look like a userId:deviceId combination`,
);
return undefined;
}

View File

@@ -0,0 +1,129 @@
/*
Copyright 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { expect, test, vitest } from "vitest";
import { FC } from "react";
import { render } from "@testing-library/react";
import { afterEach } from "node:test";
import userEvent from "@testing-library/user-event";
import { deviceStub, MediaDevicesContext } from "./livekit/MediaDevicesContext";
import { useAudioContext } from "./useAudioContext";
import { soundEffectVolumeSetting } from "./settings/settings";
const staticSounds = Promise.resolve({
aSound: new ArrayBuffer(0),
});
const TestComponent: FC = () => {
const audioCtx = useAudioContext({
sounds: staticSounds,
latencyHint: "balanced",
});
if (!audioCtx) {
return null;
}
return (
<>
<button onClick={() => audioCtx.playSound("aSound")}>Valid sound</button>
{/* eslint-disable-next-line @typescript-eslint/no-explicit-any*/}
<button onClick={() => audioCtx.playSound("not-valid" as any)}>
Invalid sound
</button>
</>
);
};
class MockAudioContext {
public static testContext: MockAudioContext;
public constructor() {
MockAudioContext.testContext = this;
}
public gain = vitest.mocked(
{
connect: () => {},
gain: {
setValueAtTime: vitest.fn(),
},
},
true,
);
public setSinkId = vitest.fn().mockResolvedValue(undefined);
public decodeAudioData = vitest.fn().mockReturnValue(1);
public createBufferSource = vitest.fn().mockReturnValue(
vitest.mocked({
connect: (v: unknown) => v,
start: () => {},
}),
);
public createGain = vitest.fn().mockReturnValue(this.gain);
public close = vitest.fn().mockResolvedValue(undefined);
}
afterEach(() => {
vitest.unstubAllGlobals();
});
test("can play a single sound", async () => {
const user = userEvent.setup();
vitest.stubGlobal("AudioContext", MockAudioContext);
const { findByText } = render(<TestComponent />);
await user.click(await findByText("Valid sound"));
expect(
MockAudioContext.testContext.createBufferSource,
).toHaveBeenCalledOnce();
});
test("will ignore sounds that are not registered", async () => {
const user = userEvent.setup();
vitest.stubGlobal("AudioContext", MockAudioContext);
const { findByText } = render(<TestComponent />);
await user.click(await findByText("Invalid sound"));
expect(
MockAudioContext.testContext.createBufferSource,
).not.toHaveBeenCalled();
});
test("will use the correct device", () => {
vitest.stubGlobal("AudioContext", MockAudioContext);
render(
<MediaDevicesContext.Provider
value={{
audioInput: deviceStub,
audioOutput: {
selectedId: "chosen-device",
available: [],
select: () => {},
},
videoInput: deviceStub,
startUsingDeviceNames: () => {},
stopUsingDeviceNames: () => {},
}}
>
<TestComponent />
</MediaDevicesContext.Provider>,
);
expect(
MockAudioContext.testContext.createBufferSource,
).not.toHaveBeenCalled();
expect(MockAudioContext.testContext.setSinkId).toHaveBeenCalledWith(
"chosen-device",
);
});
test("will use the correct volume level", async () => {
const user = userEvent.setup();
vitest.stubGlobal("AudioContext", MockAudioContext);
soundEffectVolumeSetting.setValue(0.33);
const { findByText } = render(<TestComponent />);
await user.click(await findByText("Valid sound"));
expect(
MockAudioContext.testContext.gain.gain.setValueAtTime,
).toHaveBeenCalledWith(0.33, 0);
});

124
src/useAudioContext.tsx Normal file
View File

@@ -0,0 +1,124 @@
/*
Copyright 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { logger } from "matrix-js-sdk/src/logger";
import { useState, useEffect } from "react";
import {
soundEffectVolumeSetting as effectSoundVolumeSetting,
useSetting,
} from "./settings/settings";
import { useMediaDevices } from "./livekit/MediaDevicesContext";
import { PrefetchedSounds } from "./soundUtils";
/**
* Play a sound though a given AudioContext. Will take
* care of connecting the correct buffer and gating
* through gain.
* @param volume The volume to play at.
* @param ctx The context to play through.
* @param buffer The buffer to play.
*/
function playSound(
ctx: AudioContext,
buffer: AudioBuffer,
volume: number,
): void {
const gain = ctx.createGain();
gain.gain.setValueAtTime(volume, 0);
const src = ctx.createBufferSource();
src.buffer = buffer;
src.connect(gain).connect(ctx.destination);
src.start();
}
interface Props<S extends string> {
/**
* The sounds to play. If no sounds should be played then
* this can be set to null, which will prevent the audio
* context from being created.
*/
sounds: PrefetchedSounds<S> | null;
latencyHint: AudioContextLatencyCategory;
}
interface UseAudioContext<S> {
playSound(soundName: S): void;
}
/**
* Add an audio context which can be used to play
* a set of preloaded sounds.
* @param props
* @returns Either an instance that can be used to play sounds, or null if not ready.
*/
export function useAudioContext<S extends string>(
props: Props<S>,
): UseAudioContext<S> | null {
const [effectSoundVolume] = useSetting(effectSoundVolumeSetting);
const devices = useMediaDevices();
const [audioContext, setAudioContext] = useState<AudioContext>();
const [audioBuffers, setAudioBuffers] = useState<Record<S, AudioBuffer>>();
useEffect(() => {
const sounds = props.sounds;
if (!sounds) {
return;
}
const ctx = new AudioContext({
// We want low latency for these effects.
latencyHint: props.latencyHint,
});
// We want to clone the content of our preloaded
// sound buffers into this context. The context may
// close during this process, so it's okay if it throws.
(async (): Promise<void> => {
const buffers: Record<string, AudioBuffer> = {};
for (const [name, buffer] of Object.entries<ArrayBuffer>(await sounds)) {
const audioBuffer = await ctx.decodeAudioData(buffer.slice(0));
buffers[name] = audioBuffer;
}
setAudioBuffers(buffers as Record<S, AudioBuffer>);
})().catch((ex) => {
logger.debug("Failed to setup audio context", ex);
});
setAudioContext(ctx);
return (): void => {
void ctx.close().catch((ex) => {
logger.debug("Failed to close audio engine", ex);
});
setAudioContext(undefined);
};
}, [props.sounds, props.latencyHint]);
// Update the sink ID whenever we change devices.
useEffect(() => {
if (audioContext && "setSinkId" in audioContext) {
// https://developer.mozilla.org/en-US/docs/Web/API/AudioContext/setSinkId
// @ts-expect-error - setSinkId doesn't exist yet in types, maybe because it's not supported everywhere.
audioContext.setSinkId(devices.audioOutput.selectedId).catch((ex) => {
logger.warn("Unable to change sink for audio context", ex);
});
}
}, [audioContext, devices]);
// Don't return a function until we're ready.
if (!audioContext || !audioBuffers) {
return null;
}
return {
playSound: (name): void => {
if (!audioBuffers[name]) {
logger.debug(`Tried to play a sound that wasn't buffered (${name})`);
return;
}
return playSound(audioContext, audioBuffers[name], effectSoundVolume);
},
};
}

View File

@@ -333,15 +333,3 @@ export function getRelativeRoomUrl(
: "";
return `/room/#${roomPart}?${generateUrlSearchParams(roomId, encryptionSystem, viaServers).toString()}`;
}
export function getAvatarUrl(
client: MatrixClient,
mxcUrl: string,
avatarSize = 96,
): string {
const width = Math.floor(avatarSize * window.devicePixelRatio);
const height = Math.floor(avatarSize * window.devicePixelRatio);
// scale is more suitable for larger sizes
const resizeMethod = avatarSize <= 96 ? "crop" : "scale";
return mxcUrl && client.mxcUrlToHttp(mxcUrl, width, height, resizeMethod)!;
}

View File

@@ -27,9 +27,7 @@ import {
RemoteParticipant,
RemoteTrackPublication,
Room as LivekitRoom,
RoomEvent,
} from "livekit-client";
import { EventEmitter } from "stream";
import {
LocalUserMediaViewModel,
@@ -153,33 +151,6 @@ export function mockMatrixRoom(room: Partial<MatrixRoom>): MatrixRoom {
return { ...mockEmitter(), ...room } as Partial<MatrixRoom> as MatrixRoom;
}
/**
* A mock of a Livekit Room that can emit events.
*/
export class EmittableMockLivekitRoom extends EventEmitter {
public localParticipant?: LocalParticipant;
public remoteParticipants: Map<string, RemoteParticipant>;
public constructor(room: {
localParticipant?: LocalParticipant;
remoteParticipants: Map<string, RemoteParticipant>;
}) {
super();
this.localParticipant = room.localParticipant;
this.remoteParticipants = room.remoteParticipants ?? new Map();
}
public addParticipant(remoteParticipant: RemoteParticipant): void {
this.remoteParticipants.set(remoteParticipant.identity, remoteParticipant);
this.emit(RoomEvent.ParticipantConnected, remoteParticipant);
}
public removeParticipant(remoteParticipant: RemoteParticipant): void {
this.remoteParticipants.delete(remoteParticipant.identity);
this.emit(RoomEvent.ParticipantDisconnected, remoteParticipant);
}
}
export function mockLivekitRoom(
room: Partial<LivekitRoom>,
{
@@ -280,15 +251,6 @@ export function mockConfig(config: Partial<ResolvedConfigOptions> = {}): void {
});
}
export function mockMediaPlay(): string[] {
const audioIsPlaying: string[] = [];
window.HTMLMediaElement.prototype.play = async function (): Promise<void> {
audioIsPlaying.push((this.children[0] as HTMLSourceElement).src);
return Promise.resolve();
};
return audioIsPlaying;
}
export class MockRTCSession extends TypedEventEmitter<
MatrixRTCSessionEvent,
MatrixRTCSessionEventHandlerMap

View File

@@ -32,7 +32,7 @@ export const TestReactionsWrapper = ({
rtcSession,
children,
}: PropsWithChildren<{
rtcSession: MockRTCSession;
rtcSession: MockRTCSession | MatrixRTCSession;
}>): ReactNode => {
return (
<ReactionsProvider rtcSession={rtcSession as unknown as MatrixRTCSession}>
@@ -203,4 +203,12 @@ export class MockRoom extends EventEmitter {
});
return evt.getId()!;
}
public getMember(): void {
return;
}
public testGetAsMatrixRoom(): Room {
return this as unknown as Room;
}
}