Matrix/element-call-Github
1
0
Fork 0
mirror of https://github.com/vector-im/element-call.git synced 2026-03-31 07:00:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

merge

Browse Source
This commit is contained in:
Half-Shot
2024-12-09 11:52:22 +00:00
parent b7f55a43f8 a8a95c3f00
commit 59e8e76969
49 changed files with 1317 additions and 556 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/@types/i18next.d.ts vendored
View File

@@ -7,7 +7,7 @@ Please see LICENSE in the repository root for full details.
import "i18next";
// import all namespaces (for the default language, only)
import app from "../../locales/en-GB/app.json";
import app from "../../locales/en/app.json";
declare module "i18next" {
interface CustomTypeOptions {

156
src/Avatar.test.tsx Normal file
View File

@@ -0,0 +1,156 @@
/*
Copyright 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { afterEach, expect, test, vi } from "vitest";
import { render, screen } from "@testing-library/react";
import { MatrixClient } from "matrix-js-sdk/src/client";
import { FC, PropsWithChildren } from "react";
import { ClientContextProvider } from "./ClientContext";
import { Avatar } from "./Avatar";
import { mockMatrixRoomMember, mockRtcMembership } from "./utils/test";
const TestComponent: FC<
PropsWithChildren<{ client: MatrixClient; supportsThumbnails?: boolean }>
> = ({ client, children, supportsThumbnails }) => {
return (
<ClientContextProvider
value={{
state: "valid",
disconnected: false,
supportedFeatures: {
reactions: true,
thumbnails: supportsThumbnails ?? true,
},
setClient: vi.fn(),
authenticated: {
client,
isPasswordlessUser: true,
changePassword: vi.fn(),
logout: vi.fn(),
},
}}
>
{children}
</ClientContextProvider>
);
};
afterEach(() => {
vi.unstubAllGlobals();
});
test("should just render a placeholder when the user has no avatar", () => {
const client = vi.mocked<MatrixClient>({
getAccessToken: () => "my-access-token",
mxcUrlToHttp: () => vi.fn(),
} as unknown as MatrixClient);
vi.spyOn(client, "mxcUrlToHttp");
const member = mockMatrixRoomMember(
mockRtcMembership("@alice:example.org", "AAAA"),
{
getMxcAvatarUrl: () => undefined,
},
);
const displayName = "Alice";
render(
<TestComponent client={client}>
<Avatar
id={member.userId}
name={displayName}
size={96}
src={member.getMxcAvatarUrl()}
/>
</TestComponent>,
);
const element = screen.getByRole("img", { name: "@alice:example.org" });
expect(element.tagName).toEqual("SPAN");
expect(client.mxcUrlToHttp).toBeCalledTimes(0);
});
test("should just render a placeholder when thumbnails are not supported", () => {
const client = vi.mocked<MatrixClient>({
getAccessToken: () => "my-access-token",
mxcUrlToHttp: () => vi.fn(),
} as unknown as MatrixClient);
vi.spyOn(client, "mxcUrlToHttp");
const member = mockMatrixRoomMember(
mockRtcMembership("@alice:example.org", "AAAA"),
{
getMxcAvatarUrl: () => "mxc://example.org/alice-avatar",
},
);
const displayName = "Alice";
render(
<TestComponent client={client} supportsThumbnails={false}>
<Avatar
id={member.userId}
name={displayName}
size={96}
src={member.getMxcAvatarUrl()}
/>
</TestComponent>,
);
const element = screen.getByRole("img", { name: "@alice:example.org" });
expect(element.tagName).toEqual("SPAN");
expect(client.mxcUrlToHttp).toBeCalledTimes(0);
});
test("should attempt to fetch authenticated media", async () => {
const expectedAuthUrl = "http://example.org/media/alice-avatar";
const expectedObjectURL = "my-object-url";
const accessToken = "my-access-token";
const theBlob = new Blob([]);
// vitest doesn't have a implementation of create/revokeObjectURL, so we need
// to delete the property. It's a bit odd, but it works.
Reflect.deleteProperty(global.window.URL, "createObjectURL");
globalThis.URL.createObjectURL = vi.fn().mockReturnValue(expectedObjectURL);
Reflect.deleteProperty(global.window.URL, "revokeObjectURL");
globalThis.URL.revokeObjectURL = vi.fn();
const fetchFn = vi.fn().mockResolvedValue({
blob: async () => Promise.resolve(theBlob),
});
vi.stubGlobal("fetch", fetchFn);
const client = vi.mocked<MatrixClient>({
getAccessToken: () => accessToken,
mxcUrlToHttp: () => vi.fn(),
} as unknown as MatrixClient);
vi.spyOn(client, "mxcUrlToHttp").mockReturnValue(expectedAuthUrl);
const member = mockMatrixRoomMember(
mockRtcMembership("@alice:example.org", "AAAA"),
{
getMxcAvatarUrl: () => "mxc://example.org/alice-avatar",
},
);
const displayName = "Alice";
render(
<TestComponent client={client}>
<Avatar
id={member.userId}
name={displayName}
size={96}
src={member.getMxcAvatarUrl()}
/>
</TestComponent>,
);
// Fetch is asynchronous, so wait for this to resolve.
await vi.waitUntil(() =>
document.querySelector(`img[src='${expectedObjectURL}']`),
);
expect(client.mxcUrlToHttp).toBeCalledTimes(1);
expect(globalThis.fetch).toBeCalledWith(expectedAuthUrl, {
headers: { Authorization: `Bearer ${accessToken}` },
});
});

80
src/Avatar.tsx
View File

@@ -5,11 +5,11 @@ SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { useMemo, FC, CSSProperties } from "react";
import { useMemo, FC, CSSProperties, useState, useEffect } from "react";
import { Avatar as CompoundAvatar } from "@vector-im/compound-web";
import { MatrixClient } from "matrix-js-sdk/src/client";
import { getAvatarUrl } from "./utils/matrix";
import { useClient } from "./ClientContext";
import { useClientState } from "./ClientContext";
export enum Size {
XS = "xs",
@@ -36,6 +36,28 @@ interface Props {
style?: CSSProperties;
}
export function getAvatarUrl(
client: MatrixClient,
mxcUrl: string | null,
avatarSize = 96,
): string | null {
const width = Math.floor(avatarSize * window.devicePixelRatio);
const height = Math.floor(avatarSize * window.devicePixelRatio);
// scale is more suitable for larger sizes
const resizeMethod = avatarSize <= 96 ? "crop" : "scale";
return mxcUrl
? client.mxcUrlToHttp(
mxcUrl,
width,
height,
resizeMethod,
false,
true,
true,
)
: null;
}
export const Avatar: FC<Props> = ({
className,
id,
@@ -45,7 +67,7 @@ export const Avatar: FC<Props> = ({
style,
...props
}) => {
const { client } = useClient();
const clientState = useClientState();
const sizePx = useMemo(
() =>
@@ -55,10 +77,50 @@ export const Avatar: FC<Props> = ({
[size],
);
const resolvedSrc = useMemo(() => {
if (!client || !src || !sizePx) return undefined;
return src.startsWith("mxc://") ? getAvatarUrl(client, src, sizePx) : src;
}, [client, src, sizePx]);
const [avatarUrl, setAvatarUrl] = useState<string | undefined>(undefined);
useEffect(() => {
if (clientState?.state !== "valid") {
return;
}
const { authenticated, supportedFeatures } = clientState;
const client = authenticated?.client;
if (!client || !src || !sizePx || !supportedFeatures.thumbnails) {
return;
}
const token = client.getAccessToken();
if (!token) {
return;
}
const resolveSrc = getAvatarUrl(client, src, sizePx);
if (!resolveSrc) {
setAvatarUrl(undefined);
return;
}
let objectUrl: string | undefined;
fetch(resolveSrc, {
headers: {
Authorization: `Bearer ${token}`,
},
})
.then(async (req) => req.blob())
.then((blob) => {
objectUrl = URL.createObjectURL(blob);
setAvatarUrl(objectUrl);
})
.catch((ex) => {
setAvatarUrl(undefined);
});
return (): void => {
if (objectUrl) {
URL.revokeObjectURL(objectUrl);
}
};
}, [clientState, src, sizePx]);
return (
<CompoundAvatar
@@ -66,7 +128,7 @@ export const Avatar: FC<Props> = ({
id={id}
name={name}
size={`${sizePx}px`}
src={resolvedSrc}
src={avatarUrl}
style={style}
{...props}
/>

9
src/ClientContext.tsx
View File

@@ -48,6 +48,7 @@ export type ValidClientState = {
disconnected: boolean;
supportedFeatures: {
reactions: boolean;
thumbnails: boolean;
};
setClient: (params?: SetClientParams) => void;
};
@@ -71,6 +72,8 @@ export type SetClientParams = {
const ClientContext = createContext<ClientState | undefined>(undefined);
export const ClientContextProvider = ClientContext.Provider;
export const useClientState = (): ClientState | undefined =>
useContext(ClientContext);
@@ -253,6 +256,7 @@ export const ClientProvider: FC<Props> = ({ children }) => {
const [isDisconnected, setIsDisconnected] = useState(false);
const [supportsReactions, setSupportsReactions] = useState(false);
const [supportsThumbnails, setSupportsThumbnails] = useState(false);
const state: ClientState | undefined = useMemo(() => {
if (alreadyOpenedErr) {
@@ -278,6 +282,7 @@ export const ClientProvider: FC<Props> = ({ children }) => {
disconnected: isDisconnected,
supportedFeatures: {
reactions: supportsReactions,
thumbnails: supportsThumbnails,
},
};
}, [
@@ -288,6 +293,7 @@ export const ClientProvider: FC<Props> = ({ children }) => {
setClient,
isDisconnected,
supportsReactions,
supportsThumbnails,
]);
const onSync = useCallback(
@@ -313,6 +319,8 @@ export const ClientProvider: FC<Props> = ({ children }) => {
}
if (initClientState.widgetApi) {
// There is currently no widget API for authenticated media thumbnails.
setSupportsThumbnails(false);
const reactSend = initClientState.widgetApi.hasCapability(
"org.matrix.msc2762.send.event:m.reaction",
);
@@ -334,6 +342,7 @@ export const ClientProvider: FC<Props> = ({ children }) => {
}
} else {
setSupportsReactions(true);
setSupportsThumbnails(true);
}
return (): void => {

2
src/TranslatedError.ts
View File

@@ -22,7 +22,7 @@ export abstract class TranslatedError extends Error {
messageKey: ParseKeys<DefaultNamespace, TOptions>,
translationFn: TFunction<DefaultNamespace>,
) {
super(translationFn(messageKey, { lng: "en-GB" } as TOptions));
super(translationFn(messageKey, { lng: "en" } as TOptions));
this.translatedMessage = translationFn(messageKey);
}
}

2
src/config/ConfigOptions.ts
View File

@@ -51,7 +51,7 @@ export interface ConfigOptions {
// a livekit service url in the client well-known.
// The well known needs to be formatted like so:
// {"type":"livekit", "livekit_service_url":"https://livekit.example.com"}
// and stored under the key: "livekit_focus"
// and stored under the key: "org.matrix.msc4143.rtc_foci"
livekit_service_url: string;
};

5
src/index.css
View File

@@ -47,6 +47,11 @@ layer(compound);
--background-gradient: url("graphics/backgroundGradient.svg");
}
:root,
[class*="cpd-theme-"] {
--video-tile-background: var(--cpd-color-bg-subtle-secondary);
}
.cpd-theme-dark {
--cpd-color-border-accent: var(--cpd-color-green-1100);
--stopgap-color-on-solid-accent: var(--cpd-color-text-primary);

6
src/initializer.tsx
View File

@@ -24,7 +24,7 @@ import { platform } from "./Platform";
// This generates a map of locale names to their URL (based on import.meta.url), which looks like this:
// {
// "../locales/en-GB/app.json": "/whatever/assets/root/locales/en-aabbcc.json",
// "../locales/en/app.json": "/whatever/assets/root/locales/en-aabbcc.json",
// ...
// }
const locales = import.meta.glob<string>("../locales/*/*.json", {
@@ -41,7 +41,7 @@ const getLocaleUrl = (
const supportedLngs = [
...new Set(
Object.keys(locales).map((url) => {
// The URLs are of the form ../locales/en-GB/app.json
// The URLs are of the form ../locales/en/app.json
// This extracts the language code from the URL
const lang = url.match(/\/([^/]+)\/[^/]+\.json$/)?.[1];
if (!lang) {
@@ -133,7 +133,7 @@ export class Initializer {
.use(languageDetector)
.use(initReactI18next)
.init({
fallbackLng: "en-GB",
fallbackLng: "en",
defaultNS: "app",
keySeparator: ".",
nsSeparator: false,

207
src/room/CallEventAudioRenderer.test.tsx
View File

@@ -8,23 +8,31 @@ Please see LICENSE in the repository root for full details.
import { render } from "@testing-library/react";
import {
afterAll,
afterEach,
beforeEach,
expect,
Mock,
MockedFunction,
test,
vitest,
} from "vitest";
import { ConnectionState, RemoteParticipant, Room } from "livekit-client";
import { of } from "rxjs";
import { MatrixClient } from "matrix-js-sdk/src/client";
import { ConnectionState } from "livekit-client";
import { BehaviorSubject, of } from "rxjs";
import { afterEach } from "node:test";
import { act, ReactNode } from "react";
import {
CallMembership,
type MatrixRTCSession,
} from "matrix-js-sdk/src/matrixrtc";
import { RoomMember } from "matrix-js-sdk/src/matrix";
import {
EmittableMockLivekitRoom,
mockLivekitRoom,
mockLocalParticipant,
mockMatrixRoom,
mockMatrixRoomMember,
mockRemoteParticipant,
mockRtcMembership,
MockRTCSession,
} from "../utils/test";
import { E2eeType } from "../e2ee/e2eeType";
import { CallViewModel } from "../state/CallViewModel";
@@ -32,22 +40,21 @@ import {
CallEventAudioRenderer,
MAX_PARTICIPANT_COUNT_FOR_SOUND,
} from "./CallEventAudioRenderer";
import { prefetchSounds, useAudioContext } from "../useAudioContext";
import {
MockRoom,
MockRTCSession,
TestReactionsWrapper,
} from "../utils/testReactions";
import { useAudioContext } from "../useAudioContext";
import { TestReactionsWrapper } from "../utils/testReactions";
import { prefetchSounds } from "../soundUtils";
const alice = mockMatrixRoomMember({ userId: "@alice:example.org" });
const bob = mockMatrixRoomMember({ userId: "@bob:example.org" });
const aliceId = `${alice.userId}:AAAA`;
const bobId = `${bob.userId}:BBBB`;
const localRtcMember = mockRtcMembership("@carol:example.org", "CCCC");
const local = mockMatrixRoomMember(localRtcMember);
const aliceRtcMember = mockRtcMembership("@alice:example.org", "AAAA");
const alice = mockMatrixRoomMember(aliceRtcMember);
const bobRtcMember = mockRtcMembership("@bob:example.org", "BBBB");
const localParticipant = mockLocalParticipant({ identity: "" });
const aliceId = `${alice.userId}:${aliceRtcMember.deviceId}`;
const aliceParticipant = mockRemoteParticipant({ identity: aliceId });
const bobParticipant = mockRemoteParticipant({ identity: bobId });
vitest.mock("../useAudioContext");
vitest.mock("../soundUtils");
afterEach(() => {
vitest.resetAllMocks();
@@ -57,7 +64,7 @@ afterAll(() => {
vitest.restoreAllMocks();
});
let playSound: Mock<
let playSound: MockedFunction<
NonNullable<ReturnType<typeof useAudioContext>>["playSound"]
>;
@@ -72,19 +79,65 @@ beforeEach(() => {
});
function TestComponent({
room,
rtcSession,
vm,
}: {
room: MockRoom;
rtcSession: MockRTCSession;
vm: CallViewModel;
}): ReactNode {
return (
<TestReactionsWrapper rtcSession={new MockRTCSession(room, {})}>
<TestReactionsWrapper
rtcSession={rtcSession as unknown as MatrixRTCSession}
>
<CallEventAudioRenderer vm={vm} />
</TestReactionsWrapper>
);
}
function getMockEnv(
members: RoomMember[],
initialRemoteRtcMemberships: CallMembership[] = [aliceRtcMember],
): {
vm: CallViewModel;
session: MockRTCSession;
remoteRtcMemberships: BehaviorSubject<CallMembership[]>;
} {
const matrixRoomMembers = new Map(members.map((p) => [p.userId, p]));
const remoteParticipants = of([aliceParticipant]);
const liveKitRoom = mockLivekitRoom(
{ localParticipant },
{ remoteParticipants },
);
const matrixRoom = mockMatrixRoom({
client: {
getUserId: () => localRtcMember.sender,
getDeviceId: () => localRtcMember.deviceId,
on: vitest.fn(),
off: vitest.fn(),
} as Partial<MatrixClient> as MatrixClient,
getMember: (userId) => matrixRoomMembers.get(userId) ?? null,
});
const remoteRtcMemberships = new BehaviorSubject<CallMembership[]>(
initialRemoteRtcMemberships,
);
const session = new MockRTCSession(
matrixRoom,
localRtcMember,
).withMemberships(remoteRtcMemberships);
const vm = new CallViewModel(
session as unknown as MatrixRTCSession,
liveKitRoom,
{
kind: E2eeType.PER_PARTICIPANT,
},
of(ConnectionState.Connected),
);
return { vm, session, remoteRtcMemberships };
}
/**
* We don't want to play a sound when loading the call state
* because typically this occurs in two stages. We first join
@@ -93,118 +146,56 @@ function TestComponent({
* a noise every time.
*/
test("plays one sound when entering a call", () => {
const liveKitRoom = new EmittableMockLivekitRoom({
localParticipant,
remoteParticipants: new Map(),
});
const room = new MockRoom(alice.userId);
const vm = new CallViewModel(
room.testGetAsMatrixRoom(),
liveKitRoom.getAsLivekitRoom(),
{
kind: E2eeType.PER_PARTICIPANT,
},
of(ConnectionState.Connected),
);
const { session, vm, remoteRtcMemberships } = getMockEnv([local, alice]);
render(<TestComponent rtcSession={session} vm={vm} />);
// Joining a call usually means remote participants are added later.
act(() => {
liveKitRoom.addParticipant(bobParticipant);
remoteRtcMemberships.next([aliceRtcMember, bobRtcMember]);
});
render(<TestComponent room={room} vm={vm} />);
expect(playSound).toBeCalled();
expect(playSound).toHaveBeenCalledOnce();
});
// TODO: Same test?
test("plays a sound when a user joins", () => {
const remoteParticipants = new Map(
[aliceParticipant].map((p) => [p.identity, p]),
);
const liveKitRoom = new EmittableMockLivekitRoom({
localParticipant,
remoteParticipants,
});
const room = new MockRoom(alice.userId);
const vm = new CallViewModel(
room.testGetAsMatrixRoom(),
liveKitRoom as unknown as Room,
{
kind: E2eeType.PER_PARTICIPANT,
},
of(ConnectionState.Connected),
);
render(<TestComponent room={room} vm={vm} />);
const { session, vm, remoteRtcMemberships } = getMockEnv([local, alice]);
render(<TestComponent rtcSession={session} vm={vm} />);
act(() => {
liveKitRoom.addParticipant(bobParticipant);
remoteRtcMemberships.next([aliceRtcMember, bobRtcMember]);
});
// Play a sound when joining a call.
expect(playSound).toBeCalledWith("join");
});
test("plays a sound when a user leaves", () => {
const remoteParticipants = new Map(
[aliceParticipant].map((p) => [p.identity, p]),
);
const liveKitRoom = new EmittableMockLivekitRoom({
localParticipant,
remoteParticipants,
});
const room = new MockRoom(alice.userId);
const vm = new CallViewModel(
room.testGetAsMatrixRoom(),
liveKitRoom.getAsLivekitRoom(),
{
kind: E2eeType.PER_PARTICIPANT,
},
of(ConnectionState.Connected),
);
render(<TestComponent room={room} vm={vm} />);
const { session, vm, remoteRtcMemberships } = getMockEnv([local, alice]);
render(<TestComponent rtcSession={session} vm={vm} />);
act(() => {
liveKitRoom.removeParticipant(aliceParticipant);
remoteRtcMemberships.next([]);
});
expect(playSound).toBeCalledWith("left");
});
test("plays no sound when the participant list is more than the maximum size", () => {
expect(playSound).not.toBeCalled();
const remoteParticipants = new Map<string, RemoteParticipant>([
[aliceParticipant.identity, aliceParticipant],
// You + other participants to hit the max.
...Array.from({ length: MAX_PARTICIPANT_COUNT_FOR_SOUND - 1 }).map<
[string, RemoteParticipant]
>((_, index) => {
const p = mockRemoteParticipant({
identity: `@user${index}:example.com:DEV${index}`,
});
return [p.identity, p];
}),
]);
const mockRtcMemberships: CallMembership[] = [];
for (let i = 0; i < MAX_PARTICIPANT_COUNT_FOR_SOUND; i++) {
mockRtcMemberships.push(
mockRtcMembership(`@user${i}:example.org`, `DEVICE${i}`),
);
}
// Preload the call with the maximum members, assume that
// we're already in the call by this point rather than
// joining.
const liveKitRoom = new EmittableMockLivekitRoom({
localParticipant,
remoteParticipants,
});
const room = new MockRoom(alice.userId);
const vm = new CallViewModel(
room.testGetAsMatrixRoom(),
liveKitRoom.getAsLivekitRoom(),
{
kind: E2eeType.PER_PARTICIPANT,
},
of(ConnectionState.Connected),
const { session, vm, remoteRtcMemberships } = getMockEnv(
[local, alice],
mockRtcMemberships,
);
render(<TestComponent room={room} vm={vm} />);
// When the count drops, play a leave sound.
render(<TestComponent rtcSession={session} vm={vm} />);
expect(playSound).not.toBeCalled();
act(() => {
liveKitRoom.removeParticipant(aliceParticipant);
remoteRtcMemberships.next(
mockRtcMemberships.slice(0, MAX_PARTICIPANT_COUNT_FOR_SOUND - 1),
);
});
expect(playSound).toBeCalledWith("left");
});

7
src/room/CallEventAudioRenderer.tsx
View File

@@ -15,7 +15,8 @@ import leftCallSoundMp3 from "../sound/left_call.mp3";
import leftCallSoundOgg from "../sound/left_call.ogg";
import handSoundOgg from "../sound/raise_hand.ogg?url";
import handSoundMp3 from "../sound/raise_hand.mp3?url";
import { prefetchSounds, useAudioContext } from "../useAudioContext";
import { useAudioContext } from "../useAudioContext";
import { prefetchSounds } from "../soundUtils";
import { useReactions } from "../useReactions";
import { useLatest } from "../useLatest";
@@ -24,7 +25,7 @@ import { useLatest } from "../useLatest";
export const MAX_PARTICIPANT_COUNT_FOR_SOUND = 8;
export const THROTTLE_SOUND_EFFECT_MS = 500;
export const CallEventAudioSounds = prefetchSounds({
const sounds = prefetchSounds({
join: {
mp3: joinCallSoundMp3,
ogg: joinCallSoundOgg,
@@ -45,7 +46,7 @@ export function CallEventAudioRenderer({
vm: CallViewModel;
}): ReactNode {
const audioEngineCtx = useAudioContext({
sounds: CallEventAudioSounds,
sounds,
latencyHint: "interactive",
});
const audioEngineRef = useLatest(audioEngineCtx);

2
src/room/GroupCallView.tsx
View File

@@ -343,7 +343,7 @@ export const GroupCallView: FC<Props> = ({
<ActiveCall
client={client}
matrixInfo={matrixInfo}
rtcSession={rtcSession}
rtcSession={rtcSession as unknown as MatrixRTCSession}
participantCount={participantCount}
onLeave={onLeave}
hideHeader={hideHeader}

9
src/room/InCallView.tsx
View File

@@ -120,7 +120,7 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
useEffect(() => {
if (livekitRoom !== undefined) {
const vm = new CallViewModel(
props.rtcSession.room,
props.rtcSession,
livekitRoom,
props.e2eeSystem,
connStateObservable,
@@ -128,12 +128,7 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
setVm(vm);
return (): void => vm.destroy();
}
}, [
props.rtcSession.room,
livekitRoom,
props.e2eeSystem,
connStateObservable,
]);
}, [props.rtcSession, livekitRoom, props.e2eeSystem, connStateObservable]);
if (livekitRoom === undefined || vm === null) return null;

4
src/room/ReactionAudioRenderer.test.tsx
View File

@@ -29,8 +29,9 @@ import {
playReactionsSound,
soundEffectVolumeSetting,
} from "../settings/settings";
import { prefetchSounds, useAudioContext } from "../useAudioContext";
import { useAudioContext } from "../useAudioContext";
import { GenericReaction, ReactionSet } from "../reactions";
import { prefetchSounds } from "../soundUtils";
const memberUserIdAlice = "@alice:example.org";
const memberUserIdBob = "@bob:example.org";
@@ -60,6 +61,7 @@ function TestComponent({
}
vitest.mock("../useAudioContext");
vitest.mock("../soundUtils");
afterEach(() => {
vitest.resetAllMocks();

28
src/room/ReactionAudioRenderer.tsx
View File

@@ -5,15 +5,16 @@ SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { ReactNode, useDeferredValue, useEffect } from "react";
import { ReactNode, useDeferredValue, useEffect, useState } from "react";
import { useReactions } from "../useReactions";
import { playReactionsSound, useSetting } from "../settings/settings";
import { GenericReaction, ReactionSet } from "../reactions";
import { prefetchSounds, useAudioContext } from "../useAudioContext";
import { useAudioContext } from "../useAudioContext";
import { prefetchSounds } from "../soundUtils";
import { useLatest } from "../useLatest";
const SoundMap = Object.fromEntries([
const soundMap = Object.fromEntries([
...ReactionSet.filter((v) => v.sound !== undefined).map((v) => [
v.name,
v.sound!,
@@ -21,18 +22,29 @@ const SoundMap = Object.fromEntries([
[GenericReaction.name, GenericReaction.sound],
]);
const Sounds = prefetchSounds(SoundMap);
export function ReactionsAudioRenderer(): ReactNode {
const { reactions } = useReactions();
const [shouldPlay] = useSetting(playReactionsSound);
const [soundCache, setSoundCache] = useState<ReturnType<
typeof prefetchSounds
> | null>(null);
const audioEngineCtx = useAudioContext({
sounds: Sounds,
sounds: soundCache,
latencyHint: "interactive",
});
const audioEngineRef = useLatest(audioEngineCtx);
const oldReactions = useDeferredValue(reactions);
useEffect(() => {
if (!shouldPlay || soundCache) {
return;
}
// This is fine even if we load the component multiple times,
// as the browser's cache should ensure once the media is loaded
// once that future fetches come via the cache.
setSoundCache(prefetchSounds(soundMap));
}, [soundCache, shouldPlay]);
useEffect(() => {
if (!shouldPlay || !audioEngineRef.current) {
return;
@@ -47,7 +59,7 @@ export function ReactionsAudioRenderer(): ReactNode {
// Don't replay old reactions
return;
}
if (SoundMap[reactionName]) {
if (soundMap[reactionName]) {
void audioEngineRef.current.playSound(reactionName);
} else {
// Fallback sounds.
@@ -55,5 +67,5 @@ export function ReactionsAudioRenderer(): ReactNode {
}
}
}, [audioEngineRef, shouldPlay, oldReactions, reactions]);
return <></>;
return null;
}

5
src/room/VideoPreview.module.css
View File

@@ -18,8 +18,7 @@ Please see LICENSE in the repository root for full details.
width: 100%;
height: 100%;
object-fit: cover;
background-color: black;
background-color: var(--cpd-color-bg-subtle-primary);
background-color: var(--video-tile-background);
}
video.mirror {
@@ -35,7 +34,7 @@ video.mirror {
display: flex;
justify-content: center;
align-items: center;
background-color: var(--cpd-color-bg-subtle-secondary);
background-color: var(--video-tile-background);
}
.buttonBar {

5
src/settings/SettingsModal.tsx
View File

@@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { ChangeEvent, FC, useCallback, useEffect, useState } from "react";
import { ChangeEvent, FC, useCallback, useState } from "react";
import { Trans, useTranslation } from "react-i18next";
import { MatrixClient } from "matrix-js-sdk/src/matrix";
import { Root as Form, Text } from "@vector-im/compound-web";
@@ -86,9 +86,6 @@ export const SettingsModal: FC<Props> = ({
const [soundVolume, setSoundVolume] = useSetting(soundEffectVolumeSetting);
const [soundVolumeRaw, setSoundVolumeRaw] = useState(soundVolume);
// Debounce saving the sound volume as it triggers certain components to reload.
useEffect(() => {});
const audioTab: Tab<SettingsTab> = {
key: "audio",
name: t("common.audio"),

63
src/soundUtils.ts Normal file
View File

@@ -0,0 +1,63 @@
/*
Copyright 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { logger } from "matrix-js-sdk/src/logger";
type SoundDefinition = { mp3?: string; ogg: string };
export type PrefetchedSounds<S extends string> = Promise<
Record<S, ArrayBuffer>
>;
/**
* Determine the best format we can use to play our sounds
* through. We prefer ogg support if possible, but will fall
* back to MP3.
* @returns "ogg" if the browser is likely to support it, or "mp3" otherwise.
*/
function getPreferredAudioFormat(): "ogg" | "mp3" {
const a = document.createElement("audio");
if (a.canPlayType("audio/ogg") === "maybe") {
return "ogg";
}
// Otherwise just assume MP3, as that has a chance of being more widely supported.
return "mp3";
}
const preferredFormat = getPreferredAudioFormat();
/**
* Prefetch sounds to be used by the AudioContext. This can
* be called outside the scope of a component to ensure the
* sounds load ahead of time.
* @param sounds A set of sound files that may be played.
* @returns A map of sound files to buffers.
*/
export async function prefetchSounds<S extends string>(
sounds: Record<S, SoundDefinition>,
): PrefetchedSounds<S> {
const buffers: Record<string, ArrayBuffer> = {};
await Promise.all(
Object.entries(sounds).map(async ([name, file]) => {
const { mp3, ogg } = file as SoundDefinition;
// Use preferred format, fallback to ogg if no mp3 is provided.
// Load an audio file
const response = await fetch(
preferredFormat === "ogg" ? ogg : (mp3 ?? ogg),
);
if (!response.ok) {
// If the sound doesn't load, it's not the end of the world. We won't play
// the sound when requested, but it's better than failing the whole application.
logger.warn(`Could not load sound ${name}, response was not okay`);
return;
}
// Decode it
buffers[name] = await response.arrayBuffer();
}),
);
return buffers as Record<S, ArrayBuffer>;
}

157
src/state/CallViewModel.test.ts
View File

@@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { test, vi, onTestFinished } from "vitest";
import { test, vi, onTestFinished, it } from "vitest";
import {
combineLatest,
debounceTime,
@@ -25,6 +25,7 @@ import {
} from "livekit-client";
import * as ComponentsCore from "@livekit/components-core";
import { isEqual } from "lodash-es";
import { CallMembership, MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc";
import { CallViewModel, Layout } from "./CallViewModel";
import {
@@ -34,6 +35,8 @@ import {
mockMatrixRoomMember,
mockRemoteParticipant,
withTestScheduler,
mockRtcMembership,
MockRTCSession,
} from "../utils/test";
import {
ECAddonConnectionState,
@@ -43,14 +46,19 @@ import { E2eeType } from "../e2ee/e2eeType";
vi.mock("@livekit/components-core");
const alice = mockMatrixRoomMember({ userId: "@alice:example.org" });
const bob = mockMatrixRoomMember({ userId: "@bob:example.org" });
const carol = mockMatrixRoomMember({ userId: "@carol:example.org" });
const dave = mockMatrixRoomMember({ userId: "@dave:example.org" });
const localRtcMember = mockRtcMembership("@carol:example.org", "CCCC");
const aliceRtcMember = mockRtcMembership("@alice:example.org", "AAAA");
const bobRtcMember = mockRtcMembership("@bob:example.org", "BBBB");
const daveRtcMember = mockRtcMembership("@dave:example.org", "DDDD");
const aliceId = `${alice.userId}:AAAA`;
const bobId = `${bob.userId}:BBBB`;
const daveId = `${dave.userId}:DDDD`;
const alice = mockMatrixRoomMember(aliceRtcMember);
const bob = mockMatrixRoomMember(bobRtcMember);
const carol = mockMatrixRoomMember(localRtcMember);
const dave = mockMatrixRoomMember(daveRtcMember);
const aliceId = `${alice.userId}:${aliceRtcMember.deviceId}`;
const bobId = `${bob.userId}:${bobRtcMember.deviceId}`;
const daveId = `${dave.userId}:${daveRtcMember.deviceId}`;
const localParticipant = mockLocalParticipant({ identity: "" });
const aliceParticipant = mockRemoteParticipant({ identity: aliceId });
@@ -65,7 +73,9 @@ const bobSharingScreen = mockRemoteParticipant({
});
const daveParticipant = mockRemoteParticipant({ identity: daveId });
const members = new Map([alice, bob, carol, dave].map((p) => [p.userId, p]));
const roomMembers = new Map(
[alice, bob, carol, dave].map((p) => [p.userId, p]),
);
export interface GridLayoutSummary {
type: "grid";
@@ -173,10 +183,23 @@ function summarizeLayout(l: Observable<Layout>): Observable<LayoutSummary> {
function withCallViewModel(
remoteParticipants: Observable<RemoteParticipant[]>,
rtcMembers: Observable<Partial<CallMembership>[]>,
connectionState: Observable<ECConnectionState>,
speaking: Map<Participant, Observable<boolean>>,
continuation: (vm: CallViewModel) => void,
): void {
const room = mockMatrixRoom({
client: {
getUserId: () => localRtcMember.sender,
getDeviceId: () => localRtcMember.deviceId,
} as Partial<MatrixClient> as MatrixClient,
getMember: (userId) => roomMembers.get(userId) ?? null,
});
const rtcSession = new MockRTCSession(
room,
localRtcMember,
[],
).withMemberships(rtcMembers);
const participantsSpy = vi
.spyOn(ComponentsCore, "connectedParticipantsObserver")
.mockReturnValue(remoteParticipants);
@@ -209,12 +232,7 @@ function withCallViewModel(
);
const vm = new CallViewModel(
mockMatrixRoom({
client: {
getUserId: () => "@carol:example.org",
} as Partial<MatrixClient> as MatrixClient,
getMember: (userId) => members.get(userId) ?? null,
}),
rtcSession as unknown as MatrixRTCSession,
liveKitRoom,
{
kind: E2eeType.PER_PARTICIPANT,
@@ -247,6 +265,7 @@ test("participants are retained during a focus switch", () => {
a: [aliceParticipant, bobParticipant],
b: [],
}),
of([aliceRtcMember, bobRtcMember]),
hot(connectionInputMarbles, {
c: ConnectionState.Connected,
s: ECAddonConnectionState.ECSwitchingFocus,
@@ -288,6 +307,7 @@ test("screen sharing activates spotlight layout", () => {
c: [aliceSharingScreen, bobSharingScreen],
d: [aliceParticipant, bobSharingScreen],
}),
of([aliceRtcMember, bobRtcMember]),
of(ConnectionState.Connected),
new Map(),
(vm) => {
@@ -356,7 +376,7 @@ test("participants stay in the same order unless to appear/disappear", () => {
const modeInputMarbles = " a";
// First Bob speaks, then Dave, then Alice
const aSpeakingInputMarbles = "n- 1998ms - 1999ms y";
const bSpeakingInputMarbles = "ny 1998ms n 1999ms ";
const bSpeakingInputMarbles = "ny 1998ms n 1999ms -";
const dSpeakingInputMarbles = "n- 1998ms y 1999ms n";
// Nothing should change when Bob speaks, because Bob is already on screen.
// When Dave speaks he should switch with Alice because she's the one who
@@ -366,6 +386,7 @@ test("participants stay in the same order unless to appear/disappear", () => {
withCallViewModel(
of([aliceParticipant, bobParticipant, daveParticipant]),
of([aliceRtcMember, bobRtcMember, daveRtcMember]),
of(ConnectionState.Connected),
new Map([
[aliceParticipant, hot(aSpeakingInputMarbles, { y: true, n: false })],
@@ -427,6 +448,7 @@ test("spotlight speakers swap places", () => {
withCallViewModel(
of([aliceParticipant, bobParticipant, daveParticipant]),
of([aliceRtcMember, bobRtcMember, daveRtcMember]),
of(ConnectionState.Connected),
new Map([
[aliceParticipant, hot(aSpeakingInputMarbles, { y: true, n: false })],
@@ -475,6 +497,7 @@ test("layout enters picture-in-picture mode when requested", () => {
withCallViewModel(
of([aliceParticipant, bobParticipant]),
of([aliceRtcMember, bobRtcMember]),
of(ConnectionState.Connected),
new Map(),
(vm) => {
@@ -515,6 +538,7 @@ test("spotlight remembers whether it's expanded", () => {
withCallViewModel(
of([aliceParticipant, bobParticipant]),
of([aliceRtcMember, bobRtcMember]),
of(ConnectionState.Connected),
new Map(),
(vm) => {
@@ -559,3 +583,104 @@ test("spotlight remembers whether it's expanded", () => {
);
});
});
test("participants must have a MatrixRTCSession to be visible", () => {
withTestScheduler(({ hot, expectObservable }) => {
// iterate through a number of combinations of participants and MatrixRTC memberships
// Bob never has an MatrixRTC membership
const scenarioInputMarbles = " abcdec";
// Bob should never be visible
const expectedLayoutMarbles = "a-bc-b";
withCallViewModel(
hot(scenarioInputMarbles, {
a: [],
b: [bobParticipant],
c: [aliceParticipant, bobParticipant],
d: [aliceParticipant, daveParticipant, bobParticipant],
e: [aliceParticipant, daveParticipant, bobSharingScreen],
}),
hot(scenarioInputMarbles, {
a: [],
b: [],
c: [aliceRtcMember],
d: [aliceRtcMember, daveRtcMember],
e: [aliceRtcMember, daveRtcMember],
}),
of(ConnectionState.Connected),
new Map(),
(vm) => {
vm.setGridMode("grid");
expectObservable(summarizeLayout(vm.layout)).toBe(
expectedLayoutMarbles,
{
a: {
type: "grid",
spotlight: undefined,
grid: ["local:0"],
},
b: {
type: "one-on-one",
local: "local:0",
remote: `${aliceId}:0`,
},
c: {
type: "grid",
spotlight: undefined,
grid: ["local:0", `${aliceId}:0`, `${daveId}:0`],
},
},
);
},
);
});
});
it("should show at least one tile per MatrixRTCSession", () => {
withTestScheduler(({ hot, expectObservable }) => {
// iterate through some combinations of MatrixRTC memberships
const scenarioInputMarbles = " abcd";
// There should always be one tile for each MatrixRTCSession
const expectedLayoutMarbles = "abcd";
withCallViewModel(
of([]),
hot(scenarioInputMarbles, {
a: [],
b: [aliceRtcMember],
c: [aliceRtcMember, daveRtcMember],
d: [daveRtcMember],
}),
of(ConnectionState.Connected),
new Map(),
(vm) => {
vm.setGridMode("grid");
expectObservable(summarizeLayout(vm.layout)).toBe(
expectedLayoutMarbles,
{
a: {
type: "grid",
spotlight: undefined,
grid: ["local:0"],
},
b: {
type: "one-on-one",
local: "local:0",
remote: `${aliceId}:0`,
},
c: {
type: "grid",
spotlight: undefined,
grid: ["local:0", `${aliceId}:0`, `${daveId}:0`],
},
d: {
type: "one-on-one",
local: "local:0",
remote: `${daveId}:0`,
},
},
);
},
);
});
});

275
src/state/CallViewModel.ts
View File

@@ -18,12 +18,9 @@ import {
RemoteParticipant,
Track,
} from "livekit-client";
import { Room as MatrixRoom, RoomMember } from "matrix-js-sdk/src/matrix";
import {
Room as MatrixRoom,
RoomMember,
RoomStateEvent,
} from "matrix-js-sdk/src/matrix";
import {
BehaviorSubject,
EMPTY,
Observable,
Subject,
@@ -49,6 +46,10 @@ import {
withLatestFrom,
} from "rxjs";
import { logger } from "matrix-js-sdk/src/logger";
import {
MatrixRTCSession,
MatrixRTCSessionEvent,
} from "matrix-js-sdk/src/matrixrtc";
import { ViewModel } from "./ViewModel";
import {
@@ -222,41 +223,67 @@ interface LayoutScanState {
class UserMedia {
private readonly scope = new ObservableScope();
public readonly vm: UserMediaViewModel;
private readonly participant: BehaviorSubject<
LocalParticipant | RemoteParticipant | undefined
>;
public readonly speaker: Observable<boolean>;
public readonly presenter: Observable<boolean>;
public constructor(
public readonly id: string,
member: RoomMember | undefined,
participant: LocalParticipant | RemoteParticipant,
participant: LocalParticipant | RemoteParticipant | undefined,
encryptionSystem: EncryptionSystem,
livekitRoom: LivekitRoom,
) {
this.vm = participant.isLocal
? new LocalUserMediaViewModel(
id,
member,
participant as LocalParticipant,
encryptionSystem,
livekitRoom,
)
: new RemoteUserMediaViewModel(
id,
member,
participant as RemoteParticipant,
encryptionSystem,
livekitRoom,
);
this.participant = new BehaviorSubject(participant);
if (participant?.isLocal) {
this.vm = new LocalUserMediaViewModel(
this.id,
member,
this.participant.asObservable() as Observable<LocalParticipant>,
encryptionSystem,
livekitRoom,
);
} else {
this.vm = new RemoteUserMediaViewModel(
id,
member,
this.participant.asObservable() as Observable<
RemoteParticipant | undefined
>,
encryptionSystem,
livekitRoom,
);
}
this.speaker = observeSpeaker(this.vm.speaking).pipe(this.scope.state());
this.presenter = observeParticipantEvents(
participant,
ParticipantEvent.TrackPublished,
ParticipantEvent.TrackUnpublished,
ParticipantEvent.LocalTrackPublished,
ParticipantEvent.LocalTrackUnpublished,
).pipe(map((p) => p.isScreenShareEnabled));
this.presenter = this.participant.pipe(
switchMap(
(p) =>
(p &&
observeParticipantEvents(
p,
ParticipantEvent.TrackPublished,
ParticipantEvent.TrackUnpublished,
ParticipantEvent.LocalTrackPublished,
ParticipantEvent.LocalTrackUnpublished,
).pipe(map((p) => p.isScreenShareEnabled))) ??
of(false),
),
this.scope.state(),
);
}
public updateParticipant(
newParticipant: LocalParticipant | RemoteParticipant | undefined,
): void {
if (this.participant.value !== newParticipant) {
// Update the BehaviourSubject in the UserMedia.
this.participant.next(newParticipant);
}
}
public destroy(): void {
@@ -267,6 +294,9 @@ class UserMedia {
class ScreenShare {
public readonly vm: ScreenShareViewModel;
private readonly participant: BehaviorSubject<
LocalParticipant | RemoteParticipant
>;
public constructor(
id: string,
@@ -275,12 +305,15 @@ class ScreenShare {
encryptionSystem: EncryptionSystem,
liveKitRoom: LivekitRoom,
) {
this.participant = new BehaviorSubject(participant);
this.vm = new ScreenShareViewModel(
id,
member,
participant,
this.participant.asObservable(),
encryptionSystem,
liveKitRoom,
participant.isLocal,
);
}
@@ -317,11 +350,11 @@ function findMatrixRoomMember(
export class CallViewModel extends ViewModel {
public readonly localVideo: Observable<LocalVideoTrack | null> =
observeTrackReference(
this.livekitRoom.localParticipant,
of(this.livekitRoom.localParticipant),
Track.Source.Camera,
).pipe(
map((trackRef) => {
const track = trackRef.publication?.track;
const track = trackRef?.publication?.track;
return track instanceof LocalVideoTrack ? track : null;
}),
);
@@ -401,49 +434,87 @@ export class CallViewModel extends ViewModel {
this.remoteParticipants,
observeParticipantMedia(this.livekitRoom.localParticipant),
duplicateTiles.value,
// Also react to changes in the list of members
fromEvent(this.matrixRoom, RoomStateEvent.Update).pipe(startWith(null)),
// Also react to changes in the MatrixRTC session list.
// The session list will also be update if a room membership changes.
// No additional RoomState event listener needs to be set up.
fromEvent(
this.matrixRTCSession,
MatrixRTCSessionEvent.MembershipsChanged,
).pipe(startWith(null)),
]).pipe(
scan(
(
prevItems,
[remoteParticipants, { participant: localParticipant }, duplicateTiles],
[
remoteParticipants,
{ participant: localParticipant },
duplicateTiles,
_membershipsChanged,
],
) => {
const newItems = new Map(
function* (this: CallViewModel): Iterable<[string, MediaItem]> {
for (const p of [localParticipant, ...remoteParticipants]) {
const id = p === localParticipant ? "local" : p.identity;
const member = findMatrixRoomMember(this.matrixRoom, id);
if (member === undefined)
logger.warn(
`Ruh, roh! No matrix member found for SFU participant '${p.identity}': creating g-g-g-ghost!`,
);
// m.rtc.members are the basis for calculating what is visible in the call
for (const rtcMember of this.matrixRTCSession.memberships) {
const room = this.matrixRTCSession.room;
// WARN! This is not exactly the sender but the user defined in the state key.
// This will be available once we change to the new "member as object" format in the MatrixRTC object.
let livekitParticipantId =
rtcMember.sender + ":" + rtcMember.deviceId;
// Create as many tiles for this participant as called for by
// the duplicateTiles option
let participant:
| LocalParticipant
| RemoteParticipant
| undefined = undefined;
if (
rtcMember.sender === room.client.getUserId()! &&
rtcMember.deviceId === room.client.getDeviceId()
) {
livekitParticipantId = "local";
participant = localParticipant;
} else {
participant = remoteParticipants.find(
(p) => p.identity === livekitParticipantId,
);
}
const member = findMatrixRoomMember(room, livekitParticipantId);
if (!member) {
logger.error(
"Could not find member for media id: ",
livekitParticipantId,
);
}
for (let i = 0; i < 1 + duplicateTiles; i++) {
const userMediaId = `${id}:${i}`;
const indexedMediaId = `${livekitParticipantId}:${i}`;
const prevMedia = prevItems.get(indexedMediaId);
if (prevMedia && prevMedia instanceof UserMedia) {
prevMedia.updateParticipant(participant);
}
yield [
userMediaId,
prevItems.get(userMediaId) ??
indexedMediaId,
// We create UserMedia with or without a participant.
// This will be the initial value of a BehaviourSubject.
// Once a participant appears we will update the BehaviourSubject. (see above)
prevMedia ??
new UserMedia(
userMediaId,
indexedMediaId,
member,
p,
participant,
this.encryptionSystem,
this.livekitRoom,
),
];
if (p.isScreenShareEnabled) {
const screenShareId = `${userMediaId}:screen-share`;
if (participant?.isScreenShareEnabled) {
const screenShareId = `${indexedMediaId}:screen-share`;
yield [
screenShareId,
prevItems.get(screenShareId) ??
new ScreenShare(
screenShareId,
member,
p,
participant,
this.encryptionSystem,
this.livekitRoom,
),
@@ -454,7 +525,6 @@ export class CallViewModel extends ViewModel {
}.bind(this)(),
);
for (const [id, t] of prevItems) if (!newItems.has(id)) t.destroy();
return newItems;
},
new Map<string, MediaItem>(),
@@ -488,11 +558,6 @@ export class CallViewModel extends ViewModel {
),
);
private readonly localUserMedia: Observable<LocalUserMediaViewModel> =
this.mediaItems.pipe(
map((ms) => ms.find((m) => m.vm.local)!.vm as LocalUserMediaViewModel),
);
/**
* List of MediaItems that we want to display, that are of type ScreenShare
*/
@@ -504,7 +569,7 @@ export class CallViewModel extends ViewModel {
this.scope.state(),
);
private readonly spotlightSpeaker: Observable<UserMediaViewModel> =
private readonly spotlightSpeaker: Observable<UserMediaViewModel | null> =
this.userMedia.pipe(
switchMap((mediaItems) =>
mediaItems.length === 0
@@ -515,7 +580,7 @@ export class CallViewModel extends ViewModel {
),
),
),
scan<(readonly [UserMedia, boolean])[], UserMedia, null>(
scan<(readonly [UserMedia, boolean])[], UserMedia | undefined, null>(
(prev, mediaItems) => {
// Only remote users that are still in the call should be sticky
const [stickyMedia, stickySpeaking] =
@@ -532,11 +597,11 @@ export class CallViewModel extends ViewModel {
// Otherwise, spotlight an arbitrary remote user
mediaItems.find(([m]) => !m.vm.local)?.[0] ??
// Otherwise, spotlight the local user
mediaItems.find(([m]) => m.vm.local)![0]);
mediaItems.find(([m]) => m.vm.local)?.[0]);
},
null,
),
map((speaker) => speaker.vm),
map((speaker) => speaker?.vm ?? null),
this.scope.state(),
);
@@ -576,37 +641,57 @@ export class CallViewModel extends ViewModel {
}),
);
private readonly spotlightAndPip: Observable<
[Observable<MediaViewModel[]>, Observable<UserMediaViewModel | null>]
> = this.screenShares.pipe(
map((screenShares) =>
screenShares.length > 0
? ([of(screenShares.map((m) => m.vm)), this.spotlightSpeaker] as const)
: ([
this.spotlightSpeaker.pipe(map((speaker) => [speaker!])),
this.spotlightSpeaker.pipe(
switchMap((speaker) =>
speaker.local
? of(null)
: this.localUserMedia.pipe(
switchMap((vm) =>
vm.alwaysShow.pipe(
map((alwaysShow) => (alwaysShow ? vm : null)),
),
),
),
),
),
] as const),
),
);
private readonly spotlight: Observable<MediaViewModel[]> =
this.spotlightAndPip.pipe(
switchMap(([spotlight]) => spotlight),
this.screenShares.pipe(
switchMap((screenShares) => {
if (screenShares.length > 0) {
return of(screenShares.map((m) => m.vm));
}
return this.spotlightSpeaker.pipe(
map((speaker) => (speaker ? [speaker] : [])),
);
}),
this.scope.state(),
);
private readonly pip: Observable<UserMediaViewModel | null> = combineLatest([
this.screenShares,
this.spotlightSpeaker,
this.mediaItems,
]).pipe(
switchMap(([screenShares, spotlight, mediaItems]) => {
if (screenShares.length > 0) {
return this.spotlightSpeaker;
}
if (!spotlight || spotlight.local) {
return of(null);
}
const localUserMedia = mediaItems.find(
(m) => m.vm instanceof LocalUserMediaViewModel,
) as UserMedia | undefined;
const localUserMediaViewModel = localUserMedia?.vm as
| LocalUserMediaViewModel
| undefined;
if (!localUserMediaViewModel) {
return of(null);
}
return localUserMediaViewModel.alwaysShow.pipe(
map((alwaysShow) => {
if (alwaysShow) {
return localUserMediaViewModel;
}
return null;
}),
);
}),
this.scope.state(),
);
private readonly hasRemoteScreenShares: Observable<boolean> =
this.spotlight.pipe(
map((spotlight) =>
@@ -615,9 +700,6 @@ export class CallViewModel extends ViewModel {
distinctUntilChanged(),
);
private readonly pip: Observable<UserMediaViewModel | null> =
this.spotlightAndPip.pipe(switchMap(([, pip]) => pip));
private readonly pipEnabled: Observable<boolean> = setPipEnabled.pipe(
startWith(false),
);
@@ -721,15 +803,16 @@ export class CallViewModel extends ViewModel {
this.mediaItems.pipe(
map((mediaItems) => {
if (mediaItems.length !== 2) return null;
const local = mediaItems.find((vm) => vm.vm.local)!
.vm as LocalUserMediaViewModel;
const local = mediaItems.find((vm) => vm.vm.local)?.vm as
| LocalUserMediaViewModel
| undefined;
const remote = mediaItems.find((vm) => !vm.vm.local)?.vm as
| RemoteUserMediaViewModel
| undefined;
// There might not be a remote tile if there are screen shares, or if
// only the local user is in the call and they're using the duplicate
// tiles option
if (remote === undefined) return null;
if (!remote || !local) return null;
return { type: "one-on-one", local, remote };
}),
@@ -1010,7 +1093,7 @@ export class CallViewModel extends ViewModel {
public constructor(
// A call is permanently tied to a single Matrix room and LiveKit room
private readonly matrixRoom: MatrixRoom,
private readonly matrixRTCSession: MatrixRTCSession,
private readonly livekitRoom: LivekitRoom,
private readonly encryptionSystem: EncryptionSystem,
private readonly connectionState: Observable<ECConnectionState>,

23
src/state/MediaViewModel.test.ts
View File

@@ -8,14 +8,17 @@ Please see LICENSE in the repository root for full details.
import { expect, test, vi } from "vitest";
import {
mockRtcMembership,
withLocalMedia,
withRemoteMedia,
withTestScheduler,
} from "../utils/test";
const rtcMembership = mockRtcMembership("@alice:example.org", "AAAA");
test("control a participant's volume", async () => {
const setVolumeSpy = vi.fn();
await withRemoteMedia({}, { setVolume: setVolumeSpy }, (vm) =>
await withRemoteMedia(rtcMembership, {}, { setVolume: setVolumeSpy }, (vm) =>
withTestScheduler(({ expectObservable, schedule }) => {
schedule("-ab---c---d|", {
a() {
@@ -60,7 +63,7 @@ test("control a participant's volume", async () => {
});
test("toggle fit/contain for a participant's video", async () => {
await withRemoteMedia({}, {}, (vm) =>
await withRemoteMedia(rtcMembership, {}, {}, (vm) =>
withTestScheduler(({ expectObservable, schedule }) => {
schedule("-ab|", {
a: () => vm.toggleFitContain(),
@@ -76,17 +79,21 @@ test("toggle fit/contain for a participant's video", async () => {
});
test("local media remembers whether it should always be shown", async () => {
await withLocalMedia({}, (vm) =>
await withLocalMedia(rtcMembership, {}, (vm) =>
withTestScheduler(({ expectObservable, schedule }) => {
schedule("-a|", { a: () => vm.setAlwaysShow(false) });
expectObservable(vm.alwaysShow).toBe("ab", { a: true, b: false });
}),
);
// Next local media should start out *not* always shown
await withLocalMedia({}, (vm) =>
withTestScheduler(({ expectObservable, schedule }) => {
schedule("-a|", { a: () => vm.setAlwaysShow(true) });
expectObservable(vm.alwaysShow).toBe("ab", { a: false, b: true });
}),
await withLocalMedia(
rtcMembership,
{},
(vm) =>
withTestScheduler(({ expectObservable, schedule }) => {
schedule("-a|", { a: () => vm.setAlwaysShow(true) });
expectObservable(vm.alwaysShow).toBe("ab", { a: false, b: true });
}),
);
});

223
src/state/MediaViewModel.ts
View File

@@ -32,7 +32,6 @@ import {
Observable,
Subject,
combineLatest,
distinctUntilChanged,
distinctUntilKeyChanged,
filter,
fromEvent,
@@ -40,7 +39,6 @@ import {
map,
merge,
of,
shareReplay,
startWith,
switchMap,
throttleTime,
@@ -77,16 +75,24 @@ export function useDisplayName(vm: MediaViewModel): string {
}
export function observeTrackReference(
participant: Participant,
participant: Observable<Participant | undefined>,
source: Track.Source,
): Observable<TrackReferenceOrPlaceholder> {
return observeParticipantMedia(participant).pipe(
map(() => ({
participant,
publication: participant.getTrackPublication(source),
source,
})),
distinctUntilKeyChanged("publication"),
): Observable<TrackReferenceOrPlaceholder | undefined> {
return participant.pipe(
switchMap((p) => {
if (p) {
return observeParticipantMedia(p).pipe(
map(() => ({
participant: p,
publication: p.getTrackPublication(source),
source,
})),
distinctUntilKeyChanged("publication"),
);
} else {
return of(undefined);
}
}),
);
}
@@ -105,11 +111,11 @@ function observeRemoteTrackReceivingOkay(
};
return combineLatest([
observeTrackReference(participant, source),
observeTrackReference(of(participant), source),
interval(1000).pipe(startWith(0)),
]).pipe(
switchMap(async ([trackReference]) => {
const track = trackReference.publication?.track;
const track = trackReference?.publication?.track;
if (!track || !(track instanceof RemoteTrack)) {
return undefined;
}
@@ -200,14 +206,10 @@ export enum EncryptionStatus {
}
abstract class BaseMediaViewModel extends ViewModel {
/**
* Whether the media belongs to the local user.
*/
public readonly local = this.participant.isLocal;
/**
* The LiveKit video track for this media.
*/
public readonly video: Observable<TrackReferenceOrPlaceholder>;
public readonly video: Observable<TrackReferenceOrPlaceholder | undefined>;
/**
* Whether there should be a warning that this media is unencrypted.
*/
@@ -215,6 +217,11 @@ abstract class BaseMediaViewModel extends ViewModel {
public readonly encryptionStatus: Observable<EncryptionStatus>;
/**
* Whether this media corresponds to the local participant.
*/
public abstract readonly local: boolean;
public constructor(
/**
* An opaque identifier for this media.
@@ -226,7 +233,12 @@ abstract class BaseMediaViewModel extends ViewModel {
// TODO: Fully separate the data layer from the UI layer by keeping the
// member object internal
public readonly member: RoomMember | undefined,
protected readonly participant: LocalParticipant | RemoteParticipant,
// We don't necessarily have a participant if a user connects via MatrixRTC but not (yet) through
// livekit.
protected readonly participant: Observable<
LocalParticipant | RemoteParticipant | undefined
>,
encryptionSystem: EncryptionSystem,
audioSource: AudioSource,
videoSource: VideoSource,
@@ -243,69 +255,72 @@ abstract class BaseMediaViewModel extends ViewModel {
[audio, this.video],
(a, v) =>
encryptionSystem.kind !== E2eeType.NONE &&
(a.publication?.isEncrypted === false ||
v.publication?.isEncrypted === false),
).pipe(
distinctUntilChanged(),
shareReplay({ bufferSize: 1, refCount: false }),
);
(a?.publication?.isEncrypted === false ||
v?.publication?.isEncrypted === false),
).pipe(this.scope.state());
if (participant.isLocal || encryptionSystem.kind === E2eeType.NONE) {
this.encryptionStatus = of(EncryptionStatus.Okay).pipe(
this.scope.state(),
);
} else if (encryptionSystem.kind === E2eeType.PER_PARTICIPANT) {
this.encryptionStatus = combineLatest([
encryptionErrorObservable(
livekitRoom,
participant,
encryptionSystem,
"MissingKey",
),
encryptionErrorObservable(
livekitRoom,
participant,
encryptionSystem,
"InvalidKey",
),
observeRemoteTrackReceivingOkay(participant, audioSource),
observeRemoteTrackReceivingOkay(participant, videoSource),
]).pipe(
map(([keyMissing, keyInvalid, audioOkay, videoOkay]) => {
if (keyMissing) return EncryptionStatus.KeyMissing;
if (keyInvalid) return EncryptionStatus.KeyInvalid;
if (audioOkay || videoOkay) return EncryptionStatus.Okay;
return undefined; // no change
}),
filter((x) => !!x),
startWith(EncryptionStatus.Connecting),
this.scope.state(),
);
} else {
this.encryptionStatus = combineLatest([
encryptionErrorObservable(
livekitRoom,
participant,
encryptionSystem,
"InvalidKey",
),
observeRemoteTrackReceivingOkay(participant, audioSource),
observeRemoteTrackReceivingOkay(participant, videoSource),
]).pipe(
map(
([keyInvalid, audioOkay, videoOkay]):
| EncryptionStatus
| undefined => {
if (keyInvalid) return EncryptionStatus.PasswordInvalid;
if (audioOkay || videoOkay) return EncryptionStatus.Okay;
return undefined; // no change
},
),
filter((x) => !!x),
startWith(EncryptionStatus.Connecting),
this.scope.state(),
);
}
this.encryptionStatus = this.participant.pipe(
switchMap((participant): Observable<EncryptionStatus> => {
if (!participant) {
return of(EncryptionStatus.Connecting);
} else if (
participant.isLocal ||
encryptionSystem.kind === E2eeType.NONE
) {
return of(EncryptionStatus.Okay);
} else if (encryptionSystem.kind === E2eeType.PER_PARTICIPANT) {
return combineLatest([
encryptionErrorObservable(
livekitRoom,
participant,
encryptionSystem,
"MissingKey",
),
encryptionErrorObservable(
livekitRoom,
participant,
encryptionSystem,
"InvalidKey",
),
observeRemoteTrackReceivingOkay(participant, audioSource),
observeRemoteTrackReceivingOkay(participant, videoSource),
]).pipe(
map(([keyMissing, keyInvalid, audioOkay, videoOkay]) => {
if (keyMissing) return EncryptionStatus.KeyMissing;
if (keyInvalid) return EncryptionStatus.KeyInvalid;
if (audioOkay || videoOkay) return EncryptionStatus.Okay;
return undefined; // no change
}),
filter((x) => !!x),
startWith(EncryptionStatus.Connecting),
);
} else {
return combineLatest([
encryptionErrorObservable(
livekitRoom,
participant,
encryptionSystem,
"InvalidKey",
),
observeRemoteTrackReceivingOkay(participant, audioSource),
observeRemoteTrackReceivingOkay(participant, videoSource),
]).pipe(
map(
([keyInvalid, audioOkay, videoOkay]):
| EncryptionStatus
| undefined => {
if (keyInvalid) return EncryptionStatus.PasswordInvalid;
if (audioOkay || videoOkay) return EncryptionStatus.Okay;
return undefined; // no change
},
),
filter((x) => !!x),
startWith(EncryptionStatus.Connecting),
);
}
}),
this.scope.state(),
);
}
}
@@ -324,11 +339,14 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel {
/**
* Whether the participant is speaking.
*/
public readonly speaking = observeParticipantEvents(
this.participant,
ParticipantEvent.IsSpeakingChanged,
).pipe(
map((p) => p.isSpeaking),
public readonly speaking = this.participant.pipe(
switchMap((p) =>
p
? observeParticipantEvents(p, ParticipantEvent.IsSpeakingChanged).pipe(
map((p) => p.isSpeaking),
)
: of(false),
),
this.scope.state(),
);
@@ -350,7 +368,7 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel {
public constructor(
id: string,
member: RoomMember | undefined,
participant: LocalParticipant | RemoteParticipant,
participant: Observable<LocalParticipant | RemoteParticipant | undefined>,
encryptionSystem: EncryptionSystem,
livekitRoom: LivekitRoom,
) {
@@ -364,18 +382,25 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel {
livekitRoom,
);
const media = observeParticipantMedia(participant).pipe(this.scope.state());
const media = participant.pipe(
switchMap((p) => (p && observeParticipantMedia(p)) ?? of(undefined)),
this.scope.state(),
);
this.audioEnabled = media.pipe(
map((m) => m.microphoneTrack?.isMuted === false),
map((m) => m?.microphoneTrack?.isMuted === false),
);
this.videoEnabled = media.pipe(
map((m) => m.cameraTrack?.isMuted === false),
map((m) => m?.cameraTrack?.isMuted === false),
);
}
public toggleFitContain(): void {
this._cropVideo.next(!this._cropVideo.value);
}
public get local(): boolean {
return this instanceof LocalUserMediaViewModel;
}
}
/**
@@ -387,7 +412,7 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel {
*/
public readonly mirror = this.video.pipe(
switchMap((v) => {
const track = v.publication?.track;
const track = v?.publication?.track;
if (!(track instanceof LocalTrack)) return of(false);
// Watch for track restarts, because they indicate a camera switch
return fromEvent(track, TrackEvent.Restarted).pipe(
@@ -409,7 +434,7 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel {
public constructor(
id: string,
member: RoomMember | undefined,
participant: LocalParticipant,
participant: Observable<LocalParticipant | undefined>,
encryptionSystem: EncryptionSystem,
livekitRoom: LivekitRoom,
) {
@@ -470,18 +495,17 @@ export class RemoteUserMediaViewModel extends BaseUserMediaViewModel {
public constructor(
id: string,
member: RoomMember | undefined,
participant: RemoteParticipant,
participant: Observable<RemoteParticipant | undefined>,
encryptionSystem: EncryptionSystem,
livekitRoom: LivekitRoom,
) {
super(id, member, participant, encryptionSystem, livekitRoom);
// Sync the local volume with LiveKit
this.localVolume
.pipe(this.scope.bind())
.subscribe((volume) =>
(this.participant as RemoteParticipant).setVolume(volume),
);
combineLatest([
participant,
this.localVolume.pipe(this.scope.bind()),
]).subscribe(([p, volume]) => p && p.setVolume(volume));
}
public toggleLocallyMuted(): void {
@@ -504,9 +528,10 @@ export class ScreenShareViewModel extends BaseMediaViewModel {
public constructor(
id: string,
member: RoomMember | undefined,
participant: LocalParticipant | RemoteParticipant,
participant: Observable<LocalParticipant | RemoteParticipant>,
encryptionSystem: EncryptionSystem,
livekitRoom: LivekitRoom,
public readonly local: boolean,
) {
super(
id,

3
src/tile/GridTile.test.tsx
View File

@@ -13,7 +13,7 @@ import { of } from "rxjs";
import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession";
import { GridTile } from "./GridTile";
import { withRemoteMedia } from "../utils/test";
import { mockRtcMembership, withRemoteMedia } from "../utils/test";
import { GridTileViewModel } from "../state/TileViewModel";
import { ReactionsProvider } from "../useReactions";
@@ -25,6 +25,7 @@ global.IntersectionObserver = class MockIntersectionObserver {
test("GridTile is accessible", async () => {
await withRemoteMedia(
mockRtcMembership("@alice:example.org", "AAAA"),
{
rawDisplayName: "Alice",
getMxcAvatarUrl: () => "mxc://adfsg",

1
src/tile/GridTile.tsx
View File

@@ -175,6 +175,7 @@ const UserMediaTile = forwardRef<HTMLDivElement, UserMediaTileProps>(
raisedHandTime={handRaised}
currentReaction={currentReaction}
raisedHandOnClick={raisedHandOnClick}
localParticipant={vm.local}
{...props}
/>
);

18
src/tile/MediaView.module.css
View File

@@ -15,7 +15,7 @@ Please see LICENSE in the repository root for full details.
inline-size: 100%;
block-size: 100%;
object-fit: contain;
background-color: var(--cpd-color-bg-subtle-primary);
background-color: var(--video-tile-background);
/* This transform is a no-op, but it forces Firefox to use a different
rendering path, one that actually clips the corners of <video> elements into
the intended rounded shape. We can remove this if Firefox stops being broken. */
@@ -35,7 +35,7 @@ Please see LICENSE in the repository root for full details.
}
.bg {
background-color: var(--cpd-color-bg-subtle-secondary);
background-color: var(--video-tile-background);
inline-size: 100%;
block-size: 100%;
border-radius: inherit;
@@ -74,9 +74,9 @@ unconditionally select the container so we can use cqmin units */
calc(var(--media-view-border-radius) - var(--cpd-space-3x))
);
display: grid;
grid-template-columns: 1fr auto;
grid-template-columns: 30px 1fr 30px;
grid-template-rows: 1fr auto;
grid-template-areas: "status status" "nameTag button";
grid-template-areas: "reactions status ." "nameTag nameTag button";
gap: var(--cpd-space-1x);
place-items: start;
}
@@ -101,8 +101,8 @@ unconditionally select the container so we can use cqmin units */
grid-area: status;
justify-self: center;
align-self: start;
padding: var(--cpd-space-1x);
padding-block: var(--cpd-space-1x);
padding: var(--cpd-space-2x);
padding-block: var(--cpd-space-2x);
color: var(--cpd-color-text-primary);
background-color: var(--cpd-color-bg-canvas-default);
display: flex;
@@ -116,6 +116,12 @@ unconditionally select the container so we can use cqmin units */
text-align: center;
}
.reactions {
grid-area: reactions;
display: flex;
gap: var(--cpd-space-1x);
}
.nameTag > svg,
.nameTag > span {
flex-shrink: 0;

22
src/tile/MediaView.test.tsx
View File

@@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { describe, expect, test } from "vitest";
import { describe, expect, it, test } from "vitest";
import { render, screen } from "@testing-library/react";
import { axe } from "vitest-axe";
import { TooltipProvider } from "@vector-im/compound-web";
@@ -42,6 +42,7 @@ describe("MediaView", () => {
unencryptedWarning: false,
video: trackReference,
member: undefined,
localParticipant: false,
};
test("is accessible", async () => {
@@ -59,6 +60,25 @@ describe("MediaView", () => {
});
});
describe("with no participant", () => {
it("shows avatar for local user", () => {
render(
<MediaView {...baseProps} video={undefined} localParticipant={true} />,
);
expect(screen.getByRole("img", { name: "some name" })).toBeVisible();
expect(screen.queryAllByText("video_tile.waiting_for_media").length).toBe(
0,
);
});
it("shows avatar and label for remote user", () => {
render(
<MediaView {...baseProps} video={undefined} localParticipant={false} />,
);
expect(screen.getByRole("img", { name: "some name" })).toBeVisible();
expect(screen.getByText("video_tile.waiting_for_media")).toBeVisible();
});
});
describe("name tag", () => {
test("is shown with name", () => {
render(<MediaView {...baseProps} displayName="Bob" />);

17
src/tile/MediaView.tsx
View File

@@ -28,7 +28,7 @@ interface Props extends ComponentProps<typeof animated.div> {
style?: ComponentProps<typeof animated.div>["style"];
targetWidth: number;
targetHeight: number;
video: TrackReferenceOrPlaceholder;
video: TrackReferenceOrPlaceholder | undefined;
videoFit: "cover" | "contain";
mirror: boolean;
member: RoomMember | undefined;
@@ -41,6 +41,7 @@ interface Props extends ComponentProps<typeof animated.div> {
raisedHandTime?: Date;
currentReaction?: ReactionOption;
raisedHandOnClick?: () => void;
localParticipant: boolean;
}
export const MediaView = forwardRef<HTMLDivElement, Props>(
@@ -63,6 +64,7 @@ export const MediaView = forwardRef<HTMLDivElement, Props>(
raisedHandTime,
currentReaction,
raisedHandOnClick,
localParticipant,
...props
},
ref,
@@ -90,21 +92,21 @@ export const MediaView = forwardRef<HTMLDivElement, Props>(
size={avatarSize}
src={member?.getMxcAvatarUrl()}
className={styles.avatar}
style={{ display: videoEnabled ? "none" : "initial" }}
style={{ display: video && videoEnabled ? "none" : "initial" }}
/>
{video.publication !== undefined && (
{video?.publication !== undefined && (
<VideoTrack
trackRef={video}
// There's no reason for this to be focusable
tabIndex={-1}
disablePictureInPicture
style={{ display: videoEnabled ? "block" : "none" }}
style={{ display: video && videoEnabled ? "block" : "none" }}
data-testid="video"
/>
)}
</div>
<div className={styles.fg}>
<div style={{ display: "flex", gap: "var(--cpd-space-1x)" }}>
<div className={styles.reactions}>
<RaisedHandIndicator
raisedHandTime={raisedHandTime}
miniature={avatarSize < 96}
@@ -118,6 +120,11 @@ export const MediaView = forwardRef<HTMLDivElement, Props>(
/>
)}
</div>
{!video && !localParticipant && (
<div className={styles.status}>
{t("video_tile.waiting_for_media")}
</div>
)}
{/* TODO: Bring this back once encryption status is less broken */}
{/*encryptionStatus !== EncryptionStatus.Okay && (
<div className={styles.status}>

8
src/tile/SpotlightTile.test.tsx
View File

@@ -12,7 +12,11 @@ import userEvent from "@testing-library/user-event";
import { of } from "rxjs";
import { SpotlightTile } from "./SpotlightTile";
import { withLocalMedia, withRemoteMedia } from "../utils/test";
import {
mockRtcMembership,
withLocalMedia,
withRemoteMedia,
} from "../utils/test";
import { SpotlightTileViewModel } from "../state/TileViewModel";
global.IntersectionObserver = class MockIntersectionObserver {
@@ -22,6 +26,7 @@ global.IntersectionObserver = class MockIntersectionObserver {
test("SpotlightTile is accessible", async () => {
await withRemoteMedia(
mockRtcMembership("@alice:example.org", "AAAA"),
{
rawDisplayName: "Alice",
getMxcAvatarUrl: () => "mxc://adfsg",
@@ -29,6 +34,7 @@ test("SpotlightTile is accessible", async () => {
{},
async (vm1) => {
await withLocalMedia(
mockRtcMembership("@bob:example.org", "BBBB"),
{
rawDisplayName: "Bob",
getMxcAvatarUrl: () => "mxc://dlskf",

8
src/tile/SpotlightTile.tsx
View File

@@ -49,12 +49,13 @@ interface SpotlightItemBaseProps {
"data-id": string;
targetWidth: number;
targetHeight: number;
video: TrackReferenceOrPlaceholder;
video: TrackReferenceOrPlaceholder | undefined;
member: RoomMember | undefined;
unencryptedWarning: boolean;
encryptionStatus: EncryptionStatus;
displayName: string;
"aria-hidden"?: boolean;
localParticipant: boolean;
}
interface SpotlightUserMediaItemBaseProps extends SpotlightItemBaseProps {
@@ -163,6 +164,7 @@ const SpotlightItem = forwardRef<HTMLDivElement, SpotlightItemProps>(
displayName,
encryptionStatus,
"aria-hidden": ariaHidden,
localParticipant: vm.local,
};
return vm instanceof ScreenShareViewModel ? (
@@ -210,7 +212,9 @@ export const SpotlightTile = forwardRef<HTMLDivElement, Props>(
const ref = useMergedRefs(ourRef, theirRef);
const maximised = useObservableEagerState(vm.maximised);
const media = useObservableEagerState(vm.media);
const [visibleId, setVisibleId] = useState(media[0].id);
const [visibleId, setVisibleId] = useState<string | undefined>(
media[0]?.id,
);
const latestMedia = useLatest(media);
const latestVisibleId = useLatest(visibleId);
const visibleIndex = media.findIndex((vm) => vm.id === visibleId);

16
src/useAudioContext.test.tsx
View File

@@ -15,11 +15,13 @@ import { deviceStub, MediaDevicesContext } from "./livekit/MediaDevicesContext";
import { useAudioContext } from "./useAudioContext";
import { soundEffectVolumeSetting } from "./settings/settings";
const staticSounds = Promise.resolve({
aSound: new ArrayBuffer(0),
});
const TestComponent: FC = () => {
const audioCtx = useAudioContext({
sounds: Promise.resolve({
aSound: new ArrayBuffer(0),
}),
sounds: staticSounds,
latencyHint: "balanced",
});
if (!audioCtx) {
@@ -27,11 +29,9 @@ const TestComponent: FC = () => {
}
return (
<>
<button onClick={() => void audioCtx.playSound("aSound")}>
Valid sound
</button>
<button onClick={() => audioCtx.playSound("aSound")}>Valid sound</button>
{/* eslint-disable-next-line @typescript-eslint/no-explicit-any*/}
<button onClick={() => void audioCtx.playSound("not-valid" as any)}>
<button onClick={() => audioCtx.playSound("not-valid" as any)}>
Invalid sound
</button>
</>
@@ -61,7 +61,7 @@ class MockAudioContext {
vitest.mocked({
connect: (v: unknown) => v,
start: () => {},
addEventListener: (_eventType: string, cb: () => void) => cb(),
addEventListener: (_name: string, cb: () => void) => cb(),
}),
);
public createGain = vitest.fn().mockReturnValue(this.gain);

78
src/useAudioContext.tsx
View File

@@ -13,9 +13,7 @@ import {
useSetting,
} from "./settings/settings";
import { useMediaDevices } from "./livekit/MediaDevicesContext";
import { useInitial } from "./useInitial";
type SoundDefinition = { mp3?: string; ogg: string };
import { PrefetchedSounds } from "./soundUtils";
/**
* Play a sound though a given AudioContext. Will take
@@ -41,61 +39,13 @@ async function playSound(
return p;
}
/**
* Determine the best format we can use to play our sounds
* through. We prefer ogg support if possible, but will fall
* back to MP3.
* @returns "ogg" if the browser is likely to support it, or "mp3" otherwise.
*/
function getPreferredAudioFormat(): "ogg" | "mp3" {
const a = document.createElement("audio");
if (a.canPlayType("audio/ogg") === "maybe") {
return "ogg";
}
// Otherwise just assume MP3, as that has a chance of being more widely supported.
return "mp3";
}
type PrefetchedSounds<S extends string> = Promise<Record<S, ArrayBuffer>>;
// We prefer to load these sounds ahead of time, so there
// is no delay on call join.
const PreferredFormat = getPreferredAudioFormat();
/**
* Prefetch sounds to be used by the AudioContext. This should
* be called outside the scope of a component to ensure the
* sounds load ahead of time.
* @param sounds A set of sound files that may be played.
* @returns A map of sound files to buffers.
*/
export async function prefetchSounds<S extends string>(
sounds: Record<S, SoundDefinition>,
): PrefetchedSounds<S> {
const buffers: Record<string, ArrayBuffer> = {};
await Promise.all(
Object.entries(sounds).map(async ([name, file]) => {
const { mp3, ogg } = file as SoundDefinition;
// Use preferred format, fallback to ogg if no mp3 is provided.
// Load an audio file
const response = await fetch(
PreferredFormat === "ogg" ? ogg : (mp3 ?? ogg),
);
if (!response.ok) {
// If the sound doesn't load, it's not the end of the world. We won't play
// the sound when requested, but it's better than failing the whole application.
logger.warn(`Could not load sound ${name}, resposne was not okay`);
return;
}
// Decode it
buffers[name] = await response.arrayBuffer();
}),
);
return buffers as Record<S, ArrayBuffer>;
}
interface Props<S extends string> {
sounds: PrefetchedSounds<S>;
/**
* The sounds to play. If no sounds should be played then
* this can be set to null, which will prevent the audio
* context from being created.
*/
sounds: PrefetchedSounds<S> | null;
latencyHint: AudioContextLatencyCategory;
}
@@ -116,9 +66,12 @@ export function useAudioContext<S extends string>(
const devices = useMediaDevices();
const [audioContext, setAudioContext] = useState<AudioContext>();
const [audioBuffers, setAudioBuffers] = useState<Record<S, AudioBuffer>>();
const soundCache = useInitial(async () => props.sounds);
useEffect(() => {
const sounds = props.sounds;
if (!sounds) {
return;
}
const ctx = new AudioContext({
// We want low latency for these effects.
latencyHint: props.latencyHint,
@@ -129,11 +82,8 @@ export function useAudioContext<S extends string>(
// close during this process, so it's okay if it throws.
(async (): Promise<void> => {
const buffers: Record<string, AudioBuffer> = {};
for (const [name, buffer] of Object.entries(await soundCache)) {
const audioBuffer = await ctx.decodeAudioData(
// Type quirk, this is *definitely* a ArrayBuffer.
(buffer as ArrayBuffer).slice(0),
);
for (const [name, buffer] of Object.entries<ArrayBuffer>(await sounds)) {
const audioBuffer = await ctx.decodeAudioData(buffer.slice(0));
buffers[name] = audioBuffer;
}
setAudioBuffers(buffers as Record<S, AudioBuffer>);
@@ -148,7 +98,7 @@ export function useAudioContext<S extends string>(
});
setAudioContext(undefined);
};
}, [soundCache, props.latencyHint]);
}, [props.sounds, props.latencyHint]);
// Update the sink ID whenever we change devices.
useEffect(() => {

12
src/utils/matrix.ts
View File

@@ -333,15 +333,3 @@ export function getRelativeRoomUrl(
: "";
return `/room/#${roomPart}?${generateUrlSearchParams(roomId, encryptionSystem, viaServers).toString()}`;
}
export function getAvatarUrl(
client: MatrixClient,
mxcUrl: string,
avatarSize = 96,
): string {
const width = Math.floor(avatarSize * window.devicePixelRatio);
const height = Math.floor(avatarSize * window.devicePixelRatio);
// scale is more suitable for larger sizes
const resizeMethod = avatarSize <= 96 ? "crop" : "scale";
return mxcUrl && client.mxcUrlToHttp(mxcUrl, width, height, resizeMethod)!;
}

122
src/utils/test.ts
View File

@@ -7,16 +7,27 @@ Please see LICENSE in the repository root for full details.
import { map, Observable, of, SchedulerLike } from "rxjs";
import { RunHelpers, TestScheduler } from "rxjs/testing";
import { expect, vi } from "vitest";
import { RoomMember, Room as MatrixRoom } from "matrix-js-sdk/src/matrix";
import {
RoomMember,
Room as MatrixRoom,
MatrixEvent,
Room,
TypedEventEmitter,
} from "matrix-js-sdk/src/matrix";
import {
CallMembership,
Focus,
MatrixRTCSessionEvent,
MatrixRTCSessionEventHandlerMap,
SessionMembershipData,
} from "matrix-js-sdk/src/matrixrtc";
import {
LocalParticipant,
LocalTrackPublication,
RemoteParticipant,
RemoteTrackPublication,
Room as LivekitRoom,
RoomEvent,
} from "livekit-client";
import { EventEmitter } from "stream";
import {
LocalUserMediaViewModel,
@@ -100,48 +111,46 @@ function mockEmitter<T>(): EmitterMock<T> {
};
}
export function mockRtcMembership(
user: string | RoomMember,
deviceId: string,
callId = "",
fociPreferred: Focus[] = [],
focusActive: Focus = { type: "oldest_membership" },
membership: Partial<SessionMembershipData> = {},
): CallMembership {
const data: SessionMembershipData = {
application: "m.call",
call_id: callId,
device_id: deviceId,
foci_preferred: fociPreferred,
focus_active: focusActive,
...membership,
};
const event = new MatrixEvent({
sender: typeof user === "string" ? user : user.userId,
});
return new CallMembership(event, data);
}
// Maybe it'd be good to move this to matrix-js-sdk? Our testing needs are
// rather simple, but if one util to mock a member is good enough for us, maybe
// it's useful for matrix-js-sdk consumers in general.
export function mockMatrixRoomMember(member: Partial<RoomMember>): RoomMember {
return { ...mockEmitter(), ...member } as RoomMember;
export function mockMatrixRoomMember(
rtcMembership: CallMembership,
member: Partial<RoomMember> = {},
): RoomMember {
return {
...mockEmitter(),
userId: rtcMembership.sender,
...member,
} as RoomMember;
}
export function mockMatrixRoom(room: Partial<MatrixRoom>): MatrixRoom {
return { ...mockEmitter(), ...room } as Partial<MatrixRoom> as MatrixRoom;
}
/**
* A mock of a Livekit Room that can emit events.
*/
export class EmittableMockLivekitRoom extends EventEmitter {
public localParticipant?: LocalParticipant;
public remoteParticipants: Map<string, RemoteParticipant>;
public constructor(room: {
localParticipant?: LocalParticipant;
remoteParticipants: Map<string, RemoteParticipant>;
}) {
super();
this.localParticipant = room.localParticipant;
this.remoteParticipants = room.remoteParticipants ?? new Map();
}
public addParticipant(remoteParticipant: RemoteParticipant): void {
this.remoteParticipants.set(remoteParticipant.identity, remoteParticipant);
this.emit(RoomEvent.ParticipantConnected, remoteParticipant);
}
public removeParticipant(remoteParticipant: RemoteParticipant): void {
this.remoteParticipants.delete(remoteParticipant.identity);
this.emit(RoomEvent.ParticipantDisconnected, remoteParticipant);
}
public getAsLivekitRoom(): LivekitRoom {
return this as unknown as LivekitRoom;
}
}
export function mockLivekitRoom(
room: Partial<LivekitRoom>,
{
@@ -178,14 +187,15 @@ export function mockLocalParticipant(
}
export async function withLocalMedia(
member: Partial<RoomMember>,
localRtcMember: CallMembership,
roomMember: Partial<RoomMember>,
continuation: (vm: LocalUserMediaViewModel) => void | Promise<void>,
): Promise<void> {
const localParticipant = mockLocalParticipant({});
const vm = new LocalUserMediaViewModel(
"local",
mockMatrixRoomMember(member),
localParticipant,
mockMatrixRoomMember(localRtcMember, roomMember),
of(localParticipant),
{
kind: E2eeType.PER_PARTICIPANT,
},
@@ -212,15 +222,16 @@ export function mockRemoteParticipant(
}
export async function withRemoteMedia(
member: Partial<RoomMember>,
localRtcMember: CallMembership,
roomMember: Partial<RoomMember>,
participant: Partial<RemoteParticipant>,
continuation: (vm: RemoteUserMediaViewModel) => void | Promise<void>,
): Promise<void> {
const remoteParticipant = mockRemoteParticipant(participant);
const vm = new RemoteUserMediaViewModel(
"remote",
mockMatrixRoomMember(member),
remoteParticipant,
mockMatrixRoomMember(localRtcMember, roomMember),
of(remoteParticipant),
{
kind: E2eeType.PER_PARTICIPANT,
},
@@ -239,3 +250,30 @@ export function mockConfig(config: Partial<ResolvedConfigOptions> = {}): void {
...config,
});
}
export class MockRTCSession extends TypedEventEmitter<
MatrixRTCSessionEvent,
MatrixRTCSessionEventHandlerMap
> {
public constructor(
public readonly room: Room,
private localMembership: CallMembership,
public memberships: CallMembership[] = [],
) {
super();
}
public withMemberships(
rtcMembers: Observable<Partial<CallMembership>[]>,
): MockRTCSession {
rtcMembers.subscribe((m) => {
const old = this.memberships;
// always prepend the local participant
const updated = [this.localMembership, ...(m as CallMembership[])];
this.memberships = updated;
this.emit(MatrixRTCSessionEvent.MembershipsChanged, old, updated);
});
return this;
}
}

2
src/utils/testReactions.tsx
View File

@@ -32,7 +32,7 @@ export const TestReactionsWrapper = ({
rtcSession,
children,
}: PropsWithChildren<{
rtcSession: MockRTCSession;
rtcSession: MockRTCSession | MatrixRTCSession;
}>): ReactNode => {
return (
<ReactionsProvider rtcSession={rtcSession as unknown as MatrixRTCSession}>

12
src/vitest.setup.ts
View File

@@ -17,20 +17,20 @@ import "vitest-axe/extend-expect";
import { logger } from "matrix-js-sdk/src/logger";
import "@testing-library/jest-dom/vitest";
import EN_GB from "../locales/en-GB/app.json";
import EN from "../locales/en/app.json";
import { Config } from "./config/Config";
// Bare-minimum i18n config
i18n
.use(initReactI18next)
.init({
lng: "en-GB",
fallbackLng: "en-GB",
supportedLngs: ["en-GB"],
lng: "en",
fallbackLng: "en",
supportedLngs: ["en"],
// We embed the translations, so that it never needs to fetch
resources: {
"en-GB": {
app: EN_GB,
en: {
app: EN,
},
},
interpolation: {