Refactor reactions / hand raised to use rxjs and start ordering tiles based on hand raised. (#2885)

* Add support for using CallViewModel for reactions sounds.

* Drop setting

* Convert reaction sounds to call view model / rxjs

* Use call view model for hand raised reactions

* Support raising reactions for matrix rtc members.

* Tie up last bits of useReactions

* linting

* Update calleventaudiorenderer

* Update reaction audio renderer

* more test bits

* All the test bits and pieces

* More refactors

* Refactor reactions into a sender and receiver.

* Fixup reaction toggle button

* Adapt reactions test

* Tests all pass.

* lint

* fix a couple of bugs

* remove unused helper file

* lint

* finnish notation

* Add tests for useReactionsReader

* remove mistaken vitest file

* fix

* filter

* invert

* fixup tests with fake timers

* Port useReactionsReader hook to ReactionsReader class.

* lint

* exclude some files from coverage

* Add screen share sound effect.

* cancel sub on destroy

* tidy tidy
This commit is contained in:
Will Hunt
2024-12-19 15:54:28 +00:00
committed by GitHub
parent 7d00f85abc
commit abf2ecd521
28 changed files with 1835 additions and 1184 deletions

View File

@@ -5,47 +5,47 @@ SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { render } from "@testing-library/react";
import { act, render } from "@testing-library/react";
import { expect, test } from "vitest";
import { TooltipProvider } from "@vector-im/compound-web";
import { userEvent } from "@testing-library/user-event";
import { type ReactNode } from "react";
import { type MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession";
import {
MockRoom,
MockRTCSession,
TestReactionsWrapper,
} from "../utils/testReactions";
import { ReactionToggleButton } from "./ReactionToggleButton";
import { ElementCallReactionEventType } from "../reactions";
import { type CallViewModel } from "../state/CallViewModel";
import { getBasicCallViewModelEnvironment } from "../utils/test-viewmodel";
import { alice, local, localRtcMember } from "../utils/test-fixtures";
import { type MockRTCSession } from "../utils/test";
import { ReactionsSenderProvider } from "../reactions/useReactionsSender";
const memberUserIdAlice = "@alice:example.org";
const memberEventAlice = "$membership-alice:example.org";
const membership: Record<string, string> = {
[memberEventAlice]: memberUserIdAlice,
};
const localIdent = `${localRtcMember.sender}:${localRtcMember.deviceId}`;
function TestComponent({
rtcSession,
vm,
}: {
rtcSession: MockRTCSession;
vm: CallViewModel;
}): ReactNode {
return (
<TooltipProvider>
<TestReactionsWrapper rtcSession={rtcSession}>
<ReactionToggleButton userId={memberUserIdAlice} />
</TestReactionsWrapper>
<ReactionsSenderProvider
vm={vm}
rtcSession={rtcSession as unknown as MatrixRTCSession}
>
<ReactionToggleButton vm={vm} identifier={localIdent} />
</ReactionsSenderProvider>
</TooltipProvider>
);
}
test("Can open menu", async () => {
const user = userEvent.setup();
const room = new MockRoom(memberUserIdAlice);
const rtcSession = new MockRTCSession(room, membership);
const { vm, rtcSession } = getBasicCallViewModelEnvironment([alice]);
const { getByLabelText, container } = render(
<TestComponent rtcSession={rtcSession} />,
<TestComponent vm={vm} rtcSession={rtcSession} />,
);
await user.click(getByLabelText("common.reactions"));
expect(container).toMatchSnapshot();
@@ -53,102 +53,120 @@ test("Can open menu", async () => {
test("Can raise hand", async () => {
const user = userEvent.setup();
const room = new MockRoom(memberUserIdAlice);
const rtcSession = new MockRTCSession(room, membership);
const { vm, rtcSession, handRaisedSubject$ } =
getBasicCallViewModelEnvironment([local, alice]);
const { getByLabelText, container } = render(
<TestComponent rtcSession={rtcSession} />,
<TestComponent vm={vm} rtcSession={rtcSession} />,
);
await user.click(getByLabelText("common.reactions"));
await user.click(getByLabelText("action.raise_hand"));
expect(room.testSentEvents).toEqual([
[
undefined,
"m.reaction",
{
"m.relates_to": {
event_id: memberEventAlice,
key: "🖐️",
rel_type: "m.annotation",
},
expect(rtcSession.room.client.sendEvent).toHaveBeenCalledWith(
rtcSession.room.roomId,
"m.reaction",
{
"m.relates_to": {
event_id: localRtcMember.eventId,
key: "🖐️",
rel_type: "m.annotation",
},
],
]);
},
);
act(() => {
// Mock receiving a reaction.
handRaisedSubject$.next({
[localIdent]: {
time: new Date(),
reactionEventId: "",
membershipEventId: localRtcMember.eventId!,
},
});
});
expect(container).toMatchSnapshot();
});
test("Can lower hand", async () => {
const reactionEventId = "$my-reaction-event:example.org";
const user = userEvent.setup();
const room = new MockRoom(memberUserIdAlice);
const rtcSession = new MockRTCSession(room, membership);
const { vm, rtcSession, handRaisedSubject$ } =
getBasicCallViewModelEnvironment([local, alice]);
const { getByLabelText, container } = render(
<TestComponent rtcSession={rtcSession} />,
<TestComponent vm={vm} rtcSession={rtcSession} />,
);
const reactionEvent = room.testSendHandRaise(memberEventAlice, membership);
await user.click(getByLabelText("common.reactions"));
await user.click(getByLabelText("action.raise_hand"));
act(() => {
handRaisedSubject$.next({
[localIdent]: {
time: new Date(),
reactionEventId,
membershipEventId: localRtcMember.eventId!,
},
});
});
await user.click(getByLabelText("common.reactions"));
await user.click(getByLabelText("action.lower_hand"));
expect(room.testRedactedEvents).toEqual([[undefined, reactionEvent]]);
expect(rtcSession.room.client.redactEvent).toHaveBeenCalledWith(
rtcSession.room.roomId,
reactionEventId,
);
act(() => {
// Mock receiving a redacted reaction.
handRaisedSubject$.next({});
});
expect(container).toMatchSnapshot();
});
test("Can react with emoji", async () => {
const user = userEvent.setup();
const room = new MockRoom(memberUserIdAlice);
const rtcSession = new MockRTCSession(room, membership);
const { vm, rtcSession } = getBasicCallViewModelEnvironment([local, alice]);
const { getByLabelText, getByText } = render(
<TestComponent rtcSession={rtcSession} />,
<TestComponent vm={vm} rtcSession={rtcSession} />,
);
await user.click(getByLabelText("common.reactions"));
await user.click(getByText("🐶"));
expect(room.testSentEvents).toEqual([
[
undefined,
ElementCallReactionEventType,
{
"m.relates_to": {
event_id: memberEventAlice,
rel_type: "m.reference",
},
name: "dog",
emoji: "🐶",
expect(rtcSession.room.client.sendEvent).toHaveBeenCalledWith(
rtcSession.room.roomId,
ElementCallReactionEventType,
{
"m.relates_to": {
event_id: localRtcMember.eventId,
rel_type: "m.reference",
},
],
]);
name: "dog",
emoji: "🐶",
},
);
});
test("Can fully expand emoji picker", async () => {
const user = userEvent.setup();
const room = new MockRoom(memberUserIdAlice);
const rtcSession = new MockRTCSession(room, membership);
const { getByText, container, getByLabelText } = render(
<TestComponent rtcSession={rtcSession} />,
const { vm, rtcSession } = getBasicCallViewModelEnvironment([local, alice]);
const { getByLabelText, container, getByText } = render(
<TestComponent vm={vm} rtcSession={rtcSession} />,
);
await user.click(getByLabelText("common.reactions"));
await user.click(getByLabelText("action.show_more"));
expect(container).toMatchSnapshot();
await user.click(getByText("🦗"));
expect(room.testSentEvents).toEqual([
[
undefined,
ElementCallReactionEventType,
{
"m.relates_to": {
event_id: memberEventAlice,
rel_type: "m.reference",
},
name: "crickets",
emoji: "🦗",
expect(rtcSession.room.client.sendEvent).toHaveBeenCalledWith(
rtcSession.room.roomId,
ElementCallReactionEventType,
{
"m.relates_to": {
event_id: localRtcMember.eventId,
rel_type: "m.reference",
},
],
]);
name: "crickets",
emoji: "🦗",
},
);
});
test("Can close reaction dialog", async () => {
const user = userEvent.setup();
const room = new MockRoom(memberUserIdAlice);
const rtcSession = new MockRTCSession(room, membership);
const { vm, rtcSession } = getBasicCallViewModelEnvironment([local, alice]);
const { getByLabelText, container } = render(
<TestComponent rtcSession={rtcSession} />,
<TestComponent vm={vm} rtcSession={rtcSession} />,
);
await user.click(getByLabelText("common.reactions"));
await user.click(getByLabelText("action.show_more"));

View File

@@ -24,8 +24,10 @@ import {
import { useTranslation } from "react-i18next";
import { logger } from "matrix-js-sdk/src/logger";
import classNames from "classnames";
import { useObservableState } from "observable-hooks";
import { map } from "rxjs";
import { useReactions } from "../useReactions";
import { useReactionsSender } from "../reactions/useReactionsSender";
import styles from "./ReactionToggleButton.module.css";
import {
type ReactionOption,
@@ -33,6 +35,7 @@ import {
ReactionsRowSize,
} from "../reactions";
import { Modal } from "../Modal";
import { type CallViewModel } from "../state/CallViewModel";
interface InnerButtonProps extends ComponentPropsWithoutRef<"button"> {
raised: boolean;
@@ -162,22 +165,27 @@ export function ReactionPopupMenu({
}
interface ReactionToggleButtonProps extends ComponentPropsWithoutRef<"button"> {
userId: string;
identifier: string;
vm: CallViewModel;
}
export function ReactionToggleButton({
userId,
identifier,
vm,
...props
}: ReactionToggleButtonProps): ReactNode {
const { t } = useTranslation();
const { raisedHands, toggleRaisedHand, sendReaction, reactions } =
useReactions();
const { toggleRaisedHand, sendReaction } = useReactionsSender();
const [busy, setBusy] = useState(false);
const [showReactionsMenu, setShowReactionsMenu] = useState(false);
const [errorText, setErrorText] = useState<string>();
const isHandRaised = !!raisedHands[userId];
const canReact = !reactions[userId];
const isHandRaised = useObservableState(
vm.handsRaised$.pipe(map((v) => !!v[identifier])),
);
const canReact = useObservableState(
vm.reactions$.pipe(map((v) => !v[identifier])),
);
useEffect(() => {
// Clear whenever the reactions menu state changes.
@@ -223,7 +231,7 @@ export function ReactionToggleButton({
<InnerButton
disabled={busy}
onClick={() => setShowReactionsMenu((show) => !show)}
raised={isHandRaised}
raised={!!isHandRaised}
open={showReactionsMenu}
{...props}
/>
@@ -237,8 +245,8 @@ export function ReactionToggleButton({
>
<ReactionPopupMenu
errorText={errorText}
isHandRaised={isHandRaised}
canReact={!busy && canReact}
isHandRaised={!!isHandRaised}
canReact={!busy && !!canReact}
sendReaction={(reaction) => void sendRelation(reaction)}
toggleRaisedHand={wrappedToggleRaisedHand}
/>

View File

@@ -9,7 +9,7 @@ exports[`Can close reaction dialog 1`] = `
aria-disabled="false"
aria-expanded="true"
aria-haspopup="true"
aria-labelledby=":r9l:"
aria-labelledby=":rav:"
class="_button_i91xf_17 _has-icon_i91xf_66 _icon-only_i91xf_59"
data-kind="primary"
data-size="lg"
@@ -43,7 +43,7 @@ exports[`Can fully expand emoji picker 1`] = `
aria-disabled="false"
aria-expanded="true"
aria-haspopup="true"
aria-labelledby=":r6c:"
aria-labelledby=":r7m:"
class="_button_i91xf_17 _has-icon_i91xf_66 _icon-only_i91xf_59"
data-kind="primary"
data-size="lg"
@@ -75,8 +75,8 @@ exports[`Can lower hand 1`] = `
aria-expanded="false"
aria-haspopup="true"
aria-labelledby=":r36:"
class="_button_i91xf_17 raisedButton _has-icon_i91xf_66 _icon-only_i91xf_59"
data-kind="primary"
class="_button_i91xf_17 _has-icon_i91xf_66 _icon-only_i91xf_59"
data-kind="secondary"
data-size="lg"
role="button"
tabindex="0"
@@ -90,7 +90,9 @@ exports[`Can lower hand 1`] = `
xmlns="http://www.w3.org/2000/svg"
>
<path
d="M11 3a1 1 0 1 1 2 0v8.5a.5.5 0 0 0 1 0V4a1 1 0 1 1 2 0v10.2l3.284-2.597a1.081 1.081 0 0 1 1.47 1.577c-.613.673-1.214 1.367-1.818 2.064-1.267 1.463-2.541 2.934-3.944 4.235A6 6 0 0 1 5 15V7a1 1 0 0 1 2 0v5.5a.5.5 0 0 0 1 0V4a1 1 0 0 1 2 0v7.5a.5.5 0 0 0 1 0V3Z"
clip-rule="evenodd"
d="M12 22c5.523 0 10-4.477 10-10S17.523 2 12 2 2 6.477 2 12s4.477 10 10 10Zm3.536-6.464a1 1 0 0 0-1.415-1.415A2.988 2.988 0 0 1 12 15a2.988 2.988 0 0 1-2.121-.879 1 1 0 1 0-1.414 1.415A4.987 4.987 0 0 0 12 17c1.38 0 2.632-.56 3.536-1.464ZM10 10.5a1.5 1.5 0 1 1-3 0 1.5 1.5 0 0 1 3 0Zm5.5 1.5a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3Z"
fill-rule="evenodd"
/>
</svg>
</button>
@@ -138,8 +140,8 @@ exports[`Can raise hand 1`] = `
aria-expanded="false"
aria-haspopup="true"
aria-labelledby=":r1j:"
class="_button_i91xf_17 _has-icon_i91xf_66 _icon-only_i91xf_59"
data-kind="secondary"
class="_button_i91xf_17 raisedButton _has-icon_i91xf_66 _icon-only_i91xf_59"
data-kind="primary"
data-size="lg"
role="button"
tabindex="0"
@@ -153,9 +155,7 @@ exports[`Can raise hand 1`] = `
xmlns="http://www.w3.org/2000/svg"
>
<path
clip-rule="evenodd"
d="M12 22c5.523 0 10-4.477 10-10S17.523 2 12 2 2 6.477 2 12s4.477 10 10 10Zm3.536-6.464a1 1 0 0 0-1.415-1.415A2.988 2.988 0 0 1 12 15a2.988 2.988 0 0 1-2.121-.879 1 1 0 1 0-1.414 1.415A4.987 4.987 0 0 0 12 17c1.38 0 2.632-.56 3.536-1.464ZM10 10.5a1.5 1.5 0 1 1-3 0 1.5 1.5 0 0 1 3 0Zm5.5 1.5a1.5 1.5 0 1 0 0-3 1.5 1.5 0 0 0 0 3Z"
fill-rule="evenodd"
d="M11 3a1 1 0 1 1 2 0v8.5a.5.5 0 0 0 1 0V4a1 1 0 1 1 2 0v10.2l3.284-2.597a1.081 1.081 0 0 1 1.47 1.577c-.613.673-1.214 1.367-1.818 2.064-1.267 1.463-2.541 2.934-3.944 4.235A6 6 0 0 1 5 15V7a1 1 0 0 1 2 0v5.5a.5.5 0 0 0 1 0V4a1 1 0 0 1 2 0v7.5a.5.5 0 0 0 1 0V3Z"
/>
</svg>
</button>

View File

@@ -0,0 +1,515 @@
/*
Copyright 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { renderHook } from "@testing-library/react";
import { afterEach, test, vitest } from "vitest";
import { type MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc";
import {
RoomEvent as MatrixRoomEvent,
MatrixEvent,
type IRoomTimelineData,
EventType,
MatrixEventEvent,
} from "matrix-js-sdk/src/matrix";
import { ReactionsReader, REACTION_ACTIVE_TIME_MS } from "./ReactionsReader";
import {
alice,
aliceRtcMember,
local,
localRtcMember,
} from "../utils/test-fixtures";
import { getBasicRTCSession } from "../utils/test-viewmodel";
import { withTestScheduler } from "../utils/test";
import { ElementCallReactionEventType, ReactionSet } from ".";
afterEach(() => {
vitest.useRealTimers();
});
test("handles a hand raised reaction", () => {
const { rtcSession } = getBasicRTCSession([local, alice]);
const reactionEventId = "$my_event_id:example.org";
const localTimestamp = new Date();
withTestScheduler(({ schedule, expectObservable }) => {
renderHook(() => {
const { raisedHands$ } = new ReactionsReader(
rtcSession as unknown as MatrixRTCSession,
);
schedule("ab", {
a: () => {},
b: () => {
rtcSession.room.emit(
MatrixRoomEvent.Timeline,
new MatrixEvent({
room_id: rtcSession.room.roomId,
event_id: reactionEventId,
sender: localRtcMember.sender,
type: EventType.Reaction,
origin_server_ts: localTimestamp.getTime(),
content: {
"m.relates_to": {
event_id: localRtcMember.eventId,
key: "🖐️",
},
},
}),
rtcSession.room,
undefined,
false,
{} as IRoomTimelineData,
);
},
});
expectObservable(raisedHands$).toBe("ab", {
a: {},
b: {
[`${localRtcMember.sender}:${localRtcMember.deviceId}`]: {
reactionEventId,
membershipEventId: localRtcMember.eventId,
time: localTimestamp,
},
},
});
});
});
});
test("handles a redaction", () => {
const { rtcSession } = getBasicRTCSession([local, alice]);
const reactionEventId = "$my_event_id:example.org";
const localTimestamp = new Date();
withTestScheduler(({ schedule, expectObservable }) => {
renderHook(() => {
const { raisedHands$ } = new ReactionsReader(
rtcSession as unknown as MatrixRTCSession,
);
schedule("abc", {
a: () => {},
b: () => {
rtcSession.room.emit(
MatrixRoomEvent.Timeline,
new MatrixEvent({
room_id: rtcSession.room.roomId,
event_id: reactionEventId,
sender: localRtcMember.sender,
type: EventType.Reaction,
origin_server_ts: localTimestamp.getTime(),
content: {
"m.relates_to": {
event_id: localRtcMember.eventId,
key: "🖐️",
},
},
}),
rtcSession.room,
undefined,
false,
{} as IRoomTimelineData,
);
},
c: () => {
rtcSession.room.emit(
MatrixRoomEvent.Redaction,
new MatrixEvent({
room_id: rtcSession.room.roomId,
event_id: reactionEventId,
sender: localRtcMember.sender,
type: EventType.RoomRedaction,
redacts: reactionEventId,
}),
rtcSession.room,
undefined,
);
},
});
expectObservable(raisedHands$).toBe("abc", {
a: {},
b: {
[`${localRtcMember.sender}:${localRtcMember.deviceId}`]: {
reactionEventId,
membershipEventId: localRtcMember.eventId,
time: localTimestamp,
},
},
c: {},
});
});
});
});
test("handles waiting for event decryption", () => {
const { rtcSession } = getBasicRTCSession([local, alice]);
const reactionEventId = "$my_event_id:example.org";
const localTimestamp = new Date();
withTestScheduler(({ schedule, expectObservable }) => {
renderHook(() => {
const { raisedHands$ } = new ReactionsReader(
rtcSession as unknown as MatrixRTCSession,
);
schedule("abc", {
a: () => {},
b: () => {
const encryptedEvent = new MatrixEvent({
room_id: rtcSession.room.roomId,
event_id: reactionEventId,
sender: localRtcMember.sender,
type: EventType.Reaction,
origin_server_ts: localTimestamp.getTime(),
content: {
"m.relates_to": {
event_id: localRtcMember.eventId,
key: "🖐️",
},
},
});
// Should ignore encrypted events that are still encrypting
encryptedEvent["decryptionPromise"] = Promise.resolve();
rtcSession.room.emit(
MatrixRoomEvent.Timeline,
encryptedEvent,
rtcSession.room,
undefined,
false,
{} as IRoomTimelineData,
);
},
c: () => {
rtcSession.room.client.emit(
MatrixEventEvent.Decrypted,
new MatrixEvent({
room_id: rtcSession.room.roomId,
event_id: reactionEventId,
sender: localRtcMember.sender,
type: EventType.Reaction,
origin_server_ts: localTimestamp.getTime(),
content: {
"m.relates_to": {
event_id: localRtcMember.eventId,
key: "🖐️",
},
},
}),
);
},
});
expectObservable(raisedHands$).toBe("a-c", {
a: {},
c: {
[`${localRtcMember.sender}:${localRtcMember.deviceId}`]: {
reactionEventId,
membershipEventId: localRtcMember.eventId,
time: localTimestamp,
},
},
});
});
});
});
test("hands rejecting events without a proper membership", () => {
const { rtcSession } = getBasicRTCSession([local, alice]);
const reactionEventId = "$my_event_id:example.org";
const localTimestamp = new Date();
withTestScheduler(({ schedule, expectObservable }) => {
renderHook(() => {
const { raisedHands$ } = new ReactionsReader(
rtcSession as unknown as MatrixRTCSession,
);
schedule("ab", {
a: () => {},
b: () => {
rtcSession.room.emit(
MatrixRoomEvent.Timeline,
new MatrixEvent({
room_id: rtcSession.room.roomId,
event_id: reactionEventId,
sender: localRtcMember.sender,
type: EventType.Reaction,
origin_server_ts: localTimestamp.getTime(),
content: {
"m.relates_to": {
event_id: "$not-this-one:example.org",
key: "🖐️",
},
},
}),
rtcSession.room,
undefined,
false,
{} as IRoomTimelineData,
);
},
});
expectObservable(raisedHands$).toBe("a-", {
a: {},
});
});
});
});
test("handles a reaction", () => {
const { rtcSession } = getBasicRTCSession([local, alice]);
const reactionEventId = "$my_event_id:example.org";
const reaction = ReactionSet[1];
vitest.useFakeTimers();
vitest.setSystemTime(0);
withTestScheduler(({ schedule, time, expectObservable }) => {
renderHook(() => {
const { reactions$ } = new ReactionsReader(
rtcSession as unknown as MatrixRTCSession,
);
schedule(`abc`, {
a: () => {},
b: () => {
rtcSession.room.emit(
MatrixRoomEvent.Timeline,
new MatrixEvent({
room_id: rtcSession.room.roomId,
event_id: reactionEventId,
sender: localRtcMember.sender,
type: ElementCallReactionEventType,
content: {
emoji: reaction.emoji,
name: reaction.name,
"m.relates_to": {
event_id: localRtcMember.eventId,
},
},
}),
rtcSession.room,
undefined,
false,
{} as IRoomTimelineData,
);
},
c: () => {
vitest.advanceTimersByTime(REACTION_ACTIVE_TIME_MS);
},
});
expectObservable(reactions$).toBe(
`ab ${REACTION_ACTIVE_TIME_MS - 1}ms c`,
{
a: {},
b: {
[`${localRtcMember.sender}:${localRtcMember.deviceId}`]: {
reactionOption: reaction,
expireAfter: new Date(REACTION_ACTIVE_TIME_MS),
},
},
// Expect reaction to expire.
c: {},
},
);
});
});
});
test("ignores bad reaction events", () => {
const { rtcSession } = getBasicRTCSession([local, alice]);
const reactionEventId = "$my_event_id:example.org";
const reaction = ReactionSet[1];
vitest.setSystemTime(0);
withTestScheduler(({ schedule, expectObservable }) => {
renderHook(() => {
const { reactions$ } = new ReactionsReader(
rtcSession as unknown as MatrixRTCSession,
);
schedule("ab", {
a: () => {},
b: () => {
// Missing content
rtcSession.room.emit(
MatrixRoomEvent.Timeline,
new MatrixEvent({
room_id: rtcSession.room.roomId,
event_id: reactionEventId,
sender: localRtcMember.sender,
type: ElementCallReactionEventType,
content: {},
}),
rtcSession.room,
undefined,
false,
{} as IRoomTimelineData,
);
// Wrong relates event
rtcSession.room.emit(
MatrixRoomEvent.Timeline,
new MatrixEvent({
room_id: rtcSession.room.roomId,
event_id: reactionEventId,
sender: localRtcMember.sender,
type: ElementCallReactionEventType,
content: {
emoji: reaction.emoji,
name: reaction.name,
"m.relates_to": {
event_id: "wrong-event",
},
},
}),
rtcSession.room,
undefined,
false,
{} as IRoomTimelineData,
);
// Wrong rtc member event
rtcSession.room.emit(
MatrixRoomEvent.Timeline,
new MatrixEvent({
room_id: rtcSession.room.roomId,
event_id: reactionEventId,
sender: aliceRtcMember.sender,
type: ElementCallReactionEventType,
content: {
emoji: reaction.emoji,
name: reaction.name,
"m.relates_to": {
event_id: localRtcMember.eventId,
},
},
}),
rtcSession.room,
undefined,
false,
{} as IRoomTimelineData,
);
// No emoji
rtcSession.room.emit(
MatrixRoomEvent.Timeline,
new MatrixEvent({
room_id: rtcSession.room.roomId,
event_id: reactionEventId,
sender: localRtcMember.sender,
type: ElementCallReactionEventType,
content: {
name: reaction.name,
"m.relates_to": {
event_id: localRtcMember.eventId,
},
},
}),
rtcSession.room,
undefined,
false,
{} as IRoomTimelineData,
);
// Invalid emoji
rtcSession.room.emit(
MatrixRoomEvent.Timeline,
new MatrixEvent({
room_id: rtcSession.room.roomId,
event_id: reactionEventId,
sender: localRtcMember.sender,
type: ElementCallReactionEventType,
content: {
emoji: " ",
name: reaction.name,
"m.relates_to": {
event_id: localRtcMember.eventId,
},
},
}),
rtcSession.room,
undefined,
false,
{} as IRoomTimelineData,
);
},
});
expectObservable(reactions$).toBe("a-", {
a: {},
});
});
});
});
test("that reactions cannot be spammed", () => {
const { rtcSession } = getBasicRTCSession([local, alice]);
const reactionEventId = "$my_event_id:example.org";
const reactionA = ReactionSet[1];
const reactionB = ReactionSet[2];
vitest.useFakeTimers();
vitest.setSystemTime(0);
withTestScheduler(({ schedule, expectObservable }) => {
renderHook(() => {
const { reactions$ } = new ReactionsReader(
rtcSession as unknown as MatrixRTCSession,
);
schedule("abcd", {
a: () => {},
b: () => {
rtcSession.room.emit(
MatrixRoomEvent.Timeline,
new MatrixEvent({
room_id: rtcSession.room.roomId,
event_id: reactionEventId,
sender: localRtcMember.sender,
type: ElementCallReactionEventType,
content: {
emoji: reactionA.emoji,
name: reactionA.name,
"m.relates_to": {
event_id: localRtcMember.eventId,
},
},
}),
rtcSession.room,
undefined,
false,
{} as IRoomTimelineData,
);
},
c: () => {
rtcSession.room.emit(
MatrixRoomEvent.Timeline,
new MatrixEvent({
room_id: rtcSession.room.roomId,
event_id: reactionEventId,
sender: localRtcMember.sender,
type: ElementCallReactionEventType,
content: {
emoji: reactionB.emoji,
name: reactionB.name,
"m.relates_to": {
event_id: localRtcMember.eventId,
},
},
}),
rtcSession.room,
undefined,
false,
{} as IRoomTimelineData,
);
},
d: () => {
vitest.advanceTimersByTime(REACTION_ACTIVE_TIME_MS);
},
});
expectObservable(reactions$).toBe(
`ab- ${REACTION_ACTIVE_TIME_MS - 2}ms d`,
{
a: {},
b: {
[`${localRtcMember.sender}:${localRtcMember.deviceId}`]: {
reactionOption: reactionA,
expireAfter: new Date(REACTION_ACTIVE_TIME_MS),
},
},
d: {},
},
);
});
});
});

View File

@@ -0,0 +1,339 @@
/*
Copyright 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import {
type CallMembership,
MatrixRTCSessionEvent,
type MatrixRTCSession,
} from "matrix-js-sdk/src/matrixrtc";
import { logger } from "matrix-js-sdk/src/logger";
import { type MatrixEvent, MatrixEventEvent } from "matrix-js-sdk/src/matrix";
import { type ReactionEventContent } from "matrix-js-sdk/src/types";
import {
RelationType,
EventType,
RoomEvent as MatrixRoomEvent,
} from "matrix-js-sdk/src/matrix";
import { BehaviorSubject, delay, type Subscription } from "rxjs";
import {
ElementCallReactionEventType,
type ECallReactionEventContent,
GenericReaction,
ReactionSet,
type RaisedHandInfo,
type ReactionInfo,
} from ".";
export const REACTION_ACTIVE_TIME_MS = 3000;
/**
* Listens for reactions from a RTCSession and populates subjects
* for consumption by the CallViewModel.
* @param rtcSession
*/
export class ReactionsReader {
private readonly raisedHandsSubject$ = new BehaviorSubject<
Record<string, RaisedHandInfo>
>({});
private readonly reactionsSubject$ = new BehaviorSubject<
Record<string, ReactionInfo>
>({});
/**
* The latest set of raised hands.
*/
public readonly raisedHands$ = this.raisedHandsSubject$.asObservable();
/**
* The latest set of reactions.
*/
public readonly reactions$ = this.reactionsSubject$.asObservable();
private readonly reactionsSub: Subscription;
public constructor(private readonly rtcSession: MatrixRTCSession) {
// Hide reactions after a given time.
this.reactionsSub = this.reactionsSubject$
.pipe(delay(REACTION_ACTIVE_TIME_MS))
.subscribe((reactions) => {
const date = new Date();
const nextEntries = Object.fromEntries(
Object.entries(reactions).filter(([_, hr]) => hr.expireAfter > date),
);
if (Object.keys(reactions).length === Object.keys(nextEntries).length) {
return;
}
this.reactionsSubject$.next(nextEntries);
});
this.rtcSession.room.on(MatrixRoomEvent.Timeline, this.handleReactionEvent);
this.rtcSession.room.on(
MatrixRoomEvent.Redaction,
this.handleReactionEvent,
);
this.rtcSession.room.client.on(
MatrixEventEvent.Decrypted,
this.handleReactionEvent,
);
// We listen for a local echo to get the real event ID, as timeline events
// may still be sending.
this.rtcSession.room.on(
MatrixRoomEvent.LocalEchoUpdated,
this.handleReactionEvent,
);
rtcSession.on(
MatrixRTCSessionEvent.MembershipsChanged,
this.onMembershipsChanged,
);
// Run this once to ensure we have fetched the state from the call.
this.onMembershipsChanged([]);
}
/**
* Fetchest any hand wave reactions by the given sender on the given
* membership event.
* @param membershipEventId
* @param expectedSender
* @returns A MatrixEvent if one was found.
*/
private getLastReactionEvent(
membershipEventId: string,
expectedSender: string,
): MatrixEvent | undefined {
const relations = this.rtcSession.room.relations.getChildEventsForEvent(
membershipEventId,
RelationType.Annotation,
EventType.Reaction,
);
const allEvents = relations?.getRelations() ?? [];
return allEvents.find(
(reaction) =>
reaction.event.sender === expectedSender &&
reaction.getType() === EventType.Reaction &&
reaction.getContent()?.["m.relates_to"]?.key === "🖐️",
);
}
/**
* Will remove any hand raises by old members, and look for any
* existing hand raises by new members.
* @param oldMemberships Any members who have left the call.
*/
private onMembershipsChanged = (oldMemberships: CallMembership[]): void => {
// Remove any raised hands for users no longer joined to the call.
for (const identifier of Object.keys(this.raisedHandsSubject$.value).filter(
(rhId) => oldMemberships.find((u) => u.sender == rhId),
)) {
this.removeRaisedHand(identifier);
}
// For each member in the call, check to see if a reaction has
// been raised and adjust.
for (const m of this.rtcSession.memberships) {
if (!m.sender || !m.eventId) {
continue;
}
const identifier = `${m.sender}:${m.deviceId}`;
if (
this.raisedHandsSubject$.value[identifier] &&
this.raisedHandsSubject$.value[identifier].membershipEventId !==
m.eventId
) {
// Membership event for sender has changed since the hand
// was raised, reset.
this.removeRaisedHand(identifier);
}
const reaction = this.getLastReactionEvent(m.eventId, m.sender);
if (reaction) {
const eventId = reaction?.getId();
if (!eventId) {
continue;
}
this.addRaisedHand(`${m.sender}:${m.deviceId}`, {
membershipEventId: m.eventId,
reactionEventId: eventId,
time: new Date(reaction.localTimestamp),
});
}
}
};
/**
* Add a raised hand
* @param identifier A userId:deviceId combination.
* @param info The event information.
*/
private addRaisedHand(identifier: string, info: RaisedHandInfo): void {
this.raisedHandsSubject$.next({
...this.raisedHandsSubject$.value,
[identifier]: info,
});
}
/**
* Remove a raised hand
* @param identifier A userId:deviceId combination.
*/
private removeRaisedHand(identifier: string): void {
this.raisedHandsSubject$.next(
Object.fromEntries(
Object.entries(this.raisedHandsSubject$.value).filter(
([uId]) => uId !== identifier,
),
),
);
}
/**
* Handle a new reaction event, validating it's contents and potentially
* updating the hand raise or reaction observers.
* @param event The incoming matrix event, which may or may not be decrypted.
*/
private handleReactionEvent = (event: MatrixEvent): void => {
const room = this.rtcSession.room;
// Decrypted events might come from a different room
if (event.getRoomId() !== room.roomId) return;
// Skip any events that are still sending.
if (event.isSending()) return;
const sender = event.getSender();
const reactionEventId = event.getId();
// Skip any event without a sender or event ID.
if (!sender || !reactionEventId) return;
room.client
.decryptEventIfNeeded(event)
.catch((e) => logger.warn(`Failed to decrypt ${event.getId()}`, e));
if (event.isBeingDecrypted() || event.isDecryptionFailure()) return;
if (event.getType() === ElementCallReactionEventType) {
const content: ECallReactionEventContent = event.getContent();
const membershipEventId = content?.["m.relates_to"]?.event_id;
const membershipEvent = this.rtcSession.memberships.find(
(e) => e.eventId === membershipEventId && e.sender === sender,
);
// Check to see if this reaction was made to a membership event (and the
// sender of the reaction matches the membership)
if (!membershipEvent) {
logger.warn(
`Reaction target was not a membership event for ${sender}, ignoring`,
);
return;
}
const identifier = `${membershipEvent.sender}:${membershipEvent.deviceId}`;
if (!content.emoji) {
logger.warn(`Reaction had no emoji from ${reactionEventId}`);
return;
}
const segment = new Intl.Segmenter(undefined, {
granularity: "grapheme",
})
.segment(content.emoji)
[Symbol.iterator]();
const emoji = segment.next().value?.segment;
if (!emoji?.trim()) {
logger.warn(
`Reaction had no emoji from ${reactionEventId} after splitting`,
);
return;
}
// One of our custom reactions
const reaction = {
...GenericReaction,
emoji,
// If we don't find a reaction, we can fallback to the generic sound.
...ReactionSet.find((r) => r.name === content.name),
};
const currentReactions = this.reactionsSubject$.value;
if (currentReactions[identifier]) {
// We've still got a reaction from this user, ignore it to prevent spamming
logger.warn(`Got reaction from ${identifier} but one is still playing`);
return;
}
this.reactionsSubject$.next({
...currentReactions,
[identifier]: {
reactionOption: reaction,
expireAfter: new Date(Date.now() + REACTION_ACTIVE_TIME_MS),
},
});
} else if (event.getType() === EventType.Reaction) {
const content = event.getContent() as ReactionEventContent;
const membershipEventId = content["m.relates_to"].event_id;
// Check to see if this reaction was made to a membership event (and the
// sender of the reaction matches the membership)
const membershipEvent = this.rtcSession.memberships.find(
(e) => e.eventId === membershipEventId && e.sender === sender,
);
if (!membershipEvent) {
logger.warn(
`Reaction target was not a membership event for ${sender}, ignoring`,
);
return;
}
if (content?.["m.relates_to"].key === "🖐️") {
this.addRaisedHand(
`${membershipEvent.sender}:${membershipEvent.deviceId}`,
{
reactionEventId,
membershipEventId,
time: new Date(event.localTimestamp),
},
);
}
} else if (event.getType() === EventType.RoomRedaction) {
const targetEvent = event.event.redacts;
const targetUser = Object.entries(this.raisedHandsSubject$.value).find(
([_u, r]) => r.reactionEventId === targetEvent,
)?.[0];
if (!targetUser) {
// Reaction target was not for us, ignoring
return;
}
this.removeRaisedHand(targetUser);
}
};
/**
* Stop listening for events.
*/
public destroy(): void {
this.rtcSession.off(
MatrixRTCSessionEvent.MembershipsChanged,
this.onMembershipsChanged,
);
this.rtcSession.room.off(
MatrixRoomEvent.Timeline,
this.handleReactionEvent,
);
this.rtcSession.room.off(
MatrixRoomEvent.Redaction,
this.handleReactionEvent,
);
this.rtcSession.room.client.off(
MatrixEventEvent.Decrypted,
this.handleReactionEvent,
);
this.rtcSession.room.off(
MatrixRoomEvent.LocalEchoUpdated,
this.handleReactionEvent,
);
this.reactionsSub.unsubscribe();
}
}

View File

@@ -181,3 +181,23 @@ export const ReactionSet: ReactionOption[] = [
},
},
];
export interface RaisedHandInfo {
/**
* Call membership event that was reacted to.
*/
membershipEventId: string;
/**
* Event ID of the reaction itself.
*/
reactionEventId: string;
/**
* The time when the reaction was raised.
*/
time: Date;
}
export interface ReactionInfo {
expireAfter: Date;
reactionOption: ReactionOption;
}

View File

@@ -0,0 +1,174 @@
/*
Copyright 2024 Milton Moura <miltonmoura@gmail.com>
SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { EventType, RelationType } from "matrix-js-sdk/src/matrix";
import {
createContext,
useContext,
type ReactNode,
useCallback,
useMemo,
} from "react";
import { type MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession";
import { logger } from "matrix-js-sdk/src/logger";
import { useObservableEagerState } from "observable-hooks";
import { useMatrixRTCSessionMemberships } from "../useMatrixRTCSessionMemberships";
import { useClientState } from "../ClientContext";
import { ElementCallReactionEventType, type ReactionOption } from ".";
import { type CallViewModel } from "../state/CallViewModel";
interface ReactionsSenderContextType {
supportsReactions: boolean;
toggleRaisedHand: () => Promise<void>;
sendReaction: (reaction: ReactionOption) => Promise<void>;
}
const ReactionsSenderContext = createContext<
ReactionsSenderContextType | undefined
>(undefined);
export const useReactionsSender = (): ReactionsSenderContextType => {
const context = useContext(ReactionsSenderContext);
if (!context) {
throw new Error("useReactions must be used within a ReactionsProvider");
}
return context;
};
/**
* Provider that handles sending a reaction or hand raised event to a call.
*/
export const ReactionsSenderProvider = ({
children,
rtcSession,
vm,
}: {
children: ReactNode;
rtcSession: MatrixRTCSession;
vm: CallViewModel;
}): JSX.Element => {
const memberships = useMatrixRTCSessionMemberships(rtcSession);
const clientState = useClientState();
const supportsReactions =
clientState?.state === "valid" && clientState.supportedFeatures.reactions;
const room = rtcSession.room;
const myUserId = room.client.getUserId();
const myDeviceId = room.client.getDeviceId();
const myMembershipEvent = useMemo(
() =>
memberships.find(
(m) => m.sender === myUserId && m.deviceId === myDeviceId,
)?.eventId,
[memberships, myUserId, myDeviceId],
);
const myMembershipIdentifier = useMemo(() => {
const membership = memberships.find((m) => m.sender === myUserId);
return membership
? `${membership.sender}:${membership.deviceId}`
: undefined;
}, [memberships, myUserId]);
const reactions = useObservableEagerState(vm.reactions$);
const myReaction = useMemo(
() =>
myMembershipIdentifier !== undefined
? reactions[myMembershipIdentifier]
: undefined,
[myMembershipIdentifier, reactions],
);
const handsRaised = useObservableEagerState(vm.handsRaised$);
const myRaisedHand = useMemo(
() =>
myMembershipIdentifier !== undefined
? handsRaised[myMembershipIdentifier]
: undefined,
[myMembershipIdentifier, handsRaised],
);
const toggleRaisedHand = useCallback(async () => {
if (!myMembershipIdentifier) {
return;
}
const myReactionId = myRaisedHand?.reactionEventId;
if (!myReactionId) {
try {
if (!myMembershipEvent) {
throw new Error("Cannot find own membership event");
}
const reaction = await room.client.sendEvent(
rtcSession.room.roomId,
EventType.Reaction,
{
"m.relates_to": {
rel_type: RelationType.Annotation,
event_id: myMembershipEvent,
key: "🖐️",
},
},
);
logger.debug("Sent raise hand event", reaction.event_id);
} catch (ex) {
logger.error("Failed to send raised hand", ex);
}
} else {
try {
await room.client.redactEvent(rtcSession.room.roomId, myReactionId);
logger.debug("Redacted raise hand event");
} catch (ex) {
logger.error("Failed to redact reaction event", myReactionId, ex);
throw ex;
}
}
}, [
myMembershipEvent,
myMembershipIdentifier,
myRaisedHand,
rtcSession,
room,
]);
const sendReaction = useCallback(
async (reaction: ReactionOption) => {
if (!myMembershipIdentifier || myReaction) {
// We're still reacting
return;
}
if (!myMembershipEvent) {
throw new Error("Cannot find own membership event");
}
await room.client.sendEvent(
rtcSession.room.roomId,
ElementCallReactionEventType,
{
"m.relates_to": {
rel_type: RelationType.Reference,
event_id: myMembershipEvent,
},
emoji: reaction.emoji,
name: reaction.name,
},
);
},
[myMembershipEvent, myReaction, room, myMembershipIdentifier, rtcSession],
);
return (
<ReactionsSenderContext.Provider
value={{
supportsReactions,
toggleRaisedHand,
sendReaction,
}}
>
{children}
</ReactionsSenderContext.Provider>
);
};

View File

@@ -15,43 +15,23 @@ import {
vitest,
afterEach,
} from "vitest";
import { type MatrixClient } from "matrix-js-sdk/src/client";
import { ConnectionState } from "livekit-client";
import { BehaviorSubject, of } from "rxjs";
import { act, type ReactNode } from "react";
import {
type CallMembership,
type MatrixRTCSession,
} from "matrix-js-sdk/src/matrixrtc";
import { type RoomMember } from "matrix-js-sdk/src/matrix";
import { act } from "react";
import { type CallMembership } from "matrix-js-sdk/src/matrixrtc";
import {
mockLivekitRoom,
mockLocalParticipant,
mockMatrixRoom,
mockMatrixRoomMember,
mockRemoteParticipant,
mockRtcMembership,
MockRTCSession,
} from "../utils/test";
import { E2eeType } from "../e2ee/e2eeType";
import { CallViewModel } from "../state/CallViewModel";
import { mockRtcMembership } from "../utils/test";
import {
CallEventAudioRenderer,
MAX_PARTICIPANT_COUNT_FOR_SOUND,
} from "./CallEventAudioRenderer";
import { useAudioContext } from "../useAudioContext";
import { TestReactionsWrapper } from "../utils/testReactions";
import { prefetchSounds } from "../soundUtils";
const localRtcMember = mockRtcMembership("@carol:example.org", "CCCC");
const local = mockMatrixRoomMember(localRtcMember);
const aliceRtcMember = mockRtcMembership("@alice:example.org", "AAAA");
const alice = mockMatrixRoomMember(aliceRtcMember);
const bobRtcMember = mockRtcMembership("@bob:example.org", "BBBB");
const localParticipant = mockLocalParticipant({ identity: "" });
const aliceId = `${alice.userId}:${aliceRtcMember.deviceId}`;
const aliceParticipant = mockRemoteParticipant({ identity: aliceId });
import { getBasicCallViewModelEnvironment } from "../utils/test-viewmodel";
import {
alice,
aliceRtcMember,
bobRtcMember,
local,
} from "../utils/test-fixtures";
vitest.mock("../useAudioContext");
vitest.mock("../soundUtils");
@@ -78,66 +58,6 @@ beforeEach(() => {
});
});
function TestComponent({
rtcSession,
vm,
}: {
rtcSession: MockRTCSession;
vm: CallViewModel;
}): ReactNode {
return (
<TestReactionsWrapper
rtcSession={rtcSession as unknown as MatrixRTCSession}
>
<CallEventAudioRenderer vm={vm} />
</TestReactionsWrapper>
);
}
function getMockEnv(
members: RoomMember[],
initialRemoteRtcMemberships: CallMembership[] = [aliceRtcMember],
): {
vm: CallViewModel;
session: MockRTCSession;
remoteRtcMemberships$: BehaviorSubject<CallMembership[]>;
} {
const matrixRoomMembers = new Map(members.map((p) => [p.userId, p]));
const remoteParticipants$ = of([aliceParticipant]);
const liveKitRoom = mockLivekitRoom(
{ localParticipant },
{ remoteParticipants$ },
);
const matrixRoom = mockMatrixRoom({
client: {
getUserId: () => localRtcMember.sender,
getDeviceId: () => localRtcMember.deviceId,
on: vitest.fn(),
off: vitest.fn(),
} as Partial<MatrixClient> as MatrixClient,
getMember: (userId) => matrixRoomMembers.get(userId) ?? null,
});
const remoteRtcMemberships$ = new BehaviorSubject<CallMembership[]>(
initialRemoteRtcMemberships,
);
const session = new MockRTCSession(
matrixRoom,
localRtcMember,
).withMemberships(remoteRtcMemberships$);
const vm = new CallViewModel(
session as unknown as MatrixRTCSession,
liveKitRoom,
{
kind: E2eeType.PER_PARTICIPANT,
},
of(ConnectionState.Connected),
);
return { vm, session, remoteRtcMemberships$ };
}
/**
* We don't want to play a sound when loading the call state
* because typically this occurs in two stages. We first join
@@ -146,8 +66,12 @@ function getMockEnv(
* a noise every time.
*/
test("plays one sound when entering a call", () => {
const { session, vm, remoteRtcMemberships$ } = getMockEnv([local, alice]);
render(<TestComponent rtcSession={session} vm={vm} />);
const { vm, remoteRtcMemberships$ } = getBasicCallViewModelEnvironment([
local,
alice,
]);
render(<CallEventAudioRenderer vm={vm} />);
// Joining a call usually means remote participants are added later.
act(() => {
remoteRtcMemberships$.next([aliceRtcMember, bobRtcMember]);
@@ -155,10 +79,12 @@ test("plays one sound when entering a call", () => {
expect(playSound).toHaveBeenCalledOnce();
});
// TODO: Same test?
test("plays a sound when a user joins", () => {
const { session, vm, remoteRtcMemberships$ } = getMockEnv([local, alice]);
render(<TestComponent rtcSession={session} vm={vm} />);
const { vm, remoteRtcMemberships$ } = getBasicCallViewModelEnvironment([
local,
alice,
]);
render(<CallEventAudioRenderer vm={vm} />);
act(() => {
remoteRtcMemberships$.next([aliceRtcMember, bobRtcMember]);
@@ -168,8 +94,11 @@ test("plays a sound when a user joins", () => {
});
test("plays a sound when a user leaves", () => {
const { session, vm, remoteRtcMemberships$ } = getMockEnv([local, alice]);
render(<TestComponent rtcSession={session} vm={vm} />);
const { vm, remoteRtcMemberships$ } = getBasicCallViewModelEnvironment([
local,
alice,
]);
render(<CallEventAudioRenderer vm={vm} />);
act(() => {
remoteRtcMemberships$.next([]);
@@ -185,12 +114,12 @@ test("plays no sound when the participant list is more than the maximum size", (
);
}
const { session, vm, remoteRtcMemberships$ } = getMockEnv(
const { vm, remoteRtcMemberships$ } = getBasicCallViewModelEnvironment(
[local, alice],
mockRtcMemberships,
);
render(<TestComponent rtcSession={session} vm={vm} />);
render(<CallEventAudioRenderer vm={vm} />);
expect(playSound).not.toBeCalled();
act(() => {
remoteRtcMemberships$.next(
@@ -199,3 +128,56 @@ test("plays no sound when the participant list is more than the maximum size", (
});
expect(playSound).toBeCalledWith("left");
});
test("plays one sound when a hand is raised", () => {
const { vm, handRaisedSubject$ } = getBasicCallViewModelEnvironment([
local,
alice,
]);
render(<CallEventAudioRenderer vm={vm} />);
act(() => {
handRaisedSubject$.next({
[bobRtcMember.callId]: {
time: new Date(),
membershipEventId: "",
reactionEventId: "",
},
});
});
expect(playSound).toBeCalledWith("raiseHand");
});
test("should not play a sound when a hand raise is retracted", () => {
const { vm, handRaisedSubject$ } = getBasicCallViewModelEnvironment([
local,
alice,
]);
render(<CallEventAudioRenderer vm={vm} />);
act(() => {
handRaisedSubject$.next({
["foo"]: {
time: new Date(),
membershipEventId: "",
reactionEventId: "",
},
["bar"]: {
time: new Date(),
membershipEventId: "",
reactionEventId: "",
},
});
});
expect(playSound).toHaveBeenCalledTimes(2);
act(() => {
handRaisedSubject$.next({
["foo"]: {
time: new Date(),
membershipEventId: "",
reactionEventId: "",
},
});
});
expect(playSound).toHaveBeenCalledTimes(2);
});

View File

@@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { type ReactNode, useDeferredValue, useEffect, useMemo } from "react";
import { type ReactNode, useEffect } from "react";
import { filter, interval, throttle } from "rxjs";
import { type CallViewModel } from "../state/CallViewModel";
@@ -19,7 +19,6 @@ import screenShareStartedOgg from "../sound/screen_share_started.ogg";
import screenShareStartedMp3 from "../sound/screen_share_started.mp3";
import { useAudioContext } from "../useAudioContext";
import { prefetchSounds } from "../soundUtils";
import { useReactions } from "../useReactions";
import { useLatest } from "../useLatest";
// Do not play any sounds if the participant count has exceeded this
@@ -57,19 +56,6 @@ export function CallEventAudioRenderer({
});
const audioEngineRef = useLatest(audioEngineCtx);
const { raisedHands } = useReactions();
const raisedHandCount = useMemo(
() => Object.keys(raisedHands).length,
[raisedHands],
);
const previousRaisedHandCount = useDeferredValue(raisedHandCount);
useEffect(() => {
if (audioEngineRef.current && previousRaisedHandCount < raisedHandCount) {
void audioEngineRef.current.playSound("raiseHand");
}
}, [audioEngineRef, previousRaisedHandCount, raisedHandCount]);
useEffect(() => {
const joinSub = vm.memberChanges$
.pipe(
@@ -95,6 +81,10 @@ export function CallEventAudioRenderer({
void audioEngineRef.current?.playSound("left");
});
const handRaisedSub = vm.newHandRaised$.subscribe(() => {
void audioEngineRef.current?.playSound("raiseHand");
});
const screenshareSub = vm.newScreenShare$.subscribe(() => {
void audioEngineRef.current?.playSound("screenshareStarted");
});
@@ -102,6 +92,7 @@ export function CallEventAudioRenderer({
return (): void => {
joinSub.unsubscribe();
leftSub.unsubscribe();
handRaisedSub.unsubscribe();
screenshareSub.unsubscribe();
};
}, [audioEngineRef, vm]);

View File

@@ -14,6 +14,7 @@ import { JoinRule, type RoomState } from "matrix-js-sdk/src/matrix";
import { Router } from "react-router-dom";
import { createBrowserHistory } from "history";
import userEvent from "@testing-library/user-event";
import { type RelationsContainer } from "matrix-js-sdk/src/models/relations-container";
import { type MuteStates } from "./MuteStates";
import { prefetchSounds } from "../soundUtils";
@@ -85,6 +86,12 @@ function createGroupCallView(widget: WidgetHelpers | null): {
getRoom: (rId) => (rId === roomId ? room : null),
} as Partial<MatrixClient> as MatrixClient;
const room = mockMatrixRoom({
relations: {
getChildEventsForEvent: () =>
vitest.mocked({
getRelations: () => [],
}),
} as unknown as RelationsContainer,
client,
roomId,
getMember: (userId) => roomMembers.get(userId) ?? null,

View File

@@ -366,7 +366,7 @@ export const GroupCallView: FC<Props> = ({
<ActiveCall
client={client}
matrixInfo={matrixInfo}
rtcSession={rtcSession as unknown as MatrixRTCSession}
rtcSession={rtcSession as MatrixRTCSession}
participantCount={participantCount}
onLeave={onLeave}
hideHeader={hideHeader}

View File

@@ -83,7 +83,10 @@ import { makeSpotlightExpandedLayout } from "../grid/SpotlightExpandedLayout";
import { makeSpotlightLandscapeLayout } from "../grid/SpotlightLandscapeLayout";
import { makeSpotlightPortraitLayout } from "../grid/SpotlightPortraitLayout";
import { GridTileViewModel, type TileViewModel } from "../state/TileViewModel";
import { ReactionsProvider, useReactions } from "../useReactions";
import {
ReactionsSenderProvider,
useReactionsSender,
} from "../reactions/useReactionsSender";
import { ReactionsAudioRenderer } from "./ReactionAudioRenderer";
import { useSwitchCamera } from "./useSwitchCamera";
import { ReactionsOverlay } from "./ReactionsOverlay";
@@ -92,6 +95,7 @@ import {
debugTileLayout as debugTileLayoutSetting,
useSetting,
} from "../settings/settings";
import { ReactionsReader } from "../reactions/ReactionsReader";
const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {});
@@ -127,14 +131,20 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
useEffect(() => {
if (livekitRoom !== undefined) {
const reactionsReader = new ReactionsReader(props.rtcSession);
const vm = new CallViewModel(
props.rtcSession,
livekitRoom,
props.e2eeSystem,
connStateObservable$,
reactionsReader.raisedHands$,
reactionsReader.reactions$,
);
setVm(vm);
return (): void => vm.destroy();
return (): void => {
vm.destroy();
reactionsReader.destroy();
};
}
}, [props.rtcSession, livekitRoom, props.e2eeSystem, connStateObservable$]);
@@ -142,14 +152,14 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
return (
<RoomContext.Provider value={livekitRoom}>
<ReactionsProvider rtcSession={props.rtcSession}>
<ReactionsSenderProvider vm={vm} rtcSession={props.rtcSession}>
<InCallView
{...props}
vm={vm}
livekitRoom={livekitRoom}
connState={connState}
/>
</ReactionsProvider>
</ReactionsSenderProvider>
</RoomContext.Provider>
);
};
@@ -182,7 +192,8 @@ export const InCallView: FC<InCallViewProps> = ({
connState,
onShareClick,
}) => {
const { supportsReactions, sendReaction, toggleRaisedHand } = useReactions();
const { supportsReactions, sendReaction, toggleRaisedHand } =
useReactionsSender();
useWakeLock();
@@ -551,9 +562,10 @@ export const InCallView: FC<InCallViewProps> = ({
if (supportsReactions) {
buttons.push(
<ReactionToggleButton
vm={vm}
key="raise_hand"
className={styles.raiseHand}
userId={client.getUserId()!}
identifier={`${client.getUserId()}:${client.getDeviceId()}`}
onTouchEnd={onControlsTouchEnd}
/>,
);
@@ -653,8 +665,8 @@ export const InCallView: FC<InCallViewProps> = ({
<RoomAudioRenderer />
{renderContent()}
<CallEventAudioRenderer vm={vm} />
<ReactionsAudioRenderer />
<ReactionsOverlay />
<ReactionsAudioRenderer vm={vm} />
<ReactionsOverlay vm={vm} />
{footer}
{layout.type !== "pip" && (
<>

View File

@@ -19,11 +19,6 @@ import {
import { TooltipProvider } from "@vector-im/compound-web";
import { act, type ReactNode } from "react";
import {
MockRoom,
MockRTCSession,
TestReactionsWrapper,
} from "../utils/testReactions";
import { ReactionsAudioRenderer } from "./ReactionAudioRenderer";
import {
playReactionsSound,
@@ -32,30 +27,20 @@ import {
import { useAudioContext } from "../useAudioContext";
import { GenericReaction, ReactionSet } from "../reactions";
import { prefetchSounds } from "../soundUtils";
import { type CallViewModel } from "../state/CallViewModel";
import { getBasicCallViewModelEnvironment } from "../utils/test-viewmodel";
import {
alice,
aliceRtcMember,
bobRtcMember,
local,
localRtcMember,
} from "../utils/test-fixtures";
const memberUserIdAlice = "@alice:example.org";
const memberUserIdBob = "@bob:example.org";
const memberUserIdCharlie = "@charlie:example.org";
const memberEventAlice = "$membership-alice:example.org";
const memberEventBob = "$membership-bob:example.org";
const memberEventCharlie = "$membership-charlie:example.org";
const membership: Record<string, string> = {
[memberEventAlice]: memberUserIdAlice,
[memberEventBob]: memberUserIdBob,
[memberEventCharlie]: memberUserIdCharlie,
};
function TestComponent({
rtcSession,
}: {
rtcSession: MockRTCSession;
}): ReactNode {
function TestComponent({ vm }: { vm: CallViewModel }): ReactNode {
return (
<TooltipProvider>
<TestReactionsWrapper rtcSession={rtcSession}>
<ReactionsAudioRenderer />
</TestReactionsWrapper>
<ReactionsAudioRenderer vm={vm} />
</TooltipProvider>
);
}
@@ -88,20 +73,19 @@ beforeEach(() => {
});
test("preloads all audio elements", () => {
const { vm } = getBasicCallViewModelEnvironment([local, alice]);
playReactionsSound.setValue(true);
const rtcSession = new MockRTCSession(
new MockRoom(memberUserIdAlice),
membership,
);
render(<TestComponent rtcSession={rtcSession} />);
render(<TestComponent vm={vm} />);
expect(prefetchSounds).toHaveBeenCalledOnce();
});
test("will play an audio sound when there is a reaction", () => {
const { vm, reactionsSubject$ } = getBasicCallViewModelEnvironment([
local,
alice,
]);
playReactionsSound.setValue(true);
const room = new MockRoom(memberUserIdAlice);
const rtcSession = new MockRTCSession(room, membership);
render(<TestComponent rtcSession={rtcSession} />);
render(<TestComponent vm={vm} />);
// Find the first reaction with a sound effect
const chosenReaction = ReactionSet.find((r) => !!r.sound);
@@ -111,16 +95,23 @@ test("will play an audio sound when there is a reaction", () => {
);
}
act(() => {
room.testSendReaction(memberEventAlice, chosenReaction, membership);
reactionsSubject$.next({
[aliceRtcMember.deviceId]: {
reactionOption: chosenReaction,
expireAfter: new Date(0),
},
});
});
expect(playSound).toHaveBeenCalledWith(chosenReaction.name);
});
test("will play the generic audio sound when there is soundless reaction", () => {
const { vm, reactionsSubject$ } = getBasicCallViewModelEnvironment([
local,
alice,
]);
playReactionsSound.setValue(true);
const room = new MockRoom(memberUserIdAlice);
const rtcSession = new MockRTCSession(room, membership);
render(<TestComponent rtcSession={rtcSession} />);
render(<TestComponent vm={vm} />);
// Find the first reaction with a sound effect
const chosenReaction = ReactionSet.find((r) => !r.sound);
@@ -130,17 +121,23 @@ test("will play the generic audio sound when there is soundless reaction", () =>
);
}
act(() => {
room.testSendReaction(memberEventAlice, chosenReaction, membership);
reactionsSubject$.next({
[aliceRtcMember.deviceId]: {
reactionOption: chosenReaction,
expireAfter: new Date(0),
},
});
});
expect(playSound).toHaveBeenCalledWith(GenericReaction.name);
});
test("will play multiple audio sounds when there are multiple different reactions", () => {
const { vm, reactionsSubject$ } = getBasicCallViewModelEnvironment([
local,
alice,
]);
playReactionsSound.setValue(true);
const room = new MockRoom(memberUserIdAlice);
const rtcSession = new MockRTCSession(room, membership);
render(<TestComponent rtcSession={rtcSession} />);
render(<TestComponent vm={vm} />);
// Find the first reaction with a sound effect
const [reaction1, reaction2] = ReactionSet.filter((r) => !!r.sound);
@@ -150,9 +147,20 @@ test("will play multiple audio sounds when there are multiple different reaction
);
}
act(() => {
room.testSendReaction(memberEventAlice, reaction1, membership);
room.testSendReaction(memberEventBob, reaction2, membership);
room.testSendReaction(memberEventCharlie, reaction1, membership);
reactionsSubject$.next({
[aliceRtcMember.deviceId]: {
reactionOption: reaction1,
expireAfter: new Date(0),
},
[bobRtcMember.deviceId]: {
reactionOption: reaction2,
expireAfter: new Date(0),
},
[localRtcMember.deviceId]: {
reactionOption: reaction1,
expireAfter: new Date(0),
},
});
});
expect(playSound).toHaveBeenCalledWith(reaction1.name);
expect(playSound).toHaveBeenCalledWith(reaction2.name);

View File

@@ -5,14 +5,14 @@ SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { type ReactNode, useDeferredValue, useEffect, useState } from "react";
import { type ReactNode, useEffect, useState } from "react";
import { useReactions } from "../useReactions";
import { playReactionsSound, useSetting } from "../settings/settings";
import { GenericReaction, ReactionSet } from "../reactions";
import { useAudioContext } from "../useAudioContext";
import { prefetchSounds } from "../soundUtils";
import { useLatest } from "../useLatest";
import { type CallViewModel } from "../state/CallViewModel";
const soundMap = Object.fromEntries([
...ReactionSet.filter((v) => v.sound !== undefined).map((v) => [
@@ -22,8 +22,11 @@ const soundMap = Object.fromEntries([
[GenericReaction.name, GenericReaction.sound],
]);
export function ReactionsAudioRenderer(): ReactNode {
const { reactions } = useReactions();
export function ReactionsAudioRenderer({
vm,
}: {
vm: CallViewModel;
}): ReactNode {
const [shouldPlay] = useSetting(playReactionsSound);
const [soundCache, setSoundCache] = useState<ReturnType<
typeof prefetchSounds
@@ -33,7 +36,6 @@ export function ReactionsAudioRenderer(): ReactNode {
latencyHint: "interactive",
});
const audioEngineRef = useLatest(audioEngineCtx);
const oldReactions = useDeferredValue(reactions);
useEffect(() => {
if (!shouldPlay || soundCache) {
@@ -46,26 +48,19 @@ export function ReactionsAudioRenderer(): ReactNode {
}, [soundCache, shouldPlay]);
useEffect(() => {
if (!shouldPlay || !audioEngineRef.current) {
return;
}
const oldReactionSet = new Set(
Object.values(oldReactions).map((r) => r.name),
);
for (const reactionName of new Set(
Object.values(reactions).map((r) => r.name),
)) {
if (oldReactionSet.has(reactionName)) {
// Don't replay old reactions
return;
const sub = vm.audibleReactions$.subscribe((newReactions) => {
for (const reactionName of newReactions) {
if (soundMap[reactionName]) {
void audioEngineRef.current?.playSound(reactionName);
} else {
// Fallback sounds.
void audioEngineRef.current?.playSound("generic");
}
}
if (soundMap[reactionName]) {
void audioEngineRef.current.playSound(reactionName);
} else {
// Fallback sounds.
void audioEngineRef.current.playSound("generic");
}
}
}, [audioEngineRef, shouldPlay, oldReactions, reactions]);
});
return (): void => {
sub.unsubscribe();
};
}, [vm, audioEngineRef]);
return null;
}

View File

@@ -7,44 +7,18 @@ Please see LICENSE in the repository root for full details.
import { render } from "@testing-library/react";
import { expect, test, afterEach } from "vitest";
import { TooltipProvider } from "@vector-im/compound-web";
import { act, type ReactNode } from "react";
import { act } from "react";
import {
MockRoom,
MockRTCSession,
TestReactionsWrapper,
} from "../utils/testReactions";
import { showReactions } from "../settings/settings";
import { ReactionsOverlay } from "./ReactionsOverlay";
import { ReactionSet } from "../reactions";
const memberUserIdAlice = "@alice:example.org";
const memberUserIdBob = "@bob:example.org";
const memberUserIdCharlie = "@charlie:example.org";
const memberEventAlice = "$membership-alice:example.org";
const memberEventBob = "$membership-bob:example.org";
const memberEventCharlie = "$membership-charlie:example.org";
const membership: Record<string, string> = {
[memberEventAlice]: memberUserIdAlice,
[memberEventBob]: memberUserIdBob,
[memberEventCharlie]: memberUserIdCharlie,
};
function TestComponent({
rtcSession,
}: {
rtcSession: MockRTCSession;
}): ReactNode {
return (
<TooltipProvider>
<TestReactionsWrapper rtcSession={rtcSession}>
<ReactionsOverlay />
</TestReactionsWrapper>
</TooltipProvider>
);
}
import {
local,
alice,
aliceRtcMember,
bobRtcMember,
} from "../utils/test-fixtures";
import { getBasicCallViewModelEnvironment } from "../utils/test-viewmodel";
afterEach(() => {
showReactions.setValue(showReactions.defaultValue);
@@ -52,22 +26,26 @@ afterEach(() => {
test("defaults to showing no reactions", () => {
showReactions.setValue(true);
const rtcSession = new MockRTCSession(
new MockRoom(memberUserIdAlice),
membership,
);
const { container } = render(<TestComponent rtcSession={rtcSession} />);
const { vm } = getBasicCallViewModelEnvironment([local, alice]);
const { container } = render(<ReactionsOverlay vm={vm} />);
expect(container.getElementsByTagName("span")).toHaveLength(0);
});
test("shows a reaction when sent", () => {
showReactions.setValue(true);
const { vm, reactionsSubject$ } = getBasicCallViewModelEnvironment([
local,
alice,
]);
const { getByRole } = render(<ReactionsOverlay vm={vm} />);
const reaction = ReactionSet[0];
const room = new MockRoom(memberUserIdAlice);
const rtcSession = new MockRTCSession(room, membership);
const { getByRole } = render(<TestComponent rtcSession={rtcSession} />);
act(() => {
room.testSendReaction(memberEventAlice, reaction, membership);
reactionsSubject$.next({
[aliceRtcMember.deviceId]: {
reactionOption: reaction,
expireAfter: new Date(0),
},
});
});
const span = getByRole("presentation");
expect(getByRole("presentation")).toBeTruthy();
@@ -77,29 +55,45 @@ test("shows a reaction when sent", () => {
test("shows two of the same reaction when sent", () => {
showReactions.setValue(true);
const reaction = ReactionSet[0];
const room = new MockRoom(memberUserIdAlice);
const rtcSession = new MockRTCSession(room, membership);
const { getAllByRole } = render(<TestComponent rtcSession={rtcSession} />);
const { vm, reactionsSubject$ } = getBasicCallViewModelEnvironment([
local,
alice,
]);
const { getAllByRole } = render(<ReactionsOverlay vm={vm} />);
act(() => {
room.testSendReaction(memberEventAlice, reaction, membership);
});
act(() => {
room.testSendReaction(memberEventBob, reaction, membership);
reactionsSubject$.next({
[aliceRtcMember.deviceId]: {
reactionOption: reaction,
expireAfter: new Date(0),
},
[bobRtcMember.deviceId]: {
reactionOption: reaction,
expireAfter: new Date(0),
},
});
});
expect(getAllByRole("presentation")).toHaveLength(2);
});
test("shows two different reactions when sent", () => {
showReactions.setValue(true);
const room = new MockRoom(memberUserIdAlice);
const rtcSession = new MockRTCSession(room, membership);
const [reactionA, reactionB] = ReactionSet;
const { getAllByRole } = render(<TestComponent rtcSession={rtcSession} />);
const { vm, reactionsSubject$ } = getBasicCallViewModelEnvironment([
local,
alice,
]);
const { getAllByRole } = render(<ReactionsOverlay vm={vm} />);
act(() => {
room.testSendReaction(memberEventAlice, reactionA, membership);
});
act(() => {
room.testSendReaction(memberEventBob, reactionB, membership);
reactionsSubject$.next({
[aliceRtcMember.deviceId]: {
reactionOption: reactionA,
expireAfter: new Date(0),
},
[bobRtcMember.deviceId]: {
reactionOption: reactionB,
expireAfter: new Date(0),
},
});
});
const [reactionElementA, reactionElementB] = getAllByRole("presentation");
expect(reactionElementA.innerHTML).toEqual(reactionA.emoji);
@@ -109,11 +103,18 @@ test("shows two different reactions when sent", () => {
test("hides reactions when reaction animations are disabled", () => {
showReactions.setValue(false);
const reaction = ReactionSet[0];
const room = new MockRoom(memberUserIdAlice);
const rtcSession = new MockRTCSession(room, membership);
const { vm, reactionsSubject$ } = getBasicCallViewModelEnvironment([
local,
alice,
]);
const { container } = render(<ReactionsOverlay vm={vm} />);
act(() => {
room.testSendReaction(memberEventAlice, reaction, membership);
reactionsSubject$.next({
[aliceRtcMember.deviceId]: {
reactionOption: reaction,
expireAfter: new Date(0),
},
});
});
const { container } = render(<TestComponent rtcSession={rtcSession} />);
expect(container.getElementsByTagName("span")).toHaveLength(0);
});

View File

@@ -5,33 +5,17 @@ SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { type ReactNode, useMemo } from "react";
import { type ReactNode } from "react";
import { useObservableState } from "observable-hooks";
import { useReactions } from "../useReactions";
import {
showReactions as showReactionsSetting,
useSetting,
} from "../settings/settings";
import styles from "./ReactionsOverlay.module.css";
import { type CallViewModel } from "../state/CallViewModel";
export function ReactionsOverlay(): ReactNode {
const { reactions } = useReactions();
const [showReactions] = useSetting(showReactionsSetting);
const reactionsIcons = useMemo(
() =>
showReactions
? Object.entries(reactions).map(([sender, { emoji }]) => ({
sender,
emoji,
startX: Math.ceil(Math.random() * 80) + 10,
}))
: [],
[showReactions, reactions],
);
export function ReactionsOverlay({ vm }: { vm: CallViewModel }): ReactNode {
const reactionsIcons = useObservableState(vm.visibleReactions$);
return (
<div className={styles.container}>
{reactionsIcons.map(({ sender, emoji, startX }) => (
{reactionsIcons?.map(({ sender, emoji, startX }) => (
<span
// Reactions effects are considered presentation elements. The reaction
// is also present on the sender's tile, which assistive technology can

View File

@@ -7,6 +7,7 @@ Please see LICENSE in the repository root for full details.
import { test, vi, onTestFinished, it } from "vitest";
import {
BehaviorSubject,
combineLatest,
debounceTime,
distinctUntilChanged,
@@ -46,6 +47,7 @@ import {
type ECConnectionState,
} from "../livekit/useECConnectionState";
import { E2eeType } from "../e2ee/e2eeType";
import type { RaisedHandInfo } from "../reactions";
import { showNonMemberTiles } from "../settings/settings";
vi.mock("@livekit/components-core");
@@ -190,7 +192,10 @@ function withCallViewModel(
rtcMembers$: Observable<Partial<CallMembership>[]>,
connectionState$: Observable<ECConnectionState>,
speaking: Map<Participant, Observable<boolean>>,
continuation: (vm: CallViewModel) => void,
continuation: (
vm: CallViewModel,
subjects: { raisedHands$: BehaviorSubject<Record<string, RaisedHandInfo>> },
) => void,
): void {
const room = mockMatrixRoom({
client: {
@@ -235,6 +240,8 @@ function withCallViewModel(
{ remoteParticipants$ },
);
const raisedHands$ = new BehaviorSubject<Record<string, RaisedHandInfo>>({});
const vm = new CallViewModel(
rtcSession as unknown as MatrixRTCSession,
liveKitRoom,
@@ -242,6 +249,8 @@ function withCallViewModel(
kind: E2eeType.PER_PARTICIPANT,
},
connectionState$,
raisedHands$,
new BehaviorSubject({}),
);
onTestFinished(() => {
@@ -252,7 +261,7 @@ function withCallViewModel(
roomEventSelectorSpy!.mockRestore();
});
continuation(vm);
continuation(vm, { raisedHands$: raisedHands$ });
}
test("participants are retained during a focus switch", () => {
@@ -782,3 +791,62 @@ it("should show at least one tile per MatrixRTCSession", () => {
);
});
});
it("should rank raised hands above video feeds and below speakers and presenters", () => {
withTestScheduler(({ schedule, expectObservable }) => {
// There should always be one tile for each MatrixRTCSession
const expectedLayoutMarbles = "ab";
withCallViewModel(
of([aliceParticipant, bobParticipant]),
of([aliceRtcMember, bobRtcMember]),
of(ConnectionState.Connected),
new Map(),
(vm, { raisedHands$ }) => {
schedule("ab", {
a: () => {
// We imagine that only two tiles (the first two) will be visible on screen at a time
vm.layout$.subscribe((layout) => {
if (layout.type === "grid") {
layout.setVisibleTiles(2);
}
});
},
b: () => {
raisedHands$.next({
[`${bobRtcMember.sender}:${bobRtcMember.deviceId}`]: {
time: new Date(),
reactionEventId: "",
membershipEventId: "",
},
});
},
});
expectObservable(summarizeLayout$(vm.layout$)).toBe(
expectedLayoutMarbles,
{
a: {
type: "grid",
spotlight: undefined,
grid: [
"local:0",
"@alice:example.org:AAAA:0",
"@bob:example.org:BBBB:0",
],
},
b: {
type: "grid",
spotlight: undefined,
grid: [
"local:0",
// Bob shifts up!
"@bob:example.org:BBBB:0",
"@alice:example.org:AAAA:0",
],
},
},
);
},
);
});
});

View File

@@ -69,7 +69,12 @@ import {
} from "./MediaViewModel";
import { accumulate, finalizeValue } from "../utils/observable";
import { ObservableScope } from "./ObservableScope";
import { duplicateTiles, showNonMemberTiles } from "../settings/settings";
import {
duplicateTiles,
playReactionsSound,
showReactions,
showNonMemberTiles,
} from "../settings/settings";
import { isFirefox } from "../Platform";
import { setPipEnabled$ } from "../controls";
import {
@@ -82,6 +87,11 @@ import { spotlightExpandedLayout } from "./SpotlightExpandedLayout";
import { oneOnOneLayout } from "./OneOnOneLayout";
import { pipLayout } from "./PipLayout";
import { type EncryptionSystem } from "../e2ee/sharedKeyManagement";
import {
type RaisedHandInfo,
type ReactionInfo,
type ReactionOption,
} from "../reactions";
import { observeSpeaker$ } from "./observeSpeaker";
import { shallowEquals } from "../utils/array";
@@ -210,6 +220,10 @@ enum SortingBin {
* Participants that have been speaking recently.
*/
Speakers,
/**
* Participants that have their hand raised.
*/
HandRaised,
/**
* Participants with video.
*/
@@ -244,6 +258,8 @@ class UserMedia {
participant: LocalParticipant | RemoteParticipant | undefined,
encryptionSystem: EncryptionSystem,
livekitRoom: LivekitRoom,
handRaised$: Observable<Date | null>,
reaction$: Observable<ReactionOption | null>,
) {
this.participant$ = new BehaviorSubject(participant);
@@ -254,6 +270,8 @@ class UserMedia {
this.participant$.asObservable() as Observable<LocalParticipant>,
encryptionSystem,
livekitRoom,
handRaised$,
reaction$,
);
} else {
this.vm = new RemoteUserMediaViewModel(
@@ -264,6 +282,8 @@ class UserMedia {
>,
encryptionSystem,
livekitRoom,
handRaised$,
reaction$,
);
}
@@ -473,6 +493,8 @@ export class CallViewModel extends ViewModel {
let livekitParticipantId =
rtcMember.sender + ":" + rtcMember.deviceId;
const matrixIdentifier = `${rtcMember.sender}:${rtcMember.deviceId}`;
let participant:
| LocalParticipant
| RemoteParticipant
@@ -522,6 +544,12 @@ export class CallViewModel extends ViewModel {
participant,
this.encryptionSystem,
this.livekitRoom,
this.handsRaised$.pipe(
map((v) => v[matrixIdentifier]?.time ?? null),
),
this.reactions$.pipe(
map((v) => v[matrixIdentifier] ?? undefined),
),
),
];
@@ -574,6 +602,8 @@ export class CallViewModel extends ViewModel {
participant,
this.encryptionSystem,
this.livekitRoom,
of(null),
of(null),
),
];
}
@@ -681,11 +711,12 @@ export class CallViewModel extends ViewModel {
m.speaker$,
m.presenter$,
m.vm.videoEnabled$,
m.vm.handRaised$,
m.vm instanceof LocalUserMediaViewModel
? m.vm.alwaysShow$
: of(false),
],
(speaker, presenter, video, alwaysShow) => {
(speaker, presenter, video, handRaised, alwaysShow) => {
let bin: SortingBin;
if (m.vm.local)
bin = alwaysShow
@@ -693,6 +724,7 @@ export class CallViewModel extends ViewModel {
: SortingBin.SelfNotAlwaysShown;
else if (presenter) bin = SortingBin.Presenters;
else if (speaker) bin = SortingBin.Speakers;
else if (handRaised) bin = SortingBin.HandRaised;
else if (video) bin = SortingBin.Video;
else bin = SortingBin.NoVideo;
@@ -1170,6 +1202,77 @@ export class CallViewModel extends ViewModel {
}),
this.scope.state(),
);
public readonly reactions$ = this.reactionsSubject$.pipe(
map((v) =>
Object.fromEntries(
Object.entries(v).map(([a, { reactionOption }]) => [a, reactionOption]),
),
),
);
public readonly handsRaised$ = this.handsRaisedSubject$.pipe();
/**
* Emits an array of reactions that should be visible on the screen.
*/
public readonly visibleReactions$ = showReactions.value$.pipe(
switchMap((show) => (show ? this.reactions$ : of({}))),
scan<
Record<string, ReactionOption>,
{ sender: string; emoji: string; startX: number }[]
>((acc, latest) => {
const newSet: { sender: string; emoji: string; startX: number }[] = [];
for (const [sender, reaction] of Object.entries(latest)) {
const startX =
acc.find((v) => v.sender === sender && v.emoji)?.startX ??
Math.ceil(Math.random() * 80) + 10;
newSet.push({ sender, emoji: reaction.emoji, startX });
}
return newSet;
}, []),
);
/**
* Emits an array of reactions that should be played.
*/
public readonly audibleReactions$ = playReactionsSound.value$.pipe(
switchMap((show) =>
show ? this.reactions$ : of<Record<string, ReactionOption>>({}),
),
map((reactions) => Object.values(reactions).map((v) => v.name)),
scan<string[], { playing: string[]; newSounds: string[] }>(
(acc, latest) => {
return {
playing: latest.filter(
(v) => acc.playing.includes(v) || acc.newSounds.includes(v),
),
newSounds: latest.filter(
(v) => !acc.playing.includes(v) && !acc.newSounds.includes(v),
),
};
},
{ playing: [], newSounds: [] },
),
map((v) => v.newSounds),
);
/**
* Emits an event every time a new hand is raised in
* the call.
*/
public readonly newHandRaised$ = this.handsRaised$.pipe(
map((v) => Object.keys(v).length),
scan(
(acc, newValue) => ({
value: newValue,
playSounds: newValue > acc.value,
}),
{ value: 0, playSounds: false },
),
filter((v) => v.playSounds),
);
/**
* Emits an event every time a new screenshare is started in
* the call.
@@ -1192,6 +1295,12 @@ export class CallViewModel extends ViewModel {
private readonly livekitRoom: LivekitRoom,
private readonly encryptionSystem: EncryptionSystem,
private readonly connectionState$: Observable<ECConnectionState>,
private readonly handsRaisedSubject$: Observable<
Record<string, RaisedHandInfo>
>,
private readonly reactionsSubject$: Observable<
Record<string, ReactionInfo>
>,
) {
super();
}

View File

@@ -51,6 +51,7 @@ import { alwaysShowSelf } from "../settings/settings";
import { accumulate } from "../utils/observable";
import { type EncryptionSystem } from "../e2ee/sharedKeyManagement";
import { E2eeType } from "../e2ee/e2eeType";
import { type ReactionOption } from "../reactions";
// TODO: Move this naming logic into the view model
export function useDisplayName(vm: MediaViewModel): string {
@@ -371,6 +372,8 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel {
participant$: Observable<LocalParticipant | RemoteParticipant | undefined>,
encryptionSystem: EncryptionSystem,
livekitRoom: LivekitRoom,
public readonly handRaised$: Observable<Date | null>,
public readonly reaction$: Observable<ReactionOption | null>,
) {
super(
id,
@@ -437,8 +440,18 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel {
participant$: Observable<LocalParticipant | undefined>,
encryptionSystem: EncryptionSystem,
livekitRoom: LivekitRoom,
handRaised$: Observable<Date | null>,
reaction$: Observable<ReactionOption | null>,
) {
super(id, member, participant$, encryptionSystem, livekitRoom);
super(
id,
member,
participant$,
encryptionSystem,
livekitRoom,
handRaised$,
reaction$,
);
}
}
@@ -498,8 +511,18 @@ export class RemoteUserMediaViewModel extends BaseUserMediaViewModel {
participant$: Observable<RemoteParticipant | undefined>,
encryptionSystem: EncryptionSystem,
livekitRoom: LivekitRoom,
handRaised$: Observable<Date | null>,
reaction$: Observable<ReactionOption | null>,
) {
super(id, member, participant$, encryptionSystem, livekitRoom);
super(
id,
member,
participant$,
encryptionSystem,
livekitRoom,
handRaised$,
reaction$,
);
// Sync the local volume with LiveKit
combineLatest([

View File

@@ -15,7 +15,8 @@ import { type MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSess
import { GridTile } from "./GridTile";
import { mockRtcMembership, withRemoteMedia } from "../utils/test";
import { GridTileViewModel } from "../state/TileViewModel";
import { ReactionsProvider } from "../useReactions";
import { ReactionsSenderProvider } from "../reactions/useReactionsSender";
import type { CallViewModel } from "../state/CallViewModel";
global.IntersectionObserver = class MockIntersectionObserver {
public observe(): void {}
@@ -44,14 +45,19 @@ test("GridTile is accessible", async () => {
off: () => {},
client: {
getUserId: () => null,
getDeviceId: () => null,
on: () => {},
off: () => {},
},
},
memberships: [],
} as unknown as MatrixRTCSession;
const cVm = {
reactions$: of({}),
handsRaised$: of({}),
} as Partial<CallViewModel> as CallViewModel;
const { container } = render(
<ReactionsProvider rtcSession={fakeRtcSession}>
<ReactionsSenderProvider vm={cVm} rtcSession={fakeRtcSession}>
<GridTile
vm={new GridTileViewModel(of(vm))}
onOpenProfile={() => {}}
@@ -59,7 +65,7 @@ test("GridTile is accessible", async () => {
targetHeight={200}
showSpeakingIndicators
/>
</ReactionsProvider>,
</ReactionsSenderProvider>,
);
expect(await axe(container)).toHaveNoViolations();
// Name should be visible

View File

@@ -34,7 +34,7 @@ import {
ToggleMenuItem,
Menu,
} from "@vector-im/compound-web";
import { useObservableEagerState } from "observable-hooks";
import { useObservableEagerState, useObservableState } from "observable-hooks";
import styles from "./GridTile.module.css";
import {
@@ -48,8 +48,7 @@ import { MediaView } from "./MediaView";
import { useLatest } from "../useLatest";
import { type GridTileViewModel } from "../state/TileViewModel";
import { useMergedRefs } from "../useMergedRefs";
import { useReactions } from "../useReactions";
import { type ReactionOption } from "../reactions";
import { useReactionsSender } from "../reactions/useReactionsSender";
interface TileProps {
className?: string;
@@ -82,6 +81,7 @@ const UserMediaTile = forwardRef<HTMLDivElement, UserMediaTileProps>(
},
ref,
) => {
const { toggleRaisedHand } = useReactionsSender();
const { t } = useTranslation();
const video = useObservableEagerState(vm.video$);
const unencryptedWarning = useObservableEagerState(vm.unencryptedWarning$);
@@ -97,7 +97,8 @@ const UserMediaTile = forwardRef<HTMLDivElement, UserMediaTileProps>(
},
[vm],
);
const { raisedHands, toggleRaisedHand, reactions } = useReactions();
const handRaised = useObservableState(vm.handRaised$);
const reaction = useObservableState(vm.reaction$);
const AudioIcon = locallyMuted
? VolumeOffSolidIcon
@@ -124,9 +125,6 @@ const UserMediaTile = forwardRef<HTMLDivElement, UserMediaTileProps>(
</>
);
const handRaised: Date | undefined = raisedHands[vm.member?.userId ?? ""];
const currentReaction: ReactionOption | undefined =
reactions[vm.member?.userId ?? ""];
const raisedHandOnClick = vm.local
? (): void => void toggleRaisedHand()
: undefined;
@@ -144,7 +142,7 @@ const UserMediaTile = forwardRef<HTMLDivElement, UserMediaTileProps>(
videoFit={cropVideo ? "cover" : "contain"}
className={classNames(className, styles.tile, {
[styles.speaking]: showSpeaking,
[styles.handRaised]: !showSpeaking && !!handRaised,
[styles.handRaised]: !showSpeaking && handRaised,
})}
nameTagLeadingIcon={
<AudioIcon
@@ -172,8 +170,8 @@ const UserMediaTile = forwardRef<HTMLDivElement, UserMediaTileProps>(
{menu}
</Menu>
}
raisedHandTime={handRaised}
currentReaction={currentReaction}
raisedHandTime={handRaised ?? undefined}
currentReaction={reaction ?? undefined}
raisedHandOnClick={raisedHandOnClick}
localParticipant={vm.local}
{...props}

View File

@@ -1,173 +0,0 @@
/*
Copyright 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { render } from "@testing-library/react";
import { act, type FC } from "react";
import { describe, expect, test } from "vitest";
import { RoomEvent } from "matrix-js-sdk/src/matrix";
import { useReactions } from "./useReactions";
import {
createHandRaisedReaction,
createRedaction,
MockRoom,
MockRTCSession,
TestReactionsWrapper,
} from "./utils/testReactions";
const memberUserIdAlice = "@alice:example.org";
const memberEventAlice = "$membership-alice:example.org";
const memberUserIdBob = "@bob:example.org";
const memberEventBob = "$membership-bob:example.org";
const membership: Record<string, string> = {
[memberEventAlice]: memberUserIdAlice,
[memberEventBob]: memberUserIdBob,
"$membership-charlie:example.org": "@charlie:example.org",
};
/**
* Test explanation.
* This test suite checks that the useReactions hook appropriately reacts
* to new reactions, redactions and membership changesin the room. There is
* a large amount of test structure used to construct a mock environment.
*/
const TestComponent: FC = () => {
const { raisedHands } = useReactions();
return (
<div>
<ul>
{Object.entries(raisedHands).map(([userId, date]) => (
<li key={userId}>
<span>{userId}</span>
<time>{date.getTime()}</time>
</li>
))}
</ul>
</div>
);
};
describe("useReactions", () => {
test("starts with an empty list", () => {
const rtcSession = new MockRTCSession(
new MockRoom(memberUserIdAlice),
membership,
);
const { queryByRole } = render(
<TestReactionsWrapper rtcSession={rtcSession}>
<TestComponent />
</TestReactionsWrapper>,
);
expect(queryByRole("list")?.children).to.have.lengthOf(0);
});
test("handles incoming raised hand", async () => {
const room = new MockRoom(memberUserIdAlice);
const rtcSession = new MockRTCSession(room, membership);
const { queryByRole } = render(
<TestReactionsWrapper rtcSession={rtcSession}>
<TestComponent />
</TestReactionsWrapper>,
);
await act(() => room.testSendHandRaise(memberEventAlice, membership));
expect(queryByRole("list")?.children).to.have.lengthOf(1);
await act(() => room.testSendHandRaise(memberEventBob, membership));
expect(queryByRole("list")?.children).to.have.lengthOf(2);
});
test("handles incoming unraised hand", async () => {
const room = new MockRoom(memberUserIdAlice);
const rtcSession = new MockRTCSession(room, membership);
const { queryByRole } = render(
<TestReactionsWrapper rtcSession={rtcSession}>
<TestComponent />
</TestReactionsWrapper>,
);
const reactionEventId = await act(() =>
room.testSendHandRaise(memberEventAlice, membership),
);
expect(queryByRole("list")?.children).to.have.lengthOf(1);
await act(() =>
room.emit(
RoomEvent.Redaction,
createRedaction(memberUserIdAlice, reactionEventId),
room,
undefined,
),
);
expect(queryByRole("list")?.children).to.have.lengthOf(0);
});
test("handles loading prior raised hand events", () => {
const room = new MockRoom(memberUserIdAlice, [
createHandRaisedReaction(memberEventAlice, membership),
]);
const rtcSession = new MockRTCSession(room, membership);
const { queryByRole } = render(
<TestReactionsWrapper rtcSession={rtcSession}>
<TestComponent />
</TestReactionsWrapper>,
);
expect(queryByRole("list")?.children).to.have.lengthOf(1);
});
// If the membership event changes for a user, we want to remove
// the raised hand event.
test("will remove reaction when a member leaves the call", () => {
const room = new MockRoom(memberUserIdAlice, [
createHandRaisedReaction(memberEventAlice, membership),
]);
const rtcSession = new MockRTCSession(room, membership);
const { queryByRole } = render(
<TestReactionsWrapper rtcSession={rtcSession}>
<TestComponent />
</TestReactionsWrapper>,
);
expect(queryByRole("list")?.children).to.have.lengthOf(1);
act(() => rtcSession.testRemoveMember(memberUserIdAlice));
expect(queryByRole("list")?.children).to.have.lengthOf(0);
});
test("will remove reaction when a member joins via a new event", () => {
const room = new MockRoom(memberUserIdAlice, [
createHandRaisedReaction(memberEventAlice, membership),
]);
const rtcSession = new MockRTCSession(room, membership);
const { queryByRole } = render(
<TestReactionsWrapper rtcSession={rtcSession}>
<TestComponent />
</TestReactionsWrapper>,
);
expect(queryByRole("list")?.children).to.have.lengthOf(1);
// Simulate leaving and rejoining
act(() => {
rtcSession.testRemoveMember(memberUserIdAlice);
rtcSession.testAddMember(memberUserIdAlice);
});
expect(queryByRole("list")?.children).to.have.lengthOf(0);
});
test("ignores invalid sender for historic event", () => {
const room = new MockRoom(memberUserIdAlice, [
createHandRaisedReaction(memberEventAlice, memberUserIdBob),
]);
const rtcSession = new MockRTCSession(room, membership);
const { queryByRole } = render(
<TestReactionsWrapper rtcSession={rtcSession}>
<TestComponent />
</TestReactionsWrapper>,
);
expect(queryByRole("list")?.children).to.have.lengthOf(0);
});
test("ignores invalid sender for new event", async () => {
const room = new MockRoom(memberUserIdAlice);
const rtcSession = new MockRTCSession(room, membership);
const { queryByRole } = render(
<TestReactionsWrapper rtcSession={rtcSession}>
<TestComponent />
</TestReactionsWrapper>,
);
await act(() => room.testSendHandRaise(memberEventAlice, memberUserIdBob));
expect(queryByRole("list")?.children).to.have.lengthOf(0);
});
});

View File

@@ -1,405 +0,0 @@
/*
Copyright 2024 Milton Moura <miltonmoura@gmail.com>
SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import {
EventType,
type MatrixEvent,
RelationType,
RoomEvent as MatrixRoomEvent,
MatrixEventEvent,
} from "matrix-js-sdk/src/matrix";
import { type ReactionEventContent } from "matrix-js-sdk/src/types";
import {
createContext,
useContext,
useState,
type ReactNode,
useCallback,
useEffect,
useMemo,
} from "react";
import { type MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession";
import { logger } from "matrix-js-sdk/src/logger";
import { useMatrixRTCSessionMemberships } from "./useMatrixRTCSessionMemberships";
import { useClientState } from "./ClientContext";
import {
type ECallReactionEventContent,
ElementCallReactionEventType,
GenericReaction,
type ReactionOption,
ReactionSet,
} from "./reactions";
import { useLatest } from "./useLatest";
interface ReactionsContextType {
raisedHands: Record<string, Date>;
supportsReactions: boolean;
reactions: Record<string, ReactionOption>;
toggleRaisedHand: () => Promise<void>;
sendReaction: (reaction: ReactionOption) => Promise<void>;
}
const ReactionsContext = createContext<ReactionsContextType | undefined>(
undefined,
);
interface RaisedHandInfo {
/**
* Call membership event that was reacted to.
*/
membershipEventId: string;
/**
* Event ID of the reaction itself.
*/
reactionEventId: string;
/**
* The time when the reaction was raised.
*/
time: Date;
}
const REACTION_ACTIVE_TIME_MS = 3000;
export const useReactions = (): ReactionsContextType => {
const context = useContext(ReactionsContext);
if (!context) {
throw new Error("useReactions must be used within a ReactionsProvider");
}
return context;
};
/**
* Provider that handles raised hand reactions for a given `rtcSession`.
*/
export const ReactionsProvider = ({
children,
rtcSession,
}: {
children: ReactNode;
rtcSession: MatrixRTCSession;
}): JSX.Element => {
const [raisedHands, setRaisedHands] = useState<
Record<string, RaisedHandInfo>
>({});
const memberships = useMatrixRTCSessionMemberships(rtcSession);
const clientState = useClientState();
const supportsReactions =
clientState?.state === "valid" && clientState.supportedFeatures.reactions;
const room = rtcSession.room;
const myUserId = room.client.getUserId();
const [reactions, setReactions] = useState<Record<string, ReactionOption>>(
{},
);
// Reduce the data down for the consumers.
const resultRaisedHands = useMemo(
() =>
Object.fromEntries(
Object.entries(raisedHands).map(([uid, data]) => [uid, data.time]),
),
[raisedHands],
);
const addRaisedHand = useCallback((userId: string, info: RaisedHandInfo) => {
setRaisedHands((prevRaisedHands) => ({
...prevRaisedHands,
[userId]: info,
}));
}, []);
const removeRaisedHand = useCallback((userId: string) => {
setRaisedHands(
({ [userId]: _removed, ...remainingRaisedHands }) => remainingRaisedHands,
);
}, []);
// This effect will check the state whenever the membership of the session changes.
useEffect(() => {
// Fetches the first reaction for a given event.
const getLastReactionEvent = (
eventId: string,
expectedSender: string,
): MatrixEvent | undefined => {
const relations = room.relations.getChildEventsForEvent(
eventId,
RelationType.Annotation,
EventType.Reaction,
);
const allEvents = relations?.getRelations() ?? [];
return allEvents.find(
(reaction) =>
reaction.event.sender === expectedSender &&
reaction.getType() === EventType.Reaction &&
reaction.getContent()?.["m.relates_to"]?.key === "🖐️",
);
};
// Remove any raised hands for users no longer joined to the call.
for (const userId of Object.keys(raisedHands).filter(
(rhId) => !memberships.find((u) => u.sender == rhId),
)) {
removeRaisedHand(userId);
}
// For each member in the call, check to see if a reaction has
// been raised and adjust.
for (const m of memberships) {
if (!m.sender || !m.eventId) {
continue;
}
if (
raisedHands[m.sender] &&
raisedHands[m.sender].membershipEventId !== m.eventId
) {
// Membership event for sender has changed since the hand
// was raised, reset.
removeRaisedHand(m.sender);
}
const reaction = getLastReactionEvent(m.eventId, m.sender);
if (reaction) {
const eventId = reaction?.getId();
if (!eventId) {
continue;
}
addRaisedHand(m.sender, {
membershipEventId: m.eventId,
reactionEventId: eventId,
time: new Date(reaction.localTimestamp),
});
}
}
// Ignoring raisedHands here because we don't want to trigger each time the raised
// hands set is updated.
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [room, memberships, myUserId, addRaisedHand, removeRaisedHand]);
const latestMemberships = useLatest(memberships);
const latestRaisedHands = useLatest(raisedHands);
const myMembership = useMemo(
() => memberships.find((m) => m.sender === myUserId)?.eventId,
[memberships, myUserId],
);
// This effect handles any *live* reaction/redactions in the room.
useEffect(() => {
const reactionTimeouts = new Set<number>();
const handleReactionEvent = (event: MatrixEvent): void => {
// Decrypted events might come from a different room
if (event.getRoomId() !== room.roomId) return;
// Skip any events that are still sending.
if (event.isSending()) return;
const sender = event.getSender();
const reactionEventId = event.getId();
// Skip any event without a sender or event ID.
if (!sender || !reactionEventId) return;
room.client
.decryptEventIfNeeded(event)
.catch((e) => logger.warn(`Failed to decrypt ${event.getId()}`, e));
if (event.isBeingDecrypted() || event.isDecryptionFailure()) return;
if (event.getType() === ElementCallReactionEventType) {
const content: ECallReactionEventContent = event.getContent();
const membershipEventId = content?.["m.relates_to"]?.event_id;
// Check to see if this reaction was made to a membership event (and the
// sender of the reaction matches the membership)
if (
!latestMemberships.current.some(
(e) => e.eventId === membershipEventId && e.sender === sender,
)
) {
logger.warn(
`Reaction target was not a membership event for ${sender}, ignoring`,
);
return;
}
if (!content.emoji) {
logger.warn(`Reaction had no emoji from ${reactionEventId}`);
return;
}
const segment = new Intl.Segmenter(undefined, {
granularity: "grapheme",
})
.segment(content.emoji)
[Symbol.iterator]();
const emoji = segment.next().value?.segment;
if (!emoji) {
logger.warn(
`Reaction had no emoji from ${reactionEventId} after splitting`,
);
return;
}
// One of our custom reactions
const reaction = {
...GenericReaction,
emoji,
// If we don't find a reaction, we can fallback to the generic sound.
...ReactionSet.find((r) => r.name === content.name),
};
setReactions((reactions) => {
if (reactions[sender]) {
// We've still got a reaction from this user, ignore it to prevent spamming
return reactions;
}
const timeout = window.setTimeout(() => {
// Clear the reaction after some time.
setReactions(({ [sender]: _unused, ...remaining }) => remaining);
reactionTimeouts.delete(timeout);
}, REACTION_ACTIVE_TIME_MS);
reactionTimeouts.add(timeout);
return {
...reactions,
[sender]: reaction,
};
});
} else if (event.getType() === EventType.Reaction) {
const content = event.getContent() as ReactionEventContent;
const membershipEventId = content["m.relates_to"].event_id;
// Check to see if this reaction was made to a membership event (and the
// sender of the reaction matches the membership)
if (
!latestMemberships.current.some(
(e) => e.eventId === membershipEventId && e.sender === sender,
)
) {
logger.warn(
`Reaction target was not a membership event for ${sender}, ignoring`,
);
return;
}
if (content?.["m.relates_to"].key === "🖐️") {
addRaisedHand(sender, {
reactionEventId,
membershipEventId,
time: new Date(event.localTimestamp),
});
}
} else if (event.getType() === EventType.RoomRedaction) {
const targetEvent = event.event.redacts;
const targetUser = Object.entries(latestRaisedHands.current).find(
([_u, r]) => r.reactionEventId === targetEvent,
)?.[0];
if (!targetUser) {
// Reaction target was not for us, ignoring
return;
}
removeRaisedHand(targetUser);
}
};
room.on(MatrixRoomEvent.Timeline, handleReactionEvent);
room.on(MatrixRoomEvent.Redaction, handleReactionEvent);
room.client.on(MatrixEventEvent.Decrypted, handleReactionEvent);
// We listen for a local echo to get the real event ID, as timeline events
// may still be sending.
room.on(MatrixRoomEvent.LocalEchoUpdated, handleReactionEvent);
return (): void => {
room.off(MatrixRoomEvent.Timeline, handleReactionEvent);
room.off(MatrixRoomEvent.Redaction, handleReactionEvent);
room.client.off(MatrixEventEvent.Decrypted, handleReactionEvent);
room.off(MatrixRoomEvent.LocalEchoUpdated, handleReactionEvent);
reactionTimeouts.forEach((t) => clearTimeout(t));
// If we're clearing timeouts, we also clear all reactions.
setReactions({});
};
}, [
room,
addRaisedHand,
removeRaisedHand,
latestMemberships,
latestRaisedHands,
]);
const toggleRaisedHand = useCallback(async () => {
if (!myUserId) {
return;
}
const myReactionId = raisedHands[myUserId]?.reactionEventId;
if (!myReactionId) {
try {
if (!myMembership) {
throw new Error("Cannot find own membership event");
}
const reaction = await room.client.sendEvent(
rtcSession.room.roomId,
EventType.Reaction,
{
"m.relates_to": {
rel_type: RelationType.Annotation,
event_id: myMembership,
key: "🖐️",
},
},
);
logger.debug("Sent raise hand event", reaction.event_id);
} catch (ex) {
logger.error("Failed to send raised hand", ex);
}
} else {
try {
await room.client.redactEvent(rtcSession.room.roomId, myReactionId);
logger.debug("Redacted raise hand event");
} catch (ex) {
logger.error("Failed to redact reaction event", myReactionId, ex);
throw ex;
}
}
}, [myMembership, myUserId, raisedHands, rtcSession, room]);
const sendReaction = useCallback(
async (reaction: ReactionOption) => {
if (!myUserId || reactions[myUserId]) {
// We're still reacting
return;
}
if (!myMembership) {
throw new Error("Cannot find own membership event");
}
await room.client.sendEvent(
rtcSession.room.roomId,
ElementCallReactionEventType,
{
"m.relates_to": {
rel_type: RelationType.Reference,
event_id: myMembership,
},
emoji: reaction.emoji,
name: reaction.name,
},
);
},
[myMembership, reactions, room, myUserId, rtcSession],
);
return (
<ReactionsContext.Provider
value={{
raisedHands: resultRaisedHands,
supportsReactions,
reactions,
toggleRaisedHand,
sendReaction,
}}
>
{children}
</ReactionsContext.Provider>
);
};

View File

@@ -0,0 +1,24 @@
/*
Copyright 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import {
mockRtcMembership,
mockMatrixRoomMember,
mockRemoteParticipant,
mockLocalParticipant,
} from "./test";
export const aliceRtcMember = mockRtcMembership("@alice:example.org", "AAAA");
export const alice = mockMatrixRoomMember(aliceRtcMember);
export const aliceId = `${alice.userId}:${aliceRtcMember.deviceId}`;
export const aliceParticipant = mockRemoteParticipant({ identity: aliceId });
export const localRtcMember = mockRtcMembership("@carol:example.org", "CCCC");
export const local = mockMatrixRoomMember(localRtcMember);
export const localParticipant = mockLocalParticipant({ identity: "" });
export const bobRtcMember = mockRtcMembership("@bob:example.org", "BBBB");

150
src/utils/test-viewmodel.ts Normal file
View File

@@ -0,0 +1,150 @@
/*
Copyright 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { ConnectionState } from "livekit-client";
import { type MatrixClient } from "matrix-js-sdk/src/client";
import { type RoomMember } from "matrix-js-sdk/src/matrix";
import {
type CallMembership,
type MatrixRTCSession,
} from "matrix-js-sdk/src/matrixrtc";
import { BehaviorSubject, of } from "rxjs";
import { vitest } from "vitest";
import { type RelationsContainer } from "matrix-js-sdk/src/models/relations-container";
import EventEmitter from "events";
import { E2eeType } from "../e2ee/e2eeType";
import { CallViewModel } from "../state/CallViewModel";
import { mockLivekitRoom, mockMatrixRoom, MockRTCSession } from "./test";
import {
aliceRtcMember,
aliceParticipant,
localParticipant,
localRtcMember,
} from "./test-fixtures";
import { type RaisedHandInfo, type ReactionInfo } from "../reactions";
export function getBasicRTCSession(
members: RoomMember[],
initialRemoteRtcMemberships: CallMembership[] = [aliceRtcMember],
): {
rtcSession: MockRTCSession;
remoteRtcMemberships$: BehaviorSubject<CallMembership[]>;
} {
const matrixRoomId = "!myRoomId:example.com";
const matrixRoomMembers = new Map(members.map((p) => [p.userId, p]));
const roomEmitter = new EventEmitter();
const clientEmitter = new EventEmitter();
const matrixRoom = mockMatrixRoom({
relations: {
getChildEventsForEvent: vitest.fn(),
} as Partial<RelationsContainer> as RelationsContainer,
client: {
getUserId: () => localRtcMember.sender,
getDeviceId: () => localRtcMember.deviceId,
sendEvent: vitest.fn().mockResolvedValue({ event_id: "$fake:event" }),
redactEvent: vitest.fn().mockResolvedValue({ event_id: "$fake:event" }),
decryptEventIfNeeded: vitest.fn().mockResolvedValue(undefined),
on: vitest
.fn()
.mockImplementation(
(eventName: string, fn: (...args: unknown[]) => void) => {
clientEmitter.on(eventName, fn);
},
),
emit: (eventName: string, ...args: unknown[]) =>
clientEmitter.emit(eventName, ...args),
off: vitest
.fn()
.mockImplementation(
(eventName: string, fn: (...args: unknown[]) => void) => {
clientEmitter.off(eventName, fn);
},
),
} as Partial<MatrixClient> as MatrixClient,
getMember: (userId) => matrixRoomMembers.get(userId) ?? null,
roomId: matrixRoomId,
on: vitest
.fn()
.mockImplementation(
(eventName: string, fn: (...args: unknown[]) => void) => {
roomEmitter.on(eventName, fn);
},
),
emit: (eventName: string, ...args: unknown[]) =>
roomEmitter.emit(eventName, ...args),
off: vitest
.fn()
.mockImplementation(
(eventName: string, fn: (...args: unknown[]) => void) => {
roomEmitter.off(eventName, fn);
},
),
});
const remoteRtcMemberships$ = new BehaviorSubject<CallMembership[]>(
initialRemoteRtcMemberships,
);
const rtcSession = new MockRTCSession(
matrixRoom,
localRtcMember,
).withMemberships(remoteRtcMemberships$);
return {
rtcSession,
remoteRtcMemberships$,
};
}
/**
* Construct a basic CallViewModel to test components that make use of it.
* @param members
* @param initialRemoteRtcMemberships
* @returns
*/
export function getBasicCallViewModelEnvironment(
members: RoomMember[],
initialRemoteRtcMemberships: CallMembership[] = [aliceRtcMember],
): {
vm: CallViewModel;
remoteRtcMemberships$: BehaviorSubject<CallMembership[]>;
rtcSession: MockRTCSession;
handRaisedSubject$: BehaviorSubject<Record<string, RaisedHandInfo>>;
reactionsSubject$: BehaviorSubject<Record<string, ReactionInfo>>;
} {
const { rtcSession, remoteRtcMemberships$ } = getBasicRTCSession(
members,
initialRemoteRtcMemberships,
);
const handRaisedSubject$ = new BehaviorSubject({});
const reactionsSubject$ = new BehaviorSubject({});
const remoteParticipants$ = of([aliceParticipant]);
const liveKitRoom = mockLivekitRoom(
{ localParticipant },
{ remoteParticipants$ },
);
const vm = new CallViewModel(
rtcSession as unknown as MatrixRTCSession,
liveKitRoom,
{
kind: E2eeType.PER_PARTICIPANT,
},
of(ConnectionState.Connected),
handRaisedSubject$,
reactionsSubject$,
);
return {
vm,
remoteRtcMemberships$,
rtcSession,
handRaisedSubject$: handRaisedSubject$,
reactionsSubject$: reactionsSubject$,
};
}

View File

@@ -28,6 +28,7 @@ import {
type RemoteTrackPublication,
type Room as LivekitRoom,
} from "livekit-client";
import { randomUUID } from "crypto";
import {
LocalUserMediaViewModel,
@@ -132,6 +133,7 @@ export function mockRtcMembership(
};
const event = new MatrixEvent({
sender: typeof user === "string" ? user : user.userId,
event_id: `$-ev-${randomUUID()}:example.org`,
});
return new CallMembership(event, data);
}
@@ -203,6 +205,8 @@ export async function withLocalMedia(
kind: E2eeType.PER_PARTICIPANT,
},
mockLivekitRoom({ localParticipant }),
of(null),
of(null),
);
try {
await continuation(vm);
@@ -239,6 +243,8 @@ export async function withRemoteMedia(
kind: E2eeType.PER_PARTICIPANT,
},
mockLivekitRoom({}, { remoteParticipants$: of([remoteParticipant]) }),
of(null),
of(null),
);
try {
await continuation(vm);

View File

@@ -1,214 +0,0 @@
/*
Copyright 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { type PropsWithChildren, type ReactNode } from "react";
import { randomUUID } from "crypto";
import EventEmitter from "events";
import { type MatrixClient } from "matrix-js-sdk/src/client";
import { EventType, RoomEvent, RelationType } from "matrix-js-sdk/src/matrix";
import {
MatrixEvent,
EventTimeline,
EventTimelineSet,
type Room,
} from "matrix-js-sdk/src/matrix";
import {
type MatrixRTCSession,
MatrixRTCSessionEvent,
} from "matrix-js-sdk/src/matrixrtc";
import { ReactionsProvider } from "../useReactions";
import {
type ECallReactionEventContent,
ElementCallReactionEventType,
type ReactionOption,
} from "../reactions";
export const TestReactionsWrapper = ({
rtcSession,
children,
}: PropsWithChildren<{
rtcSession: MockRTCSession | MatrixRTCSession;
}>): ReactNode => {
return (
<ReactionsProvider rtcSession={rtcSession as unknown as MatrixRTCSession}>
{children}
</ReactionsProvider>
);
};
export class MockRTCSession extends EventEmitter {
public memberships: {
sender: string;
eventId: string;
createdTs: () => Date;
}[];
public constructor(
public readonly room: MockRoom,
membership: Record<string, string>,
) {
super();
this.memberships = Object.entries(membership).map(([eventId, sender]) => ({
sender,
eventId,
createdTs: (): Date => new Date(),
}));
}
public testRemoveMember(userId: string): void {
this.memberships = this.memberships.filter((u) => u.sender !== userId);
this.emit(MatrixRTCSessionEvent.MembershipsChanged);
}
public testAddMember(sender: string): void {
this.memberships.push({
sender,
eventId: `!fake-${randomUUID()}:event`,
createdTs: (): Date => new Date(),
});
this.emit(MatrixRTCSessionEvent.MembershipsChanged);
}
}
export function createHandRaisedReaction(
parentMemberEvent: string,
membershipOrOverridenSender: Record<string, string> | string,
): MatrixEvent {
return new MatrixEvent({
sender:
typeof membershipOrOverridenSender === "string"
? membershipOrOverridenSender
: membershipOrOverridenSender[parentMemberEvent],
type: EventType.Reaction,
origin_server_ts: new Date().getTime(),
content: {
"m.relates_to": {
key: "🖐️",
event_id: parentMemberEvent,
},
},
event_id: randomUUID(),
});
}
export function createRedaction(
sender: string,
reactionEventId: string,
): MatrixEvent {
return new MatrixEvent({
sender,
type: EventType.RoomRedaction,
origin_server_ts: new Date().getTime(),
redacts: reactionEventId,
content: {},
event_id: randomUUID(),
});
}
export class MockRoom extends EventEmitter {
public readonly testSentEvents: Parameters<MatrixClient["sendEvent"]>[] = [];
public readonly testRedactedEvents: Parameters<
MatrixClient["redactEvent"]
>[] = [];
public constructor(
private readonly ownUserId: string,
private readonly existingRelations: MatrixEvent[] = [],
) {
super();
}
public get client(): MatrixClient {
return {
getUserId: (): string => this.ownUserId,
sendEvent: async (
...props: Parameters<MatrixClient["sendEvent"]>
): ReturnType<MatrixClient["sendEvent"]> => {
this.testSentEvents.push(props);
return Promise.resolve({ event_id: randomUUID() });
},
redactEvent: async (
...props: Parameters<MatrixClient["redactEvent"]>
): ReturnType<MatrixClient["redactEvent"]> => {
this.testRedactedEvents.push(props);
return Promise.resolve({ event_id: randomUUID() });
},
decryptEventIfNeeded: async () => {},
on() {
return this;
},
off() {
return this;
},
} as unknown as MatrixClient;
}
public get relations(): Room["relations"] {
return {
getChildEventsForEvent: (membershipEventId: string) => ({
getRelations: (): MatrixEvent[] => {
return this.existingRelations.filter(
(r) =>
r.getContent()["m.relates_to"]?.event_id === membershipEventId,
);
},
}),
} as unknown as Room["relations"];
}
public testSendHandRaise(
parentMemberEvent: string,
membershipOrOverridenSender: Record<string, string> | string,
): string {
const evt = createHandRaisedReaction(
parentMemberEvent,
membershipOrOverridenSender,
);
this.emit(RoomEvent.Timeline, evt, this, undefined, false, {
timeline: new EventTimeline(new EventTimelineSet(undefined)),
});
return evt.getId()!;
}
public testSendReaction(
parentMemberEvent: string,
reaction: ReactionOption,
membershipOrOverridenSender: Record<string, string> | string,
): string {
const evt = new MatrixEvent({
sender:
typeof membershipOrOverridenSender === "string"
? membershipOrOverridenSender
: membershipOrOverridenSender[parentMemberEvent],
type: ElementCallReactionEventType,
origin_server_ts: new Date().getTime(),
content: {
"m.relates_to": {
rel_type: RelationType.Reference,
event_id: parentMemberEvent,
},
emoji: reaction.emoji,
name: reaction.name,
} satisfies ECallReactionEventContent,
event_id: randomUUID(),
});
this.emit(RoomEvent.Timeline, evt, this, undefined, false, {
timeline: new EventTimeline(new EventTimelineSet(undefined)),
});
return evt.getId()!;
}
public getMember(): void {
return;
}
public testGetAsMatrixRoom(): Room {
return this as unknown as Room;
}
}

View File

@@ -16,7 +16,12 @@ export default defineConfig((configEnv) =>
coverage: {
reporter: ["html", "json"],
include: ["src/"],
exclude: ["src/**/*.{d,test}.{ts,tsx}", "src/utils/test.ts"],
exclude: [
"src/**/*.{d,test}.{ts,tsx}",
"src/utils/test.ts",
"src/utils/test-viewmodel.ts",
"src/utils/test-fixtures.ts",
],
},
},
}),