Hand raise feature (#2542)

* Initial support for Hand Raise feature

Signed-off-by: Milton Moura <miltonmoura@gmail.com>

* Refactored to use reaction and redaction events

Signed-off-by: Milton Moura <miltonmoura@gmail.com>

* Replacing button svg with raised hand emoji

Signed-off-by: Milton Moura <miltonmoura@gmail.com>

* SpotlightTile should not duplicate the raised hand

Signed-off-by: Milton Moura <miltonmoura@gmail.com>

* Update src/room/useRaisedHands.tsx

Element Call recently changed to AGPL-3.0

* Use relations to load existing reactions when joining the call

Signed-off-by: Milton Moura <miltonmoura@gmail.com>

* Links to sha commit of matrix-js-sdk that exposes the call membership event id and refactors some async code

Signed-off-by: Milton Moura <miltonmoura@gmail.com>

* Removing RaiseHand.svg

* Check for reaction & redaction capabilities in widget mode

Signed-off-by: Milton Moura <miltonmoura@gmail.com>

* Fix failing GridTile test

Signed-off-by: Milton Moura <miltonmoura@gmail.com>

* Center align hand raise.

* Add support for displaying the duration of a raised hand.

* Add a sound for when a hand is raised.

* Refactor raised hand indicator and add tests.

* lint

* Refactor into own files.

* Redact the right thing.

* Tidy up useEffect

* Lint tests

* Remove extra layer

* Add better sound. (woosh)

* Add a small mode for spotlight

* Fix timestamp calculation on relaod.

* Fix call border resizing video

* lint

* Fix and update tests

* Allow timer to be configurable.

* Add preferences tab for choosing to enable timer.

* Drop border from raised hand icon

* Handle cases when a new member event happens.

* Prevent infinite loop

* Major refactor to support various state problems.

* Tidy up and finish test rewrites

* Add some explanation comments.

* Even more comments.

* Use proper duration formatter

* Remove rerender

* Fix redactions not working because they pick up events in transit.

* More tidying

* Use deferred value

* linting

* Add tests for cases where we got a reaction from someone else.

* Be even less brittle.

* Transpose border to GridTile.

* lint

---------

Signed-off-by: Milton Moura <miltonmoura@gmail.com>
Co-authored-by: fkwp <fkwp@users.noreply.github.com>
Co-authored-by: Half-Shot <will@half-shot.uk>
Co-authored-by: Will Hunt <github@half-shot.uk>
This commit is contained in:
Milton Moura
2024-11-04 08:54:13 -01:00
committed by GitHub
parent f2ed07c258
commit 1897210a60
24 changed files with 1149 additions and 30 deletions

View File

@@ -24,6 +24,7 @@
"@babel/preset-env": "^7.22.20",
"@babel/preset-react": "^7.22.15",
"@babel/preset-typescript": "^7.23.0",
"@formatjs/intl-durationformat": "^0.6.1",
"@livekit/components-core": "^0.11.0",
"@livekit/components-react": "^2.0.0",
"@opentelemetry/api": "^1.4.0",

View File

@@ -53,7 +53,9 @@
"next": "Next",
"options": "Options",
"password": "Password",
"preferences": "Preferences",
"profile": "Profile",
"raise_hand": "Raise hand",
"settings": "Settings",
"unencrypted": "Not encrypted",
"username": "Username",
@@ -145,6 +147,10 @@
"feedback_tab_title": "Feedback",
"more_tab_title": "More",
"opt_in_description": "<0></0><1></1>You may withdraw consent by unchecking this box. If you are currently in a call, this setting will take effect at the end of the call.",
"preferences_tab_body": "Here you can configure extra options for an improved experience",
"preferences_tab_h4": "Preferences",
"preferences_tab_show_hand_raised_timer_description": "Show a timer when a participant raises their hand",
"preferences_tab_show_hand_raised_timer_label": "Show hand raise duration",
"speaker_device_selection_label": "Speaker"
},
"star_rating_input_label_one": "{{count}} stars",

View File

@@ -25,6 +25,7 @@ import { logger } from "matrix-js-sdk/src/logger";
import { useTranslation } from "react-i18next";
import { ISyncStateData, SyncState } from "matrix-js-sdk/src/sync";
import { MatrixError } from "matrix-js-sdk/src/matrix";
import { WidgetApi } from "matrix-widget-api";
import { ErrorView } from "./FullScreenView";
import { fallbackICEServerAllowed, initClient } from "./utils/matrix";
@@ -52,6 +53,9 @@ export type ValidClientState = {
// 'Disconnected' rather than 'connected' because it tracks specifically
// whether the client is supposed to be connected but is not
disconnected: boolean;
supportedFeatures: {
reactions: boolean;
};
setClient: (params?: SetClientParams) => void;
};
@@ -188,11 +192,11 @@ export const ClientProvider: FC<Props> = ({ children }) => {
saveSession({ ...session, passwordlessUser: false });
setInitClientState({
client: initClientState.client,
...initClientState,
passwordlessUser: false,
});
},
[initClientState?.client],
[initClientState],
);
const setClient = useCallback(
@@ -206,6 +210,7 @@ export const ClientProvider: FC<Props> = ({ children }) => {
if (clientParams) {
saveSession(clientParams.session);
setInitClientState({
widgetApi: null,
client: clientParams.client,
passwordlessUser: clientParams.session.passwordlessUser,
});
@@ -254,6 +259,7 @@ export const ClientProvider: FC<Props> = ({ children }) => {
);
const [isDisconnected, setIsDisconnected] = useState(false);
const [supportsReactions, setSupportsReactions] = useState(false);
const state: ClientState | undefined = useMemo(() => {
if (alreadyOpenedErr) {
@@ -277,6 +283,9 @@ export const ClientProvider: FC<Props> = ({ children }) => {
authenticated,
setClient,
disconnected: isDisconnected,
supportedFeatures: {
reactions: supportsReactions,
},
};
}, [
alreadyOpenedErr,
@@ -285,6 +294,7 @@ export const ClientProvider: FC<Props> = ({ children }) => {
logout,
setClient,
isDisconnected,
supportsReactions,
]);
const onSync = useCallback(
@@ -309,6 +319,30 @@ export const ClientProvider: FC<Props> = ({ children }) => {
initClientState.client.on(ClientEvent.Sync, onSync);
}
if (initClientState.widgetApi) {
const reactSend = initClientState.widgetApi.hasCapability(
"org.matrix.msc2762.send.event:m.reaction",
);
const redactSend = initClientState.widgetApi.hasCapability(
"org.matrix.msc2762.send.event:m.room.redaction",
);
const reactRcv = initClientState.widgetApi.hasCapability(
"org.matrix.msc2762.receive.event:m.reaction",
);
const redactRcv = initClientState.widgetApi.hasCapability(
"org.matrix.msc2762.receive.event:m.room.redaction",
);
if (!reactSend || !reactRcv || !redactSend || !redactRcv) {
logger.warn("Widget does not support reactions");
setSupportsReactions(false);
} else {
setSupportsReactions(true);
}
} else {
setSupportsReactions(true);
}
return (): void => {
if (initClientState.client) {
initClientState.client.removeListener(ClientEvent.Sync, onSync);
@@ -326,6 +360,7 @@ export const ClientProvider: FC<Props> = ({ children }) => {
};
type InitResult = {
widgetApi: WidgetApi | null;
client: MatrixClient;
passwordlessUser: boolean;
};
@@ -336,6 +371,7 @@ async function loadClient(): Promise<InitResult | null> {
logger.log("Using a matryoshka client");
const client = await widget.client;
return {
widgetApi: widget.api,
client,
passwordlessUser: false,
};
@@ -364,6 +400,7 @@ async function loadClient(): Promise<InitResult | null> {
try {
const client = await initClient(initClientParams, true);
return {
widgetApi: null,
client,
passwordlessUser,
};

View File

@@ -12,7 +12,7 @@ Please see LICENSE in the repository root for full details.
.dialog {
box-sizing: border-box;
inline-size: 520px;
inline-size: 580px;
max-inline-size: 90%;
max-block-size: 600px;
}

View File

@@ -0,0 +1,133 @@
/*
Copyright 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { Button as CpdButton, Tooltip } from "@vector-im/compound-web";
import {
ComponentPropsWithoutRef,
FC,
ReactNode,
useCallback,
useState,
} from "react";
import { useTranslation } from "react-i18next";
import { logger } from "matrix-js-sdk/src/logger";
import { EventType, RelationType } from "matrix-js-sdk/src/matrix";
import { MatrixClient } from "matrix-js-sdk/src/client";
import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession";
import { useReactions } from "../useReactions";
import { useMatrixRTCSessionMemberships } from "../useMatrixRTCSessionMemberships";
interface InnerButtonProps extends ComponentPropsWithoutRef<"button"> {
raised: boolean;
}
const InnerButton: FC<InnerButtonProps> = ({ raised, ...props }) => {
const { t } = useTranslation();
return (
<Tooltip label={t("common.raise_hand")}>
<CpdButton
kind={raised ? "primary" : "secondary"}
{...props}
style={{ paddingLeft: 8, paddingRight: 8 }}
>
<p
role="img"
aria-hidden
style={{
width: "30px",
height: "0px",
display: "inline-block",
fontSize: "22px",
}}
>
</p>
</CpdButton>
</Tooltip>
);
};
interface RaisedHandToggleButtonProps {
rtcSession: MatrixRTCSession;
client: MatrixClient;
}
export function RaiseHandToggleButton({
client,
rtcSession,
}: RaisedHandToggleButtonProps): ReactNode {
const { raisedHands, myReactionId } = useReactions();
const [busy, setBusy] = useState(false);
const userId = client.getUserId()!;
const isHandRaised = !!raisedHands[userId];
const memberships = useMatrixRTCSessionMemberships(rtcSession);
const toggleRaisedHand = useCallback(() => {
const raiseHand = async (): Promise<void> => {
if (isHandRaised) {
if (!myReactionId) {
logger.warn(`Hand raised but no reaction event to redact!`);
return;
}
try {
setBusy(true);
await client.redactEvent(rtcSession.room.roomId, myReactionId);
logger.debug("Redacted raise hand event");
} catch (ex) {
logger.error("Failed to redact reaction event", myReactionId, ex);
} finally {
setBusy(false);
}
} else {
const myMembership = memberships.find((m) => m.sender === userId);
if (!myMembership?.eventId) {
logger.error("Cannot find own membership event");
return;
}
const parentEventId = myMembership.eventId;
try {
setBusy(true);
const reaction = await client.sendEvent(
rtcSession.room.roomId,
EventType.Reaction,
{
"m.relates_to": {
rel_type: RelationType.Annotation,
event_id: parentEventId,
key: "🖐️",
},
},
);
logger.debug("Sent raise hand event", reaction.event_id);
} catch (ex) {
logger.error("Failed to send reaction event", ex);
} finally {
setBusy(false);
}
}
};
void raiseHand();
}, [
client,
isHandRaised,
memberships,
myReactionId,
rtcSession.room.roomId,
userId,
]);
return (
<InnerButton
disabled={busy}
onClick={toggleRaisedHand}
raised={isHandRaised}
/>
);
}

View File

@@ -7,3 +7,4 @@ Please see LICENSE in the repository root for full details.
export * from "./Button";
export * from "./LinkButton";
export * from "./RaisedHandToggleButton";

View File

@@ -0,0 +1,52 @@
.raisedHandWidget {
display: flex;
background-color: var(--cpd-color-bg-subtle-primary);
border-radius: var(--cpd-radius-pill-effect);
color: var(--cpd-color-icon-secondary);
}
.raisedHandWidget > p {
padding: none;
margin-top: auto;
margin-bottom: auto;
width: 4em;
}
.raisedHandWidgetLarge > p {
padding: var(--cpd-space-2x);
}
.raisedHandLarge {
margin: var(--cpd-space-2x);
padding: var(--cpd-space-2x);
padding-block: var(--cpd-space-2x);
}
.raisedHand {
margin: var(--cpd-space-1x);
color: var(--cpd-color-icon-secondary);
background-color: var(--cpd-color-icon-secondary);
display: flex;
align-items: center;
border-radius: var(--cpd-radius-pill-effect);
user-select: none;
overflow: hidden;
box-shadow: var(--small-drop-shadow);
box-sizing: border-box;
max-inline-size: 100%;
max-width: fit-content;
}
.raisedHand > span {
width: var(--cpd-space-6x);
height: var(--cpd-space-6x);
display: inline-block;
text-align: center;
font-size: 16px;
}
.raisedHandLarge > span {
width: var(--cpd-space-8x);
height: var(--cpd-space-8x);
font-size: 22px;
}

View File

@@ -0,0 +1,43 @@
/*
Copyright 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { describe, expect, test } from "vitest";
import { render, configure } from "@testing-library/react";
import { RaisedHandIndicator } from "./RaisedHandIndicator";
configure({
defaultHidden: true,
});
describe("RaisedHandIndicator", () => {
test("renders nothing when no hand has been raised", () => {
const { container } = render(<RaisedHandIndicator />);
expect(container.firstChild).toBeNull();
});
test("renders an indicator when a hand has been raised", () => {
const dateTime = new Date();
const { container } = render(
<RaisedHandIndicator raisedHandTime={dateTime} showTimer />,
);
expect(container.firstChild).toMatchSnapshot();
});
test("renders an indicator when a hand has been raised with the expected time", () => {
const dateTime = new Date(new Date().getTime() - 60000);
const { container } = render(
<RaisedHandIndicator raisedHandTime={dateTime} showTimer />,
);
expect(container.firstChild).toMatchSnapshot();
});
test("renders a smaller indicator when minature is specified", () => {
const dateTime = new Date();
const { container } = render(
<RaisedHandIndicator raisedHandTime={dateTime} minature showTimer />,
);
expect(container.firstChild).toMatchSnapshot();
});
});

View File

@@ -0,0 +1,77 @@
/*
Copyright 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { ReactNode, useEffect, useState } from "react";
import classNames from "classnames";
import "@formatjs/intl-durationformat/polyfill";
import { DurationFormat } from "@formatjs/intl-durationformat";
import styles from "./RaisedHandIndicator.module.css";
const durationFormatter = new DurationFormat(undefined, {
minutesDisplay: "always",
secondsDisplay: "always",
hoursDisplay: "auto",
style: "digital",
});
export function RaisedHandIndicator({
raisedHandTime,
minature,
showTimer,
}: {
raisedHandTime?: Date;
minature?: boolean;
showTimer?: boolean;
}): ReactNode {
const [raisedHandDuration, setRaisedHandDuration] = useState("");
// This effect creates a simple timer effect.
useEffect(() => {
if (!raisedHandTime || !showTimer) {
return;
}
const calculateTime = (): void => {
const totalSeconds = Math.ceil(
(new Date().getTime() - raisedHandTime.getTime()) / 1000,
);
setRaisedHandDuration(
durationFormatter.format({
seconds: totalSeconds % 60,
minutes: Math.floor(totalSeconds / 60),
}),
);
};
calculateTime();
const to = setInterval(calculateTime, 1000);
return (): void => clearInterval(to);
}, [setRaisedHandDuration, raisedHandTime, showTimer]);
if (raisedHandTime) {
return (
<div
className={classNames(styles.raisedHandWidget, {
[styles.raisedHandWidgetLarge]: !minature,
})}
>
<div
className={classNames(styles.raisedHand, {
[styles.raisedHandLarge]: !minature,
})}
>
<span role="img" aria-label="raised hand">
</span>
</div>
{showTimer && <p>{raisedHandDuration}</p>}
</div>
);
}
return null;
}

View File

@@ -0,0 +1,61 @@
// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
exports[`RaisedHandIndicator > renders a smaller indicator when minature is specified 1`] = `
<div
class="raisedHandWidget"
>
<div
class="raisedHand"
>
<span
aria-label="raised hand"
role="img"
>
</span>
</div>
<p>
00:01
</p>
</div>
`;
exports[`RaisedHandIndicator > renders an indicator when a hand has been raised 1`] = `
<div
class="raisedHandWidget raisedHandWidgetLarge"
>
<div
class="raisedHand raisedHandLarge"
>
<span
aria-label="raised hand"
role="img"
>
</span>
</div>
<p>
00:01
</p>
</div>
`;
exports[`RaisedHandIndicator > renders an indicator when a hand has been raised with the expected time 1`] = `
<div
class="raisedHandWidget raisedHandWidgetLarge"
>
<div
class="raisedHand raisedHandLarge"
>
<span
aria-label="raised hand"
role="img"
>
</span>
</div>
<p>
01:01
</p>
</div>
`;

View File

@@ -19,6 +19,7 @@ import {
TouchEvent,
forwardRef,
useCallback,
useDeferredValue,
useEffect,
useMemo,
useRef,
@@ -40,6 +41,7 @@ import {
VideoButton,
ShareScreenButton,
SettingsButton,
RaiseHandToggleButton,
SwitchCameraButton,
} from "../button";
import { Header, LeftNav, RightNav, RoomHeaderInfo } from "../Header";
@@ -79,6 +81,9 @@ import { makeOneOnOneLayout } from "../grid/OneOnOneLayout";
import { makeSpotlightExpandedLayout } from "../grid/SpotlightExpandedLayout";
import { makeSpotlightLandscapeLayout } from "../grid/SpotlightLandscapeLayout";
import { makeSpotlightPortraitLayout } from "../grid/SpotlightPortraitLayout";
import { ReactionsProvider, useReactions } from "../useReactions";
import handSoundOgg from "../sound/raise_hand.ogg?url";
import handSoundMp3 from "../sound/raise_hand.mp3?url";
import { useSwitchCamera } from "./useSwitchCamera";
const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {});
@@ -132,12 +137,14 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
return (
<RoomContext.Provider value={livekitRoom}>
<InCallView
{...props}
vm={vm}
livekitRoom={livekitRoom}
connState={connState}
/>
<ReactionsProvider rtcSession={props.rtcSession}>
<InCallView
{...props}
vm={vm}
livekitRoom={livekitRoom}
connState={connState}
/>
</ReactionsProvider>
</RoomContext.Provider>
);
};
@@ -170,6 +177,13 @@ export const InCallView: FC<InCallViewProps> = ({
connState,
onShareClick,
}) => {
const { supportsReactions, raisedHands } = useReactions();
const raisedHandCount = useMemo(
() => Object.keys(raisedHands).length,
[raisedHands],
);
const previousRaisedHandCount = useDeferredValue(raisedHandCount);
useWakeLock();
useEffect(() => {
@@ -308,6 +322,19 @@ export const InCallView: FC<InCallViewProps> = ({
[vm],
);
// Play a sound when the raised hand count increases.
const handRaisePlayer = useRef<HTMLAudioElement>(null);
useEffect(() => {
if (!handRaisePlayer.current) {
return;
}
if (previousRaisedHandCount < raisedHandCount) {
handRaisePlayer.current.play().catch((ex) => {
logger.warn("Failed to play raise hand sound", ex);
});
}
}, [raisedHandCount, handRaisePlayer, previousRaisedHandCount]);
useEffect(() => {
widget?.api.transport
.send(
@@ -527,6 +554,15 @@ export const InCallView: FC<InCallViewProps> = ({
/>,
);
}
if (supportsReactions) {
buttons.push(
<RaiseHandToggleButton
client={client}
rtcSession={rtcSession}
key="4"
/>,
);
}
buttons.push(<SettingsButton key="settings" onClick={openSettings} />);
}
@@ -608,6 +644,10 @@ export const InCallView: FC<InCallViewProps> = ({
))}
<RoomAudioRenderer />
{renderContent()}
<audio ref={handRaisePlayer} hidden>
<source src={handSoundOgg} type="audio/ogg; codecs=vorbis" />
<source src={handSoundMp3} type="audio/mpeg" />
</audio>
{footer}
{!noControls && <RageshakeRequestModal {...rageshakeRequestModalProps} />}
<SettingsModal

View File

@@ -0,0 +1,49 @@
/*
Copyright 2022-2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { ChangeEvent, FC, useCallback } from "react";
import { useTranslation } from "react-i18next";
import { Text } from "@vector-im/compound-web";
import { FieldRow, InputField } from "../input/Input";
import {
showHandRaisedTimer as showHandRaisedTimerSetting,
useSetting,
} from "./settings";
export const PreferencesSettingsTab: FC = () => {
const { t } = useTranslation();
const [showHandRaisedTimer, setShowHandRaisedTimer] = useSetting(
showHandRaisedTimerSetting,
);
const onChangeSetting = useCallback(
(e: ChangeEvent<HTMLInputElement>) => {
setShowHandRaisedTimer(e.target.checked);
},
[setShowHandRaisedTimer],
);
return (
<div>
<h4>{t("settings.preferences_tab_h4")}</h4>
<Text>{t("settings.preferences_tab_body")}</Text>
<FieldRow>
<InputField
id="showHandRaisedTimer"
label={t("settings.preferences_tab_show_hand_raised_timer_label")}
description={t(
"settings.preferences_tab_show_hand_raised_timer_description",
)}
type="checkbox"
checked={showHandRaisedTimer}
onChange={onChangeSetting}
/>
</FieldRow>
</div>
);
};

View File

@@ -30,11 +30,13 @@ import {
useOptInAnalytics,
} from "./settings";
import { isFirefox } from "../Platform";
import { PreferencesSettingsTab } from "./PreferencesSettingsTab";
type SettingsTab =
| "audio"
| "video"
| "profile"
| "preferences"
| "feedback"
| "more"
| "developer";
@@ -135,6 +137,12 @@ export const SettingsModal: FC<Props> = ({
content: generateDeviceSelection(devices.videoInput, t("common.camera")),
};
const preferencesTab: Tab<SettingsTab> = {
key: "preferences",
name: t("common.preferences"),
content: <PreferencesSettingsTab />,
};
const profileTab: Tab<SettingsTab> = {
key: "profile",
name: t("common.profile"),
@@ -234,7 +242,7 @@ export const SettingsModal: FC<Props> = ({
const tabs = [audioTab, videoTab];
if (widget === null) tabs.push(profileTab);
tabs.push(feedbackTab, moreTab);
tabs.push(preferencesTab, feedbackTab, moreTab);
if (developerSettingsTab) tabs.push(developerTab);
return (

View File

@@ -85,4 +85,9 @@ export const videoInput = new Setting<string | undefined>(
undefined,
);
export const showHandRaisedTimer = new Setting<boolean>(
"hand-raised-show-timer",
false,
);
export const alwaysShowSelf = new Setting<boolean>("always-show-self", true);

BIN
src/sound/raise_hand.mp3 Normal file

Binary file not shown.

BIN
src/sound/raise_hand.ogg Normal file

Binary file not shown.

View File

@@ -21,6 +21,15 @@ borders don't support gradients */
transition: opacity ease 0.15s;
inset: calc(-1 * var(--cpd-border-width-4));
border-radius: var(--cpd-space-5x);
background-blend-mode: overlay, normal;
}
.tile.speaking {
/* !important because speaking border should take priority over hover */
outline: var(--cpd-border-width-1) solid var(--cpd-color-bg-canvas-default) !important;
}
.tile.speaking::before {
background: linear-gradient(
119deg,
rgba(13, 92, 189, 0.7) 0%,
@@ -31,15 +40,25 @@ borders don't support gradients */
rgba(13, 92, 189, 0.9) 0%,
rgba(13, 189, 168, 0.9) 100%
);
background-blend-mode: overlay, normal;
opacity: 1;
}
.tile.speaking {
/* !important because speaking border should take priority over hover */
outline: var(--cpd-border-width-1) solid var(--cpd-color-bg-canvas-default) !important;
.tile.handRaised {
/* !important because hand raised border should take priority over hover */
outline: var(--cpd-border-width-2) solid var(--cpd-color-bg-canvas-default) !important;
}
.tile.speaking::before {
.tile.handRaised::before {
background: linear-gradient(
119deg,
var(--cpd-color-yellow-1200) 0%,
var(--cpd-color-yellow-900) 100%
),
linear-gradient(
180deg,
var(--cpd-color-yellow-1200) 0%,
var(--cpd-color-yellow-900) 100%
);
opacity: 1;
}

View File

@@ -9,9 +9,11 @@ import { RemoteTrackPublication } from "livekit-client";
import { test, expect } from "vitest";
import { render, screen } from "@testing-library/react";
import { axe } from "vitest-axe";
import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession";
import { GridTile } from "./GridTile";
import { withRemoteMedia } from "../utils/test";
import { ReactionsProvider } from "../useReactions";
test("GridTile is accessible", async () => {
await withRemoteMedia(
@@ -25,15 +27,29 @@ test("GridTile is accessible", async () => {
({}) as Partial<RemoteTrackPublication> as RemoteTrackPublication,
},
async (vm) => {
const fakeRtcSession = {
on: () => {},
off: () => {},
room: {
on: () => {},
off: () => {},
client: {
getUserId: () => null,
},
},
memberships: [],
} as unknown as MatrixRTCSession;
const { container } = render(
<GridTile
vm={vm}
onOpenProfile={() => {}}
targetWidth={300}
targetHeight={200}
showVideo
showSpeakingIndicators
/>,
<ReactionsProvider rtcSession={fakeRtcSession}>
<GridTile
vm={vm}
onOpenProfile={() => {}}
targetWidth={300}
targetHeight={200}
showVideo
showSpeakingIndicators
/>
</ReactionsProvider>,
);
expect(await axe(container)).toHaveNoViolations();
// Name should be visible

View File

@@ -44,6 +44,7 @@ import {
import { Slider } from "../Slider";
import { MediaView } from "./MediaView";
import { useLatest } from "../useLatest";
import { useReactions } from "../useReactions";
interface TileProps {
className?: string;
@@ -90,6 +91,7 @@ const UserMediaTile = forwardRef<HTMLDivElement, UserMediaTileProps>(
},
[vm],
);
const { raisedHands } = useReactions();
const MicIcon = audioEnabled ? MicOnSolidIcon : MicOffSolidIcon;
@@ -107,6 +109,10 @@ const UserMediaTile = forwardRef<HTMLDivElement, UserMediaTileProps>(
</>
);
const handRaised: Date | undefined = raisedHands[vm.member?.userId ?? ""];
const showSpeaking = showSpeakingIndicators && speaking;
const tile = (
<MediaView
ref={ref}
@@ -116,7 +122,8 @@ const UserMediaTile = forwardRef<HTMLDivElement, UserMediaTileProps>(
videoEnabled={videoEnabled && showVideo}
videoFit={cropVideo ? "cover" : "contain"}
className={classNames(className, styles.tile, {
[styles.speaking]: showSpeakingIndicators && speaking,
[styles.speaking]: showSpeaking,
[styles.handRaised]: !showSpeaking && !!handRaised,
})}
nameTagLeadingIcon={
<MicIcon
@@ -144,6 +151,7 @@ const UserMediaTile = forwardRef<HTMLDivElement, UserMediaTileProps>(
{menu}
</Menu>
}
raisedHandTime={handRaised}
{...props}
/>
);

View File

@@ -17,6 +17,8 @@ import { ErrorIcon } from "@vector-im/compound-design-tokens/assets/web/icons";
import styles from "./MediaView.module.css";
import { Avatar } from "../Avatar";
import { RaisedHandIndicator } from "../reactions/RaisedHandIndicator";
import { showHandRaisedTimer, useSetting } from "../settings/settings";
interface Props extends ComponentProps<typeof animated.div> {
className?: string;
@@ -32,6 +34,7 @@ interface Props extends ComponentProps<typeof animated.div> {
nameTagLeadingIcon?: ReactNode;
displayName: string;
primaryButton?: ReactNode;
raisedHandTime?: Date;
}
export const MediaView = forwardRef<HTMLDivElement, Props>(
@@ -50,11 +53,15 @@ export const MediaView = forwardRef<HTMLDivElement, Props>(
nameTagLeadingIcon,
displayName,
primaryButton,
raisedHandTime,
...props
},
ref,
) => {
const { t } = useTranslation();
const [handRaiseTimerVisible] = useSetting(showHandRaisedTimer);
const avatarSize = Math.round(Math.min(targetWidth, targetHeight) / 2);
return (
<animated.div
@@ -72,7 +79,7 @@ export const MediaView = forwardRef<HTMLDivElement, Props>(
<Avatar
id={member?.userId ?? displayName}
name={displayName}
size={Math.round(Math.min(targetWidth, targetHeight) / 2)}
size={avatarSize}
src={member?.getMxcAvatarUrl()}
className={styles.avatar}
/>
@@ -86,6 +93,11 @@ export const MediaView = forwardRef<HTMLDivElement, Props>(
)}
</div>
<div className={styles.fg}>
<RaisedHandIndicator
raisedHandTime={raisedHandTime}
minature={avatarSize < 96}
showTimer={handRaiseTimerVisible}
/>
<div className={styles.nameTag}>
{nameTagLeadingIcon}
<Text as="span" size="sm" weight="medium" className={styles.name}>

268
src/useReactions.test.tsx Normal file
View File

@@ -0,0 +1,268 @@
/*
Copyright 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { act, render } from "@testing-library/react";
import { FC, ReactNode } from "react";
import { describe, expect, test } from "vitest";
import {
MatrixRTCSession,
MatrixRTCSessionEvent,
} from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession";
import {
EventTimeline,
EventTimelineSet,
EventType,
MatrixClient,
MatrixEvent,
Room,
RoomEvent,
} from "matrix-js-sdk/src/matrix";
import EventEmitter from "events";
import { randomUUID } from "crypto";
import { ReactionsProvider, useReactions } from "./useReactions";
/**
* Test explanation.
* This test suite checks that the useReactions hook appropriately reacts
* to new reactions, redactions and membership changesin the room. There is
* a large amount of test structure used to construct a mock environment.
*/
const memberUserIdAlice = "@alice:example.org";
const memberEventAlice = "$membership-alice:example.org";
const memberUserIdBob = "@bob:example.org";
const memberEventBob = "$membership-bob:example.org";
const membership: Record<string, string> = {
[memberEventAlice]: memberUserIdAlice,
[memberEventBob]: memberUserIdBob,
"$membership-charlie:example.org": "@charlie:example.org",
};
const TestComponent: FC = () => {
const { raisedHands, myReactionId } = useReactions();
return (
<div>
<ul>
{Object.entries(raisedHands).map(([userId, date]) => (
<li key={userId}>
<span>{userId}</span>
<time>{date.getTime()}</time>
</li>
))}
</ul>
<p>{myReactionId ? "Local reaction" : "No local reaction"}</p>
</div>
);
};
const TestComponentWrapper = ({
rtcSession,
}: {
rtcSession: MockRTCSession;
}): ReactNode => {
return (
<ReactionsProvider rtcSession={rtcSession as unknown as MatrixRTCSession}>
<TestComponent />
</ReactionsProvider>
);
};
export class MockRTCSession extends EventEmitter {
public memberships = Object.entries(membership).map(([eventId, sender]) => ({
sender,
eventId,
createdTs: (): Date => new Date(),
}));
public constructor(public readonly room: MockRoom) {
super();
}
public testRemoveMember(userId: string): void {
this.memberships = this.memberships.filter((u) => u.sender !== userId);
this.emit(MatrixRTCSessionEvent.MembershipsChanged);
}
public testAddMember(sender: string): void {
this.memberships.push({
sender,
eventId: `!fake-${randomUUID()}:event`,
createdTs: (): Date => new Date(),
});
this.emit(MatrixRTCSessionEvent.MembershipsChanged);
}
}
function createReaction(
parentMemberEvent: string,
overridenSender?: string,
): MatrixEvent {
return new MatrixEvent({
sender: overridenSender ?? membership[parentMemberEvent],
type: EventType.Reaction,
origin_server_ts: new Date().getTime(),
content: {
"m.relates_to": {
key: "🖐️",
event_id: parentMemberEvent,
},
},
event_id: randomUUID(),
});
}
function createRedaction(sender: string, reactionEventId: string): MatrixEvent {
return new MatrixEvent({
sender,
type: EventType.RoomRedaction,
origin_server_ts: new Date().getTime(),
redacts: reactionEventId,
content: {},
event_id: randomUUID(),
});
}
export class MockRoom extends EventEmitter {
public constructor(private readonly existingRelations: MatrixEvent[] = []) {
super();
}
public get client(): MatrixClient {
return {
getUserId: (): string => memberUserIdAlice,
} as unknown as MatrixClient;
}
public get relations(): Room["relations"] {
return {
getChildEventsForEvent: (membershipEventId: string) => ({
getRelations: (): MatrixEvent[] => {
return this.existingRelations.filter(
(r) =>
r.getContent()["m.relates_to"]?.event_id === membershipEventId,
);
},
}),
} as unknown as Room["relations"];
}
public testSendReaction(
parentMemberEvent: string,
overridenSender?: string,
): string {
const evt = createReaction(parentMemberEvent, overridenSender);
this.emit(RoomEvent.Timeline, evt, this, undefined, false, {
timeline: new EventTimeline(new EventTimelineSet(undefined)),
});
return evt.getId()!;
}
}
describe("useReactions", () => {
test("starts with an empty list", () => {
const rtcSession = new MockRTCSession(new MockRoom());
const { queryByRole } = render(
<TestComponentWrapper rtcSession={rtcSession} />,
);
expect(queryByRole("list")?.children).to.have.lengthOf(0);
});
test("handles own raised hand", async () => {
const room = new MockRoom();
const rtcSession = new MockRTCSession(room);
const { queryByText } = render(
<TestComponentWrapper rtcSession={rtcSession} />,
);
await act(() => room.testSendReaction(memberEventAlice));
expect(queryByText("Local reaction")).toBeTruthy();
});
test("handles incoming raised hand", async () => {
const room = new MockRoom();
const rtcSession = new MockRTCSession(room);
const { queryByRole } = render(
<TestComponentWrapper rtcSession={rtcSession} />,
);
await act(() => room.testSendReaction(memberEventAlice));
expect(queryByRole("list")?.children).to.have.lengthOf(1);
await act(() => room.testSendReaction(memberEventBob));
expect(queryByRole("list")?.children).to.have.lengthOf(2);
});
test("handles incoming unraised hand", async () => {
const room = new MockRoom();
const rtcSession = new MockRTCSession(room);
const { queryByRole } = render(
<TestComponentWrapper rtcSession={rtcSession} />,
);
const reactionEventId = await act(() =>
room.testSendReaction(memberEventAlice),
);
expect(queryByRole("list")?.children).to.have.lengthOf(1);
await act(() =>
room.emit(
RoomEvent.Redaction,
createRedaction(memberUserIdAlice, reactionEventId),
room,
undefined,
),
);
expect(queryByRole("list")?.children).to.have.lengthOf(0);
});
test("handles loading prior raised hand events", () => {
const room = new MockRoom([createReaction(memberEventAlice)]);
const rtcSession = new MockRTCSession(room);
const { queryByRole } = render(
<TestComponentWrapper rtcSession={rtcSession} />,
);
expect(queryByRole("list")?.children).to.have.lengthOf(1);
});
// If the membership event changes for a user, we want to remove
// the raised hand event.
test("will remove reaction when a member leaves the call", () => {
const room = new MockRoom([createReaction(memberEventAlice)]);
const rtcSession = new MockRTCSession(room);
const { queryByRole } = render(
<TestComponentWrapper rtcSession={rtcSession} />,
);
expect(queryByRole("list")?.children).to.have.lengthOf(1);
act(() => rtcSession.testRemoveMember(memberUserIdAlice));
expect(queryByRole("list")?.children).to.have.lengthOf(0);
});
test("will remove reaction when a member joins via a new event", () => {
const room = new MockRoom([createReaction(memberEventAlice)]);
const rtcSession = new MockRTCSession(room);
const { queryByRole } = render(
<TestComponentWrapper rtcSession={rtcSession} />,
);
expect(queryByRole("list")?.children).to.have.lengthOf(1);
// Simulate leaving and rejoining
act(() => {
rtcSession.testRemoveMember(memberUserIdAlice);
rtcSession.testAddMember(memberUserIdAlice);
});
expect(queryByRole("list")?.children).to.have.lengthOf(0);
});
test("ignores invalid sender for historic event", () => {
const room = new MockRoom([
createReaction(memberEventAlice, memberUserIdBob),
]);
const rtcSession = new MockRTCSession(room);
const { queryByRole } = render(
<TestComponentWrapper rtcSession={rtcSession} />,
);
expect(queryByRole("list")?.children).to.have.lengthOf(0);
});
test("ignores invalid sender for new event", async () => {
const room = new MockRoom([]);
const rtcSession = new MockRTCSession(room);
const { queryByRole } = render(
<TestComponentWrapper rtcSession={rtcSession} />,
);
await act(() => room.testSendReaction(memberEventAlice, memberUserIdBob));
expect(queryByRole("list")?.children).to.have.lengthOf(0);
});
});

249
src/useReactions.tsx Normal file
View File

@@ -0,0 +1,249 @@
/*
Copyright 2024 Milton Moura <miltonmoura@gmail.com>
SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import {
EventType,
MatrixEvent,
RelationType,
RoomEvent as MatrixRoomEvent,
} from "matrix-js-sdk/src/matrix";
import { ReactionEventContent } from "matrix-js-sdk/src/types";
import {
createContext,
useContext,
useState,
ReactNode,
useCallback,
useEffect,
useMemo,
} from "react";
import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession";
import { logger } from "matrix-js-sdk/src/logger";
import { useMatrixRTCSessionMemberships } from "./useMatrixRTCSessionMemberships";
import { useClientState } from "./ClientContext";
interface ReactionsContextType {
raisedHands: Record<string, Date>;
supportsReactions: boolean;
myReactionId: string | null;
}
const ReactionsContext = createContext<ReactionsContextType | undefined>(
undefined,
);
interface RaisedHandInfo {
/**
* Call membership event that was reacted to.
*/
membershipEventId: string;
/**
* Event ID of the reaction itself.
*/
reactionEventId: string;
/**
* The time when the reaction was raised.
*/
time: Date;
}
export const useReactions = (): ReactionsContextType => {
const context = useContext(ReactionsContext);
if (!context) {
throw new Error("useReactions must be used within a ReactionsProvider");
}
return context;
};
/**
* Provider that handles raised hand reactions for a given `rtcSession`.
*/
export const ReactionsProvider = ({
children,
rtcSession,
}: {
children: ReactNode;
rtcSession: MatrixRTCSession;
}): JSX.Element => {
const [raisedHands, setRaisedHands] = useState<
Record<string, RaisedHandInfo>
>({});
const memberships = useMatrixRTCSessionMemberships(rtcSession);
const clientState = useClientState();
const supportsReactions =
clientState?.state === "valid" && clientState.supportedFeatures.reactions;
const room = rtcSession.room;
const myUserId = room.client.getUserId();
// Calculate our own reaction event.
const myReactionId = useMemo(
(): string | null =>
(myUserId && raisedHands[myUserId]?.reactionEventId) ?? null,
[raisedHands, myUserId],
);
// Reduce the data down for the consumers.
const resultRaisedHands = useMemo(
() =>
Object.fromEntries(
Object.entries(raisedHands).map(([uid, data]) => [uid, data.time]),
),
[raisedHands],
);
const addRaisedHand = useCallback((userId: string, info: RaisedHandInfo) => {
setRaisedHands((prevRaisedHands) => ({
...prevRaisedHands,
[userId]: info,
}));
}, []);
const removeRaisedHand = useCallback((userId: string) => {
setRaisedHands(
({ [userId]: _removed, ...remainingRaisedHands }) => remainingRaisedHands,
);
}, []);
// This effect will check the state whenever the membership of the session changes.
useEffect(() => {
// Fetches the first reaction for a given event.
const getLastReactionEvent = (
eventId: string,
expectedSender: string,
): MatrixEvent | undefined => {
const relations = room.relations.getChildEventsForEvent(
eventId,
RelationType.Annotation,
EventType.Reaction,
);
const allEvents = relations?.getRelations() ?? [];
return allEvents.find(
(reaction) =>
reaction.event.sender === expectedSender &&
reaction.getType() === EventType.Reaction &&
reaction.getContent()?.["m.relates_to"]?.key === "🖐️",
);
};
// Remove any raised hands for users no longer joined to the call.
for (const userId of Object.keys(raisedHands).filter(
(rhId) => !memberships.find((u) => u.sender == rhId),
)) {
removeRaisedHand(userId);
}
// For each member in the call, check to see if a reaction has
// been raised and adjust.
for (const m of memberships) {
if (!m.sender || !m.eventId) {
continue;
}
if (
raisedHands[m.sender] &&
raisedHands[m.sender].membershipEventId !== m.eventId
) {
// Membership event for sender has changed since the hand
// was raised, reset.
removeRaisedHand(m.sender);
}
const reaction = getLastReactionEvent(m.eventId, m.sender);
if (reaction) {
const eventId = reaction?.getId();
if (!eventId) {
continue;
}
addRaisedHand(m.sender, {
membershipEventId: m.eventId,
reactionEventId: eventId,
time: new Date(reaction.localTimestamp),
});
}
}
// Ignoring raisedHands here because we don't want to trigger each time the raised
// hands set is updated.
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [room, memberships, myUserId, addRaisedHand, removeRaisedHand]);
// This effect handles any *live* reaction/redactions in the room.
useEffect(() => {
const handleReactionEvent = (event: MatrixEvent): void => {
if (event.isSending()) {
// Skip any events that are still sending.
return;
}
const sender = event.getSender();
const reactionEventId = event.getId();
if (!sender || !reactionEventId) {
// Skip any event without a sender or event ID.
return;
}
if (event.getType() === EventType.Reaction) {
const content = event.getContent() as ReactionEventContent;
const membershipEventId = content["m.relates_to"].event_id;
// Check to see if this reaction was made to a membership event (and the
// sender of the reaction matches the membership)
if (
!memberships.some(
(e) => e.eventId === membershipEventId && e.sender === sender,
)
) {
logger.warn(
`Reaction target was not a membership event for ${sender}, ignoring`,
);
return;
}
if (content?.["m.relates_to"].key === "🖐️") {
addRaisedHand(sender, {
reactionEventId,
membershipEventId,
time: new Date(event.localTimestamp),
});
}
} else if (event.getType() === EventType.RoomRedaction) {
const targetEvent = event.event.redacts;
const targetUser = Object.entries(raisedHands).find(
([_u, r]) => r.reactionEventId === targetEvent,
)?.[0];
if (!targetUser) {
// Reaction target was not for us, ignoring
return;
}
removeRaisedHand(targetUser);
}
};
room.on(MatrixRoomEvent.Timeline, handleReactionEvent);
room.on(MatrixRoomEvent.Redaction, handleReactionEvent);
// We listen for a local echo to get the real event ID, as timeline events
// may still be sending.
room.on(MatrixRoomEvent.LocalEchoUpdated, handleReactionEvent);
return (): void => {
room.off(MatrixRoomEvent.Timeline, handleReactionEvent);
room.off(MatrixRoomEvent.Redaction, handleReactionEvent);
room.off(MatrixRoomEvent.LocalEchoUpdated, handleReactionEvent);
};
}, [room, addRaisedHand, removeRaisedHand, memberships, raisedHands]);
return (
<ReactionsContext.Provider
value={{
raisedHands: resultRaisedHands,
supportsReactions,
myReactionId,
}}
>
{children}
</ReactionsContext.Provider>
);
};

View File

@@ -103,6 +103,8 @@ export const widget = ((): WidgetHelpers | null => {
const sendRecvEvent = [
"org.matrix.rageshake_request",
EventType.CallEncryptionKeysPrefix,
EventType.Reaction,
EventType.RoomRedaction,
];
const sendState = [

View File

@@ -1601,6 +1601,38 @@
resolved "https://registry.yarnpkg.com/@floating-ui/utils/-/utils-0.2.8.tgz#21a907684723bbbaa5f0974cf7730bd797eb8e62"
integrity sha512-kym7SodPp8/wloecOpcmSnWJsK7M0E5Wg8UcFA+uO4B9s5d0ywXOEro/8HM9x0rW+TljRzul/14UYz3TleT3ig==
"@formatjs/ecma402-abstract@2.2.1":
version "2.2.1"
resolved "https://registry.yarnpkg.com/@formatjs/ecma402-abstract/-/ecma402-abstract-2.2.1.tgz#2e62bc5c22b0e6a5e13bfec6aac15d3d403e1065"
integrity sha512-O4ywpkdJybrjFc9zyL8qK5aklleIAi5O4nYhBVJaOFtCkNrnU+lKFeJOFC48zpsZQmR8Aok2V79hGpHnzbmFpg==
dependencies:
"@formatjs/fast-memoize" "2.2.2"
"@formatjs/intl-localematcher" "0.5.6"
tslib "2"
"@formatjs/fast-memoize@2.2.2":
version "2.2.2"
resolved "https://registry.yarnpkg.com/@formatjs/fast-memoize/-/fast-memoize-2.2.2.tgz#2409ec10f5f7d6c65f4c04e6c2d6cc56fa1e4cef"
integrity sha512-mzxZcS0g1pOzwZTslJOBTmLzDXseMLLvnh25ymRilCm8QLMObsQ7x/rj9GNrH0iUhZMlFisVOD6J1n6WQqpKPQ==
dependencies:
tslib "2"
"@formatjs/intl-durationformat@^0.6.1":
version "0.6.1"
resolved "https://registry.yarnpkg.com/@formatjs/intl-durationformat/-/intl-durationformat-0.6.1.tgz#ea376202b1dc70683a3f3e125bb07f4fab1135a5"
integrity sha512-tPSX/D/wjO5ZKnRtwLlUYtjLUBILLX1w6+arU97NpPCpZ8SRWQePu+kDAxDwFKJ/w09idqvSFkJjYGTs6hMd1A==
dependencies:
"@formatjs/ecma402-abstract" "2.2.1"
"@formatjs/intl-localematcher" "0.5.6"
tslib "2"
"@formatjs/intl-localematcher@0.5.6":
version "0.5.6"
resolved "https://registry.yarnpkg.com/@formatjs/intl-localematcher/-/intl-localematcher-0.5.6.tgz#cd0cd99483673d3196a15b4e2c924cfda7f002f8"
integrity sha512-roz1+Ba5e23AHX6KUAWmLEyTRZegM5YDuxuvkHCyK3RJddf/UXB2f+s7pOMm9ktfPGla0g+mQXOn5vsuYirnaA==
dependencies:
tslib "2"
"@gulpjs/to-absolute-glob@^4.0.0":
version "4.0.0"
resolved "https://registry.yarnpkg.com/@gulpjs/to-absolute-glob/-/to-absolute-glob-4.0.0.tgz#1fc2460d3953e1d9b9f2dfdb4bcc99da4710c021"
@@ -7631,16 +7663,16 @@ tsconfig-paths@^3.15.0:
minimist "^1.2.6"
strip-bom "^3.0.0"
tslib@2, tslib@^2.0.0, tslib@^2.1.0:
version "2.8.0"
resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.8.0.tgz#d124c86c3c05a40a91e6fdea4021bd31d377971b"
integrity sha512-jWVzBLplnCmoaTr13V9dYbiQ99wvZRd0vNWaDRg+aVYRcjDF3nDksxFDE/+fkXnKhpnUUkmx5pK/v8mCtLVqZA==
tslib@2.7.0:
version "2.7.0"
resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.7.0.tgz#d9b40c5c40ab59e8738f297df3087bf1a2690c01"
integrity sha512-gLXCKdN1/j47AiHiOkJN69hJmcbGTHI0ImLmbYLHykhgeN0jVGola9yVjFgzCUklsZQMW55o+dW7IXv3RCXDzA==
tslib@^2.0.0, tslib@^2.1.0:
version "2.8.0"
resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.8.0.tgz#d124c86c3c05a40a91e6fdea4021bd31d377971b"
integrity sha512-jWVzBLplnCmoaTr13V9dYbiQ99wvZRd0vNWaDRg+aVYRcjDF3nDksxFDE/+fkXnKhpnUUkmx5pK/v8mCtLVqZA==
tslib@^2.0.3:
version "2.6.2"
resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.6.2.tgz#703ac29425e7b37cd6fd456e92404d46d1f3e4ae"