Merge branch 'livekit' into toger5/tiles_based_on_rtc_member

This commit is contained in:
Hugh Nimmo-Smith
2024-11-13 09:31:39 +00:00
24 changed files with 851 additions and 685 deletions

View File

@@ -1,77 +0,0 @@
/*
Copyright 2022-2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { MatrixClient } from "matrix-js-sdk/src/client";
import { useTranslation } from "react-i18next";
import { MatrixError } from "matrix-js-sdk/src/matrix";
import { Heading, Text } from "@vector-im/compound-web";
import { Link } from "../button/Link";
import {
useLoadGroupCall,
GroupCallStatus,
CallTerminatedMessage,
} from "./useLoadGroupCall";
import { ErrorView, FullScreenView } from "../FullScreenView";
interface Props {
client: MatrixClient;
roomIdOrAlias: string;
viaServers: string[];
children: (groupCallState: GroupCallStatus) => JSX.Element;
}
export function GroupCallLoader({
client,
roomIdOrAlias,
viaServers,
children,
}: Props): JSX.Element {
const { t } = useTranslation();
const groupCallState = useLoadGroupCall(client, roomIdOrAlias, viaServers);
switch (groupCallState.kind) {
case "loaded":
case "waitForInvite":
case "canKnock":
return children(groupCallState);
case "loading":
return (
<FullScreenView>
<h1>{t("common.loading")}</h1>
</FullScreenView>
);
case "failed":
if ((groupCallState.error as MatrixError).errcode === "M_NOT_FOUND") {
return (
<FullScreenView>
<Heading>{t("group_call_loader.failed_heading")}</Heading>
<Text>{t("group_call_loader.failed_text")}</Text>
{/* XXX: A 'create it for me' button would be the obvious UX here. Two screens already have
dupes of this flow, let's make a common component and put it here. */}
<Link to="/">{t("common.home")}</Link>
</FullScreenView>
);
} else if (groupCallState.error instanceof CallTerminatedMessage) {
return (
<FullScreenView>
<Heading>{groupCallState.error.message}</Heading>
<Text>{groupCallState.error.messageBody}</Text>
{groupCallState.error.reason && (
<>
{t("group_call_loader.reason")}:
<Text size="sm">"{groupCallState.error.reason}"</Text>
</>
)}
<Link to="/">{t("common.home")}</Link>
</FullScreenView>
);
} else {
return <ErrorView error={groupCallState.error} />;
}
}
}

View File

@@ -177,29 +177,37 @@ export const GroupCallView: FC<Props> = ({
}
};
if (widget && preload && skipLobby) {
// In preload mode without lobby we wait for a join action before entering
const onJoin = (ev: CustomEvent<IWidgetApiRequest>): void => {
(async (): Promise<void> => {
await defaultDeviceSetup(ev.detail.data as unknown as JoinCallData);
await enterRTCSession(rtcSession, perParticipantE2EE);
widget!.api.transport.reply(ev.detail, {});
})().catch((e) => {
logger.error("Error joining RTC session", e);
});
};
widget.lazyActions.on(ElementWidgetActions.JoinCall, onJoin);
return (): void => {
widget!.lazyActions.off(ElementWidgetActions.JoinCall, onJoin);
};
} else if (widget && !preload && skipLobby) {
// No lobby and no preload: we enter the rtc session right away
(async (): Promise<void> => {
await defaultDeviceSetup({ audioInput: null, videoInput: null });
await enterRTCSession(rtcSession, perParticipantE2EE);
})().catch((e) => {
logger.error("Error joining RTC session", e);
});
if (skipLobby) {
if (widget) {
if (preload) {
// In preload mode without lobby we wait for a join action before entering
const onJoin = (ev: CustomEvent<IWidgetApiRequest>): void => {
(async (): Promise<void> => {
await defaultDeviceSetup(
ev.detail.data as unknown as JoinCallData,
);
await enterRTCSession(rtcSession, perParticipantE2EE);
widget!.api.transport.reply(ev.detail, {});
})().catch((e) => {
logger.error("Error joining RTC session", e);
});
};
widget.lazyActions.on(ElementWidgetActions.JoinCall, onJoin);
return (): void => {
widget!.lazyActions.off(ElementWidgetActions.JoinCall, onJoin);
};
} else {
// No lobby and no preload: we enter the rtc session right away
(async (): Promise<void> => {
await defaultDeviceSetup({ audioInput: null, videoInput: null });
await enterRTCSession(rtcSession, perParticipantE2EE);
})().catch((e) => {
logger.error("Error joining RTC session", e);
});
}
} else {
void enterRTCSession(rtcSession, perParticipantE2EE);
}
}
}, [rtcSession, preload, skipLobby, perParticipantE2EE]);

View File

@@ -180,48 +180,3 @@ Please see LICENSE in the repository root for full details.
position: relative;
flex-grow: 1;
}
.floatingReaction {
position: relative;
display: inline;
z-index: 2;
font-size: 32pt;
/* Reactions are "active" for 3 seconds (as per REACTION_ACTIVE_TIME_MS), give a bit more time for it to fade out. */
animation-duration: 4s;
animation-name: reaction-up;
width: fit-content;
pointer-events: none;
}
@keyframes reaction-up {
from {
opacity: 1;
translate: 100vw 0;
scale: 200%;
}
to {
opacity: 0;
translate: 100vw -100vh;
scale: 100%;
}
}
@media (prefers-reduced-motion) {
@keyframes reaction-up-reduced {
from {
opacity: 1;
}
to {
opacity: 0;
}
}
.floatingReaction {
font-size: 48pt;
animation-name: reaction-up-reduced;
top: calc(-50vh + (48pt / 2));
left: calc(50vw - (48pt / 2)) !important;
}
}

View File

@@ -85,11 +85,8 @@ import handSoundOgg from "../sound/raise_hand.ogg?url";
import handSoundMp3 from "../sound/raise_hand.mp3?url";
import { ReactionsAudioRenderer } from "./ReactionAudioRenderer";
import { useSwitchCamera } from "./useSwitchCamera";
import {
soundEffectVolumeSetting,
showReactions,
useSetting,
} from "../settings/settings";
import { soundEffectVolumeSetting, useSetting } from "../settings/settings";
import { ReactionsOverlay } from "./ReactionsOverlay";
const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {});
@@ -180,27 +177,14 @@ export const InCallView: FC<InCallViewProps> = ({
connState,
onShareClick,
}) => {
const [shouldShowReactions] = useSetting(showReactions);
const [soundEffectVolume] = useSetting(soundEffectVolumeSetting);
const { supportsReactions, raisedHands, reactions } = useReactions();
const { supportsReactions, raisedHands } = useReactions();
const raisedHandCount = useMemo(
() => Object.keys(raisedHands).length,
[raisedHands],
);
const previousRaisedHandCount = useDeferredValue(raisedHandCount);
const reactionsIcons = useMemo(
() =>
shouldShowReactions
? Object.entries(reactions).map(([sender, { emoji }]) => ({
sender,
emoji,
startX: -Math.ceil(Math.random() * 50) - 25,
}))
: [],
[shouldShowReactions, reactions],
);
useWakeLock();
useEffect(() => {
@@ -686,15 +670,7 @@ export const InCallView: FC<InCallViewProps> = ({
<source src={handSoundMp3} type="audio/mpeg" />
</audio>
<ReactionsAudioRenderer />
{reactionsIcons.map(({ sender, emoji, startX }) => (
<span
style={{ left: `${startX}vw` }}
className={styles.floatingReaction}
key={sender}
>
{emoji}
</span>
))}
<ReactionsOverlay />
{footer}
{layout.type !== "pip" && (
<>

View File

@@ -0,0 +1,54 @@
.container {
position: absolute;
display: inline;
z-index: 2;
pointer-events: none;
width: 100vw;
height: 100vh;
left: 0;
top: 0;
}
.reaction {
font-size: 32pt;
/* Reactions are "active" for 3 seconds (as per REACTION_ACTIVE_TIME_MS), give a bit more time for it to fade out. */
animation-duration: 4s;
animation-name: reaction-up;
width: fit-content;
position: relative;
top: 80vh;
}
@keyframes reaction-up {
from {
opacity: 1;
translate: 0 0;
scale: 200%;
top: 80vh;
}
to {
top: 0;
opacity: 0;
scale: 100%;
}
}
@media (prefers-reduced-motion) {
@keyframes reaction-up-reduced {
from {
opacity: 1;
}
to {
opacity: 0;
}
}
.reaction {
font-size: 48pt;
animation-name: reaction-up-reduced;
top: calc(-50vh + (48pt / 2));
left: calc(50vw - (48pt / 2)) !important;
}
}

View File

@@ -0,0 +1,120 @@
/*
Copyright 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { render } from "@testing-library/react";
import { expect, test } from "vitest";
import { TooltipProvider } from "@vector-im/compound-web";
import { act, ReactNode } from "react";
import { afterEach } from "node:test";
import {
MockRoom,
MockRTCSession,
TestReactionsWrapper,
} from "../utils/testReactions";
import { showReactions } from "../settings/settings";
import { ReactionsOverlay } from "./ReactionsOverlay";
import { ReactionSet } from "../reactions";
const memberUserIdAlice = "@alice:example.org";
const memberUserIdBob = "@bob:example.org";
const memberUserIdCharlie = "@charlie:example.org";
const memberEventAlice = "$membership-alice:example.org";
const memberEventBob = "$membership-bob:example.org";
const memberEventCharlie = "$membership-charlie:example.org";
const membership: Record<string, string> = {
[memberEventAlice]: memberUserIdAlice,
[memberEventBob]: memberUserIdBob,
[memberEventCharlie]: memberUserIdCharlie,
};
function TestComponent({
rtcSession,
}: {
rtcSession: MockRTCSession;
}): ReactNode {
return (
<TooltipProvider>
<TestReactionsWrapper rtcSession={rtcSession}>
<ReactionsOverlay />
</TestReactionsWrapper>
</TooltipProvider>
);
}
afterEach(() => {
showReactions.setValue(showReactions.defaultValue);
});
test("defaults to showing no reactions", () => {
showReactions.setValue(true);
const rtcSession = new MockRTCSession(
new MockRoom(memberUserIdAlice),
membership,
);
const { container } = render(<TestComponent rtcSession={rtcSession} />);
expect(container.getElementsByTagName("span")).toHaveLength(0);
});
test("shows a reaction when sent", () => {
showReactions.setValue(true);
const reaction = ReactionSet[0];
const room = new MockRoom(memberUserIdAlice);
const rtcSession = new MockRTCSession(room, membership);
const { getByRole } = render(<TestComponent rtcSession={rtcSession} />);
act(() => {
room.testSendReaction(memberEventAlice, reaction, membership);
});
const span = getByRole("presentation");
expect(getByRole("presentation")).toBeTruthy();
expect(span.innerHTML).toEqual(reaction.emoji);
});
test("shows two of the same reaction when sent", () => {
showReactions.setValue(true);
const reaction = ReactionSet[0];
const room = new MockRoom(memberUserIdAlice);
const rtcSession = new MockRTCSession(room, membership);
const { getAllByRole } = render(<TestComponent rtcSession={rtcSession} />);
act(() => {
room.testSendReaction(memberEventAlice, reaction, membership);
});
act(() => {
room.testSendReaction(memberEventBob, reaction, membership);
});
expect(getAllByRole("presentation")).toHaveLength(2);
});
test("shows two different reactions when sent", () => {
showReactions.setValue(true);
const room = new MockRoom(memberUserIdAlice);
const rtcSession = new MockRTCSession(room, membership);
const [reactionA, reactionB] = ReactionSet;
const { getAllByRole } = render(<TestComponent rtcSession={rtcSession} />);
act(() => {
room.testSendReaction(memberEventAlice, reactionA, membership);
});
act(() => {
room.testSendReaction(memberEventBob, reactionB, membership);
});
const [reactionElementA, reactionElementB] = getAllByRole("presentation");
expect(reactionElementA.innerHTML).toEqual(reactionA.emoji);
expect(reactionElementB.innerHTML).toEqual(reactionB.emoji);
});
test("hides reactions when reaction animations are disabled", () => {
showReactions.setValue(false);
const reaction = ReactionSet[0];
const room = new MockRoom(memberUserIdAlice);
const rtcSession = new MockRTCSession(room, membership);
act(() => {
room.testSendReaction(memberEventAlice, reaction, membership);
});
const { container } = render(<TestComponent rtcSession={rtcSession} />);
expect(container.getElementsByTagName("span")).toHaveLength(0);
});

View File

@@ -0,0 +1,50 @@
/*
Copyright 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { ReactNode, useMemo } from "react";
import { useReactions } from "../useReactions";
import {
showReactions as showReactionsSetting,
useSetting,
} from "../settings/settings";
import styles from "./ReactionsOverlay.module.css";
export function ReactionsOverlay(): ReactNode {
const { reactions } = useReactions();
const [showReactions] = useSetting(showReactionsSetting);
const reactionsIcons = useMemo(
() =>
showReactions
? Object.entries(reactions).map(([sender, { emoji }]) => ({
sender,
emoji,
startX: Math.ceil(Math.random() * 80) + 10,
}))
: [],
[showReactions, reactions],
);
return (
<div className={styles.container}>
{reactionsIcons.map(({ sender, emoji, startX }) => (
<span
// Reactions effects are considered presentation elements. The reaction
// is also present on the sender's tile, which assistive technology can
// read from instead.
role="presentation"
style={{ left: `${startX}vw` }}
className={styles.reaction}
// A sender can only send one emoji at a time.
key={sender}
>
{emoji}
</span>
))}
</div>
);
}

View File

@@ -5,15 +5,16 @@ SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { FC, useEffect, useState, useCallback, ReactNode } from "react";
import { FC, useEffect, useState, ReactNode, useRef } from "react";
import { logger } from "matrix-js-sdk/src/logger";
import { useTranslation } from "react-i18next";
import { CheckIcon } from "@vector-im/compound-design-tokens/assets/web/icons";
import { MatrixError } from "matrix-js-sdk/src/http-api";
import { Heading, Text } from "@vector-im/compound-web";
import { useClientLegacy } from "../ClientContext";
import { ErrorView, LoadingView } from "../FullScreenView";
import { ErrorView, FullScreenView, LoadingView } from "../FullScreenView";
import { RoomAuthView } from "./RoomAuthView";
import { GroupCallLoader } from "./GroupCallLoader";
import { GroupCallView } from "./GroupCallView";
import { useRoomIdentifier, useUrlParams } from "../UrlParams";
import { useRegisterPasswordlessUser } from "../auth/useRegisterPasswordlessUser";
@@ -21,13 +22,14 @@ import { HomePage } from "../home/HomePage";
import { platform } from "../Platform";
import { AppSelectionModal } from "./AppSelectionModal";
import { widget } from "../widget";
import { GroupCallStatus } from "./useLoadGroupCall";
import { CallTerminatedMessage, useLoadGroupCall } from "./useLoadGroupCall";
import { LobbyView } from "./LobbyView";
import { E2eeType } from "../e2ee/e2eeType";
import { useProfile } from "../profile/useProfile";
import { useMuteStates } from "./MuteStates";
import { useOptInAnalytics } from "../settings/settings";
import { Config } from "../config/Config";
import { Link } from "../button/Link";
export const RoomPage: FC = () => {
const {
@@ -53,6 +55,7 @@ export const RoomPage: FC = () => {
useClientLegacy();
const { avatarUrl, displayName: userDisplayName } = useProfile(client);
const groupCallState = useLoadGroupCall(client, roomIdOrAlias, viaServers);
const muteStates = useMuteStates();
useEffect(() => {
@@ -82,82 +85,112 @@ export const RoomPage: FC = () => {
if (optInAnalytics === null && setOptInAnalytics) setOptInAnalytics(true);
}, [optInAnalytics, setOptInAnalytics]);
const groupCallView = useCallback(
(groupCallState: GroupCallStatus): JSX.Element => {
switch (groupCallState.kind) {
case "loaded":
return (
<GroupCallView
client={client!}
rtcSession={groupCallState.rtcSession}
isPasswordlessUser={passwordlessUser}
confineToRoom={confineToRoom}
preload={preload}
skipLobby={skipLobby}
hideHeader={hideHeader}
muteStates={muteStates}
/>
const wasInWaitForInviteState = useRef<boolean>(false);
useEffect(() => {
if (groupCallState.kind === "loaded" && wasInWaitForInviteState.current) {
logger.log("Play join sound 'Not yet implemented'");
}
}, [groupCallState.kind]);
const groupCallView = (): JSX.Element => {
switch (groupCallState.kind) {
case "loaded":
return (
<GroupCallView
client={client!}
rtcSession={groupCallState.rtcSession}
isPasswordlessUser={passwordlessUser}
confineToRoom={confineToRoom}
preload={preload}
skipLobby={skipLobby || wasInWaitForInviteState.current}
hideHeader={hideHeader}
muteStates={muteStates}
/>
);
case "waitForInvite":
case "canKnock": {
wasInWaitForInviteState.current =
wasInWaitForInviteState.current ||
groupCallState.kind === "waitForInvite";
const knock =
groupCallState.kind === "canKnock" ? groupCallState.knock : null;
const label: string | JSX.Element =
groupCallState.kind === "canKnock" ? (
t("lobby.ask_to_join")
) : (
<>
{t("lobby.waiting_for_invite")}
<CheckIcon />
</>
);
case "waitForInvite":
case "canKnock": {
const knock =
groupCallState.kind === "canKnock" ? groupCallState.knock : null;
const label: string | JSX.Element =
groupCallState.kind === "canKnock" ? (
t("lobby.ask_to_join")
) : (
<>
{t("lobby.waiting_for_invite")}
<CheckIcon />
</>
);
return (
<LobbyView
client={client!}
matrixInfo={{
userId: client!.getUserId() ?? "",
displayName: userDisplayName ?? "",
avatarUrl: avatarUrl ?? "",
roomAlias: null,
roomId: groupCallState.roomSummary.room_id,
roomName: groupCallState.roomSummary.name ?? "",
roomAvatar: groupCallState.roomSummary.avatar_url ?? null,
e2eeSystem: {
kind: groupCallState.roomSummary[
"im.nheko.summary.encryption"
]
? E2eeType.PER_PARTICIPANT
: E2eeType.NONE,
},
}}
onEnter={(): void => knock?.()}
enterLabel={label}
waitingForInvite={groupCallState.kind === "waitForInvite"}
confineToRoom={confineToRoom}
hideHeader={hideHeader}
participantCount={null}
muteStates={muteStates}
onShareClick={null}
/>
);
}
default:
return <> </>;
return (
<LobbyView
client={client!}
matrixInfo={{
userId: client!.getUserId() ?? "",
displayName: userDisplayName ?? "",
avatarUrl: avatarUrl ?? "",
roomAlias: null,
roomId: groupCallState.roomSummary.room_id,
roomName: groupCallState.roomSummary.name ?? "",
roomAvatar: groupCallState.roomSummary.avatar_url ?? null,
e2eeSystem: {
kind: groupCallState.roomSummary["im.nheko.summary.encryption"]
? E2eeType.PER_PARTICIPANT
: E2eeType.NONE,
},
}}
onEnter={(): void => knock?.()}
enterLabel={label}
waitingForInvite={groupCallState.kind === "waitForInvite"}
confineToRoom={confineToRoom}
hideHeader={hideHeader}
participantCount={null}
muteStates={muteStates}
onShareClick={null}
/>
);
}
},
[
client,
passwordlessUser,
confineToRoom,
preload,
skipLobby,
hideHeader,
muteStates,
t,
userDisplayName,
avatarUrl,
],
);
case "loading":
return (
<FullScreenView>
<h1>{t("common.loading")}</h1>
</FullScreenView>
);
case "failed":
wasInWaitForInviteState.current = false;
if ((groupCallState.error as MatrixError).errcode === "M_NOT_FOUND") {
return (
<FullScreenView>
<Heading>{t("group_call_loader.failed_heading")}</Heading>
<Text>{t("group_call_loader.failed_text")}</Text>
{/* XXX: A 'create it for me' button would be the obvious UX here. Two screens already have
dupes of this flow, let's make a common component and put it here. */}
<Link to="/">{t("common.home")}</Link>
</FullScreenView>
);
} else if (groupCallState.error instanceof CallTerminatedMessage) {
return (
<FullScreenView>
<Heading>{groupCallState.error.message}</Heading>
<Text>{groupCallState.error.messageBody}</Text>
{groupCallState.error.reason && (
<>
{t("group_call_loader.reason")}:
<Text size="sm">"{groupCallState.error.reason}"</Text>
</>
)}
<Link to="/">{t("common.home")}</Link>
</FullScreenView>
);
} else {
return <ErrorView error={groupCallState.error} />;
}
default:
return <> </>;
}
};
let content: ReactNode;
if (loading || isRegistering) {
@@ -170,15 +203,7 @@ export const RoomPage: FC = () => {
// TODO: This doesn't belong here, the app routes need to be reworked
content = <HomePage />;
} else {
content = (
<GroupCallLoader
client={client}
roomIdOrAlias={roomIdOrAlias}
viaServers={viaServers}
>
{groupCallView}
</GroupCallLoader>
);
content = groupCallView();
}
return (

View File

@@ -117,8 +117,8 @@ export class CallTerminatedMessage extends Error {
}
export const useLoadGroupCall = (
client: MatrixClient,
roomIdOrAlias: string,
client: MatrixClient | undefined,
roomIdOrAlias: string | null,
viaServers: string[],
): GroupCallStatus => {
const [state, setState] = useState<GroupCallStatus>({ kind: "loading" });
@@ -159,6 +159,9 @@ export const useLoadGroupCall = (
?.getContent().reason;
useEffect(() => {
if (!client || !roomIdOrAlias) {
return;
}
const getRoomByAlias = async (alias: string): Promise<Room> => {
// We lowercase the localpart when we create the room, so we must lowercase
// it here too (we just do the whole alias). We can't do the same to room IDs