Merge branch 'livekit' into toger5/tiles_based_on_rtc_member

This commit is contained in:
Hugh Nimmo-Smith
2024-11-11 10:41:11 +00:00
69 changed files with 2890 additions and 655 deletions

View File

@@ -36,12 +36,14 @@ Please see LICENSE in the repository root for full details.
inset-block-end: 0;
z-index: 1;
display: grid;
grid-template-columns: 1fr auto 1fr;
grid-template-areas: "logo buttons layout";
grid-template-columns: minmax(0, var(--inline-content-inset)) 1fr auto 1fr minmax(
0,
var(--inline-content-inset)
);
grid-template-areas: ". logo buttons layout .";
align-items: center;
gap: var(--cpd-space-3x);
padding-block: var(--cpd-space-4x);
padding-inline: var(--inline-content-inset);
padding-block: var(--cpd-space-10x);
background: linear-gradient(
180deg,
rgba(0, 0, 0, 0) 0%,
@@ -64,7 +66,6 @@ Please see LICENSE in the repository root for full details.
.footer.overlay.hidden {
display: grid;
opacity: 0;
pointer-events: none;
}
.footer.overlay:has(:focus-visible) {
@@ -83,6 +84,7 @@ Please see LICENSE in the repository root for full details.
.buttons {
grid-area: buttons;
justify-self: center;
display: flex;
gap: var(--cpd-space-3x);
}
@@ -92,15 +94,49 @@ Please see LICENSE in the repository root for full details.
justify-self: end;
}
@media (min-height: 400px) {
@media (max-width: 660px) {
.footer {
padding-block: var(--cpd-space-8x);
grid-template-areas: ". buttons buttons buttons .";
}
.logo {
display: none;
}
.layout {
display: none !important;
}
}
@media (min-height: 800px) {
@media (max-width: 370px) {
.raiseHand {
display: none;
}
}
@media (max-width: 340px) {
.invite,
.switchCamera,
.shareScreen {
display: none;
}
@media (max-height: 400px) {
.footer {
display: none;
}
}
}
@media (max-height: 400px) {
.footer {
padding-block: var(--cpd-space-10x);
padding-block: var(--cpd-space-4x);
}
}
@media (max-height: 800px) {
.footer {
padding-block: var(--cpd-space-8x);
}
}
@@ -144,3 +180,48 @@ Please see LICENSE in the repository root for full details.
position: relative;
flex-grow: 1;
}
.floatingReaction {
position: relative;
display: inline;
z-index: 2;
font-size: 32pt;
/* Reactions are "active" for 3 seconds (as per REACTION_ACTIVE_TIME_MS), give a bit more time for it to fade out. */
animation-duration: 4s;
animation-name: reaction-up;
width: fit-content;
pointer-events: none;
}
@keyframes reaction-up {
from {
opacity: 1;
translate: 100vw 0;
scale: 200%;
}
to {
opacity: 0;
translate: 100vw -100vh;
scale: 100%;
}
}
@media (prefers-reduced-motion) {
@keyframes reaction-up-reduced {
from {
opacity: 1;
}
to {
opacity: 0;
}
}
.floatingReaction {
font-size: 48pt;
animation-name: reaction-up-reduced;
top: calc(-50vh + (48pt / 2));
left: calc(50vw - (48pt / 2)) !important;
}
}

View File

@@ -41,7 +41,7 @@ import {
VideoButton,
ShareScreenButton,
SettingsButton,
RaiseHandToggleButton,
ReactionToggleButton,
SwitchCameraButton,
} from "../button";
import { Header, LeftNav, RightNav, RoomHeaderInfo } from "../Header";
@@ -83,7 +83,13 @@ import { GridTileViewModel, TileViewModel } from "../state/TileViewModel";
import { ReactionsProvider, useReactions } from "../useReactions";
import handSoundOgg from "../sound/raise_hand.ogg?url";
import handSoundMp3 from "../sound/raise_hand.mp3?url";
import { ReactionsAudioRenderer } from "./ReactionAudioRenderer";
import { useSwitchCamera } from "./useSwitchCamera";
import {
soundEffectVolumeSetting,
showReactions,
useSetting,
} from "../settings/settings";
const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {});
@@ -174,13 +180,27 @@ export const InCallView: FC<InCallViewProps> = ({
connState,
onShareClick,
}) => {
const { supportsReactions, raisedHands } = useReactions();
const [shouldShowReactions] = useSetting(showReactions);
const [soundEffectVolume] = useSetting(soundEffectVolumeSetting);
const { supportsReactions, raisedHands, reactions } = useReactions();
const raisedHandCount = useMemo(
() => Object.keys(raisedHands).length,
[raisedHands],
);
const previousRaisedHandCount = useDeferredValue(raisedHandCount);
const reactionsIcons = useMemo(
() =>
shouldShowReactions
? Object.entries(reactions).map(([sender, { emoji }]) => ({
sender,
emoji,
startX: -Math.ceil(Math.random() * 50) - 25,
}))
: [],
[shouldShowReactions, reactions],
);
useWakeLock();
useEffect(() => {
@@ -194,7 +214,6 @@ export const InCallView: FC<InCallViewProps> = ({
const nonMemberItemCount = useObservableEagerState(vm.nonMemberItemCount);
const containerRef1 = useRef<HTMLDivElement | null>(null);
const [containerRef2, bounds] = useMeasure();
const boundsValid = bounds.height > 0;
// Merge the refs so they can attach to the same element
const containerRef = useMergedRefs(containerRef1, containerRef2);
@@ -222,10 +241,6 @@ export const InCallView: FC<InCallViewProps> = ({
(muted) => muteStates.audio.setEnabled?.(!muted),
);
const mobile = boundsValid && bounds.width <= 660;
const reducedControls = boundsValid && bounds.width <= 340;
const noControls = reducedControls && bounds.height <= 400;
const windowMode = useObservableEagerState(vm.windowMode);
const layout = useObservableEagerState(vm.layout);
const gridMode = useObservableEagerState(vm.gridMode);
@@ -247,12 +262,22 @@ export const InCallView: FC<InCallViewProps> = ({
}, [vm]);
const onTouchCancel = useCallback(() => (touchStart.current = null), []);
// We also need to tell the layout toggle to prevent touch events from
// bubbling up, or else the controls will be dismissed before a change event
// can be registered on the toggle
const onLayoutToggleTouchEnd = useCallback(
(e: TouchEvent) => e.stopPropagation(),
[],
// We also need to tell the footer controls to prevent touch events from
// bubbling up, or else the footer will be dismissed before a click/change
// event can be registered on the control
const onControlsTouchEnd = useCallback(
(e: TouchEvent) => {
// Somehow applying pointer-events: none to the controls when the footer
// is hidden is not enough to stop clicks from happening as the footer
// becomes visible, so we check manually whether the footer is shown
if (showFooter) {
e.stopPropagation();
vm.tapControls();
} else {
e.preventDefault();
}
},
[vm, showFooter],
);
const onPointerMove = useCallback(
@@ -330,11 +355,17 @@ export const InCallView: FC<InCallViewProps> = ({
return;
}
if (previousRaisedHandCount < raisedHandCount) {
handRaisePlayer.current.volume = soundEffectVolume;
handRaisePlayer.current.play().catch((ex) => {
logger.warn("Failed to play raise hand sound", ex);
});
}
}, [raisedHandCount, handRaisePlayer, previousRaisedHandCount]);
}, [
raisedHandCount,
handRaisePlayer,
previousRaisedHandCount,
soundEffectVolume,
]);
useEffect(() => {
widget?.api.transport
@@ -507,95 +538,106 @@ export const InCallView: FC<InCallViewProps> = ({
.catch(logger.error);
}, [localParticipant, isScreenShareEnabled]);
let footer: JSX.Element | null;
if (noControls) {
footer = null;
} else {
const buttons: JSX.Element[] = [];
const buttons: JSX.Element[] = [];
buttons.push(
<MicButton
key="audio"
muted={!muteStates.audio.enabled}
onClick={toggleMicrophone}
onTouchEnd={onControlsTouchEnd}
disabled={muteStates.audio.setEnabled === null}
data-testid="incall_mute"
/>,
<VideoButton
key="video"
muted={!muteStates.video.enabled}
onClick={toggleCamera}
onTouchEnd={onControlsTouchEnd}
disabled={muteStates.video.setEnabled === null}
data-testid="incall_videomute"
/>,
);
if (switchCamera !== null)
buttons.push(
<MicButton
key="audio"
muted={!muteStates.audio.enabled}
onClick={toggleMicrophone}
disabled={muteStates.audio.setEnabled === null}
data-testid="incall_mute"
/>,
<VideoButton
key="video"
muted={!muteStates.video.enabled}
onClick={toggleCamera}
disabled={muteStates.video.setEnabled === null}
data-testid="incall_videomute"
<SwitchCameraButton
key="switch_camera"
className={styles.switchCamera}
onClick={switchCamera}
onTouchEnd={onControlsTouchEnd}
/>,
);
if (!reducedControls) {
if (switchCamera !== null)
buttons.push(
<SwitchCameraButton key="switch_camera" onClick={switchCamera} />,
);
if (canScreenshare && !hideScreensharing) {
buttons.push(
<ShareScreenButton
key="share_screen"
enabled={isScreenShareEnabled}
onClick={toggleScreensharing}
data-testid="incall_screenshare"
/>,
);
}
if (supportsReactions) {
buttons.push(
<RaiseHandToggleButton
client={client}
rtcSession={rtcSession}
key="4"
/>,
);
}
buttons.push(<SettingsButton key="settings" onClick={openSettings} />);
}
if (canScreenshare && !hideScreensharing) {
buttons.push(
<EndCallButton
key="end_call"
onClick={function (): void {
onLeave();
}}
data-testid="incall_leave"
<ShareScreenButton
key="share_screen"
className={styles.shareScreen}
enabled={isScreenShareEnabled}
onClick={toggleScreensharing}
onTouchEnd={onControlsTouchEnd}
data-testid="incall_screenshare"
/>,
);
footer = (
<div
ref={footerRef}
className={classNames(styles.footer, {
[styles.overlay]: windowMode === "flat",
[styles.hidden]: !showFooter || (!showControls && hideHeader),
})}
>
{!mobile && !hideHeader && (
<div className={styles.logo}>
<LogoMark width={24} height={24} aria-hidden />
<LogoType
width={80}
height={11}
aria-label={import.meta.env.VITE_PRODUCT_NAME || "Element Call"}
/>
</div>
)}
{showControls && <div className={styles.buttons}>{buttons}</div>}
{!mobile && showControls && (
<LayoutToggle
className={styles.layout}
layout={gridMode}
setLayout={setGridMode}
onTouchEnd={onLayoutToggleTouchEnd}
/>
)}
</div>
);
}
if (supportsReactions) {
buttons.push(
<ReactionToggleButton
key="raise_hand"
className={styles.raiseHand}
client={client}
rtcSession={rtcSession}
onTouchEnd={onControlsTouchEnd}
/>,
);
}
if (layout.type !== "pip")
buttons.push(
<SettingsButton
key="settings"
onClick={openSettings}
onTouchEnd={onControlsTouchEnd}
/>,
);
buttons.push(
<EndCallButton
key="end_call"
onClick={function (): void {
onLeave();
}}
onTouchEnd={onControlsTouchEnd}
data-testid="incall_leave"
/>,
);
const footer = (
<div
ref={footerRef}
className={classNames(styles.footer, {
[styles.overlay]: windowMode === "flat",
[styles.hidden]: !showFooter || (!showControls && hideHeader),
})}
>
{!hideHeader && (
<div className={styles.logo}>
<LogoMark width={24} height={24} aria-hidden />
<LogoType
width={80}
height={11}
aria-label={import.meta.env.VITE_PRODUCT_NAME || "Element Call"}
/>
</div>
)}
{showControls && <div className={styles.buttons}>{buttons}</div>}
{showControls && (
<LayoutToggle
className={styles.layout}
layout={gridMode}
setLayout={setGridMode}
onTouchEnd={onControlsTouchEnd}
/>
)}
</div>
);
return (
<div
@@ -628,28 +670,45 @@ export const InCallView: FC<InCallViewProps> = ({
/>
</LeftNav>
<RightNav>
{!reducedControls && showControls && onShareClick !== null && (
<InviteButton onClick={onShareClick} />
{showControls && onShareClick !== null && (
<InviteButton
className={styles.invite}
onClick={onShareClick}
/>
)}
</RightNav>
</Header>
))}
<RoomAudioRenderer />
{renderContent()}
<audio ref={handRaisePlayer} hidden>
<audio ref={handRaisePlayer} preload="auto" hidden>
<source src={handSoundOgg} type="audio/ogg; codecs=vorbis" />
<source src={handSoundMp3} type="audio/mpeg" />
</audio>
<ReactionsAudioRenderer />
{reactionsIcons.map(({ sender, emoji, startX }) => (
<span
style={{ left: `${startX}vw` }}
className={styles.floatingReaction}
key={sender}
>
{emoji}
</span>
))}
{footer}
{!noControls && <RageshakeRequestModal {...rageshakeRequestModalProps} />}
<SettingsModal
client={client}
roomId={rtcSession.room.roomId}
open={settingsModalOpen}
onDismiss={closeSettings}
tab={settingsTab}
onTabChange={setSettingsTab}
/>
{layout.type !== "pip" && (
<>
<RageshakeRequestModal {...rageshakeRequestModalProps} />
<SettingsModal
client={client}
roomId={rtcSession.room.roomId}
open={settingsModalOpen}
onDismiss={closeSettings}
tab={settingsTab}
onTabChange={setSettingsTab}
/>
</>
)}
</div>
);
};

View File

@@ -0,0 +1,181 @@
/*
Copyright 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { render } from "@testing-library/react";
import { afterAll, expect, test } from "vitest";
import { TooltipProvider } from "@vector-im/compound-web";
import { act, ReactNode } from "react";
import {
MockRoom,
MockRTCSession,
TestReactionsWrapper,
} from "../utils/testReactions";
import { ReactionsAudioRenderer } from "./ReactionAudioRenderer";
import { GenericReaction, ReactionSet } from "../reactions";
import {
playReactionsSound,
soundEffectVolumeSetting,
} from "../settings/settings";
const memberUserIdAlice = "@alice:example.org";
const memberUserIdBob = "@bob:example.org";
const memberUserIdCharlie = "@charlie:example.org";
const memberEventAlice = "$membership-alice:example.org";
const memberEventBob = "$membership-bob:example.org";
const memberEventCharlie = "$membership-charlie:example.org";
const membership: Record<string, string> = {
[memberEventAlice]: memberUserIdAlice,
[memberEventBob]: memberUserIdBob,
[memberEventCharlie]: memberUserIdCharlie,
};
function TestComponent({
rtcSession,
}: {
rtcSession: MockRTCSession;
}): ReactNode {
return (
<TooltipProvider>
<TestReactionsWrapper rtcSession={rtcSession}>
<ReactionsAudioRenderer />
</TestReactionsWrapper>
</TooltipProvider>
);
}
const originalPlayFn = window.HTMLMediaElement.prototype.play;
afterAll(() => {
playReactionsSound.setValue(playReactionsSound.defaultValue);
soundEffectVolumeSetting.setValue(soundEffectVolumeSetting.defaultValue);
window.HTMLMediaElement.prototype.play = originalPlayFn;
});
test("preloads all audio elements", () => {
playReactionsSound.setValue(true);
const rtcSession = new MockRTCSession(
new MockRoom(memberUserIdAlice),
membership,
);
const { container } = render(<TestComponent rtcSession={rtcSession} />);
expect(container.getElementsByTagName("audio")).toHaveLength(
// All reactions plus the generic sound
ReactionSet.filter((r) => r.sound).length + 1,
);
});
test("loads no audio elements when disabled in settings", () => {
playReactionsSound.setValue(false);
const rtcSession = new MockRTCSession(
new MockRoom(memberUserIdAlice),
membership,
);
const { container } = render(<TestComponent rtcSession={rtcSession} />);
expect(container.getElementsByTagName("audio")).toHaveLength(0);
});
test("will play an audio sound when there is a reaction", () => {
const audioIsPlaying: string[] = [];
window.HTMLMediaElement.prototype.play = async function (): Promise<void> {
audioIsPlaying.push((this.children[0] as HTMLSourceElement).src);
return Promise.resolve();
};
playReactionsSound.setValue(true);
const room = new MockRoom(memberUserIdAlice);
const rtcSession = new MockRTCSession(room, membership);
render(<TestComponent rtcSession={rtcSession} />);
// Find the first reaction with a sound effect
const chosenReaction = ReactionSet.find((r) => !!r.sound);
if (!chosenReaction) {
throw Error(
"No reactions have sounds configured, this test cannot succeed",
);
}
act(() => {
room.testSendReaction(memberEventAlice, chosenReaction, membership);
});
expect(audioIsPlaying).toHaveLength(1);
expect(audioIsPlaying[0]).toContain(chosenReaction.sound?.ogg);
});
test("will play the generic audio sound when there is soundless reaction", () => {
const audioIsPlaying: string[] = [];
window.HTMLMediaElement.prototype.play = async function (): Promise<void> {
audioIsPlaying.push((this.children[0] as HTMLSourceElement).src);
return Promise.resolve();
};
playReactionsSound.setValue(true);
const room = new MockRoom(memberUserIdAlice);
const rtcSession = new MockRTCSession(room, membership);
render(<TestComponent rtcSession={rtcSession} />);
// Find the first reaction with a sound effect
const chosenReaction = ReactionSet.find((r) => !r.sound);
if (!chosenReaction) {
throw Error(
"No reactions have sounds configured, this test cannot succeed",
);
}
act(() => {
room.testSendReaction(memberEventAlice, chosenReaction, membership);
});
expect(audioIsPlaying).toHaveLength(1);
expect(audioIsPlaying[0]).toContain(GenericReaction.sound?.ogg);
});
test("will play an audio sound with the correct volume", () => {
playReactionsSound.setValue(true);
soundEffectVolumeSetting.setValue(0.5);
const room = new MockRoom(memberUserIdAlice);
const rtcSession = new MockRTCSession(room, membership);
const { getByTestId } = render(<TestComponent rtcSession={rtcSession} />);
// Find the first reaction with a sound effect
const chosenReaction = ReactionSet.find((r) => !!r.sound);
if (!chosenReaction) {
throw Error(
"No reactions have sounds configured, this test cannot succeed",
);
}
act(() => {
room.testSendReaction(memberEventAlice, chosenReaction, membership);
});
expect((getByTestId(chosenReaction.name) as HTMLAudioElement).volume).toEqual(
0.5,
);
});
test("will play multiple audio sounds when there are multiple different reactions", () => {
const audioIsPlaying: string[] = [];
window.HTMLMediaElement.prototype.play = async function (): Promise<void> {
audioIsPlaying.push((this.children[0] as HTMLSourceElement).src);
return Promise.resolve();
};
playReactionsSound.setValue(true);
const room = new MockRoom(memberUserIdAlice);
const rtcSession = new MockRTCSession(room, membership);
render(<TestComponent rtcSession={rtcSession} />);
// Find the first reaction with a sound effect
const [reaction1, reaction2] = ReactionSet.filter((r) => !!r.sound);
if (!reaction1 || !reaction2) {
throw Error(
"No reactions have sounds configured, this test cannot succeed",
);
}
act(() => {
room.testSendReaction(memberEventAlice, reaction1, membership);
room.testSendReaction(memberEventBob, reaction2, membership);
room.testSendReaction(memberEventCharlie, reaction1, membership);
});
expect(audioIsPlaying).toHaveLength(2);
expect(audioIsPlaying[0]).toContain(reaction1.sound?.ogg);
expect(audioIsPlaying[1]).toContain(reaction2.sound?.ogg);
});

View File

@@ -0,0 +1,74 @@
/*
Copyright 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { ReactNode, useEffect, useRef } from "react";
import { useReactions } from "../useReactions";
import {
playReactionsSound,
soundEffectVolumeSetting as effectSoundVolumeSetting,
useSetting,
} from "../settings/settings";
import { GenericReaction, ReactionSet } from "../reactions";
export function ReactionsAudioRenderer(): ReactNode {
const { reactions } = useReactions();
const [shouldPlay] = useSetting(playReactionsSound);
const [effectSoundVolume] = useSetting(effectSoundVolumeSetting);
const audioElements = useRef<Record<string, HTMLAudioElement | null>>({});
useEffect(() => {
if (!audioElements.current) {
return;
}
if (!shouldPlay) {
return;
}
for (const reactionName of new Set(
Object.values(reactions).map((r) => r.name),
)) {
const audioElement =
audioElements.current[reactionName] ?? audioElements.current.generic;
if (audioElement?.paused) {
audioElement.volume = effectSoundVolume;
void audioElement.play();
}
}
}, [audioElements, shouldPlay, reactions, effectSoundVolume]);
// Do not render any audio elements if playback is disabled. Will save
// audio file fetches.
if (!shouldPlay) {
return null;
}
// NOTE: We load all audio elements ahead of time to allow the cache
// to be populated, rather than risk a cache miss and have the audio
// be delayed.
return (
<>
{[GenericReaction, ...ReactionSet].map(
(r) =>
r.sound && (
<audio
ref={(el) => (audioElements.current[r.name] = el)}
data-testid={r.name}
key={r.name}
preload="auto"
hidden
>
<source src={r.sound.ogg} type="audio/ogg; codecs=vorbis" />
{r.sound.mp3 ? (
<source src={r.sound.mp3} type="audio/mpeg" />
) : null}
</audio>
),
)}
</>
);
}

View File

@@ -66,6 +66,7 @@ video.mirror {
margin-inline: 0;
border-radius: 0;
block-size: 100%;
width: 100%;
}
.buttonBar {