From b10863d582bde674e42723540594dea54209a6e1 Mon Sep 17 00:00:00 2001 From: Half-Shot Date: Mon, 9 Dec 2024 16:09:41 +0000 Subject: [PATCH] Support raising reactions for matrix rtc members. --- src/room/ReactionAudioRenderer.tsx | 3 +- src/state/CallViewModel.ts | 22 +++++++++- src/state/MediaViewModel.ts | 27 +++++++++++- src/tile/GridTile.tsx | 14 +++---- src/useReactions.tsx | 66 ++++++++++++++++++------------ 5 files changed, 93 insertions(+), 39 deletions(-) diff --git a/src/room/ReactionAudioRenderer.tsx b/src/room/ReactionAudioRenderer.tsx index 1d9f8daa..274c8c4c 100644 --- a/src/room/ReactionAudioRenderer.tsx +++ b/src/room/ReactionAudioRenderer.tsx @@ -5,9 +5,8 @@ SPDX-License-Identifier: AGPL-3.0-only Please see LICENSE in the repository root for full details. */ -import { ReactNode, useDeferredValue, useEffect, useState } from "react"; +import { ReactNode, useEffect, useState } from "react"; -import { useReactions } from "../useReactions"; import { playReactionsSound, useSetting } from "../settings/settings"; import { GenericReaction, ReactionSet } from "../reactions"; import { useAudioContext } from "../useAudioContext"; diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index df56fb00..ec2c9c4b 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -206,6 +206,10 @@ enum SortingBin { * Participants that have been speaking recently. */ Speakers, + /** + * Participants that have their hand raised. + */ + HandRaised, /** * Participants with video. */ @@ -241,6 +245,8 @@ class UserMedia { participant: LocalParticipant | RemoteParticipant | undefined, encryptionSystem: EncryptionSystem, livekitRoom: LivekitRoom, + handRaised: Observable, + reactions: Observable, ) { this.participant = new BehaviorSubject(participant); @@ -251,6 +257,8 @@ class UserMedia { this.participant.asObservable() as Observable, encryptionSystem, livekitRoom, + handRaised, + reactions, ); } else { this.vm = new RemoteUserMediaViewModel( @@ -261,6 +269,8 @@ class UserMedia { >, encryptionSystem, livekitRoom, + handRaised, + reactions, ); } @@ -468,6 +478,8 @@ export class CallViewModel extends ViewModel { let livekitParticipantId = rtcMember.sender + ":" + rtcMember.deviceId; + const matrixIdentifier = `${rtcMember.sender}:${rtcMember.deviceId}`; + let participant: | LocalParticipant | RemoteParticipant @@ -509,6 +521,12 @@ export class CallViewModel extends ViewModel { participant, this.encryptionSystem, this.livekitRoom, + this.handsRaised.pipe( + map((v) => v[matrixIdentifier] ?? undefined), + ), + this.reactions.pipe( + map((v) => v[matrixIdentifier] ?? undefined), + ), ), ]; @@ -618,12 +636,13 @@ export class CallViewModel extends ViewModel { [ m.speaker, m.presenter, + m.vm.handRaised, m.vm.videoEnabled, m.vm instanceof LocalUserMediaViewModel ? m.vm.alwaysShow : of(false), ], - (speaker, presenter, video, alwaysShow) => { + (speaker, presenter, handRaised, video, alwaysShow) => { let bin: SortingBin; if (m.vm.local) bin = alwaysShow @@ -631,6 +650,7 @@ export class CallViewModel extends ViewModel { : SortingBin.SelfNotAlwaysShown; else if (presenter) bin = SortingBin.Presenters; else if (speaker) bin = SortingBin.Speakers; + else if (handRaised) bin = SortingBin.HandRaised; else if (video) bin = SortingBin.Video; else bin = SortingBin.NoVideo; diff --git a/src/state/MediaViewModel.ts b/src/state/MediaViewModel.ts index ceaca57c..ab12b819 100644 --- a/src/state/MediaViewModel.ts +++ b/src/state/MediaViewModel.ts @@ -51,6 +51,7 @@ import { alwaysShowSelf } from "../settings/settings"; import { accumulate } from "../utils/observable"; import { EncryptionSystem } from "../e2ee/sharedKeyManagement"; import { E2eeType } from "../e2ee/e2eeType"; +import { ReactionOption } from "../reactions"; // TODO: Move this naming logic into the view model export function useDisplayName(vm: MediaViewModel): string { @@ -371,6 +372,8 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel { participant: Observable, encryptionSystem: EncryptionSystem, livekitRoom: LivekitRoom, + public readonly handRaised: Observable, + public readonly reactions: Observable, ) { super( id, @@ -437,8 +440,18 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel { participant: Observable, encryptionSystem: EncryptionSystem, livekitRoom: LivekitRoom, + handRaised: Observable, + reactions: Observable, ) { - super(id, member, participant, encryptionSystem, livekitRoom); + super( + id, + member, + participant, + encryptionSystem, + livekitRoom, + handRaised, + reactions, + ); } } @@ -498,8 +511,18 @@ export class RemoteUserMediaViewModel extends BaseUserMediaViewModel { participant: Observable, encryptionSystem: EncryptionSystem, livekitRoom: LivekitRoom, + handRaised: Observable, + reactions: Observable, ) { - super(id, member, participant, encryptionSystem, livekitRoom); + super( + id, + member, + participant, + encryptionSystem, + livekitRoom, + handRaised, + reactions, + ); // Sync the local volume with LiveKit combineLatest([ diff --git a/src/tile/GridTile.tsx b/src/tile/GridTile.tsx index 15f7c295..59263516 100644 --- a/src/tile/GridTile.tsx +++ b/src/tile/GridTile.tsx @@ -34,7 +34,7 @@ import { ToggleMenuItem, Menu, } from "@vector-im/compound-web"; -import { useObservableEagerState } from "observable-hooks"; +import { useObservableEagerState, useObservableState } from "observable-hooks"; import styles from "./GridTile.module.css"; import { @@ -49,7 +49,6 @@ import { useLatest } from "../useLatest"; import { GridTileViewModel } from "../state/TileViewModel"; import { useMergedRefs } from "../useMergedRefs"; import { useReactions } from "../useReactions"; -import { ReactionOption } from "../reactions"; interface TileProps { className?: string; @@ -82,6 +81,7 @@ const UserMediaTile = forwardRef( }, ref, ) => { + const { toggleRaisedHand } = useReactions(); const { t } = useTranslation(); const video = useObservableEagerState(vm.video); const unencryptedWarning = useObservableEagerState(vm.unencryptedWarning); @@ -97,7 +97,8 @@ const UserMediaTile = forwardRef( }, [vm], ); - const { raisedHands, toggleRaisedHand, reactions } = useReactions(); + const handRaised = useObservableState(vm.handRaised); + const reaction = useObservableState(vm.reactions); const AudioIcon = locallyMuted ? VolumeOffSolidIcon @@ -124,9 +125,6 @@ const UserMediaTile = forwardRef( ); - const handRaised: Date | undefined = raisedHands[vm.member?.userId ?? ""]; - const currentReaction: ReactionOption | undefined = - reactions[vm.member?.userId ?? ""]; const raisedHandOnClick = vm.local ? (): void => void toggleRaisedHand() : undefined; @@ -144,7 +142,7 @@ const UserMediaTile = forwardRef( videoFit={cropVideo ? "cover" : "contain"} className={classNames(className, styles.tile, { [styles.speaking]: showSpeaking, - [styles.handRaised]: !showSpeaking && !!handRaised, + [styles.handRaised]: !showSpeaking && handRaised, })} nameTagLeadingIcon={ ( } raisedHandTime={handRaised} - currentReaction={currentReaction} + currentReaction={reaction} raisedHandOnClick={raisedHandOnClick} localParticipant={vm.local} {...props} diff --git a/src/useReactions.tsx b/src/useReactions.tsx index 7195cfd0..79898b93 100644 --- a/src/useReactions.tsx +++ b/src/useReactions.tsx @@ -140,10 +140,10 @@ export const ReactionsProvider = ({ }; // Remove any raised hands for users no longer joined to the call. - for (const userId of Object.keys(raisedHands).filter( + for (const identifier of Object.keys(raisedHands).filter( (rhId) => !memberships.find((u) => u.sender == rhId), )) { - removeRaisedHand(userId); + removeRaisedHand(identifier); } // For each member in the call, check to see if a reaction has @@ -152,13 +152,14 @@ export const ReactionsProvider = ({ if (!m.sender || !m.eventId) { continue; } + const identifier = `${m.sender}:${m.deviceId}`; if ( - raisedHands[m.sender] && - raisedHands[m.sender].membershipEventId !== m.eventId + raisedHands[identifier] && + raisedHands[identifier].membershipEventId !== m.eventId ) { // Membership event for sender has changed since the hand // was raised, reset. - removeRaisedHand(m.sender); + removeRaisedHand(identifier); } const reaction = getLastReactionEvent(m.eventId, m.sender); if (reaction) { @@ -166,7 +167,7 @@ export const ReactionsProvider = ({ if (!eventId) { continue; } - addRaisedHand(m.sender, { + addRaisedHand(`${m.sender}:${m.deviceId}`, { membershipEventId: m.eventId, reactionEventId: eventId, time: new Date(reaction.localTimestamp), @@ -181,11 +182,16 @@ export const ReactionsProvider = ({ const latestMemberships = useLatest(memberships); const latestRaisedHands = useLatest(raisedHands); - const myMembership = useMemo( + const myMembershipEvent = useMemo( () => memberships.find((m) => m.sender === myUserId)?.eventId, [memberships, myUserId], ); - + const myMembershipIdentifier = useMemo(() => { + const membership = memberships.find((m) => m.sender === myUserId); + return membership + ? `${membership.sender}:${membership.deviceId}` + : undefined; + }, [memberships, myUserId]); // This effect handles any *live* reaction/redactions in the room. useEffect(() => { const reactionTimeouts = new Set(); @@ -271,11 +277,10 @@ export const ReactionsProvider = ({ // Check to see if this reaction was made to a membership event (and the // sender of the reaction matches the membership) - if ( - !latestMemberships.current.some( - (e) => e.eventId === membershipEventId && e.sender === sender, - ) - ) { + const membershipEvent = latestMemberships.current.find( + (e) => e.eventId === membershipEventId && e.sender === sender, + ); + if (!membershipEvent) { logger.warn( `Reaction target was not a membership event for ${sender}, ignoring`, ); @@ -283,11 +288,14 @@ export const ReactionsProvider = ({ } if (content?.["m.relates_to"].key === "🖐️") { - addRaisedHand(sender, { - reactionEventId, - membershipEventId, - time: new Date(event.localTimestamp), - }); + addRaisedHand( + `${membershipEvent.sender}:${membershipEvent.deviceId}`, + { + reactionEventId, + membershipEventId, + time: new Date(event.localTimestamp), + }, + ); } } else if (event.getType() === EventType.RoomRedaction) { const targetEvent = event.event.redacts; @@ -328,14 +336,14 @@ export const ReactionsProvider = ({ ]); const toggleRaisedHand = useCallback(async () => { - if (!myUserId) { + if (!myMembershipIdentifier) { return; } - const myReactionId = raisedHands[myUserId]?.reactionEventId; + const myReactionId = raisedHands[myMembershipIdentifier]?.reactionEventId; if (!myReactionId) { try { - if (!myMembership) { + if (!myMembershipEvent) { throw new Error("Cannot find own membership event"); } const reaction = await room.client.sendEvent( @@ -344,7 +352,7 @@ export const ReactionsProvider = ({ { "m.relates_to": { rel_type: RelationType.Annotation, - event_id: myMembership, + event_id: myMembershipEvent, key: "🖐️", }, }, @@ -362,7 +370,13 @@ export const ReactionsProvider = ({ throw ex; } } - }, [myMembership, myUserId, raisedHands, rtcSession, room]); + }, [ + myMembershipEvent, + myMembershipIdentifier, + raisedHands, + rtcSession, + room, + ]); const sendReaction = useCallback( async (reaction: ReactionOption) => { @@ -370,7 +384,7 @@ export const ReactionsProvider = ({ // We're still reacting return; } - if (!myMembership) { + if (!myMembershipEvent) { throw new Error("Cannot find own membership event"); } await room.client.sendEvent( @@ -379,14 +393,14 @@ export const ReactionsProvider = ({ { "m.relates_to": { rel_type: RelationType.Reference, - event_id: myMembership, + event_id: myMembershipEvent, }, emoji: reaction.emoji, name: reaction.name, }, ); }, - [myMembership, reactions, room, myUserId, rtcSession], + [myMembershipEvent, reactions, room, myUserId, rtcSession], ); return (