Support raising reactions for matrix rtc members.

src/room/ReactionAudioRenderer.tsx

@@ -5,9 +5,8 @@ SPDX-License-Identifier: AGPL-3.0-only
 Please see LICENSE in the repository root for full details.
 */
 
-import { ReactNode, useDeferredValue, useEffect, useState } from "react";
+import { ReactNode, useEffect, useState } from "react";
 
-import { useReactions } from "../useReactions";
 import { playReactionsSound, useSetting } from "../settings/settings";
 import { GenericReaction, ReactionSet } from "../reactions";
 import { useAudioContext } from "../useAudioContext";

src/state/CallViewModel.ts

@@ -206,6 +206,10 @@ enum SortingBin {
    * Participants that have been speaking recently.
    */
   Speakers,
+  /**
+   * Participants that have their hand raised.
+   */
+  HandRaised,
   /**
    * Participants with video.
    */
@@ -241,6 +245,8 @@ class UserMedia {
     participant: LocalParticipant | RemoteParticipant | undefined,
     encryptionSystem: EncryptionSystem,
     livekitRoom: LivekitRoom,
+    handRaised: Observable<Date | undefined>,
+    reactions: Observable<ReactionOption | undefined>,
   ) {
     this.participant = new BehaviorSubject(participant);
 
@@ -251,6 +257,8 @@ class UserMedia {
         this.participant.asObservable() as Observable<LocalParticipant>,
         encryptionSystem,
         livekitRoom,
+        handRaised,
+        reactions,
       );
     } else {
       this.vm = new RemoteUserMediaViewModel(
@@ -261,6 +269,8 @@ class UserMedia {
         >,
         encryptionSystem,
         livekitRoom,
+        handRaised,
+        reactions,
       );
     }
 
@@ -468,6 +478,8 @@ export class CallViewModel extends ViewModel {
               let livekitParticipantId =
                 rtcMember.sender + ":" + rtcMember.deviceId;
 
+              const matrixIdentifier = `${rtcMember.sender}:${rtcMember.deviceId}`;
+
               let participant:
                 | LocalParticipant
                 | RemoteParticipant
@@ -509,6 +521,12 @@ export class CallViewModel extends ViewModel {
                       participant,
                       this.encryptionSystem,
                       this.livekitRoom,
+                      this.handsRaised.pipe(
+                        map((v) => v[matrixIdentifier] ?? undefined),
+                      ),
+                      this.reactions.pipe(
+                        map((v) => v[matrixIdentifier] ?? undefined),
+                      ),
                     ),
                 ];
 
@@ -618,12 +636,13 @@ export class CallViewModel extends ViewModel {
           [
             m.speaker,
             m.presenter,
+            m.vm.handRaised,
             m.vm.videoEnabled,
             m.vm instanceof LocalUserMediaViewModel
               ? m.vm.alwaysShow
               : of(false),
           ],
-          (speaker, presenter, video, alwaysShow) => {
+          (speaker, presenter, handRaised, video, alwaysShow) => {
             let bin: SortingBin;
             if (m.vm.local)
               bin = alwaysShow
@@ -631,6 +650,7 @@ export class CallViewModel extends ViewModel {
                 : SortingBin.SelfNotAlwaysShown;
             else if (presenter) bin = SortingBin.Presenters;
             else if (speaker) bin = SortingBin.Speakers;
+            else if (handRaised) bin = SortingBin.HandRaised;
             else if (video) bin = SortingBin.Video;
             else bin = SortingBin.NoVideo;
 

src/state/MediaViewModel.ts

@@ -51,6 +51,7 @@ import { alwaysShowSelf } from "../settings/settings";
 import { accumulate } from "../utils/observable";
 import { EncryptionSystem } from "../e2ee/sharedKeyManagement";
 import { E2eeType } from "../e2ee/e2eeType";
+import { ReactionOption } from "../reactions";
 
 // TODO: Move this naming logic into the view model
 export function useDisplayName(vm: MediaViewModel): string {
@@ -371,6 +372,8 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel {
     participant: Observable<LocalParticipant | RemoteParticipant | undefined>,
     encryptionSystem: EncryptionSystem,
     livekitRoom: LivekitRoom,
+    public readonly handRaised: Observable<Date | undefined>,
+    public readonly reactions: Observable<ReactionOption | undefined>,
   ) {
     super(
       id,
@@ -437,8 +440,18 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel {
     participant: Observable<LocalParticipant | undefined>,
     encryptionSystem: EncryptionSystem,
     livekitRoom: LivekitRoom,
+    handRaised: Observable<Date | undefined>,
+    reactions: Observable<ReactionOption | undefined>,
   ) {
-    super(id, member, participant, encryptionSystem, livekitRoom);
+    super(
+      id,
+      member,
+      participant,
+      encryptionSystem,
+      livekitRoom,
+      handRaised,
+      reactions,
+    );
   }
 }
 
@@ -498,8 +511,18 @@ export class RemoteUserMediaViewModel extends BaseUserMediaViewModel {
     participant: Observable<RemoteParticipant | undefined>,
     encryptionSystem: EncryptionSystem,
     livekitRoom: LivekitRoom,
+    handRaised: Observable<Date | undefined>,
+    reactions: Observable<ReactionOption | undefined>,
   ) {
-    super(id, member, participant, encryptionSystem, livekitRoom);
+    super(
+      id,
+      member,
+      participant,
+      encryptionSystem,
+      livekitRoom,
+      handRaised,
+      reactions,
+    );
 
     // Sync the local volume with LiveKit
     combineLatest([

src/tile/GridTile.tsx

@@ -34,7 +34,7 @@ import {
   ToggleMenuItem,
   Menu,
 } from "@vector-im/compound-web";
-import { useObservableEagerState } from "observable-hooks";
+import { useObservableEagerState, useObservableState } from "observable-hooks";
 
 import styles from "./GridTile.module.css";
 import {
@@ -49,7 +49,6 @@ import { useLatest } from "../useLatest";
 import { GridTileViewModel } from "../state/TileViewModel";
 import { useMergedRefs } from "../useMergedRefs";
 import { useReactions } from "../useReactions";
-import { ReactionOption } from "../reactions";
 
 interface TileProps {
   className?: string;
@@ -82,6 +81,7 @@ const UserMediaTile = forwardRef<HTMLDivElement, UserMediaTileProps>(
     },
     ref,
   ) => {
+    const { toggleRaisedHand } = useReactions();
     const { t } = useTranslation();
     const video = useObservableEagerState(vm.video);
     const unencryptedWarning = useObservableEagerState(vm.unencryptedWarning);
@@ -97,7 +97,8 @@ const UserMediaTile = forwardRef<HTMLDivElement, UserMediaTileProps>(
       },
       [vm],
     );
-    const { raisedHands, toggleRaisedHand, reactions } = useReactions();
+    const handRaised = useObservableState(vm.handRaised);
+    const reaction = useObservableState(vm.reactions);
 
     const AudioIcon = locallyMuted
       ? VolumeOffSolidIcon
@@ -124,9 +125,6 @@ const UserMediaTile = forwardRef<HTMLDivElement, UserMediaTileProps>(
       </>
     );
 
-    const handRaised: Date | undefined = raisedHands[vm.member?.userId ?? ""];
-    const currentReaction: ReactionOption | undefined =
-      reactions[vm.member?.userId ?? ""];
     const raisedHandOnClick = vm.local
       ? (): void => void toggleRaisedHand()
       : undefined;
@@ -144,7 +142,7 @@ const UserMediaTile = forwardRef<HTMLDivElement, UserMediaTileProps>(
         videoFit={cropVideo ? "cover" : "contain"}
         className={classNames(className, styles.tile, {
           [styles.speaking]: showSpeaking,
-          [styles.handRaised]: !showSpeaking && !!handRaised,
+          [styles.handRaised]: !showSpeaking && handRaised,
         })}
         nameTagLeadingIcon={
           <AudioIcon
@@ -173,7 +171,7 @@ const UserMediaTile = forwardRef<HTMLDivElement, UserMediaTileProps>(
           </Menu>
         }
         raisedHandTime={handRaised}
-        currentReaction={currentReaction}
+        currentReaction={reaction}
         raisedHandOnClick={raisedHandOnClick}
         localParticipant={vm.local}
         {...props}

src/useReactions.tsx

@@ -140,10 +140,10 @@ export const ReactionsProvider = ({
     };
 
     // Remove any raised hands for users no longer joined to the call.
-    for (const userId of Object.keys(raisedHands).filter(
+    for (const identifier of Object.keys(raisedHands).filter(
       (rhId) => !memberships.find((u) => u.sender == rhId),
     )) {
-      removeRaisedHand(userId);
+      removeRaisedHand(identifier);
     }
 
     // For each member in the call, check to see if a reaction has
@@ -152,13 +152,14 @@ export const ReactionsProvider = ({
       if (!m.sender || !m.eventId) {
         continue;
       }
+      const identifier = `${m.sender}:${m.deviceId}`;
       if (
-        raisedHands[m.sender] &&
+        raisedHands[identifier] &&
-        raisedHands[m.sender].membershipEventId !== m.eventId
+        raisedHands[identifier].membershipEventId !== m.eventId
       ) {
         // Membership event for sender has changed since the hand
         // was raised, reset.
-        removeRaisedHand(m.sender);
+        removeRaisedHand(identifier);
       }
       const reaction = getLastReactionEvent(m.eventId, m.sender);
       if (reaction) {
@@ -166,7 +167,7 @@ export const ReactionsProvider = ({
         if (!eventId) {
           continue;
         }
-        addRaisedHand(m.sender, {
+        addRaisedHand(`${m.sender}:${m.deviceId}`, {
           membershipEventId: m.eventId,
           reactionEventId: eventId,
           time: new Date(reaction.localTimestamp),
@@ -181,11 +182,16 @@ export const ReactionsProvider = ({
   const latestMemberships = useLatest(memberships);
   const latestRaisedHands = useLatest(raisedHands);
 
-  const myMembership = useMemo(
+  const myMembershipEvent = useMemo(
     () => memberships.find((m) => m.sender === myUserId)?.eventId,
     [memberships, myUserId],
   );
-
+  const myMembershipIdentifier = useMemo(() => {
+    const membership = memberships.find((m) => m.sender === myUserId);
+    return membership
+      ? `${membership.sender}:${membership.deviceId}`
+      : undefined;
+  }, [memberships, myUserId]);
   // This effect handles any *live* reaction/redactions in the room.
   useEffect(() => {
     const reactionTimeouts = new Set<number>();
@@ -271,11 +277,10 @@ export const ReactionsProvider = ({
 
         // Check to see if this reaction was made to a membership event (and the
         // sender of the reaction matches the membership)
-        if (
+        const membershipEvent = latestMemberships.current.find(
-          !latestMemberships.current.some(
+          (e) => e.eventId === membershipEventId && e.sender === sender,
-            (e) => e.eventId === membershipEventId && e.sender === sender,
+        );
-          )
+        if (!membershipEvent) {
-        ) {
           logger.warn(
             `Reaction target was not a membership event for ${sender}, ignoring`,
           );
@@ -283,11 +288,14 @@ export const ReactionsProvider = ({
         }
 
         if (content?.["m.relates_to"].key === "🖐️") {
-          addRaisedHand(sender, {
+          addRaisedHand(
-            reactionEventId,
+            `${membershipEvent.sender}:${membershipEvent.deviceId}`,
-            membershipEventId,
+            {
-            time: new Date(event.localTimestamp),
+              reactionEventId,
-          });
+              membershipEventId,
+              time: new Date(event.localTimestamp),
+            },
+          );
         }
       } else if (event.getType() === EventType.RoomRedaction) {
         const targetEvent = event.event.redacts;
@@ -328,14 +336,14 @@ export const ReactionsProvider = ({
   ]);
 
   const toggleRaisedHand = useCallback(async () => {
-    if (!myUserId) {
+    if (!myMembershipIdentifier) {
       return;
     }
-    const myReactionId = raisedHands[myUserId]?.reactionEventId;
+    const myReactionId = raisedHands[myMembershipIdentifier]?.reactionEventId;
 
     if (!myReactionId) {
       try {
-        if (!myMembership) {
+        if (!myMembershipEvent) {
           throw new Error("Cannot find own membership event");
         }
         const reaction = await room.client.sendEvent(
@@ -344,7 +352,7 @@ export const ReactionsProvider = ({
           {
             "m.relates_to": {
               rel_type: RelationType.Annotation,
-              event_id: myMembership,
+              event_id: myMembershipEvent,
               key: "🖐️",
             },
           },
@@ -362,7 +370,13 @@ export const ReactionsProvider = ({
         throw ex;
       }
     }
-  }, [myMembership, myUserId, raisedHands, rtcSession, room]);
+  }, [
+    myMembershipEvent,
+    myMembershipIdentifier,
+    raisedHands,
+    rtcSession,
+    room,
+  ]);
 
   const sendReaction = useCallback(
     async (reaction: ReactionOption) => {
@@ -370,7 +384,7 @@ export const ReactionsProvider = ({
         // We're still reacting
         return;
       }
-      if (!myMembership) {
+      if (!myMembershipEvent) {
         throw new Error("Cannot find own membership event");
       }
       await room.client.sendEvent(
@@ -379,14 +393,14 @@ export const ReactionsProvider = ({
         {
           "m.relates_to": {
             rel_type: RelationType.Reference,
-            event_id: myMembership,
+            event_id: myMembershipEvent,
           },
           emoji: reaction.emoji,
           name: reaction.name,
         },
       );
     },
-    [myMembership, reactions, room, myUserId, rtcSession],
+    [myMembershipEvent, reactions, room, myUserId, rtcSession],
   );
 
   return (