From 4242d45ba2c7336d02813d0629cf2a0fae769ed8 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 16 Aug 2023 18:41:27 +0100 Subject: [PATCH 01/50] WIP refactor for removing m.call events --- src/Header.tsx | 37 +------- src/IncompatibleVersionModal.tsx | 60 ------------ src/matrix-utils.ts | 1 - src/room/GroupCallLoader.tsx | 6 +- src/room/GroupCallView.tsx | 132 ++++++++++++++------------ src/room/InCallView.tsx | 64 +++++-------- src/room/RoomPage.tsx | 6 +- src/room/useGroupCall.ts | 18 ++-- src/room/useLoadGroupCall.ts | 70 ++------------ src/room/useSentryGroupCallHandler.ts | 46 --------- src/useMatrixRTCSessionJoinState.ts | 44 +++++++++ src/useMatrixRTCSessionMemberships.ts | 48 ++++++++++ 12 files changed, 209 insertions(+), 323 deletions(-) delete mode 100644 src/IncompatibleVersionModal.tsx delete mode 100644 src/room/useSentryGroupCallHandler.ts create mode 100644 src/useMatrixRTCSessionJoinState.ts create mode 100644 src/useMatrixRTCSessionMemberships.ts diff --git a/src/Header.tsx b/src/Header.tsx index 8d754b92..80a6ab15 100644 --- a/src/Header.tsx +++ b/src/Header.tsx @@ -15,17 +15,13 @@ limitations under the License. */ import classNames from "classnames"; -import { HTMLAttributes, ReactNode, useCallback } from "react"; +import { HTMLAttributes, ReactNode } from "react"; import { Link } from "react-router-dom"; -import { Room } from "matrix-js-sdk/src/models/room"; import { useTranslation } from "react-i18next"; import styles from "./Header.module.css"; -import { useModalTriggerState } from "./Modal"; -import { Button } from "./button"; import { ReactComponent as Logo } from "./icons/Logo.svg"; import { Subtitle } from "./typography/Typography"; -import { IncompatibleVersionModal } from "./IncompatibleVersionModal"; interface HeaderProps extends HTMLAttributes { children: ReactNode; @@ -125,34 +121,3 @@ export function RoomHeaderInfo({ roomName }: RoomHeaderInfo) { ); } - -interface VersionMismatchWarningProps { - users: Set; - room: Room; -} - -export function VersionMismatchWarning({ - users, - room, -}: VersionMismatchWarningProps) { - const { t } = useTranslation(); - const { modalState, modalProps } = useModalTriggerState(); - - const onDetailsClick = useCallback(() => { - modalState.open(); - }, [modalState]); - - if (users.size === 0) return null; - - return ( - - {t("Incompatible versions!")} - - {modalState.isOpen && ( - - )} - - ); -} diff --git a/src/IncompatibleVersionModal.tsx b/src/IncompatibleVersionModal.tsx deleted file mode 100644 index d11822de..00000000 --- a/src/IncompatibleVersionModal.tsx +++ /dev/null @@ -1,60 +0,0 @@ -/* -Copyright 2022 New Vector Ltd - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -import { Room } from "matrix-js-sdk/src/models/room"; -import { FC, useMemo } from "react"; -import { Trans, useTranslation } from "react-i18next"; - -import { Modal, ModalContent } from "./Modal"; -import { Body } from "./typography/Typography"; - -interface Props { - userIds: Set; - room: Room; - onClose: () => void; -} - -export const IncompatibleVersionModal: FC = ({ - userIds, - room, - onClose, - ...rest -}) => { - const { t } = useTranslation(); - const userLis = useMemo( - () => [...userIds].map((u) =>
  • {room.getMember(u)?.name ?? u}
    • ), - [userIds, room] - ); - - return ( - - - - - Other users are trying to join this call from incompatible versions. - These users should ensure that they have refreshed their browsers: -
      {userLis}
    -     - -     -     - ); -}; diff --git a/src/matrix-utils.ts b/src/matrix-utils.ts index 5f672c57..73ab4151 100644 --- a/src/matrix-utils.ts +++ b/src/matrix-utils.ts @@ -172,7 +172,6 @@ export async function initClient( localTimeoutMs: 5000, useE2eForGroupCall: e2eEnabled, fallbackICEServerAllowed: fallbackICEServerAllowed, - useLivekitForGroupCalls: true, }); try { diff --git a/src/room/GroupCallLoader.tsx b/src/room/GroupCallLoader.tsx index ca52a163..78fa780c 100644 --- a/src/room/GroupCallLoader.tsx +++ b/src/room/GroupCallLoader.tsx @@ -16,8 +16,8 @@ limitations under the License. import { ReactNode } from "react"; import { MatrixClient } from "matrix-js-sdk/src/client"; -import { GroupCall } from "matrix-js-sdk/src/webrtc/groupCall"; import { useTranslation } from "react-i18next"; +import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession"; import { useLoadGroupCall } from "./useLoadGroupCall"; import { ErrorView, FullScreenView } from "../FullScreenView"; @@ -26,7 +26,7 @@ interface Props { client: MatrixClient; roomIdOrAlias: string; viaServers: string[]; - children: (groupCall: GroupCall) => ReactNode; + children: (rtcSession: MatrixRTCSession) => ReactNode; createPtt: boolean; } @@ -53,7 +53,7 @@ export function GroupCallLoader({ ); case "loaded": - return <>{children(groupCallState.groupCall)}; + return <>{children(groupCallState.rtcSession)}; case "failed": return ; } diff --git a/src/room/GroupCallView.tsx b/src/room/GroupCallView.tsx index 8f533d72..9a91bfcf 100644 --- a/src/room/GroupCallView.tsx +++ b/src/room/GroupCallView.tsx @@ -16,33 +16,35 @@ limitations under the License. import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { useHistory } from "react-router-dom"; -import { GroupCall, GroupCallState } from "matrix-js-sdk/src/webrtc/groupCall"; import { MatrixClient } from "matrix-js-sdk/src/client"; import { useTranslation } from "react-i18next"; import { Room } from "livekit-client"; import { logger } from "matrix-js-sdk/src/logger"; +import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession"; +import { Focus } from "matrix-js-sdk/src/matrixrtc/focus"; import type { IWidgetApiRequest } from "matrix-widget-api"; import { widget, ElementWidgetActions, JoinCallData } from "../widget"; -import { useGroupCall } from "./useGroupCall"; import { ErrorView, FullScreenView } from "../FullScreenView"; import { LobbyView } from "./LobbyView"; import { MatrixInfo } from "./VideoPreview"; import { CallEndedView } from "./CallEndedView"; -import { useSentryGroupCallHandler } from "./useSentryGroupCallHandler"; import { PosthogAnalytics } from "../analytics/PosthogAnalytics"; import { useProfile } from "../profile/useProfile"; import { E2EEConfig } from "../livekit/useLiveKit"; import { findDeviceByName } from "../media-utils"; -import { OpenIDLoader } from "../livekit/OpenIDLoader"; +//import { OpenIDLoader } from "../livekit/OpenIDLoader"; import { ActiveCall } from "./InCallView"; -import { Config } from "../config/Config"; import { MuteStates, useMuteStates } from "./MuteStates"; import { useMediaDevices, MediaDevices } from "../livekit/MediaDevicesContext"; +import { LivekitFocus } from "../livekit/LivekitFocus"; +import { useMatrixRTCSessionMemberships } from "../useMatrixRTCSessionMemberships"; +import { enterRTCSession, leaveRTCSession } from "../rtcSessionHelpers"; +import { useMatrixRTCSessionJoinState } from "../useMatrixRTCSessionJoinState"; declare global { interface Window { - groupCall?: GroupCall; + rtcSession?: MatrixRTCSession; } } @@ -52,7 +54,7 @@ interface Props { isEmbedded: boolean; preload: boolean; hideHeader: boolean; - groupCall: GroupCall; + rtcSession: MatrixRTCSession; } export function GroupCallView({ @@ -61,9 +63,9 @@ export function GroupCallView({ isEmbedded, preload, hideHeader, - groupCall, + rtcSession, }: Props) { - const { + /*const { state, error, enter, @@ -71,33 +73,36 @@ export function GroupCallView({ participants, unencryptedEventsFromUsers, otelGroupCallMembership, - } = useGroupCall(groupCall, client); + } = useGroupCall(groupCall, client);*/ + + const memberships = useMatrixRTCSessionMemberships(rtcSession); + const isJoined = useMatrixRTCSessionJoinState(rtcSession); const { t } = useTranslation(); useEffect(() => { - window.groupCall = groupCall; + window.rtcSession = rtcSession; return () => { - delete window.groupCall; + delete window.rtcSession; }; - }, [groupCall]); + }, [rtcSession]); const { displayName, avatarUrl } = useProfile(client); const matrixInfo = useMemo((): MatrixInfo => { return { displayName: displayName!, avatarUrl: avatarUrl!, - roomId: groupCall.room.roomId, - roomName: groupCall.room.name, - roomAlias: groupCall.room.getCanonicalAlias(), + roomId: rtcSession.room.roomId, + roomName: rtcSession.room.name, + roomAlias: rtcSession.room.getCanonicalAlias(), }; - }, [displayName, avatarUrl, groupCall]); + }, [displayName, avatarUrl, rtcSession]); const deviceContext = useMediaDevices(); const latestDevices = useRef(); latestDevices.current = deviceContext; - const muteStates = useMuteStates(participants.size); + const muteStates = useMuteStates(memberships.length); const latestMuteStates = useRef(); latestMuteStates.current = muteStates; @@ -154,10 +159,13 @@ export function GroupCallView({ } } - await enter(); + enterRTCSession(rtcSession); PosthogAnalytics.instance.eventCallEnded.cacheStartCall(new Date()); - PosthogAnalytics.instance.eventCallStarted.track(groupCall.groupCallId); + // we only have room sessions right now, so call ID is the emprty string - we use the room ID + PosthogAnalytics.instance.eventCallStarted.track( + rtcSession.room.roomId + ); await Promise.all([ widget!.api.setAlwaysOnScreen(true), @@ -170,19 +178,18 @@ export function GroupCallView({ widget!.lazyActions.off(ElementWidgetActions.JoinCall, onJoin); }; } - }, [groupCall, preload, enter]); + }, [rtcSession, preload]); useEffect(() => { if (isEmbedded && !preload) { // In embedded mode, bypass the lobby and just enter the call straight away - enter(); + enterRTCSession(rtcSession); PosthogAnalytics.instance.eventCallEnded.cacheStartCall(new Date()); - PosthogAnalytics.instance.eventCallStarted.track(groupCall.groupCallId); + // use the room ID as above + PosthogAnalytics.instance.eventCallStarted.track(rtcSession.room.roomId); } - }, [groupCall, isEmbedded, preload, enter]); - - useSentryGroupCallHandler(groupCall); + }, [rtcSession, isEmbedded, preload]); const [left, setLeft] = useState(false); const [leaveError, setLeaveError] = useState(undefined); @@ -193,21 +200,16 @@ export function GroupCallView({ setLeaveError(leaveError); setLeft(true); - let participantCount = 0; - for (const deviceMap of groupCall.participants.values()) { - participantCount += deviceMap.size; - } - // In embedded/widget mode the iFrame will be killed right after the call ended prohibiting the posthog event from getting sent, // therefore we want the event to be sent instantly without getting queued/batched. const sendInstantly = !!widget; PosthogAnalytics.instance.eventCallEnded.track( - groupCall.groupCallId, - participantCount, + rtcSession.room.roomId, + rtcSession.memberships.length, sendInstantly ); - leave(); + leaveRTCSession(rtcSession); if (widget) { // we need to wait until the callEnded event is tracked. Otherwise the iFrame gets killed before the callEnded event got tracked. await new Promise((resolve) => window.setTimeout(resolve, 10)); // 10ms @@ -224,13 +226,13 @@ export function GroupCallView({ history.push("/"); } }, - [groupCall, leave, isPasswordlessUser, isEmbedded, history] + [rtcSession, isPasswordlessUser, isEmbedded, history] ); useEffect(() => { - if (widget && state === GroupCallState.Entered) { + if (widget && isJoined) { const onHangup = async (ev: CustomEvent) => { - leave(); + leaveRTCSession(rtcSession); await widget!.api.transport.reply(ev.detail, {}); widget!.api.setAlwaysOnScreen(false); }; @@ -239,7 +241,7 @@ export function GroupCallView({ widget!.lazyActions.off(ElementWidgetActions.HangupCall, onHangup); }; } - }, [groupCall, state, leave]); + }, [isJoined, rtcSession]); const [e2eeConfig, setE2EEConfig] = useState( undefined @@ -248,36 +250,40 @@ export function GroupCallView({ const onReconnect = useCallback(() => { setLeft(false); setLeaveError(undefined); - groupCall.enter(); - }, [groupCall]); + rtcSession.joinRoomSession(); + }, [rtcSession]); - const livekitServiceURL = - groupCall.livekitServiceURL ?? Config.get().livekit?.livekit_service_url; - if (!livekitServiceURL) { - return ; + const focus: Focus | undefined = rtcSession + .getOldestMembership() + ?.getActiveFoci()?.[0]; + if ( + !focus || + focus.type !== "livekit" || + !(focus as LivekitFocus).livekit_alias || + !(focus as LivekitFocus).livekit_service_url + ) { + logger.error("Incompatible focus on call", focus); + return ; } - if (error) { - return ; - } else if (state === GroupCallState.Entered) { + if (isJoined) { return ( - - - + >*/ + + // ); } else if (left) { // The call ended view is shown for two reasons: prompting guests to create @@ -293,7 +299,7 @@ export function GroupCallView({ ) { return ( { setE2EEConfig(e2eeConfig); - enter(); + enterRTCSession(rtcSession); }} isEmbedded={isEmbedded} hideHeader={hideHeader} diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index d8a9ef17..556ec4c4 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -27,7 +27,6 @@ import classNames from "classnames"; import { DisconnectReason, Room, RoomEvent, Track } from "livekit-client"; import { MatrixClient } from "matrix-js-sdk/src/client"; import { RoomMember } from "matrix-js-sdk/src/models/room-member"; -import { GroupCall } from "matrix-js-sdk/src/webrtc/groupCall"; import { Ref, useCallback, useEffect, useMemo, useRef } from "react"; import { useTranslation } from "react-i18next"; import useMeasure from "react-use-measure"; @@ -35,6 +34,8 @@ import { OverlayTriggerState } from "@react-stately/overlays"; import { JoinRule } from "matrix-js-sdk/src/@types/partials"; import { logger } from "matrix-js-sdk/src/logger"; import { RoomEventCallbacks } from "livekit-client/dist/src/room/Room"; +import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession"; +import { CallMembership } from "matrix-js-sdk/src/matrixrtc/CallMembership"; import type { IWidgetApiRequest } from "matrix-widget-api"; import { @@ -45,22 +46,13 @@ import { SettingsButton, InviteButton, } from "../button"; -import { - Header, - LeftNav, - RightNav, - RoomHeaderInfo, - VersionMismatchWarning, -} from "../Header"; +import { Header, LeftNav, RightNav, RoomHeaderInfo } from "../Header"; import { useVideoGridLayout, TileDescriptor, VideoGrid, } from "../video-grid/VideoGrid"; -import { - useShowInspector, - useShowConnectionStats, -} from "../settings/useSetting"; +import { useShowConnectionStats } from "../settings/useSetting"; import { useModalTriggerState } from "../Modal"; import { PosthogAnalytics } from "../analytics/PosthogAnalytics"; import { useUrlParams } from "../UrlParams"; @@ -68,10 +60,8 @@ import { useCallViewKeyboardShortcuts } from "../useCallViewKeyboardShortcuts"; import { usePrefersReducedMotion } from "../usePrefersReducedMotion"; import { ElementWidgetActions, widget } from "../widget"; import { GridLayoutMenu } from "./GridLayoutMenu"; -import { GroupCallInspector } from "./GroupCallInspector"; import styles from "./InCallView.module.css"; import { useJoinRule } from "./useJoinRule"; -import { ParticipantInfo } from "./useGroupCall"; import { ItemData, TileContent, VideoTile } from "../video-grid/VideoTile"; import { NewVideoGrid } from "../video-grid/NewVideoGrid"; import { OTelGroupCallMembership } from "../otel/OTelGroupCallMembership"; @@ -120,24 +110,22 @@ export function ActiveCall(props: ActiveCallProps) { export interface InCallViewProps { client: MatrixClient; - groupCall: GroupCall; + rtcSession: MatrixRTCSession; livekitRoom: Room; muteStates: MuteStates; - participants: Map>; + memberships: CallMembership[]; onLeave: (error?: Error) => void; - unencryptedEventsFromUsers: Set; hideHeader: boolean; otelGroupCallMembership?: OTelGroupCallMembership; } export function InCallView({ client, - groupCall, + rtcSession, livekitRoom, muteStates, - participants, + memberships, onLeave, - unencryptedEventsFromUsers, hideHeader, otelGroupCallMembership, }: InCallViewProps) { @@ -161,7 +149,7 @@ export function InCallView({ screenSharingTracks.length > 0 ); - const [showInspector] = useShowInspector(); + //const [showInspector] = useShowInspector(); const [showConnectionStats] = useShowConnectionStats(); const { hideScreensharing } = useUrlParams(); @@ -179,7 +167,7 @@ export function InCallView({ [muteStates] ); - const joinRule = useJoinRule(groupCall.room); + const joinRule = useJoinRule(rtcSession.room); // This function incorrectly assumes that there is a camera and microphone, which is not always the case. // TODO: Make sure that this module is resilient when it comes to camera/microphone availability! @@ -250,7 +238,7 @@ export function InCallView({ const reducedControls = boundsValid && bounds.width <= 400; const noControls = reducedControls && bounds.height <= 400; - const items = useParticipantTiles(livekitRoom, participants); + const items = useParticipantTiles(livekitRoom, memberships); const { fullscreenItem, toggleFullscreen, exitFullscreen } = useFullscreen(items); @@ -324,7 +312,7 @@ export function InCallView({ const { modalState: rageshakeRequestModalState, modalProps: rageshakeRequestModalProps, - } = useRageshakeRequestModal(groupCall.room.roomId); + } = useRageshakeRequestModal(rtcSession.room.roomId); const { modalState: settingsModalState, @@ -419,11 +407,7 @@ export function InCallView({ {!hideHeader && maximisedParticipant === null && (
    - - + @@ -439,31 +423,31 @@ export function InCallView({ {renderContent()} {footer} - {otelGroupCallMembership && ( + {/*otelGroupCallMembership && ( - )} + )*/} {rageshakeRequestModalState.isOpen && !noControls && ( )} {settingsModalState.isOpen && ( )} {inviteModalState.isOpen && ( @@ -474,22 +458,26 @@ export function InCallView({ function useParticipantTiles( livekitRoom: Room, - participants: Map> + memberships: CallMembership[] ): TileDescriptor[] { const sfuParticipants = useParticipants({ room: livekitRoom, }); const items = useMemo(() => { - // The IDs of the participants who published membership event to the room (i.e. are present from Matrix perspective). const matrixParticipants: Map = new Map( + memberships.map((m) => [`${m.member.userId}:${m.deviceId}`, m.member]) + ); + + // The IDs of the participants who published membership event to the room (i.e. are present from Matrix perspective). + /*const matrixParticipants: Map = new Map( [...participants.entries()].flatMap(([user, devicesMap]) => { return [...devicesMap.keys()].map((deviceId) => [ `${user.userId}:${deviceId}`, user, ]); }) - ); + );*/ const hasPresenter = sfuParticipants.find((p) => p.isScreenShareEnabled) !== undefined; @@ -558,7 +546,7 @@ function useParticipantTiles( // If every item is a ghost, that probably means we're still connecting and // shouldn't bother showing anything yet return allGhosts ? [] : tiles; - }, [participants, sfuParticipants]); + }, [memberships, sfuParticipants]); return items; } diff --git a/src/room/RoomPage.tsx b/src/room/RoomPage.tsx index ddf92959..50156e70 100644 --- a/src/room/RoomPage.tsx +++ b/src/room/RoomPage.tsx @@ -15,8 +15,8 @@ limitations under the License. */ import { FC, useEffect, useState, useCallback } from "react"; +import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession"; -import type { GroupCall } from "matrix-js-sdk/src/webrtc/groupCall"; import { useClientLegacy } from "../ClientContext"; import { ErrorView, LoadingView } from "../FullScreenView"; import { RoomAuthView } from "./RoomAuthView"; @@ -73,10 +73,10 @@ export const RoomPage: FC = () => { ]); const groupCallView = useCallback( - (groupCall: GroupCall) => ( + (rtcSession: MatrixRTCSession) => ( >; hasLocalParticipant: boolean; - unencryptedEventsFromUsers: Set; otelGroupCallMembership?: OTelGroupCallMembership; } @@ -103,14 +103,14 @@ interface State { let groupCallOTelMembership: OTelGroupCallMembership | undefined; let groupCallOTelMembershipGroupCallId: string; -function getParticipants( - groupCall: GroupCall +/*function getParticipants( + rtcSession: MatrixRTCSession ): Map> { const participants = new Map>(); - for (const [member, participantsStateMap] of groupCall.participants) { + for (const membership of rtcSession.memberships) { const participantInfoMap = new Map(); - participants.set(member, participantInfoMap); + participants.set(membership.member, participantInfoMap); for (const [deviceId, participant] of participantsStateMap) { const feed = groupCall.userMediaFeeds.find( @@ -141,10 +141,10 @@ function getParticipants( } return participants; -} +}*/ export function useGroupCall( - groupCall: GroupCall, + rtcSession: MatrixRTCSession, client: MatrixClient ): UseGroupCallReturnType { const [ @@ -171,7 +171,7 @@ export function useGroupCall( isScreensharing: false, screenshareFeeds: [], requestingScreenshare: false, - participants: getParticipants(groupCall), + participants: getParticipants(rtcSession), hasLocalParticipant: false, }); diff --git a/src/room/useLoadGroupCall.ts b/src/room/useLoadGroupCall.ts index 4aa82f11..cae3d409 100644 --- a/src/room/useLoadGroupCall.ts +++ b/src/room/useLoadGroupCall.ts @@ -15,28 +15,19 @@ limitations under the License. */ import { useState, useEffect } from "react"; -import { EventType } from "matrix-js-sdk/src/@types/event"; -import { - GroupCallType, - GroupCallIntent, -} from "matrix-js-sdk/src/webrtc/groupCall"; -import { GroupCallEventHandlerEvent } from "matrix-js-sdk/src/webrtc/groupCallEventHandler"; import { logger } from "matrix-js-sdk/src/logger"; import { ClientEvent, MatrixClient } from "matrix-js-sdk/src/client"; import { SyncState } from "matrix-js-sdk/src/sync"; import { useTranslation } from "react-i18next"; +import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession"; import type { Room } from "matrix-js-sdk/src/models/room"; import type { GroupCall } from "matrix-js-sdk/src/webrtc/groupCall"; import { isLocalRoomId, createRoom, roomNameFromRoomId } from "../matrix-utils"; -import { translatedError } from "../TranslatedError"; -import { widget } from "../widget"; - -const STATS_COLLECT_INTERVAL_TIME_MS = 10000; export type GroupCallLoaded = { kind: "loaded"; - groupCall: GroupCall; + rtcSession: MatrixRTCSession; }; export type GroupCallLoadFailed = { @@ -115,61 +106,12 @@ export const useLoadGroupCall = ( } }; - const fetchOrCreateGroupCall = async (): Promise => { + const fetchOrCreateGroupCall = async (): Promise => { const room = await fetchOrCreateRoom(); logger.debug(`Fetched / joined room ${roomIdOrAlias}`); - let groupCall = client.getGroupCallForRoom(room.roomId); - logger.debug("Got group call", groupCall?.groupCallId); - if (groupCall) { - groupCall.setGroupCallStatsInterval(STATS_COLLECT_INTERVAL_TIME_MS); - return groupCall; - } - - if ( - !widget && - room.currentState.mayClientSendStateEvent( - EventType.GroupCallPrefix, - client - ) - ) { - // The call doesn't exist, but we can create it - console.log( - `No call found in ${roomIdOrAlias}: creating ${ - createPtt ? "PTT" : "video" - } call` - ); - groupCall = await client.createGroupCall( - room.roomId, - createPtt ? GroupCallType.Voice : GroupCallType.Video, - createPtt, - GroupCallIntent.Room - ); - groupCall.setGroupCallStatsInterval(STATS_COLLECT_INTERVAL_TIME_MS); - return groupCall; - } - - // We don't have permission to create the call, so all we can do is wait - // for one to come in - return new Promise((resolve, reject) => { - const onGroupCallIncoming = (groupCall: GroupCall) => { - if (groupCall?.room.roomId === room.roomId) { - clearTimeout(timeout); - groupCall.setGroupCallStatsInterval(STATS_COLLECT_INTERVAL_TIME_MS); - client.off( - GroupCallEventHandlerEvent.Incoming, - onGroupCallIncoming - ); - resolve(groupCall); - } - }; - client.on(GroupCallEventHandlerEvent.Incoming, onGroupCallIncoming); - - const timeout = setTimeout(() => { - client.off(GroupCallEventHandlerEvent.Incoming, onGroupCallIncoming); - reject(translatedError("Fetching group call timed out.", t)); - }, 30000); - }); + const rtcSession = client.matrixRTC.getRoomSession(room); + return rtcSession; }; const waitForClientSyncing = async () => { @@ -192,7 +134,7 @@ export const useLoadGroupCall = ( waitForClientSyncing() .then(fetchOrCreateGroupCall) - .then((groupCall) => setState({ kind: "loaded", groupCall })) + .then((rtcSession) => setState({ kind: "loaded", rtcSession })) .catch((error) => setState({ kind: "failed", error })); }, [client, roomIdOrAlias, viaServers, createPtt, t]); diff --git a/src/room/useSentryGroupCallHandler.ts b/src/room/useSentryGroupCallHandler.ts deleted file mode 100644 index 188a2934..00000000 --- a/src/room/useSentryGroupCallHandler.ts +++ /dev/null @@ -1,46 +0,0 @@ -/* -Copyright 2022 New Vector Ltd - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -import { useEffect } from "react"; -import * as Sentry from "@sentry/react"; -import { GroupCall, GroupCallEvent } from "matrix-js-sdk/src/webrtc/groupCall"; -import { CallEvent, MatrixCall } from "matrix-js-sdk/src/webrtc/call"; - -export function useSentryGroupCallHandler(groupCall: GroupCall) { - useEffect(() => { - function onHangup(call: MatrixCall) { - if (call.hangupReason === "ice_failed") { - Sentry.captureException(new Error("Call hangup due to ICE failure.")); - } - } - - function onError(error: Error) { - Sentry.captureException(error); - } - - if (groupCall) { - groupCall.on(CallEvent.Hangup, onHangup); - groupCall.on(GroupCallEvent.Error, onError); - } - - return () => { - if (groupCall) { - groupCall.removeListener(CallEvent.Hangup, onHangup); - groupCall.removeListener(GroupCallEvent.Error, onError); - } - }; - }, [groupCall]); -} diff --git a/src/useMatrixRTCSessionJoinState.ts b/src/useMatrixRTCSessionJoinState.ts new file mode 100644 index 00000000..7afd6b52 --- /dev/null +++ b/src/useMatrixRTCSessionJoinState.ts @@ -0,0 +1,44 @@ +/* +Copyright 2023 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { + MatrixRTCSession, + MatrixRTCSessionEvent, +} from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession"; +import { useCallback, useEffect, useState } from "react"; + +export function useMatrixRTCSessionJoinState( + rtcSession: MatrixRTCSession +): boolean { + const [isJoined, setJoined] = useState(rtcSession.isJoined()); + + const onJoinStateChanged = useCallback(() => { + setJoined(rtcSession.isJoined()); + }, [rtcSession]); + + useEffect(() => { + rtcSession.on(MatrixRTCSessionEvent.JoinStateChanged, onJoinStateChanged); + + return () => { + rtcSession.off( + MatrixRTCSessionEvent.JoinStateChanged, + onJoinStateChanged + ); + }; + }, [rtcSession, onJoinStateChanged]); + + return isJoined; +} diff --git a/src/useMatrixRTCSessionMemberships.ts b/src/useMatrixRTCSessionMemberships.ts new file mode 100644 index 00000000..149aa8f6 --- /dev/null +++ b/src/useMatrixRTCSessionMemberships.ts @@ -0,0 +1,48 @@ +/* +Copyright 2023 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { CallMembership } from "matrix-js-sdk/src/matrixrtc/CallMembership"; +import { + MatrixRTCSession, + MatrixRTCSessionEvent, +} from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession"; +import { useCallback, useEffect, useState } from "react"; + +export function useMatrixRTCSessionMemberships( + rtcSession: MatrixRTCSession +): CallMembership[] { + const [memberships, setMemberships] = useState(rtcSession.memberships); + + const onMembershipsChanged = useCallback(() => { + setMemberships(rtcSession.memberships); + }, [rtcSession]); + + useEffect(() => { + rtcSession.on( + MatrixRTCSessionEvent.MembershipsChanged, + onMembershipsChanged + ); + + return () => { + rtcSession.off( + MatrixRTCSessionEvent.MembershipsChanged, + onMembershipsChanged + ); + }; + }, [rtcSession, onMembershipsChanged]); + + return memberships; +} From e39d00154d50ad7dc87d0fab1e583d362262b3ea Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 18 Aug 2023 09:03:21 +0100 Subject: [PATCH 02/50] More hacking on rtcsession --- src/livekit/LivekitFocus.ts | 23 +++++++ src/livekit/openIDSFU.ts | 87 ++++++++++++--------------- src/room/GroupCallView.tsx | 40 ++---------- src/room/useActiveFocus.ts | 68 +++++++++++++++++++++ src/rtcSessionHelpers.ts | 53 ++++++++++++++++ src/useMatrixRTCSessionJoinState.ts | 6 ++ src/useMatrixRTCSessionMemberships.ts | 4 ++ 7 files changed, 196 insertions(+), 85 deletions(-) create mode 100644 src/livekit/LivekitFocus.ts create mode 100644 src/room/useActiveFocus.ts create mode 100644 src/rtcSessionHelpers.ts diff --git a/src/livekit/LivekitFocus.ts b/src/livekit/LivekitFocus.ts new file mode 100644 index 00000000..f3a1e532 --- /dev/null +++ b/src/livekit/LivekitFocus.ts @@ -0,0 +1,23 @@ +/* +Copyright 2023 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { Focus } from "matrix-js-sdk/src/matrixrtc/focus"; + +export interface LivekitFocus extends Focus { + type: "livekit"; + livekit_service_url: string; + livekit_alias: string; +} diff --git a/src/livekit/openIDSFU.ts b/src/livekit/openIDSFU.ts index 083e95b2..c122a844 100644 --- a/src/livekit/openIDSFU.ts +++ b/src/livekit/openIDSFU.ts @@ -14,10 +14,13 @@ See the License for the specific language governing permissions and limitations under the License. */ -import { GroupCall, IOpenIDToken, MatrixClient } from "matrix-js-sdk"; +import { IOpenIDToken, MatrixClient } from "matrix-js-sdk"; import { logger } from "matrix-js-sdk/src/logger"; +import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession"; +import { useEffect, useState } from "react"; -import { Config } from "../config/Config"; +import { LivekitFocus } from "./LivekitFocus"; +import { useActiveFocus } from "../room/useActiveFocus"; export interface SFUConfig { url: string; @@ -30,66 +33,52 @@ export type OpenIDClientParts = Pick< "getOpenIdToken" | "getDeviceId" >; +export function useOpenIDSFU( + client: OpenIDClientParts, + rtcSession: MatrixRTCSession +) { + const [sfuConfig, setSFUConfig] = useState(undefined); + + const activeFocus = useActiveFocus(rtcSession); + + useEffect(() => { + (async () => { + const sfuConfig = activeFocus + ? await getSFUConfigWithOpenID(client, activeFocus) + : undefined; + setSFUConfig(sfuConfig); + })(); + }, [client, activeFocus]); + + return sfuConfig; +} + export async function getSFUConfigWithOpenID( client: OpenIDClientParts, - groupCall: GroupCall, - roomName: string -): Promise { + activeFocus: LivekitFocus +): Promise { const openIdToken = await client.getOpenIdToken(); logger.debug("Got openID token", openIdToken); - // if the call has a livekit service URL, try it. - if (groupCall.livekitServiceURL) { - try { - logger.info( - `Trying to get JWT from call's configured URL of ${groupCall.livekitServiceURL}...` - ); - const sfuConfig = await getLiveKitJWT( - client, - groupCall.livekitServiceURL, - roomName, - openIdToken - ); - logger.info(`Got JWT from call state event URL.`); - - return sfuConfig; - } catch (e) { - logger.warn( - `Failed to get JWT from group call's configured URL of ${groupCall.livekitServiceURL}.`, - e - ); - } - } - - // otherwise, try our configured one and, if it works, update the call's service URL in the state event - // NB. This wuill update it for everyone so we may end up with multiple clients updating this when they - // join at similar times, but we don't have a huge number of options here. - const urlFromConf = Config.get().livekit!.livekit_service_url; - logger.info(`Trying livekit service URL from our config: ${urlFromConf}...`); try { + logger.info( + `Trying to get JWT from call's active focus URL of ${activeFocus.livekit_service_url}...` + ); const sfuConfig = await getLiveKitJWT( client, - urlFromConf, - roomName, + activeFocus.livekit_service_url, + activeFocus.livekit_alias, openIdToken ); - - logger.info( - `Got JWT, updating call livekit service URL with: ${urlFromConf}...` - ); - try { - await groupCall.updateLivekitServiceURL(urlFromConf); - logger.info(`Call livekit service URL updated.`); - } catch (e) { - logger.warn( - `Failed to update call livekit service URL: continuing anyway.` - ); - } + logger.info(`Got JWT from call's active focus URL.`); return sfuConfig; } catch (e) { - logger.error("Failed to get JWT from URL defined in Config.", e); - throw e; + logger.warn( + `Failed to get JWT from RTC session's active focus URL of ${activeFocus.livekit_service_url}.`, + e + ); + return undefined; } } diff --git a/src/room/GroupCallView.tsx b/src/room/GroupCallView.tsx index eaf334ce..57182b04 100644 --- a/src/room/GroupCallView.tsx +++ b/src/room/GroupCallView.tsx @@ -21,7 +21,6 @@ import { useTranslation } from "react-i18next"; import { Room } from "livekit-client"; import { logger } from "matrix-js-sdk/src/logger"; import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession"; -import { Focus } from "matrix-js-sdk/src/matrixrtc/focus"; import type { IWidgetApiRequest } from "matrix-widget-api"; import { widget, ElementWidgetActions, JoinCallData } from "../widget"; @@ -32,11 +31,9 @@ import { CallEndedView } from "./CallEndedView"; import { PosthogAnalytics } from "../analytics/PosthogAnalytics"; import { useProfile } from "../profile/useProfile"; import { findDeviceByName } from "../media-utils"; -//import { OpenIDLoader } from "../livekit/OpenIDLoader"; import { ActiveCall } from "./InCallView"; import { MuteStates, useMuteStates } from "./MuteStates"; import { useMediaDevices, MediaDevices } from "../livekit/MediaDevicesContext"; -import { LivekitFocus } from "../livekit/LivekitFocus"; import { useMatrixRTCSessionMemberships } from "../useMatrixRTCSessionMemberships"; import { enterRTCSession, leaveRTCSession } from "../rtcSessionHelpers"; import { useMatrixRTCSessionJoinState } from "../useMatrixRTCSessionJoinState"; @@ -69,21 +66,11 @@ export function GroupCallView({ hideHeader, rtcSession, }: Props) { - /*const { - state, - error, - enter, - leave, - participants, - unencryptedEventsFromUsers, - otelGroupCallMembership, - } = useGroupCall(groupCall, client);*/ - const memberships = useMatrixRTCSessionMemberships(rtcSession); const isJoined = useMatrixRTCSessionJoinState(rtcSession); - const e2eeSharedKey = useManageRoomSharedKey(groupCall.room.roomId); - const isRoomE2EE = useIsRoomE2EE(groupCall.room.roomId); + const e2eeSharedKey = useManageRoomSharedKey(rtcSession.room.roomId); + const isRoomE2EE = useIsRoomE2EE(rtcSession.room.roomId); const { t } = useTranslation(); @@ -260,22 +247,9 @@ export function GroupCallView({ const onReconnect = useCallback(() => { setLeft(false); setLeaveError(undefined); - rtcSession.joinRoomSession(); + enterRTCSession(rtcSession); }, [rtcSession]); - const focus: Focus | undefined = rtcSession - .getOldestMembership() - ?.getActiveFoci()?.[0]; - if ( - !focus || - focus.type !== "livekit" || - !(focus as LivekitFocus).livekit_alias || - !(focus as LivekitFocus).livekit_service_url - ) { - logger.error("Incompatible focus on call", focus); - return ; - } - if (e2eeEnabled && isRoomE2EE && !e2eeSharedKey) { return ( */ - // ); } else if (left) { // The call ended view is shown for two reasons: prompting guests to create @@ -351,7 +319,7 @@ export function GroupCallView({ enter()} + onEnter={() => enterRTCSession(rtcSession)} isEmbedded={isEmbedded} hideHeader={hideHeader} /> diff --git a/src/room/useActiveFocus.ts b/src/room/useActiveFocus.ts new file mode 100644 index 00000000..a62ffafd --- /dev/null +++ b/src/room/useActiveFocus.ts @@ -0,0 +1,68 @@ +/* +Copyright 2023 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { + MatrixRTCSession, + MatrixRTCSessionEvent, +} from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession"; +import { useCallback, useEffect, useState } from "react"; +import { deepCompare } from "matrix-js-sdk/src/utils"; + +import { LivekitFocus } from "../livekit/LivekitFocus"; + +function getActiveFocus( + rtcSession: MatrixRTCSession +): LivekitFocus | undefined { + const oldestMembership = rtcSession.getOldestMembership(); + return oldestMembership?.getActiveFoci()[0] as LivekitFocus; +} + +/** + * Gets the currently active (livekit) focus for a MatrixRTC session + * This logic is specific to livekit foci where the whole call must use one + * and the same focus. + */ +export function useActiveFocus( + rtcSession: MatrixRTCSession +): LivekitFocus | undefined { + const [activeFocus, setActiveFocus] = useState(() => + getActiveFocus(rtcSession) + ); + + const onMembershipsChanged = useCallback(() => { + const newActiveFocus = getActiveFocus(rtcSession); + + if (!deepCompare(activeFocus, newActiveFocus)) { + setActiveFocus(newActiveFocus); + } + }, [activeFocus, rtcSession]); + + useEffect(() => { + rtcSession.on( + MatrixRTCSessionEvent.MembershipsChanged, + onMembershipsChanged + ); + + return () => { + rtcSession.off( + MatrixRTCSessionEvent.MembershipsChanged, + onMembershipsChanged + ); + }; + }); + + return activeFocus; +} diff --git a/src/rtcSessionHelpers.ts b/src/rtcSessionHelpers.ts new file mode 100644 index 00000000..3d62f980 --- /dev/null +++ b/src/rtcSessionHelpers.ts @@ -0,0 +1,53 @@ +/* +Copyright 2023 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession"; + +import { PosthogAnalytics } from "./analytics/PosthogAnalytics"; +import { LivekitFocus } from "./livekit/LivekitFocus"; +import { Config } from "./config/Config"; + +function makeFocus(livekitAlias: string): LivekitFocus { + const urlFromConf = Config.get().livekit!.livekit_service_url; + if (!urlFromConf) { + throw new Error("No livekit_service_url is configured!"); + } + + return { + type: "livekit", + livekit_service_url: urlFromConf, + livekit_alias: livekitAlias, + }; +} + +export function enterRTCSession(rtcSession: MatrixRTCSession) { + PosthogAnalytics.instance.eventCallEnded.cacheStartCall(new Date()); + PosthogAnalytics.instance.eventCallStarted.track(rtcSession.room.roomId); + + // This must be called before we start trying to join the call, as we need to + // have started tracking by the time calls start getting created. + //groupCallOTelMembership?.onJoinCall(); + + // right now we asume everything is a room-scoped call + const livekitAlias = rtcSession.room.roomId; + + rtcSession.joinRoomSession([makeFocus(livekitAlias)]); +} + +export function leaveRTCSession(rtcSession: MatrixRTCSession) { + //groupCallOTelMembership?.onLeaveCall(); + rtcSession.leaveRoomSession(); +} diff --git a/src/useMatrixRTCSessionJoinState.ts b/src/useMatrixRTCSessionJoinState.ts index 7afd6b52..c7c54563 100644 --- a/src/useMatrixRTCSessionJoinState.ts +++ b/src/useMatrixRTCSessionJoinState.ts @@ -14,6 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ +import { logger } from "matrix-js-sdk/src/logger"; import { MatrixRTCSession, MatrixRTCSessionEvent, @@ -26,6 +27,11 @@ export function useMatrixRTCSessionJoinState( const [isJoined, setJoined] = useState(rtcSession.isJoined()); const onJoinStateChanged = useCallback(() => { + logger.info( + `Session in room ${rtcSession.room.roomId} changed to ${ + rtcSession.isJoined() ? "joined" : "left" + }` + ); setJoined(rtcSession.isJoined()); }, [rtcSession]); diff --git a/src/useMatrixRTCSessionMemberships.ts b/src/useMatrixRTCSessionMemberships.ts index 149aa8f6..a0d5a513 100644 --- a/src/useMatrixRTCSessionMemberships.ts +++ b/src/useMatrixRTCSessionMemberships.ts @@ -14,6 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ +import { logger } from "matrix-js-sdk/src/logger"; import { CallMembership } from "matrix-js-sdk/src/matrixrtc/CallMembership"; import { MatrixRTCSession, @@ -27,6 +28,9 @@ export function useMatrixRTCSessionMemberships( const [memberships, setMemberships] = useState(rtcSession.memberships); const onMembershipsChanged = useCallback(() => { + logger.info( + `Memberships changed for call in room ${rtcSession.room.roomId} (${rtcSession.memberships.length} members)` + ); setMemberships(rtcSession.memberships); }, [rtcSession]); From 274349f2e2e50fab851766747aceac6b3207767f Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 18 Aug 2023 10:20:14 +0100 Subject: [PATCH 03/50] Use the right sfu config hook --- src/room/InCallView.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index e8e6c71c..61146d79 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -72,13 +72,13 @@ import { RageshakeRequestModal } from "./RageshakeRequestModal"; import { E2EEConfig, useLiveKit } from "../livekit/useLiveKit"; import { useFullscreen } from "./useFullscreen"; import { useLayoutStates } from "../video-grid/Layout"; -import { useSFUConfig } from "../livekit/OpenIDLoader"; import { E2EELock } from "../E2EELock"; import { useEventEmitterThree } from "../useEvents"; import { useWakeLock } from "../useWakeLock"; import { useMergedRefs } from "../useMergedRefs"; import { MuteStates } from "./MuteStates"; import { useIsRoomE2EE } from "../e2ee/sharedKeyManagement"; +import { useOpenIDSFU } from "../livekit/openIDSFU"; const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {}); // There is currently a bug in Safari our our code with cloning and sending MediaStreams @@ -91,7 +91,7 @@ export interface ActiveCallProps extends Omit { } export function ActiveCall(props: ActiveCallProps) { - const sfuConfig = useSFUConfig(); + const sfuConfig = useOpenIDSFU(props.client, props.rtcSession); const livekitRoom = useLiveKit(props.muteStates, sfuConfig, props.e2eeConfig); if (!livekitRoom) { From 7a197a2700f79ae5e8bae57087436936b92edfc9 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 18 Aug 2023 14:14:24 +0100 Subject: [PATCH 04/50] Use js-sdk branch --- package.json | 2 +- yarn.lock | 26 +++++++++++++++++--------- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/package.json b/package.json index eff3786d..5bee62c3 100644 --- a/package.json +++ b/package.json @@ -57,7 +57,7 @@ "i18next-http-backend": "^1.4.4", "livekit-client": "1.12.3", "lodash": "^4.17.21", - "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#b698217445318f453e0b1086364a33113eaa85d9", + "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#f5f63d67f74a936f971199e76935000d2d938c21", "matrix-widget-api": "^1.3.1", "mermaid": "^8.13.8", "normalize.css": "^8.0.1", diff --git a/yarn.lock b/yarn.lock index 2e9bfbdc..6f2b59b9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2222,10 +2222,10 @@ "@react-hook/latest" "^1.0.3" clsx "^1.2.1" -"@matrix-org/matrix-sdk-crypto-js@^0.1.1": - version "0.1.4" - resolved "https://registry.yarnpkg.com/@matrix-org/matrix-sdk-crypto-js/-/matrix-sdk-crypto-js-0.1.4.tgz#c13c7c8c3a1d8da08e6ad195d25e5e61cc402df7" - integrity sha512-OxG84iSeR89zYLFeb+DCaFtZT+DDiIu+kTkqY8OYfhE5vpGLFX2sDVBRrAdos1IUqEoboDloDBR9+yU7hNRyog== +"@matrix-org/matrix-sdk-crypto-wasm@^1.2.1": + version "1.2.1" + resolved "https://registry.yarnpkg.com/@matrix-org/matrix-sdk-crypto-wasm/-/matrix-sdk-crypto-wasm-1.2.1.tgz#5b546c8a0e53b614f10b77b3b649818aed9d0db1" + integrity sha512-DCb7Q83PCQK0uav5vB3KNV/hJPlxAhT/ddar+VHz2kC39hMLKGzWYVhprpLYVcavaE/6OX+Q/xFkAoV/3QtUHQ== "@matrix-org/olm@https://gitlab.matrix.org/api/v4/projects/27/packages/npm/@matrix-org/olm/-/@matrix-org/olm-3.2.14.tgz": version "3.2.14" @@ -10943,19 +10943,19 @@ matrix-events-sdk@0.0.1: resolved "https://registry.yarnpkg.com/matrix-events-sdk/-/matrix-events-sdk-0.0.1.tgz#c8c38911e2cb29023b0bbac8d6f32e0de2c957dd" integrity sha512-1QEOsXO+bhyCroIe2/A5OwaxHvBm7EsSQ46DEDn8RBIfQwN5HWBpFvyWWR4QY0KHPPnnJdI99wgRiAl7Ad5qaA== -"matrix-js-sdk@github:matrix-org/matrix-js-sdk#b698217445318f453e0b1086364a33113eaa85d9": - version "26.2.0" - resolved "https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/b698217445318f453e0b1086364a33113eaa85d9" +"matrix-js-sdk@github:matrix-org/matrix-js-sdk#f5f63d67f74a936f971199e76935000d2d938c21": + version "27.2.0" + resolved "https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/f5f63d67f74a936f971199e76935000d2d938c21" dependencies: "@babel/runtime" "^7.12.5" - "@matrix-org/matrix-sdk-crypto-js" "^0.1.1" + "@matrix-org/matrix-sdk-crypto-wasm" "^1.2.1" another-json "^0.2.0" bs58 "^5.0.0" content-type "^1.0.4" jwt-decode "^3.1.2" loglevel "^1.7.1" matrix-events-sdk "0.0.1" - matrix-widget-api "^1.3.1" + matrix-widget-api "^1.5.0" oidc-client-ts "^2.2.4" p-retry "4" sdp-transform "^2.14.1" @@ -10970,6 +10970,14 @@ matrix-widget-api@^1.3.1: "@types/events" "^3.0.0" events "^3.2.0" +matrix-widget-api@^1.5.0: + version "1.5.0" + resolved "https://registry.yarnpkg.com/matrix-widget-api/-/matrix-widget-api-1.5.0.tgz#4ae3e46a7f2854f944ddaf8a5af63d72fba76c45" + integrity sha512-hKGfqQKK5qVMwW0Sp8l2TiuW8UuHafTvUZNSWBPghedB/rSFbVLlr0mufuEV0iq/pQ7ChW96q/WEC6Llie4SnA== + dependencies: + "@types/events" "^3.0.0" + events "^3.2.0" + md5.js@^1.3.4: version "1.3.5" resolved "https://registry.yarnpkg.com/md5.js/-/md5.js-1.3.5.tgz#b5d07b8e3216e3e27cd728d72f70d1e6a342005f" From ee77820e5be143cb14d8f45d929944f48f9b67f9 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 23 Aug 2023 09:59:02 +0100 Subject: [PATCH 05/50] Use latest js-sdk branch --- package.json | 2 +- yarn.lock | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 5bee62c3..77d10cfa 100644 --- a/package.json +++ b/package.json @@ -57,7 +57,7 @@ "i18next-http-backend": "^1.4.4", "livekit-client": "1.12.3", "lodash": "^4.17.21", - "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#f5f63d67f74a936f971199e76935000d2d938c21", + "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#0d46aeb5a4d44685885adc502f46688a4bed65b6", "matrix-widget-api": "^1.3.1", "mermaid": "^8.13.8", "normalize.css": "^8.0.1", diff --git a/yarn.lock b/yarn.lock index 6f2b59b9..4c2c0dac 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10943,9 +10943,9 @@ matrix-events-sdk@0.0.1: resolved "https://registry.yarnpkg.com/matrix-events-sdk/-/matrix-events-sdk-0.0.1.tgz#c8c38911e2cb29023b0bbac8d6f32e0de2c957dd" integrity sha512-1QEOsXO+bhyCroIe2/A5OwaxHvBm7EsSQ46DEDn8RBIfQwN5HWBpFvyWWR4QY0KHPPnnJdI99wgRiAl7Ad5qaA== -"matrix-js-sdk@github:matrix-org/matrix-js-sdk#f5f63d67f74a936f971199e76935000d2d938c21": +"matrix-js-sdk@github:matrix-org/matrix-js-sdk#0d46aeb5a4d44685885adc502f46688a4bed65b6": version "27.2.0" - resolved "https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/f5f63d67f74a936f971199e76935000d2d938c21" + resolved "https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/0d46aeb5a4d44685885adc502f46688a4bed65b6" dependencies: "@babel/runtime" "^7.12.5" "@matrix-org/matrix-sdk-crypto-wasm" "^1.2.1" From 095753c6a03c4ca223bdf717544d3d126d80ecab Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 23 Aug 2023 10:03:05 +0100 Subject: [PATCH 06/50] i18n --- public/locales/en-GB/app.json | 5 ----- 1 file changed, 5 deletions(-) diff --git a/public/locales/en-GB/app.json b/public/locales/en-GB/app.json index 076c65b8..fc461337 100644 --- a/public/locales/en-GB/app.json +++ b/public/locales/en-GB/app.json @@ -32,7 +32,6 @@ "Create account": "Create account", "Debug log": "Debug log", "Debug log request": "Debug log request", - "Details": "Details", "Developer": "Developer", "Developer Settings": "Developer Settings", "Display name": "Display name", @@ -44,7 +43,6 @@ "Exit full screen": "Exit full screen", "Expose developer settings in the settings window.": "Expose developer settings in the settings window.", "Feedback": "Feedback", - "Fetching group call timed out.": "Fetching group call timed out.", "Freedom": "Freedom", "Full screen": "Full screen", "Go": "Go", @@ -53,8 +51,6 @@ "How did it go?": "How did it go?", "If you are experiencing issues or simply would like to provide some feedback, please send us a short description below.": "If you are experiencing issues or simply would like to provide some feedback, please send us a short description below.", "Include debug logs": "Include debug logs", - "Incompatible versions": "Incompatible versions", - "Incompatible versions!": "Incompatible versions!", "Inspector": "Inspector", "Invite": "Invite", "Invite people": "Invite people", @@ -73,7 +69,6 @@ "No": "No", "Not now, return to home screen": "Not now, return to home screen", "Not registered yet? <2>Create an account": "Not registered yet? <2>Create an account", - "Other users are trying to join this call from incompatible versions. These users should ensure that they have refreshed their browsers:<1>{userLis}": "Other users are trying to join this call from incompatible versions. These users should ensure that they have refreshed their browsers:<1>{userLis}", "Password": "Password", "Passwords must match": "Passwords must match", "Profile": "Profile", From 918736e758082720d0a7c7585d8f261414d160fe Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 23 Aug 2023 10:06:08 +0100 Subject: [PATCH 07/50] Removed now unused useGroupCall & OpenIDLoader --- src/livekit/OpenIDLoader.tsx | 92 ----- src/room/useGroupCall.ts | 644 ----------------------------------- 2 files changed, 736 deletions(-) delete mode 100644 src/livekit/OpenIDLoader.tsx delete mode 100644 src/room/useGroupCall.ts diff --git a/src/livekit/OpenIDLoader.tsx b/src/livekit/OpenIDLoader.tsx deleted file mode 100644 index 911a1ae7..00000000 --- a/src/livekit/OpenIDLoader.tsx +++ /dev/null @@ -1,92 +0,0 @@ -/* -Copyright 2023 New Vector Ltd - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -import { - ReactNode, - createContext, - useContext, - useEffect, - useState, -} from "react"; -import { logger } from "matrix-js-sdk/src/logger"; -import { GroupCall } from "matrix-js-sdk"; - -import { - OpenIDClientParts, - SFUConfig, - getSFUConfigWithOpenID, -} from "./openIDSFU"; -import { ErrorView, LoadingView } from "../FullScreenView"; - -interface Props { - client: OpenIDClientParts; - groupCall: GroupCall; - roomName: string; - children: ReactNode; -} - -const SFUConfigContext = createContext(undefined); - -export const useSFUConfig = () => useContext(SFUConfigContext); - -export function OpenIDLoader({ client, groupCall, roomName, children }: Props) { - const [state, setState] = useState< - SFUConfigLoading | SFUConfigLoaded | SFUConfigFailed - >({ kind: "loading" }); - - useEffect(() => { - (async () => { - try { - const result = await getSFUConfigWithOpenID( - client, - groupCall, - roomName - ); - setState({ kind: "loaded", sfuConfig: result }); - } catch (e) { - logger.error("Failed to fetch SFU config: ", e); - setState({ kind: "failed", error: e as Error }); - } - })(); - }, [client, groupCall, roomName]); - - switch (state.kind) { - case "loading": - return ; - case "failed": - return ; - case "loaded": - return ( - - {children} - - ); - } -} - -type SFUConfigLoading = { - kind: "loading"; -}; - -type SFUConfigLoaded = { - kind: "loaded"; - sfuConfig: SFUConfig; -}; - -type SFUConfigFailed = { - kind: "failed"; - error: Error; -}; diff --git a/src/room/useGroupCall.ts b/src/room/useGroupCall.ts deleted file mode 100644 index 4f6d3ad7..00000000 --- a/src/room/useGroupCall.ts +++ /dev/null @@ -1,644 +0,0 @@ -/* -Copyright 2022-2023 New Vector Ltd - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -import { useCallback, useEffect, useReducer, useState } from "react"; -import * as Sentry from "@sentry/react"; -import { - GroupCallEvent, - GroupCallState, - GroupCall, - GroupCallErrorCode, - GroupCallUnknownDeviceError, - GroupCallError, - GroupCallStatsReportEvent, - GroupCallStatsReport, -} from "matrix-js-sdk/src/webrtc/groupCall"; -import { CallFeed, CallFeedEvent } from "matrix-js-sdk/src/webrtc/callFeed"; -import { RoomMember } from "matrix-js-sdk/src/models/room-member"; -import { useTranslation } from "react-i18next"; -import { IWidgetApiRequest } from "matrix-widget-api"; -import { MatrixClient, RoomStateEvent } from "matrix-js-sdk"; -import { - ByteSentStatsReport, - ConnectionStatsReport, - SummaryStatsReport, - CallFeedReport, -} from "matrix-js-sdk/src/webrtc/stats/statsReport"; -import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession"; - -import { usePageUnload } from "./usePageUnload"; -import { PosthogAnalytics } from "../analytics/PosthogAnalytics"; -import { TranslatedError, translatedError } from "../TranslatedError"; -import { ElementWidgetActions, ScreenshareStartData, widget } from "../widget"; -import { OTelGroupCallMembership } from "../otel/OTelGroupCallMembership"; -import { ElementCallOpenTelemetry } from "../otel/otel"; -import { checkForParallelCalls } from "./checkForParallelCalls"; - -enum ConnectionState { - EstablishingCall = "establishing call", // call hasn't been established yet - WaitMedia = "wait_media", // call is set up, waiting for ICE to connect - Connected = "connected", // media is flowing -} - -export interface ParticipantInfo { - connectionState: ConnectionState; - presenter: boolean; -} - -interface UseGroupCallReturnType { - state: GroupCallState; - localCallFeed?: CallFeed; - activeSpeaker?: CallFeed; - userMediaFeeds: CallFeed[]; - microphoneMuted: boolean; - localVideoMuted: boolean; - error?: TranslatedError; - initLocalCallFeed: () => void; - enter: () => Promise; - leave: () => void; - toggleLocalVideoMuted: () => void; - toggleMicrophoneMuted: () => void; - toggleScreensharing: () => void; - setMicrophoneMuted: (muted: boolean) => void; - requestingScreenshare: boolean; - isScreensharing: boolean; - screenshareFeeds: CallFeed[]; - participants: Map>; - hasLocalParticipant: boolean; - otelGroupCallMembership?: OTelGroupCallMembership; -} - -interface State { - state: GroupCallState; - localCallFeed?: CallFeed; - activeSpeaker?: CallFeed; - userMediaFeeds: CallFeed[]; - error?: TranslatedError; - microphoneMuted: boolean; - localVideoMuted: boolean; - screenshareFeeds: CallFeed[]; - isScreensharing: boolean; - requestingScreenshare: boolean; - participants: Map>; - hasLocalParticipant: boolean; -} - -// This is a bit of a hack, but we keep the opentelemetry tracker object at the file -// level so that it doesn't pop in & out of existence as react mounts & unmounts -// components. The right solution is probably for this to live in the js-sdk and have -// the same lifetime as groupcalls themselves. -let groupCallOTelMembership: OTelGroupCallMembership | undefined; -let groupCallOTelMembershipGroupCallId: string; - -/*function getParticipants( - rtcSession: MatrixRTCSession -): Map> { - const participants = new Map>(); - - for (const membership of rtcSession.memberships) { - const participantInfoMap = new Map(); - participants.set(membership.member, participantInfoMap); - - for (const [deviceId, participant] of participantsStateMap) { - const feed = groupCall.userMediaFeeds.find( - (f) => f.userId === member.userId && f.deviceId === deviceId - ); - - let connectionState: ConnectionState; - // If we allow calls without media, we have no feeds and cannot read the connection status from them. - // @TODO: The connection state should generally not be determined by the feed. - if ( - groupCall.allowCallWithoutVideoAndAudio && - !feed && - !participant.screensharing - ) { - connectionState = ConnectionState.Connected; - } else { - connectionState = feed - ? feed.connected - ? ConnectionState.Connected - : ConnectionState.WaitMedia - : ConnectionState.EstablishingCall; - } - participantInfoMap.set(deviceId, { - connectionState, - presenter: participant.screensharing, - }); - } - } - - return participants; -}*/ - -export function useGroupCall( - rtcSession: MatrixRTCSession, - client: MatrixClient -): UseGroupCallReturnType { - const [ - { - state, - localCallFeed, - activeSpeaker, - userMediaFeeds, - error, - microphoneMuted, - localVideoMuted, - isScreensharing, - screenshareFeeds, - participants, - hasLocalParticipant, - requestingScreenshare, - }, - setState, - ] = useState({ - state: GroupCallState.LocalCallFeedUninitialized, - userMediaFeeds: [], - microphoneMuted: false, - localVideoMuted: false, - isScreensharing: false, - screenshareFeeds: [], - requestingScreenshare: false, - participants: getParticipants(rtcSession), - hasLocalParticipant: false, - }); - - if (groupCallOTelMembershipGroupCallId !== groupCall.groupCallId) { - if (groupCallOTelMembership) groupCallOTelMembership.dispose(); - - // If the user disables analytics, this will stay around until they leave the call - // so analytics will be disabled once they leave. - if (ElementCallOpenTelemetry.instance) { - groupCallOTelMembership = new OTelGroupCallMembership(groupCall, client); - groupCallOTelMembershipGroupCallId = groupCall.groupCallId; - } else { - groupCallOTelMembership = undefined; - } - } - - const [unencryptedEventsFromUsers, addUnencryptedEventUser] = useReducer( - (state: Set, newVal: string) => { - return new Set(state).add(newVal); - }, - new Set() - ); - - const updateState = useCallback( - (state: Partial) => setState((prev) => ({ ...prev, ...state })), - [setState] - ); - - const doNothingMediaActionCallback = useCallback( - (details: MediaSessionActionDetails) => {}, - [] - ); - - const leaveCall = useCallback(() => { - groupCallOTelMembership?.onLeaveCall(); - groupCall.leave(); - }, [groupCall]); - - useEffect(() => { - // disable the media action keys, otherwise audio elements get paused when - // the user presses media keys or unplugs headphones, etc. - // Note there are actions for muting / unmuting a microphone & hanging up - // which we could wire up. - const mediaActions: MediaSessionAction[] = [ - "play", - "pause", - "stop", - "nexttrack", - "previoustrack", - ]; - - for (const mediaAction of mediaActions) { - navigator.mediaSession?.setActionHandler( - mediaAction, - doNothingMediaActionCallback - ); - } - - return () => { - for (const mediaAction of mediaActions) { - navigator.mediaSession?.setActionHandler(mediaAction, null); - } - }; - }, [doNothingMediaActionCallback]); - - useEffect(() => { - function onGroupCallStateChanged() { - updateState({ - state: groupCall.state, - localCallFeed: groupCall.localCallFeed, - activeSpeaker: groupCall.activeSpeaker, - userMediaFeeds: [...groupCall.userMediaFeeds], - microphoneMuted: groupCall.isMicrophoneMuted(), - localVideoMuted: groupCall.isLocalVideoMuted(), - isScreensharing: groupCall.isScreensharing(), - screenshareFeeds: [...groupCall.screenshareFeeds], - }); - } - - const prevUserMediaFeeds = new Set(); - - function onUserMediaFeedsChanged(userMediaFeeds: CallFeed[]): void { - for (const feed of prevUserMediaFeeds) { - feed.off(CallFeedEvent.ConnectedChanged, onConnectedChanged); - } - prevUserMediaFeeds.clear(); - - for (const feed of userMediaFeeds) { - feed.on(CallFeedEvent.ConnectedChanged, onConnectedChanged); - prevUserMediaFeeds.add(feed); - } - - updateState({ - userMediaFeeds: [...userMediaFeeds], - participants: getParticipants(groupCall), - }); - } - - const prevScreenshareFeeds = new Set(); - - function onScreenshareFeedsChanged(screenshareFeeds: CallFeed[]): void { - for (const feed of prevScreenshareFeeds) { - feed.off(CallFeedEvent.ConnectedChanged, onConnectedChanged); - } - prevScreenshareFeeds.clear(); - - for (const feed of screenshareFeeds) { - feed.on(CallFeedEvent.ConnectedChanged, onConnectedChanged); - prevScreenshareFeeds.add(feed); - } - - updateState({ - screenshareFeeds: [...screenshareFeeds], - }); - } - - function onConnectedChanged(connected: boolean): void { - updateState({ - participants: getParticipants(groupCall), - }); - } - - function onActiveSpeakerChanged(activeSpeaker: CallFeed | undefined): void { - updateState({ - activeSpeaker: activeSpeaker, - }); - } - - function onLocalMuteStateChanged( - microphoneMuted: boolean, - localVideoMuted: boolean - ): void { - updateState({ - microphoneMuted, - localVideoMuted, - }); - } - - function onLocalScreenshareStateChanged( - isScreensharing: boolean, - _localScreenshareFeed?: CallFeed, - localDesktopCapturerSourceId?: string - ): void { - updateState({ - isScreensharing, - }); - } - - function onCallsChanged(): void { - updateState({ participants: getParticipants(groupCall) }); - } - - function onParticipantsChanged(): void { - updateState({ - participants: getParticipants(groupCall), - hasLocalParticipant: groupCall.hasLocalParticipant(), - }); - } - - function onError(e: GroupCallError): void { - Sentry.captureException(e); - if (e.code === GroupCallErrorCode.UnknownDevice) { - const unknownDeviceError = e as GroupCallUnknownDeviceError; - addUnencryptedEventUser(unknownDeviceError.userId); - } - } - - function onConnectionStatsReport( - report: GroupCallStatsReport - ): void { - groupCallOTelMembership?.onConnectionStatsReport(report); - } - - function onByteSentStatsReport( - report: GroupCallStatsReport - ): void { - groupCallOTelMembership?.onByteSentStatsReport(report); - } - - function onSummaryStatsReport( - report: GroupCallStatsReport - ): void { - groupCallOTelMembership?.onSummaryStatsReport(report); - } - - function onCallFeedStatsReport( - report: GroupCallStatsReport - ): void { - groupCallOTelMembership?.onCallFeedStatsReport(report); - } - - groupCall.on(GroupCallEvent.GroupCallStateChanged, onGroupCallStateChanged); - groupCall.on(GroupCallEvent.UserMediaFeedsChanged, onUserMediaFeedsChanged); - groupCall.on( - GroupCallEvent.ScreenshareFeedsChanged, - onScreenshareFeedsChanged - ); - groupCall.on(GroupCallEvent.ActiveSpeakerChanged, onActiveSpeakerChanged); - groupCall.on(GroupCallEvent.LocalMuteStateChanged, onLocalMuteStateChanged); - groupCall.on( - GroupCallEvent.LocalScreenshareStateChanged, - onLocalScreenshareStateChanged - ); - groupCall.on(GroupCallEvent.CallsChanged, onCallsChanged); - groupCall.on(GroupCallEvent.ParticipantsChanged, onParticipantsChanged); - groupCall.on(GroupCallEvent.Error, onError); - groupCall.on( - GroupCallStatsReportEvent.ConnectionStats, - onConnectionStatsReport - ); - groupCall.on( - GroupCallStatsReportEvent.ByteSentStats, - onByteSentStatsReport - ); - groupCall.on(GroupCallStatsReportEvent.SummaryStats, onSummaryStatsReport); - groupCall.on( - GroupCallStatsReportEvent.CallFeedStats, - onCallFeedStatsReport - ); - - groupCall.room.currentState.on( - RoomStateEvent.Update, - checkForParallelCalls - ); - - updateState({ - error: undefined, - state: groupCall.state, - localCallFeed: groupCall.localCallFeed, - activeSpeaker: groupCall.activeSpeaker, - userMediaFeeds: [...groupCall.userMediaFeeds], - microphoneMuted: groupCall.isMicrophoneMuted(), - localVideoMuted: groupCall.isLocalVideoMuted(), - isScreensharing: groupCall.isScreensharing(), - screenshareFeeds: [...groupCall.screenshareFeeds], - participants: getParticipants(groupCall), - hasLocalParticipant: groupCall.hasLocalParticipant(), - }); - - return () => { - groupCall.removeListener( - GroupCallEvent.GroupCallStateChanged, - onGroupCallStateChanged - ); - groupCall.removeListener( - GroupCallEvent.UserMediaFeedsChanged, - onUserMediaFeedsChanged - ); - groupCall.removeListener( - GroupCallEvent.ScreenshareFeedsChanged, - onScreenshareFeedsChanged - ); - groupCall.removeListener( - GroupCallEvent.ActiveSpeakerChanged, - onActiveSpeakerChanged - ); - groupCall.removeListener( - GroupCallEvent.LocalMuteStateChanged, - onLocalMuteStateChanged - ); - groupCall.removeListener( - GroupCallEvent.LocalScreenshareStateChanged, - onLocalScreenshareStateChanged - ); - groupCall.removeListener(GroupCallEvent.CallsChanged, onCallsChanged); - groupCall.removeListener( - GroupCallEvent.ParticipantsChanged, - onParticipantsChanged - ); - groupCall.removeListener(GroupCallEvent.Error, onError); - groupCall.removeListener( - GroupCallStatsReportEvent.ConnectionStats, - onConnectionStatsReport - ); - groupCall.removeListener( - GroupCallStatsReportEvent.ByteSentStats, - onByteSentStatsReport - ); - groupCall.removeListener( - GroupCallStatsReportEvent.SummaryStats, - onSummaryStatsReport - ); - groupCall.removeListener( - GroupCallStatsReportEvent.CallFeedStats, - onCallFeedStatsReport - ); - groupCall.room.currentState.off( - RoomStateEvent.Update, - checkForParallelCalls - ); - leaveCall(); - }; - }, [groupCall, updateState, leaveCall]); - - usePageUnload(() => { - leaveCall(); - }); - - const initLocalCallFeed = useCallback( - () => groupCall.initLocalCallFeed(), - [groupCall] - ); - - const enter = useCallback(async () => { - if ( - groupCall.state !== GroupCallState.LocalCallFeedUninitialized && - groupCall.state !== GroupCallState.LocalCallFeedInitialized - ) { - return; - } - - PosthogAnalytics.instance.eventCallEnded.cacheStartCall(new Date()); - PosthogAnalytics.instance.eventCallStarted.track(groupCall.groupCallId); - - // This must be called before we start trying to join the call, as we need to - // have started tracking by the time calls start getting created. - groupCallOTelMembership?.onJoinCall(); - - await groupCall.enter().catch((error) => { - console.error(error); - updateState({ error }); - }); - }, [groupCall, updateState]); - - const toggleLocalVideoMuted = useCallback(() => { - const toggleToMute = !groupCall.isLocalVideoMuted(); - groupCall.setLocalVideoMuted(toggleToMute); - groupCallOTelMembership?.onToggleLocalVideoMuted(toggleToMute); - // TODO: These explict posthog calls should be unnecessary now with the posthog otel exporter? - PosthogAnalytics.instance.eventMuteCamera.track( - toggleToMute, - groupCall.groupCallId - ); - }, [groupCall]); - - const setMicrophoneMuted = useCallback( - (setMuted: boolean) => { - groupCall.setMicrophoneMuted(setMuted); - groupCallOTelMembership?.onSetMicrophoneMuted(setMuted); - PosthogAnalytics.instance.eventMuteMicrophone.track( - setMuted, - groupCall.groupCallId - ); - }, - [groupCall] - ); - - const toggleMicrophoneMuted = useCallback(() => { - const toggleToMute = !groupCall.isMicrophoneMuted(); - groupCallOTelMembership?.onToggleMicrophoneMuted(toggleToMute); - setMicrophoneMuted(toggleToMute); - }, [groupCall, setMicrophoneMuted]); - - const toggleScreensharing = useCallback(async () => { - groupCallOTelMembership?.onToggleScreensharing(!groupCall.isScreensharing); - - if (!groupCall.isScreensharing()) { - // toggling on - updateState({ requestingScreenshare: true }); - - try { - await groupCall.setScreensharingEnabled(true, { - audio: true, - throwOnFail: true, - }); - updateState({ requestingScreenshare: false }); - } catch (e) { - // this will fail in Electron because getDisplayMedia just throws a permission - // error, so if we have a widget API, try requesting via that. - if (widget) { - const reply = await widget.api.transport.send( - ElementWidgetActions.ScreenshareRequest, - {} - ); - if (!reply.pending) { - updateState({ requestingScreenshare: false }); - } - } - } - } else { - // toggling off - groupCall.setScreensharingEnabled(false); - } - }, [groupCall, updateState]); - - const onScreenshareStart = useCallback( - async (ev: CustomEvent) => { - updateState({ requestingScreenshare: false }); - - const data = ev.detail.data as unknown as ScreenshareStartData; - - await groupCall.setScreensharingEnabled(true, { - desktopCapturerSourceId: data.desktopCapturerSourceId as string, - audio: !data.desktopCapturerSourceId, - }); - await widget?.api.transport.reply(ev.detail, {}); - }, - [groupCall, updateState] - ); - - const onScreenshareStop = useCallback( - async (ev: CustomEvent) => { - updateState({ requestingScreenshare: false }); - await groupCall.setScreensharingEnabled(false); - await widget?.api.transport.reply(ev.detail, {}); - }, - [groupCall, updateState] - ); - - useEffect(() => { - if (widget) { - widget.lazyActions.on( - ElementWidgetActions.ScreenshareStart, - onScreenshareStart - ); - widget.lazyActions.on( - ElementWidgetActions.ScreenshareStop, - onScreenshareStop - ); - - return () => { - widget?.lazyActions.off( - ElementWidgetActions.ScreenshareStart, - onScreenshareStart - ); - widget?.lazyActions.off( - ElementWidgetActions.ScreenshareStop, - onScreenshareStop - ); - }; - } - }, [onScreenshareStart, onScreenshareStop]); - - const { t } = useTranslation(); - - useEffect(() => { - if (window.RTCPeerConnection === undefined) { - const error = translatedError( - "WebRTC is not supported or is being blocked in this browser.", - t - ); - console.error(error); - updateState({ error }); - } - }, [t, updateState]); - - return { - state, - localCallFeed, - activeSpeaker, - userMediaFeeds, - microphoneMuted, - localVideoMuted, - error, - initLocalCallFeed, - enter, - leave: leaveCall, - toggleLocalVideoMuted, - toggleMicrophoneMuted, - toggleScreensharing, - setMicrophoneMuted, - requestingScreenshare, - isScreensharing, - screenshareFeeds, - participants, - hasLocalParticipant, - unencryptedEventsFromUsers, - otelGroupCallMembership: groupCallOTelMembership, - }; -} From c516207199dd6e4d2430b38b670ab1c9efb612d4 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 23 Aug 2023 10:12:09 +0100 Subject: [PATCH 08/50] Remove matrix-widget-api as a dep & remove livekit stuff from widget init We'll always have matrix-widget-api as a dep through js-sdk so also specifyin it ourselves just means we'll end up using a different version when the js-sdk upgrade their copy and get wierd errors. We could add a peerDependency if we really felt the need? --- package.json | 1 - src/widget.ts | 16 ---------------- yarn.lock | 8 -------- 3 files changed, 25 deletions(-) diff --git a/package.json b/package.json index 77d10cfa..f592e9ee 100644 --- a/package.json +++ b/package.json @@ -58,7 +58,6 @@ "livekit-client": "1.12.3", "lodash": "^4.17.21", "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#0d46aeb5a4d44685885adc502f46688a4bed65b6", - "matrix-widget-api": "^1.3.1", "mermaid": "^8.13.8", "normalize.css": "^8.0.1", "pako": "^2.0.4", diff --git a/src/widget.ts b/src/widget.ts index de03d78e..324dbe4a 100644 --- a/src/widget.ts +++ b/src/widget.ts @@ -157,15 +157,6 @@ export const widget: WidgetHelpers | null = (() => { timelineSupport: true, useE2eForGroupCall: e2eEnabled, fallbackICEServerAllowed: allowIceFallback, - // XXX: The client expects the list of foci in its constructor, but we don't - // know this until we fetch the config file. However, we can't wait to construct - // the client object or we'll miss the 'capabilities' request from the host app. - // As of writing this, I have made the embedded widget client send the 'contentLoaded' - // message so that we can use the widget API in less racy mode, but we need to change - // element-web to use waitForIFrameLoad=false. Once that change has rolled out, - // we can just start the client after we've fetched the config. - livekitServiceURL: undefined, - useLivekitForGroupCalls: true, } ); @@ -174,13 +165,6 @@ export const widget: WidgetHelpers | null = (() => { // wait for the config file to be ready (we load very early on so it might not // be otherwise) await Config.init(); - const livekit = Config.get().livekit; - const focus = livekit?.livekit_service_url; - // Now we've fetched the config, be evil and use the getter to inject the focus - // into the client (see above XXX). - if (focus) { - client.setLivekitServiceURL(livekit.livekit_service_url); - } await client.startClient(); resolve(client); })(); diff --git a/yarn.lock b/yarn.lock index 4c2c0dac..2e336c79 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10962,14 +10962,6 @@ matrix-events-sdk@0.0.1: unhomoglyph "^1.0.6" uuid "9" -matrix-widget-api@^1.3.1: - version "1.4.0" - resolved "https://registry.yarnpkg.com/matrix-widget-api/-/matrix-widget-api-1.4.0.tgz#e426ec16a013897f3a4a9c2bff423f54ab0ba745" - integrity sha512-dw0dRylGQzDUoiaY/g5xx1tBbS7aoov31PRtFMAvG58/4uerYllV9Gfou7w+I1aglwB6hihTREzKltVjARWV6A== - dependencies: - "@types/events" "^3.0.0" - events "^3.2.0" - matrix-widget-api@^1.5.0: version "1.5.0" resolved "https://registry.yarnpkg.com/matrix-widget-api/-/matrix-widget-api-1.5.0.tgz#4ae3e46a7f2854f944ddaf8a5af63d72fba76c45" From af5e679089033cfffcdc31120a0d6b1c8285a019 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 23 Aug 2023 10:15:29 +0100 Subject: [PATCH 09/50] i18n --- public/locales/en-GB/app.json | 1 - 1 file changed, 1 deletion(-) diff --git a/public/locales/en-GB/app.json b/public/locales/en-GB/app.json index fc461337..30c770dd 100644 --- a/public/locales/en-GB/app.json +++ b/public/locales/en-GB/app.json @@ -114,7 +114,6 @@ "Waiting for other participants…": "Waiting for other participants…", "Walkie-talkie call": "Walkie-talkie call", "Walkie-talkie call name": "Walkie-talkie call name", - "WebRTC is not supported or is being blocked in this browser.": "WebRTC is not supported or is being blocked in this browser.", "Yes, join call": "Yes, join call", "You were disconnected from the call": "You were disconnected from the call", "Your feedback": "Your feedback", From 1955765cfdabee709d8c07d824a1a12d54965f8e Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 23 Aug 2023 14:03:38 +0100 Subject: [PATCH 10/50] Bump js-sdk --- package.json | 2 +- yarn.lock | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index f592e9ee..bf6220df 100644 --- a/package.json +++ b/package.json @@ -57,7 +57,7 @@ "i18next-http-backend": "^1.4.4", "livekit-client": "1.12.3", "lodash": "^4.17.21", - "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#0d46aeb5a4d44685885adc502f46688a4bed65b6", + "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#c8ae665eb07079d9f0d568aa1316e792062b5439", "mermaid": "^8.13.8", "normalize.css": "^8.0.1", "pako": "^2.0.4", diff --git a/yarn.lock b/yarn.lock index 2e336c79..8cce81d8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10943,9 +10943,9 @@ matrix-events-sdk@0.0.1: resolved "https://registry.yarnpkg.com/matrix-events-sdk/-/matrix-events-sdk-0.0.1.tgz#c8c38911e2cb29023b0bbac8d6f32e0de2c957dd" integrity sha512-1QEOsXO+bhyCroIe2/A5OwaxHvBm7EsSQ46DEDn8RBIfQwN5HWBpFvyWWR4QY0KHPPnnJdI99wgRiAl7Ad5qaA== -"matrix-js-sdk@github:matrix-org/matrix-js-sdk#0d46aeb5a4d44685885adc502f46688a4bed65b6": +"matrix-js-sdk@github:matrix-org/matrix-js-sdk#c8ae665eb07079d9f0d568aa1316e792062b5439": version "27.2.0" - resolved "https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/0d46aeb5a4d44685885adc502f46688a4bed65b6" + resolved "https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/c8ae665eb07079d9f0d568aa1316e792062b5439" dependencies: "@babel/runtime" "^7.12.5" "@matrix-org/matrix-sdk-crypto-wasm" "^1.2.1" From dc80a7e35072aaf55842605f35db24f87659488a Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 23 Aug 2023 15:04:41 +0100 Subject: [PATCH 11/50] Log ghosts and remove unsused code --- src/room/InCallView.tsx | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 61146d79..7e6aafd7 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -467,16 +467,6 @@ function useParticipantTiles( memberships.map((m) => [`${m.member.userId}:${m.deviceId}`, m.member]) ); - // The IDs of the participants who published membership event to the room (i.e. are present from Matrix perspective). - /*const matrixParticipants: Map = new Map( - [...participants.entries()].flatMap(([user, devicesMap]) => { - return [...devicesMap.keys()].map((deviceId) => [ - `${user.userId}:${deviceId}`, - user, - ]); - }) - );*/ - const hasPresenter = sfuParticipants.find((p) => p.isScreenShareEnabled) !== undefined; let allGhosts = true; @@ -493,6 +483,11 @@ function useParticipantTiles( const id = sfuParticipant.identity; const member = matrixParticipants.get(id); + if (member === undefined) { + logger.warn( + `Ruh, roh! No matrix member found for SFU participant ${id}: creating g-g-g-ghost!` + ); + } allGhosts &&= member === undefined; const userMediaTile = { From e4e35eecfd5bde60d8d4be0dcd2a21eb1bcb2f91 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 25 Aug 2023 13:19:23 +0100 Subject: [PATCH 12/50] Use Matrix room membership to look up room member Rather than the matrixRTC memberships. We're essentially trusting LiveKit's view of weho is connected here, so we may as well include the real names of anyone we don't think is a matrixRTC participant, for whatever reason. --- src/room/InCallView.tsx | 37 +++++++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 7e6aafd7..e5f56668 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -27,6 +27,7 @@ import classNames from "classnames"; import { DisconnectReason, Room, RoomEvent, Track } from "livekit-client"; import { MatrixClient } from "matrix-js-sdk/src/client"; import { RoomMember } from "matrix-js-sdk/src/models/room-member"; +import { Room as MatrixRoom } from "matrix-js-sdk/src/models/room"; import { Ref, useCallback, useEffect, useMemo, useRef } from "react"; import { useTranslation } from "react-i18next"; import useMeasure from "react-use-measure"; @@ -241,7 +242,7 @@ export function InCallView({ const reducedControls = boundsValid && bounds.width <= 400; const noControls = reducedControls && bounds.height <= 400; - const items = useParticipantTiles(livekitRoom, memberships); + const items = useParticipantTiles(livekitRoom, rtcSession.room); const { fullscreenItem, toggleFullscreen, exitFullscreen } = useFullscreen(items); @@ -454,19 +455,33 @@ export function InCallView({ ); } +function findMatrixMember( + room: MatrixRoom, + id: string +): RoomMember | undefined { + const parts = id.split(":"); + if (parts.length < 2) { + logger.warn( + "Livekit participants ID doesn't look like a userId:deviceId combination" + ); + return undefined; + } + + parts.pop(); + const userId = parts.join(":"); + + return room.getMember(userId) ?? undefined; +} + function useParticipantTiles( livekitRoom: Room, - memberships: CallMembership[] + matrixRoom: MatrixRoom ): TileDescriptor[] { const sfuParticipants = useParticipants({ room: livekitRoom, }); const items = useMemo(() => { - const matrixParticipants: Map = new Map( - memberships.map((m) => [`${m.member.userId}:${m.deviceId}`, m.member]) - ); - const hasPresenter = sfuParticipants.find((p) => p.isScreenShareEnabled) !== undefined; let allGhosts = true; @@ -482,10 +497,12 @@ function useParticipantTiles( : false; const id = sfuParticipant.identity; - const member = matrixParticipants.get(id); - if (member === undefined) { + const member = findMatrixMember(matrixRoom, id); + // We always start with a local participant wit the empty string as their ID before we're + // connected, this is fine and we'll be in "all ghosts" mode. + if (id !== "" && member === undefined) { logger.warn( - `Ruh, roh! No matrix member found for SFU participant ${id}: creating g-g-g-ghost!` + `Ruh, roh! No matrix member found for SFU participant '${id}': creating g-g-g-ghost!` ); } allGhosts &&= member === undefined; @@ -539,7 +556,7 @@ function useParticipantTiles( // If every item is a ghost, that probably means we're still connecting and // shouldn't bother showing anything yet return allGhosts ? [] : tiles; - }, [memberships, sfuParticipants]); + }, [matrixRoom, sfuParticipants]); return items; } From 992e6aa2a324fb07e5ce2d314f89ea3cb705181f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Mon, 28 Aug 2023 17:01:58 +0200 Subject: [PATCH 13/50] Update js-sdk MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index bf6220df..7a0ec680 100644 --- a/package.json +++ b/package.json @@ -57,7 +57,7 @@ "i18next-http-backend": "^1.4.4", "livekit-client": "1.12.3", "lodash": "^4.17.21", - "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#c8ae665eb07079d9f0d568aa1316e792062b5439", + "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#1a0718fc66d7f5684a1763c5fee9b9f5d0dafbb8", "mermaid": "^8.13.8", "normalize.css": "^8.0.1", "pako": "^2.0.4", From 4cd274b91eb6cf165926a5198d9178f85945de6a Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 29 Aug 2023 12:44:30 +0100 Subject: [PATCH 14/50] Manually disconnect & reconnect the livekit call if our focus changes Without breaking the 'disconnected' screen --- src/livekit/openIDSFU.ts | 7 +++ src/livekit/useECConnectionState.ts | 90 +++++++++++++++++++++++++++++ src/livekit/useLiveKit.ts | 20 +++++-- src/room/InCallView.tsx | 45 +++++++-------- 4 files changed, 135 insertions(+), 27 deletions(-) create mode 100644 src/livekit/useECConnectionState.ts diff --git a/src/livekit/openIDSFU.ts b/src/livekit/openIDSFU.ts index c122a844..d10f56fb 100644 --- a/src/livekit/openIDSFU.ts +++ b/src/livekit/openIDSFU.ts @@ -27,6 +27,13 @@ export interface SFUConfig { jwt: string; } +export function sfuConfigEquals(a?: SFUConfig, b?: SFUConfig): boolean { + if (a === undefined && b === undefined) return true; + if (a === undefined || b === undefined) return false; + + return a.jwt === b.jwt && a.url === b.url; +} + // The bits we need from MatrixClient export type OpenIDClientParts = Pick< MatrixClient, diff --git a/src/livekit/useECConnectionState.ts b/src/livekit/useECConnectionState.ts new file mode 100644 index 00000000..2e8bb430 --- /dev/null +++ b/src/livekit/useECConnectionState.ts @@ -0,0 +1,90 @@ +/* +Copyright 2023 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { ConnectionState, Room, RoomEvent } from "livekit-client"; +import { useCallback, useEffect, useRef, useState } from "react"; +import { logger } from "matrix-js-sdk/src/logger"; + +import { SFUConfig, sfuConfigEquals } from "./openIDSFU"; + +/* + * Additional values for states that a call can be in, beyond what livekit + * provides in ConnectionState. Also reconnects the call if the SFU Config + * changes. + */ +export enum ECAddonConnectionState { + // We are switching from one focus to another (or between livekit room aliases on the same focus) + ECSwitchingFocus = "ec_switching_focus", + // The call has just been initialised and is waiting for credentials to arrive before attempting + // to connect. This distinguishes from the 'Disconected' state which is now just for when livekit + // gives up on connectivity and we consider the call to have failed. + ECWaiting = "ec_waiting", +} + +export type ECConnectionState = ConnectionState | ECAddonConnectionState; + +export function useECConnectionState( + livekitRoom?: Room, + sfuConfig?: SFUConfig +): ECConnectionState { + const [connState, setConnState] = useState( + sfuConfig && livekitRoom + ? livekitRoom.state + : ECAddonConnectionState.ECWaiting + ); + + const [isSwitchingFocus, setSwitchingFocus] = useState(false); + + const onConnStateChanged = useCallback((state: ConnectionState) => { + if (state == ConnectionState.Connected) setSwitchingFocus(false); + setConnState(state); + }, []); + + useEffect(() => { + const oldRoom = livekitRoom; + + if (livekitRoom) { + livekitRoom.on(RoomEvent.ConnectionStateChanged, onConnStateChanged); + } + + return () => { + if (oldRoom) + oldRoom.off(RoomEvent.ConnectionStateChanged, onConnStateChanged); + }; + }, [livekitRoom, onConnStateChanged]); + + const currentSFUConfig = useRef(Object.assign({}, sfuConfig)); + + useEffect(() => { + if ( + sfuConfig && + currentSFUConfig.current && + !sfuConfigEquals(currentSFUConfig.current, sfuConfig) + ) { + logger.info("JWT changed!"); + + (async () => { + setSwitchingFocus(true); + await livekitRoom?.disconnect(); + await livekitRoom?.connect(sfuConfig.url, sfuConfig.jwt); + })(); + } + + currentSFUConfig.current = Object.assign({}, sfuConfig); + }, [sfuConfig, livekitRoom]); + + return isSwitchingFocus ? ECAddonConnectionState.ECSwitchingFocus : connState; +} diff --git a/src/livekit/useLiveKit.ts b/src/livekit/useLiveKit.ts index 14acfff3..1f5704b3 100644 --- a/src/livekit/useLiveKit.ts +++ b/src/livekit/useLiveKit.ts @@ -22,7 +22,7 @@ import { RoomOptions, setLogLevel, } from "livekit-client"; -import { useConnectionState, useLiveKitRoom } from "@livekit/components-react"; +import { useLiveKitRoom } from "@livekit/components-react"; import { useEffect, useMemo, useRef } from "react"; import E2EEWorker from "livekit-client/e2ee-worker?worker"; import { logger } from "matrix-js-sdk/src/logger"; @@ -35,6 +35,10 @@ import { MediaDevices, useMediaDevices, } from "./MediaDevicesContext"; +import { + ECConnectionState, + useECConnectionState, +} from "./useECConnectionState"; export type E2EEConfig = { sharedKey: string; @@ -42,11 +46,16 @@ export type E2EEConfig = { setLogLevel("debug"); +interface UseLivekitResult { + livekitRoom?: Room; + connState: ECConnectionState; +} + export function useLiveKit( muteStates: MuteStates, sfuConfig?: SFUConfig, e2eeConfig?: E2EEConfig -): Room | undefined { +): UseLivekitResult { const e2eeOptions = useMemo(() => { if (!e2eeConfig?.sharedKey) return undefined; @@ -101,7 +110,7 @@ export function useLiveKit( room: roomWithoutProps, }); - const connectionState = useConnectionState(roomWithoutProps); + const connectionState = useECConnectionState(room, sfuConfig); useEffect(() => { // Sync the requested mute states with LiveKit's mute states. We do it this @@ -149,5 +158,8 @@ export function useLiveKit( } }, [room, devices, connectionState]); - return room; + return { + connState: connectionState, + livekitRoom: room, + }; } diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index e5f56668..76cb7c07 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -24,7 +24,7 @@ import { } from "@livekit/components-react"; import { usePreventScroll } from "@react-aria/overlays"; import classNames from "classnames"; -import { DisconnectReason, Room, RoomEvent, Track } from "livekit-client"; +import { Room, Track, ConnectionState } from "livekit-client"; import { MatrixClient } from "matrix-js-sdk/src/client"; import { RoomMember } from "matrix-js-sdk/src/models/room-member"; import { Room as MatrixRoom } from "matrix-js-sdk/src/models/room"; @@ -34,7 +34,6 @@ import useMeasure from "react-use-measure"; import { OverlayTriggerState } from "@react-stately/overlays"; import { JoinRule } from "matrix-js-sdk/src/@types/partials"; import { logger } from "matrix-js-sdk/src/logger"; -import { RoomEventCallbacks } from "livekit-client/dist/src/room/Room"; import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession"; import { CallMembership } from "matrix-js-sdk/src/matrixrtc/CallMembership"; @@ -74,12 +73,12 @@ import { E2EEConfig, useLiveKit } from "../livekit/useLiveKit"; import { useFullscreen } from "./useFullscreen"; import { useLayoutStates } from "../video-grid/Layout"; import { E2EELock } from "../E2EELock"; -import { useEventEmitterThree } from "../useEvents"; import { useWakeLock } from "../useWakeLock"; import { useMergedRefs } from "../useMergedRefs"; import { MuteStates } from "./MuteStates"; import { useIsRoomE2EE } from "../e2ee/sharedKeyManagement"; import { useOpenIDSFU } from "../livekit/openIDSFU"; +import { ECConnectionState } from "../livekit/useECConnectionState"; const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {}); // There is currently a bug in Safari our our code with cloning and sending MediaStreams @@ -87,13 +86,18 @@ const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {}); // For now we can disable screensharing in Safari. const isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent); -export interface ActiveCallProps extends Omit { +export interface ActiveCallProps + extends Omit { e2eeConfig?: E2EEConfig; } export function ActiveCall(props: ActiveCallProps) { const sfuConfig = useOpenIDSFU(props.client, props.rtcSession); - const livekitRoom = useLiveKit(props.muteStates, sfuConfig, props.e2eeConfig); + const { livekitRoom, connState } = useLiveKit( + props.muteStates, + sfuConfig, + props.e2eeConfig + ); if (!livekitRoom) { return null; @@ -105,7 +109,7 @@ export function ActiveCall(props: ActiveCallProps) { return ( - + ); } @@ -119,6 +123,7 @@ export interface InCallViewProps { onLeave: (error?: Error) => void; hideHeader: boolean; otelGroupCallMembership?: OTelGroupCallMembership; + connState: ECConnectionState; } export function InCallView({ @@ -130,11 +135,20 @@ export function InCallView({ onLeave, hideHeader, otelGroupCallMembership, + connState, }: InCallViewProps) { const { t } = useTranslation(); usePreventScroll(); useWakeLock(); + useEffect(() => { + if (connState === ConnectionState.Disconnected) { + // annoyingly we don't get the disconnection reason this way, + // only by listening for the emitted event + onLeave(new Error("Disconnected from call server")); + } + }, [connState, onLeave]); + const isRoomE2EE = useIsRoomE2EE(rtcSession.room.roomId); const containerRef1 = useRef(null); @@ -182,27 +196,10 @@ export function InCallView({ async (muted) => await localParticipant.setMicrophoneEnabled(!muted) ); - const onDisconnected = useCallback( - (reason?: DisconnectReason) => { - PosthogAnalytics.instance.eventCallDisconnected.track(reason); - logger.info("Disconnected from livekit call with reason ", reason); - onLeave( - new Error("Disconnected from LiveKit call with reason " + reason) - ); - }, - [onLeave] - ); - const onLeavePress = useCallback(() => { onLeave(); }, [onLeave]); - useEventEmitterThree( - livekitRoom, - RoomEvent.Disconnected, - onDisconnected - ); - useEffect(() => { widget?.api.transport.send( layout === "freedom" @@ -459,6 +456,8 @@ function findMatrixMember( room: MatrixRoom, id: string ): RoomMember | undefined { + if (!id) return undefined; + const parts = id.split(":"); if (parts.length < 2) { logger.warn( From 192c3295c9593472853d6fa74dfe7676702118c4 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 29 Aug 2023 14:19:08 +0100 Subject: [PATCH 15/50] Update js-sdk --- package.json | 2 +- yarn.lock | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 7a0ec680..99e6588c 100644 --- a/package.json +++ b/package.json @@ -57,7 +57,7 @@ "i18next-http-backend": "^1.4.4", "livekit-client": "1.12.3", "lodash": "^4.17.21", - "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#1a0718fc66d7f5684a1763c5fee9b9f5d0dafbb8", + "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#c444e374407cee40643438e01270e1e6e2835e82", "mermaid": "^8.13.8", "normalize.css": "^8.0.1", "pako": "^2.0.4", diff --git a/yarn.lock b/yarn.lock index 8cce81d8..ffca4a3c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10943,9 +10943,9 @@ matrix-events-sdk@0.0.1: resolved "https://registry.yarnpkg.com/matrix-events-sdk/-/matrix-events-sdk-0.0.1.tgz#c8c38911e2cb29023b0bbac8d6f32e0de2c957dd" integrity sha512-1QEOsXO+bhyCroIe2/A5OwaxHvBm7EsSQ46DEDn8RBIfQwN5HWBpFvyWWR4QY0KHPPnnJdI99wgRiAl7Ad5qaA== -"matrix-js-sdk@github:matrix-org/matrix-js-sdk#c8ae665eb07079d9f0d568aa1316e792062b5439": +"matrix-js-sdk@github:matrix-org/matrix-js-sdk#c444e374407cee40643438e01270e1e6e2835e82": version "27.2.0" - resolved "https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/c8ae665eb07079d9f0d568aa1316e792062b5439" + resolved "https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/c444e374407cee40643438e01270e1e6e2835e82" dependencies: "@babel/runtime" "^7.12.5" "@matrix-org/matrix-sdk-crypto-wasm" "^1.2.1" From b256755a0d98ee1d0d931fc8060fdcb397347212 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 31 Aug 2023 09:44:23 +0100 Subject: [PATCH 16/50] Don't treat empty object as a valid sfu config This was causing an extra reconnect cycle when the call was first joined because it thought the previous SFU config was valid. This was probably causing some client to fail to connect at all. --- src/livekit/useECConnectionState.ts | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/livekit/useECConnectionState.ts b/src/livekit/useECConnectionState.ts index 2e8bb430..01adb64d 100644 --- a/src/livekit/useECConnectionState.ts +++ b/src/livekit/useECConnectionState.ts @@ -36,6 +36,12 @@ export enum ECAddonConnectionState { export type ECConnectionState = ConnectionState | ECAddonConnectionState; +// This is mostly necessary because an empty useRef is an empty object +// which is truthy, so we can't just use Boolean(currentSFUConfig.current) +function sfuConfigValid(sfuConfig?: SFUConfig): boolean { + return Boolean(sfuConfig?.url) && Boolean(sfuConfig?.jwt); +} + export function useECConnectionState( livekitRoom?: Room, sfuConfig?: SFUConfig @@ -68,18 +74,22 @@ export function useECConnectionState( const currentSFUConfig = useRef(Object.assign({}, sfuConfig)); + // Id we are transitioning from a valid config to another valid one, we need + // to explicitly switch focus useEffect(() => { if ( - sfuConfig && - currentSFUConfig.current && + sfuConfigValid(sfuConfig) && + sfuConfigValid(currentSFUConfig.current) && !sfuConfigEquals(currentSFUConfig.current, sfuConfig) ) { - logger.info("JWT changed!"); + logger.info( + `SFU config changed! URL was ${currentSFUConfig.current?.url} now ${sfuConfig?.url}` + ); (async () => { setSwitchingFocus(true); await livekitRoom?.disconnect(); - await livekitRoom?.connect(sfuConfig.url, sfuConfig.jwt); + await livekitRoom?.connect(sfuConfig!.url, sfuConfig!.jwt); })(); } From 0535a35a1cc968f45fa044355067e4317550c894 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 31 Aug 2023 14:36:54 +0100 Subject: [PATCH 17/50] Remove unused memberships prop --- src/room/GroupCallView.tsx | 1 - src/room/InCallView.tsx | 3 --- 2 files changed, 4 deletions(-) diff --git a/src/room/GroupCallView.tsx b/src/room/GroupCallView.tsx index 57182b04..1a449e9d 100644 --- a/src/room/GroupCallView.tsx +++ b/src/room/GroupCallView.tsx @@ -271,7 +271,6 @@ export function GroupCallView({ void; hideHeader: boolean; otelGroupCallMembership?: OTelGroupCallMembership; @@ -131,7 +129,6 @@ export function InCallView({ rtcSession, livekitRoom, muteStates, - memberships, onLeave, hideHeader, otelGroupCallMembership, From 981a29ebf78a95fa73e5caf8e36435f27238bde6 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 31 Aug 2023 14:38:31 +0100 Subject: [PATCH 18/50] yarn --- yarn.lock | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/yarn.lock b/yarn.lock index b2f396ee..fb5a53a8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11057,6 +11057,14 @@ matrix-events-sdk@0.0.1: unhomoglyph "^1.0.6" uuid "9" +matrix-widget-api@^1.3.1: + version "1.6.0" + resolved "https://registry.yarnpkg.com/matrix-widget-api/-/matrix-widget-api-1.6.0.tgz#f0075411edffc6de339580ade7e6e6e6edb01af4" + integrity sha512-VXIJyAZ/WnBmT4C7ePqevgMYGneKMCP/0JuCOqntSsaNlCRHJvwvTxmqUU+ufOpzIF5gYNyIrAjbgrEbK3iqJQ== + dependencies: + "@types/events" "^3.0.0" + events "^3.2.0" + matrix-widget-api@^1.5.0: version "1.5.0" resolved "https://registry.yarnpkg.com/matrix-widget-api/-/matrix-widget-api-1.5.0.tgz#4ae3e46a7f2854f944ddaf8a5af63d72fba76c45" From fe4d42f4d0de4d1959ab7e58cfabfe539f1290b3 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 31 Aug 2023 14:43:53 +0100 Subject: [PATCH 19/50] Update js-sdk --- package.json | 2 +- yarn.lock | 18 +++++------------- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/package.json b/package.json index 7539a17c..6abbcba2 100644 --- a/package.json +++ b/package.json @@ -58,7 +58,7 @@ "i18next-http-backend": "^1.4.4", "livekit-client": "^1.12.3", "lodash": "^4.17.21", - "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#c444e374407cee40643438e01270e1e6e2835e82", + "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#fc2671d8538a3339ca5925ef42e3ba6102f7e8f2", "matrix-widget-api": "^1.3.1", "mermaid": "^9.0.0", "normalize.css": "^8.0.1", diff --git a/yarn.lock b/yarn.lock index fb5a53a8..62da65eb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11038,9 +11038,9 @@ matrix-events-sdk@0.0.1: resolved "https://registry.yarnpkg.com/matrix-events-sdk/-/matrix-events-sdk-0.0.1.tgz#c8c38911e2cb29023b0bbac8d6f32e0de2c957dd" integrity sha512-1QEOsXO+bhyCroIe2/A5OwaxHvBm7EsSQ46DEDn8RBIfQwN5HWBpFvyWWR4QY0KHPPnnJdI99wgRiAl7Ad5qaA== -"matrix-js-sdk@github:matrix-org/matrix-js-sdk#c444e374407cee40643438e01270e1e6e2835e82": - version "27.2.0" - resolved "https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/c444e374407cee40643438e01270e1e6e2835e82" +"matrix-js-sdk@github:matrix-org/matrix-js-sdk#fc2671d8538a3339ca5925ef42e3ba6102f7e8f2": + version "28.0.0" + resolved "https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/fc2671d8538a3339ca5925ef42e3ba6102f7e8f2" dependencies: "@babel/runtime" "^7.12.5" "@matrix-org/matrix-sdk-crypto-wasm" "^1.2.1" @@ -11050,14 +11050,14 @@ matrix-events-sdk@0.0.1: jwt-decode "^3.1.2" loglevel "^1.7.1" matrix-events-sdk "0.0.1" - matrix-widget-api "^1.5.0" + matrix-widget-api "^1.6.0" oidc-client-ts "^2.2.4" p-retry "4" sdp-transform "^2.14.1" unhomoglyph "^1.0.6" uuid "9" -matrix-widget-api@^1.3.1: +matrix-widget-api@^1.3.1, matrix-widget-api@^1.6.0: version "1.6.0" resolved "https://registry.yarnpkg.com/matrix-widget-api/-/matrix-widget-api-1.6.0.tgz#f0075411edffc6de339580ade7e6e6e6edb01af4" integrity sha512-VXIJyAZ/WnBmT4C7ePqevgMYGneKMCP/0JuCOqntSsaNlCRHJvwvTxmqUU+ufOpzIF5gYNyIrAjbgrEbK3iqJQ== @@ -11065,14 +11065,6 @@ matrix-widget-api@^1.3.1: "@types/events" "^3.0.0" events "^3.2.0" -matrix-widget-api@^1.5.0: - version "1.5.0" - resolved "https://registry.yarnpkg.com/matrix-widget-api/-/matrix-widget-api-1.5.0.tgz#4ae3e46a7f2854f944ddaf8a5af63d72fba76c45" - integrity sha512-hKGfqQKK5qVMwW0Sp8l2TiuW8UuHafTvUZNSWBPghedB/rSFbVLlr0mufuEV0iq/pQ7ChW96q/WEC6Llie4SnA== - dependencies: - "@types/events" "^3.0.0" - events "^3.2.0" - md5.js@^1.3.4: version "1.3.5" resolved "https://registry.yarnpkg.com/md5.js/-/md5.js-1.3.5.tgz#b5d07b8e3216e3e27cd728d72f70d1e6a342005f" From 40283ab7609700d0293db880210f8f436e29507a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Thu, 31 Aug 2023 15:24:25 +0200 Subject: [PATCH 20/50] Remove unused `Facepile` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/Facepile.module.css | 42 ---------------- src/Facepile.tsx | 97 ------------------------------------- src/home/CallList.tsx | 22 +-------- src/home/RegisteredView.tsx | 2 +- 4 files changed, 3 insertions(+), 160 deletions(-) delete mode 100644 src/Facepile.module.css delete mode 100644 src/Facepile.tsx diff --git a/src/Facepile.module.css b/src/Facepile.module.css deleted file mode 100644 index 0c911658..00000000 --- a/src/Facepile.module.css +++ /dev/null @@ -1,42 +0,0 @@ -/* -Copyright 2022 New Vector Ltd - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -.facepile { - width: 100%; - position: relative; -} - -.facepile.xs { - height: 24px; -} - -.facepile.sm { - height: 32px; -} - -.facepile.md { - height: 36px; -} - -.facepile .avatar { - position: absolute; - top: 0; - border: 1px solid var(--cpd-color-bg-canvas-default); -} - -.facepile.md .avatar { - border-width: 2px; -} diff --git a/src/Facepile.tsx b/src/Facepile.tsx deleted file mode 100644 index 0c9ec239..00000000 --- a/src/Facepile.tsx +++ /dev/null @@ -1,97 +0,0 @@ -/* -Copyright 2022 New Vector Ltd - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -import { HTMLAttributes, useMemo } from "react"; -import classNames from "classnames"; -import { MatrixClient } from "matrix-js-sdk/src/client"; -import { RoomMember } from "matrix-js-sdk/src/models/room-member"; -import { useTranslation } from "react-i18next"; - -import styles from "./Facepile.module.css"; -import { Avatar, Size, sizes } from "./Avatar"; - -const overlapMap: Partial> = { - [Size.XS]: 2, - [Size.SM]: 4, - [Size.MD]: 8, -}; - -interface Props extends HTMLAttributes { - className: string; - client: MatrixClient; - members: RoomMember[]; - max?: number; - size?: Size; -} - -export function Facepile({ - className, - client, - members, - max = 3, - size = Size.XS, - ...rest -}: Props) { - const { t } = useTranslation(); - - const _size = sizes.get(size)!; - const _overlap = overlapMap[size]!; - - const title = useMemo(() => { - return members.reduce( - (prev, curr) => - prev === null - ? curr.name - : t("{{names}}, {{name}}", { names: prev, name: curr.name }), - null - ) as string; - }, [members, t]); - - return ( -
    - {members.slice(0, max).map((member, i) => { - const avatarUrl = member.getMxcAvatarUrl(); - return ( - - ); - })} - {members.length > max && ( - - )} -
    - ); -} diff --git a/src/home/CallList.tsx b/src/home/CallList.tsx index 1e36360e..01b84369 100644 --- a/src/home/CallList.tsx +++ b/src/home/CallList.tsx @@ -19,7 +19,6 @@ import { MatrixClient } from "matrix-js-sdk/src/client"; import { RoomMember } from "matrix-js-sdk/src/models/room-member"; import { CopyButton } from "../button"; -import { Facepile } from "../Facepile"; import { Avatar, Size } from "../Avatar"; import styles from "./CallList.module.css"; import { getRoomUrl } from "../matrix-utils"; @@ -30,9 +29,8 @@ import { useRoomSharedKey } from "../e2ee/sharedKeyManagement"; interface CallListProps { rooms: GroupCallRoom[]; client: MatrixClient; - disableFacepile?: boolean; } -export function CallList({ rooms, client, disableFacepile }: CallListProps) { +export function CallList({ rooms, client }: CallListProps) { return ( <>
    @@ -44,7 +42,6 @@ export function CallList({ rooms, client, disableFacepile }: CallListProps) { avatarUrl={avatarUrl} roomId={room.roomId} participants={participants} - disableFacepile={disableFacepile} /> ))} {rooms.length > 3 && ( @@ -63,16 +60,8 @@ interface CallTileProps { roomId: string; participants: RoomMember[]; client: MatrixClient; - disableFacepile?: boolean; } -function CallTile({ - name, - avatarUrl, - roomId, - participants, - client, - disableFacepile, -}: CallTileProps) { +function CallTile({ name, avatarUrl, roomId }: CallTileProps) { const roomSharedKey = useRoomSharedKey(roomId); return ( @@ -89,13 +78,6 @@ function CallTile({ {name} - {participants && !disableFacepile && ( - - )}
    diff --git a/src/home/RegisteredView.tsx b/src/home/RegisteredView.tsx index 42aba9e8..0f8cc93c 100644 --- a/src/home/RegisteredView.tsx +++ b/src/home/RegisteredView.tsx @@ -170,7 +170,7 @@ export function RegisteredView({ client, isPasswordlessUser }: Props) { {t("Your recent calls")} - + )} From 973d3962495ccd1776921d73e9e4eb97073929cd Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 5 Sep 2023 17:34:55 +0100 Subject: [PATCH 21/50] Remove extra device request on the video preview page As per comment, livekit mutates the object that's passed in, so we ended up re-requesting the devices in the next render because we effectively passed in different options. --- src/main.tsx | 6 +++--- src/room/VideoPreview.tsx | 7 ++++++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/main.tsx b/src/main.tsx index d04a3a94..efc6e8d7 100644 --- a/src/main.tsx +++ b/src/main.tsx @@ -57,7 +57,7 @@ Initializer.initBeforeReact(); const history = createBrowserHistory(); root.render( - - - + // + + // ); diff --git a/src/room/VideoPreview.tsx b/src/room/VideoPreview.tsx index 1b5c07ce..044a3221 100644 --- a/src/room/VideoPreview.tsx +++ b/src/room/VideoPreview.tsx @@ -68,6 +68,9 @@ export const VideoPreview: FC = ({ matrixInfo, muteStates }) => { const devices = useMediaDevices(); + // Capture the audio options as they were when we first mounted, because + // we're not doing anything with the audio anyway so we don't need to + // re-open the devices when they change (see below). const initialAudioOptions = useRef(); initialAudioOptions.current ??= muteStates.audio.enabled && { deviceId: devices.audioInput.selectedId, @@ -79,7 +82,9 @@ export const VideoPreview: FC = ({ matrixInfo, muteStates }) => { // request over with at the same time. But changing the audio settings // shouldn't cause this hook to recreate the track, which is why we // reference the initial values here. - audio: initialAudioOptions.current, + // We also pass in a clone because livekit mutates the object passed in, + // which would cause the devices to be re-opened on the next render. + audio: Object.assign({}, initialAudioOptions.current), video: muteStates.video.enabled && { deviceId: devices.videoInput.selectedId, }, From 3e726043914365d9999d165837a0f4f031057677 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 5 Sep 2023 17:38:43 +0100 Subject: [PATCH 22/50] Remove unintentional change --- src/main.tsx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main.tsx b/src/main.tsx index efc6e8d7..d04a3a94 100644 --- a/src/main.tsx +++ b/src/main.tsx @@ -57,7 +57,7 @@ Initializer.initBeforeReact(); const history = createBrowserHistory(); root.render( - // - - // + + + ); From 1cb0ad2f65306430e73a5b6da4b8c97a5d6189f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Thu, 31 Aug 2023 15:46:09 +0200 Subject: [PATCH 23/50] Switch to `Avatar` from Compound MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/Avatar.module.css | 76 ------------------------ src/Avatar.tsx | 89 +++++++---------------------- src/UserMenu.module.css | 14 ----- src/UserMenu.tsx | 6 +- src/UserMenuContainer.tsx | 1 + src/home/CallList.tsx | 8 +-- src/input/AvatarInputField.tsx | 15 ++++- src/room/GroupCallView.tsx | 3 +- src/room/VideoPreview.tsx | 4 +- src/settings/ProfileSettingsTab.tsx | 6 +- src/video-grid/VideoTile.tsx | 5 +- 11 files changed, 52 insertions(+), 175 deletions(-) delete mode 100644 src/Avatar.module.css diff --git a/src/Avatar.module.css b/src/Avatar.module.css deleted file mode 100644 index accff6ae..00000000 --- a/src/Avatar.module.css +++ /dev/null @@ -1,76 +0,0 @@ -/* -Copyright 2022 New Vector Ltd - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -.avatar { - position: relative; - color: var(--stopgap-color-on-solid-accent); - display: flex; - align-items: center; - justify-content: center; - pointer-events: none; - font-weight: 600; - overflow: hidden; - flex-shrink: 0; -} - -.avatar img { - width: 100%; - height: 100%; - object-fit: cover; -} - -.avatar svg * { - fill: var(--cpd-color-text-primary); -} - -.avatar span { - padding-top: 1px; -} - -.xs { - width: 22px; - height: 22px; - border-radius: 22px; - font-size: 14px; -} - -.sm { - width: 32px; - height: 32px; - border-radius: 32px; - font-size: 15px; -} - -.md { - width: 36px; - height: 36px; - border-radius: 36px; - font-size: 20px; -} - -.lg { - width: 42px; - height: 42px; - border-radius: 42px; - font-size: 24px; -} - -.xl { - width: 90px; - height: 90px; - border-radius: 90px; - font-size: 48px; -} diff --git a/src/Avatar.tsx b/src/Avatar.tsx index c11d71d4..e23bd909 100644 --- a/src/Avatar.tsx +++ b/src/Avatar.tsx @@ -14,23 +14,11 @@ See the License for the specific language governing permissions and limitations under the License. */ -import { useMemo, CSSProperties, HTMLAttributes, FC } from "react"; -import classNames from "classnames"; +import { useMemo, FC } from "react"; +import { Avatar as CompoundAvatar } from "@vector-im/compound-web"; import { getAvatarUrl } from "./matrix-utils"; import { useClient } from "./ClientContext"; -import styles from "./Avatar.module.css"; - -const backgroundColors = [ - "#5C56F5", - "#03B381", - "#368BD6", - "#AC3BA8", - "#E64F7A", - "#FF812D", - "#2DC2C5", - "#74D12C", -]; export enum Size { XS = "xs", @@ -48,50 +36,28 @@ export const sizes = new Map([ [Size.XL, 90], ]); -function hashStringToArrIndex(str: string, arrLength: number) { - let sum = 0; - - for (let i = 0; i < str.length; i++) { - sum += str.charCodeAt(i); - } - - return sum % arrLength; -} - -interface Props extends HTMLAttributes { - bgKey?: string; +interface Props { + id: string; + name: string; + className?: string; src?: string; size?: Size | number; - className?: string; - style?: CSSProperties; - fallback: string; } export const Avatar: FC = ({ - bgKey, - src, - fallback, - size = Size.MD, className, - style = {}, - ...rest + id, + name, + src, + size = Size.MD, }) => { const { client } = useClient(); - const [sizeClass, sizePx, sizeStyle] = useMemo( + const sizePx = useMemo( () => Object.values(Size).includes(size as Size) - ? [styles[size as string], sizes.get(size as Size), {}] - : [ - null, - size as number, - { - width: size, - height: size, - borderRadius: size, - fontSize: Math.round((size as number) / 2), - }, - ], + ? sizes.get(size as Size) + : (size as number), [size] ); @@ -100,28 +66,13 @@ export const Avatar: FC = ({ return src.startsWith("mxc://") ? getAvatarUrl(client, src, sizePx) : src; }, [client, src, sizePx]); - const backgroundColor = useMemo(() => { - const index = hashStringToArrIndex( - bgKey || fallback || src || "", - backgroundColors.length - ); - return backgroundColors[index]; - }, [bgKey, src, fallback]); - - /* eslint-disable jsx-a11y/alt-text */ return ( -
    - {resolvedSrc ? ( - - ) : typeof fallback === "string" ? ( - {fallback} - ) : ( - fallback - )} -
    + ); }; diff --git a/src/UserMenu.module.css b/src/UserMenu.module.css index d1db1071..575b71b9 100644 --- a/src/UserMenu.module.css +++ b/src/UserMenu.module.css @@ -24,17 +24,3 @@ limitations under the License. .userButton svg * { fill: var(--cpd-color-icon-primary); } - -.avatar { - width: 24px; - height: 24px; - font-size: var(--font-size-caption); -} - -@media (min-width: 800px) { - .avatar { - width: 32px; - height: 32px; - font-size: var(--font-size-body); - } -} diff --git a/src/UserMenu.tsx b/src/UserMenu.tsx index 9df3309d..515e71f0 100644 --- a/src/UserMenu.tsx +++ b/src/UserMenu.tsx @@ -35,6 +35,7 @@ interface UserMenuProps { preventNavigation: boolean; isAuthenticated: boolean; isPasswordlessUser: boolean; + userId: string; displayName: string; avatarUrl?: string; onAction: (value: string) => void; @@ -44,6 +45,7 @@ export function UserMenu({ preventNavigation, isAuthenticated, isPasswordlessUser, + userId, displayName, avatarUrl, onAction, @@ -109,10 +111,10 @@ export function UserMenu({ > {isAuthenticated && (!isPasswordlessUser || avatarUrl) ? ( ) : ( diff --git a/src/UserMenuContainer.tsx b/src/UserMenuContainer.tsx index 6a83133e..a03e5b5a 100644 --- a/src/UserMenuContainer.tsx +++ b/src/UserMenuContainer.tsx @@ -67,6 +67,7 @@ export function UserMenuContainer({ preventNavigation = false }: Props) { isPasswordlessUser={passwordlessUser} avatarUrl={avatarUrl} onAction={onAction} + userId={client?.getUserId() ?? ""} displayName={displayName || (userName ? userName.replace("@", "") : "")} /> {modalState.isOpen && client && ( diff --git a/src/home/CallList.tsx b/src/home/CallList.tsx index 01b84369..a82e4c13 100644 --- a/src/home/CallList.tsx +++ b/src/home/CallList.tsx @@ -67,13 +67,7 @@ function CallTile({ name, avatarUrl, roomId }: CallTileProps) { return (
    - +
    {name} diff --git a/src/input/AvatarInputField.tsx b/src/input/AvatarInputField.tsx index 0218258e..8a069718 100644 --- a/src/input/AvatarInputField.tsx +++ b/src/input/AvatarInputField.tsx @@ -35,13 +35,23 @@ interface Props extends AllHTMLAttributes { id: string; label: string; avatarUrl: string | undefined; + userId: string; displayName: string; onRemoveAvatar: () => void; } export const AvatarInputField = forwardRef( ( - { id, label, className, avatarUrl, displayName, onRemoveAvatar, ...rest }, + { + id, + label, + className, + avatarUrl, + userId, + displayName, + onRemoveAvatar, + ...rest + }, ref ) => { const { t } = useTranslation(); @@ -80,9 +90,10 @@ export const AvatarInputField = forwardRef(
    { return { + userId: client.getUserId()!, displayName: displayName!, avatarUrl: avatarUrl!, roomId: groupCall.room.roomId, roomName: groupCall.room.name, roomAlias: groupCall.room.getCanonicalAlias(), }; - }, [displayName, avatarUrl, groupCall]); + }, [client, displayName, avatarUrl, groupCall]); const deviceContext = useMediaDevices(); const latestDevices = useRef(); diff --git a/src/room/VideoPreview.tsx b/src/room/VideoPreview.tsx index 1b5c07ce..807d402f 100644 --- a/src/room/VideoPreview.tsx +++ b/src/room/VideoPreview.tsx @@ -35,6 +35,7 @@ import { useMediaDevices } from "../livekit/MediaDevicesContext"; import { MuteStates } from "./MuteStates"; export type MatrixInfo = { + userId: string; displayName: string; avatarUrl: string; roomId: string; @@ -124,9 +125,10 @@ export const VideoPreview: FC = ({ matrixInfo, muteStates }) => { {!muteStates.video.enabled && (
    )} diff --git a/src/settings/ProfileSettingsTab.tsx b/src/settings/ProfileSettingsTab.tsx index e6a59634..4286a960 100644 --- a/src/settings/ProfileSettingsTab.tsx +++ b/src/settings/ProfileSettingsTab.tsx @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import { useCallback, useEffect, useRef } from "react"; +import { useCallback, useEffect, useMemo, useRef } from "react"; import { MatrixClient } from "matrix-js-sdk/src/client"; import { useTranslation } from "react-i18next"; @@ -29,6 +29,7 @@ interface Props { export function ProfileSettingsTab({ client }: Props) { const { t } = useTranslation(); const { error, displayName, avatarUrl, saveProfile } = useProfile(client); + const userId = useMemo(() => client.getUserId(), [client]); const formRef = useRef(null); @@ -77,12 +78,13 @@ export function ProfileSettingsTab({ client }: Props) { return (
    - {displayName && ( + {userId && displayName && ( diff --git a/src/video-grid/VideoTile.tsx b/src/video-grid/VideoTile.tsx index f15052e0..3433f8aa 100644 --- a/src/video-grid/VideoTile.tsx +++ b/src/video-grid/VideoTile.tsx @@ -145,6 +145,8 @@ export const VideoTile = forwardRef( // Firefox doesn't respect the disablePictureInPicture attribute // https://bugzilla.mozilla.org/show_bug.cgi?id=1611831 + console.log(`LOG VideoTIle mxcSrc=${member?.getMxcAvatarUrl()}`); + return ( (
    From 40e31607d21d5aab15556662fad0daaeb6cf9f45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Mon, 4 Sep 2023 20:04:24 +0200 Subject: [PATCH 24/50] i18n MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- public/locales/en-GB/app.json | 1 - 1 file changed, 1 deletion(-) diff --git a/public/locales/en-GB/app.json b/public/locales/en-GB/app.json index f2605959..9864bcf4 100644 --- a/public/locales/en-GB/app.json +++ b/public/locales/en-GB/app.json @@ -3,7 +3,6 @@ "{{count}} stars|other": "{{count}} stars", "{{displayName}} is presenting": "{{displayName}} is presenting", "{{displayName}}, your call has ended.": "{{displayName}}, your call has ended.", - "{{names}}, {{name}}": "{{names}}, {{name}}", "<0><1>You may withdraw consent by unchecking this box. If you are currently in a call, this setting will take effect at the end of the call.": "<0><1>You may withdraw consent by unchecking this box. If you are currently in a call, this setting will take effect at the end of the call.", "<0>Already have an account?<1><0>Log in Or <2>Access as a guest": "<0>Already have an account?<1><0>Log in Or <2>Access as a guest", "<0>Create an account Or <2>Access as a guest": "<0>Create an account Or <2>Access as a guest", From 8f570b7893179988566501aaa2ce8cd3e472a4b7 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 6 Sep 2023 09:12:04 +0100 Subject: [PATCH 25/50] Include the colon in the user ID in the numebr of parts we check for --- src/room/InCallView.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index c7e17036..68e4e80a 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -456,7 +456,8 @@ function findMatrixMember( if (!id) return undefined; const parts = id.split(":"); - if (parts.length < 2) { + // must be at least 3 parts because we know the first part is a userId which must necessarily contain a colon + if (parts.length < 3) { logger.warn( "Livekit participants ID doesn't look like a userId:deviceId combination" ); From ef037c5c1db48d699ab42411373189cd488c4fe3 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 6 Sep 2023 10:22:02 +0100 Subject: [PATCH 26/50] Add apple site association --- public/apple-app-site-association.json | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 public/apple-app-site-association.json diff --git a/public/apple-app-site-association.json b/public/apple-app-site-association.json new file mode 100644 index 00000000..eff625a4 --- /dev/null +++ b/public/apple-app-site-association.json @@ -0,0 +1,17 @@ +{ + "applinks": { + "apps": [], + "details": [ + { + "appIDs":[ + "7J4U792NQT.io.element.elementx", + "7J4U792NQT.io.element.elementx.nightly", + "7J4U792NQT.io.element.elementx.pr" + ], + "paths": [ + "*" + ] + } + ] + } +} From 5d969563f739062b5b96fe43561c6b475a22d162 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 6 Sep 2023 10:35:41 +0100 Subject: [PATCH 27/50] Prettier --- public/apple-app-site-association.json | 28 ++++++++++++-------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/public/apple-app-site-association.json b/public/apple-app-site-association.json index eff625a4..b27b5c71 100644 --- a/public/apple-app-site-association.json +++ b/public/apple-app-site-association.json @@ -1,17 +1,15 @@ { - "applinks": { - "apps": [], - "details": [ - { - "appIDs":[ - "7J4U792NQT.io.element.elementx", - "7J4U792NQT.io.element.elementx.nightly", - "7J4U792NQT.io.element.elementx.pr" - ], - "paths": [ - "*" - ] - } - ] - } + "applinks": { + "apps": [], + "details": [ + { + "appIDs": [ + "7J4U792NQT.io.element.elementx", + "7J4U792NQT.io.element.elementx.nightly", + "7J4U792NQT.io.element.elementx.pr" + ], + "paths": ["*"] + } + ] + } } From 00f14c98143c0511c85a8b4a7d87f28fca970c43 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 6 Sep 2023 11:45:01 +0100 Subject: [PATCH 28/50] Fix the mute icon being incorrect when using PTT We were manipulating the participant's mute state directly for some reason, just for setting the mute state directly, which bypased the mutestates hook. --- src/room/InCallView.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 1d3701bc..9bb2a4d9 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -190,7 +190,8 @@ export function InCallView({ containerRef1, toggleMicrophone, toggleCamera, - async (muted) => await localParticipant.setMicrophoneEnabled(!muted) + (muted) => + muteStates?.audio?.setEnabled && muteStates.audio.setEnabled(!muted) ); const onDisconnected = useCallback( From 1757bc546dc1f328f1de1aa7f731497da30b390c Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 6 Sep 2023 13:59:37 +0100 Subject: [PATCH 29/50] Rename to not have file extension As this is what apple need. Also make the docker container serve it with the right content type. --- config/nginx.conf | 4 ++++ ...e-app-site-association.json => apple-app-site-association} | 0 2 files changed, 4 insertions(+) rename public/{apple-app-site-association.json => apple-app-site-association} (100%) diff --git a/config/nginx.conf b/config/nginx.conf index f7253465..5dc69b2f 100644 --- a/config/nginx.conf +++ b/config/nginx.conf @@ -21,5 +21,9 @@ server { expires 1w; add_header Cache-Control "public, no-transform"; } + + location /apple-app-site-association { + default_type application/json; + } } diff --git a/public/apple-app-site-association.json b/public/apple-app-site-association similarity index 100% rename from public/apple-app-site-association.json rename to public/apple-app-site-association From b0e7b949daca13fe79f0de0845b62431feb2ac5f Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 6 Sep 2023 16:03:54 +0100 Subject: [PATCH 30/50] Assetlinks for android too --- public/.well-known/assetlinks.json | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 public/.well-known/assetlinks.json diff --git a/public/.well-known/assetlinks.json b/public/.well-known/assetlinks.json new file mode 100644 index 00000000..f49328c2 --- /dev/null +++ b/public/.well-known/assetlinks.json @@ -0,0 +1,12 @@ +[ + { + "relation": ["delegate_permission/common.handle_all_urls"], + "target": { + "namespace": "android_app", + "package_name": "io.element.android.x.debug", + "sha256_cert_fingerprints": [ + "BC:9D:56:30:0C:3C:DB:80:16:EE:0D:3F:40:E6:65:C0:78:CA:B8:99:2E:DD:6E:FB:2B:CA:FB:60:48:FE:BF:58" + ] + } + } +] From da5bef1206c7bd13316d8afddae3bef0195ff4cc Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 6 Sep 2023 16:23:49 +0100 Subject: [PATCH 31/50] Wrong finger or something --- public/.well-known/assetlinks.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/.well-known/assetlinks.json b/public/.well-known/assetlinks.json index f49328c2..0eb37bf0 100644 --- a/public/.well-known/assetlinks.json +++ b/public/.well-known/assetlinks.json @@ -5,7 +5,7 @@ "namespace": "android_app", "package_name": "io.element.android.x.debug", "sha256_cert_fingerprints": [ - "BC:9D:56:30:0C:3C:DB:80:16:EE:0D:3F:40:E6:65:C0:78:CA:B8:99:2E:DD:6E:FB:2B:CA:FB:60:48:FE:BF:58" + "B0:B0:51:DC:56:5C:81:2F:E1:7F:6F:3E:94:5B:4D:79:04:71:23:AB:0D:A6:12:86:76:9E:B2:94:91:97:13:0E" ] } } From 38bb2895db59f25517eec9692eed72ae95f52436 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 7 Sep 2023 13:16:40 +0100 Subject: [PATCH 32/50] Tidier syntax Co-authored-by: Robin --- src/room/InCallView.tsx | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 9bb2a4d9..1eeafece 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -190,8 +190,7 @@ export function InCallView({ containerRef1, toggleMicrophone, toggleCamera, - (muted) => - muteStates?.audio?.setEnabled && muteStates.audio.setEnabled(!muted) + (muted) => muteStates.audio.setEnabled?.(!muted) ); const onDisconnected = useCallback( From 3545a8152a2a49d922375dfedfeebd0423d14d2c Mon Sep 17 00:00:00 2001 From: Robin Date: Fri, 8 Sep 2023 15:39:49 -0400 Subject: [PATCH 33/50] Improve accessibility of buttons I noticed that none of these buttons had accessible labels, which is obviously no good since they rely on icons alone to convey purpose when not focused. --- src/button/Button.tsx | 64 +++++++++++++++---------------------------- 1 file changed, 22 insertions(+), 42 deletions(-) diff --git a/src/button/Button.tsx b/src/button/Button.tsx index ac02c6ed..2b8049e5 100644 --- a/src/button/Button.tsx +++ b/src/button/Button.tsx @@ -13,7 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -import { forwardRef, useCallback } from "react"; +import { forwardRef } from "react"; import { PressEvent } from "@react-types/shared"; import classNames from "classnames"; import { useButton } from "@react-aria/button"; @@ -27,13 +27,11 @@ import { ReactComponent as VideoCallOffIcon } from "@vector-im/compound-design-t import { ReactComponent as EndCallIcon } from "@vector-im/compound-design-tokens/icons/end-call.svg"; import { ReactComponent as ShareScreenSolidIcon } from "@vector-im/compound-design-tokens/icons/share-screen-solid.svg"; import { ReactComponent as SettingsSolidIcon } from "@vector-im/compound-design-tokens/icons/settings-solid.svg"; -import { ReactComponent as UserAddSolidIcon } from "@vector-im/compound-design-tokens/icons/user-add-solid.svg"; import { ReactComponent as ChevronDownIcon } from "@vector-im/compound-design-tokens/icons/chevron-down.svg"; import styles from "./Button.module.css"; import { ReactComponent as Fullscreen } from "../icons/Fullscreen.svg"; import { ReactComponent as FullscreenExit } from "../icons/FullscreenExit.svg"; -import { TooltipTrigger } from "../Tooltip"; import { VolumeIcon } from "./VolumeIcon"; export type ButtonVariant = @@ -146,11 +144,13 @@ export function MicButton({ [index: string]: unknown; }) { const { t } = useTranslation(); + const Icon = muted ? MicOffSolidIcon : MicOnSolidIcon; + const label = muted ? t("Microphone off") : t("Microphone on"); return ( - + ); @@ -165,11 +165,13 @@ export function VideoButton({ [index: string]: unknown; }) { const { t } = useTranslation(); + const Icon = muted ? VideoCallOffIcon : VideoCallIcon; + const label = muted ? t("Video off") : t("Video on"); return ( - + ); @@ -186,11 +188,12 @@ export function ScreenshareButton({ [index: string]: unknown; }) { const { t } = useTranslation(); + const label = enabled ? t("Sharing screen") : t("Share screen"); return ( - + ); @@ -213,7 +216,7 @@ export function HangupButton({ className={classNames(styles.hangupButton, className)} {...rest} > - + ); @@ -232,28 +235,7 @@ export function SettingsButton({ return ( - - ); -} - -export function InviteButton({ - className, - variant = "toolbar", - ...rest -}: { - className?: string; - variant?: string; - // TODO: add all props for ); @@ -268,14 +250,13 @@ interface AudioButtonProps extends Omit { export function AudioButton({ volume, ...rest }: AudioButtonProps) { const { t } = useTranslation(); - const tooltip = useCallback(() => t("Local volume"), [t]); return ( - + - + ); } @@ -288,15 +269,14 @@ export function FullscreenButton({ ...rest }: FullscreenButtonProps) { const { t } = useTranslation(); - const tooltip = useCallback(() => { - return fullscreen ? t("Exit full screen") : t("Full screen"); - }, [fullscreen, t]); + const Icon = fullscreen ? FullscreenExit : Fullscreen; + const label = fullscreen ? t("Exit full screen") : t("Full screen"); return ( - + - + ); } From 1445c42cc32644a1b573a800418febb81912ed83 Mon Sep 17 00:00:00 2001 From: Robin Date: Fri, 8 Sep 2023 15:39:10 -0400 Subject: [PATCH 34/50] Implement new in-call header and footer designs --- public/locales/en-GB/app.json | 16 +-- src/E2EELock.tsx | 55 --------- src/Facepile.tsx | 66 +++++++++++ src/Header.module.css | 77 ++++++------- src/Header.stories.jsx | 105 ------------------ src/Header.tsx | 92 ++++++++------- src/button/ShareButton.tsx | 31 ++++++ src/button/VolumeIcon.tsx | 14 ++- src/icons/AddUser.svg | 4 - src/icons/AlertTriangleFilled.svg | 3 - src/icons/ArrowLeft.svg | 3 - src/icons/DisableVideo.svg | 6 - src/icons/Freedom.svg | 6 - src/icons/Hangup.svg | 4 - src/icons/LogoMark.svg | 11 ++ src/icons/LogoType.svg | 17 +++ src/icons/Screenshare.svg | 3 - src/icons/Spotlight.svg | 3 - src/icons/VideoMuted.svg | 6 - src/index.css | 4 + .../EncryptionLock.module.css} | 28 +++-- src/room/EncryptionLock.tsx | 46 ++++++++ src/room/GridLayoutMenu.module.css | 0 src/room/GridLayoutMenu.tsx | 82 -------------- src/room/GroupCallView.tsx | 84 ++++++++++---- src/room/InCallView.module.css | 50 +++++---- src/room/InCallView.tsx | 102 ++++++++--------- src/room/LayoutToggle.module.css | 77 +++++++++++++ src/room/LayoutToggle.tsx | 75 +++++++++++++ src/room/LobbyView.tsx | 20 +++- ...Modal.module.css => ShareModal.module.css} | 0 src/room/{InviteModal.tsx => ShareModal.tsx} | 6 +- src/room/VideoPreview.tsx | 2 + src/room/useGroupCall.ts | 17 +-- src/room/useRoomName.ts | 27 +++++ src/useMediaQuery.ts | 37 ++++++ src/usePrefersReducedMotion.ts | 27 +---- src/video-grid/NewVideoGrid.module.css | 10 +- src/video-grid/VideoGrid.tsx | 18 +-- vite.config.js | 8 +- yarn.lock | 6 +- 41 files changed, 700 insertions(+), 548 deletions(-) delete mode 100644 src/E2EELock.tsx create mode 100644 src/Facepile.tsx delete mode 100644 src/Header.stories.jsx create mode 100644 src/button/ShareButton.tsx delete mode 100644 src/icons/AddUser.svg delete mode 100644 src/icons/AlertTriangleFilled.svg delete mode 100644 src/icons/ArrowLeft.svg delete mode 100644 src/icons/DisableVideo.svg delete mode 100644 src/icons/Freedom.svg delete mode 100644 src/icons/Hangup.svg create mode 100644 src/icons/LogoMark.svg create mode 100644 src/icons/LogoType.svg delete mode 100644 src/icons/Screenshare.svg delete mode 100644 src/icons/Spotlight.svg delete mode 100644 src/icons/VideoMuted.svg rename src/{E2EELock.module.css => room/EncryptionLock.module.css} (62%) create mode 100644 src/room/EncryptionLock.tsx delete mode 100644 src/room/GridLayoutMenu.module.css delete mode 100644 src/room/GridLayoutMenu.tsx create mode 100644 src/room/LayoutToggle.module.css create mode 100644 src/room/LayoutToggle.tsx rename src/room/{InviteModal.module.css => ShareModal.module.css} (100%) rename src/room/{InviteModal.tsx => ShareModal.tsx} (90%) create mode 100644 src/room/useRoomName.ts create mode 100644 src/useMediaQuery.ts diff --git a/public/locales/en-GB/app.json b/public/locales/en-GB/app.json index 9864bcf4..69127826 100644 --- a/public/locales/en-GB/app.json +++ b/public/locales/en-GB/app.json @@ -1,8 +1,11 @@ { + "{{count, number}}|one": "{{count, number}}", + "{{count, number}}|other": "{{count, number}}", "{{count}} stars|one": "{{count}} star", "{{count}} stars|other": "{{count}} stars", "{{displayName}} is presenting": "{{displayName}} is presenting", "{{displayName}}, your call has ended.": "{{displayName}}, your call has ended.", + "{{names, list(style: short;)}}": "{{names, list(style: short;)}}", "<0><1>You may withdraw consent by unchecking this box. If you are currently in a call, this setting will take effect at the end of the call.": "<0><1>You may withdraw consent by unchecking this box. If you are currently in a call, this setting will take effect at the end of the call.", "<0>Already have an account?<1><0>Log in Or <2>Access as a guest": "<0>Already have an account?<1><0>Log in Or <2>Access as a guest", "<0>Create an account Or <2>Access as a guest": "<0>Create an account Or <2>Access as a guest", @@ -21,7 +24,6 @@ "Call link copied": "Call link copied", "Call type menu": "Call type menu", "Camera": "Camera", - "Change layout": "Change layout", "Close": "Close", "Confirm password": "Confirm password", "Connectivity to the server has been lost.": "Connectivity to the server has been lost.", @@ -31,7 +33,6 @@ "Create account": "Create account", "Debug log": "Debug log", "Debug log request": "Debug log request", - "Details": "Details", "Developer": "Developer", "Developer Settings": "Developer Settings", "Display name": "Display name", @@ -39,25 +40,22 @@ "Element Call Home": "Element Call Home", "Element Call is temporarily not end-to-end encrypted while we test scalability.": "Element Call is temporarily not end-to-end encrypted while we test scalability.", "Enable end-to-end encryption (password protected calls)": "Enable end-to-end encryption (password protected calls)", + "Encrypted": "Encrypted", "End call": "End call", "End-to-end encryption isn't supported on your browser.": "End-to-end encryption isn't supported on your browser.", "Exit full screen": "Exit full screen", "Expose developer settings in the settings window.": "Expose developer settings in the settings window.", "Feedback": "Feedback", "Fetching group call timed out.": "Fetching group call timed out.", - "Freedom": "Freedom", "Full screen": "Full screen", "Go": "Go", - "Grid layout menu": "Grid layout menu", + "Grid": "Grid", "Home": "Home", "How did it go?": "How did it go?", "If you are experiencing issues or simply would like to provide some feedback, please send us a short description below.": "If you are experiencing issues or simply would like to provide some feedback, please send us a short description below.", "Include debug logs": "Include debug logs", "Incompatible versions": "Incompatible versions", - "Incompatible versions!": "Incompatible versions!", "Inspector": "Inspector", - "Invite": "Invite", - "Invite people": "Invite people", "Join call": "Join call", "Join call now": "Join call now", "Join existing call?": "Join existing call?", @@ -71,6 +69,7 @@ "Microphone on": "Microphone on", "More": "More", "No": "No", + "Not encrypted": "Not encrypted", "Not now, return to home screen": "Not now, return to home screen", "Not registered yet? <2>Create an account": "Not registered yet? <2>Create an account", "Other users are trying to join this call from incompatible versions. These users should ensure that they have refreshed their browsers:<1>{userLis}": "Other users are trying to join this call from incompatible versions. These users should ensure that they have refreshed their browsers:<1>{userLis}", @@ -90,7 +89,9 @@ "Sending debug logs…": "Sending debug logs…", "Sending…": "Sending…", "Settings": "Settings", + "Share": "Share", "Share screen": "Share screen", + "Share this call": "Share this call", "Sharing screen": "Sharing screen", "Show call inspector": "Show call inspector", "Show connection stats": "Show connection stats", @@ -105,7 +106,6 @@ "Thanks, we received your feedback!": "Thanks, we received your feedback!", "Thanks!": "Thanks!", "This call already exists, would you like to join?": "This call already exists, would you like to join?", - "This call is not end-to-end encrypted.": "This call is not end-to-end encrypted.", "This site is protected by ReCAPTCHA and the Google <2>Privacy Policy and <6>Terms of Service apply.<9>By clicking \"Register\", you agree to our <12>End User Licensing Agreement (EULA)": "This site is protected by ReCAPTCHA and the Google <2>Privacy Policy and <6>Terms of Service apply.<9>By clicking \"Register\", you agree to our <12>End User Licensing Agreement (EULA)", "User menu": "User menu", "Username": "Username", diff --git a/src/E2EELock.tsx b/src/E2EELock.tsx deleted file mode 100644 index 9a9a55e9..00000000 --- a/src/E2EELock.tsx +++ /dev/null @@ -1,55 +0,0 @@ -/* -Copyright 2023 New Vector Ltd - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -import { useTranslation } from "react-i18next"; -import { useCallback } from "react"; -import { useObjectRef } from "@react-aria/utils"; -import { useButton } from "@react-aria/button"; - -import styles from "./E2EELock.module.css"; -import { ReactComponent as LockOffIcon } from "./icons/LockOff.svg"; -import { TooltipTrigger } from "./Tooltip"; - -export const E2EELock = () => { - const { t } = useTranslation(); - const tooltip = useCallback( - () => t("This call is not end-to-end encrypted."), - [t] - ); - - return ( - - - - ); -}; - -/** - * This component is a bit of hack - for some reason for the TooltipTrigger to - * work, it needs to contain a component which uses the useButton hook; please - * note that for some reason this also needs to be a separate component and we - * cannot just use the useButton hook inside the E2EELock. - */ -const Icon = () => { - const buttonRef = useObjectRef(); - const { buttonProps } = useButton({}, buttonRef); - - return ( -
    - -
    - ); -}; diff --git a/src/Facepile.tsx b/src/Facepile.tsx new file mode 100644 index 00000000..7ed995ce --- /dev/null +++ b/src/Facepile.tsx @@ -0,0 +1,66 @@ +/* +Copyright 2022 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { HTMLAttributes } from "react"; +import { MatrixClient } from "matrix-js-sdk/src/client"; +import { RoomMember } from "matrix-js-sdk/src/models/room-member"; +import { useTranslation } from "react-i18next"; +import { AvatarStack } from "@vector-im/compound-web"; + +import { Avatar, Size } from "./Avatar"; + +interface Props extends HTMLAttributes { + className?: string; + client: MatrixClient; + members: RoomMember[]; + max?: number; + size?: Size | number; +} + +export function Facepile({ + className, + client, + members, + max = 3, + size = Size.XS, + ...rest +}: Props) { + const { t } = useTranslation(); + + const displayedMembers = members.slice(0, max); + + return ( + m.name), + })} + {...rest} + > + {displayedMembers.map((member, i) => { + const avatarUrl = member.getMxcAvatarUrl(); + return ( + + ); + })} + + ); +} diff --git a/src/Header.module.css b/src/Header.module.css index e54edff5..84b9df7b 100644 --- a/src/Header.module.css +++ b/src/Header.module.css @@ -28,8 +28,8 @@ limitations under the License. flex: 1; align-items: center; white-space: nowrap; - padding: 0 20px; - height: 64px; + padding-inline: var(--inline-content-inset); + height: 80px; } .headerLogo { @@ -66,51 +66,56 @@ limitations under the License. margin-right: 0; } +.roomHeaderInfo { + display: grid; + column-gap: var(--cpd-space-4x); + grid-template-columns: auto auto; + grid-template-rows: 1fr auto; +} + +.roomHeaderInfo[data-size="sm"] { + grid-template-areas: "avatar name" ". participants"; +} + +.roomHeaderInfo[data-size="lg"] { + grid-template-areas: "avatar name" "avatar participants"; +} + .roomAvatar { - position: relative; - display: none; - justify-content: center; + align-self: flex-start; + grid-area: avatar; +} + +.nameLine { + grid-area: name; + flex-grow: 1; + display: flex; align-items: center; - width: 36px; - height: 36px; - border-radius: 36px; - background-color: #5c56f5; + gap: var(--cpd-space-1x); } -.roomAvatar > * { - fill: white; - stroke: white; -} - -.userName { - font-weight: 600; - margin-right: 8px; - text-overflow: ellipsis; +.nameLine > h1 { + margin: 0; + /* XXX I can't actually get this ellipsis overflow to trigger, because + constraint propagation in a nested flexbox layout is a massive pain */ overflow: hidden; - flex-shrink: 1; + text-overflow: ellipsis; } -.versionMismatchWarning { - padding-left: 15px; +.nameLine > svg { + flex-shrink: 0; } -.versionMismatchWarning::before { - content: ""; - display: inline-block; - position: relative; - top: 1px; - width: 16px; - height: 16px; - mask-image: url("./icons/AlertTriangleFilled.svg"); - mask-repeat: no-repeat; - mask-size: contain; - background-color: var(--cpd-color-icon-critical-primary); - padding-right: 5px; +.participantsLine { + grid-area: participants; + display: flex; + align-items: center; + gap: var(--cpd-space-1-5x); + font: var(--cpd-font-body-sm-medium); } @media (min-width: 800px) { .headerLogo, - .roomAvatar, .leftNav.hideMobile, .rightNav.hideMobile { display: flex; @@ -119,8 +124,4 @@ limitations under the License. .leftNav h3 { font-size: var(--font-size-subtitle); } - - .nav { - height: 76px; - } } diff --git a/src/Header.stories.jsx b/src/Header.stories.jsx deleted file mode 100644 index e5c4473a..00000000 --- a/src/Header.stories.jsx +++ /dev/null @@ -1,105 +0,0 @@ -import { GridLayoutMenu } from "./room/GridLayoutMenu"; -import { - Header, - HeaderLogo, - LeftNav, - RightNav, - RoomHeaderInfo, -} from "./Header"; -import { UserMenu } from "./UserMenu"; - -export default { - title: "Header", - component: Header, - parameters: { - layout: "fullscreen", - }, -}; - -export const HomeAnonymous = () => ( -
    - - - - - - -
    -); - -export const HomeNamedGuest = () => ( -
    - - - - - - -
    -); - -export const HomeLoggedIn = () => ( -
    - - - - - - -
    -); - -export const LobbyNamedGuest = () => ( -
    - - - - - - -
    -); - -export const LobbyLoggedIn = () => ( -
    - - - - - - -
    -); - -export const InRoomNamedGuest = () => ( -
    - - - - - - - -
    -); - -export const InRoomLoggedIn = () => ( -
    - - - - - - - -
    -); - -export const CreateAccount = () => ( -
    - - - - -
    -); diff --git a/src/Header.tsx b/src/Header.tsx index 8d754b92..aea3da71 100644 --- a/src/Header.tsx +++ b/src/Header.tsx @@ -15,17 +15,18 @@ limitations under the License. */ import classNames from "classnames"; -import { HTMLAttributes, ReactNode, useCallback } from "react"; +import { FC, HTMLAttributes, ReactNode } from "react"; import { Link } from "react-router-dom"; -import { Room } from "matrix-js-sdk/src/models/room"; import { useTranslation } from "react-i18next"; +import { MatrixClient, RoomMember } from "matrix-js-sdk/src/matrix"; +import { Heading } from "@vector-im/compound-web"; import styles from "./Header.module.css"; -import { useModalTriggerState } from "./Modal"; -import { Button } from "./button"; import { ReactComponent as Logo } from "./icons/Logo.svg"; -import { Subtitle } from "./typography/Typography"; -import { IncompatibleVersionModal } from "./IncompatibleVersionModal"; +import { Avatar, Size } from "./Avatar"; +import { Facepile } from "./Facepile"; +import { EncryptionLock } from "./room/EncryptionLock"; +import { useMediaQuery } from "./useMediaQuery"; interface HeaderProps extends HTMLAttributes { children: ReactNode; @@ -112,47 +113,52 @@ export function HeaderLogo({ className }: HeaderLogoProps) { ); } -interface RoomHeaderInfo { - roomName: string; +interface RoomHeaderInfoProps { + id: string; + name: string; + avatarUrl: string | null; + encrypted: boolean; + participants: RoomMember[]; + client: MatrixClient; } -export function RoomHeaderInfo({ roomName }: RoomHeaderInfo) { - return ( - <> - - {roomName} - - - ); -} - -interface VersionMismatchWarningProps { - users: Set; - room: Room; -} - -export function VersionMismatchWarning({ - users, - room, -}: VersionMismatchWarningProps) { +export const RoomHeaderInfo: FC = ({ + id, + name, + avatarUrl, + encrypted, + participants, + client, +}) => { const { t } = useTranslation(); - const { modalState, modalProps } = useModalTriggerState(); - - const onDetailsClick = useCallback(() => { - modalState.open(); - }, [modalState]); - - if (users.size === 0) return null; + const size = useMediaQuery("(max-width: 550px)") ? "sm" : "lg"; return ( - - {t("Incompatible versions!")} - - {modalState.isOpen && ( - +
    + +
    + + {name} + + +
    + {participants.length > 0 && ( +
    + + {t("{{count, number}}", { count: participants.length })} +
    )} - +
    ); -} +}; diff --git a/src/button/ShareButton.tsx b/src/button/ShareButton.tsx new file mode 100644 index 00000000..2f7f1334 --- /dev/null +++ b/src/button/ShareButton.tsx @@ -0,0 +1,31 @@ +/* +Copyright 2023 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { ComponentPropsWithoutRef, FC } from "react"; +import { Button } from "@vector-im/compound-web"; +import { useTranslation } from "react-i18next"; +import { ReactComponent as UserAddSolidIcon } from "@vector-im/compound-design-tokens/icons/user-add-solid.svg"; + +export const ShareButton: FC< + Omit, "children"> +> = (props) => { + const { t } = useTranslation(); + return ( + + ); +}; diff --git a/src/button/VolumeIcon.tsx b/src/button/VolumeIcon.tsx index 163699f6..00aebb06 100644 --- a/src/button/VolumeIcon.tsx +++ b/src/button/VolumeIcon.tsx @@ -15,19 +15,21 @@ See the License for the specific language governing permissions and limitations under the License. */ +import { ComponentPropsWithoutRef, FC } from "react"; + import { ReactComponent as AudioMuted } from "../icons/AudioMuted.svg"; import { ReactComponent as AudioLow } from "../icons/AudioLow.svg"; import { ReactComponent as Audio } from "../icons/Audio.svg"; -interface Props { +interface Props extends ComponentPropsWithoutRef<"svg"> { /** * Number between 0 and 1 */ volume: number; } -export function VolumeIcon({ volume }: Props) { - if (volume <= 0) return ; - if (volume <= 0.5) return ; - return