From c0443288c52dad7cc7d48c53392fca7a774367ec Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 4 Sep 2023 12:46:06 +0100 Subject: [PATCH 1/4] Smooth Focus Switching For a few seconds after a focusn switch, keep old tiles from the previous focus so that it doesn't look like everyone leaves & comes back. Based on https://github.com/vector-im/element-call/pull/1348 Requires https://github.com/livekit/components-js/pull/620 --- src/room/InCallView.tsx | 52 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 48 insertions(+), 4 deletions(-) diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index f827b4fd..1e6abe97 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -28,7 +28,7 @@ import { Room, Track, ConnectionState } from "livekit-client"; import { MatrixClient } from "matrix-js-sdk/src/client"; import { RoomMember } from "matrix-js-sdk/src/models/room-member"; import { Room as MatrixRoom } from "matrix-js-sdk/src/models/room"; -import { Ref, useCallback, useEffect, useMemo, useRef } from "react"; +import { Ref, useCallback, useEffect, useMemo, useRef, useState } from "react"; import { useTranslation } from "react-i18next"; import useMeasure from "react-use-measure"; import { OverlayTriggerState } from "@react-stately/overlays"; @@ -77,7 +77,10 @@ import { useMergedRefs } from "../useMergedRefs"; import { MuteStates } from "./MuteStates"; import { useIsRoomE2EE } from "../e2ee/sharedKeyManagement"; import { useOpenIDSFU } from "../livekit/openIDSFU"; -import { ECConnectionState } from "../livekit/useECConnectionState"; +import { + ECAddonConnectionState, + ECConnectionState, +} from "../livekit/useECConnectionState"; const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {}); // There is currently a bug in Safari our our code with cloning and sending MediaStreams @@ -85,6 +88,9 @@ const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {}); // For now we can disable screensharing in Safari. const isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent); +// How long we wait after a focus switch before showing the real participant list again +const POST_FOCUS_PARTICIPANT_UPDATE_DELAY_MS = 3000; + export interface ActiveCallProps extends Omit { e2eeConfig?: E2EEConfig; @@ -236,7 +242,7 @@ export function InCallView({ const reducedControls = boundsValid && bounds.width <= 400; const noControls = reducedControls && bounds.height <= 400; - const items = useParticipantTiles(livekitRoom, rtcSession.room); + const items = useParticipantTiles(livekitRoom, rtcSession.room, connState); const { fullscreenItem, toggleFullscreen, exitFullscreen } = useFullscreen(items); @@ -471,8 +477,11 @@ function findMatrixMember( function useParticipantTiles( livekitRoom: Room, - matrixRoom: MatrixRoom + matrixRoom: MatrixRoom, + connState: ECConnectionState ): TileDescriptor[] { + const previousTiles = useRef[]>([]); + const sfuParticipants = useParticipants({ room: livekitRoom, }); @@ -554,5 +563,40 @@ function useParticipantTiles( return allGhosts ? [] : tiles; }, [matrixRoom, sfuParticipants]); + // We carry over old tiles from the previous focus for some time after a focus switch + // so that the video tiles don't all disappear and reappear. + const [isSwitchingFocus, setIsSwitchingFocus] = useState(false); + + useEffect(() => { + if (connState === ECAddonConnectionState.ECSwitchingFocus) { + setIsSwitchingFocus(true); + } else if (isSwitchingFocus) { + setTimeout(() => { + setIsSwitchingFocus(false); + }, POST_FOCUS_PARTICIPANT_UPDATE_DELAY_MS); + } + }, [connState, setIsSwitchingFocus, isSwitchingFocus]); + + if ( + connState === ECAddonConnectionState.ECSwitchingFocus || + isSwitchingFocus + ) { + logger.debug("Switching focus: injecting previous tiles"); + + // inject the previous tile for members that haven't rejoined yet + const newItems = items.slice(0); + const rejoined = new Set(newItems.map((p) => p.id)); + + for (const prevTile of previousTiles.current) { + if (!rejoined.has(prevTile.id)) { + newItems.push(prevTile); + } + } + + return newItems; + } + + previousTiles.current = items; + return items; } From 378ce403f51ed372bf6cb7587e1682d6797f023b Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 12 Sep 2023 10:34:13 +0100 Subject: [PATCH 2/4] Update livekit components --- yarn.lock | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/yarn.lock b/yarn.lock index 8b48bc4d..07f71eb6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1820,17 +1820,17 @@ "@floating-ui/utils" "^0.1.1" "@floating-ui/dom@^1.1.0": - version "1.5.1" - resolved "https://registry.yarnpkg.com/@floating-ui/dom/-/dom-1.5.1.tgz#88b70defd002fe851f17b4a25efb2d3c04d7a8d7" - integrity sha512-KwvVcPSXg6mQygvA1TjbN/gh///36kKtllIF8SUm0qpFj8+rvYrpvlYdL1JoA71SHpDqgSSdGOSoQ0Mp3uY5aw== + version "1.5.2" + resolved "https://registry.yarnpkg.com/@floating-ui/dom/-/dom-1.5.2.tgz#6812e89d1d4d4ea32f10d15c3b81feb7f9836d89" + integrity sha512-6ArmenS6qJEWmwzczWyhvrXRdI/rI78poBcW0h/456+onlabit+2G+QxHx5xTOX60NBJQXjsCLFbW2CmsXpUog== dependencies: "@floating-ui/core" "^1.4.1" "@floating-ui/utils" "^0.1.1" "@floating-ui/utils@^0.1.1": - version "0.1.1" - resolved "https://registry.yarnpkg.com/@floating-ui/utils/-/utils-0.1.1.tgz#1a5b1959a528e374e8037c4396c3e825d6cf4a83" - integrity sha512-m0G6wlnhm/AX0H12IOWtK8gASEMffnX08RtKkCgTdHb9JpHKGloI7icFfLg9ZmQeavcvR0PKmzxClyuFPSjKWw== + version "0.1.2" + resolved "https://registry.yarnpkg.com/@floating-ui/utils/-/utils-0.1.2.tgz#b7e9309ccce5a0a40ac482cb894f120dba2b357f" + integrity sha512-ou3elfqG/hZsbmF4bxeJhPHIf3G2pm0ujc39hYEZrfVqt7Vk/Zji6CXc3W0pmYM8BW1g40U+akTl9DKZhFhInQ== "@formatjs/ecma402-abstract@1.11.4": version "1.11.4" @@ -2265,9 +2265,9 @@ rxjs "^7.8.0" "@livekit/components-react@^1.1.0": - version "1.1.6" - resolved "https://registry.yarnpkg.com/@livekit/components-react/-/components-react-1.1.6.tgz#4e832570bd3fb0c15e36ca7513f736683de62fd4" - integrity sha512-wA5wKVsKM4cBskXkTbgQ8UhEWCq7hYn/ElOlm9IsGf0U3KpsXpyiW/h2hd/aDr/ufTbBWYjtPWa8RrvoBImnyg== + version "1.1.8" + resolved "https://registry.yarnpkg.com/@livekit/components-react/-/components-react-1.1.8.tgz#34f67a6646bcd5c44d87d3fed5be5010b297f111" + integrity sha512-Ljlcqkgg7IqdQIDr7/ViEsL8GZkmSkoMC38hz3CkCcmHP+e8U4+4be2C2feiHb4hHw+tLd1DPuElyEWuTeiQKQ== dependencies: "@livekit/components-core" "0.6.15" "@react-hook/latest" "^1.0.3" From 5570e3f80671112fac8c01f3b562c3e83502ef48 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 13 Sep 2023 14:32:21 +0100 Subject: [PATCH 3/4] More obvious syntax Co-authored-by: Timo <16718859+toger5@users.noreply.github.com> --- src/room/InCallView.tsx | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 3a91d383..24724fa3 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -595,9 +595,8 @@ function useParticipantTiles( } return newItems; - } - - previousTiles.current = items; - - return items; + } else { + previousTiles.current = items; + return items; +} } From a02261561cd71fbf1bfaadc4577df4f963099bee Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 13 Sep 2023 15:03:54 +0100 Subject: [PATCH 4/4] Add comment --- src/room/InCallView.tsx | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 24724fa3..0bf5bca7 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -566,6 +566,11 @@ function useParticipantTiles( // We carry over old tiles from the previous focus for some time after a focus switch // so that the video tiles don't all disappear and reappear. + // This is set to true when the state transitions to Switching Focus and remains + // true for a short time after it changes (ie. connState is only switching focus for + // the time it takes us to reconnect to the conference). + // If there are still members that haven't reconnected after that time, they'll just + // appear to disconnect and will reappear once they reconnect. const [isSwitchingFocus, setIsSwitchingFocus] = useState(false); useEffect(() => { @@ -596,7 +601,7 @@ function useParticipantTiles( return newItems; } else { - previousTiles.current = items; - return items; -} + previousTiles.current = items; + return items; + } }