From 0f77b1c42d6535405a9377328648e5cd923cc1fa Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Thu, 25 Sep 2025 17:44:45 +0100 Subject: [PATCH] Add media hints for notification events. (#3493) (#3514) * Add media hints for notification events. * Prevent showing calling view when disconnected from Livekit. (#3491) * Refactor disconnection handling * Use "unknown" * Update signature * Add tests * Expose livekitConnectionState directly * fix whoopsie * Update dependency livekit-client to v2.15.7 (#3496) * Fix the interactivity of buttons while reconnecting or in earpiece mode (#3486) * Fix the interactivity of buttons while reconnecting or in earpiece mode When we're in one of these modes, we need to ensure that everything above the overlay (the header and footer buttons) is interactive, while everything obscured by the overlay (the media tiles) is non-interactive and removed from the accessibility tree. It's not a very easy task to trap focus *outside* an element, so the best solution I could come up with is to set tabindex="-1" manually on all interactive elements belonging to the media tiles. * Write a Playwright test for reconnecting * fix lints * fix test * enable http2 for matrx-rtc host to allow the jwt service to talk to the SFU * remove rate limit for delayed events * more time to connect to livekit SFU * Due to a Firefox issue we set the start anchor for the tab test to the Mute microphone button * adapt to most recent Element Web version * Use the "End call" button as proofe for a started call * Currrenty disabled due to recent Element Web - not indicating the number of participants - bypassing Lobby * linting * disable 'can only interact with header and footer while reconnecting' for firefox --------- * Log when a track is unpublished or runs into an error (#3495) * default mute states (unmuted!) in widget mode (embedded + intent) (#3494) * default mute states (unmuted!) in widget mode (embedded + intent) * review * introduce a cache for the url params. * Add an option to skip the cache. --------- * Apply new hint code * missed a bit * fix intent * Automatically update intent on mute change * update packages * lint * Fix tests * fix merge fails --------- Signed-off-by: Timo K Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: Robin Co-authored-by: Timo <16718859+toger5@users.noreply.github.com> Co-authored-by: Timo K Co-authored-by: fkwp --- src/UrlParams.ts | 29 ++++++++++++++++-- src/room/GroupCallView.tsx | 10 ++++++ src/rtcSessionHelpers.ts | 4 ++- src/utils/test.ts | 2 ++ yarn.lock | 62 +++++++++++++++++--------------------- 5 files changed, 69 insertions(+), 38 deletions(-) diff --git a/src/UrlParams.ts b/src/UrlParams.ts index d184a1ae..4eb69298 100644 --- a/src/UrlParams.ts +++ b/src/UrlParams.ts @@ -8,7 +8,10 @@ Please see LICENSE in the repository root for full details. import { useMemo } from "react"; import { useLocation } from "react-router-dom"; import { logger } from "matrix-js-sdk/lib/logger"; -import { type RTCNotificationType } from "matrix-js-sdk/lib/matrixrtc"; +import { + type RTCCallIntent, + type RTCNotificationType, +} from "matrix-js-sdk/lib/matrixrtc"; import { pickBy } from "lodash-es"; import { Config } from "./config/Config"; @@ -26,7 +29,9 @@ export enum UserIntent { StartNewCall = "start_call", JoinExistingCall = "join_existing", StartNewCallDM = "start_call_dm", + StartNewCallDMVoice = "start_call_dm_voice", JoinExistingCallDM = "join_existing_dm", + JoinExistingCallDMVoice = "join_existing_dm_voice", Unknown = "unknown", } @@ -227,6 +232,12 @@ export interface UrlConfiguration { * - auto-dismiss the call widget once the notification lifetime expires on the receivers side. */ waitForCallPickup: boolean; + + callIntent?: RTCCallIntent; +} +interface IntentAndPlatformDerivedConfiguration { + defaultAudioEnabled?: boolean; + defaultVideoEnabled?: boolean; } interface IntentAndPlatformDerivedConfiguration { defaultAudioEnabled?: boolean; @@ -395,22 +406,31 @@ export const computeUrlParams = (search = "", hash = ""): UrlParams => { switch (intent) { case UserIntent.StartNewCall: intentPreset.skipLobby = false; + intentPreset.callIntent = "video"; break; case UserIntent.JoinExistingCall: // On desktop this will be overridden based on which button was used to join the call intentPreset.skipLobby = false; + intentPreset.callIntent = "video"; break; + case UserIntent.StartNewCallDMVoice: + intentPreset.callIntent = "audio"; + // Fall through case UserIntent.StartNewCallDM: intentPreset.skipLobby = true; intentPreset.sendNotificationType = "ring"; intentPreset.autoLeaveWhenOthersLeft = true; intentPreset.waitForCallPickup = true; - + intentPreset.callIntent = intentPreset.callIntent ?? "video"; break; + case UserIntent.JoinExistingCallDMVoice: + intentPreset.callIntent = "audio"; + // Fall through case UserIntent.JoinExistingCallDM: // On desktop this will be overridden based on which button was used to join the call intentPreset.skipLobby = true; intentPreset.autoLeaveWhenOthersLeft = true; + intentPreset.callIntent = intentPreset.callIntent ?? "video"; break; // Non widget usecase defaults default: @@ -447,6 +467,11 @@ export const computeUrlParams = (search = "", hash = ""): UrlParams => { intentAndPlatformDerivedConfiguration.defaultAudioEnabled = true; intentAndPlatformDerivedConfiguration.defaultVideoEnabled = true; break; + case UserIntent.StartNewCallDMVoice: + case UserIntent.JoinExistingCallDMVoice: + intentAndPlatformDerivedConfiguration.defaultAudioEnabled = true; + intentAndPlatformDerivedConfiguration.defaultVideoEnabled = false; + break; } } diff --git a/src/room/GroupCallView.tsx b/src/room/GroupCallView.tsx index dbc3ea18..e4a6a997 100644 --- a/src/room/GroupCallView.tsx +++ b/src/room/GroupCallView.tsx @@ -128,6 +128,16 @@ export const GroupCallView: FC = ({ // eslint-disable-next-line react-hooks/exhaustive-deps }, []); + // Update our member event when our mute state changes. + useEffect(() => { + if (!isJoined) { + return; + } + void rtcSession.updateCallIntent( + muteStates.video.enabled ? "video" : "audio", + ); + }, [rtcSession, isJoined, muteStates.video.enabled]); + useEffect(() => { logger.info("[Lifecycle] GroupCallView Component mounted"); return (): void => { diff --git a/src/rtcSessionHelpers.ts b/src/rtcSessionHelpers.ts index 73f58cea..f02759b8 100644 --- a/src/rtcSessionHelpers.ts +++ b/src/rtcSessionHelpers.ts @@ -120,11 +120,13 @@ export async function enterRTCSession( const { features, matrix_rtc_session: matrixRtcSessionConfig } = Config.get(); const useDeviceSessionMemberEvents = features?.feature_use_device_session_member_events; + const { sendNotificationType: notificationType, callIntent } = getUrlParams(); rtcSession.joinRoomSession( await makePreferredLivekitFoci(rtcSession, livekitAlias), makeActiveFocus(), { - notificationType: getUrlParams().sendNotificationType, + notificationType, + callIntent, useNewMembershipManager, manageMediaKeys: encryptMedia, ...(useDeviceSessionMemberEvents !== undefined && { diff --git a/src/utils/test.ts b/src/utils/test.ts index 31c6068a..2a9ca176 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -360,6 +360,8 @@ export class MockRTCSession extends TypedEventEmitter< return this; } + public updateCallIntent = vitest.fn(); + private _membershipStatus = Status.Connected; public get membershipStatus(): Status { return this._membershipStatus; diff --git a/yarn.lock b/yarn.lock index 5f224576..69a1036e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2793,10 +2793,10 @@ __metadata: languageName: node linkType: hard -"@matrix-org/matrix-sdk-crypto-wasm@npm:^15.1.0": - version: 15.1.0 - resolution: "@matrix-org/matrix-sdk-crypto-wasm@npm:15.1.0" - checksum: 10c0/19edc6d0045ff49fad8d77b6e561cee994f7513f8c18a7176ae2d3f0116c1a91980e02d10300b09c2b72dea4da4a8c3392f2bf1752057f2d6b53030a056d76d8 +"@matrix-org/matrix-sdk-crypto-wasm@npm:^15.3.0": + version: 15.3.0 + resolution: "@matrix-org/matrix-sdk-crypto-wasm@npm:15.3.0" + checksum: 10c0/45628f36b7b0e54a8777ae67a7233dbdf3e3cf14e0d95d21f62f89a7ea7e3f907232f1eb7b1262193b1e227759fad47af829dcccc103ded89011f13c66f01d76 languageName: node linkType: hard @@ -5290,13 +5290,6 @@ __metadata: languageName: node linkType: hard -"@types/retry@npm:0.12.0": - version: 0.12.0 - resolution: "@types/retry@npm:0.12.0" - checksum: 10c0/7c5c9086369826f569b83a4683661557cab1361bac0897a1cefa1a915ff739acd10ca0d62b01071046fe3f5a3f7f2aec80785fe283b75602dc6726781ea3e328 - languageName: node - linkType: hard - "@types/sdp-transform@npm:^2.4.5": version: 2.4.10 resolution: "@types/sdp-transform@npm:2.4.10" @@ -9528,6 +9521,13 @@ __metadata: languageName: node linkType: hard +"is-network-error@npm:^1.1.0": + version: 1.3.0 + resolution: "is-network-error@npm:1.3.0" + checksum: 10c0/3e85a69e957988db66d5af5412efdd531a5a63e150d1bdd5647cfd4dc54fd89b1dbdd472621f8915233c3176ba1e6922afa8a51a9e363ba4693edf96a294f898 + languageName: node + linkType: hard + "is-number-object@npm:^1.1.1": version: 1.1.1 resolution: "is-number-object@npm:1.1.1" @@ -10298,11 +10298,11 @@ __metadata: linkType: hard "matrix-js-sdk@github:matrix-org/matrix-js-sdk#head=develop": - version: 37.13.0 - resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=2f1d654f14be8dd03896e9e76f12017b6f9eec1c" + version: 38.3.0 + resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=41d70d0b5d3f0eba92686f8089cb329d875b26b5" dependencies: "@babel/runtime": "npm:^7.12.5" - "@matrix-org/matrix-sdk-crypto-wasm": "npm:^15.1.0" + "@matrix-org/matrix-sdk-crypto-wasm": "npm:^15.3.0" another-json: "npm:^0.2.0" bs58: "npm:^6.0.0" content-type: "npm:^1.0.4" @@ -10311,11 +10311,11 @@ __metadata: matrix-events-sdk: "npm:0.0.1" matrix-widget-api: "npm:^1.10.0" oidc-client-ts: "npm:^3.0.1" - p-retry: "npm:4" + p-retry: "npm:7" sdp-transform: "npm:^2.14.1" unhomoglyph: "npm:^1.0.6" - uuid: "npm:11" - checksum: 10c0/ecd019c677c272c5598617dcde407dbe4b1b11460863b2a577e33f3fd8732c9d9073ec0221b471ec1eb24e2839eec20728db7f92c9348be83126547286e50805 + uuid: "npm:13" + checksum: 10c0/b48528fec573f3e14d1297f360a56d52d7f313da0d4cf82ab51e4c29798b86995b8a6bd72409779746e7bcf02949bc2788bffa9aba276bfb1a76dbcbe89900a0 languageName: node linkType: hard @@ -10922,13 +10922,12 @@ __metadata: languageName: node linkType: hard -"p-retry@npm:4": - version: 4.6.2 - resolution: "p-retry@npm:4.6.2" +"p-retry@npm:7": + version: 7.0.0 + resolution: "p-retry@npm:7.0.0" dependencies: - "@types/retry": "npm:0.12.0" - retry: "npm:^0.13.1" - checksum: 10c0/d58512f120f1590cfedb4c2e0c42cb3fa66f3cea8a4646632fcb834c56055bb7a6f138aa57b20cc236fb207c9d694e362e0b5c2b14d9b062f67e8925580c73b0 + is-network-error: "npm:^1.1.0" + checksum: 10c0/3c090ac72bbe00fd2f062ee6178c44f7302f298936ab2290a458575e73650e7834b556beb2b09fa9fbebedab2ec3358cb474c09710cf828972b670c3c0cb89e4 languageName: node linkType: hard @@ -12230,13 +12229,6 @@ __metadata: languageName: node linkType: hard -"retry@npm:^0.13.1": - version: 0.13.1 - resolution: "retry@npm:0.13.1" - checksum: 10c0/9ae822ee19db2163497e074ea919780b1efa00431d197c7afdb950e42bf109196774b92a49fc9821f0b8b328a98eea6017410bfc5e8a0fc19c85c6d11adb3772 - languageName: node - linkType: hard - "reusify@npm:^1.0.4": version: 1.1.0 resolution: "reusify@npm:1.1.0" @@ -13766,12 +13758,12 @@ __metadata: languageName: node linkType: hard -"uuid@npm:11": - version: 11.0.5 - resolution: "uuid@npm:11.0.5" +"uuid@npm:13": + version: 13.0.0 + resolution: "uuid@npm:13.0.0" bin: - uuid: dist/esm/bin/uuid - checksum: 10c0/6f59f0c605e02c14515401084ca124b9cb462b4dcac866916a49862bcf831874508a308588c23a7718269226ad11a92da29b39d761ad2b86e736623e3a33b6e7 + uuid: dist-node/bin/uuid + checksum: 10c0/950e4c18d57fef6c69675344f5700a08af21e26b9eff2bf2180427564297368c538ea11ac9fb2e6528b17fc3966a9fd2c5049361b0b63c7d654f3c550c9b3d67 languageName: node linkType: hard