diff --git a/sdk/main.ts b/sdk/main.ts index 55aa4a022..de15a5759 100644 --- a/sdk/main.ts +++ b/sdk/main.ts @@ -60,6 +60,7 @@ import { initializeWidget, } from "../src/widget"; import { type Connection } from "../src/state/CallViewModel/remoteMembers/Connection"; +import { createWidgetHostBridge } from "../src/HostBridge"; interface MatrixRTCSdk { /** @@ -113,6 +114,7 @@ export async function createMatrixRTCSdk( const widget = _widget; if (!widget) throw Error("No widget. This webapp can only start as a widget"); const client = await widget.client; + const hostBridge = createWidgetHostBridge(widget); logger.info("client created"); // url params @@ -132,10 +134,12 @@ export async function createMatrixRTCSdk( controlledAudioDevices, callIntent, }); - const muteStates = new MuteStates(scope, mediaDevices, { - audioEnabled: false, - videoEnabled: false, - }); + const muteStates = new MuteStates( + scope, + mediaDevices, + { audioEnabled: false, videoEnabled: false }, + hostBridge, + ); // call view model const callViewModel = createCallViewModel$( @@ -146,6 +150,7 @@ export async function createMatrixRTCSdk( muteStates, { encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + hostBridge, controlledAudioDevices, callIntent, }, diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index e291e001b..e2edde844 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -29,6 +29,7 @@ import { Header, LeftNav, RightNav, RoomHeaderInfo } from "../Header"; import { HeaderStyle, useUrlParams } from "../UrlParams"; import { useCallViewKeyboardShortcuts } from "../useCallViewKeyboardShortcuts"; import { widget } from "../widget"; +import { useHostBridge } from "../HostBridge.ts"; import styles from "./InCallView.module.css"; import { GridTile } from "../tile/GridTile"; import { SettingsModal, defaultSettingsTab } from "../settings/SettingsModal"; @@ -116,6 +117,7 @@ export const ActiveCall: FC = (props) => { useState | null>(null); const urlParams = useUrlParams(); + const hostBridge = useHostBridge(); const mediaDevices = useMediaDevices(); const trackProcessorState$ = useTrackProcessorObservable$(); useEffect(() => { @@ -141,6 +143,7 @@ export const ActiveCall: FC = (props) => { props.muteStates, { encryptionSystem: props.e2eeSystem, + hostBridge, autoLeaveWhenOthersLeft, waitForCallPickup: waitForCallPickup && sendNotificationType === "ring", matrixRTCMode$: matrixRTCModeSetting.value$, @@ -171,6 +174,7 @@ export const ActiveCall: FC = (props) => { props.e2eeSystem, props.onLeft, urlParams, + hostBridge, mediaDevices, trackProcessorState$, props.client, diff --git a/src/room/RoomPage.tsx b/src/room/RoomPage.tsx index b5f943ce8..00905ac4d 100644 --- a/src/room/RoomPage.tsx +++ b/src/room/RoomPage.tsx @@ -30,6 +30,7 @@ import { useRoomIdentifier, useUrlParams } from "../UrlParams"; import { useRegisterPasswordlessUser } from "../auth/useRegisterPasswordlessUser"; import { HomePage } from "../home/HomePage"; import { widget } from "../widget"; +import { useHostBridge } from "../HostBridge.ts"; import { CallTerminatedMessage, useLoadGroupCall } from "./useLoadGroupCall"; import { LobbyView } from "./LobbyView"; import { E2eeType } from "../e2ee/e2eeType"; @@ -44,6 +45,7 @@ import { calculateInitialMuteState } from "../state/initialMuteState.ts"; export const RoomPage: FC = (): ReactNode => { const urlParams = useUrlParams(); + const hostBridge = useHostBridge(); const { confineToRoom, preload, header, displayName, skipLobby } = urlParams; const { t } = useTranslation(); const { roomAlias, roomId, viaServers } = useRoomIdentifier(); @@ -77,10 +79,11 @@ export const RoomPage: FC = (): ReactNode => { urlParams.callIntent, widget !== null, ), + hostBridge, ), ); return (): void => scope.end(); - }, [devices, urlParams]); + }, [devices, urlParams, hostBridge]); useEffect(() => { // If we've finished loading, are not already authed and we've been given a display name as diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index e9abeae59..b40a45231 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -48,7 +48,6 @@ import { type RTCCallIntent, type RTCNotificationType, } from "matrix-js-sdk/lib/matrixrtc"; -import { type IWidgetApiRequest } from "matrix-widget-api"; import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; import { v4 as uuidv4 } from "uuid"; import { type IMembershipManager } from "matrix-js-sdk/lib/matrixrtc/IMembershipManager"; @@ -89,7 +88,7 @@ import { MatrixKeyProvider } from "../../e2ee/matrixKeyProvider"; import { type MuteStates } from "../MuteStates"; import { HeaderStyle } from "../../UrlParams"; import { type ProcessorState } from "../../livekit/TrackProcessorContext"; -import { ElementWidgetActions, widget } from "../../widget"; +import { type HostBridge, nullHostBridge } from "../../HostBridge"; import { layoutShallowEquals, type Alignment, @@ -173,6 +172,11 @@ import { type GridTileViewModel } from "../TileViewModel.ts"; // callMembership -> rtcMembership export interface CallViewModelOptions { encryptionSystem: EncryptionSystem; + /** + * The application hosting Element Call, which can ask it to hang up and wants + * to know when the user joins or leaves. Defaults to no host. + */ + hostBridge?: HostBridge; /** * Whether the app hosting Element Call controls the audio output devices, * rather than the browser. Defaults to false. @@ -457,6 +461,7 @@ export function createCallViewModel$( // Defaults match what the URL parameters resolve to outside of widget mode, // so that callers which don't care (chiefly tests) behave as they always have. const { + hostBridge = nullHostBridge, controlledAudioDevices = false, header = HeaderStyle.Standard, showControls = true, @@ -632,6 +637,7 @@ export function createCallViewModel$( localTransport$, roomId: matrixRoom.roomId, hideScreensharing, + hostBridge, logger: logger.getChild(`[${Date.now()}]`), }); @@ -923,24 +929,16 @@ export function createCallViewModel$( const userHangup$ = new Subject(); - const widgetHangup$ = - widget === null - ? NEVER - : ( - fromEvent( - widget.lazyActions, - ElementWidgetActions.HangupCall, - ) as Observable> - ).pipe( - tap((ev) => { - widget!.api.transport.reply(ev.detail, {}); - }), - ); + const hostHangup$ = hostBridge.hangUp$.pipe( + tap((request) => { + request.reply(); + }), + ); const leave$: Observable<"user" | "timeout" | "decline" | "allOthersLeft"> = merge( autoLeave$, - merge(userHangup$, widgetHangup$).pipe(map(() => "user" as const)), + merge(userHangup$, hostHangup$).pipe(map(() => "user" as const)), ).pipe(scope.share); const spotlightSpeaker$ = scope.behavior( diff --git a/src/state/CallViewModel/localMember/LocalMember.test.ts b/src/state/CallViewModel/localMember/LocalMember.test.ts index 2b89d50b5..4826dcdd1 100644 --- a/src/state/CallViewModel/localMember/LocalMember.test.ts +++ b/src/state/CallViewModel/localMember/LocalMember.test.ts @@ -52,6 +52,7 @@ import { ConnectionManagerData } from "../remoteMembers/ConnectionManager"; import { ConnectionState, type Connection } from "../remoteMembers/Connection"; import { type Publisher } from "./Publisher"; import { initializeWidget } from "../../../widget"; +import { nullHostBridge } from "../../../HostBridge"; import { type LocalTransport, type LocalTransportWithSFUConfig, @@ -200,6 +201,7 @@ describe("LocalMembership", () => { }, roomId: "!test-room-id:example.org", hideScreensharing: false, + hostBridge: nullHostBridge, }; it("throws error on missing RTC config error", () => { diff --git a/src/state/CallViewModel/localMember/LocalMember.ts b/src/state/CallViewModel/localMember/LocalMember.ts index 844b66dee..e28f5fe51 100644 --- a/src/state/CallViewModel/localMember/LocalMember.ts +++ b/src/state/CallViewModel/localMember/LocalMember.ts @@ -53,7 +53,7 @@ import { MembershipManagerError, UnknownCallError, } from "../../../utils/errors.ts"; -import { ElementWidgetActions, widget } from "../../../widget.ts"; +import { type HostBridge } from "../../../HostBridge.ts"; import { PosthogAnalytics } from "../../../analytics/PosthogAnalytics.ts"; import { @@ -145,6 +145,8 @@ interface Props { >; /** Whether to hide the screen-sharing button. */ hideScreensharing: boolean; + /** The application hosting Element Call, to be kept informed of join/leave. */ + hostBridge: HostBridge; logger: Logger; } @@ -165,6 +167,7 @@ interface Props { * @param props.matrixRTCSession The matrix RTC session to join. * @param props.roomId The room ID used as the call identifier in analytics events. * @param props.hideScreensharing Whether to hide the screen-sharing button. + * @param props.hostBridge The application hosting Element Call. * @returns * - publisher: The handle to create tracks and publish them to the room. * - connected$: the current connection state. Including matrix server and livekit server connection. (only considering the livekit server we are using for our own media publication) @@ -184,6 +187,7 @@ export const createLocalMembership$ = ({ matrixRTCSession, roomId, hideScreensharing, + hostBridge, }: Props): { /** * This request to start audio and video tracks. @@ -576,29 +580,24 @@ export const createLocalMembership$ = ({ } }); - // inform the widget about the connect and disconnect intent from the user. + // inform the host about the connect and disconnect intent from the user. scope .behavior(joinAndPublishRequested$.pipe(pairwise(), scope.bind()), [ undefined, joinAndPublishRequested$.value, ]) .subscribe(([prev, current]) => { - if (!widget) return; // JOIN prev=false (was left) => current-true (now joiend) if (!prev && current) { - widget.api.transport - .send(ElementWidgetActions.JoinCall, {}) - .catch((e) => { - logger.error("Failed to send join action", e); - }); + hostBridge.notifyJoined().catch((e) => { + logger.error("Failed to notify the host that we joined", e); + }); } // LEAVE prev=false (was joined) => current-true (now left) if (prev && !current) { - widget.api.transport - .send(ElementWidgetActions.HangupCall, {}) - .catch((e) => { - logger.error("Failed to send hangup action", e); - }); + hostBridge.notifyHungUp().catch((e) => { + logger.error("Failed to notify the host that we hung up", e); + }); } }); @@ -845,7 +844,7 @@ interface EnterRTCSessionOptions { * @param options - `encryptMedia`: Whether to encrypt media. `matrixRTCMode`: The * Matrix RTC mode to use. `sendNotificationType`: Whether and what kind of * notification to send on join. `callIntent`: The kind of call being placed. - * @throws If the widget could not send ElementWidgetActions.JoinCall action. + * @throws If the host could not be told that we are joining. */ // Exported for unit testing export function enterRTCSession( diff --git a/src/state/CallViewModelWidget.test.ts b/src/state/CallViewModelWidget.test.ts index 2f331bd32..dd2e75ab8 100644 --- a/src/state/CallViewModelWidget.test.ts +++ b/src/state/CallViewModelWidget.test.ts @@ -6,39 +6,31 @@ Please see LICENSE in the repository root for full details. */ import { it, vi, expect } from "vitest"; -import EventEmitter from "events"; +import { Subject } from "rxjs"; // import * as ComponentsCore from "@livekit/components-core"; import { withCallViewModel } from "./CallViewModel/CallViewModelTestUtils.ts"; import { type CallViewModel } from "./CallViewModel/CallViewModel.ts"; import { constant } from "./Behavior.ts"; import { aliceParticipant, localRtcMember } from "../utils/test-fixtures.ts"; -import { ElementWidgetActions, widget } from "../widget.ts"; +import { + type HostBridge, + type HostRequest, + nullHostBridge, +} from "../HostBridge.ts"; import { E2eeType } from "../e2ee/e2eeType.ts"; import { MatrixRTCMode } from "../config/ConfigOptions.ts"; vi.mock("@livekit/components-core", { spy: true }); -vi.mock("../widget", () => ({ - ElementWidgetActions: { - HangupCall: "HangupCall", - // Add other actions if needed - }, - widget: { - api: { - transport: { - send: vi.fn().mockResolvedValue(undefined), - reply: vi.fn().mockResolvedValue(undefined), - }, - }, - lazyActions: new EventEmitter(), - }, -})); - it.each([[MatrixRTCMode.Compatibility], [MatrixRTCMode.Matrix_2_0]])( - "expect leave when ElementWidgetActions.HangupCall is called (%s mode)", + "expect leave when the host asks us to hang up (%s mode)", async (mode) => { const pr = Promise.withResolvers(); + const hangUp$ = new Subject>>(); + const hostBridge: HostBridge = { ...nullHostBridge, hangUp$ }; + const reply = vi.fn(); + withCallViewModel(mode)( { remoteParticipants$: constant([aliceParticipant]), @@ -49,25 +41,17 @@ it.each([[MatrixRTCMode.Compatibility], [MatrixRTCMode.Matrix_2_0]])( pr.resolve(s); }); - widget!.lazyActions!.emit( - ElementWidgetActions.HangupCall, - new CustomEvent(ElementWidgetActions.HangupCall, { - detail: { - action: "im.vector.hangup", - api: "toWidget", - data: {}, - requestId: "widgetapi-1761237395918", - widgetId: "mrUjS9T6uKUOWHMxXvLbSv0F", - }, - }), - ); + hangUp$.next({ data: {}, reply }); }, { encryptionSystem: { kind: E2eeType.PER_PARTICIPANT }, + hostBridge, }, ); const source = await pr.promise; expect(source).toBe("user"); + // The host expects to hear back that we acted on its request + expect(reply).toHaveBeenCalledOnce(); }, ); diff --git a/src/state/MuteStates.test.ts b/src/state/MuteStates.test.ts index f594cb05c..239f0381f 100644 --- a/src/state/MuteStates.test.ts +++ b/src/state/MuteStates.test.ts @@ -9,6 +9,7 @@ import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; import { BehaviorSubject } from "rxjs"; import { logger } from "matrix-js-sdk/lib/logger"; +import { nullHostBridge } from "../HostBridge"; import { MuteStates, MuteState } from "./MuteStates"; import { type AudioOutputDeviceLabel, @@ -228,10 +229,12 @@ describe("MuteStates", () => { videoInput: aVideoInput(), // other devices are not relevant for this test }); - const muteStates = new MuteStates(testScope, mediaDevices, { - audioEnabled: false, - videoEnabled: false, - }); + const muteStates = new MuteStates( + testScope, + mediaDevices, + { audioEnabled: false, videoEnabled: false }, + nullHostBridge, + ); let latestSyncedState: boolean | null = null; muteStates.video.setHandler(async (enabled: boolean): Promise => { diff --git a/src/state/MuteStates.ts b/src/state/MuteStates.ts index d89cb8442..59413b036 100644 --- a/src/state/MuteStates.ts +++ b/src/state/MuteStates.ts @@ -6,14 +6,12 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { type IWidgetApiRequest } from "matrix-widget-api"; import { logger } from "matrix-js-sdk/lib/logger"; import { BehaviorSubject, combineLatest, distinctUntilChanged, firstValueFrom, - fromEvent, map, merge, Observable, @@ -24,7 +22,7 @@ import { } from "rxjs"; import { type MediaDevices, type MediaDevice } from "../state/MediaDevices"; -import { ElementWidgetActions, widget } from "../widget"; +import { type DeviceMuteState, type HostBridge } from "../HostBridge"; import { type ObservableScope } from "./ObservableScope"; import { type Behavior, constant } from "./Behavior"; @@ -216,58 +214,51 @@ export class MuteStates { audioEnabled: boolean; videoEnabled: boolean; }, + hostBridge: HostBridge, ) { - if (widget !== null) { - // Sync our mute states with the hosting client - const widgetApiState$ = combineLatest( - [this.audio.enabled$, this.video.enabled$], - (audio, video) => ({ audio_enabled: audio, video_enabled: video }), - ); - widgetApiState$.pipe(this.scope.bind()).subscribe((state) => { - widget!.api.transport - .send(ElementWidgetActions.DeviceMute, state) - .catch((e) => - logger.warn("Could not send DeviceMute action to widget", e), - ); - }); + // Keep the host informed of our mute state + const muteState$ = combineLatest( + [this.audio.enabled$, this.video.enabled$], + (audio, video): DeviceMuteState => ({ + audio_enabled: audio, + video_enabled: video, + }), + ); + muteState$.pipe(this.scope.bind()).subscribe((state) => { + hostBridge + .notifyDeviceMute(state) + .catch((e) => logger.warn("Could not send mute state to the host", e)); + }); - // Also sync the hosting client's mute states back with ours - const muteActions$ = fromEvent( - widget.lazyActions, - ElementWidgetActions.DeviceMute, - ) as Observable>; - muteActions$ - .pipe( - withLatestFrom( - widgetApiState$, - this.audio.setEnabled$, - this.video.setEnabled$, - ), - this.scope.bind(), - ) - .subscribe(([ev, state, setAudioEnabled, setVideoEnabled]) => { - // First copy the current state into our new state - const newState = { ...state }; - // Update new state if there are any requested changes from the widget - // action in `ev.detail.data`. - if ( - ev.detail.data.audio_enabled != null && - typeof ev.detail.data.audio_enabled === "boolean" && - setAudioEnabled !== null - ) { - newState.audio_enabled = ev.detail.data.audio_enabled; - setAudioEnabled(newState.audio_enabled); - } - if ( - ev.detail.data.video_enabled != null && - typeof ev.detail.data.video_enabled === "boolean" && - setVideoEnabled !== null - ) { - newState.video_enabled = ev.detail.data.video_enabled; - setVideoEnabled(newState.video_enabled); - } - widget!.api.transport.reply(ev.detail, newState); - }); - } + // And apply the changes the host asks for + hostBridge.deviceMute$ + .pipe( + withLatestFrom( + muteState$, + this.audio.setEnabled$, + this.video.setEnabled$, + ), + this.scope.bind(), + ) + .subscribe(([request, state, setAudioEnabled, setVideoEnabled]) => { + // First copy the current state into our new state + const newState = { ...state }; + // Then apply whichever changes the host asked for + if ( + typeof request.data.audio_enabled === "boolean" && + setAudioEnabled !== null + ) { + newState.audio_enabled = request.data.audio_enabled; + setAudioEnabled(newState.audio_enabled); + } + if ( + typeof request.data.video_enabled === "boolean" && + setVideoEnabled !== null + ) { + newState.video_enabled = request.data.video_enabled; + setVideoEnabled(newState.video_enabled); + } + request.reply(newState); + }); } } diff --git a/src/utils/test.ts b/src/utils/test.ts index de74ac53e..49ae0a3f1 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -63,6 +63,7 @@ import { type MediaDevices } from "../state/MediaDevices"; import { type Behavior, constant } from "../state/Behavior"; import { ObservableScope } from "../state/ObservableScope"; import { MuteStates } from "../state/MuteStates"; +import { nullHostBridge } from "../HostBridge"; import { createLocalUserMedia, type LocalUserMediaViewModel, @@ -577,10 +578,12 @@ export function mockMuteStates( joined$: Observable = of(true), ): MuteStates { const observableScope = new ObservableScope(); - return new MuteStates(observableScope, mockMediaDevices({}), { - audioEnabled: false, - videoEnabled: false, - }); + return new MuteStates( + observableScope, + mockMediaDevices({}), + { audioEnabled: false, videoEnabled: false }, + nullHostBridge, + ); } export class MockConnection extends Connection {