diff --git a/sdk/README.md b/sdk/README.md index 65f77b3aa..7c30f9950 100644 --- a/sdk/README.md +++ b/sdk/README.md @@ -30,8 +30,7 @@ const rtcClient = createMatrixRTCClient( client, // a matrix-js-sdk MatrixClient, logged in and syncing room, // the matrix-js-sdk Room, from client.getRoom() once the join has synced { - microphoneEnabled: true, - cameraEnabled: true, + publish: [{ source: "microphone" }, { source: "camera" }], audioOutputDeviceId$: constant(undefined), }, { diff --git a/sdk/SdkArchitecture.md b/sdk/SdkArchitecture.md index 5fffba2eb..834cc9950 100644 --- a/sdk/SdkArchitecture.md +++ b/sdk/SdkArchitecture.md @@ -189,8 +189,6 @@ export interface MatrixRTCClientOptions { timings?: Partial; /** Limits on what is published; LiveKit's defaults otherwise. */ mediaQuality?: MediaQuality; - /** How the local tracks are captured and encoded: audio processing, camera and screen share resolution, bitrate, codec. */ - capture?: CaptureSettings; /** Use this transport instead of asking the homeserver. */ transportUrl?: string; /** Use this transport when the homeserver advertises none. */ @@ -198,18 +196,33 @@ export interface MatrixRTCClientOptions { } /** - * What the local member publishes when it joins. Initial values: from then on - * `setMicrophoneEnabled` and `setCameraEnabled` on the client, and the - * controls on the local tracks, change it. Device enumeration, permission - * prompts and the lobby preview stay with the host. + * One thing to publish: a source, where to capture it from and how to encode + * it. Where the device and the settings are left out, the browser's and + * LiveKit's defaults apply. + */ +export type PublishRequest = + | { source: "microphone"; deviceId?: string; capture?: AudioCaptureSettings } + | { + source: "camera"; + deviceId?: string; + /** Background blur and the like. */ + processor?: TrackProcessor; + capture?: VideoCaptureSettings; + } + | { + source: "screenShare"; + /** Whether to capture the screen's audio too. Default true. */ + audio?: boolean; + capture?: VideoCaptureSettings; + }; + +/** + * Device enumeration, permission prompts and the lobby preview stay with the + * host. */ export interface LocalMediaInputs { - microphoneEnabled: boolean; - cameraEnabled: boolean; - audioInputDeviceId?: string; - videoInputDeviceId?: string; - /** Background blur and the like. */ - videoProcessor?: TrackProcessor; + /** Published at the join; `publish` on the local member adds to it from then on. */ + publish: PublishRequest[]; /** Undefined where the host routes audio itself, or to leave the browser's choice. */ audioOutputDeviceId$: Behavior; } @@ -268,14 +281,6 @@ export interface MatrixRTCClient { /** Null while connected, the first failing link otherwise. */ disconnectReason$: Behavior; - /** - * Publishes or mutes the microphone. Resolves with the state that - * resulted, which differs from the request where the device could not be - * used. Before the transport is connected the request is remembered and - * applied once it is. - */ - setMicrophoneEnabled(enabled: boolean): Promise; - setCameraEnabled(enabled: boolean): Promise; /** * A transport, Matrix or connection error that stops the session. Null * while fine. A failed publication is not fatal: the member can still @@ -367,12 +372,18 @@ export interface RemoteRTCMember extends RTCMember { export interface LocalRTCMember extends RTCMember { local: true; media$: Behavior; - sharingScreen$: Behavior; - /** Null when the platform cannot share a screen (no getDisplayMedia). Hiding the button is the host's policy. */ - toggleScreenSharing: (() => void) | null; - /** The last error from toggling, until dismissed. The user closing the picker is not one. */ - screenShareError$: Behavior; - dismissScreenShareError(): void; + /** + * Publishes a source and resolves with its track once it is in `tracks$`. + * Before the transport is connected the request is remembered and applied + * once it is. Rejects where the device could not be used, including the + * user closing the picker. One publication per source: publishing a source + * again unmutes it. + */ + publish( + request: PublishRequest, + ): Promise; + /** Removes one of our tracks; a screen share takes its audio with it. */ + unpublish(id: string): Promise; } ``` @@ -426,7 +437,7 @@ they only make sense for a call: | `autoLeave$` / `AutoLeaveReason` (`"allOthersLeft" \| "timeout" \| "decline"`) | hanging up because the other side declined or everyone left is call etiquette, not session semantics | `CallViewModel.leave$`, built from `remoteMembers$` and the room timeline | | `ringAttempts$`, `createCallNotificationLifecycle$`, `createSentCallNotification$`, `createReceivedDecline$` | ringing, pickup timeouts and decline events are MSC4075 call-notify UX | `src/state/CallViewModel/CallNotificationLifecycle.ts`, fed by `remoteMembers$` and the Matrix room | | `waitForCallPickup`, `autoLeaveWhenOthersLeft` options | inputs to the above | `CallViewModelOptions` | -| `hideScreensharing` option | a UI policy; the session can always share a screen if the platform can | `CallViewModel` nulls `toggleScreenSharing` | +| `hideScreensharing` option | a UI policy; the session can always share a screen if the platform can | `CallViewModel` offers no screen share control | | `ringingVm$`, `ringingStatusLocation` | view models | `CallViewModel` | | `hostBridge.hangUp$`, `hangup()`, `userHangup$` | widget / host integration | `CallViewModel` | | `ElementCallError` | carries Element Call translation keys | the view model wraps `MatrixRTCError`, reading `cause` | @@ -645,11 +656,8 @@ public API only. ## Open decisions -- **`LocalMediaInputs` shape.** Everything in it is an initial value; the - setters on the client and the controls on the local tracks change it from - then on, because they have to report what the device allowed. Creating the - tracks before `join()` for a preview is still missing: today they are created - at the join. +- **Publishing before `join()`.** The initial requests are published at the + join; creating the tracks earlier, for a lobby preview, is still missing. - **`applicationData` after the join.** The option is read once. The client still updates `m.call.intent` from the camera state, as Element Call did; that moves to the host once the data can be updated. diff --git a/sdk/dev/index.html b/sdk/dev/index.html index 37153285e..60e4b585a 100644 --- a/sdk/dev/index.html +++ b/sdk/dev/index.html @@ -60,6 +60,9 @@ Please see LICENSE in the repository root for full details. +

diff --git a/sdk/dev/main.ts b/sdk/dev/main.ts index f73cc206a..5a464a33c 100644 --- a/sdk/dev/main.ts +++ b/sdk/dev/main.ts @@ -36,6 +36,7 @@ const messages = document.getElementById("messages")!; const buttons = { microphone: document.getElementById("microphone") as HTMLButtonElement, camera: document.getElementById("camera") as HTMLButtonElement, + screenShare: document.getElementById("screenShare") as HTMLButtonElement, leave: document.getElementById("leave") as HTMLButtonElement, }; @@ -74,8 +75,7 @@ async function start( client, room, { - microphoneEnabled: true, - cameraEnabled: true, + publish: [{ source: "microphone" }, { source: "camera" }], audioOutputDeviceId$: constant(undefined), }, { @@ -94,10 +94,13 @@ async function start( rtcClient.join(); toggle(buttons.microphone, async (enabled) => - rtcClient.setMicrophoneEnabled(enabled), + setPublished(rtcClient, "microphone", enabled), ); toggle(buttons.camera, async (enabled) => - rtcClient.setCameraEnabled(enabled), + setPublished(rtcClient, "camera", enabled), + ); + toggle(buttons.screenShare, async (enabled) => + setPublished(rtcClient, "screenShare", enabled), ); buttons.leave.hidden = false; buttons.leave.onclick = (): void => { @@ -115,6 +118,31 @@ async function start( } } +/** + * Turns one of our sources on or off: a microphone or camera is muted and + * unmuted while it exists and published when it does not; a screen share is + * published and unpublished. + */ +async function setPublished( + rtcClient: MatrixRTCClient, + source: "microphone" | "camera" | "screenShare", + enabled: boolean, +): Promise { + const member = rtcClient.localMember$.value; + if (member === null) return !enabled; + const track = member.media$.value?.tracks$.value.find( + (t) => t.source === source, + ); + if (track && source === "screenShare") { + if (!enabled) await member.unpublish(track.id); + return enabled; + } + if (track) return track.setEnabled(enabled); + if (!enabled) return false; + await member.publish({ source }); + return true; +} + /** A pressed button means the source is on; the SDK says what it could do. */ function toggle( button: HTMLButtonElement, diff --git a/sdk/src/api.ts b/sdk/src/api.ts index aa468f583..1cc7ab2a8 100644 --- a/sdk/src/api.ts +++ b/sdk/src/api.ts @@ -57,23 +57,16 @@ export interface MatrixRTCClientOptions { timings?: Partial; /** Limits on what is published; LiveKit's defaults otherwise. */ mediaQuality?: MediaQuality; - /** How the local tracks are captured and encoded. */ - capture?: CaptureSettings; /** Use this transport instead of asking the homeserver. */ transportUrl?: string; /** Use this transport when the homeserver advertises none. */ fallbackTransportUrl?: string; } -/** Capture and encoding settings for the local tracks, applied at creation. */ -export interface CaptureSettings { - audio?: { - echoCancellation?: boolean; - noiseSuppression?: boolean; - autoGainControl?: boolean; - }; - camera?: VideoCaptureSettings; - screenShare?: VideoCaptureSettings; +export interface AudioCaptureSettings { + echoCancellation?: boolean; + noiseSuppression?: boolean; + autoGainControl?: boolean; } export interface VideoCaptureSettings { @@ -84,17 +77,29 @@ export interface VideoCaptureSettings { } /** - * What the local member publishes when it joins. Initial values: from then on - * `setMicrophoneEnabled` and `setCameraEnabled` on the client, and the - * controls on the local tracks, change it. + * One thing to publish: a source, where to capture it from and how to encode + * it. Where the device and the settings are left out, the browser's and + * LiveKit's defaults apply. */ +export type PublishRequest = + | { source: "microphone"; deviceId?: string; capture?: AudioCaptureSettings } + | { + source: "camera"; + deviceId?: string; + /** Background blur and the like. */ + processor?: TrackProcessor; + capture?: VideoCaptureSettings; + } + | { + source: "screenShare"; + /** Whether to capture the screen's audio too. Default true. */ + audio?: boolean; + capture?: VideoCaptureSettings; + }; + export interface LocalMediaInputs { - microphoneEnabled: boolean; - cameraEnabled: boolean; - audioInputDeviceId?: string; - videoInputDeviceId?: string; - /** Background blur and the like. */ - videoProcessor?: TrackProcessor; + /** Published at the join; `publish` on the local member adds to it from then on. */ + publish: PublishRequest[]; /** Undefined where the host routes audio itself, or to leave the browser's choice. */ audioOutputDeviceId$: Behavior; } @@ -129,14 +134,6 @@ export interface MatrixRTCClient { /** Null while connected, the first failing link otherwise. */ disconnectReason$: Behavior; - /** - * Publishes or mutes the microphone. Resolves with the state that - * resulted, which differs from the request where the device could not be - * used. Before the transport is connected the request is remembered and - * applied once it is. - */ - setMicrophoneEnabled(enabled: boolean): Promise; - setCameraEnabled(enabled: boolean): Promise; /** A transport, Matrix or connection error that stops the session. */ fatalError$: Behavior; @@ -239,11 +236,18 @@ export interface RemoteRTCMember extends RTCMember { export interface LocalRTCMember extends RTCMember { local: true; media$: Behavior; - sharingScreen$: Behavior; - /** Null when the platform cannot share a screen. */ - toggleScreenSharing: (() => void) | null; - screenShareError$: Behavior; - dismissScreenShareError(): void; + /** + * Publishes a source and resolves with its track once it is in `tracks$`. + * Before the transport is connected the request is remembered and applied + * once it is. Rejects where the device could not be used, including the + * user closing the picker. One publication per source: publishing a source + * again unmutes it. + */ + publish( + request: PublishRequest, + ): Promise; + /** Removes one of our tracks; a screen share takes its audio with it. */ + unpublish(id: string): Promise; } // --------------------------------------------------------------------------- diff --git a/sdk/src/media/LivekitMediaTrack.ts b/sdk/src/media/LivekitMediaTrack.ts index b71be7ed2..1e019e2be 100644 --- a/sdk/src/media/LivekitMediaTrack.ts +++ b/sdk/src/media/LivekitMediaTrack.ts @@ -52,7 +52,7 @@ import { import { MatrixRTCError } from "../errors"; import { LazyBehavior } from "../utils/LazyBehavior"; -const sources: Record = { +export const mediaSources: Record = { [Track.Source.Microphone]: "microphone", [Track.Source.Camera]: "camera", [Track.Source.ScreenShare]: "screenShare", @@ -114,7 +114,7 @@ export function createLivekitMediaTrack( publication.isMuted, ); const base: MediaTrack = { - source: sources[publication.source], + source: mediaSources[publication.source], kind: publication.kind === Track.Kind.Audio ? "audio" : "video", id: publication.trackSid, muted$, diff --git a/sdk/src/session/ConnectionFactory.ts b/sdk/src/session/ConnectionFactory.ts index 4e4038d7b..532368858 100644 --- a/sdk/src/session/ConnectionFactory.ts +++ b/sdk/src/session/ConnectionFactory.ts @@ -23,12 +23,14 @@ import { type OpenIDClientParts, type SFUConfig } from "./openIDSFU"; import { type Behavior } from "../reactive/Behavior"; import { Connection, type ConnectionOpts, ConnectionState } from "./Connection"; import { type ObservableScope } from "../reactive/ObservableScope"; -import { - type CaptureSettings, - type LocalMediaInputs, - type ResolvedTransport, -} from "../api"; +import { type LocalMediaInputs, type ResolvedTransport } from "../api"; import { type MediaQuality } from "../config"; +import { + audioCaptureOptions, + type PublishRequestFor, + videoCaptureOptions, + videoPublishOptions, +} from "./publishOptions"; export interface ConnectionFactory { createConnection( @@ -93,7 +95,6 @@ export class LivekitConnectionFactory implements ConnectionFactory { private readonly localMedia: LocalMediaInputs, private readonly keyProvider: BaseKeyProvider | undefined, private readonly mediaQuality: MediaQuality | undefined, - private readonly capture: CaptureSettings | undefined, ) {} public createConnection( @@ -112,12 +113,7 @@ export class LivekitConnectionFactory implements ConnectionFactory { scope, livekitRoomFactory: () => new LivekitRoom( - roomOptions( - this.localMedia, - this.keyProvider, - this.mediaQuality, - this.capture, - ), + roomOptions(this.localMedia, this.keyProvider, this.mediaQuality), ), ownMembershipIdentity, }, @@ -126,36 +122,28 @@ export class LivekitConnectionFactory implements ConnectionFactory { } } +/** The initial requests become the room's defaults, so that one permission prompt covers both. */ function roomOptions( localMedia: LocalMediaInputs, keyProvider: BaseKeyProvider | undefined, mediaQuality: MediaQuality | undefined, - capture: CaptureSettings | undefined, ): RoomOptions { const base = buildLiveKitOptions(mediaQuality); - const camera = capture?.camera; + const microphone = initialRequest(localMedia, "microphone"); + const camera = initialRequest(localMedia, "camera"); return { ...base, videoCaptureDefaults: { ...base.videoCaptureDefaults, - deviceId: localMedia.videoInputDeviceId, - processor: localMedia.videoProcessor, - ...(camera?.resolution && { resolution: camera.resolution }), + ...(camera && videoCaptureOptions(camera)), }, publishDefaults: { ...base.publishDefaults, - ...(camera?.maxBitrate !== undefined && { - videoEncoding: { - maxBitrate: camera.maxBitrate, - maxFramerate: camera.maxFramerate, - }, - }), - ...(camera?.codec && { videoCodec: camera.codec }), + ...(camera && videoPublishOptions(camera.capture)), }, audioCaptureDefaults: { ...base.audioCaptureDefaults, - deviceId: localMedia.audioInputDeviceId, - ...capture?.audio, + ...(microphone && audioCaptureOptions(microphone)), }, audioOutput: { deviceId: localMedia.audioOutputDeviceId$.value }, // Every room needs a worker of its own: one gets confused by streams from @@ -163,3 +151,12 @@ function roomOptions( e2ee: keyProvider && { keyProvider, worker: new E2EEWorker() }, }; } + +function initialRequest( + localMedia: LocalMediaInputs, + source: S, +): PublishRequestFor | undefined { + return localMedia.publish.find( + (request): request is PublishRequestFor => request.source === source, + ); +} diff --git a/sdk/src/session/LocalMember.ts b/sdk/src/session/LocalMember.ts index ce8dd6c23..b54cb5d6f 100644 --- a/sdk/src/session/LocalMember.ts +++ b/sdk/src/session/LocalMember.ts @@ -8,11 +8,8 @@ Please see LICENSE in the repository root for full details. import { observeParticipantEvents } from "@livekit/components-core"; import { type LocalParticipant, - type Participant, + type LocalTrackPublication, ParticipantEvent, - type ScreenShareCaptureOptions, - Track, - type TrackPublishOptions, } from "livekit-client"; import { type MatrixClient } from "matrix-js-sdk"; import { type Logger } from "matrix-js-sdk/lib/logger"; @@ -29,6 +26,8 @@ import { combineLatest, concat, distinctUntilChanged, + filter, + firstValueFrom, map, NEVER, type Observable, @@ -49,7 +48,7 @@ import { import { type DisconnectReason, type MediaSource, - type VideoCaptureSettings, + type PublishRequest, } from "../api"; import { FailToStartLivekitConnection, @@ -105,7 +104,6 @@ interface Props { matrixRTCMode: MatrixRTCMode; timings: SessionTimings; desired: DesiredMedia; - screenShare?: VideoCaptureSettings; logger: Logger; } @@ -121,14 +119,11 @@ export interface LocalMembership { /** Connected once, and currently not. */ reconnecting$: Behavior; disconnectReason$: Behavior; - setMicrophoneEnabled: (enabled: boolean) => Promise; - setCameraEnabled: (enabled: boolean) => Promise; + /** Resolves with the publication, once a publisher has made it. */ + publish: (request: PublishRequest) => Promise; + unpublish: (source: MediaSource) => Promise; /** Mutes or unmutes one of our publications; the result is the state that resulted. */ setEnabled: (source: MediaSource, enabled: boolean) => Promise; - sharingScreen$: Behavior; - toggleScreenSharing: (() => void) | null; - screenShareError$: Behavior; - dismissScreenShareError: () => void; } /** @@ -152,7 +147,6 @@ export function createLocalMembership$({ matrixRTCMode, timings, desired, - screenShare, logger: parentLogger, }: Props): LocalMembership { const logger = parentLogger.getChild("[LocalMember]"); @@ -222,7 +216,7 @@ export function createLocalMembership$({ if (publisher === null) return; try { if (shouldPublish) { - publisher.createAndSetupTracks(); + publisher.start(); await publisher.startPublishing(); } else if (publisher.shouldPublish) await publisher.stopPublishing(); } catch (e) { @@ -398,74 +392,62 @@ export function createLocalMembership$({ } }); + const publish = async ( + request: PublishRequest, + ): Promise => { + desired.set(request.source, { request, enabled: true }); + const publisher = publisher$.value; + try { + // Before the start, the publisher publishes everything desired itself + if (publisher?.started) { + const publication = await publisher.publish(request); + if (publication) return publication; + } + return await firstValueFrom(publication$(request.source)); + } catch (e) { + desired.delete(request.source); + throw toMatrixRTCError(e); + } + }; + + const publication$ = ( + source: MediaSource, + ): Observable => + participant$.pipe( + switchMap((participant) => + participant === null + ? NEVER + : observeParticipantEvents( + participant, + ParticipantEvent.LocalTrackPublished, + ).pipe( + map(() => + participant.getTrackPublication(livekitSources[source]), + ), + ), + ), + filter((publication) => publication !== undefined), + ); + + const unpublish = async (source: MediaSource): Promise => { + desired.delete(source); + await publisher$.value?.unpublish(source); + }; + const setEnabled = async ( source: MediaSource, enabled: boolean, ): Promise => { - if (source !== "microphone" && source !== "camera") - return setPublicationEnabled(source, enabled); - const desired$ = - source === "microphone" ? desired.microphone$ : desired.camera$; - desired$.next(enabled); + const wanted = desired.get(source); + if (wanted) wanted.enabled = enabled; // Without a publisher the request waits for the tracks to be created const publisher = publisher$.value; if (publisher === null) return enabled; - const result = await publisher.setEnabled( - source === "microphone" ? Track.Source.Microphone : Track.Source.Camera, - enabled, - ); - if (result !== enabled) desired$.next(result); + const result = await publisher.setEnabled(source, enabled); + if (wanted && result !== enabled) wanted.enabled = result; return result; }; - // A publication the publisher does not recreate on reconnection, so there - // is nothing to remember for it - const setPublicationEnabled = async ( - source: MediaSource, - enabled: boolean, - ): Promise => { - const publication = participant$.value?.getTrackPublication( - livekitSources[source], - ); - if (!publication) return false; - if (enabled) await publication.unmute(); - else await publication.mute(); - return !publication.isMuted; - }; - - const sharingScreen$ = scope.behavior( - participant$.pipe( - switchMap((p) => (p === null ? of(false) : observeSharingScreen$(p))), - ), - ); - const screenShareError$ = new BehaviorSubject(null); - const toggleScreenSharing = - "getDisplayMedia" in (navigator.mediaDevices ?? {}) - ? (): void => { - const participant = participant$.value; - if (participant === null) return; - const enable = !sharingScreen$.value; - participant - .setScreenShareEnabled( - enable, - screenShareCaptureOptions(screenShare), - screenSharePublishOptions(screenShare), - ) - .catch((e: unknown) => { - logger.error( - `Screen share ${enable ? "start" : "stop"} failed`, - e, - ); - // The user closing the picker is not an error worth showing - if (e instanceof DOMException && e.name === "NotAllowedError") - return; - screenShareError$.next( - e instanceof Error ? e : new Error(String(e)), - ); - }); - } - : null; - return { requestJoinAndPublish: () => joinRequested$.next(true), requestDisconnect: () => joinRequested$.next(false), @@ -476,13 +458,9 @@ export function createLocalMembership$({ connected$, reconnecting$, disconnectReason$, - setMicrophoneEnabled: async (enabled) => setEnabled("microphone", enabled), - setCameraEnabled: async (enabled) => setEnabled("camera", enabled), + publish, + unpublish, setEnabled, - sharingScreen$, - toggleScreenSharing, - screenShareError$, - dismissScreenShareError: () => screenShareError$.next(null), }; } @@ -511,46 +489,3 @@ async function checkDelegationSupport( return false; } } - -function screenShareCaptureOptions( - settings: VideoCaptureSettings | undefined, -): ScreenShareCaptureOptions { - return { - // No echo cancellation: it would cancel the other members' voices out of - // the shared audio - audio: { - autoGainControl: false, - noiseSuppression: false, - voiceIsolation: false, - }, - selfBrowserSurface: "include", - surfaceSwitching: "include", - systemAudio: "include", - ...(settings?.resolution && { resolution: settings.resolution }), - }; -} - -function screenSharePublishOptions( - settings: VideoCaptureSettings | undefined, -): TrackPublishOptions | undefined { - if (settings === undefined) return undefined; - return { - ...(settings.maxBitrate !== undefined && { - screenShareEncoding: { - maxBitrate: settings.maxBitrate, - maxFramerate: settings.maxFramerate, - }, - }), - ...(settings.codec && { videoCodec: settings.codec }), - }; -} - -function observeSharingScreen$(participant: Participant): Observable { - return observeParticipantEvents( - participant, - ParticipantEvent.TrackPublished, - ParticipantEvent.TrackUnpublished, - ParticipantEvent.LocalTrackPublished, - ParticipantEvent.LocalTrackUnpublished, - ).pipe(map((p) => p.isScreenShareEnabled)); -} diff --git a/sdk/src/session/MatrixRTCClient.ts b/sdk/src/session/MatrixRTCClient.ts index d76cf409f..de200fba7 100644 --- a/sdk/src/session/MatrixRTCClient.ts +++ b/sdk/src/session/MatrixRTCClient.ts @@ -15,14 +15,7 @@ import { import { type IMembershipManager } from "matrix-js-sdk/lib/matrixrtc/IMembershipManager"; import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager"; import { v4 as uuidv4 } from "uuid"; -import { - BehaviorSubject, - combineLatest, - from, - fromEvent, - map, - type Observable, -} from "rxjs"; +import { combineLatest, from, fromEvent, map, type Observable } from "rxjs"; import { defaultSessionTimings, MatrixRTCMode } from "../config"; import { type ObservableScope } from "../reactive/ObservableScope"; @@ -123,10 +116,12 @@ export function createMatrixRTCClient( }), ); - const desired: DesiredMedia = { - microphone$: new BehaviorSubject(localMedia.microphoneEnabled), - camera$: new BehaviorSubject(localMedia.cameraEnabled), - }; + const desired: DesiredMedia = new Map( + localMedia.publish.map((request) => [ + request.source, + { request, enabled: true }, + ]), + ); const connectionManager = createConnectionManager$({ scope, @@ -136,7 +131,6 @@ export function createMatrixRTCClient( localMedia, keyProvider, options.mediaQuality, - options.capture, ), localTransport$, remoteTransports$: transports$, @@ -194,7 +188,6 @@ export function createMatrixRTCClient( matrixRTCMode, timings, desired, - screenShare: options.capture?.screenShare, logger, }); @@ -276,8 +269,6 @@ export function createMatrixRTCClient( connected$: localMembership.connected$, reconnecting$: localMembership.reconnecting$, disconnectReason$: localMembership.disconnectReason$, - setMicrophoneEnabled: localMembership.setMicrophoneEnabled, - setCameraEnabled: localMembership.setCameraEnabled, fatalError$: scope.behavior(localMembership.state$.pipe(map(fatalError))), localMember$, remoteMembers$, diff --git a/sdk/src/session/Members.ts b/sdk/src/session/Members.ts index f2429ac61..eb99c79ed 100644 --- a/sdk/src/session/Members.ts +++ b/sdk/src/session/Members.ts @@ -11,7 +11,15 @@ import { type Room as LivekitRoom, } from "livekit-client"; import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc"; -import { combineLatest, distinctUntilChanged, map } from "rxjs"; +import { + combineLatest, + distinctUntilChanged, + filter, + firstValueFrom, + map, + of, + switchMap, +} from "rxjs"; import { type Behavior } from "../reactive/Behavior"; import { type ObservableScope } from "../reactive/ObservableScope"; @@ -80,26 +88,40 @@ export function createLocalRTCMember( localMembership: LocalMembership, context: MemberContext, ): LocalRTCMember { + const media$ = mediaFor( + scope, + localMembership.participant$, + localMembership.connection$, + (mediaScope, participant, room) => + createLocalLivekitMemberMedia( + mediaScope, + participant, + room, + context.encryptionSystem, + localMembership.setEnabled, + ), + ); return { ...createRTCMember(scope, membership$, context), local: true, - media$: mediaFor( - scope, - localMembership.participant$, - localMembership.connection$, - (mediaScope, participant, room) => - createLocalLivekitMemberMedia( - mediaScope, - participant, - room, - context.encryptionSystem, - localMembership.setEnabled, + media$, + publish: async (request) => { + const { trackSid } = await localMembership.publish(request); + return firstValueFrom( + media$.pipe( + switchMap((media) => media?.tracks$ ?? of([])), + map((tracks) => tracks.find((track) => track.id === trackSid)), + filter((track) => track !== undefined), ), - ), - sharingScreen$: localMembership.sharingScreen$, - toggleScreenSharing: localMembership.toggleScreenSharing, - screenShareError$: localMembership.screenShareError$, - dismissScreenShareError: localMembership.dismissScreenShareError, + ); + }, + unpublish: async (id) => { + const track = media$.value?.tracks$.value.find((t) => t.id === id); + if (track === undefined) return; + await localMembership.unpublish( + track.source === "screenShareAudio" ? "screenShare" : track.source, + ); + }, }; } diff --git a/sdk/src/session/Publisher.ts b/sdk/src/session/Publisher.ts index e85c41312..6f19a5f1a 100644 --- a/sdk/src/session/Publisher.ts +++ b/sdk/src/session/Publisher.ts @@ -10,20 +10,33 @@ import { type LocalTrackPublication, ParticipantEvent, type Room as LivekitRoom, - Track, + type Track, } from "livekit-client"; import { type Logger } from "matrix-js-sdk/lib/logger"; -import { type BehaviorSubject } from "rxjs"; import { ObservableScope } from "../reactive/ObservableScope"; -import { type LocalMediaInputs } from "../api"; +import { + type LocalMediaInputs, + type MediaSource, + type PublishRequest, +} from "../api"; +import { livekitSources, mediaSources } from "../media/LivekitMediaTrack"; +import { + audioCaptureOptions, + screenShareCaptureOptions, + screenSharePublishOptions, + videoCaptureOptions, + videoPublishOptions, +} from "./publishOptions"; -/** What the host last asked for, kept across publishers. */ -export interface DesiredMedia { - microphone$: BehaviorSubject; - camera$: BehaviorSubject; +/** What the host asked to publish, kept across publishers for a reconnection. */ +export interface DesiredPublication { + request: PublishRequest; + enabled: boolean; } +export type DesiredMedia = Map; + /** * Publishes the local media on one LiveKit room, following the host's * requests. @@ -35,7 +48,8 @@ export interface DesiredMedia { */ export class Publisher { public shouldPublish = false; - private tracksRequested = false; + /** Whether the initial requests have been published; `publish` is immediate from then on. */ + public started = false; private readonly scope = new ObservableScope(); private readonly room: LivekitRoom; @@ -64,53 +78,106 @@ export class Publisher { this.onLocalTrackPublished, ); try { - await this.stopTracks(); + for (const { track } of this.publications()) + if (track) await this.room.localParticipant.unpublishTrack(track, true); } catch (e) { this.logger.error("Failed to stop the local tracks", e); } } - /** - * Creates the microphone and camera tracks the host asked for. Both are - * enabled in one call so that the browser asks for permission once. Safe to - * call more than once. - */ - public createAndSetupTracks(): void { - if (this.tracksRequested) return; - this.tracksRequested = true; + /** Publishes what the host asked for, less what it has muted since. Safe to call more than once. */ + public start(): void { + if (this.started) return; + this.started = true; + const requests = [...this.desired.values()] + .filter(({ enabled }) => enabled) + .map(({ request }) => request); + const sources = new Set(requests.map((request) => request.source)); + // Microphone and camera in one call so that the browser asks for + // permission once; the room defaults carry their options. LiveKit + // resolves these once the track is published, which may block on the + // connection; LocalTrackPublished is what tells us a track exists. + if (sources.has("microphone") && sources.has("camera")) + void this.room.localParticipant.enableCameraAndMicrophone(); + for (const request of requests) + if ( + request.source === "screenShare" || + !(sources.has("microphone") && sources.has("camera")) + ) + this.publish(request).catch((e) => { + this.logger.error(`Failed to publish the ${request.source}`, e); + }); + } + + /** Publishes one request and resolves with its publication once LiveKit has it. */ + public async publish( + request: PublishRequest, + ): Promise { const participant = this.room.localParticipant; - const audio = this.desired.microphone$.value; - const video = this.desired.camera$.value; - // LiveKit resolves these once the track is published, which may block on - // the connection; LocalTrackPublished is what tells us a track exists. - if (audio && video) void participant.enableCameraAndMicrophone(); - else if (audio) void participant.setMicrophoneEnabled(true); - else if (video) void participant.setCameraEnabled(true); + let publication: LocalTrackPublication | undefined; + if (request.source === "microphone") + publication = await participant.setMicrophoneEnabled( + true, + audioCaptureOptions(request), + ); + else if (request.source === "camera") + publication = await participant.setCameraEnabled( + true, + videoCaptureOptions(request), + videoPublishOptions(request.capture), + ); + else + publication = await participant.setScreenShareEnabled( + true, + screenShareCaptureOptions(request), + screenSharePublishOptions(request.capture), + ); + // Until the member has joined, the upstream has to stay paused + if (!this.shouldPublish) await this.pauseUpstreams(); + return publication; + } + + public async unpublish(source: MediaSource): Promise { + const participant = this.room.localParticipant; + // Takes the screen share audio with it + if (source === "screenShare") { + await participant.setScreenShareEnabled(false); + return; + } + const track = participant.getTrackPublication( + livekitSources[source], + )?.track; + if (track) await participant.unpublishTrack(track, true); } /** - * Enables or disables a source and reports what resulted: the request where + * Mutes or unmutes a source and reports what resulted: the request where * it succeeded, the state LiveKit is left in where it did not (a denied * permission, a missing device). */ public async setEnabled( - source: Track.Source.Microphone | Track.Source.Camera, + source: MediaSource, enabled: boolean, ): Promise { const participant = this.room.localParticipant; + const livekitSource = livekitSources[source]; try { - if (source === Track.Source.Microphone) + if (source === "microphone") await participant.setMicrophoneEnabled(enabled); - else await participant.setCameraEnabled(enabled); + else if (source === "camera") await participant.setCameraEnabled(enabled); + else { + const publication = participant.getTrackPublication(livekitSource); + if (!publication) return false; + if (enabled) await publication.unmute(); + else await publication.mute(); + } // Unmuting restarts the upstream; until the member has joined, it has // to stay paused - if (enabled && !this.shouldPublish) await this.pauseUpstreams([source]); + if (enabled && !this.shouldPublish) await this.pauseUpstreams(); return enabled; } catch (e) { this.logger.error(`Failed to set ${source} enabled=${enabled}`, e); - return source === Track.Source.Microphone - ? participant.isMicrophoneEnabled - : participant.isCameraEnabled; + return !(participant.getTrackPublication(livekitSource)?.isMuted ?? true); } } @@ -119,64 +186,34 @@ export class Publisher { this.shouldPublish = true; // Enabling a track does not resume an upstream that was paused while it // was already enabled, so it is done explicitly - await this.resumeUpstreams([Track.Source.Microphone, Track.Source.Camera]); + for (const { track } of this.publications()) + if (track?.isUpstreamPaused) await track.resumeUpstream(); } public async stopPublishing(): Promise { this.shouldPublish = false; - await this.pauseUpstreams([ - Track.Source.Microphone, - Track.Source.Camera, - Track.Source.ScreenShare, - ]); - } - - private async stopTracks(): Promise { - const participant = this.room.localParticipant; - for (const source of [ - Track.Source.Microphone, - Track.Source.Camera, - Track.Source.ScreenShare, - ]) { - const track = participant.getTrackPublication(source)?.track; - if (track) await participant.unpublishTrack(track, true); - } + await this.pauseUpstreams(); } private onLocalTrackPublished(publication: LocalTrackPublication): void { this.logger.info(`Local ${publication.source} track published`); if (!this.shouldPublish) - this.pauseUpstreams([publication.source]).catch((e) => { + this.pauseUpstreams().catch((e) => { this.logger.error("Failed to pause the upstream", e); }); // The host may have changed its mind while the track was being created - const desired = - publication.source === Track.Source.Microphone - ? this.desired.microphone$.value - : publication.source === Track.Source.Camera - ? this.desired.camera$.value - : undefined; - if (desired === false) - void this.setEnabled( - publication.source as Track.Source.Microphone | Track.Source.Camera, - false, - ); + const source = mediaSources[publication.source as Track.Source]; + if (this.desired.get(source)?.enabled === false) + void this.setEnabled(source, false); } - private async pauseUpstreams(sources: Track.Source[]): Promise { - for (const source of sources) { - const track = - this.room.localParticipant.getTrackPublication(source)?.track; + private async pauseUpstreams(): Promise { + for (const { track } of this.publications()) if (track && !track.isUpstreamPaused) await track.pauseUpstream(); - } } - private async resumeUpstreams(sources: Track.Source[]): Promise { - for (const source of sources) { - const track = - this.room.localParticipant.getTrackPublication(source)?.track; - if (track?.isUpstreamPaused) await track.resumeUpstream(); - } + private publications(): LocalTrackPublication[] { + return [...this.room.localParticipant.trackPublications.values()]; } private followAudioOutput(): void { diff --git a/sdk/src/session/publishOptions.ts b/sdk/src/session/publishOptions.ts new file mode 100644 index 000000000..4e433f453 --- /dev/null +++ b/sdk/src/session/publishOptions.ts @@ -0,0 +1,91 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { + type AudioCaptureOptions, + type ScreenShareCaptureOptions, + type TrackPublishOptions, + type VideoCaptureOptions, +} from "livekit-client"; + +import { type PublishRequest, type VideoCaptureSettings } from "../api"; + +export type PublishRequestFor = Extract< + PublishRequest, + { source: S } +>; + +/** LiveKit fills in the room defaults for whatever a request leaves out. */ +export function audioCaptureOptions({ + deviceId, + capture, +}: PublishRequestFor<"microphone">): AudioCaptureOptions | undefined { + if (deviceId === undefined && capture === undefined) return undefined; + return { ...(deviceId && { deviceId }), ...capture }; +} + +export function videoCaptureOptions({ + deviceId, + processor, + capture, +}: PublishRequestFor<"camera">): VideoCaptureOptions | undefined { + if (!deviceId && !processor && !capture?.resolution) return undefined; + return { + ...(deviceId && { deviceId }), + ...(processor && { processor }), + ...(capture?.resolution && { resolution: capture.resolution }), + }; +} + +export function videoPublishOptions( + settings: VideoCaptureSettings | undefined, +): TrackPublishOptions | undefined { + if (settings === undefined) return undefined; + return { + ...(settings.maxBitrate !== undefined && { + videoEncoding: { + maxBitrate: settings.maxBitrate, + maxFramerate: settings.maxFramerate, + }, + }), + ...(settings.codec && { videoCodec: settings.codec }), + }; +} + +export function screenShareCaptureOptions({ + audio = true, + capture, +}: PublishRequestFor<"screenShare">): ScreenShareCaptureOptions { + return { + // No echo cancellation: it would cancel the other members' voices out of + // the shared audio + audio: audio && { + autoGainControl: false, + noiseSuppression: false, + voiceIsolation: false, + }, + selfBrowserSurface: "include", + surfaceSwitching: "include", + systemAudio: "include", + ...(capture?.resolution && { resolution: capture.resolution }), + }; +} + +export function screenSharePublishOptions( + settings: VideoCaptureSettings | undefined, +): TrackPublishOptions | undefined { + if (settings === undefined) return undefined; + return { + ...(settings.maxBitrate !== undefined && { + screenShareEncoding: { + maxBitrate: settings.maxBitrate, + maxFramerate: settings.maxFramerate, + }, + }), + ...(settings.codec && { videoCodec: settings.codec }), + }; +}