Matrix/element-call-Github
1
0
Fork 0
mirror of https://github.com/vector-im/element-call.git synced 2026-10-16 23:45:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Publish and unpublish on the local member

Browse Source 
One way to put media on the session: publish(request) resolves with the local track once LiveKit has it, and unpublish(id) removes it. A request names a source with its device, processor and capture settings, so CaptureSettings on the client options goes, as do setMicrophoneEnabled and setCameraEnabled on the client and the four screen share fields on the local member; sharing is tracks$ having a screenShare track, and a capture failure rejects the promise. LocalMediaInputs becomes the list of initial requests, which also become the room defaults so that one permission prompt covers microphone and camera. The publisher keeps the requests across a reconnection and republishes what is not muted. The harness gains a Share screen button.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Timo K.
2026-10-08 13:53:57 +02:00
co-authored by Claude Fable 5.1
parent 5a43eaf5a6
commit b9f79be106
12 changed files with 441 additions and 326 deletions
Download Patch File Download Diff File Expand all files Collapse all files
sdk/README.md
+1 -2
View File
@@ -30,8 +30,7 @@ const rtcClient = createMatrixRTCClient(
client, // a matrix-js-sdk MatrixClient, logged in and syncing
room, // the matrix-js-sdk Room, from client.getRoom() once the join has synced
{
microphoneEnabled: true,
cameraEnabled: true,
publish: [{ source: "microphone" }, { source: "camera" }],
audioOutputDeviceId$: constant(undefined),
},
{
sdk/SdkArchitecture.md
+40 -32
View File
@@ -189,8 +189,6 @@ export interface MatrixRTCClientOptions {
timings?: Partial<SessionTimings>;
/** Limits on what is published; LiveKit's defaults otherwise. */
mediaQuality?: MediaQuality;
/** How the local tracks are captured and encoded: audio processing, camera and screen share resolution, bitrate, codec. */
capture?: CaptureSettings;
/** Use this transport instead of asking the homeserver. */
transportUrl?: string;
/** Use this transport when the homeserver advertises none. */
@@ -198,18 +196,33 @@ export interface MatrixRTCClientOptions {
}
/**
* What the local member publishes when it joins. Initial values: from then on
* `setMicrophoneEnabled` and `setCameraEnabled` on the client, and the
* controls on the local tracks, change it. Device enumeration, permission
* prompts and the lobby preview stay with the host.
* One thing to publish: a source, where to capture it from and how to encode
* it. Where the device and the settings are left out, the browser's and
* LiveKit's defaults apply.
*/
export type PublishRequest =
| { source: "microphone"; deviceId?: string; capture?: AudioCaptureSettings }
| {
source: "camera";
deviceId?: string;
/** Background blur and the like. */
processor?: TrackProcessor<Track.Kind.Video>;
capture?: VideoCaptureSettings;
}
| {
source: "screenShare";
/** Whether to capture the screen's audio too. Default true. */
audio?: boolean;
capture?: VideoCaptureSettings;
};
/**
* Device enumeration, permission prompts and the lobby preview stay with the
* host.
*/
export interface LocalMediaInputs {
microphoneEnabled: boolean;
cameraEnabled: boolean;
audioInputDeviceId?: string;
videoInputDeviceId?: string;
/** Background blur and the like. */
videoProcessor?: TrackProcessor<Track.Kind.Video>;
/** Published at the join; `publish` on the local member adds to it from then on. */
publish: PublishRequest[];
/** Undefined where the host routes audio itself, or to leave the browser's choice. */
audioOutputDeviceId$: Behavior<string | undefined>;
}
@@ -268,14 +281,6 @@ export interface MatrixRTCClient {
/** Null while connected, the first failing link otherwise. */
disconnectReason$: Behavior<DisconnectReason | null>;
/**
* Publishes or mutes the microphone. Resolves with the state that
* resulted, which differs from the request where the device could not be
* used. Before the transport is connected the request is remembered and
* applied once it is.
*/
setMicrophoneEnabled(enabled: boolean): Promise<boolean>;
setCameraEnabled(enabled: boolean): Promise<boolean>;
/**
* A transport, Matrix or connection error that stops the session. Null
* while fine. A failed publication is not fatal: the member can still
@@ -367,12 +372,18 @@ export interface RemoteRTCMember extends RTCMember {
export interface LocalRTCMember extends RTCMember {
local: true;
media$: Behavior<LocalMemberMedia | null>;
sharingScreen$: Behavior<boolean>;
/** Null when the platform cannot share a screen (no getDisplayMedia). Hiding the button is the host's policy. */
toggleScreenSharing: (() => void) | null;
/** The last error from toggling, until dismissed. The user closing the picker is not one. */
screenShareError$: Behavior<Error | null>;
dismissScreenShareError(): void;
/**
* Publishes a source and resolves with its track once it is in `tracks$`.
* Before the transport is connected the request is remembered and applied
* once it is. Rejects where the device could not be used, including the
* user closing the picker. One publication per source: publishing a source
* again unmutes it.
*/
publish(
request: PublishRequest,
): Promise<LocalAudioMediaTrack | LocalVideoMediaTrack>;
/** Removes one of our tracks; a screen share takes its audio with it. */
unpublish(id: string): Promise<void>;
}
```
@@ -426,7 +437,7 @@ they only make sense for a call:
| `autoLeave$` / `AutoLeaveReason` (`"allOthersLeft" \| "timeout" \| "decline"`) | hanging up because the other side declined or everyone left is call etiquette, not session semantics | `CallViewModel.leave$`, built from `remoteMembers$` and the room timeline |
| `ringAttempts$`, `createCallNotificationLifecycle$`, `createSentCallNotification$`, `createReceivedDecline$` | ringing, pickup timeouts and decline events are MSC4075 call-notify UX | `src/state/CallViewModel/CallNotificationLifecycle.ts`, fed by `remoteMembers$` and the Matrix room |
| `waitForCallPickup`, `autoLeaveWhenOthersLeft` options | inputs to the above | `CallViewModelOptions` |
| `hideScreensharing` option | a UI policy; the session can always share a screen if the platform can | `CallViewModel` nulls `toggleScreenSharing` |
| `hideScreensharing` option | a UI policy; the session can always share a screen if the platform can | `CallViewModel` offers no screen share control |
| `ringingVm$`, `ringingStatusLocation` | view models | `CallViewModel` |
| `hostBridge.hangUp$`, `hangup()`, `userHangup$` | widget / host integration | `CallViewModel` |
| `ElementCallError` | carries Element Call translation keys | the view model wraps `MatrixRTCError`, reading `cause` |
@@ -645,11 +656,8 @@ public API only.
## Open decisions
- **`LocalMediaInputs` shape.** Everything in it is an initial value; the
setters on the client and the controls on the local tracks change it from
then on, because they have to report what the device allowed. Creating the
tracks before `join()` for a preview is still missing: today they are created
at the join.
- **Publishing before `join()`.** The initial requests are published at the
join; creating the tracks earlier, for a lobby preview, is still missing.
- **`applicationData` after the join.** The option is read once. The client
still updates `m.call.intent` from the camera state, as Element Call did; that
moves to the host once the data can be updated.
sdk/dev/index.html
+3
View File
@@ -60,6 +60,9 @@ Please see LICENSE in the repository root for full details.
<button type="button" id="camera" aria-pressed="true" hidden>
Camera
</button>
<button type="button" id="screenShare" aria-pressed="false" hidden>
Share screen
</button>
<button type="button" id="leave" hidden>Leave</button>
</form>
<p id="status" data-testid="status"></p>
sdk/dev/main.ts
+32 -4
View File
@@ -36,6 +36,7 @@ const messages = document.getElementById("messages")!;
const buttons = {
microphone: document.getElementById("microphone") as HTMLButtonElement,
camera: document.getElementById("camera") as HTMLButtonElement,
screenShare: document.getElementById("screenShare") as HTMLButtonElement,
leave: document.getElementById("leave") as HTMLButtonElement,
};
@@ -74,8 +75,7 @@ async function start(
client,
room,
{
microphoneEnabled: true,
cameraEnabled: true,
publish: [{ source: "microphone" }, { source: "camera" }],
audioOutputDeviceId$: constant(undefined),
},
{
@@ -94,10 +94,13 @@ async function start(
rtcClient.join();
toggle(buttons.microphone, async (enabled) =>
rtcClient.setMicrophoneEnabled(enabled),
setPublished(rtcClient, "microphone", enabled),
);
toggle(buttons.camera, async (enabled) =>
rtcClient.setCameraEnabled(enabled),
setPublished(rtcClient, "camera", enabled),
);
toggle(buttons.screenShare, async (enabled) =>
setPublished(rtcClient, "screenShare", enabled),
);
buttons.leave.hidden = false;
buttons.leave.onclick = (): void => {
@@ -115,6 +118,31 @@ async function start(
}
}
/**
* Turns one of our sources on or off: a microphone or camera is muted and
* unmuted while it exists and published when it does not; a screen share is
* published and unpublished.
*/
async function setPublished(
rtcClient: MatrixRTCClient,
source: "microphone" | "camera" | "screenShare",
enabled: boolean,
): Promise<boolean> {
const member = rtcClient.localMember$.value;
if (member === null) return !enabled;
const track = member.media$.value?.tracks$.value.find(
(t) => t.source === source,
);
if (track && source === "screenShare") {
if (!enabled) await member.unpublish(track.id);
return enabled;
}
if (track) return track.setEnabled(enabled);
if (!enabled) return false;
await member.publish({ source });
return true;
}
/** A pressed button means the source is on; the SDK says what it could do. */
function toggle(
button: HTMLButtonElement,
sdk/src/api.ts
+37 -33
View File
@@ -57,23 +57,16 @@ export interface MatrixRTCClientOptions {
timings?: Partial<SessionTimings>;
/** Limits on what is published; LiveKit's defaults otherwise. */
mediaQuality?: MediaQuality;
/** How the local tracks are captured and encoded. */
capture?: CaptureSettings;
/** Use this transport instead of asking the homeserver. */
transportUrl?: string;
/** Use this transport when the homeserver advertises none. */
fallbackTransportUrl?: string;
}
/** Capture and encoding settings for the local tracks, applied at creation. */
export interface CaptureSettings {
audio?: {
echoCancellation?: boolean;
noiseSuppression?: boolean;
autoGainControl?: boolean;
};
camera?: VideoCaptureSettings;
screenShare?: VideoCaptureSettings;
export interface AudioCaptureSettings {
echoCancellation?: boolean;
noiseSuppression?: boolean;
autoGainControl?: boolean;
}
export interface VideoCaptureSettings {
@@ -84,17 +77,29 @@ export interface VideoCaptureSettings {
}
/**
* What the local member publishes when it joins. Initial values: from then on
* `setMicrophoneEnabled` and `setCameraEnabled` on the client, and the
* controls on the local tracks, change it.
* One thing to publish: a source, where to capture it from and how to encode
* it. Where the device and the settings are left out, the browser's and
* LiveKit's defaults apply.
*/
export type PublishRequest =
| { source: "microphone"; deviceId?: string; capture?: AudioCaptureSettings }
| {
source: "camera";
deviceId?: string;
/** Background blur and the like. */
processor?: TrackProcessor<Track.Kind.Video>;
capture?: VideoCaptureSettings;
}
| {
source: "screenShare";
/** Whether to capture the screen's audio too. Default true. */
audio?: boolean;
capture?: VideoCaptureSettings;
};
export interface LocalMediaInputs {
microphoneEnabled: boolean;
cameraEnabled: boolean;
audioInputDeviceId?: string;
videoInputDeviceId?: string;
/** Background blur and the like. */
videoProcessor?: TrackProcessor<Track.Kind.Video>;
/** Published at the join; `publish` on the local member adds to it from then on. */
publish: PublishRequest[];
/** Undefined where the host routes audio itself, or to leave the browser's choice. */
audioOutputDeviceId$: Behavior<string | undefined>;
}
@@ -129,14 +134,6 @@ export interface MatrixRTCClient {
/** Null while connected, the first failing link otherwise. */
disconnectReason$: Behavior<DisconnectReason | null>;
/**
* Publishes or mutes the microphone. Resolves with the state that
* resulted, which differs from the request where the device could not be
* used. Before the transport is connected the request is remembered and
* applied once it is.
*/
setMicrophoneEnabled(enabled: boolean): Promise<boolean>;
setCameraEnabled(enabled: boolean): Promise<boolean>;
/** A transport, Matrix or connection error that stops the session. */
fatalError$: Behavior<MatrixRTCError | null>;
@@ -239,11 +236,18 @@ export interface RemoteRTCMember extends RTCMember {
export interface LocalRTCMember extends RTCMember {
local: true;
media$: Behavior<LocalMemberMedia | null>;
sharingScreen$: Behavior<boolean>;
/** Null when the platform cannot share a screen. */
toggleScreenSharing: (() => void) | null;
screenShareError$: Behavior<Error | null>;
dismissScreenShareError(): void;
/**
* Publishes a source and resolves with its track once it is in `tracks$`.
* Before the transport is connected the request is remembered and applied
* once it is. Rejects where the device could not be used, including the
* user closing the picker. One publication per source: publishing a source
* again unmutes it.
*/
publish(
request: PublishRequest,
): Promise<LocalAudioMediaTrack | LocalVideoMediaTrack>;
/** Removes one of our tracks; a screen share takes its audio with it. */
unpublish(id: string): Promise<void>;
}
// ---------------------------------------------------------------------------
sdk/src/media/LivekitMediaTrack.ts
+2 -2
View File
@@ -52,7 +52,7 @@ import {
import { MatrixRTCError } from "../errors";
import { LazyBehavior } from "../utils/LazyBehavior";
const sources: Record<Track.Source, MediaSource> = {
export const mediaSources: Record<Track.Source, MediaSource> = {
[Track.Source.Microphone]: "microphone",
[Track.Source.Camera]: "camera",
[Track.Source.ScreenShare]: "screenShare",
@@ -114,7 +114,7 @@ export function createLivekitMediaTrack(
publication.isMuted,
);
const base: MediaTrack = {
source: sources[publication.source],
source: mediaSources[publication.source],
kind: publication.kind === Track.Kind.Audio ? "audio" : "video",
id: publication.trackSid,
muted$,
sdk/src/session/ConnectionFactory.ts
+23 -26
View File
@@ -23,12 +23,14 @@ import { type OpenIDClientParts, type SFUConfig } from "./openIDSFU";
import { type Behavior } from "../reactive/Behavior";
import { Connection, type ConnectionOpts, ConnectionState } from "./Connection";
import { type ObservableScope } from "../reactive/ObservableScope";
import {
type CaptureSettings,
type LocalMediaInputs,
type ResolvedTransport,
} from "../api";
import { type LocalMediaInputs, type ResolvedTransport } from "../api";
import { type MediaQuality } from "../config";
import {
audioCaptureOptions,
type PublishRequestFor,
videoCaptureOptions,
videoPublishOptions,
} from "./publishOptions";
export interface ConnectionFactory {
createConnection(
@@ -93,7 +95,6 @@ export class LivekitConnectionFactory implements ConnectionFactory {
private readonly localMedia: LocalMediaInputs,
private readonly keyProvider: BaseKeyProvider | undefined,
private readonly mediaQuality: MediaQuality | undefined,
private readonly capture: CaptureSettings | undefined,
) {}
public createConnection(
@@ -112,12 +113,7 @@ export class LivekitConnectionFactory implements ConnectionFactory {
scope,
livekitRoomFactory: () =>
new LivekitRoom(
roomOptions(
this.localMedia,
this.keyProvider,
this.mediaQuality,
this.capture,
),
roomOptions(this.localMedia, this.keyProvider, this.mediaQuality),
),
ownMembershipIdentity,
},
@@ -126,36 +122,28 @@ export class LivekitConnectionFactory implements ConnectionFactory {
}
}
/** The initial requests become the room's defaults, so that one permission prompt covers both. */
function roomOptions(
localMedia: LocalMediaInputs,
keyProvider: BaseKeyProvider | undefined,
mediaQuality: MediaQuality | undefined,
capture: CaptureSettings | undefined,
): RoomOptions {
const base = buildLiveKitOptions(mediaQuality);
const camera = capture?.camera;
const microphone = initialRequest(localMedia, "microphone");
const camera = initialRequest(localMedia, "camera");
return {
...base,
videoCaptureDefaults: {
...base.videoCaptureDefaults,
deviceId: localMedia.videoInputDeviceId,
processor: localMedia.videoProcessor,
...(camera?.resolution && { resolution: camera.resolution }),
...(camera && videoCaptureOptions(camera)),
},
publishDefaults: {
...base.publishDefaults,
...(camera?.maxBitrate !== undefined && {
videoEncoding: {
maxBitrate: camera.maxBitrate,
maxFramerate: camera.maxFramerate,
},
}),
...(camera?.codec && { videoCodec: camera.codec }),
...(camera && videoPublishOptions(camera.capture)),
},
audioCaptureDefaults: {
...base.audioCaptureDefaults,
deviceId: localMedia.audioInputDeviceId,
...capture?.audio,
...(microphone && audioCaptureOptions(microphone)),
},
audioOutput: { deviceId: localMedia.audioOutputDeviceId$.value },
// Every room needs a worker of its own: one gets confused by streams from
@@ -163,3 +151,12 @@ function roomOptions(
e2ee: keyProvider && { keyProvider, worker: new E2EEWorker() },
};
}
function initialRequest<S extends "microphone" | "camera">(
localMedia: LocalMediaInputs,
source: S,
): PublishRequestFor<S> | undefined {
return localMedia.publish.find(
(request): request is PublishRequestFor<S> => request.source === source,
);
}
sdk/src/session/LocalMember.ts
+56 -121
View File
@@ -8,11 +8,8 @@ Please see LICENSE in the repository root for full details.
import { observeParticipantEvents } from "@livekit/components-core";
import {
type LocalParticipant,
type Participant,
type LocalTrackPublication,
ParticipantEvent,
type ScreenShareCaptureOptions,
Track,
type TrackPublishOptions,
} from "livekit-client";
import { type MatrixClient } from "matrix-js-sdk";
import { type Logger } from "matrix-js-sdk/lib/logger";
@@ -29,6 +26,8 @@ import {
combineLatest,
concat,
distinctUntilChanged,
filter,
firstValueFrom,
map,
NEVER,
type Observable,
@@ -49,7 +48,7 @@ import {
import {
type DisconnectReason,
type MediaSource,
type VideoCaptureSettings,
type PublishRequest,
} from "../api";
import {
FailToStartLivekitConnection,
@@ -105,7 +104,6 @@ interface Props {
matrixRTCMode: MatrixRTCMode;
timings: SessionTimings;
desired: DesiredMedia;
screenShare?: VideoCaptureSettings;
logger: Logger;
}
@@ -121,14 +119,11 @@ export interface LocalMembership {
/** Connected once, and currently not. */
reconnecting$: Behavior<boolean>;
disconnectReason$: Behavior<DisconnectReason | null>;
setMicrophoneEnabled: (enabled: boolean) => Promise<boolean>;
setCameraEnabled: (enabled: boolean) => Promise<boolean>;
/** Resolves with the publication, once a publisher has made it. */
publish: (request: PublishRequest) => Promise<LocalTrackPublication>;
unpublish: (source: MediaSource) => Promise<void>;
/** Mutes or unmutes one of our publications; the result is the state that resulted. */
setEnabled: (source: MediaSource, enabled: boolean) => Promise<boolean>;
sharingScreen$: Behavior<boolean>;
toggleScreenSharing: (() => void) | null;
screenShareError$: Behavior<Error | null>;
dismissScreenShareError: () => void;
}
/**
@@ -152,7 +147,6 @@ export function createLocalMembership$({
matrixRTCMode,
timings,
desired,
screenShare,
logger: parentLogger,
}: Props): LocalMembership {
const logger = parentLogger.getChild("[LocalMember]");
@@ -222,7 +216,7 @@ export function createLocalMembership$({
if (publisher === null) return;
try {
if (shouldPublish) {
publisher.createAndSetupTracks();
publisher.start();
await publisher.startPublishing();
} else if (publisher.shouldPublish) await publisher.stopPublishing();
} catch (e) {
@@ -398,74 +392,62 @@ export function createLocalMembership$({
}
});
const publish = async (
request: PublishRequest,
): Promise<LocalTrackPublication> => {
desired.set(request.source, { request, enabled: true });
const publisher = publisher$.value;
try {
// Before the start, the publisher publishes everything desired itself
if (publisher?.started) {
const publication = await publisher.publish(request);
if (publication) return publication;
}
return await firstValueFrom(publication$(request.source));
} catch (e) {
desired.delete(request.source);
throw toMatrixRTCError(e);
}
};
const publication$ = (
source: MediaSource,
): Observable<LocalTrackPublication> =>
participant$.pipe(
switchMap((participant) =>
participant === null
? NEVER
: observeParticipantEvents(
participant,
ParticipantEvent.LocalTrackPublished,
).pipe(
map(() =>
participant.getTrackPublication(livekitSources[source]),
),
),
),
filter((publication) => publication !== undefined),
);
const unpublish = async (source: MediaSource): Promise<void> => {
desired.delete(source);
await publisher$.value?.unpublish(source);
};
const setEnabled = async (
source: MediaSource,
enabled: boolean,
): Promise<boolean> => {
if (source !== "microphone" && source !== "camera")
return setPublicationEnabled(source, enabled);
const desired$ =
source === "microphone" ? desired.microphone$ : desired.camera$;
desired$.next(enabled);
const wanted = desired.get(source);
if (wanted) wanted.enabled = enabled;
// Without a publisher the request waits for the tracks to be created
const publisher = publisher$.value;
if (publisher === null) return enabled;
const result = await publisher.setEnabled(
source === "microphone" ? Track.Source.Microphone : Track.Source.Camera,
enabled,
);
if (result !== enabled) desired$.next(result);
const result = await publisher.setEnabled(source, enabled);
if (wanted && result !== enabled) wanted.enabled = result;
return result;
};
// A publication the publisher does not recreate on reconnection, so there
// is nothing to remember for it
const setPublicationEnabled = async (
source: MediaSource,
enabled: boolean,
): Promise<boolean> => {
const publication = participant$.value?.getTrackPublication(
livekitSources[source],
);
if (!publication) return false;
if (enabled) await publication.unmute();
else await publication.mute();
return !publication.isMuted;
};
const sharingScreen$ = scope.behavior(
participant$.pipe(
switchMap((p) => (p === null ? of(false) : observeSharingScreen$(p))),
),
);
const screenShareError$ = new BehaviorSubject<Error | null>(null);
const toggleScreenSharing =
"getDisplayMedia" in (navigator.mediaDevices ?? {})
? (): void => {
const participant = participant$.value;
if (participant === null) return;
const enable = !sharingScreen$.value;
participant
.setScreenShareEnabled(
enable,
screenShareCaptureOptions(screenShare),
screenSharePublishOptions(screenShare),
)
.catch((e: unknown) => {
logger.error(
`Screen share ${enable ? "start" : "stop"} failed`,
e,
);
// The user closing the picker is not an error worth showing
if (e instanceof DOMException && e.name === "NotAllowedError")
return;
screenShareError$.next(
e instanceof Error ? e : new Error(String(e)),
);
});
}
: null;
return {
requestJoinAndPublish: () => joinRequested$.next(true),
requestDisconnect: () => joinRequested$.next(false),
@@ -476,13 +458,9 @@ export function createLocalMembership$({
connected$,
reconnecting$,
disconnectReason$,
setMicrophoneEnabled: async (enabled) => setEnabled("microphone", enabled),
setCameraEnabled: async (enabled) => setEnabled("camera", enabled),
publish,
unpublish,
setEnabled,
sharingScreen$,
toggleScreenSharing,
screenShareError$,
dismissScreenShareError: () => screenShareError$.next(null),
};
}
@@ -511,46 +489,3 @@ async function checkDelegationSupport(
return false;
}
}
function screenShareCaptureOptions(
settings: VideoCaptureSettings | undefined,
): ScreenShareCaptureOptions {
return {
// No echo cancellation: it would cancel the other members' voices out of
// the shared audio
audio: {
autoGainControl: false,
noiseSuppression: false,
voiceIsolation: false,
},
selfBrowserSurface: "include",
surfaceSwitching: "include",
systemAudio: "include",
...(settings?.resolution && { resolution: settings.resolution }),
};
}
function screenSharePublishOptions(
settings: VideoCaptureSettings | undefined,
): TrackPublishOptions | undefined {
if (settings === undefined) return undefined;
return {
...(settings.maxBitrate !== undefined && {
screenShareEncoding: {
maxBitrate: settings.maxBitrate,
maxFramerate: settings.maxFramerate,
},
}),
...(settings.codec && { videoCodec: settings.codec }),
};
}
function observeSharingScreen$(participant: Participant): Observable<boolean> {
return observeParticipantEvents(
participant,
ParticipantEvent.TrackPublished,
ParticipantEvent.TrackUnpublished,
ParticipantEvent.LocalTrackPublished,
ParticipantEvent.LocalTrackUnpublished,
).pipe(map((p) => p.isScreenShareEnabled));
}
sdk/src/session/MatrixRTCClient.ts
+7 -16
View File
@@ -15,14 +15,7 @@ import {
import { type IMembershipManager } from "matrix-js-sdk/lib/matrixrtc/IMembershipManager";
import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
import { v4 as uuidv4 } from "uuid";
import {
BehaviorSubject,
combineLatest,
from,
fromEvent,
map,
type Observable,
} from "rxjs";
import { combineLatest, from, fromEvent, map, type Observable } from "rxjs";
import { defaultSessionTimings, MatrixRTCMode } from "../config";
import { type ObservableScope } from "../reactive/ObservableScope";
@@ -123,10 +116,12 @@ export function createMatrixRTCClient(
}),
);
const desired: DesiredMedia = {
microphone$: new BehaviorSubject(localMedia.microphoneEnabled),
camera$: new BehaviorSubject(localMedia.cameraEnabled),
};
const desired: DesiredMedia = new Map(
localMedia.publish.map((request) => [
request.source,
{ request, enabled: true },
]),
);
const connectionManager = createConnectionManager$({
scope,
@@ -136,7 +131,6 @@ export function createMatrixRTCClient(
localMedia,
keyProvider,
options.mediaQuality,
options.capture,
),
localTransport$,
remoteTransports$: transports$,
@@ -194,7 +188,6 @@ export function createMatrixRTCClient(
matrixRTCMode,
timings,
desired,
screenShare: options.capture?.screenShare,
logger,
});
@@ -276,8 +269,6 @@ export function createMatrixRTCClient(
connected$: localMembership.connected$,
reconnecting$: localMembership.reconnecting$,
disconnectReason$: localMembership.disconnectReason$,
setMicrophoneEnabled: localMembership.setMicrophoneEnabled,
setCameraEnabled: localMembership.setCameraEnabled,
fatalError$: scope.behavior(localMembership.state$.pipe(map(fatalError))),
localMember$,
remoteMembers$,
sdk/src/session/Members.ts
+39 -17
View File
@@ -11,7 +11,15 @@ import {
type Room as LivekitRoom,
} from "livekit-client";
import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc";
import { combineLatest, distinctUntilChanged, map } from "rxjs";
import {
combineLatest,
distinctUntilChanged,
filter,
firstValueFrom,
map,
of,
switchMap,
} from "rxjs";
import { type Behavior } from "../reactive/Behavior";
import { type ObservableScope } from "../reactive/ObservableScope";
@@ -80,26 +88,40 @@ export function createLocalRTCMember(
localMembership: LocalMembership,
context: MemberContext,
): LocalRTCMember {
const media$ = mediaFor(
scope,
localMembership.participant$,
localMembership.connection$,
(mediaScope, participant, room) =>
createLocalLivekitMemberMedia(
mediaScope,
participant,
room,
context.encryptionSystem,
localMembership.setEnabled,
),
);
return {
...createRTCMember(scope, membership$, context),
local: true,
media$: mediaFor(
scope,
localMembership.participant$,
localMembership.connection$,
(mediaScope, participant, room) =>
createLocalLivekitMemberMedia(
mediaScope,
participant,
room,
context.encryptionSystem,
localMembership.setEnabled,
media$,
publish: async (request) => {
const { trackSid } = await localMembership.publish(request);
return firstValueFrom(
media$.pipe(
switchMap((media) => media?.tracks$ ?? of([])),
map((tracks) => tracks.find((track) => track.id === trackSid)),
filter((track) => track !== undefined),
),
),
sharingScreen$: localMembership.sharingScreen$,
toggleScreenSharing: localMembership.toggleScreenSharing,
screenShareError$: localMembership.screenShareError$,
dismissScreenShareError: localMembership.dismissScreenShareError,
);
},
unpublish: async (id) => {
const track = media$.value?.tracks$.value.find((t) => t.id === id);
if (track === undefined) return;
await localMembership.unpublish(
track.source === "screenShareAudio" ? "screenShare" : track.source,
);
},
};
}
sdk/src/session/Publisher.ts
+110 -73
View File
@@ -10,20 +10,33 @@ import {
type LocalTrackPublication,
ParticipantEvent,
type Room as LivekitRoom,
Track,
type Track,
} from "livekit-client";
import { type Logger } from "matrix-js-sdk/lib/logger";
import { type BehaviorSubject } from "rxjs";
import { ObservableScope } from "../reactive/ObservableScope";
import { type LocalMediaInputs } from "../api";
import {
type LocalMediaInputs,
type MediaSource,
type PublishRequest,
} from "../api";
import { livekitSources, mediaSources } from "../media/LivekitMediaTrack";
import {
audioCaptureOptions,
screenShareCaptureOptions,
screenSharePublishOptions,
videoCaptureOptions,
videoPublishOptions,
} from "./publishOptions";
/** What the host last asked for, kept across publishers. */
export interface DesiredMedia {
microphone$: BehaviorSubject<boolean>;
camera$: BehaviorSubject<boolean>;
/** What the host asked to publish, kept across publishers for a reconnection. */
export interface DesiredPublication {
request: PublishRequest;
enabled: boolean;
}
export type DesiredMedia = Map<MediaSource, DesiredPublication>;
/**
* Publishes the local media on one LiveKit room, following the host's
* requests.
@@ -35,7 +48,8 @@ export interface DesiredMedia {
*/
export class Publisher {
public shouldPublish = false;
private tracksRequested = false;
/** Whether the initial requests have been published; `publish` is immediate from then on. */
public started = false;
private readonly scope = new ObservableScope();
private readonly room: LivekitRoom;
@@ -64,53 +78,106 @@ export class Publisher {
this.onLocalTrackPublished,
);
try {
await this.stopTracks();
for (const { track } of this.publications())
if (track) await this.room.localParticipant.unpublishTrack(track, true);
} catch (e) {
this.logger.error("Failed to stop the local tracks", e);
}
}
/**
* Creates the microphone and camera tracks the host asked for. Both are
* enabled in one call so that the browser asks for permission once. Safe to
* call more than once.
*/
public createAndSetupTracks(): void {
if (this.tracksRequested) return;
this.tracksRequested = true;
/** Publishes what the host asked for, less what it has muted since. Safe to call more than once. */
public start(): void {
if (this.started) return;
this.started = true;
const requests = [...this.desired.values()]
.filter(({ enabled }) => enabled)
.map(({ request }) => request);
const sources = new Set(requests.map((request) => request.source));
// Microphone and camera in one call so that the browser asks for
// permission once; the room defaults carry their options. LiveKit
// resolves these once the track is published, which may block on the
// connection; LocalTrackPublished is what tells us a track exists.
if (sources.has("microphone") && sources.has("camera"))
void this.room.localParticipant.enableCameraAndMicrophone();
for (const request of requests)
if (
request.source === "screenShare" ||
!(sources.has("microphone") && sources.has("camera"))
)
this.publish(request).catch((e) => {
this.logger.error(`Failed to publish the ${request.source}`, e);
});
}
/** Publishes one request and resolves with its publication once LiveKit has it. */
public async publish(
request: PublishRequest,
): Promise<LocalTrackPublication | undefined> {
const participant = this.room.localParticipant;
const audio = this.desired.microphone$.value;
const video = this.desired.camera$.value;
// LiveKit resolves these once the track is published, which may block on
// the connection; LocalTrackPublished is what tells us a track exists.
if (audio && video) void participant.enableCameraAndMicrophone();
else if (audio) void participant.setMicrophoneEnabled(true);
else if (video) void participant.setCameraEnabled(true);
let publication: LocalTrackPublication | undefined;
if (request.source === "microphone")
publication = await participant.setMicrophoneEnabled(
true,
audioCaptureOptions(request),
);
else if (request.source === "camera")
publication = await participant.setCameraEnabled(
true,
videoCaptureOptions(request),
videoPublishOptions(request.capture),
);
else
publication = await participant.setScreenShareEnabled(
true,
screenShareCaptureOptions(request),
screenSharePublishOptions(request.capture),
);
// Until the member has joined, the upstream has to stay paused
if (!this.shouldPublish) await this.pauseUpstreams();
return publication;
}
public async unpublish(source: MediaSource): Promise<void> {
const participant = this.room.localParticipant;
// Takes the screen share audio with it
if (source === "screenShare") {
await participant.setScreenShareEnabled(false);
return;
}
const track = participant.getTrackPublication(
livekitSources[source],
)?.track;
if (track) await participant.unpublishTrack(track, true);
}
/**
* Enables or disables a source and reports what resulted: the request where
* Mutes or unmutes a source and reports what resulted: the request where
* it succeeded, the state LiveKit is left in where it did not (a denied
* permission, a missing device).
*/
public async setEnabled(
source: Track.Source.Microphone | Track.Source.Camera,
source: MediaSource,
enabled: boolean,
): Promise<boolean> {
const participant = this.room.localParticipant;
const livekitSource = livekitSources[source];
try {
if (source === Track.Source.Microphone)
if (source === "microphone")
await participant.setMicrophoneEnabled(enabled);
else await participant.setCameraEnabled(enabled);
else if (source === "camera") await participant.setCameraEnabled(enabled);
else {
const publication = participant.getTrackPublication(livekitSource);
if (!publication) return false;
if (enabled) await publication.unmute();
else await publication.mute();
}
// Unmuting restarts the upstream; until the member has joined, it has
// to stay paused
if (enabled && !this.shouldPublish) await this.pauseUpstreams([source]);
if (enabled && !this.shouldPublish) await this.pauseUpstreams();
return enabled;
} catch (e) {
this.logger.error(`Failed to set ${source} enabled=${enabled}`, e);
return source === Track.Source.Microphone
? participant.isMicrophoneEnabled
: participant.isCameraEnabled;
return !(participant.getTrackPublication(livekitSource)?.isMuted ?? true);
}
}
@@ -119,64 +186,34 @@ export class Publisher {
this.shouldPublish = true;
// Enabling a track does not resume an upstream that was paused while it
// was already enabled, so it is done explicitly
await this.resumeUpstreams([Track.Source.Microphone, Track.Source.Camera]);
for (const { track } of this.publications())
if (track?.isUpstreamPaused) await track.resumeUpstream();
}
public async stopPublishing(): Promise<void> {
this.shouldPublish = false;
await this.pauseUpstreams([
Track.Source.Microphone,
Track.Source.Camera,
Track.Source.ScreenShare,
]);
}
private async stopTracks(): Promise<void> {
const participant = this.room.localParticipant;
for (const source of [
Track.Source.Microphone,
Track.Source.Camera,
Track.Source.ScreenShare,
]) {
const track = participant.getTrackPublication(source)?.track;
if (track) await participant.unpublishTrack(track, true);
}
await this.pauseUpstreams();
}
private onLocalTrackPublished(publication: LocalTrackPublication): void {
this.logger.info(`Local ${publication.source} track published`);
if (!this.shouldPublish)
this.pauseUpstreams([publication.source]).catch((e) => {
this.pauseUpstreams().catch((e) => {
this.logger.error("Failed to pause the upstream", e);
});
// The host may have changed its mind while the track was being created
const desired =
publication.source === Track.Source.Microphone
? this.desired.microphone$.value
: publication.source === Track.Source.Camera
? this.desired.camera$.value
: undefined;
if (desired === false)
void this.setEnabled(
publication.source as Track.Source.Microphone | Track.Source.Camera,
false,
);
const source = mediaSources[publication.source as Track.Source];
if (this.desired.get(source)?.enabled === false)
void this.setEnabled(source, false);
}
private async pauseUpstreams(sources: Track.Source[]): Promise<void> {
for (const source of sources) {
const track =
this.room.localParticipant.getTrackPublication(source)?.track;
private async pauseUpstreams(): Promise<void> {
for (const { track } of this.publications())
if (track && !track.isUpstreamPaused) await track.pauseUpstream();
}
}
private async resumeUpstreams(sources: Track.Source[]): Promise<void> {
for (const source of sources) {
const track =
this.room.localParticipant.getTrackPublication(source)?.track;
if (track?.isUpstreamPaused) await track.resumeUpstream();
}
private publications(): LocalTrackPublication[] {
return [...this.room.localParticipant.trackPublications.values()];
}
private followAudioOutput(): void {
sdk/src/session/publishOptions.ts
+91
View File
@@ -0,0 +1,91 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
type AudioCaptureOptions,
type ScreenShareCaptureOptions,
type TrackPublishOptions,
type VideoCaptureOptions,
} from "livekit-client";
import { type PublishRequest, type VideoCaptureSettings } from "../api";
export type PublishRequestFor<S extends PublishRequest["source"]> = Extract<
PublishRequest,
{ source: S }
>;
/** LiveKit fills in the room defaults for whatever a request leaves out. */
export function audioCaptureOptions({
deviceId,
capture,
}: PublishRequestFor<"microphone">): AudioCaptureOptions | undefined {
if (deviceId === undefined && capture === undefined) return undefined;
return { ...(deviceId && { deviceId }), ...capture };
}
export function videoCaptureOptions({
deviceId,
processor,
capture,
}: PublishRequestFor<"camera">): VideoCaptureOptions | undefined {
if (!deviceId && !processor && !capture?.resolution) return undefined;
return {
...(deviceId && { deviceId }),
...(processor && { processor }),
...(capture?.resolution && { resolution: capture.resolution }),
};
}
export function videoPublishOptions(
settings: VideoCaptureSettings | undefined,
): TrackPublishOptions | undefined {
if (settings === undefined) return undefined;
return {
...(settings.maxBitrate !== undefined && {
videoEncoding: {
maxBitrate: settings.maxBitrate,
maxFramerate: settings.maxFramerate,
},
}),
...(settings.codec && { videoCodec: settings.codec }),
};
}
export function screenShareCaptureOptions({
audio = true,
capture,
}: PublishRequestFor<"screenShare">): ScreenShareCaptureOptions {
return {
// No echo cancellation: it would cancel the other members' voices out of
// the shared audio
audio: audio && {
autoGainControl: false,
noiseSuppression: false,
voiceIsolation: false,
},
selfBrowserSurface: "include",
surfaceSwitching: "include",
systemAudio: "include",
...(capture?.resolution && { resolution: capture.resolution }),
};
}
export function screenSharePublishOptions(
settings: VideoCaptureSettings | undefined,
): TrackPublishOptions | undefined {
if (settings === undefined) return undefined;
return {
...(settings.maxBitrate !== undefined && {
screenShareEncoding: {
maxBitrate: settings.maxBitrate,
maxFramerate: settings.maxFramerate,
},
}),
...(settings.codec && { videoCodec: settings.codec }),
};
}