flatten media api (no MemberMedia wrapper object

This commit is contained in:
Timo K.
2026-10-08 13:53:57 +02:00
parent 6ffab92f0f
commit 4790af08a4
13 changed files with 214 additions and 232 deletions
+11 -6
View File
@@ -20,11 +20,12 @@ import { type Observable } from "rxjs";
import { type Behavior } from "./reactive/Behavior";
import {
type AudioMediaTrack,
type EncryptionError,
type LocalAudioMediaTrack,
type LocalMemberMedia,
type LocalVideoMediaTrack,
type MemberMedia,
type PublishRequest,
type VideoMediaTrack,
} from "./media-api";
import { type MediaBackendFactory } from "./media-backend/api";
import { type EncryptionSystem } from "./encryption";
@@ -219,10 +220,14 @@ export interface RTCMember {
/** Which transport this member is on; undefined when the membership has none. */
transport$: Behavior<TransportMetadata | undefined>;
/**
* Null while the member has a transport but no media has arrived on it yet
* ("waiting for media").
* The member's tracks, in publication order, once it has shown up on its
* transport; null until then ("waiting for media"). An entry stays the same
* object for as long as the same publication is behind it. Which track is
* which is in its `source`; `trackBySource$` picks one out.
*/
media$: Behavior<MemberMedia | null>;
tracks$: Behavior<(AudioMediaTrack | VideoMediaTrack)[] | null>;
/** Emits when the SFU reports a key problem for this member. */
encryptionError$: Observable<EncryptionError>;
}
export interface RemoteRTCMember extends RTCMember {
@@ -231,7 +236,7 @@ export interface RemoteRTCMember extends RTCMember {
export interface LocalRTCMember extends RTCMember {
local: true;
media$: Behavior<LocalMemberMedia | null>;
tracks$: Behavior<(LocalAudioMediaTrack | LocalVideoMediaTrack)[] | null>;
/**
* Publishes a source and resolves with its track once it is in `tracks$`.
* Before the transport is connected the request is remembered and applied
+9 -2
View File
@@ -7,7 +7,13 @@ Please see LICENSE in the repository root for full details.
import { logger } from "matrix-js-sdk/lib/logger";
import { Status, type Transport } from "matrix-js-sdk/lib/matrixrtc";
import { BehaviorSubject, type Observable, Subject, throwError } from "rxjs";
import {
BehaviorSubject,
EMPTY,
type Observable,
Subject,
throwError,
} from "rxjs";
import { describe, expect, it, vi } from "vitest";
import { defaultSessionTimings } from "../config";
@@ -192,7 +198,8 @@ function fakeLocalMediaBackend(): FakeLocalMediaBackend {
connectionState$: new BehaviorSubject<MediaConnectionState | Error>(
MediaConnectionState.Initialized,
),
media$: new BehaviorSubject(null),
tracks$: new BehaviorSubject(null),
encryptionError$: EMPTY,
setPublishing: vi.fn<LocalMediaBackend["setPublishing"]>(),
publishError$: new BehaviorSubject<Error | null>(null),
publish: vi.fn<LocalMediaBackend["publish"]>(),
+14 -7
View File
@@ -16,8 +16,10 @@ import {
type RemoteRTCMember,
type RTCMember,
} from "../api";
import { type MemberMedia } from "../media-api";
import { type LocalMediaBackend } from "../media-backend/api";
import {
type LocalMediaBackend,
type MemberMediaFields,
} from "../media-backend/api";
import { type TransportRegistry } from "./Transports";
/** What every member is built from, besides its own membership and media. */
@@ -41,26 +43,31 @@ export function membershipKeys(
export function createRemoteRTCMember(
scope: ObservableScope,
membership$: Behavior<CallMembership>,
media$: Behavior<MemberMedia | null>,
media: MemberMediaFields,
context: MemberContext,
): RemoteRTCMember {
return {
...createRTCMember(scope, membership$, context),
local: false,
media$,
tracks$: media.tracks$,
encryptionError$: media.encryptionError$,
};
}
export function createLocalRTCMember(
scope: ObservableScope,
membership$: Behavior<CallMembership>,
local: Pick<LocalMediaBackend, "media$" | "publish" | "unpublish">,
local: Pick<
LocalMediaBackend,
"tracks$" | "encryptionError$" | "publish" | "unpublish"
>,
context: MemberContext,
): LocalRTCMember {
return {
...createRTCMember(scope, membership$, context),
local: true,
media$: local.media$,
tracks$: local.tracks$,
encryptionError$: local.encryptionError$,
publish: async (request) => local.publish(request),
unpublish: async (id) => local.unpublish(id),
};
@@ -70,7 +77,7 @@ function createRTCMember(
scope: ObservableScope,
membership$: Behavior<CallMembership>,
{ metadata, transports }: MemberContext,
): Omit<RTCMember, "local" | "media$"> {
): Omit<RTCMember, "local" | "tracks$" | "encryptionError$"> {
const { userId, deviceId, rtcBackendIdentity } = membership$.value;
return {
id: rtcBackendIdentity,
-24
View File
@@ -16,8 +16,6 @@ import {
type TrackProcessor,
type VideoCodec,
} from "livekit-client";
import { type Observable } from "rxjs";
import { type Behavior } from "./reactive/Behavior";
export interface AudioCaptureSettings {
@@ -137,25 +135,3 @@ export interface LocalVideoMediaTrack extends VideoMediaTrack, LocalMediaTrack {
}
export type EncryptionError = "MissingKey" | "InvalidKey";
/**
* The media of one member, backed by a LiveKit participant inside the SDK.
* There is no identity field: the LiveKit identity is the member's `id`.
*/
export interface MemberMedia {
local: boolean;
/**
* One entry per published track, in publication order. An entry stays the
* same object for as long as the same publication is behind it. Which
* track is which is in its `source`; `trackBySource$` picks one out.
*/
tracks$: Behavior<(AudioMediaTrack | VideoMediaTrack)[]>;
/** Emits when the SFU reports a key problem for this member. */
encryptionError$: Observable<EncryptionError>;
}
export interface LocalMemberMedia extends MemberMedia {
local: true;
tracks$: Behavior<(LocalAudioMediaTrack | LocalVideoMediaTrack)[]>;
}
+24 -15
View File
@@ -17,12 +17,15 @@ import { type Transport } from "matrix-js-sdk/lib/matrixrtc";
import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
import { type Observable } from "rxjs";
import { type ResolvedTransport, type RTCMembership } from "../api";
import {
type LocalRTCMember,
type ResolvedTransport,
type RTCMember,
type RTCMembership,
} from "../api";
import {
type LocalAudioMediaTrack,
type LocalMemberMedia,
type LocalVideoMediaTrack,
type MemberMedia,
type PublishRequest,
} from "../media-api";
import { type SessionTimings } from "../config";
@@ -50,6 +53,13 @@ export interface MediaKey {
key: Uint8Array<ArrayBuffer>;
}
/** The fields of a member the backend supplies, as the member exposes them. */
export type MemberMediaFields = Pick<RTCMember, "tracks$" | "encryptionError$">;
export type LocalMemberMediaFields = Pick<
LocalRTCMember,
"tracks$" | "encryptionError$"
>;
export interface MediaBackendContext {
roomId: string;
ownMembershipIdentity: CallMembershipIdentityParts;
@@ -132,16 +142,17 @@ export interface MediaBackend {
readonly local: LocalMediaBackend;
/**
* The media of one remote member, null while nothing has arrived for it on
* its transport. Built in `scope`, which the caller ends with the member.
* This is also how the backend learns which remote transports exist: it
* follows the membership's transport for as long as `scope` lives, and
* connects to every transport some member is on.
* The media of one remote member: its tracks, null while nothing has
* arrived for it on its transport, and its key errors. Built in `scope`,
* which the caller ends with the member. This is also how the backend
* learns which remote transports exist: it follows the membership's
* transport for as long as `scope` lives, and connects to every transport
* some member is on.
*/
mediaFor$(
scope: ObservableScope,
membership$: Behavior<RTCMembership>,
): Behavior<MemberMedia | null>;
): MemberMediaFields;
/** Every connection the backend holds. For debugging and devtool purposes.*/
readonly connections$: Behavior<BackendConnection[]>;
@@ -158,16 +169,14 @@ export interface MediaBackend {
readonly data$: Observable<DataPacket>;
}
export interface LocalMediaBackend {
/** `tracks$` is null until the connection carries a local participant. */
export interface LocalMediaBackend extends LocalMemberMediaFields {
/**
* State of the connection the local member publishes on. `Initialized`
* while there is none; an `Error` is the connection's failure.
*/
readonly connectionState$: Behavior<MediaConnectionState | Error>;
/** The local member's own media; null until the connection carries a local participant. */
readonly media$: Behavior<LocalMemberMedia | null>;
/**
* Whether the local tracks reach the transport. The session sets it to
* "joined and the homeserver is reachable". Remembered across connections:
@@ -179,8 +188,8 @@ export interface LocalMediaBackend {
readonly publishError$: Behavior<Error | null>;
/**
* Publishes a source and resolves with its track once it is in `media$`'s
* `tracks$`. Remembered across connections, so a request before the
* Publishes a source and resolves with its track once it is in `tracks$`.
* Remembered across connections, so a request before the
* transport is up is applied once it is, and republished after a
* reconnection. Rejects where the device could not be used.
*/
@@ -24,7 +24,7 @@ import {
} from "rxjs";
import { type ResolvedTransport, type RTCMembership } from "../../api";
import { type MemberMedia } from "../../media-api";
import { type MediaQuality } from "../../config";
import { MatrixRTCError } from "../../errors";
import { type Behavior } from "../../reactive/Behavior";
@@ -33,11 +33,11 @@ import {
type ObservableScope,
trackEpoch,
} from "../../reactive/ObservableScope";
import { mapScoped } from "../../utils/mapScoped";
import {
type BackendConnection,
type MediaBackend,
type MediaBackendContext,
type MemberMediaFields,
type TransportCapabilities,
} from "../api";
import { type Connection } from "./Connection";
@@ -150,7 +150,7 @@ export function createLivekitBackend(
const mediaFor$ = (
memberScope: ObservableScope,
membership$: Behavior<RTCMembership>,
): Behavior<MemberMedia | null> => {
): MemberMediaFields => {
const identity = membership$.value.rtcBackendIdentity;
const transport$ = memberScope.behavior(
membership$.pipe(
@@ -190,16 +190,7 @@ export function createLivekitBackend(
`${identity}: LiveKit participant ${source ? `matched (${source.participant.sid})` : "missing"}`,
);
});
return memberScope.behavior(
mapScoped(memberScope, source$, (mediaScope, { participant, room }) =>
createLivekitMemberMedia(
mediaScope,
participant,
room,
encryptionSystem,
),
).pipe(map((media) => media ?? null)),
);
return createLivekitMemberMedia(memberScope, source$, encryptionSystem);
};
const connections$ = scope.behavior<BackendConnection[]>(
@@ -16,29 +16,32 @@ import {
import { describe, expect, it } from "vitest";
import { E2eeType } from "../../encryption";
import { constant } from "../../reactive/Behavior";
import { testScope } from "../../utils/test";
import { createLivekitMemberMedia } from "./LivekitMemberMedia";
describe("createLivekitMemberMedia", () => {
it("lists one stable track per publication, in publication order", () => {
const { participant, publications, room, emitter } = fakes();
const media = createLivekitMemberMedia(testScope(), participant, room, {
kind: E2eeType.NONE,
});
const media = createLivekitMemberMedia(
testScope(),
constant({ participant, room }),
{ kind: E2eeType.NONE },
);
expect(media.tracks$.value).toEqual([]);
const microphone = publish(publications, "TR_1", Track.Source.Microphone);
const camera = publish(publications, "TR_2", Track.Source.Camera);
emitter.emit(ParticipantEvent.TrackPublished, camera);
const [first, second] = media.tracks$.value;
const [first, second] = media.tracks$.value!;
expect(first.source).toBe("microphone");
expect(second.source).toBe("camera");
// An unrelated media event keeps the same objects
microphone.isMuted = true;
emitter.emit(ParticipantEvent.TrackMuted, microphone);
expect(media.tracks$.value[0]).toBe(first);
expect(media.tracks$.value[1]).toBe(second);
expect(media.tracks$.value![0]).toBe(first);
expect(media.tracks$.value![1]).toBe(second);
publications.delete("TR_1");
emitter.emit(ParticipantEvent.TrackUnpublished, microphone);
@@ -47,14 +50,16 @@ describe("createLivekitMemberMedia", () => {
it("includes a track published without a source", () => {
const { participant, publications, room, emitter } = fakes();
const media = createLivekitMemberMedia(testScope(), participant, room, {
kind: E2eeType.NONE,
});
const media = createLivekitMemberMedia(
testScope(),
constant({ participant, room }),
{ kind: E2eeType.NONE },
);
emitter.emit(
ParticipantEvent.TrackPublished,
publish(publications, "TR_1", Track.Source.Unknown),
);
expect(media.tracks$.value.map((t) => t.source)).toEqual(["unknown"]);
expect(media.tracks$.value!.map((t) => t.source)).toEqual(["unknown"]);
});
});
@@ -16,120 +16,130 @@ import {
RoomEvent,
type TrackPublication,
} from "livekit-client";
import { filter, map, type Observable } from "rxjs";
import { EMPTY, filter, map, type Observable, of, switchMap } from "rxjs";
import { type ObservableScope } from "../../reactive/ObservableScope";
import { generateItems } from "../../reactive/observable";
import { E2eeType } from "../../encryption";
import { type EncryptionSystem } from "../../encryption";
import { E2eeType, type EncryptionSystem } from "../../encryption";
import { type Behavior } from "../../reactive/Behavior";
import {
type EncryptionError,
type LocalMemberMedia,
type MediaSource,
type MemberMedia,
} from "../../media-api";
import { mapScoped } from "../../utils/mapScoped";
import { type EncryptionError, type MediaSource } from "../../media-api";
import { type LocalMemberMediaFields, type MemberMediaFields } from "../api";
import {
createLivekitMediaTrack,
createLocalLivekitMediaTrack,
} from "./LivekitMediaTrack";
/** Where a member's media comes from: its participant on a room, null while it has none. */
export type ParticipantSource<P extends Participant = Participant> = {
participant: P;
room: LivekitRoom;
} | null;
/**
* A participant as `MemberMedia`. The only place, with `LivekitMediaTrack`,
* that reads a LiveKit participant; everything above sees tracks and
* behaviors.
* A member's media from its participant. The only place, with
* `LivekitMediaTrack`, that reads a LiveKit participant; everything above
* sees tracks and behaviors.
*/
export function createLivekitMemberMedia(
scope: ObservableScope,
participant: Participant,
room: LivekitRoom,
source$: Behavior<ParticipantSource>,
encryptionSystem: EncryptionSystem,
): MemberMedia {
): MemberMediaFields {
return {
local: false,
tracks$: memberTracks$(scope, participant, (trackScope, publication) =>
createLivekitMediaTrack(trackScope, participant, publication, room),
),
encryptionError$: encryptionErrors$(
tracks$: memberTracks$(
scope,
participant,
room,
encryptionSystem,
source$,
(trackScope, participant, publication, room) =>
createLivekitMediaTrack(trackScope, participant, publication, room),
),
encryptionError$: encryptionErrors$(source$, encryptionSystem),
};
}
export function createLocalLivekitMemberMedia(
scope: ObservableScope,
participant: LocalParticipant,
room: LivekitRoom,
source$: Behavior<ParticipantSource<LocalParticipant>>,
encryptionSystem: EncryptionSystem,
setEnabled: (source: MediaSource, enabled: boolean) => Promise<boolean>,
): LocalMemberMedia {
): LocalMemberMediaFields {
return {
local: true,
tracks$: memberTracks$(scope, participant, (trackScope, publication) =>
createLocalLivekitMediaTrack(
trackScope,
participant,
publication,
room,
setEnabled,
),
),
encryptionError$: encryptionErrors$(
tracks$: memberTracks$(
scope,
participant,
room,
encryptionSystem,
source$,
(trackScope, participant, publication, room) =>
createLocalLivekitMediaTrack(
trackScope,
participant,
publication,
room,
setEnabled,
),
),
encryptionError$: encryptionErrors$(source$, encryptionSystem),
};
}
/** One track per publication, each living as long as its publication is in the participant's map. */
function memberTracks$<T>(
/**
* One track per publication, each living as long as its publication is in
* the participant's map; null while there is no participant.
*/
function memberTracks$<P extends Participant, T>(
scope: ObservableScope,
participant: Participant,
factory: (scope: ObservableScope, publication: TrackPublication) => T,
): Behavior<T[]> {
source$: Behavior<ParticipantSource<P>>,
factory: (
scope: ObservableScope,
participant: P,
publication: TrackPublication,
room: LivekitRoom,
) => T,
): Behavior<T[] | null> {
return scope.behavior(
observeParticipantMedia(participant).pipe(
generateItems(
`${participant.identity} tracks$`,
function* () {
for (const publication of participant.trackPublications.values())
yield { keys: [publication.trackSid], data: publication };
},
(trackScope, publication$) => factory(trackScope, publication$.value),
mapScoped(scope, source$, (participantScope, { participant, room }) =>
participantScope.behavior(
observeParticipantMedia(participant).pipe(
generateItems(
`${participant.identity} tracks$`,
function* () {
for (const publication of participant.trackPublications.values())
yield { keys: [publication.trackSid], data: publication };
},
(trackScope, publication$) =>
factory(trackScope, participant, publication$.value, room),
),
),
),
),
).pipe(switchMap((tracks$) => tracks$ ?? of(null))),
);
}
function encryptionErrors$(
scope: ObservableScope,
participant: Participant,
room: LivekitRoom,
source$: Behavior<ParticipantSource>,
encryptionSystem: EncryptionSystem,
): Observable<EncryptionError> {
return roomEventSelector(room, RoomEvent.EncryptionError).pipe(
map(([error]) => error?.message ?? ""),
// The participant the error is about does not survive the trip from the
// worker, so the identity is matched in the message. A shared key is the
// same for everyone, so its errors are too.
filter(
(message) =>
encryptionSystem.kind === E2eeType.SHARED_KEY ||
message.includes(participant.identity),
return source$.pipe(
switchMap((source) =>
source === null
? EMPTY
: roomEventSelector(source.room, RoomEvent.EncryptionError).pipe(
map(([error]) => error?.message ?? ""),
// The participant the error is about does not survive the trip
// from the worker, so the identity is matched in the message. A
// shared key is the same for everyone, so its errors are too.
filter(
(message) =>
encryptionSystem.kind === E2eeType.SHARED_KEY ||
message.includes(source.participant.identity),
),
map((message): EncryptionError | undefined =>
message.includes("MissingKey")
? "MissingKey"
: message.includes("InvalidKey")
? "InvalidKey"
: undefined,
),
filter((error) => error !== undefined),
),
),
map((message): EncryptionError | undefined =>
message.includes("MissingKey")
? "MissingKey"
: message.includes("InvalidKey")
? "InvalidKey"
: undefined,
),
filter((error) => error !== undefined),
scope.bind(),
);
}
+13 -23
View File
@@ -28,7 +28,6 @@ import {
import {
type LocalAudioMediaTrack,
type LocalMemberMedia,
type LocalVideoMediaTrack,
type MediaSource,
type PublishRequest,
@@ -37,7 +36,6 @@ import { type EncryptionSystem } from "../../encryption";
import { FailToStartLivekitConnection, toMatrixRTCError } from "../../errors";
import { type Behavior } from "../../reactive/Behavior";
import { type ObservableScope } from "../../reactive/ObservableScope";
import { mapScoped } from "../../utils/mapScoped";
import { type LocalMediaBackend, MediaConnectionState } from "../api";
import { type Connection } from "./Connection";
import {
@@ -169,19 +167,11 @@ export function createLivekitLocalMedia({
return result;
};
const media$ = scope.behavior<LocalMemberMedia | null>(
mapScoped(
scope,
participantAndRoom$(scope, participant$, connection$),
(mediaScope, { participant, room }) =>
createLocalLivekitMemberMedia(
mediaScope,
participant,
room,
encryptionSystem,
setEnabled,
),
).pipe(map((media) => media ?? null)),
const { tracks$, encryptionError$ } = createLocalLivekitMemberMedia(
scope,
participantAndRoom$(scope, participant$, connection$),
encryptionSystem,
setEnabled,
);
const publish = async (
@@ -194,7 +184,7 @@ export function createLivekitLocalMedia({
const publication =
(publisher?.started ? await publisher.publish(request) : undefined) ??
(await firstValueFrom(publication$(participant$, request.source)));
return await firstValueFrom(trackWithId$(media$, publication.trackSid));
return await firstValueFrom(trackWithId$(tracks$, publication.trackSid));
} catch (e) {
desired.delete(request.source);
throw toMatrixRTCError(e);
@@ -202,7 +192,7 @@ export function createLivekitLocalMedia({
};
const unpublish = async (id: string): Promise<void> => {
const track = media$.value?.tracks$.value.find((t) => t.id === id);
const track = tracks$.value?.find((t) => t.id === id);
if (track === undefined) return;
// The screen share audio goes with its video
const source =
@@ -219,7 +209,8 @@ export function createLivekitLocalMedia({
return {
connection$,
connectionState$,
media$,
tracks$,
encryptionError$,
setPublishing: (publish) => publishing$.next(publish),
publishError$,
publish,
@@ -228,7 +219,7 @@ export function createLivekitLocalMedia({
};
}
/** The pair a `MemberMedia` is built on, changing only when one of the two does. */
/** The pair the local tracks are built on, changing only when one of the two does. */
function participantAndRoom$(
scope: ObservableScope,
participant$: Behavior<LocalParticipant | null>,
@@ -269,12 +260,11 @@ function publication$(
}
function trackWithId$(
media$: Behavior<LocalMemberMedia | null>,
tracks$: Behavior<(LocalAudioMediaTrack | LocalVideoMediaTrack)[] | null>,
id: string,
): Observable<LocalAudioMediaTrack | LocalVideoMediaTrack> {
return media$.pipe(
switchMap((media) => media?.tracks$ ?? of([])),
map((tracks) => tracks.find((track) => track.id === id)),
return tracks$.pipe(
map((tracks) => tracks?.find((track) => track.id === id)),
filter((track) => track !== undefined),
);
}
+3 -3
View File
@@ -24,17 +24,17 @@ export type TrackOfSource<S extends MediaSource> = S extends
? VideoMediaTrack
: AudioMediaTrack | VideoMediaTrack;
/** The member's first track of a source; undefined while there is none. */
/** The member's first track of a source; undefined while there is none, or no media yet. */
export function trackBySource$<S extends MediaSource>(
scope: ObservableScope,
tracks$: Behavior<MediaTrack[]>,
tracks$: Behavior<MediaTrack[] | null>,
source: S,
): Behavior<TrackOfSource<S> | undefined> {
return scope.behavior(
tracks$.pipe(
map(
(tracks) =>
tracks.find((t) => t.source === source) as
tracks?.find((t) => t.source === source) as
| TrackOfSource<S>
| undefined,
),