Matrix/element-call-Github
1
0
Fork 0
mirror of https://github.com/vector-im/element-call.git synced 2026-10-16 23:45:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

flatten media api (no MemberMedia wrapper object

Browse Source
This commit is contained in:
Timo K.
2026-10-08 13:53:57 +02:00
parent 6ffab92f0f
commit 4790af08a4
13 changed files with 214 additions and 232 deletions
Download Patch File Download Diff File Expand all files Collapse all files
sdk/src/media-backend/api.ts
+24 -15
View File
@@ -17,12 +17,15 @@ import { type Transport } from "matrix-js-sdk/lib/matrixrtc";
import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
import { type Observable } from "rxjs";
import { type ResolvedTransport, type RTCMembership } from "../api";
import {
type LocalRTCMember,
type ResolvedTransport,
type RTCMember,
type RTCMembership,
} from "../api";
import {
type LocalAudioMediaTrack,
type LocalMemberMedia,
type LocalVideoMediaTrack,
type MemberMedia,
type PublishRequest,
} from "../media-api";
import { type SessionTimings } from "../config";
@@ -50,6 +53,13 @@ export interface MediaKey {
key: Uint8Array<ArrayBuffer>;
}
/** The fields of a member the backend supplies, as the member exposes them. */
export type MemberMediaFields = Pick<RTCMember, "tracks$" | "encryptionError$">;
export type LocalMemberMediaFields = Pick<
LocalRTCMember,
"tracks$" | "encryptionError$"
>;
export interface MediaBackendContext {
roomId: string;
ownMembershipIdentity: CallMembershipIdentityParts;
@@ -132,16 +142,17 @@ export interface MediaBackend {
readonly local: LocalMediaBackend;
/**
* The media of one remote member, null while nothing has arrived for it on
* its transport. Built in `scope`, which the caller ends with the member.
* This is also how the backend learns which remote transports exist: it
* follows the membership's transport for as long as `scope` lives, and
* connects to every transport some member is on.
* The media of one remote member: its tracks, null while nothing has
* arrived for it on its transport, and its key errors. Built in `scope`,
* which the caller ends with the member. This is also how the backend
* learns which remote transports exist: it follows the membership's
* transport for as long as `scope` lives, and connects to every transport
* some member is on.
*/
mediaFor$(
scope: ObservableScope,
membership$: Behavior<RTCMembership>,
): Behavior<MemberMedia | null>;
): MemberMediaFields;
/** Every connection the backend holds. For debugging and devtool purposes.*/
readonly connections$: Behavior<BackendConnection[]>;
@@ -158,16 +169,14 @@ export interface MediaBackend {
readonly data$: Observable<DataPacket>;
}
export interface LocalMediaBackend {
/** `tracks$` is null until the connection carries a local participant. */
export interface LocalMediaBackend extends LocalMemberMediaFields {
/**
* State of the connection the local member publishes on. `Initialized`
* while there is none; an `Error` is the connection's failure.
*/
readonly connectionState$: Behavior<MediaConnectionState | Error>;
/** The local member's own media; null until the connection carries a local participant. */
readonly media$: Behavior<LocalMemberMedia | null>;
/**
* Whether the local tracks reach the transport. The session sets it to
* "joined and the homeserver is reachable". Remembered across connections:
@@ -179,8 +188,8 @@ export interface LocalMediaBackend {
readonly publishError$: Behavior<Error | null>;
/**
* Publishes a source and resolves with its track once it is in `media$`'s
* `tracks$`. Remembered across connections, so a request before the
* Publishes a source and resolves with its track once it is in `tracks$`.
* Remembered across connections, so a request before the
* transport is up is applied once it is, and republished after a
* reconnection. Rejects where the device could not be used.
*/
sdk/src/media-backend/livekit/LivekitBackend.ts
+4 -13
View File
@@ -24,7 +24,7 @@ import {
} from "rxjs";
import { type ResolvedTransport, type RTCMembership } from "../../api";
import { type MemberMedia } from "../../media-api";
import { type MediaQuality } from "../../config";
import { MatrixRTCError } from "../../errors";
import { type Behavior } from "../../reactive/Behavior";
@@ -33,11 +33,11 @@ import {
type ObservableScope,
trackEpoch,
} from "../../reactive/ObservableScope";
import { mapScoped } from "../../utils/mapScoped";
import {
type BackendConnection,
type MediaBackend,
type MediaBackendContext,
type MemberMediaFields,
type TransportCapabilities,
} from "../api";
import { type Connection } from "./Connection";
@@ -150,7 +150,7 @@ export function createLivekitBackend(
const mediaFor$ = (
memberScope: ObservableScope,
membership$: Behavior<RTCMembership>,
): Behavior<MemberMedia | null> => {
): MemberMediaFields => {
const identity = membership$.value.rtcBackendIdentity;
const transport$ = memberScope.behavior(
membership$.pipe(
@@ -190,16 +190,7 @@ export function createLivekitBackend(
`${identity}: LiveKit participant ${source ? `matched (${source.participant.sid})` : "missing"}`,
);
});
return memberScope.behavior(
mapScoped(memberScope, source$, (mediaScope, { participant, room }) =>
createLivekitMemberMedia(
mediaScope,
participant,
room,
encryptionSystem,
),
).pipe(map((media) => media ?? null)),
);
return createLivekitMemberMedia(memberScope, source$, encryptionSystem);
};
const connections$ = scope.behavior<BackendConnection[]>(
sdk/src/media-backend/livekit/LivekitMemberMedia.test.ts
+15 -10
View File
@@ -16,29 +16,32 @@ import {
import { describe, expect, it } from "vitest";
import { E2eeType } from "../../encryption";
import { constant } from "../../reactive/Behavior";
import { testScope } from "../../utils/test";
import { createLivekitMemberMedia } from "./LivekitMemberMedia";
describe("createLivekitMemberMedia", () => {
it("lists one stable track per publication, in publication order", () => {
const { participant, publications, room, emitter } = fakes();
const media = createLivekitMemberMedia(testScope(), participant, room, {
kind: E2eeType.NONE,
});
const media = createLivekitMemberMedia(
testScope(),
constant({ participant, room }),
{ kind: E2eeType.NONE },
);
expect(media.tracks$.value).toEqual([]);
const microphone = publish(publications, "TR_1", Track.Source.Microphone);
const camera = publish(publications, "TR_2", Track.Source.Camera);
emitter.emit(ParticipantEvent.TrackPublished, camera);
const [first, second] = media.tracks$.value;
const [first, second] = media.tracks$.value!;
expect(first.source).toBe("microphone");
expect(second.source).toBe("camera");
// An unrelated media event keeps the same objects
microphone.isMuted = true;
emitter.emit(ParticipantEvent.TrackMuted, microphone);
expect(media.tracks$.value[0]).toBe(first);
expect(media.tracks$.value[1]).toBe(second);
expect(media.tracks$.value![0]).toBe(first);
expect(media.tracks$.value![1]).toBe(second);
publications.delete("TR_1");
emitter.emit(ParticipantEvent.TrackUnpublished, microphone);
@@ -47,14 +50,16 @@ describe("createLivekitMemberMedia", () => {
it("includes a track published without a source", () => {
const { participant, publications, room, emitter } = fakes();
const media = createLivekitMemberMedia(testScope(), participant, room, {
kind: E2eeType.NONE,
});
const media = createLivekitMemberMedia(
testScope(),
constant({ participant, room }),
{ kind: E2eeType.NONE },
);
emitter.emit(
ParticipantEvent.TrackPublished,
publish(publications, "TR_1", Track.Source.Unknown),
);
expect(media.tracks$.value.map((t) => t.source)).toEqual(["unknown"]);
expect(media.tracks$.value!.map((t) => t.source)).toEqual(["unknown"]);
});
});
sdk/src/media-backend/livekit/LivekitMemberMedia.ts
+85 -75
View File
@@ -16,120 +16,130 @@ import {
RoomEvent,
type TrackPublication,
} from "livekit-client";
import { filter, map, type Observable } from "rxjs";
import { EMPTY, filter, map, type Observable, of, switchMap } from "rxjs";
import { type ObservableScope } from "../../reactive/ObservableScope";
import { generateItems } from "../../reactive/observable";
import { E2eeType } from "../../encryption";
import { type EncryptionSystem } from "../../encryption";
import { E2eeType, type EncryptionSystem } from "../../encryption";
import { type Behavior } from "../../reactive/Behavior";
import {
type EncryptionError,
type LocalMemberMedia,
type MediaSource,
type MemberMedia,
} from "../../media-api";
import { mapScoped } from "../../utils/mapScoped";
import { type EncryptionError, type MediaSource } from "../../media-api";
import { type LocalMemberMediaFields, type MemberMediaFields } from "../api";
import {
createLivekitMediaTrack,
createLocalLivekitMediaTrack,
} from "./LivekitMediaTrack";
/** Where a member's media comes from: its participant on a room, null while it has none. */
export type ParticipantSource<P extends Participant = Participant> = {
participant: P;
room: LivekitRoom;
} | null;
/**
* A participant as `MemberMedia`. The only place, with `LivekitMediaTrack`,
* that reads a LiveKit participant; everything above sees tracks and
* behaviors.
* A member's media from its participant. The only place, with
* `LivekitMediaTrack`, that reads a LiveKit participant; everything above
* sees tracks and behaviors.
*/
export function createLivekitMemberMedia(
scope: ObservableScope,
participant: Participant,
room: LivekitRoom,
source$: Behavior<ParticipantSource>,
encryptionSystem: EncryptionSystem,
): MemberMedia {
): MemberMediaFields {
return {
local: false,
tracks$: memberTracks$(scope, participant, (trackScope, publication) =>
createLivekitMediaTrack(trackScope, participant, publication, room),
),
encryptionError$: encryptionErrors$(
tracks$: memberTracks$(
scope,
participant,
room,
encryptionSystem,
source$,
(trackScope, participant, publication, room) =>
createLivekitMediaTrack(trackScope, participant, publication, room),
),
encryptionError$: encryptionErrors$(source$, encryptionSystem),
};
}
export function createLocalLivekitMemberMedia(
scope: ObservableScope,
participant: LocalParticipant,
room: LivekitRoom,
source$: Behavior<ParticipantSource<LocalParticipant>>,
encryptionSystem: EncryptionSystem,
setEnabled: (source: MediaSource, enabled: boolean) => Promise<boolean>,
): LocalMemberMedia {
): LocalMemberMediaFields {
return {
local: true,
tracks$: memberTracks$(scope, participant, (trackScope, publication) =>
createLocalLivekitMediaTrack(
trackScope,
participant,
publication,
room,
setEnabled,
),
),
encryptionError$: encryptionErrors$(
tracks$: memberTracks$(
scope,
participant,
room,
encryptionSystem,
source$,
(trackScope, participant, publication, room) =>
createLocalLivekitMediaTrack(
trackScope,
participant,
publication,
room,
setEnabled,
),
),
encryptionError$: encryptionErrors$(source$, encryptionSystem),
};
}
/** One track per publication, each living as long as its publication is in the participant's map. */
function memberTracks$<T>(
/**
* One track per publication, each living as long as its publication is in
* the participant's map; null while there is no participant.
*/
function memberTracks$<P extends Participant, T>(
scope: ObservableScope,
participant: Participant,
factory: (scope: ObservableScope, publication: TrackPublication) => T,
): Behavior<T[]> {
source$: Behavior<ParticipantSource<P>>,
factory: (
scope: ObservableScope,
participant: P,
publication: TrackPublication,
room: LivekitRoom,
) => T,
): Behavior<T[] | null> {
return scope.behavior(
observeParticipantMedia(participant).pipe(
generateItems(
`${participant.identity} tracks$`,
function* () {
for (const publication of participant.trackPublications.values())
yield { keys: [publication.trackSid], data: publication };
},
(trackScope, publication$) => factory(trackScope, publication$.value),
mapScoped(scope, source$, (participantScope, { participant, room }) =>
participantScope.behavior(
observeParticipantMedia(participant).pipe(
generateItems(
`${participant.identity} tracks$`,
function* () {
for (const publication of participant.trackPublications.values())
yield { keys: [publication.trackSid], data: publication };
},
(trackScope, publication$) =>
factory(trackScope, participant, publication$.value, room),
),
),
),
),
).pipe(switchMap((tracks$) => tracks$ ?? of(null))),
);
}
function encryptionErrors$(
scope: ObservableScope,
participant: Participant,
room: LivekitRoom,
source$: Behavior<ParticipantSource>,
encryptionSystem: EncryptionSystem,
): Observable<EncryptionError> {
return roomEventSelector(room, RoomEvent.EncryptionError).pipe(
map(([error]) => error?.message ?? ""),
// The participant the error is about does not survive the trip from the
// worker, so the identity is matched in the message. A shared key is the
// same for everyone, so its errors are too.
filter(
(message) =>
encryptionSystem.kind === E2eeType.SHARED_KEY ||
message.includes(participant.identity),
return source$.pipe(
switchMap((source) =>
source === null
? EMPTY
: roomEventSelector(source.room, RoomEvent.EncryptionError).pipe(
map(([error]) => error?.message ?? ""),
// The participant the error is about does not survive the trip
// from the worker, so the identity is matched in the message. A
// shared key is the same for everyone, so its errors are too.
filter(
(message) =>
encryptionSystem.kind === E2eeType.SHARED_KEY ||
message.includes(source.participant.identity),
),
map((message): EncryptionError | undefined =>
message.includes("MissingKey")
? "MissingKey"
: message.includes("InvalidKey")
? "InvalidKey"
: undefined,
),
filter((error) => error !== undefined),
),
),
map((message): EncryptionError | undefined =>
message.includes("MissingKey")
? "MissingKey"
: message.includes("InvalidKey")
? "InvalidKey"
: undefined,
),
filter((error) => error !== undefined),
scope.bind(),
);
}
sdk/src/media-backend/livekit/LocalMedia.ts
+13 -23
View File
@@ -28,7 +28,6 @@ import {
import {
type LocalAudioMediaTrack,
type LocalMemberMedia,
type LocalVideoMediaTrack,
type MediaSource,
type PublishRequest,
@@ -37,7 +36,6 @@ import { type EncryptionSystem } from "../../encryption";
import { FailToStartLivekitConnection, toMatrixRTCError } from "../../errors";
import { type Behavior } from "../../reactive/Behavior";
import { type ObservableScope } from "../../reactive/ObservableScope";
import { mapScoped } from "../../utils/mapScoped";
import { type LocalMediaBackend, MediaConnectionState } from "../api";
import { type Connection } from "./Connection";
import {
@@ -169,19 +167,11 @@ export function createLivekitLocalMedia({
return result;
};
const media$ = scope.behavior<LocalMemberMedia | null>(
mapScoped(
scope,
participantAndRoom$(scope, participant$, connection$),
(mediaScope, { participant, room }) =>
createLocalLivekitMemberMedia(
mediaScope,
participant,
room,
encryptionSystem,
setEnabled,
),
).pipe(map((media) => media ?? null)),
const { tracks$, encryptionError$ } = createLocalLivekitMemberMedia(
scope,
participantAndRoom$(scope, participant$, connection$),
encryptionSystem,
setEnabled,
);
const publish = async (
@@ -194,7 +184,7 @@ export function createLivekitLocalMedia({
const publication =
(publisher?.started ? await publisher.publish(request) : undefined) ??
(await firstValueFrom(publication$(participant$, request.source)));
return await firstValueFrom(trackWithId$(media$, publication.trackSid));
return await firstValueFrom(trackWithId$(tracks$, publication.trackSid));
} catch (e) {
desired.delete(request.source);
throw toMatrixRTCError(e);
@@ -202,7 +192,7 @@ export function createLivekitLocalMedia({
};
const unpublish = async (id: string): Promise<void> => {
const track = media$.value?.tracks$.value.find((t) => t.id === id);
const track = tracks$.value?.find((t) => t.id === id);
if (track === undefined) return;
// The screen share audio goes with its video
const source =
@@ -219,7 +209,8 @@ export function createLivekitLocalMedia({
return {
connection$,
connectionState$,
media$,
tracks$,
encryptionError$,
setPublishing: (publish) => publishing$.next(publish),
publishError$,
publish,
@@ -228,7 +219,7 @@ export function createLivekitLocalMedia({
};
}
/** The pair a `MemberMedia` is built on, changing only when one of the two does. */
/** The pair the local tracks are built on, changing only when one of the two does. */
function participantAndRoom$(
scope: ObservableScope,
participant$: Behavior<LocalParticipant | null>,
@@ -269,12 +260,11 @@ function publication$(
}
function trackWithId$(
media$: Behavior<LocalMemberMedia | null>,
tracks$: Behavior<(LocalAudioMediaTrack | LocalVideoMediaTrack)[] | null>,
id: string,
): Observable<LocalAudioMediaTrack | LocalVideoMediaTrack> {
return media$.pipe(
switchMap((media) => media?.tracks$ ?? of([])),
map((tracks) => tracks.find((track) => track.id === id)),
return tracks$.pipe(
map((tracks) => tracks?.find((track) => track.id === id)),
filter((track) => track !== undefined),
);
}