Matrix/element-call-Github
1
0
Fork 0
mirror of https://github.com/vector-im/element-call.git synced 2026-10-16 23:45:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update to changes in the sdk.

Browse Source 
- plain tracks instead of media$
This commit is contained in:
Timo K.
2026-10-14 10:23:29 +02:00
committed by Timo
parent 52aaf84c3e
commit 7c5d1324e7
19 changed files with 358 additions and 259 deletions
Download Patch File Download Diff File Expand all files Collapse all files
sdk-target-based-on-call-view-model/main.ts
+6 -18
View File
@@ -37,7 +37,6 @@ import {
import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
import {
type Behavior,
constant,
createMatrixRTCClient,
E2eeType,
ObservableScope,
@@ -60,7 +59,6 @@ import { observeElementSize$ } from "../src/utils/elementSize";
import {
captureSettings,
matrixRTCClientOptions,
selectedDeviceId$,
} from "../src/room/InCallView";
/**
@@ -143,22 +141,12 @@ export async function createMatrixRTCSdk(
// rtc client: the session, the transport and the media, as in the app
const encryptionSystem = { kind: E2eeType.PER_PARTICIPANT } as const;
const capture = captureSettings();
const rtcClient = createMatrixRTCClient(
scope,
client,
room,
{
publish: initialPublishRequests(muteStates, mediaDevices, capture),
audioOutputDeviceId$: controlledAudioDevices
? constant(undefined)
: selectedDeviceId$(scope, mediaDevices.audioOutput),
},
{
...matrixRTCClientOptions(urlParams, encryptionSystem),
application,
slot: id,
},
);
const rtcClient = createMatrixRTCClient(scope, client, room, {
...matrixRTCClientOptions(urlParams, encryptionSystem),
publish: initialPublishRequests(muteStates, mediaDevices, capture),
application,
slot: id,
});
// call view model: the host bridge's hang-up handling and the leave flow
const callViewModel = createCallViewModel$(
src/room/InCallView.test.tsx
+2 -3
View File
@@ -27,7 +27,6 @@ import {
mockRtcMembership,
type MockRTCSession,
mockMatrixRTCClient,
mockMemberMedia,
mockRTCMember,
} from "../utils/test";
import { E2eeType } from "../e2ee/e2eeType";
@@ -103,13 +102,13 @@ beforeEach(() => {
localMember$: constant(
mockRTCMember(true, {
membership: localRtcMember,
media$: constant(mockMemberMedia({ local: true })),
tracks$: constant([]),
}),
),
remoteMembers$: constant([
mockRTCMember(false, {
membership: aliceRtcMember,
media$: constant(mockMemberMedia()),
tracks$: constant([]),
}),
]),
}),
src/room/InCallView.tsx
+5 -18
View File
@@ -72,7 +72,6 @@ import {
createMatrixRTCClient,
type MatrixRTCClientOptions,
constant,
type Behavior,
} from "@element-hq/matrixrtc-sdk";
import { map } from "rxjs";
import {
@@ -94,7 +93,6 @@ import {
screenShareResolution,
} from "../settings/settings";
import { Config } from "../config/Config";
import { type MediaDevice } from "../state/MediaDevices";
import { createSentCallNotification$ } from "../state/CallViewModel/CallNotificationLifecycle.ts";
import { ReactionsReader } from "../reactions/ReactionsReader";
import { MemberAudioRenderer } from "../tracks/MemberAudioRenderer.tsx";
@@ -168,17 +166,13 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
props.client,
props.matrixRoom,
{
...matrixRTCClientOptions(urlParams, props.e2eeSystem),
publish: initialPublishRequests(
props.muteStates,
mediaDevices,
capture,
),
// A host that routes audio itself leaves the browser's output alone
audioOutputDeviceId$: urlParams.controlledAudioDevices
? constant(undefined)
: selectedDeviceId$(scope, mediaDevices.audioOutput),
},
matrixRTCClientOptions(urlParams, props.e2eeSystem),
);
const vm = createCallViewModel$(
scope,
@@ -744,14 +738,6 @@ export const InCallView: FC<InCallViewProps> = ({
);
};
/** The id of the device the user picked, for the client to capture or play on. */
export function selectedDeviceId$(
scope: ObservableScope,
device: MediaDevice<unknown, { id: string }>,
): Behavior<string | undefined> {
return scope.behavior(device.selected$.pipe(map((d) => d?.id)));
}
/**
* How the tracks are captured and encoded, from Element Call's settings and
* configuration. Shared with the legacy SDK bundle so that a widget built from
@@ -803,13 +789,14 @@ export function captureSettings(): CaptureSettings {
/**
* What the client needs from Element Call's configuration, settings and URL:
* the SDK reads none of them itself. Shared with the legacy SDK bundle so that
* a widget built from it behaves like the app.
* the SDK reads none of them itself. What to publish depends on the mute
* switches and the devices, so the caller adds that. Shared with the legacy
* SDK bundle so that a widget built from it behaves like the app.
*/
export function matrixRTCClientOptions(
urlParams: ReturnType<typeof useUrlParams>,
encryptionSystem: EncryptionSystem,
): MatrixRTCClientOptions {
): Omit<MatrixRTCClientOptions, "publish"> {
const config = Config.get();
const session = config.matrix_rtc_session;
return {
src/state/CallViewModel/CallViewModel.test.ts
+51 -1
View File
@@ -8,6 +8,7 @@ Please see LICENSE in the repository root for full details.
import { test, vi, onTestFinished, it, describe, expect } from "vitest";
import {
BehaviorSubject,
combineLatest,
debounceTime,
distinctUntilChanged,
@@ -26,6 +27,7 @@ import { deepCompare } from "matrix-js-sdk/lib/utils";
import { type Layout } from "../layout-types.ts";
import {
mockMatrixRoomMember,
mockMediaDevices,
mockRemoteParticipant,
withTestScheduler,
mockRtcMembership,
@@ -46,7 +48,10 @@ import {
localRtcMember,
localRtcMemberDevice2,
} from "../../utils/test-fixtures.ts";
import { MediaDevices } from "../MediaDevices.ts";
import {
MediaDevices,
type SelectedAudioOutputDevice,
} from "../MediaDevices.ts";
import { getValue } from "../../utils/observable.ts";
import { type Behavior, constant } from "../Behavior.ts";
import {
@@ -236,6 +241,51 @@ const modes = [[MatrixRTCMode.Compatibility], [MatrixRTCMode.Matrix_2_0]];
describe.each(modes)("CallViewModel (%s mode)", (mode) => {
const withCallViewModel = withCallViewModelInMode(mode);
/** Media devices whose audio output the test selects. */
const withAudioOutput = (): {
mediaDevices: MediaDevices;
selected$: BehaviorSubject<SelectedAudioOutputDevice | undefined>;
} => {
const selected$ = new BehaviorSubject<
SelectedAudioOutputDevice | undefined
>(undefined);
const mediaDevices = mockMediaDevices({
audioOutput: {
available$: constant(new Map()),
selected$,
select: vi.fn(),
},
});
return { mediaDevices, selected$ };
};
test("the selected audio output is handed to the client as it changes", () => {
const { mediaDevices, selected$ } = withAudioOutput();
withCallViewModel({ mediaDevices }, (_vm, _session, { rtcClient }) => {
// Nothing selected yet, so the browser's choice stands
expect(rtcClient.setAudioOutputDeviceId).not.toHaveBeenCalled();
selected$.next({ id: "speaker", virtualEarpiece: false });
expect(rtcClient.setAudioOutputDeviceId).toHaveBeenCalledWith("speaker");
selected$.next({ id: "headphones", virtualEarpiece: false });
expect(rtcClient.setAudioOutputDeviceId).toHaveBeenLastCalledWith(
"headphones",
);
expect(rtcClient.setAudioOutputDeviceId).toHaveBeenCalledTimes(2);
});
});
test("a host that controls the audio devices keeps the output from the client", () => {
const { mediaDevices, selected$ } = withAudioOutput();
withCallViewModel(
{ mediaDevices },
(_vm, _session, { rtcClient }) => {
selected$.next({ id: "earpiece", virtualEarpiece: true });
expect(rtcClient.setAudioOutputDeviceId).not.toHaveBeenCalled();
},
{ controlledAudioDevices: true },
);
});
test("participants are retained during a focus switch", () => {
withTestScheduler(({ behavior, expectObservable }) => {
// Participants disappear on frame 2 and come back on frame 3
src/state/CallViewModel/CallViewModel.ts
+28 -8
View File
@@ -22,7 +22,6 @@ import {
pauseWhen,
type PublishRequest,
type RemoteRTCMember,
type RTCMember,
trackBySource$,
type TransportMetadata,
type VideoCaptureSettings,
@@ -193,6 +192,12 @@ export interface CallViewModelOptions {
showControls?: boolean;
/** Whether to hide the screen-sharing button. Defaults to false. */
hideScreensharing?: boolean;
/**
* Whether the host routes the audio itself, outside the browser, so that
* the selected output device is never handed to the client. Defaults to
* false.
*/
controlledAudioDevices?: boolean;
autoLeaveWhenOthersLeft?: boolean;
/**
* If the call is started in a way where we want it to behave like a telephone usecase
@@ -232,11 +237,15 @@ export interface CallViewModelOptions {
*/
export function callViewModelOptionsFromParams(
params: UrlParams,
): Pick<CallViewModelOptions, "header" | "showControls" | "hideScreensharing"> {
): Pick<
CallViewModelOptions,
"header" | "showControls" | "hideScreensharing" | "controlledAudioDevices"
> {
return {
header: params.header,
showControls: params.showControls,
hideScreensharing: params.hideScreensharing,
controlledAudioDevices: params.controlledAudioDevices,
};
}
@@ -484,8 +493,8 @@ export function createCallViewModel$(
(LocalAudioMediaTrack | LocalVideoMediaTrack)[]
>(
rtcClient.localMember$.pipe(
switchMap((member) => member?.media$ ?? of(null)),
switchMap((media) => media?.tracks$ ?? of([])),
switchMap((member) => member?.tracks$ ?? of(null)),
map((tracks) => tracks ?? []),
),
);
// Our own tracks, which carry the controls remote ones lack
@@ -570,6 +579,19 @@ export function createCallViewModel$(
followDevice(microphone$, mediaDevices.audioInput, "microphone");
followDevice(camera$, mediaDevices.videoInput, "camera");
// The output device follows the selection onto the client the same way. A
// host that routes audio itself picks the output outside the browser, so the
// client is left alone
if (!options.controlledAudioDevices)
mediaDevices.audioOutput.selected$
.pipe(scope.bind())
.subscribe((selected) => {
if (selected === undefined) return;
rtcClient.setAudioOutputDeviceId(selected.id).catch((e) => {
logger.error("Failed to switch the audio output device", e);
});
});
// Attached once the camera track exists rather than carried in the publish
// request, so that a new track and a change of processor take the same path
combineLatest([camera$, videoProcessor$])
@@ -580,7 +602,7 @@ export function createCallViewModel$(
});
});
const members$ = scope.behavior<RTCMember[]>(
const members$ = scope.behavior<(LocalRTCMember | RemoteRTCMember)[]>(
combineLatest(
[rtcClient.localMember$, rtcClient.remoteMembers$],
(local, remote) => (local === null ? remote : [local, ...remote]),
@@ -651,9 +673,7 @@ export function createCallViewModel$(
id: `${mediaId}:${dup}`,
userId: member.userId,
rtcBackendIdentity: member.id,
...(member.local
? { local: true, media$: (member as LocalRTCMember).media$ }
: { local: false, media$: member.media$ }),
member,
encryptionSystem: options.encryptionSystem,
focusUrl$: scope.behavior(
member.transport$.pipe(map((transport) => transport?.id)),
src/state/CallViewModel/CallViewModelTestUtils.ts
+17 -21
View File
@@ -10,8 +10,8 @@ import {
type Behavior,
constant,
E2eeType,
type LocalMemberMedia,
type LocalRTCMember,
type MatrixRTCClient,
type MatrixRTCMode,
type ObservableScope,
type RemoteRTCMember,
@@ -37,7 +37,7 @@ import {
mockAudioTrack,
mockMediaDevices,
mockVideoTrack,
mockMemberMedia,
type MockTracks,
mockMuteStates,
MockRTCSession,
mockRTCMember,
@@ -112,6 +112,8 @@ export function withCallViewModel(mode: MatrixRTCMode) {
rtcSession: MockRTCSession,
subjects: {
raisedHands$: BehaviorSubject<Record<string, RaisedHandInfo>>;
/** The client the view model was built on, with spies for its calls. */
rtcClient: MatrixRTCClient;
},
setSyncState: (value: SyncState) => void,
) => void,
@@ -151,11 +153,10 @@ export function withCallViewModel(mode: MatrixRTCMode) {
);
const scope = testScope();
const mediaOf = (
const tracksOf = (
scope: ObservableScope,
participant: Participant,
local: boolean,
): LocalMemberMedia => {
): Behavior<MockTracks> => {
// A microphone and a camera are always published; the screen share
// comes and goes
const microphone = mockAudioTrack({
@@ -170,18 +171,13 @@ export function withCallViewModel(mode: MatrixRTCMode) {
),
});
const screenShare = mockVideoTrack({ source: "screenShare" });
return mockMemberMedia({
local,
tracks$: scope.behavior(
(sharingScreen.get(participant) ?? constant(false)).pipe(
map((sharing) =>
sharing
? [microphone, camera, screenShare]
: [microphone, camera],
),
return scope.behavior(
(sharingScreen.get(participant) ?? constant(false)).pipe(
map((sharing) =>
sharing ? [microphone, camera, screenShare] : [microphone, camera],
),
),
});
);
};
const participantOf = (
scope: ObservableScope,
@@ -204,12 +200,12 @@ export function withCallViewModel(mode: MatrixRTCMode) {
membership,
roomMember: roomMembers.find((m) => m.userId === membership.userId),
transportUrl: "http://my-default-service-url.com",
media$: scope.behavior(
tracks$: scope.behavior(
participantOf(scope, membership).pipe(
map((participant) =>
switchMap((participant) =>
participant === undefined
? null
: mediaOf(scope, participant, false),
? of(null)
: tracksOf(scope, participant),
),
),
),
@@ -239,7 +235,7 @@ export function withCallViewModel(mode: MatrixRTCMode) {
membership: localRtcMember,
roomMember: carol,
transportUrl: "http://my-default-service-url.com",
media$: constant(mediaOf(scope, localParticipant, true)),
tracks$: tracksOf(scope, localParticipant),
})
: null,
),
@@ -283,6 +279,6 @@ export function withCallViewModel(mode: MatrixRTCMode) {
);
void mode;
continuation(vm, rtcSession, { raisedHands$: raisedHands$ }, setSyncState);
continuation(vm, rtcSession, { raisedHands$, rtcClient }, setSyncState);
};
}
src/state/media/LocalUserMediaViewModel.ts
+7 -7
View File
@@ -8,9 +8,10 @@ Please see LICENSE in the repository root for full details.
import {
type Behavior,
type LocalMemberMedia,
type LocalRTCMember,
type LocalVideoMediaTrack,
type ObservableScope,
trackBySource$,
} from "@element-hq/matrixrtc-sdk";
import { map, of, switchMap } from "rxjs";
import { logger } from "matrix-js-sdk/lib/logger";
@@ -23,7 +24,6 @@ import {
import { alwaysShowSelf } from "../../settings/settings";
import { platform } from "../../Platform";
import { type MediaDevices } from "../MediaDevices";
import { memberTrack$ } from "./MemberMediaViewModel";
export interface LocalUserMediaViewModel extends BaseUserMediaViewModel {
local: true;
@@ -42,9 +42,9 @@ export interface LocalUserMediaViewModel extends BaseUserMediaViewModel {
export interface LocalUserMediaInputs extends Omit<
BaseUserMediaInputs,
"media$"
"member"
> {
media$: Behavior<LocalMemberMedia | null>;
member: Pick<LocalRTCMember, "local" | "tracks$" | "encryptionError$">;
mediaDevices: MediaDevices;
}
@@ -54,11 +54,11 @@ export function createLocalUserMedia(
): LocalUserMediaViewModel {
const baseUserMedia = createBaseUserMedia(scope, inputs);
// Our own camera track, which carries the controls a remote one lacks
const camera$ = memberTrack$<LocalVideoMediaTrack>(
const camera$ = trackBySource$(
scope,
inputs.media$,
inputs.member.tracks$,
"camera",
);
) as Behavior<LocalVideoMediaTrack | undefined>;
const facingMode$ = scope.behavior(
camera$.pipe(switchMap((camera) => camera?.facingMode$ ?? of(undefined))),
);
src/state/media/MediaViewModel.test.ts
+37 -57
View File
@@ -18,14 +18,15 @@ import {
mockAudioTrack,
mockLocalMedia,
mockMediaDevices,
mockMemberMedia,
mockRemoteMedia,
mockRemoteScreenShare,
mockRtcMembership,
type MockTracks,
mockVideoTrack,
withTestScheduler,
} from "../../utils/test";
import { showConnectionStats } from "../../settings/settings";
import { EncryptionStatus } from "./MemberMediaViewModel";
const platformMock = vi.hoisted(() => vi.fn(() => "desktop"));
vi.mock("../../Platform", () => ({
@@ -38,13 +39,9 @@ const rtcMembership = mockRtcMembership("@alice:example.org", "AAAA");
test("control a participant's volume", () => {
const setVolume = vi.fn();
const vm = mockRemoteMedia(
rtcMembership,
{},
mockMemberMedia({
tracks$: constant([mockAudioTrack({ setVolume })]),
}),
);
const vm = mockRemoteMedia(rtcMembership, {}, [
mockAudioTrack({ setVolume }),
]);
withTestScheduler(({ expectObservable, schedule }) => {
schedule("-ab---c---d|", {
a() {
@@ -89,15 +86,9 @@ test("control a participant's volume", () => {
test("control a participant's screen share volume", () => {
const setVolume = vi.fn();
const vm = mockRemoteScreenShare(
rtcMembership,
{},
mockMemberMedia({
tracks$: constant([
mockAudioTrack({ source: "screenShareAudio", setVolume }),
]),
}),
);
const vm = mockRemoteScreenShare(rtcMembership, {}, [
mockAudioTrack({ source: "screenShareAudio", setVolume }),
]);
withTestScheduler(({ expectObservable, schedule }) => {
schedule("-ab---c---d|", {
a() {
@@ -137,23 +128,13 @@ test("control a participant's screen share volume", () => {
});
test("local media remembers whether it should always be shown", () => {
const vm1 = mockLocalMedia(
rtcMembership,
{},
mockMemberMedia({ local: true }),
mockMediaDevices({}),
);
const vm1 = mockLocalMedia(rtcMembership, {}, [], mockMediaDevices({}));
withTestScheduler(({ expectObservable, schedule }) => {
schedule("-a|", { a: () => vm1.setAlwaysShow(false) });
expectObservable(vm1.alwaysShow$).toBe("ab", { a: true, b: false });
});
// Next local media should start out *not* always shown
const vm2 = mockLocalMedia(
rtcMembership,
{},
mockMemberMedia({ local: true }),
mockMediaDevices({}),
);
const vm2 = mockLocalMedia(rtcMembership, {}, [], mockMediaDevices({}));
withTestScheduler(({ expectObservable, schedule }) => {
schedule("-a|", { a: () => vm2.setAlwaysShow(true) });
expectObservable(vm2.alwaysShow$).toBe("ab", { a: false, b: true });
@@ -178,10 +159,7 @@ test("switch cameras", async () => {
const vm = mockLocalMedia(
rtcMembership,
{},
mockMemberMedia({
local: true,
tracks$: constant([mockVideoTrack({ facingMode$, switchFacingMode })]),
}),
[mockVideoTrack({ facingMode$, switchFacingMode })],
mockMediaDevices({
videoInput: {
available$: constant(new Map()),
@@ -215,7 +193,7 @@ test("no camera switch where the facing mode is unknown", () => {
const vm = mockLocalMedia(
rtcMembership,
{},
mockMemberMedia({ local: true, tracks$: constant([mockVideoTrack()]) }),
[mockVideoTrack()],
mockMediaDevices({}),
);
expect(vm.switchCamera$.value).toBeNull();
@@ -227,26 +205,34 @@ test("remote media is in waiting state while its media has not arrived", () => {
});
test("remote media is not in waiting state once its media is there", () => {
const vm = mockRemoteMedia(rtcMembership, {}, mockMemberMedia());
const vm = mockRemoteMedia(rtcMembership, {}, []);
expect(vm.waitingForMedia$.value).toBe(false);
});
test("remote media stops waiting once its tracks arrive, however few", () => {
const tracks$ = new BehaviorSubject<MockTracks>(null);
const vm = mockRemoteMedia(rtcMembership, {}, tracks$);
expect(vm.waitingForMedia$.value).toBe(true);
expect(vm.encryptionStatus$.value).toBe(EncryptionStatus.Connecting);
tracks$.next([]);
expect(vm.waitingForMedia$.value).toBe(false);
});
test("local media never reports an encryption problem", () => {
const vm = mockLocalMedia(rtcMembership, {}, [], mockMediaDevices({}));
expect(vm.encryptionStatus$.value).toBe(EncryptionStatus.Okay);
});
test("remote media is not in waiting state when user does not intend to publish anywhere", () => {
const vm = mockRemoteMedia(rtcMembership, {}, null, {});
expect(vm.waitingForMedia$.value).toBe(false);
});
test("audio and video follow the tracks' mute state", () => {
const vm = mockRemoteMedia(
rtcMembership,
{},
mockMemberMedia({
tracks$: constant([
mockAudioTrack({ muted$: constant(false) }),
mockVideoTrack({ muted$: constant(true) }),
]),
}),
);
const vm = mockRemoteMedia(rtcMembership, {}, [
mockAudioTrack({ muted$: constant(false) }),
mockVideoTrack({ muted$: constant(true) }),
]);
expect(vm.audioEnabled$.value).toBe(true);
expect(vm.videoEnabled$.value).toBe(false);
});
@@ -256,19 +242,13 @@ test("user media polls stream stats only while the setting is on", () => {
withTestScheduler(({ cold, expectObservable, schedule }) => {
const stats = { type: "inbound-rtp" } as RTCInboundRtpStreamStats;
// A behavior that only runs its source while subscribed, as the SDK's does
const vm = mockRemoteMedia(
rtcMembership,
{},
mockMemberMedia({
tracks$: constant([
mockAudioTrack({
stats$: cold("-s", {
s: stats,
}) as unknown as Behavior<MediaStreamStats>,
}),
]),
const vm = mockRemoteMedia(rtcMembership, {}, [
mockAudioTrack({
stats$: cold("-s", {
s: stats,
}) as unknown as Behavior<MediaStreamStats>,
}),
);
]);
schedule("-a-b", {
a() {
showConnectionStats.setValue(true);
src/state/media/MemberMediaViewModel.ts
+19 -30
View File
@@ -7,15 +7,13 @@ Please see LICENSE in the repository root for full details.
*/
import {
type AudioMediaTrack,
type Behavior,
E2eeType,
type EncryptionError,
type EncryptionSystem,
type MediaSource,
type MediaTrack,
type MemberMedia,
type ObservableScope,
type RTCMember,
trackBySource$,
type VideoMediaTrack,
} from "@element-hq/matrixrtc-sdk";
@@ -65,33 +63,24 @@ export interface BaseMemberMediaViewModel extends BaseMediaViewModel {
encryptionStatus$: Behavior<EncryptionStatus>;
}
/** The member whose media this is, as far as the media view models read it. */
export type MediaMember = Pick<
RTCMember,
"local" | "tracks$" | "encryptionError$"
>;
export interface MemberMediaInputs extends BaseMediaViewModel {
media$: Behavior<MemberMedia | null>;
member: MediaMember;
audioSource: "microphone" | "screenShareAudio";
videoSource: "camera" | "screenShare";
focusUrl$: Behavior<string | undefined>;
encryptionSystem: EncryptionSystem;
}
/**
* The member's first track of a source: `trackBySource$` over media that may
* not have arrived yet. `T` narrows to a local track where the media is ours.
*/
export function memberTrack$<T extends MediaTrack>(
scope: ObservableScope,
media$: Behavior<MemberMedia | null>,
source: MediaSource,
): Behavior<T | undefined> {
const tracks$ = scope.behavior<MediaTrack[]>(
media$.pipe(switchMap((media) => media?.tracks$ ?? of([]))),
);
return trackBySource$(scope, tracks$, source) as Behavior<T | undefined>;
}
export function createMemberMedia(
scope: ObservableScope,
{
media$,
member,
audioSource,
videoSource,
transport$,
@@ -99,8 +88,8 @@ export function createMemberMedia(
...inputs
}: MemberMediaInputs,
): BaseMemberMediaViewModel {
const audio$ = memberTrack$<AudioMediaTrack>(scope, media$, audioSource);
const video$ = memberTrack$<VideoMediaTrack>(scope, media$, videoSource);
const audio$ = trackBySource$(scope, member.tracks$, audioSource);
const video$ = trackBySource$(scope, member.tracks$, videoSource);
const unencrypted$ = (
track$: Behavior<MediaTrack | undefined>,
): Observable<boolean> =>
@@ -123,16 +112,16 @@ export function createMemberMedia(
),
),
encryptionStatus$: scope.behavior(
media$.pipe(
switchMap((media): Observable<EncryptionStatus> => {
if (media === null) return of(EncryptionStatus.Connecting);
if (media.local || encryptionSystem.kind === E2eeType.NONE)
member.tracks$.pipe(
switchMap((tracks): Observable<EncryptionStatus> => {
if (tracks === null) return of(EncryptionStatus.Connecting);
if (member.local || encryptionSystem.kind === E2eeType.NONE)
return of(EncryptionStatus.Okay);
const perParticipant =
encryptionSystem.kind === E2eeType.PER_PARTICIPANT;
return combineLatest([
encryptionError$(media, "MissingKey"),
encryptionError$(media, "InvalidKey"),
encryptionError$(member.encryptionError$, "MissingKey"),
encryptionError$(member.encryptionError$, "InvalidKey"),
receivingOkay$(audio$),
receivingOkay$(video$),
]).pipe(
@@ -156,10 +145,10 @@ export function createMemberMedia(
}
function encryptionError$(
media: MemberMedia,
errors$: Observable<EncryptionError>,
criteria: EncryptionError,
): Observable<boolean> {
return media.encryptionError$.pipe(
return errors$.pipe(
map((error) => error === criteria),
distinctUntilChanged(),
throttleTime(1000), // Throttle to avoid spamming the UI
src/state/media/RemoteScreenShareViewModel.ts
+3 -4
View File
@@ -7,9 +7,9 @@ Please see LICENSE in the repository root for full details.
*/
import {
type AudioMediaTrack,
type Behavior,
type ObservableScope,
trackBySource$,
} from "@element-hq/matrixrtc-sdk";
import { map } from "rxjs";
@@ -19,7 +19,6 @@ import {
createBaseScreenShare,
} from "./ScreenShareViewModel";
import { createVolumeControls, type VolumeControls } from "../VolumeControls";
import { memberTrack$ } from "./MemberMediaViewModel";
export interface RemoteScreenShareViewModel
extends BaseScreenShareViewModel, VolumeControls {
@@ -42,9 +41,9 @@ export function createRemoteScreenShare(
scope: ObservableScope,
{ pretendToBeDisconnected$, ...inputs }: RemoteScreenShareInputs,
): RemoteScreenShareViewModel {
const audio$ = memberTrack$<AudioMediaTrack>(
const audio$ = trackBySource$(
scope,
inputs.media$,
inputs.member.tracks$,
"screenShareAudio",
);
return {
src/state/media/RemoteUserMediaViewModel.ts
+4 -5
View File
@@ -7,9 +7,9 @@ Please see LICENSE in the repository root for full details.
*/
import {
type AudioMediaTrack,
type Behavior,
type ObservableScope,
trackBySource$,
} from "@element-hq/matrixrtc-sdk";
import { combineLatest, map, of, switchMap } from "rxjs";
import { logger } from "matrix-js-sdk/lib/logger";
@@ -20,7 +20,6 @@ import {
type BaseUserMediaViewModel,
createBaseUserMedia,
} from "./UserMediaViewModel";
import { memberTrack$ } from "./MemberMediaViewModel";
export interface RemoteUserMediaViewModel
extends BaseUserMediaViewModel, VolumeControls {
@@ -44,11 +43,11 @@ export function createRemoteUserMedia(
const waitingForMedia$ = scope.behavior(
combineLatest(
[inputs.focusUrl$, inputs.media$],
[inputs.focusUrl$, inputs.member.tracks$],
// Without a transport the user is not attempting to publish anywhere
// and so we shouldn't expect media. (They might be a subscribe-only bot
// for example.)
(focusUrl, media) => focusUrl !== undefined && media === null,
(focusUrl, tracks) => focusUrl !== undefined && tracks === null,
),
);
waitingForMedia$.pipe(scope.bind()).subscribe((waiting) => {
@@ -60,7 +59,7 @@ export function createRemoteUserMedia(
...createVolumeControls(scope, {
pretendToBeDisconnected$,
sink$: scope.behavior(
memberTrack$<AudioMediaTrack>(scope, inputs.media$, "microphone").pipe(
trackBySource$(scope, inputs.member.tracks$, "microphone").pipe(
map((track) => (volume: number) => track?.setVolume(volume)),
),
),
src/state/media/UserMediaViewModel.ts
+5 -7
View File
@@ -7,11 +7,10 @@ Please see LICENSE in the repository root for full details.
*/
import {
type AudioMediaTrack,
type Behavior,
type MediaStreamStats,
type MediaTrack,
type ObservableScope,
trackBySource$,
} from "@element-hq/matrixrtc-sdk";
import {
BehaviorSubject,
@@ -27,7 +26,6 @@ import { type LocalUserMediaViewModel } from "./LocalUserMediaViewModel";
import {
createMemberMedia,
type MemberMediaInputs,
memberTrack$,
type BaseMemberMediaViewModel,
} from "./MemberMediaViewModel";
import { type RemoteUserMediaViewModel } from "./RemoteUserMediaViewModel";
@@ -79,7 +77,7 @@ export function createBaseUserMedia(
...inputs
}: BaseUserMediaInputs,
): BaseUserMediaViewModel {
const { media$ } = inputs;
const { tracks$ } = inputs.member;
const toggleCropVideo$ = new Subject<void>();
const videoAspectRatio$ = new BehaviorSubject(NaN);
const enabled$ = (
@@ -87,7 +85,7 @@ export function createBaseUserMedia(
source: "microphone" | "camera",
): Behavior<boolean> =>
scope.behavior(
memberTrack$<MediaTrack>(scope, media$, source).pipe(
trackBySource$(scope, tracks$, source).pipe(
switchMap((track) =>
track === undefined
? of(false)
@@ -102,7 +100,7 @@ export function createBaseUserMedia(
): Behavior<MediaStreamStats> =>
scope.behavior(
combineLatest([
memberTrack$<MediaTrack>(scope, media$, source),
trackBySource$(scope, tracks$, source),
showConnectionStats.value$,
]).pipe(
switchMap(([track, show]) =>
@@ -120,7 +118,7 @@ export function createBaseUserMedia(
type: "user",
// What a call calls speaking is the microphone carrying sound
speaking$: scope.behavior(
memberTrack$<AudioMediaTrack>(scope, media$, "microphone").pipe(
trackBySource$(scope, tracks$, "microphone").pipe(
switchMap((track) => track?.isActive$ ?? of(false)),
),
),
src/state/media/WrappedUserMediaViewModel.ts
+18 -24
View File
@@ -8,9 +8,10 @@ Please see LICENSE in the repository root for full details.
import {
type Behavior,
generateItems,
type LocalMemberMedia,
type MemberMedia,
type LocalRTCMember,
type ObservableScope,
type RemoteRTCMember,
trackBySource$,
} from "@element-hq/matrixrtc-sdk";
import { combineLatest, distinctUntilChanged, map, of } from "rxjs";
@@ -27,7 +28,6 @@ import {
} from "./RemoteUserMediaViewModel.ts";
import { createLocalScreenShare } from "./LocalScreenShareViewModel.ts";
import { createRemoteScreenShare } from "./RemoteScreenShareViewModel.ts";
import { memberTrack$ } from "./MemberMediaViewModel.ts";
/**
* Sorting bins defining the order in which media tiles appear in the layout.
@@ -81,34 +81,28 @@ export type WrappedUserMediaViewModel = UserMediaViewModel & {
type WrappedUserMediaInputs = Omit<
LocalUserMediaInputs & RemoteUserMediaInputs,
"media$"
> &
(
| { local: true; media$: Behavior<LocalMemberMedia | null> }
| { local: false; media$: Behavior<MemberMedia | null> }
);
"member"
> & { member: LocalRTCMember | RemoteRTCMember };
export function createWrappedUserMedia(
scope: ObservableScope,
{ mediaDevices, pretendToBeDisconnected$, ...rest }: WrappedUserMediaInputs,
{
member,
mediaDevices,
pretendToBeDisconnected$,
...inputs
}: WrappedUserMediaInputs,
): WrappedUserMediaViewModel {
const { local, media$: _media$, ...inputs } = rest;
const userMedia = rest.local
? createLocalUserMedia(scope, {
media$: rest.media$,
mediaDevices,
...inputs,
})
const userMedia = member.local
? createLocalUserMedia(scope, { member, mediaDevices, ...inputs })
: createRemoteUserMedia(scope, {
media$: rest.media$,
member,
pretendToBeDisconnected$,
...inputs,
});
// TypeScript needs this widening of the type to happen in a separate statement
const media$: Behavior<MemberMedia | null> = _media$;
const screenShares$ = scope.behavior(
memberTrack$(scope, media$, "screenShare").pipe(
trackBySource$(scope, member.tracks$, "screenShare").pipe(
map((track) => track !== undefined),
distinctUntilChanged(),
// Technically more than one screen share might be possible... our
@@ -121,12 +115,12 @@ export function createWrappedUserMedia(
},
(scope, _data$, key) => {
const id = `${inputs.id}:${key}`;
return local
? createLocalScreenShare(scope, { ...inputs, id, media$ })
return member.local
? createLocalScreenShare(scope, { ...inputs, id, member })
: createRemoteScreenShare(scope, {
...inputs,
id,
media$,
member,
pretendToBeDisconnected$,
});
},
src/tile/GridTile.test.tsx
+2 -3
View File
@@ -17,7 +17,6 @@ import {
mockRemoteMedia,
mockLocalMedia,
mockMediaDevices,
mockMemberMedia,
} from "../utils/test";
import { GridTileViewModel } from "../state/TileViewModel";
import { ReactionsSenderProvider } from "../reactions/useReactionsSender";
@@ -62,7 +61,7 @@ test("GridTile displays remote media", async () => {
rawDisplayName: "Alice",
getMxcAvatarUrl: () => "mxc://adfsg",
},
mockMemberMedia(),
[],
);
const { container } = render(
@@ -92,7 +91,7 @@ test("GridTile displays local media", async () => {
rawDisplayName: "Alice",
getMxcAvatarUrl: () => "mxc://adfsg",
},
mockMemberMedia({ local: true }),
[],
mockMediaDevices({}),
);
src/tile/SpotlightTile.test.tsx
+4 -5
View File
@@ -19,7 +19,6 @@ import {
mockLocalMedia,
mockRemoteMedia,
mockRemoteScreenShare,
mockMemberMedia,
} from "../utils/test";
import { SpotlightTileViewModel } from "../state/TileViewModel";
import { constant } from "../state/Behavior";
@@ -41,7 +40,7 @@ test("SpotlightTile is accessible", async () => {
rawDisplayName: "Alice",
getMxcAvatarUrl: () => "mxc://adfsg",
},
mockMemberMedia(),
[],
);
const vm2 = mockLocalMedia(
@@ -50,7 +49,7 @@ test("SpotlightTile is accessible", async () => {
rawDisplayName: "Bob",
getMxcAvatarUrl: () => "mxc://dlskf",
},
mockMemberMedia({ local: true }),
[],
mockMediaDevices({}),
);
@@ -98,7 +97,7 @@ test("Screen share volume UI is shown when screen share has audio", async () =>
const vm = mockRemoteScreenShare(
mockRtcMembership("@alice:example.org", "AAAA"),
{},
mockMemberMedia(),
[],
);
vi.spyOn(vm, "audioEnabled$", "get").mockReturnValue(constant(true));
@@ -136,7 +135,7 @@ test("Screen share volume UI is hidden when screen share has no audio", async ()
const vm = mockRemoteScreenShare(
mockRtcMembership("@alice:example.org", "AAAA"),
{},
mockMemberMedia(),
[],
);
vi.spyOn(vm, "audioEnabled$", "get").mockReturnValue(constant(false));
src/tracks/MemberAudioRenderer.test.tsx
+101
View File
@@ -0,0 +1,101 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { afterEach, beforeEach, expect, test, vi } from "vitest";
import { act, render, type RenderResult } from "@testing-library/react";
import { type Behavior, type RemoteRTCMember } from "@element-hq/matrixrtc-sdk";
import { BehaviorSubject } from "rxjs";
import { MediaDevicesContext } from "../MediaDevicesContext";
import { MemberAudioRenderer } from "./MemberAudioRenderer";
import {
mockAudioTrack,
mockMediaDevices,
mockRTCMember,
mockRtcMembership,
type MockTracks,
mockVideoTrack,
} from "../utils/test";
/** The renderer creates one for the earpiece graph; jsdom has none. */
class FakeAudioContext {
public createGain(): unknown {
return { gain: { value: 1 } };
}
public createStereoPanner(): unknown {
return { pan: { value: 0 } };
}
public async close(): Promise<void> {}
}
beforeEach(() => {
vi.stubGlobal("AudioContext", FakeAudioContext);
});
afterEach(() => {
vi.unstubAllGlobals();
});
function renderMembers(
members: { id: string; tracks$: Behavior<MockTracks> }[],
): RenderResult {
const rtcMembers: RemoteRTCMember[] = members.map(({ id, tracks$ }) =>
mockRTCMember(false, {
membership: mockRtcMembership(`@${id}:example.org`, "DEVICE"),
tracks$,
}),
);
return render(
<MediaDevicesContext.Provider value={mockMediaDevices({})}>
<MemberAudioRenderer members={rtcMembers} />
</MediaDevicesContext.Provider>,
);
}
test("plays nothing for a member whose media has not arrived", () => {
const { queryAllByTestId } = renderMembers([
{ id: "alice", tracks$: new BehaviorSubject<MockTracks>(null) },
]);
expect(queryAllByTestId("audio")).toHaveLength(0);
});
test("plays the microphone and the screen share audio, and nothing else", () => {
const microphone = mockAudioTrack({ id: "mic" });
const screenShareAudio = mockAudioTrack({
id: "share-audio",
source: "screenShareAudio",
});
const camera = mockVideoTrack({ id: "camera" });
const unknownAudio = mockAudioTrack({ id: "unknown", source: "unknown" });
const { queryAllByTestId } = renderMembers([
{
id: "alice",
tracks$: new BehaviorSubject<MockTracks>([
microphone,
camera,
screenShareAudio,
unknownAudio,
]),
},
]);
expect(queryAllByTestId("audio")).toHaveLength(2);
expect(microphone.attach).toHaveBeenCalledTimes(1);
expect(screenShareAudio.attach).toHaveBeenCalledTimes(1);
expect(camera.attach).not.toHaveBeenCalled();
expect(unknownAudio.attach).not.toHaveBeenCalled();
});
test("starts playing once the member's tracks arrive", () => {
const tracks$ = new BehaviorSubject<MockTracks>(null);
const { queryAllByTestId } = renderMembers([{ id: "alice", tracks$ }]);
expect(queryAllByTestId("audio")).toHaveLength(0);
const microphone = mockAudioTrack({ id: "mic" });
act(() => tracks$.next([microphone]));
expect(queryAllByTestId("audio")).toHaveLength(1);
expect(microphone.attach).toHaveBeenCalledTimes(1);
});
src/tracks/MemberAudioRenderer.tsx
+5 -22
View File
@@ -9,7 +9,6 @@ Please see LICENSE in the repository root for full details.
import {
type AudioMediaTrack,
type MediaSource,
type MemberMedia,
type RemoteRTCMember,
} from "@element-hq/matrixrtc-sdk";
import { type FC, useEffect, useMemo, useState } from "react";
@@ -102,7 +101,7 @@ export const MemberAudioRenderer: FC<MemberAudioRendererProps> = ({
{members.map((member) => (
<MemberAudio
key={member.id}
media$={member.media$}
tracks$={member.tracks$}
muted={muted}
plugins={plugins}
/>
@@ -116,32 +115,16 @@ interface AudioPlugins {
nodes: AudioNode[];
}
const MemberAudio: FC<{
media$: RemoteRTCMember["media$"];
muted?: boolean;
plugins: AudioPlugins;
}> = ({ media$, muted, plugins }) => {
const media = useBehavior(media$);
return (
media !== null && (
<MemberTracksAudio
tracks$={media.tracks$}
muted={muted}
plugins={plugins}
/>
)
);
};
/** The sources a call plays; anything published without one stays silent. */
const playedSources = new Set<MediaSource>(["microphone", "screenShareAudio"]);
const MemberTracksAudio: FC<{
tracks$: MemberMedia["tracks$"];
/** One member's audio tracks; nothing while its media has not arrived. */
const MemberAudio: FC<{
tracks$: RemoteRTCMember["tracks$"];
muted?: boolean;
plugins: AudioPlugins;
}> = ({ tracks$, muted, plugins }) => {
const tracks = useBehavior(tracks$);
const tracks = useBehavior(tracks$) ?? [];
return tracks
.filter(
(track): track is AudioMediaTrack =>
src/utils/test-viewmodel.ts
+2 -3
View File
@@ -28,7 +28,6 @@ import {
mockMatrixRoom,
mockMatrixRTCClient,
mockMediaDevices,
mockMemberMedia,
mockMuteStates,
MockRTCSession,
mockRTCMember,
@@ -172,7 +171,7 @@ export function getBasicCallViewModelEnvironment(
(m) => m.userId === localRtcMember.userId,
),
transportUrl: "https://example.com",
media$: constant(mockMemberMedia({ local: true })),
tracks$: constant([]),
})
: null,
),
@@ -188,7 +187,7 @@ export function getBasicCallViewModelEnvironment(
membership,
roomMember: members.find((m) => m.userId === membership.userId),
transportUrl: "https://example.com",
media$: constant(mockMemberMedia()),
tracks$: constant([]),
}),
),
),
src/utils/test.ts
+42 -23
View File
@@ -59,19 +59,20 @@ import {
import { Config } from "../config/Config";
import { type MediaDevices } from "../state/MediaDevices";
import {
type AudioMediaTrack,
type Behavior,
constant,
type DisconnectReason,
type EncryptionError,
type LocalAudioMediaTrack,
type LocalMemberMedia,
type LocalRTCMember,
type LocalVideoMediaTrack,
type MatrixRTCClient,
type MatrixRTCError,
type MemberMedia,
type PublishRequest,
type RemoteRTCMember,
type TransportMetadata,
type VideoMediaTrack,
} from "@element-hq/matrixrtc-sdk";
import { ObservableScope } from "../state/ObservableScope";
import { MuteStates } from "../state/MuteStates";
@@ -337,17 +338,8 @@ export const mockAudioTrack = (
...track,
}) as LocalAudioMediaTrack;
/** A member's media as the SDK hands it out, with nothing published unless given. */
export function mockMemberMedia(
media: Partial<Omit<LocalMemberMedia, "local">> & { local?: boolean } = {},
): LocalMemberMedia {
return {
local: false,
tracks$: constant([]),
encryptionError$: NEVER,
...media,
} as LocalMemberMedia;
}
/** The tracks a member has, or null while its media has not arrived. */
export type MockTracks = (AudioMediaTrack | VideoMediaTrack)[] | null;
/** Whoever publishes media, known by identity only. */
export function mockRemoteParticipant(
@@ -363,7 +355,7 @@ export function mockRemoteParticipant(
export function mockLocalMedia(
rtcMember: CallMembership,
roomMember: Partial<RoomMember>,
media: LocalMemberMedia | null,
tracks: MockTracks,
mediaDevices: MediaDevices,
): LocalUserMediaViewModel {
const member = mockMatrixRoomMember(rtcMember, roomMember);
@@ -371,7 +363,11 @@ export function mockLocalMedia(
id: "local",
userId: member.userId,
rtcBackendIdentity: rtcMember.rtcBackendIdentity,
media$: constant(media),
member: mockRTCMember(true, {
membership: rtcMember,
roomMember,
tracks$: constant(tracks),
}),
encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
focusUrl$: constant("https://rtc-example.org"),
mediaDevices,
@@ -385,7 +381,8 @@ export function mockLocalMedia(
export function mockRemoteMedia(
rtcMember: CallMembership,
roomMember: Partial<RoomMember>,
media: MemberMedia | null,
/** The tracks, or a behavior of them for a test that has them arrive. */
tracks: MockTracks | Behavior<MockTracks>,
{ focusUrl }: { focusUrl?: string } = { focusUrl: "https://rtc-example.org" },
): RemoteUserMediaViewModel {
const member = mockMatrixRoomMember(rtcMember, roomMember);
@@ -393,7 +390,12 @@ export function mockRemoteMedia(
id: "remote",
userId: member.userId,
rtcBackendIdentity: rtcMember.rtcBackendIdentity,
media$: constant(media),
member: mockRTCMember(false, {
membership: rtcMember,
roomMember,
tracks$:
tracks === null || Array.isArray(tracks) ? constant(tracks) : tracks,
}),
encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
focusUrl$: constant(focusUrl),
pretendToBeDisconnected$: constant(false),
@@ -407,13 +409,17 @@ export function mockRemoteMedia(
export function mockRemoteScreenShare(
rtcMember: CallMembership,
roomMember: Partial<RoomMember>,
media: MemberMedia | null,
tracks: MockTracks,
): RemoteScreenShareViewModel {
const member = mockMatrixRoomMember(rtcMember, roomMember);
return createRemoteScreenShare(testScope(), {
id: "screenshare",
userId: member.userId,
media$: constant(media),
member: mockRTCMember(false, {
membership: rtcMember,
roomMember,
tracks$: constant(tracks),
}),
encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
focusUrl$: constant("https://rtc-example.org"),
pretendToBeDisconnected$: constant(false),
@@ -425,7 +431,9 @@ export function mockRemoteScreenShare(
export interface MockMemberInputs {
membership: CallMembership;
roomMember?: Partial<RoomMember>;
media$?: Behavior<LocalMemberMedia | null>;
/** Null, the default, is a member whose media has not arrived. */
tracks$?: Behavior<MockTracks>;
encryptionError$?: Observable<EncryptionError>;
transportUrl?: string | undefined;
}
@@ -440,7 +448,13 @@ export function mockRTCMember(
): RemoteRTCMember;
export function mockRTCMember(
local: boolean,
{ membership, roomMember, media$, transportUrl }: MockMemberInputs,
{
membership,
roomMember,
tracks$ = constant(null),
encryptionError$ = NEVER,
transportUrl,
}: MockMemberInputs,
): LocalRTCMember | RemoteRTCMember {
const member = mockMatrixRoomMember(membership, roomMember);
const transport: TransportMetadata | undefined =
@@ -460,12 +474,16 @@ export function mockRTCMember(
displayName$: constant(member.rawDisplayName ?? membership.userId),
avatarUrl$: constant(member.getMxcAvatarUrl()),
transport$: constant(transport),
media$: media$ ?? constant(null),
encryptionError$,
};
if (!local) return { ...base, local: false };
if (!local) return { ...base, local: false, tracks$ };
return {
...base,
local: true,
// The mock tracks carry the local controls, whichever side they stand in for
tracks$: tracks$ as Behavior<
(LocalAudioMediaTrack | LocalVideoMediaTrack)[] | null
>,
publish: vi.fn(async (request: PublishRequest) =>
Promise.resolve(
request.source === "microphone"
@@ -522,6 +540,7 @@ export function mockMatrixRTCClient(
),
keyRotationSuppressed$,
connectedTransports$,
setAudioOutputDeviceId: vi.fn(async () => Promise.resolve()),
sendData: vi.fn(async () => Promise.resolve()),
data$: NEVER,
};