Matrix/element-call-Github
1
0
Fork 0
mirror of https://github.com/vector-im/element-call.git synced 2026-08-29 21:15:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/livekit' into livekit

Browse Source
This commit is contained in:
Ryan Emmick
2026-08-05 11:55:38 -05:00
parent 9ef3499418 eb65a28a75
commit f7cf674152
158 changed files with 7301 additions and 8286 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/state/AndroidControlledAudioOutput.test.ts
View File

@@ -253,7 +253,7 @@ describe("Test mappings", () => {
});
describe("Test select a device", () => {
it(`Switch to correct device `, () => {
it(`Switch to correct device`, () => {
withTestScheduler(({ cold, schedule, expectObservable, flush }) => {
const controlledAudioOutput = new AndroidControlledAudioOutput(
cold("a", { a: FULL_DEVICE_LIST }),

576
src/state/CallViewModel/CallNotificationLifecycle.test.ts
View File

@@ -5,17 +5,19 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { describe, it } from "vitest";
import { test } from "vitest";
import {
EventType,
type IEvent,
type IRoomTimelineData,
MatrixEvent,
type Room,
} from "matrix-js-sdk";
import { type RTCCallIntent } from "matrix-js-sdk/lib/matrixrtc";
import { map, mergeMap, NEVER, type Observable, startWith } from "rxjs";
import { withTestScheduler } from "../../utils/test";
import {
alice,
aliceRtcMember,
local,
localRtcMember,
@@ -23,9 +25,10 @@ import {
import {
type CallNotificationWrapper,
createCallNotificationLifecycle$,
type Props as CallNotificationLifecycleProps,
type RingAttempt,
} from "./CallNotificationLifecycle";
import { trackEpoch } from "../ObservableScope";
import { Epoch, trackEpoch } from "../ObservableScope";
import { constant } from "../Behavior";
function mockRingEvent(
eventId: string,
@@ -40,311 +43,272 @@ function mockRingEvent(
} as unknown as CallNotificationWrapper;
}
describe("waitForCallPickup$", () => {
it("unknown -> ringing -> timeout when notified and nobody joins", () => {
withTestScheduler(({ scope, expectObservable, behavior, hot }) => {
// No one ever joins (only local user)
const props: CallNotificationLifecycleProps = {
scope,
memberships$: scope.behavior(
behavior("a", { a: [] }).pipe(trackEpoch()),
),
sentCallNotification$: hot("10ms a", {
a: mockRingEvent("$notif1", 30),
}),
receivedDecline$: hot(""),
options: {
waitForCallPickup: true,
autoLeaveWhenOthersLeft: false,
},
localUser: localRtcMember,
};
const defaultProps = {
memberships$: constant(new Epoch([])),
matrixRoomMembers$: constant(new Map([[alice.userId, alice]])),
receivedDecline$: NEVER,
options: {
waitForCallPickup: true,
autoLeaveWhenOthersLeft: false,
},
localUser: localRtcMember,
};
const lifecycle = createCallNotificationLifecycle$(props);
function summarizeRingAttempts$(
ringAttempts$: Observable<RingAttempt>,
): Observable<
| { intent: RTCCallIntent; recipient: string }
| { outcome: "accept" | "decline" | "timeout" }
> {
return ringAttempts$.pipe(
mergeMap(({ intent, recipient, outcome$ }) =>
outcome$.pipe(
map((outcome) => ({ outcome })),
startWith({ intent, recipient }),
),
),
);
}
expectObservable(lifecycle.callPickupState$).toBe("a 9ms b 29ms c", {
a: "unknown",
b: "ringing",
c: "timeout",
});
test("no ring attempt when waitForCallPickup=false", () => {
withTestScheduler(({ scope, expectObservable, hot }) => {
const { ringAttempts$ } = createCallNotificationLifecycle$({
scope,
...defaultProps,
sentCallNotification$: hot("-a", {
a: mockRingEvent("$notif1", 30),
}),
options: { ...defaultProps.options, waitForCallPickup: false },
});
});
it("ringing -> success if someone joins before timeout is reached", () => {
withTestScheduler(({ scope, hot, behavior, expectObservable }) => {
// Someone joins at 20ms (both LiveKit participant and MatrixRTC member)
const props: CallNotificationLifecycleProps = {
scope,
memberships$: scope.behavior(
behavior("a 19ms b", {
a: [localRtcMember],
b: [localRtcMember, aliceRtcMember],
}).pipe(trackEpoch()),
),
sentCallNotification$: hot("5ms a", {
a: mockRingEvent("$notif2", 100),
}),
receivedDecline$: hot(""),
options: {
waitForCallPickup: true,
autoLeaveWhenOthersLeft: false,
},
localUser: localRtcMember,
};
const lifecycle = createCallNotificationLifecycle$(props);
expectObservable(lifecycle.callPickupState$).toBe("a 4ms b 14ms c", {
a: "unknown",
b: "ringing",
c: "success",
});
});
});
it("success when someone joins before we notify", () => {
withTestScheduler(({ scope, hot, behavior, expectObservable }) => {
// Someone joins at 20ms (both LiveKit participant and MatrixRTC member)
const props: CallNotificationLifecycleProps = {
scope,
memberships$: scope.behavior(
behavior("a 9ms b", {
a: [localRtcMember],
b: [localRtcMember, aliceRtcMember],
}).pipe(trackEpoch()),
),
sentCallNotification$: hot("20ms a", {
a: mockRingEvent("$notif2", 50),
}),
receivedDecline$: hot(""),
options: {
waitForCallPickup: true,
autoLeaveWhenOthersLeft: false,
},
localUser: localRtcMember,
};
const lifecycle = createCallNotificationLifecycle$(props);
expectObservable(lifecycle.callPickupState$).toBe("a 9ms b", {
a: "unknown",
b: "success",
});
});
});
it("notify without lifetime -> immediate timeout", () => {
withTestScheduler(({ scope, hot, behavior, expectObservable }) => {
// Someone joins at 20ms (both LiveKit participant and MatrixRTC member)
const props: CallNotificationLifecycleProps = {
scope,
memberships$: scope.behavior(
behavior("a", {
a: [localRtcMember],
}).pipe(trackEpoch()),
),
sentCallNotification$: hot("10ms a", {
a: mockRingEvent("$notif2", undefined),
}),
receivedDecline$: hot(""),
options: {
waitForCallPickup: true,
autoLeaveWhenOthersLeft: false,
},
localUser: localRtcMember,
};
const lifecycle = createCallNotificationLifecycle$(props);
expectObservable(lifecycle.callPickupState$).toBe("a 9ms b", {
a: "unknown",
b: "timeout",
});
});
});
it("stays null when waitForCallPickup=false", () => {
withTestScheduler(({ scope, hot, behavior, expectObservable }) => {
// Someone joins at 20ms (both LiveKit participant and MatrixRTC member)
const validProps: CallNotificationLifecycleProps = {
scope,
memberships$: scope.behavior(
behavior("a--b", {
a: [localRtcMember],
b: [localRtcMember, aliceRtcMember],
}).pipe(trackEpoch()),
),
sentCallNotification$: hot("10ms a", {
a: mockRingEvent("$notif5", 30),
}),
receivedDecline$: hot(""),
options: {
waitForCallPickup: true,
autoLeaveWhenOthersLeft: false,
},
localUser: localRtcMember,
};
const propsDeactivated = {
...validProps,
options: {
...validProps.options,
waitForCallPickup: false,
},
};
const lifecycle = createCallNotificationLifecycle$(propsDeactivated);
expectObservable(lifecycle.callPickupState$).toBe("n", {
n: null,
});
const lifecycleReference = createCallNotificationLifecycle$(validProps);
expectObservable(lifecycleReference.callPickupState$).toBe("u--s", {
u: "unknown",
s: "success",
});
});
});
it("decline before timeout window ends -> decline", () => {
withTestScheduler(({ scope, hot, behavior, expectObservable }) => {
// Someone joins at 20ms (both LiveKit participant and MatrixRTC member)
const props: CallNotificationLifecycleProps = {
scope,
memberships$: scope.behavior(
behavior("a", {
a: [localRtcMember],
}).pipe(trackEpoch()),
),
sentCallNotification$: hot("10ms a", {
a: mockRingEvent("$decl1", 50),
}),
receivedDecline$: hot("40ms d", {
d: [
new MatrixEvent({
type: EventType.RTCDecline,
content: {
"m.relates_to": {
rel_type: "m.reference",
event_id: "$decl1",
},
},
}),
{} as Room,
undefined,
false,
{} as IRoomTimelineData,
],
}),
options: {
waitForCallPickup: true,
autoLeaveWhenOthersLeft: false,
},
localUser: localRtcMember,
};
const lifecycle = createCallNotificationLifecycle$(props);
expectObservable(lifecycle.callPickupState$).toBe("a 9ms b 29ms e", {
a: "unknown",
b: "ringing",
e: "decline",
});
});
});
it("decline after timeout window ends -> stays timeout", () => {
withTestScheduler(({ scope, hot, behavior, expectObservable }) => {
// Someone joins at 20ms (both LiveKit participant and MatrixRTC member)
const props: CallNotificationLifecycleProps = {
scope,
memberships$: scope.behavior(
behavior("a", {
a: [localRtcMember],
}).pipe(trackEpoch()),
),
sentCallNotification$: hot("10ms a", {
a: mockRingEvent("$decl", 20),
}),
receivedDecline$: hot("40ms d", {
d: [
new MatrixEvent({
type: EventType.RTCDecline,
content: {
"m.relates_to": {
rel_type: "m.reference",
event_id: "$decl",
},
},
}),
{} as Room,
undefined,
false,
{} as IRoomTimelineData,
],
}),
options: {
waitForCallPickup: true,
autoLeaveWhenOthersLeft: false,
},
localUser: localRtcMember,
};
const lifecycle = createCallNotificationLifecycle$(props);
expectObservable(lifecycle.callPickupState$, "50ms !").toBe(
"a 9ms b 19ms e",
{
a: "unknown",
b: "ringing",
e: "timeout",
},
);
});
});
//
function testStaysRinging(
declineEvent: Partial<IEvent>,
expectDecline: boolean,
): void {
withTestScheduler(({ scope, hot, behavior, expectObservable }) => {
// Someone joins at 20ms (both LiveKit participant and MatrixRTC member)
const props: CallNotificationLifecycleProps = {
scope,
memberships$: scope.behavior(
behavior("a", {
a: [localRtcMember],
}).pipe(trackEpoch()),
),
sentCallNotification$: hot("10ms a", {
a: mockRingEvent("$right", 50),
}),
receivedDecline$: hot("20ms d", {
d: [
new MatrixEvent(declineEvent),
{} as Room,
undefined,
false,
{} as IRoomTimelineData,
],
}),
options: {
waitForCallPickup: true,
autoLeaveWhenOthersLeft: false,
},
localUser: localRtcMember,
};
const lifecycle = createCallNotificationLifecycle$(props);
const marbles = expectDecline ? "a 9ms b 9ms d" : "a 9ms b";
expectObservable(lifecycle.callPickupState$, "21ms !").toBe(marbles, {
a: "unknown",
b: "ringing",
d: "decline",
});
});
}
const reference = (refId?: string, sender?: string): Partial<IEvent> => ({
event_id: "$decline",
type: EventType.RTCDecline,
sender: sender ?? "@other:example.org",
content: {
"m.relates_to": {
rel_type: "m.reference",
event_id: refId ?? "$right",
},
},
});
it("decline reference works", () => {
testStaysRinging(reference(), true);
});
it("decline with wrong id is ignored (stays ringing)", () => {
testStaysRinging(reference("$wrong"), false);
});
it("decline with wrong id is ignored (stays ringing)", () => {
testStaysRinging(reference(undefined, local.userId), false);
expectObservable(ringAttempts$).toBe("");
});
});
test("no ring attempt when notification type is not ring", () => {
withTestScheduler(({ scope, expectObservable, hot }) => {
const { ringAttempts$ } = createCallNotificationLifecycle$({
scope,
...defaultProps,
sentCallNotification$: hot("-a", {
a: {
...mockRingEvent("$notif1", 30),
notification_type: "notification",
},
}),
});
expectObservable(ringAttempts$).toBe("");
});
});
test("no ring attempt if lifetime is missing", () => {
withTestScheduler(({ scope, expectObservable, hot }) => {
const { ringAttempts$ } = createCallNotificationLifecycle$({
scope,
...defaultProps,
sentCallNotification$: hot("-a", {
a: mockRingEvent("$notif1", undefined),
}),
});
expectObservable(ringAttempts$).toBe("");
});
});
test("ring attempt times out after nobody joins", () => {
withTestScheduler(({ scope, expectObservable, hot }) => {
const { ringAttempts$ } = createCallNotificationLifecycle$({
scope,
...defaultProps,
// No one ever joins (only local user)
memberships$: constant(new Epoch([])),
sentCallNotification$: hot("-a", {
a: mockRingEvent("$notif1", 30),
}),
});
expectObservable(summarizeRingAttempts$(ringAttempts$)).toBe("-a 29ms A", {
a: { intent: "audio", recipient: alice.userId },
A: { outcome: "timeout" },
});
});
});
test("ring attempt is accepted once recipient joins", () => {
withTestScheduler(({ scope, expectObservable, hot, behavior }) => {
const { ringAttempts$ } = createCallNotificationLifecycle$({
scope,
...defaultProps,
memberships$: scope.behavior(
behavior("a-b", { a: [], b: [aliceRtcMember] }).pipe(trackEpoch()),
),
sentCallNotification$: hot("-a", {
a: mockRingEvent("$notif1", 30),
}),
});
expectObservable(summarizeRingAttempts$(ringAttempts$)).toBe("-aA", {
a: { intent: "audio", recipient: alice.userId },
A: { outcome: "accept" },
});
});
});
test("ring attempt is immediately accepted if recipient is already joined", () => {
withTestScheduler(({ scope, expectObservable, hot }) => {
const { ringAttempts$ } = createCallNotificationLifecycle$({
scope,
...defaultProps,
memberships$: constant(new Epoch([aliceRtcMember])),
sentCallNotification$: hot("-a", {
a: mockRingEvent("$notif1", 30),
}),
});
expectObservable(summarizeRingAttempts$(ringAttempts$)).toBe("-(aA)", {
a: { intent: "audio", recipient: alice.userId },
A: { outcome: "accept" },
});
});
});
test("ring attempt can be declined", () => {
withTestScheduler(({ scope, expectObservable, hot }) => {
const { ringAttempts$ } = createCallNotificationLifecycle$({
scope,
...defaultProps,
sentCallNotification$: hot("-a", {
a: mockRingEvent("$notif1", 30),
}),
receivedDecline$: hot("--d", {
d: [
new MatrixEvent({
type: EventType.RTCDecline,
sender: alice.userId,
content: {
"m.relates_to": {
rel_type: "m.reference",
event_id: "$notif1",
},
},
}),
{} as Room,
undefined,
false,
{} as IRoomTimelineData,
],
}),
});
expectObservable(summarizeRingAttempts$(ringAttempts$)).toBe("-aA", {
a: { intent: "audio", recipient: alice.userId },
A: { outcome: "decline" },
});
});
});
test("ring attempt times out if recipient declines too late", () => {
withTestScheduler(({ scope, expectObservable, hot }) => {
const { ringAttempts$ } = createCallNotificationLifecycle$({
scope,
...defaultProps,
sentCallNotification$: hot("-a", {
a: mockRingEvent("$notif1", 30),
}),
receivedDecline$: hot("100ms d", {
d: [
new MatrixEvent({
type: EventType.RTCDecline,
sender: alice.userId,
content: {
"m.relates_to": {
rel_type: "m.reference",
event_id: "$notif1",
},
},
}),
{} as Room,
undefined,
false,
{} as IRoomTimelineData,
],
}),
});
expectObservable(summarizeRingAttempts$(ringAttempts$)).toBe("-a 29ms A", {
a: { intent: "audio", recipient: alice.userId },
A: { outcome: "timeout" },
});
});
});
test("decline event relating to wrong event is ignored (times out)", () => {
withTestScheduler(({ scope, expectObservable, hot }) => {
const { ringAttempts$ } = createCallNotificationLifecycle$({
scope,
...defaultProps,
sentCallNotification$: hot("-a", {
a: mockRingEvent("$notif1", 30),
}),
receivedDecline$: hot("--d", {
d: [
new MatrixEvent({
type: EventType.RTCDecline,
sender: alice.userId,
content: {
"m.relates_to": {
rel_type: "m.reference",
event_id: "$other", // <---- WRONG
},
},
}),
{} as Room,
undefined,
false,
{} as IRoomTimelineData,
],
}),
});
expectObservable(summarizeRingAttempts$(ringAttempts$)).toBe("-a 29ms A", {
a: { intent: "audio", recipient: alice.userId },
A: { outcome: "timeout" },
});
});
});
test("decline event from wrong sender is ignored (times out)", () => {
withTestScheduler(({ scope, expectObservable, hot }) => {
const { ringAttempts$ } = createCallNotificationLifecycle$({
scope,
...defaultProps,
sentCallNotification$: hot("-a", {
a: mockRingEvent("$notif1", 30),
}),
receivedDecline$: hot("--d", {
d: [
new MatrixEvent({
type: EventType.RTCDecline,
sender: local.userId, // <---- WRONG
content: {
"m.relates_to": {
rel_type: "m.reference",
event_id: "$notif1",
},
},
}),
{} as Room,
undefined,
false,
{} as IRoomTimelineData,
],
}),
});
expectObservable(summarizeRingAttempts$(ringAttempts$)).toBe("-a 29ms A", {
a: { intent: "audio", recipient: alice.userId },
A: { outcome: "timeout" },
});
});
});

208
src/state/CallViewModel/CallNotificationLifecycle.ts
View File

@@ -10,24 +10,22 @@ import {
type IRTCNotificationContent,
type MatrixRTCSession,
MatrixRTCSessionEvent,
type RTCCallIntent,
} from "matrix-js-sdk/lib/matrixrtc";
import {
combineLatest,
concat,
endWith,
filter,
fromEvent,
ignoreElements,
map,
merge,
NEVER,
type Observable,
of,
pairwise,
startWith,
switchMap,
takeUntil,
timer,
EMPTY,
race,
take,
} from "rxjs";
import {
type EventTimelineSetHandlerMap,
@@ -35,18 +33,26 @@ import {
type Room as MatrixRoom,
RoomEvent,
} from "matrix-js-sdk";
import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
import { type Behavior } from "../Behavior";
import { type Epoch, mapEpoch, type ObservableScope } from "../ObservableScope";
import { type Epoch, type ObservableScope } from "../ObservableScope";
import { type RoomMemberMap } from "./remoteMembers/MatrixMemberMetadata";
export type AutoLeaveReason = "allOthersLeft" | "timeout" | "decline";
export type CallPickupState =
| "unknown"
| "ringing"
| "timeout"
| "decline"
| "success"
| null;
export interface RingAttempt {
intent: RTCCallIntent;
/**
* The user ID of the recipient being rung.
*/
recipient: string;
/**
* The eventual outcome of the ringing attempt. (Emits a single value.)
*/
// TODO: Include a callback for attempting ringing again in case of a timeout
outcome$: Observable<"accept" | "decline" | "timeout">;
}
export type CallNotificationWrapper = {
event_id: string;
@@ -76,6 +82,7 @@ export function createReceivedDecline$(
export interface Props {
scope: ObservableScope;
memberships$: Behavior<Epoch<CallMembership[]>>;
matrixRoomMembers$: Behavior<RoomMemberMap>;
sentCallNotification$: Observable<CallNotificationWrapper | null>;
receivedDecline$: Observable<
Parameters<EventTimelineSetHandlerMap[RoomEvent.Timeline]>
@@ -84,34 +91,86 @@ export interface Props {
localUser: { deviceId: string; userId: string };
}
/**
* @returns two observables:
* `callPickupState$` The current call pickup state of the call.
* - "unknown": The client has not yet sent the notification event. We don't know if it will because it first needs to send its own membership.
* Then we can conclude if we were the first one to join or not.
* - "ringing": The call is ringing on other devices in this room (This client should give audiovisual feedback that this is happening).
* - "timeout": No-one picked up in the defined time this call should be ringing on others devices.
* The call failed. If desired this can be used as a trigger to exit the call.
* - "success": Someone else joined. The call is in a normal state. No audiovisual feedback.
* - null: EC is configured to never show any waiting for answer state.
*
* `autoLeave$` An observable that emits (null) when the call should be automatically left.
* - if options.autoLeaveWhenOthersLeft is set to true it emits when all others left.
* - if options.waitForCallPickup is set to true it emits if noone picked up the ring or if the ring got declined.
* - if options.autoLeaveWhenOthersLeft && options.waitForCallPickup is false it will never emit.
*
*/
export function createCallNotificationLifecycle$({
scope,
memberships$,
matrixRoomMembers$,
sentCallNotification$,
receivedDecline$,
options,
localUser,
}: Props): {
callPickupState$: Behavior<CallPickupState>;
/**
* An observable of attempts to ring the remote participant's devices.
*/
ringAttempts$: Observable<RingAttempt>;
/**
* An observable that emits when the call should be automatically left.
* - if options.autoLeaveWhenOthersLeft is set to true it emits when all others left.
* - if options.waitForCallPickup is set to true it emits if noone picked up the ring or if the ring got declined.
* - if options.autoLeaveWhenOthersLeft && options.waitForCallPickup is false it will never emit.
*/
autoLeave$: Observable<AutoLeaveReason>;
} {
const logger = rootLogger.getChild("[CallNotificationLifecycle]");
let ringAttempts$: Observable<RingAttempt> = NEVER;
if (options.waitForCallPickup)
ringAttempts$ = sentCallNotification$.pipe(
filter(
(
notificationEvent: CallNotificationWrapper | null,
): notificationEvent is CallNotificationWrapper =>
// only care about new events (legacy do not have decline pattern)
notificationEvent?.notification_type === "ring" &&
notificationEvent.lifetime > 0,
),
switchMap((notificationEvent) => {
// We assume that there is only one other user in the room when ringing
// TODO: Respect io.element.functional_members
const recipient = [...matrixRoomMembers$.value.keys()].find(
(userId) => userId !== localUser.userId,
);
if (recipient === undefined) {
logger.warn("No recipient for notification event; not ringing.");
return EMPTY;
}
// Ringing times out after lifetime ms have passed
const timeout$ = timer(notificationEvent.lifetime).pipe(
map(() => "timeout" as const),
);
// Call is accepted when the recipient joins
const accept$ = memberships$.pipe(
filter((ms) => ms.value.some((m) => m.userId === recipient)),
map(() => "accept" as const),
);
// Call is declined when we receive a decline event
const decline$ = receivedDecline$.pipe(
filter(
([event]) =>
event.getRelation()?.rel_type === "m.reference" &&
event.getRelation()?.event_id === notificationEvent.event_id &&
event.getSender() === recipient,
),
map(() => "decline" as const),
);
return of({
intent: notificationEvent["m.call.intent"] ?? "audio",
recipient,
outcome$: race(timeout$, accept$, decline$).pipe(
take(1),
// Make this observable 'hot' to avoid running multiple timers. This
// is not actually a resource leak since there will be at most one
// active ring attempt at any given time.
// eslint-disable-next-line element-call/no-observablescope-leak
scope.share,
),
});
}),
scope.share,
);
const allOthersLeft$ = memberships$.pipe(
pairwise(),
filter(
@@ -122,87 +181,18 @@ export function createCallNotificationLifecycle$({
map(() => {}),
);
/**
* Whether some Matrix user other than ourself is joined to the call.
*/
const someoneElseJoined$ = memberships$.pipe(
mapEpoch((ms) => ms.some((m) => m.userId !== localUser.userId)),
) as Behavior<Epoch<boolean>>;
/**
* The state of the current ringing attempt, if the RTC session is indeed
* ringing the remote participant's devices. Otherwise `null`.
*/
const remoteRingState$: Behavior<"ringing" | "timeout" | "decline" | null> =
scope.behavior(
sentCallNotification$.pipe(
filter(
(notificationEventArgs: CallNotificationWrapper | null) =>
// only care about new events (legacy do not have decline pattern)
notificationEventArgs?.notification_type === "ring",
),
map((e) => e as CallNotificationWrapper),
switchMap((notificationEvent) => {
const lifetimeMs = notificationEvent?.lifetime ?? 0;
return concat(
lifetimeMs === 0
? // If no lifetime, skip the ring state
of(null)
: // Ring until lifetime ms have passed
timer(lifetimeMs).pipe(
ignoreElements(),
startWith("ringing" as const),
),
// The notification lifetime has timed out, meaning ringing has likely
// stopped on all receiving clients.
of("timeout" as const),
// This makes sure we will not drop into the `endWith("decline" as const)` state
NEVER,
).pipe(
takeUntil(
receivedDecline$.pipe(
filter(
([event]) =>
event.getRelation()?.rel_type === "m.reference" &&
event.getRelation()?.event_id ===
notificationEvent.event_id &&
event.getSender() !== localUser.userId &&
callPickupState$.value !== "timeout",
),
),
),
endWith("decline" as const),
);
}),
),
null,
);
const callPickupState$ = scope.behavior(
options.waitForCallPickup === true
? combineLatest(
[someoneElseJoined$, remoteRingState$],
(someoneElseJoined, ring) => {
if (someoneElseJoined.value === true) {
return "success" as const;
}
// Show the ringing state of the most recent ringing attempt.
// as long as we have not yet sent an RTC notification event or noone else joined,
// ring will be null -> callPickupState$ = unknown.
return ring ?? ("unknown" as const);
},
)
: NEVER,
null,
);
const autoLeave$ = merge(
options.autoLeaveWhenOthersLeft === true
? allOthersLeft$.pipe(map(() => "allOthersLeft" as const))
: NEVER,
callPickupState$.pipe(
filter((state) => state === "timeout" || state === "decline"),
ringAttempts$.pipe(
switchMap(({ outcome$ }) =>
outcome$.pipe(
filter((outcome) => outcome === "timeout" || outcome === "decline"),
),
),
),
);
return { autoLeave$, callPickupState$ };
return { ringAttempts$, autoLeave$ };
}

221
src/state/CallViewModel/CallViewModel.test.ts
View File

@@ -6,7 +6,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { test, vi, onTestFinished, it, describe } from "vitest";
import { test, vi, onTestFinished, it, describe, expect } from "vitest";
import {
BehaviorSubject,
combineLatest,
@@ -22,6 +22,7 @@ import { SyncState } from "matrix-js-sdk";
import {
ConnectionState,
type LocalTrackPublication,
type Participant,
type RemoteParticipant,
} from "livekit-client";
import * as ComponentsCore from "@livekit/components-core";
@@ -143,13 +144,13 @@ export interface SpotlightExpandedLayoutSummary {
}
export interface OneOnOneLandscapeLayoutSummary {
type: "one-on-one-landscape";
type: "one-on-one-desktop";
spotlight: string;
pip: string;
}
export interface OneOnOnePortraitLayoutSummary {
type: "one-on-one-portrait";
type: "one-on-one-mobile";
spotlight: string[];
pip?: string;
pipSize: "sm" | "lg";
@@ -204,7 +205,7 @@ function summarizeLayout$(l$: Observable<Layout>): Observable<LayoutSummary> {
pip: pip?.id,
}),
);
case "one-on-one-landscape":
case "one-on-one-desktop":
return combineLatest(
[l.spotlight.media$, l.pip.media$],
(spotlight, pip) => ({
@@ -213,7 +214,7 @@ function summarizeLayout$(l$: Observable<Layout>): Observable<LayoutSummary> {
pip: pip.id,
}),
);
case "one-on-one-portrait":
case "one-on-one-mobile":
return combineLatest(
[
l.spotlight.media$,
@@ -326,8 +327,8 @@ describe.each([
},
(vm) => {
schedule(modeInputMarbles, {
s: () => vm.setGridMode("spotlight"),
g: () => vm.setGridMode("grid"),
s: () => vm.layoutSwitchVm$.value!.setLayout("spotlight"),
g: () => vm.layoutSwitchVm$.value!.setLayout("grid"),
});
expectObservable(summarizeLayout$(vm.layout$)).toBe(
@@ -436,7 +437,7 @@ describe.each([
expectedLayoutMarbles,
{
a: {
type: "one-on-one-landscape",
type: "one-on-one-desktop",
pip: `${localId}:0`,
spotlight: `${aliceId}:0`,
},
@@ -452,7 +453,7 @@ describe.each([
});
});
test("one-on-one portrait layout shows local tile when video is enabled", () => {
test("one-on-one mobile layout shows local tile when video is enabled", () => {
withTestScheduler(({ behavior, schedule, expectObservable }) => {
// Local participant enables their video, then disables it
const videoInputMarbles = " ny--n";
@@ -479,19 +480,19 @@ describe.each([
expectedLayoutMarbles,
{
a: {
type: "one-on-one-portrait",
type: "one-on-one-mobile",
spotlight: [`${aliceId}:0`],
pip: undefined,
pipSize: "lg",
},
b: {
type: "one-on-one-portrait",
type: "one-on-one-mobile",
spotlight: [`${aliceId}:0`],
pip: `${localId}:0`,
pipSize: "lg",
},
c: {
type: "one-on-one-portrait",
type: "one-on-one-mobile",
spotlight: [`${aliceId}:0`],
pip: `${localId}:0`,
pipSize: "sm",
@@ -503,21 +504,21 @@ describe.each([
});
});
test("one-on-one portrait layout shows name tags in room with 3 members", () => {
withTestScheduler(({ behavior, schedule, expectObservable }) => {
test("one-on-one mobile layout shows name tags in room with 3 members", () => {
withTestScheduler(({ expectObservable }) => {
withCallViewModel(
{
remoteParticipants$: constant([aliceParticipant]),
// Both Alice and Bob are with us in the room
roomMembers: [local, alice, bob],
rtcMembers$: constant([localRtcMember, aliceRtcMember]),
windowSize$: constant({ width: 380, height: 700 }), // Mobile phone in portrait
windowSize$: constant({ width: 380, height: 700 }), // Mobile phone
},
(vm) => {
// Uses one-on-one portrait layout
// Uses one-on-one mobile layout
expectObservable(summarizeLayout$(vm.layout$)).toBe("a", {
a: {
type: "one-on-one-portrait",
type: "one-on-one-mobile",
spotlight: [`${aliceId}:0`],
pip: undefined,
pipSize: "lg",
@@ -531,6 +532,65 @@ describe.each([
});
});
test("landscape mobile layouts show screen shares and group call participants", () => {
withTestScheduler(({ behavior, expectObservable }) => {
// Starts as a one-on-one call, then Alice shares her screen, then Bob
// joins, and finally Alice stops sharing her screen
const participantInputMarbles = " a--b";
const aliceSharingInputMarbles = " ny-n";
// Starts in one-on-one mobile layout, then goes to spotlight layout for
// the screen sharing and group call cases
const expectedLayoutMarbles = " ab-c";
// Whether the layout switch is visible. It should be hidden while in
// one-on-one layout.
const expectedLayoutSwitchMarbles = "ny--";
withCallViewModel(
{
remoteParticipants$: behavior(participantInputMarbles, {
a: [aliceParticipant],
b: [aliceParticipant, bobParticipant],
}),
roomMembers: [local, alice, bob],
rtcMembers$: behavior(participantInputMarbles, {
a: [localRtcMember, aliceRtcMember],
b: [localRtcMember, aliceRtcMember, bobRtcMember],
}),
sharingScreen: new Map([
[aliceParticipant, behavior(aliceSharingInputMarbles, yesNo)],
]),
windowSize$: constant({ width: 700, height: 380 }), // Mobile phone in landscape
},
(vm) => {
expectObservable(summarizeLayout$(vm.layout$)).toBe(
expectedLayoutMarbles,
{
a: {
type: "one-on-one-mobile",
spotlight: [`${aliceId}:0`],
pip: undefined,
pipSize: "sm",
},
b: {
type: "spotlight-landscape",
spotlight: [`${aliceId}:0:screen-share`],
grid: [`${localId}:0`, `${aliceId}:0`],
},
c: {
type: "spotlight-landscape",
spotlight: [`${aliceId}:0`],
grid: [`${localId}:0`, `${bobId}:0`],
},
},
);
expectObservable(
vm.layoutSwitchVm$.pipe(map((vm) => vm !== null)),
).toBe(expectedLayoutSwitchMarbles, yesNo);
},
);
});
});
test("participants stay in the same order unless to appear/disappear", () => {
withTestScheduler(({ behavior, schedule, expectObservable }) => {
const visibilityInputMarbles = "a";
@@ -705,14 +765,14 @@ describe.each([
{
a: {
// This is the expected one-on-one layout for a narrow window
type: "one-on-one-portrait",
type: "one-on-one-mobile",
spotlight: [`${aliceId}:0`],
pip: undefined,
pipSize: "lg",
},
b: {
// In a larger window, expect the normal one-on-one layout
type: "one-on-one-landscape",
// In a larger window, expect the one-on-one desktop layout
type: "one-on-one-desktop",
pip: `${localId}:0`,
spotlight: `${aliceId}:0`,
},
@@ -762,7 +822,9 @@ describe.each([
]),
},
(vm) => {
schedule(modeInputMarbles, { s: () => vm.setGridMode("spotlight") });
schedule(modeInputMarbles, {
s: () => vm.layoutSwitchVm$.value!.setLayout("spotlight"),
});
expectObservable(summarizeLayout$(vm.layout$)).toBe(
expectedLayoutMarbles,
@@ -899,6 +961,41 @@ describe.each([
},
);
// TODO add media to lk mocks
test("onPipMediaOrientationUpdate is called with the spotlight media orientation", () => {
// Set the spy before creating the view model so the initial call is captured
const onPipMediaOrientationUpdate = vi.fn();
window.controls.onPipMediaOrientationUpdate = onPipMediaOrientationUpdate;
onTestFinished(() => {
window.controls.onPipMediaOrientationUpdate = undefined;
});
withTestScheduler(({ behavior }) => {
// Alice starts as a regular participant, then shares her screen, then stops
const aliceSharingInputMarbles = "nyn";
withCallViewModel(
{
remoteParticipants$: constant([aliceParticipant]),
rtcMembers$: constant([localRtcMember, aliceRtcMember]),
sharingScreen: new Map([
[aliceParticipant, behavior(aliceSharingInputMarbles, yesNo)],
]),
},
() => {},
);
});
// Should be called exactly 3 times:
// 1. Initially with "portrait" (Alice is in spotlight as a user, default portrait orientation)
// 2. With "landscape" when Alice starts screen sharing (screen shares always use landscape)
// 3. With "portrait" again when Alice stops screen sharing and returns to user tile
expect(onPipMediaOrientationUpdate).toHaveBeenCalledTimes(3);
expect(onPipMediaOrientationUpdate).toHaveBeenNthCalledWith(1, "portrait");
expect(onPipMediaOrientationUpdate).toHaveBeenNthCalledWith(2, "landscape");
expect(onPipMediaOrientationUpdate).toHaveBeenNthCalledWith(3, "portrait");
});
test("PiP tile in expanded spotlight layout switches speakers without layout shifts", () => {
withTestScheduler(({ behavior, schedule, expectObservable }) => {
// Switch to spotlight immediately
@@ -933,7 +1030,7 @@ describe.each([
},
(vm) => {
schedule(modeInputMarbles, {
s: () => vm.setGridMode("spotlight"),
s: () => vm.layoutSwitchVm$.value!.setLayout("spotlight"),
});
schedule(expandInputMarbles, {
a: () => vm.toggleSpotlightExpanded$.value!(),
@@ -996,10 +1093,14 @@ describe.each([
a: [localRtcMember],
b: [localRtcMember, aliceRtcMember],
}),
videoEnabled: new Map<Participant, Behavior<boolean>>([
[localParticipant, constant(true)],
[aliceParticipant, constant(true)],
]),
},
(vm) => {
schedule(modeInputMarbles, {
s: () => vm.setGridMode("spotlight"),
s: () => vm.layoutSwitchVm$.value!.setLayout("spotlight"),
});
schedule(expandInputMarbles, {
a: () => vm.toggleSpotlightExpanded$.value!(),
@@ -1025,6 +1126,39 @@ describe.each([
});
});
test("expanded spotlight layout hides PiP tile in one-on-one voice call", () => {
withTestScheduler(({ schedule, expectObservable }) => {
withCallViewModel(
{
remoteParticipants$: constant([aliceParticipant]),
roomMembers: [local, alice],
rtcMembers$: constant([localRtcMember, aliceRtcMember]),
videoEnabled: new Map<Participant, Behavior<boolean>>([
[localParticipant, constant(false)],
[aliceParticipant, constant(false)],
]),
},
(vm) => {
schedule("s", {
s: () => vm.layoutSwitchVm$.value!.setLayout("spotlight"),
});
schedule("a", {
a: () => vm.toggleSpotlightExpanded$.value!(),
});
// Layout should show remote tile only
expectObservable(summarizeLayout$(vm.layout$)).toBe("a", {
a: {
type: "spotlight-expanded",
spotlight: [`${aliceId}:0`],
pip: undefined,
},
});
},
);
});
});
test("spotlight remembers whether it's expanded", () => {
withTestScheduler(({ schedule, expectObservable }) => {
// Start in spotlight mode, then switch to grid and back to spotlight a
@@ -1043,8 +1177,8 @@ describe.each([
},
(vm) => {
schedule(modeInputMarbles, {
s: () => vm.setGridMode("spotlight"),
g: () => vm.setGridMode("grid"),
s: () => vm.layoutSwitchVm$.value!.setLayout("spotlight"),
g: () => vm.layoutSwitchVm$.value!.setLayout("grid"),
});
schedule(expandInputMarbles, {
a: () => vm.toggleSpotlightExpanded$.value!(),
@@ -1061,7 +1195,7 @@ describe.each([
b: {
type: "spotlight-expanded",
spotlight: [`${aliceId}:0`],
pip: `${localId}:0`,
pip: undefined,
},
c: {
type: "grid",
@@ -1110,7 +1244,7 @@ describe.each([
]),
},
(vm) => {
vm.setGridMode("grid");
vm.layoutSwitchVm$.value!.setLayout("grid");
expectObservable(summarizeLayout$(vm.layout$)).toBe(
expectedLayoutMarbles,
{
@@ -1120,7 +1254,7 @@ describe.each([
grid: [`${localId}:0`],
},
b: {
type: "one-on-one-landscape",
type: "one-on-one-desktop",
pip: `${localId}:0`,
spotlight: `${aliceId}:0`,
},
@@ -1153,7 +1287,7 @@ describe.each([
}),
},
(vm) => {
vm.setGridMode("grid");
vm.layoutSwitchVm$.value!.setLayout("grid");
expectObservable(summarizeLayout$(vm.layout$)).toBe(
expectedLayoutMarbles,
{
@@ -1163,7 +1297,7 @@ describe.each([
grid: [`${localId}:0`],
},
b: {
type: "one-on-one-landscape",
type: "one-on-one-desktop",
pip: `${localId}:0`,
spotlight: `${aliceId}:0`,
},
@@ -1173,7 +1307,7 @@ describe.each([
grid: [`${localId}:0`, `${aliceId}:0`, `${daveId}:0`],
},
d: {
type: "one-on-one-landscape",
type: "one-on-one-desktop",
pip: `${localId}:0`,
spotlight: `${daveId}:0`,
},
@@ -1385,13 +1519,19 @@ describe.each([
},
});
// Should ring for 30ms and then time out
expectObservable(vm.ringing$).toBe("(ny) 26ms n", yesNo);
expectObservable(vm.ringingVm$).toBe("(ab)", {
a: null,
b: expect.objectContaining({
type: "ringing",
userId: alice.userId,
intent: "audio",
}),
});
// Layout should show placeholder media for the participant we're
// ringing the entire time (even once timed out)
expectObservable(summarizeLayout$(vm.layout$)).toBe("a", {
a: {
type: "one-on-one-landscape",
type: "one-on-one-desktop",
spotlight: `${localId}:0`,
pip: `ringing:${aliceUserId}`,
},
@@ -1425,17 +1565,24 @@ describe.each([
});
// Should ring until Alice joins
expectObservable(vm.ringing$).toBe("(ny) 17ms n", yesNo);
expectObservable(vm.ringingVm$).toBe("(ab) 17ms a", {
a: null,
b: expect.objectContaining({
type: "ringing",
userId: alice.userId,
intent: "audio",
}),
});
// Layout should show placeholder media for the participant we're
// ringing the entire time
expectObservable(summarizeLayout$(vm.layout$)).toBe("a 20ms b", {
a: {
type: "one-on-one-landscape",
type: "one-on-one-desktop",
spotlight: `${localId}:0`,
pip: `ringing:${aliceUserId}`,
},
b: {
type: "one-on-one-landscape",
type: "one-on-one-desktop",
spotlight: `${aliceId}:0`,
pip: `${localId}:0`,
},

462
src/state/CallViewModel/CallViewModel.ts
View File

@@ -29,7 +29,6 @@ import {
pairwise,
race,
scan,
skipWhile,
startWith,
Subject,
switchAll,
@@ -39,8 +38,9 @@ import {
tap,
throttleTime,
timer,
takeUntil,
} from "rxjs";
import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
import { type Logger, logger as rootLogger } from "matrix-js-sdk/lib/logger";
import {
MembershipManagerEvent,
type LivekitTransportConfig,
@@ -70,8 +70,8 @@ import { setPipEnabled$ } from "../../controls";
import { TileStore } from "../TileStore";
import { gridLikeLayout } from "../GridLikeLayout";
import { spotlightExpandedLayout } from "../SpotlightExpandedLayout";
import { oneOnOneLandscapeLayout } from "../OneOnOneLandscapeLayout";
import { oneOnOnePortraitLayout } from "../OneOnOnePortraitLayout";
import { oneOnOneDesktopLayout } from "../OneOnOneDesktopLayout";
import { oneOnOneMobileLayout } from "../OneOnOneMobileLayout";
import { pipLayout } from "../PipLayout";
import { type EncryptionSystem } from "../../e2ee/sharedKeyManagement";
import {
@@ -93,8 +93,8 @@ import {
type GridLayoutMedia,
type Layout,
type LayoutMedia,
type OneOnOneLandscapeLayoutMedia,
type OneOnOnePortraitLayoutMedia,
type OneOnOneDesktopLayoutMedia,
type OneOnOneMobileLayoutMedia,
type SpotlightExpandedLayoutMedia,
type SpotlightLandscapeLayoutMedia,
type SpotlightPortraitLayoutMedia,
@@ -125,10 +125,9 @@ import {
createConnectionManager$,
} from "./remoteMembers/ConnectionManager.ts";
import {
createMatrixLivekitMembers$,
createRemoteMatrixLivekitMembers$,
type LocalMatrixLivekitMember,
type RemoteMatrixLivekitMember,
type MatrixLivekitMember,
} from "./remoteMembers/MatrixLivekitMembers.ts";
import {
type AutoLeaveReason,
@@ -142,7 +141,10 @@ import {
} from "./remoteMembers/MatrixMemberMetadata.ts";
import { Publisher } from "./localMember/Publisher.ts";
import { type Connection } from "./remoteMembers/Connection.ts";
import { createLayoutModeSwitch } from "./LayoutSwitch.ts";
import {
type LayoutSwitchViewModel,
createLayoutSwitchViewModel,
} from "../LayoutSwitchViewModel.ts";
import {
createWrappedUserMedia,
type WrappedUserMediaViewModel,
@@ -156,8 +158,8 @@ import {
createRingingMedia,
type RingingMediaViewModel,
} from "../media/RingingMediaViewModel.ts";
import { type GridTileViewModel } from "../TileViewModel.ts";
const logger = rootLogger.getChild("[CallViewModel]");
//TODO
// Larger rename
// member,membership -> rtcMember
@@ -202,12 +204,11 @@ const smallMobileCallThreshold = 3;
// with the interface
const showFooterMs = 4000;
export type GridMode = "grid" | "spotlight";
export type WindowMode = "normal" | "narrow" | "flat" | "pip";
interface LayoutScanState {
layout: Layout | null;
overflowing: boolean;
tiles: TileStore;
}
@@ -230,9 +231,13 @@ export interface CallViewModel {
// lifecycle
autoLeave$: Observable<AutoLeaveReason>;
/**
* Whether we are ringing a call recipient.
* View model for info relating to ringing, timing out, calling back, etc.
*/
ringing$: Behavior<boolean>;
ringingVm$: Behavior<RingingMediaViewModel | null>;
/**
* Which visual element the ringing status should be shown in.
*/
ringingStatusLocation: "app_bar" | "tile";
/** Observable that emits when the user should leave the call (hangup pressed, widget action, error).
* THIS DOES NOT LEAVE THE CALL YET. The only way to leave the call (send the hangup event) is
* - by ending the scope
@@ -300,7 +305,7 @@ export interface CallViewModel {
/** Participants sorted by livekit room so they can be used in the audio rendering */
livekitRoomItems$: Behavior<LivekitRoomItem[]>;
/** use the layout instead, this is just for the sdk export. */
matrixLivekitMembers$: Behavior<RemoteMatrixLivekitMember[]>;
remoteMatrixLivekitMembers$: Behavior<RemoteMatrixLivekitMember[]>;
localMatrixLivekitMember$: Behavior<LocalMatrixLivekitMember | null>;
/** List of participants raising their hand */
handsRaised$: Behavior<Record<string, RaisedHandInfo>>;
@@ -345,8 +350,7 @@ export interface CallViewModel {
showNameTags$: Behavior<boolean>;
spotlightExpanded$: Behavior<boolean>;
toggleSpotlightExpanded$: Behavior<(() => void) | null>;
gridMode$: Behavior<GridMode>;
setGridMode: (value: GridMode) => void;
layoutSwitchVm$: Behavior<LayoutSwitchViewModel | null>;
// header/footer visibility
showHeader$: Behavior<boolean>;
@@ -356,6 +360,10 @@ export interface CallViewModel {
* and header as overlays.
*/
edgeToEdge$: Behavior<boolean>;
/**
* Whether the call layout is overflowing the interface (causing it to scroll).
*/
overflowing$: Behavior<boolean>;
settingsOpen$: Behavior<boolean>;
setSettingsOpen$: Behavior<(open: boolean) => void>;
@@ -407,6 +415,7 @@ export function createCallViewModel$(
reactionsSubject$: Observable<Record<string, ReactionInfo>>,
trackProcessorState$: Behavior<ProcessorState>,
): CallViewModel {
const logger = rootLogger.getChild("[CallViewModel]");
const client = matrixRoom.client;
const userId = client.getUserId();
const deviceId = client.getDeviceId();
@@ -416,6 +425,7 @@ export function createCallViewModel$(
const livekitKeyProvider = getE2eeKeyProvider(
options.encryptionSystem,
matrixRTCSession,
logger,
);
// matrix_rtc_mode in config.json overrides the user's Developer Settings choice.
// It is validated at config load (src/config/Config.ts) so the cast is safe.
@@ -525,13 +535,15 @@ export function createCallViewModel$(
ownMembershipIdentity,
});
const matrixLivekitMembers$: Behavior<Epoch<RemoteMatrixLivekitMember[]>> =
createMatrixLivekitMembers$({
scope: scope,
membershipsWithTransport$:
membershipsAndTransports.membershipsWithTransport$,
connectionManager: connectionManager,
});
const remoteMatrixLivekitMembers$: Behavior<
Epoch<RemoteMatrixLivekitMember[]>
> = createRemoteMatrixLivekitMembers$({
scope: scope,
membershipsWithTransport$:
membershipsAndTransports.membershipsWithTransport$,
connectionManager: connectionManager,
localUser: { userId, deviceId },
});
const connectOptions$ = scope.behavior(
matrixRTCMode$.pipe(
@@ -608,19 +620,12 @@ export function createCallViewModel$(
),
);
// ------------------------------------------------------------------------
// callLifecycle
// TODO if we are in "unknown" state we need a loading rendering (or empty screen)
// Otherwise it looks like we already connected and only than the ringing starts which is weird.
const { callPickupState$, autoLeave$ } = createCallNotificationLifecycle$({
scope: scope,
memberships$: memberships$,
sentCallNotification$: createSentCallNotification$(scope, matrixRTCSession),
receivedDecline$: createReceivedDecline$(matrixRoom),
options: options,
localUser: { userId: userId, deviceId: deviceId },
});
const matrixLivekitMembers$ = scope.behavior(
combineLatest(
[localMatrixLivekitMember$, remoteMatrixLivekitMembers$],
(local, remote) => [...(local === null ? [] : [local]), ...remote.value],
),
);
// ------------------------------------------------------------------------
// matrixMemberMetadataStore
@@ -632,11 +637,26 @@ export function createCallViewModel$(
matrixRoomMembers$,
);
// ------------------------------------------------------------------------
// callLifecycle
// TODO if we are in "unknown" state we need a loading rendering (or empty screen)
// Otherwise it looks like we already connected and only than the ringing starts which is weird.
const { ringAttempts$, autoLeave$ } = createCallNotificationLifecycle$({
scope,
memberships$,
matrixRoomMembers$,
sentCallNotification$: createSentCallNotification$(scope, matrixRTCSession),
receivedDecline$: createReceivedDecline$(matrixRoom),
options,
localUser: { userId, deviceId },
});
const allConnections$ = scope.behavior(
connectionManager.connectionManagerData$.pipe(map((d) => d.value)),
);
const livekitRoomItems$ = scope.behavior(
matrixLivekitMembers$.pipe(
remoteMatrixLivekitMembers$.pipe(
switchMap((members) => {
const a$ = combineLatest(
members.value.map((member) =>
@@ -702,43 +722,20 @@ export function createCallViewModel$(
* List of user media (camera feeds) that we want tiles for.
*/
const userMedia$ = scope.behavior<WrappedUserMediaViewModel[]>(
combineLatest([
localMatrixLivekitMember$,
matrixLivekitMembers$,
duplicateTiles.value$,
]).pipe(
combineLatest([matrixLivekitMembers$, duplicateTiles.value$]).pipe(
// Generate a collection of user media from the list of expected (whether
// present or missing) LiveKit participants.
generateItems(
"CallViewModel userMedia$",
function* ([
localMatrixLivekitMember,
matrixLivekitMembers,
duplicateTiles,
]) {
const computeMediaId = (m: MatrixLivekitMember): string =>
`${m.userId}:${m.membership$.value.deviceId}`;
const localUserMediaId = localMatrixLivekitMember
? computeMediaId(localMatrixLivekitMember)
: undefined;
const localAsArray = localMatrixLivekitMember
? [localMatrixLivekitMember]
: [];
const remoteWithoutLocal = matrixLivekitMembers.value.filter(
(m) => computeMediaId(m) !== localUserMediaId,
);
const allMatrixLivekitMembers = [
...localAsArray,
...remoteWithoutLocal,
];
for (const matrixLivekitMember of allMatrixLivekitMembers) {
const { userId, participant, connection$, membership$ } =
matrixLivekitMember;
const rtcId = membership$.value.rtcBackendIdentity; // rtcBackendIdentity
const mediaId = computeMediaId(matrixLivekitMember);
function* ([members, duplicateTiles]) {
for (const {
userId,
participant,
connection$,
membership$,
} of members) {
const rtcId = membership$.value.rtcBackendIdentity;
const mediaId = `${userId}:${membership$.value.deviceId}`;
for (let dup = 0; dup < 1 + duplicateTiles; dup++) {
yield {
keys: [dup, mediaId, userId, participant, connection$, rtcId],
@@ -764,11 +761,13 @@ export function createCallViewModel$(
pretendToBeDisconnected$: localMembership.reconnecting$,
displayName$: scope.behavior(
matrixMemberMetadataStore
.createDisplayNameBehavior$(userId)
.createDisplayNameBehavior$(scope, userId)
.pipe(map((name) => name ?? userId)),
),
mxcAvatarUrl$:
matrixMemberMetadataStore.createAvatarUrlBehavior$(userId),
mxcAvatarUrl$: matrixMemberMetadataStore.createAvatarUrlBehavior$(
scope,
userId,
),
handRaised$: scope.behavior(
handsRaised$.pipe(map((v) => v[mediaId]?.time ?? null)),
),
@@ -780,49 +779,42 @@ export function createCallViewModel$(
),
);
const ringingMedia$ = scope.behavior<RingingMediaViewModel[]>(
combineLatest([userMedia$, matrixRoomMembers$, callPickupState$]).pipe(
generateItems(
"CallViewModel ringingMedia$",
function* ([userMedia, roomMembers, callPickupState]) {
if (
callPickupState === "ringing" ||
callPickupState === "timeout" ||
callPickupState === "decline"
) {
// TODO: Respect io.element.functional_members
for (const member of roomMembers.values()) {
if (!userMedia.some((vm) => vm.userId === member.userId))
yield {
keys: [member.userId],
data: callPickupState,
};
}
}
},
(scope, pickupState$, userId) =>
createRingingMedia({
id: `ringing:${userId}`,
userId,
displayName$: scope.behavior(
matrixRoomMembers$.pipe(
map((members) => members.get(userId)?.rawDisplayName || userId),
),
),
mxcAvatarUrl$:
matrixMemberMetadataStore.createAvatarUrlBehavior$(userId),
pickupState$,
muteStates,
}),
const ringingMedia$ = scope.behavior<RingingMediaViewModel | null>(
ringAttempts$.pipe(
switchMap(({ intent, recipient, outcome$ }) =>
outcome$.pipe(
startWith("ringing" as const),
generateItems(
"CallViewModel ringingMedia$",
function* (pickupState) {
if (pickupState !== "accept")
yield { keys: [intent, recipient], data: pickupState };
},
(scope, pickupState$, intent, userId) =>
createRingingMedia({
id: `ringing:${userId}`,
userId,
displayName$: scope.behavior(
matrixRoomMembers$.pipe(
map(
(members) =>
members.get(userId)?.rawDisplayName || userId,
),
),
),
mxcAvatarUrl$:
matrixMemberMetadataStore.createAvatarUrlBehavior$(
scope,
userId,
),
pickupState$,
intent,
}),
),
map(([media]) => media ?? null),
),
),
distinctUntilChanged(shallowEquals),
tap((ringingMedia) => {
if (ringingMedia.length > 1)
// Warn that UI may do something unexpected in this case
logger.warn(
`Ringing more than one participant is not supported (ringing ${ringingMedia.map((vm) => vm.userId).join(", ")})`,
);
}),
startWith(null),
),
);
@@ -861,14 +853,10 @@ export function createCallViewModel$(
* multiple devices.
*/
const participantCount$ = scope.behavior(
matrixLivekitMembers$.pipe(map((ms) => ms.value.length)),
matrixLivekitMembers$.pipe(map((ms) => ms.length)),
);
const leaveSoundEffect$ = combineLatest([callPickupState$, userMedia$]).pipe(
// Until the call is successful, do not play a leave sound.
// If callPickupState$ is null, then we always play the sound as it will not conflict with a decline sound.
skipWhile(([c]) => c !== null && c !== "success"),
map(([, userMedia]) => userMedia),
const leaveSoundEffect$ = userMedia$.pipe(
pairwise(),
filter(
([prev, current]) =>
@@ -877,6 +865,9 @@ export function createCallViewModel$(
),
map(() => {}),
throttleTime(THROTTLE_SOUND_EFFECT_MS),
// Avoid doubling up on any auto-leave sounds (e.g. the decline sound),
// which are handled elsewhere
takeUntil(autoLeave$),
);
const userHangup$ = new Subject<void>();
@@ -955,8 +946,8 @@ export function createCallViewModel$(
);
/**
* Local user media suitable for displaying in a PiP (undefined if not found
* or if user prefers to not see themselves).
* Local user media suitable for displaying in a PiP (undefined if not found,
* video is muted, or if user prefers to not see themselves).
*/
const localUserMediaForPip$ = scope.behavior<
LocalUserMediaViewModel | undefined
@@ -968,8 +959,10 @@ export function createCallViewModel$(
m.type === "user" && m.local,
);
if (!localUserMedia) return of(undefined);
return localUserMedia.alwaysShow$.pipe(
map((alwaysShow) => (alwaysShow ? localUserMedia : undefined)),
return combineLatest(
[localUserMedia.videoEnabled$, localUserMedia.alwaysShow$],
(videoEnabled, alwaysShow) =>
videoEnabled && alwaysShow ? localUserMedia : undefined,
);
}),
),
@@ -981,8 +974,8 @@ export function createCallViewModel$(
}>(
ringingMedia$.pipe(
switchMap((ringingMedia) => {
if (ringingMedia.length > 0)
return of({ spotlight: ringingMedia, pip$: localUserMediaForPip$ });
if (ringingMedia !== null)
return of({ spotlight: [ringingMedia], pip$: localUserMediaForPip$ });
return screenShares$.pipe(
switchMap((screenShares) => {
@@ -1061,7 +1054,7 @@ export function createCallViewModel$(
spotlightExpandedToggle$,
);
const { setGridMode, gridMode$ } = createLayoutModeSwitch(
const layoutSwitchVm = createLayoutSwitchViewModel(
scope,
windowMode$,
hasRemoteScreenShares$,
@@ -1113,60 +1106,60 @@ export function createCallViewModel$(
),
);
const oneOnOneLayoutMedia$: Observable<{
const oneOnOneLayoutMedia$: Behavior<{
local: LocalUserMediaViewModel;
remote: UserMediaViewModel | RingingMediaViewModel;
} | null> = combineLatest([userMedia$, screenShares$]).pipe(
switchMap(([userMedia, screenShares]) => {
// One-on-one layout only supports 2 user media, no screen shares
if (userMedia.length <= 2 && screenShares.length === 0) {
const local = userMedia.find(
(vm): vm is WrappedUserMediaViewModel & LocalUserMediaViewModel =>
vm.type === "user" && vm.local,
);
if (local !== undefined) {
const remote = userMedia.find(
(vm): vm is WrappedUserMediaViewModel & RemoteUserMediaViewModel =>
vm.type === "user" && !vm.local,
} | null> = scope.behavior(
combineLatest([userMedia$, screenShares$]).pipe(
switchMap(([userMedia, screenShares]) => {
// One-on-one layout only supports 2 user media, no screen shares
if (userMedia.length <= 2 && screenShares.length === 0) {
const local = userMedia.find(
(vm): vm is WrappedUserMediaViewModel & LocalUserMediaViewModel =>
vm.type === "user" && vm.local,
);
if (remote !== undefined) return of({ local, remote });
// If there's no other user media in the call (could still happen in
// this branch due to the duplicate tiles option), we could possibly
// show ringing media instead
if (userMedia.length === 1)
return ringingMedia$.pipe(
map((ringingMedia) => {
return ringingMedia.length === 1
? {
local,
remote: ringingMedia[0],
}
: null;
}),
if (local !== undefined) {
const remote = userMedia.find(
(
vm,
): vm is WrappedUserMediaViewModel & RemoteUserMediaViewModel =>
vm.type === "user" && !vm.local,
);
}
}
return of(null);
}),
if (remote !== undefined) return of({ local, remote });
// If there's no other user media in the call (could still happen in
// this branch due to the duplicate tiles option), we could possibly
// show ringing media instead
if (userMedia.length === 1)
return ringingMedia$.pipe(
map(
(ringingMedia) =>
ringingMedia && { local, remote: ringingMedia },
),
);
}
}
return of(null);
}),
),
);
const oneOnOneLandscapeLayoutMedia$: Observable<OneOnOneLandscapeLayoutMedia | null> =
const oneOnOneDesktopLayoutMedia$: Observable<OneOnOneDesktopLayoutMedia | null> =
oneOnOneLayoutMedia$.pipe(
map((media) => {
if (media === null) return null;
return media.remote.type === "ringing"
? {
type: "one-on-one-landscape" as const,
type: "one-on-one-desktop" as const,
edgeToEdge: false,
spotlight: media.local,
pip: media.remote,
}
: {
type: "one-on-one-landscape" as const,
type: "one-on-one-desktop" as const,
edgeToEdge: false,
spotlight: media.remote,
pip: media.local,
@@ -1174,13 +1167,13 @@ export function createCallViewModel$(
}),
);
const oneOnOnePortraitLayoutMedia$: Observable<OneOnOnePortraitLayoutMedia | null> =
const oneOnOneMobileLayoutMedia$: Observable<OneOnOneMobileLayoutMedia | null> =
oneOnOneLayoutMedia$.pipe(
switchMap((media) => {
if (media === null) return of(null);
return media.local.videoEnabled$.pipe(
map((videoEnabled) => ({
type: "one-on-one-portrait" as const,
type: "one-on-one-mobile" as const,
edgeToEdge: true as const,
spotlight: media.remote,
pip: videoEnabled ? media.local : undefined,
@@ -1197,6 +1190,33 @@ export function createCallViewModel$(
})),
);
spotlight$
.pipe(
switchMap((media) => {
let layout;
const pipMedia = media[0];
if (pipMedia === undefined) return of(undefined);
switch (pipMedia.type) {
case "user":
layout = pipMedia.videoOrientation$;
break;
case "ringing":
layout = of("landscape" as const);
break;
case "screen share":
layout = of("landscape" as const);
break;
}
return layout;
}),
scope.bind(),
)
.subscribe((orientation) => {
if (orientation === undefined) return;
logger.info("controls api pip orientation updated:", orientation);
window.controls.onPipMediaOrientationUpdate?.(orientation);
});
/**
* The media to be used to produce a layout.
*/
@@ -1205,11 +1225,11 @@ export function createCallViewModel$(
switchMap((windowMode) => {
switch (windowMode) {
case "normal":
return gridMode$.pipe(
switchMap((gridMode) => {
switch (gridMode) {
return layoutSwitchVm.layout$.pipe(
switchMap((layout) => {
switch (layout) {
case "grid":
return oneOnOneLandscapeLayoutMedia$.pipe(
return oneOnOneDesktopLayoutMedia$.pipe(
switchMap((oneOnOne) =>
oneOnOne === null ? gridLayoutMedia$ : of(oneOnOne),
),
@@ -1226,7 +1246,7 @@ export function createCallViewModel$(
}),
);
case "narrow":
return oneOnOnePortraitLayoutMedia$.pipe(
return oneOnOneMobileLayoutMedia$.pipe(
switchMap((oneOnOne) =>
oneOnOne === null
? combineLatest([grid$, spotlight$], (grid, spotlight) =>
@@ -1239,17 +1259,23 @@ export function createCallViewModel$(
),
);
case "flat":
return gridMode$.pipe(
switchMap((gridMode) => {
switch (gridMode) {
case "grid":
// Yes, grid mode actually gets you a "spotlight" layout in
// this window mode.
return spotlightLandscapeLayoutMedia$(true);
case "spotlight":
return spotlightExpandedLayoutMedia$(true);
}
}),
return oneOnOneMobileLayoutMedia$.pipe(
switchMap((oneOnOne) =>
oneOnOne === null
? layoutSwitchVm.layout$.pipe(
switchMap((layout) => {
switch (layout) {
case "grid":
// Yes, grid mode actually gets you a "spotlight" layout in
// this window mode.
return spotlightLandscapeLayoutMedia$(true);
case "spotlight":
return spotlightExpandedLayoutMedia$(true);
}
}),
)
: of(oneOnOne),
),
);
case "pip":
return pipLayoutMedia$;
@@ -1277,8 +1303,8 @@ export function createCallViewModel$(
// indicators. And in one-on-one layout there's no question as to who is
// speaking.
case "spotlight-expanded":
case "one-on-one-landscape":
case "one-on-one-portrait":
case "one-on-one-desktop":
case "one-on-one-mobile":
return false;
default:
return true;
@@ -1290,7 +1316,7 @@ export function createCallViewModel$(
const showNameTags$ = scope.behavior<boolean>(
layoutMedia$.pipe(
switchMap((l) =>
l.type === "pip" || l.type === "one-on-one-portrait"
l.type === "pip" || l.type === "one-on-one-mobile"
? matrixRoomMembers$.pipe(
map(
(members) =>
@@ -1334,6 +1360,22 @@ export function createCallViewModel$(
layoutMedia$.pipe(map(({ edgeToEdge }) => edgeToEdge)),
);
// Only show the layout switch in cases where it has an effect on the layout
const showLayoutSwitch$ = windowMode$.pipe(
switchMap((windowMode) => {
switch (windowMode) {
case "normal":
return of(true);
case "flat":
return oneOnOneLayoutMedia$.pipe(
map((oneOnOne) => oneOnOne === null),
);
default:
return of(false);
}
}),
);
const screenTap$ = new Subject<void>();
const controlsTap$ = new Subject<void>();
const screenHover$ = new Subject<void>();
@@ -1415,7 +1457,7 @@ export function createCallViewModel$(
windowMode$.pipe(
switchMap((mode) => {
// In small windows the header would be too obstructive
if (mode === "pip" || mode === "flat") return of(false);
if (mode === "pip") return of(false);
// In edge-to-edge layouts, couple the visibility of the header
// to that of the footer
return edgeToEdge$.pipe(
@@ -1455,7 +1497,7 @@ export function createCallViewModel$(
// There is a cyclical dependency here: the layout algorithms want to know
// which tiles are on screen, but to know which tiles are on screen we have to
// first render a layout. To deal with this we assume initially that no tiles
// first render a layout. To deal with this we assume initially that all tiles
// are visible, and loop the data back into the layouts with a Subject.
const visibleTiles$ = new Subject<number>();
const setVisibleTiles = (value: number): void => visibleTiles$.next(value);
@@ -1463,7 +1505,7 @@ export function createCallViewModel$(
const layoutInternals$ = scope.behavior<LayoutScanState & { layout: Layout }>(
combineLatest([
layoutMedia$,
visibleTiles$.pipe(startWith(0), distinctUntilChanged()),
visibleTiles$.pipe(startWith(Infinity), distinctUntilChanged()),
]).pipe(
scan<
[LayoutMedia, number],
@@ -1473,6 +1515,8 @@ export function createCallViewModel$(
({ tiles: prevTiles }, [media, visibleTiles]) => {
let layout: Layout;
let newTiles: TileStore;
let pip: GridTileViewModel | undefined;
let overflowing = false;
switch (media.type) {
case "grid":
case "spotlight-landscape":
@@ -1484,6 +1528,7 @@ export function createCallViewModel$(
setVisibleTiles,
prevTiles,
);
overflowing = newTiles.gridTiles.length > visibleTiles;
break;
case "spotlight-expanded":
[layout, newTiles] = spotlightExpandedLayout(
@@ -1492,29 +1537,35 @@ export function createCallViewModel$(
prevTiles,
);
break;
case "one-on-one-landscape":
[layout, newTiles] = oneOnOneLandscapeLayout(
case "one-on-one-desktop":
[layout, newTiles] = oneOnOneDesktopLayout(
media,
landscapePipAlignment$,
prevTiles,
);
pip = layout.pip;
break;
case "one-on-one-portrait":
[layout, newTiles] = oneOnOnePortraitLayout(
case "one-on-one-mobile":
[layout, newTiles] = oneOnOneMobileLayout(
media,
portraitPipSize$,
portraitPipAlignment$,
prevTiles,
);
pip = layout.pip;
break;
case "pip":
[layout, newTiles] = pipLayout(media, prevTiles);
break;
}
return { layout, tiles: newTiles };
for (const tile of newTiles.gridTiles) {
tile.setShowOutline(tile === pip);
}
return { layout, overflowing, tiles: newTiles };
},
{ layout: null, tiles: TileStore.empty() },
{ layout: null, overflowing: false, tiles: TileStore.empty() },
),
),
);
@@ -1526,6 +1577,10 @@ export function createCallViewModel$(
layoutInternals$.pipe(map(({ layout }) => layout)),
);
const overflowing$ = scope.behavior<boolean>(
layoutInternals$.pipe(map(({ overflowing }) => overflowing)),
);
/**
* The current generation of the tile store, exposed for debugging purposes.
*/
@@ -1695,9 +1750,9 @@ export function createCallViewModel$(
return {
autoLeave$: autoLeave$,
ringing$: scope.behavior(
callPickupState$.pipe(map((state) => state === "ringing")),
),
ringingVm$: ringingMedia$,
ringingStatusLocation:
urlParams.header === HeaderStyle.AppBar ? "app_bar" : "tile",
leave$: leave$,
hangup: (): void => userHangup$.next(),
join: localMembership.requestJoinAndPublish,
@@ -1738,12 +1793,13 @@ export function createCallViewModel$(
spotlightExpanded$: spotlightExpanded$,
toggleSpotlightExpanded$: toggleSpotlightExpanded$,
gridMode$: gridMode$,
setGridMode: setGridMode,
layoutSwitchVm$: scope.behavior(
showLayoutSwitch$.pipe(map((show) => (show ? layoutSwitchVm : null))),
),
layout$: layout$,
localMatrixLivekitMember$,
matrixLivekitMembers$: scope.behavior(
matrixLivekitMembers$.pipe(
remoteMatrixLivekitMembers$: scope.behavior(
remoteMatrixLivekitMembers$.pipe(
map((members) => members.value),
tap((v) => {
const listForLogs = v
@@ -1767,6 +1823,7 @@ export function createCallViewModel$(
settingsOpen$: settingsOpen$,
setSettingsOpen$: setSettingsOpen$,
edgeToEdge$,
overflowing$,
earpieceMode$: earpieceMode$,
audioOutputSwitcher$: audioOutputSwitcher$,
reconnecting$: localMembership.reconnecting$,
@@ -1778,6 +1835,7 @@ export function createCallViewModel$(
function getE2eeKeyProvider(
e2eeSystem: EncryptionSystem,
rtcSession: MatrixRTCSession,
logger: Logger,
): BaseKeyProvider | undefined {
if (e2eeSystem.kind === E2eeType.NONE) return undefined;

93
src/state/CallViewModel/LayoutSwitch.ts
View File

@@ -1,93 +0,0 @@
/*
Copyright 2025 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
combineLatest,
map,
Subject,
startWith,
skipWhile,
switchMap,
} from "rxjs";
import { type GridMode, type WindowMode } from "./CallViewModel.ts";
import { constant, type Behavior } from "../Behavior.ts";
import { type ObservableScope } from "../ObservableScope.ts";
/**
* Creates a layout mode switch that allows switching between grid and spotlight modes.
* The actual layout mode might switch automatically to spotlight if there is a
* remote screen share active or if the window mode is flat.
*
* @param scope - The observable scope to manage subscriptions.
* @param windowMode$ - The current window mode.
* @param hasRemoteScreenShares$ - A behavior indicating if there are remote screen shares active.
*/
export function createLayoutModeSwitch(
scope: ObservableScope,
windowMode$: Behavior<WindowMode>,
hasRemoteScreenShares$: Behavior<boolean>,
): {
gridMode$: Behavior<GridMode>;
setGridMode: (value: GridMode) => void;
} {
const userSelection$ = new Subject<GridMode>();
// Callback to set the grid mode desired by the user.
// Notice that this is only a preference, the actual grid mode can be overridden
// if there is a remote screen share active.
const setGridMode = (value: GridMode): void => userSelection$.next(value);
/**
* The natural grid mode - the mode that the grid would prefer to be in,
* not accounting for the user's manual selections.
*/
const naturalGridMode$ = scope.behavior<GridMode>(
combineLatest(
[hasRemoteScreenShares$, windowMode$],
(hasRemoteScreenShares, windowMode) =>
// When there are screen shares or the window is flat (as with a phone
// in landscape orientation), spotlight is a better experience.
// We want screen shares to be big and readable, and we want flipping
// your phone into landscape to be a quick way of maximising the
// spotlight tile.
hasRemoteScreenShares || windowMode === "flat" ? "spotlight" : "grid",
),
);
/**
* The layout mode of the media tile grid.
*/
const gridMode$ = scope.behavior<GridMode>(
// Whenever the user makes a selection, we enter a new mode of behavior:
userSelection$.pipe(
map((selection) => {
if (selection === "grid")
// The user has selected grid mode. Start by respecting their choice,
// but then follow the natural mode again as soon as it matches.
return naturalGridMode$.pipe(
skipWhile((naturalMode) => naturalMode !== selection),
startWith(selection),
);
// The user has selected spotlight mode. If this matches the natural
// mode, then follow the natural mode going forward.
return selection === naturalGridMode$.value
? naturalGridMode$
: constant(selection);
}),
// Initially the mode of behavior is to just follow the natural grid mode.
startWith(naturalGridMode$),
// Switch between each mode of behavior.
switchMap((mode$) => mode$),
),
);
return {
gridMode$,
setGridMode,
};
}

6
src/state/CallViewModel/localMember/HomeserverConnected.ts
View File

@@ -31,11 +31,6 @@ import { type ObservableScope } from "../../ObservableScope";
import { type Behavior } from "../../Behavior";
import { type NodeStyleEventEmitter } from "../../../utils/test";
/**
* Logger instance (scoped child) for homeserver connection updates.
*/
const logger = rootLogger.getChild("[HomeserverConnected]");
export type HomeserverDisconnectReason = "sync" | "membership" | "probablyLeft";
export interface HomeserverConnected {
@@ -70,6 +65,7 @@ export function createHomeserverConnected$(
Pick<MatrixRTCSession, "membershipStatus" | "probablyLeft">,
gracePeriodMs?: number,
): HomeserverConnected {
const logger = rootLogger.getChild("[HomeserverConnected]");
// Get grace period from parameter or config (default 10000ms)
const graceMs = gracePeriodMs ?? Config.get().sync_disconnect_grace_period_ms;

2
src/state/CallViewModel/localMember/LocalMember.ts
View File

@@ -179,7 +179,7 @@ export const createLocalMembership$ = ({
logger: parentLogger,
muteStates,
matrixRTCSession,
roomId: roomId,
roomId,
}: Props): {
/**
* This request to start audio and video tracks.

17
src/state/CallViewModel/localMember/LocalTransport.ts
View File

@@ -25,7 +25,7 @@ import {
switchMap,
tap,
} from "rxjs";
import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
import { logger as rootLogger, type Logger } from "matrix-js-sdk/lib/logger";
import { AutoDiscovery } from "matrix-js-sdk/lib/autodiscovery";
import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
@@ -46,8 +46,6 @@ import { areLivekitTransportsEqual } from "../remoteMembers/MatrixLivekitMembers
import { customLivekitUrl } from "../../../settings/settings.ts";
import { RtcTransportAutoDiscovery } from "./RtcTransportAutoDiscovery.ts";
const logger = rootLogger.getChild("[LocalTransport]");
/*
* It figures out “which LiveKit focus URL/alias the local user should use,”
* optionally aligning with the oldest member, and ensures the SFU path is primed
@@ -140,9 +138,14 @@ export const createLocalTransport$ = ({
forceJwtEndpoint,
delayId$,
}: Props): LocalTransport => {
const logger = rootLogger.getChild("[LocalTransport]");
// The LiveKit transport in use by the oldest RTC membership. `null` when the
// oldest member has no such transport.
const oldestMemberTransport$ = observerOldestMembership$(scope, memberships$);
const oldestMemberTransport$ = observerOldestMembership$(
scope,
memberships$,
logger,
);
const transportDiscovery = new RtcTransportAutoDiscovery({
client: client,
@@ -190,6 +193,7 @@ export const createLocalTransport$ = ({
roomId,
client,
delayId ?? undefined,
logger,
);
} catch (e) {
logger.error(
@@ -209,6 +213,7 @@ export const createLocalTransport$ = ({
client,
ownMembershipIdentity,
roomId,
logger,
);
}
@@ -248,6 +253,7 @@ export const createLocalTransport$ = ({
function observerOldestMembership$(
scope: ObservableScope,
memberships$: Behavior<Epoch<CallMembership[]>>,
logger: Logger,
): Behavior<LivekitTransportConfig | null> {
return scope.behavior<LivekitTransportConfig | null>(
memberships$.pipe(
@@ -307,6 +313,7 @@ async function doOpenIdAndJWTFromUrl(
> &
OpenIDClientParts,
delayId?: string,
logger?: Logger,
): Promise<LocalTransportWithSFUConfig> {
const sfuConfig = await getSFUConfigWithOpenID(
client,
@@ -337,6 +344,7 @@ function observeLocalTransportForOldestMembership(
OpenIDClientParts,
ownMembershipIdentity: CallMembershipIdentityParts,
roomId: string,
logger: Logger,
): LocalTransport {
// Ensure we can authenticate with the SFU.
const authenticatedOldestMemberTransport$ = oldestMemberTransport$.pipe(
@@ -355,6 +363,7 @@ function observeLocalTransportForOldestMembership(
roomId,
client,
undefined,
logger,
),
).pipe(
catchError((e: unknown) => {

2
src/state/CallViewModel/localMember/RtcTransportAutoDiscovery.ts
View File

@@ -102,7 +102,7 @@ export class RtcTransportAutoDiscovery {
const transportList = await doNetworkOperationWithRetry(async () =>
client._unstable_getRTCTransports(),
);
const first = transportList.filter(isLivekitTransportConfig)[0];
const first = transportList.find(isLivekitTransportConfig);
if (first) {
return first;
} else {

28
src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts
View File

@@ -15,7 +15,7 @@ import { BehaviorSubject, combineLatest, map, type Observable } from "rxjs";
import { type IConnectionManager } from "./ConnectionManager.ts";
import {
type RemoteMatrixLivekitMember,
createMatrixLivekitMembers$,
createRemoteMatrixLivekitMembers$,
} from "./MatrixLivekitMembers.ts";
import {
Epoch,
@@ -31,6 +31,7 @@ import {
} from "../../../utils/test.ts";
import { type Connection } from "./Connection.ts";
import { constant } from "../../Behavior.ts";
import { localRtcMember } from "../../../utils/test-fixtures.ts";
let testScope: ObservableScope;
@@ -88,16 +89,17 @@ test("should signal participant not yet connected to livekit", async () => {
mockConnectionManagerData$,
);
const matrixLivekitMember$ = createMatrixLivekitMembers$({
const remoteMatrixLivekitMembers$ = createRemoteMatrixLivekitMembers$({
scope: testScope,
membershipsWithTransport$: testScope.behavior(membershipsWithTransport$),
connectionManager: {
connectionManagerData$: connectionManagerData$,
} as unknown as IConnectionManager,
localUser: localRtcMember,
});
await flushPromises();
expect(matrixLivekitMember$.value.value).toSatisfy(
expect(remoteMatrixLivekitMembers$.value.value).toSatisfy(
(data: RemoteMatrixLivekitMember[]) => {
expect(data.length).toEqual(1);
expect(data[0].membership$.value).toBe(bobMembership);
@@ -157,16 +159,17 @@ test("should signal participant on a connection that is publishing", async () =>
constant(dataWithPublisher),
);
const matrixLivekitMember$ = createMatrixLivekitMembers$({
const remoteMatrixLivekitMembers$ = createRemoteMatrixLivekitMembers$({
scope: testScope,
membershipsWithTransport$: testScope.behavior(membershipsWithTransport$),
connectionManager: {
connectionManagerData$: connectionManagerData$,
} as unknown as IConnectionManager,
localUser: localRtcMember,
});
await flushPromises();
expect(matrixLivekitMember$.value.value).toSatisfy(
expect(remoteMatrixLivekitMembers$.value.value).toSatisfy(
(data: RemoteMatrixLivekitMember[]) => {
expect(data.length).toEqual(1);
expect(data[0].membership$.value).toBe(bobMembership);
@@ -197,15 +200,16 @@ test("should signal participant on a connection that is not publishing", async (
constant(dataWithPublisher),
);
const matrixLivekitMember$ = createMatrixLivekitMembers$({
const remoteMatrixLivekitMembers$ = createRemoteMatrixLivekitMembers$({
scope: testScope,
membershipsWithTransport$: testScope.behavior(membershipsWithTransport$),
connectionManager: {
connectionManagerData$: connectionManagerData$,
} as unknown as IConnectionManager,
localUser: localRtcMember,
});
await flushPromises();
expect(matrixLivekitMember$.value.value).toSatisfy(
expect(remoteMatrixLivekitMembers$.value.value).toSatisfy(
(data: RemoteMatrixLivekitMember[]) => {
expect(data.length).toEqual(1);
expect(data[0].membership$.value).toBe(bobMembership);
@@ -245,15 +249,16 @@ describe("Publication edge case", () => {
constant(connectionWithPublisher),
);
const matrixLivekitMembers$ = createMatrixLivekitMembers$({
const remoteMatrixLivekitMembers$ = createRemoteMatrixLivekitMembers$({
scope: testScope,
membershipsWithTransport$: testScope.behavior(membershipsWithTransport$),
connectionManager: {
connectionManagerData$: connectionManagerData$,
} as unknown as IConnectionManager,
localUser: localRtcMember,
});
await flushPromises();
expect(matrixLivekitMembers$.value.value).toSatisfy(
expect(remoteMatrixLivekitMembers$.value.value).toSatisfy(
(data: RemoteMatrixLivekitMember[]) => {
expect(data.length).toEqual(2);
expect(data[0].membership$.value).toBe(bobMembership);
@@ -303,16 +308,17 @@ test("bob is publishing in the wrong connection", async () => {
connectionsWithPublisher$,
);
const matrixLivekitMember$ = createMatrixLivekitMembers$({
const remoteMatrixLivekitMembers$ = createRemoteMatrixLivekitMembers$({
scope: testScope,
membershipsWithTransport$: testScope.behavior(membershipsWithTransport$),
connectionManager: {
connectionManagerData$: connectionManagerData$,
} as unknown as IConnectionManager,
localUser: localRtcMember,
});
await flushPromises();
expect(matrixLivekitMember$.value.value).toSatisfy(
expect(remoteMatrixLivekitMembers$.value.value).toSatisfy(
(data: RemoteMatrixLivekitMember[]) => {
expect(data.length).toEqual(2);
expect(data[0].membership$.value).toBe(bobMembership);

16
src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts
View File

@@ -62,7 +62,9 @@ interface Props {
Epoch<{ membership: CallMembership; transport?: LivekitTransportConfig }[]>
>;
connectionManager: IConnectionManager;
localUser: { deviceId: string; userId: string };
}
/**
* Combines MatrixRTC and Livekit worlds.
*
@@ -73,13 +75,14 @@ interface Props {
* - out (via public Observable):
* - `remoteMatrixLivekitMember` an observable of MatrixLivekitMember[] to track the remote members and associated livekit data.
*/
export function createMatrixLivekitMembers$({
export function createRemoteMatrixLivekitMembers$({
scope,
membershipsWithTransport$,
connectionManager,
localUser,
}: Props): Behavior<Epoch<RemoteMatrixLivekitMember[]>> {
/**
* Stream of all the call members and their associated livekit data (if available).
* Behavior of all the remote call members and their associated livekit data (if available).
*/
return scope.behavior(
combineLatest([
@@ -91,12 +94,19 @@ export function createMatrixLivekitMembers$({
),
map(([ms, data]) => new Epoch([ms.value, data.value] as const, ms.epoch)),
generateItemsWithEpoch(
"MatrixLivekitMembers",
"RemoteMatrixLivekitMembers",
// Generator function.
// creates an array of `{key, data}[]`
// Each change in the keys (new key) will result in a call to the factory function.
function* ([membershipsWithTransport, managerData]) {
for (const { membership, transport } of membershipsWithTransport) {
// Exclude the local membership
if (
membership.userId === localUser.userId &&
membership.deviceId === localUser.deviceId
)
continue;
const participants = transport
? managerData.getParticipantsForTransport(transport)
: [];

69
src/state/CallViewModel/remoteMembers/MatrixMemberMetadata.test.ts
View File

@@ -105,7 +105,7 @@ describe("MatrixMemberMetadata", () => {
}
it("should show our own user if present in rtc session and room", () => {
withTestScheduler(({ behavior, expectObservable }) => {
withTestScheduler(({ scope, behavior, expectObservable }) => {
fakeMemberWith({
userId: "@local:example.com",
rawDisplayName: "it's a me",
@@ -118,8 +118,10 @@ describe("MatrixMemberMetadata", () => {
memberships$,
createRoomMembers$(testScope, mockMatrixRoom),
);
const dn$ =
metadataStore.createDisplayNameBehavior$("@local:example.com");
const dn$ = metadataStore.createDisplayNameBehavior$(
scope,
"@local:example.com",
);
expectObservable(dn$).toBe("a", {
a: "it's a me",
@@ -146,7 +148,7 @@ describe("MatrixMemberMetadata", () => {
it("should get displayName for users", () => {
setUpBasicRoom();
withTestScheduler(({ behavior, expectObservable }) => {
withTestScheduler(({ scope, behavior, expectObservable }) => {
const memberships$ = behavior("a", {
a: [
mockRtcMembership("@alice:example.com", "DEVICE1"),
@@ -158,8 +160,10 @@ describe("MatrixMemberMetadata", () => {
memberships$,
createRoomMembers$(testScope, mockMatrixRoom),
);
const aliceDispName$ =
metadataStore.createDisplayNameBehavior$("@alice:example.com");
const aliceDispName$ = metadataStore.createDisplayNameBehavior$(
scope,
"@alice:example.com",
);
expectObservable(aliceDispName$).toBe("a", {
a: "Alice",
@@ -322,7 +326,7 @@ describe("MatrixMemberMetadata", () => {
});
it("should track individual member id with createDisplayNameBehavior", () => {
withTestScheduler(({ behavior, schedule, expectObservable }) => {
withTestScheduler(({ scope, behavior, schedule, expectObservable }) => {
setUpBasicRoom();
const BOB = "@bob:example.com";
const CARL = "@carl:example.com";
@@ -356,8 +360,8 @@ describe("MatrixMemberMetadata", () => {
createRoomMembers$(testScope, mockMatrixRoom),
);
const bob$ = metadataStore.createDisplayNameBehavior$(BOB);
const carl$ = metadataStore.createDisplayNameBehavior$(CARL);
const bob$ = metadataStore.createDisplayNameBehavior$(scope, BOB);
const carl$ = metadataStore.createDisplayNameBehavior$(scope, CARL);
expectObservable(bob$).toBe("abc-", {
a: undefined,
@@ -378,7 +382,7 @@ describe("MatrixMemberMetadata", () => {
});
it("should disambiguate users with invisible characters", () => {
withTestScheduler(({ behavior, expectObservable }) => {
withTestScheduler(({ scope, behavior, expectObservable }) => {
const bobRtcMember = mockRtcMembership("@bob:example.org", "BBBB");
const bobZeroWidthSpaceRtcMember = mockRtcMembership(
"@bob2:example.org",
@@ -411,12 +415,18 @@ describe("MatrixMemberMetadata", () => {
createRoomMembers$(testScope, mockMatrixRoom),
);
const bob$ =
metadataStore.createDisplayNameBehavior$("@bob:example.org");
const bob2$ =
metadataStore.createDisplayNameBehavior$("@bob2:example.org");
const carol$ =
metadataStore.createDisplayNameBehavior$("@carol:example.org");
const bob$ = metadataStore.createDisplayNameBehavior$(
scope,
"@bob:example.org",
);
const bob2$ = metadataStore.createDisplayNameBehavior$(
scope,
"@bob2:example.org",
);
const carol$ = metadataStore.createDisplayNameBehavior$(
scope,
"@carol:example.org",
);
expectObservable(bob$).toBe("ab", {
a: "Bob",
b: "Bob (@bob:example.org)",
@@ -517,7 +527,7 @@ describe("MatrixMemberMetadata", () => {
}
it("should use avatar url from room members", () => {
withTestScheduler(({ behavior, expectObservable }) => {
withTestScheduler(({ scope, behavior, expectObservable }) => {
fakeMemberWith({
userId: "@local:example.com",
});
@@ -536,11 +546,15 @@ describe("MatrixMemberMetadata", () => {
memberships$,
createRoomMembers$(testScope, mockMatrixRoom),
);
const local$ =
metadataStore.createAvatarUrlBehavior$("@local:example.com");
const local$ = metadataStore.createAvatarUrlBehavior$(
scope,
"@local:example.com",
);
const alice$ =
metadataStore.createAvatarUrlBehavior$("@alice:example.com");
const alice$ = metadataStore.createAvatarUrlBehavior$(
scope,
"@alice:example.com",
);
expectObservable(local$).toBe("a", {
a: "mxc://example.com/@local:example.com",
@@ -558,7 +572,7 @@ describe("MatrixMemberMetadata", () => {
});
it("should update on avatar change and user join/leave", () => {
withTestScheduler(({ behavior, schedule, expectObservable }) => {
withTestScheduler(({ scope, behavior, schedule, expectObservable }) => {
fakeMemberWith({ userId: "@carl:example.com" });
fakeMemberWith({ userId: "@bob:example.com" });
const memberships$ = behavior("ab-d", {
@@ -585,9 +599,14 @@ describe("MatrixMemberMetadata", () => {
},
});
const bob$ = metadataStore.createAvatarUrlBehavior$("@bob:example.com");
const carl$ =
metadataStore.createAvatarUrlBehavior$("@carl:example.com");
const bob$ = metadataStore.createAvatarUrlBehavior$(
scope,
"@bob:example.com",
);
const carl$ = metadataStore.createAvatarUrlBehavior$(
scope,
"@carl:example.com",
);
expectObservable(bob$).toBe("a---", {
a: "mxc://example.com/@bob:example.com",
});

17
src/state/CallViewModel/remoteMembers/MatrixMemberMetadata.ts
View File

@@ -22,8 +22,6 @@ import {
} from "../../../utils/displayname";
import { type Behavior } from "../../Behavior";
const logger = rootLogger.getChild("[MatrixMemberMetadata]");
export type RoomMemberMap = Map<
string,
Pick<RoomMember, "userId" | "getMxcAvatarUrl" | "rawDisplayName">
@@ -67,6 +65,7 @@ export const memberDisplaynames$ = (
memberships$: Behavior<Pick<CallMembership, "userId">[]>,
roomMembers$: Behavior<RoomMemberMap>,
): Behavior<Map<string, string>> => {
const logger = rootLogger.getChild("[MatrixMemberMetadata]");
// This map tracks userIds that at some point needed disambiguation.
// This is a memory leak bound to the number of participants.
// A call application will always increase the memory if there have been more members in a call.
@@ -115,8 +114,14 @@ export const createMatrixMemberMetadata$ = (
memberships$: Behavior<Pick<CallMembership, "userId">[]>,
roomMembers$: Behavior<RoomMemberMap>,
): {
createDisplayNameBehavior$: (userId: string) => Behavior<string | undefined>;
createAvatarUrlBehavior$: (userId: string) => Behavior<string | undefined>;
createDisplayNameBehavior$: (
scope: ObservableScope,
userId: string,
) => Behavior<string | undefined>;
createAvatarUrlBehavior$: (
scope: ObservableScope,
userId: string,
) => Behavior<string | undefined>;
displaynameMap$: Behavior<Map<string, string>>;
avatarMap$: Behavior<Map<string, string | undefined>>;
} => {
@@ -136,13 +141,13 @@ export const createMatrixMemberMetadata$ = (
),
);
return {
createDisplayNameBehavior$: (userId: string) =>
createDisplayNameBehavior$: (scope: ObservableScope, userId: string) =>
scope.behavior(
displaynameMap$.pipe(
map((displaynameMap) => displaynameMap.get(userId)),
),
),
createAvatarUrlBehavior$: (userId: string) =>
createAvatarUrlBehavior$: (scope: ObservableScope, userId: string) =>
scope.behavior(
roomMembers$.pipe(
map((roomMembers) => roomMembers.get(userId)?.getMxcAvatarUrl()),

9
src/state/CallViewModel/remoteMembers/integration.test.ts
View File

@@ -29,13 +29,13 @@ import {
import { type ProcessorState } from "../../../livekit/TrackProcessorContext.tsx";
import {
areLivekitTransportsEqual,
createMatrixLivekitMembers$,
createRemoteMatrixLivekitMembers$,
type RemoteMatrixLivekitMember,
} from "./MatrixLivekitMembers.ts";
import { createConnectionManager$ } from "./ConnectionManager.ts";
import { membershipsAndTransports$ } from "../../SessionBehaviors.ts";
import { constant } from "../../Behavior.ts";
import { testJWTToken } from "../../../utils/test-fixtures.ts";
import { localRtcMember, testJWTToken } from "../../../utils/test-fixtures.ts";
// Test the integration of ConnectionManager and MatrixLivekitMerger
@@ -130,14 +130,15 @@ test("bob, carl, then bob joining no tracks yet", () => {
ownMembershipIdentity: ownMemberMock,
});
const matrixLivekitMembers$ = createMatrixLivekitMembers$({
const remoteMatrixLivekitMembers$ = createRemoteMatrixLivekitMembers$({
scope: testScope,
membershipsWithTransport$:
membershipsAndTransports.membershipsWithTransport$,
connectionManager,
localUser: localRtcMember,
});
expectObservable(matrixLivekitMembers$).toBe(vMarble, {
expectObservable(remoteMatrixLivekitMembers$).toBe(vMarble, {
a: expect.toSatisfy((e: Epoch<RemoteMatrixLivekitMember[]>) => {
const items = e.value;
expect(items.length).toBe(1);

162
src/state/IOSControlledAudioOutput.test.ts Normal file
View File

@@ -0,0 +1,162 @@
/*
Copyright 2026 Element Corp.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { type Observable, of } from "rxjs";
import { ObservableScope } from "./ObservableScope";
import { constant } from "./Behavior";
import { type SelectedAudioOutputDevice } from "./MediaDevices";
import {
availableOutputDevices$,
type Controls,
type OutputDevice,
outputDevice$,
} from "../controls";
import {
EARPIECE_CONFIG_ID,
IOSControlledAudioOutput,
} from "./IOSControlledAudioOutput";
// `vi.mock` calls are hoisted above all imports, so the static imports below
// already see these mocks. Force the iOS platform so that the virtual earpiece
// is available, and stub the livekit device observer (only subscribed for its
// side effects).
vi.mock("../Platform", () => ({ platform: "ios" }));
vi.mock("@livekit/components-core", () => ({
createMediaDeviceObserver: (): Observable<MediaDeviceInfo[]> => of([]),
}));
// On iOS the host reports a single device for the current route. When output is
// on the loudspeaker it is flagged `forEarpiece`, which makes the controller
// expose a virtual earpiece device.
const SPEAKER: OutputDevice = {
id: "speaker",
name: "Speaker",
isSpeaker: true,
forEarpiece: true,
};
// A connected headset (e.g. Bluetooth) is reported as a plain named device,
// with neither the speaker nor earpiece flag set.
const HEADSET: OutputDevice = {
id: "bt",
name: "AirPods",
};
let testScope: ObservableScope;
beforeEach(() => {
testScope = new ObservableScope();
window.controls = {
onAudioDeviceSelect: vi.fn(),
onOutputDeviceSelect: vi.fn(),
} as unknown as Controls;
});
afterEach(() => {
testScope.end();
});
/**
* Subscribe to the controller's `selected$` and return a getter for the latest
* emitted value.
*/
function latestSelection(
output: InstanceType<typeof IOSControlledAudioOutput>,
): () => SelectedAudioOutputDevice | undefined {
let latest: SelectedAudioOutputDevice | undefined;
output.selected$.subscribe((s) => {
latest = s;
});
return () => latest;
}
describe("Default selection", () => {
it("defaults to the earpiece for voice (audio) calls", () => {
const output = new IOSControlledAudioOutput(
constant(false),
testScope,
"audio",
);
const selected = latestSelection(output);
availableOutputDevices$.next([SPEAKER]);
expect(selected()).toEqual({
id: EARPIECE_CONFIG_ID,
virtualEarpiece: true,
});
expect(window.controls.onAudioDeviceSelect).toHaveBeenLastCalledWith(
EARPIECE_CONFIG_ID,
);
});
it("defaults to the speaker for video calls", () => {
const output = new IOSControlledAudioOutput(
constant(false),
testScope,
"video",
);
const selected = latestSelection(output);
availableOutputDevices$.next([SPEAKER]);
expect(selected()).toEqual({ id: SPEAKER.id, virtualEarpiece: false });
});
it("keeps a headset for voice calls instead of forcing the earpiece", () => {
const output = new IOSControlledAudioOutput(
constant(false),
testScope,
"audio",
);
const selected = latestSelection(output);
// The host proposes the headset as the route (listed first), even though a
// forEarpiece device is also present so the virtual earpiece exists.
availableOutputDevices$.next([HEADSET, SPEAKER]);
expect(selected()).toEqual({ id: HEADSET.id, virtualEarpiece: false });
});
});
describe("Explicit selection", () => {
it("an explicit user selection overrides the earpiece default", () => {
const output = new IOSControlledAudioOutput(
constant(false),
testScope,
"audio",
);
const selected = latestSelection(output);
availableOutputDevices$.next([SPEAKER]);
// Earpiece by default for a voice call...
expect(selected()).toEqual({
id: EARPIECE_CONFIG_ID,
virtualEarpiece: true,
});
// ...until the user explicitly picks the speaker.
output.select(SPEAKER.id);
expect(selected()).toEqual({ id: SPEAKER.id, virtualEarpiece: false });
});
it("a host selection overrides the earpiece default", () => {
const output = new IOSControlledAudioOutput(
constant(false),
testScope,
"audio",
);
const selected = latestSelection(output);
availableOutputDevices$.next([SPEAKER]);
outputDevice$.next(SPEAKER.id);
expect(selected()).toEqual({ id: SPEAKER.id, virtualEarpiece: false });
});
});

37
src/state/IOSControlledAudioOutput.ts
View File

@@ -8,6 +8,7 @@ Please see LICENSE in the repository root for full details.
import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
import { combineLatest, merge, startWith, Subject, tap } from "rxjs";
import type { RTCCallIntent } from "matrix-js-sdk/lib/matrixrtc";
import {
availableOutputDevices$ as controlledAvailableOutputDevices$,
outputDevice$ as controlledOutputSelection$,
@@ -24,7 +25,7 @@ import {
// This hardcoded id is used in EX ios! It can only be changed in coordination with
// the ios swift team.
const EARPIECE_CONFIG_ID = "earpiece-id";
export const EARPIECE_CONFIG_ID = "earpiece-id";
/**
* A special implementation of audio output that allows the hosting application
@@ -94,7 +95,7 @@ export class IOSControlledAudioOutput implements MediaDevice<
),
],
(available, preferredId) => {
const id = preferredId ?? available.keys().next().value;
const id = preferredId ?? this.chooseDefaultId(available);
return id === undefined
? undefined
: { id, virtualEarpiece: id === EARPIECE_CONFIG_ID };
@@ -106,9 +107,41 @@ export class IOSControlledAudioOutput implements MediaDevice<
),
);
/**
* Chooses the default output device when no explicit selection (from the user
* or the hosting application) has been made yet.
*
* For voice calls (`initialIntent === "audio"`) we want to start on the
* earpiece rather than the speaker, like a regular phone call. We only
* override when the device that would otherwise be the default is the
* speaker: if the host already routed to a headset (e.g. Bluetooth) — which
* is reported as a plain named device, not "speaker"/"earpiece" — we keep it.
* This mirrors the Android behaviour in {@link AndroidControlledAudioOutput}.
*/
private chooseDefaultId(
available: Map<string, AudioOutputDeviceLabel>,
): string | undefined {
const firstId = available.keys().next().value;
if (this.initialIntent === "audio") {
const firstLabel =
firstId !== undefined ? available.get(firstId) : undefined;
if (firstLabel?.type === "speaker") {
for (const [id, label] of available)
if (label.type === "earpiece") {
this.logger.info(
`IOS routing: default to earpiece ${id} instead of speaker for voice call`,
);
return id;
}
}
}
return firstId;
}
public constructor(
private readonly usingNames$: Behavior<boolean>,
private readonly scope: ObservableScope,
private readonly initialIntent: RTCCallIntent | undefined = undefined,
) {
this.selected$.subscribe((device) => {
// Let the hosting application know which output device has been selected.

92
src/state/CallViewModel/LayoutSwitch.test.ts → src/state/LayoutSwitchViewModel.test.ts
View File

@@ -7,31 +7,31 @@ Please see LICENSE in the repository root for full details.
import { describe, test } from "vitest";
import { createLayoutModeSwitch } from "./LayoutSwitch";
import { testScope, withTestScheduler } from "../../utils/test";
import { createLayoutSwitchViewModel } from "./LayoutSwitchViewModel";
import { testScope, withTestScheduler } from "../utils/test";
function testLayoutSwitch({
windowMode = "n",
hasScreenShares = "n",
userSelection = "",
expectedGridMode,
expectedLayout,
}: {
windowMode?: string;
hasScreenShares?: string;
userSelection?: string;
expectedGridMode: string;
expectedLayout: string;
}): void {
withTestScheduler(({ behavior, schedule, expectObservable }) => {
const { gridMode$, setGridMode } = createLayoutModeSwitch(
const { layout$, setLayout } = createLayoutSwitchViewModel(
testScope(),
behavior(windowMode, { n: "normal", N: "narrow", f: "flat" }),
behavior(hasScreenShares, { y: true, n: false }),
);
schedule(userSelection, {
g: () => setGridMode("grid"),
s: () => setGridMode("spotlight"),
g: () => setLayout("grid"),
s: () => setLayout("spotlight"),
});
expectObservable(gridMode$).toBe(expectedGridMode, {
expectObservable(layout$).toBe(expectedLayout, {
g: "grid",
s: "spotlight",
});
@@ -39,94 +39,88 @@ function testLayoutSwitch({
}
describe("default mode", () => {
test("uses grid layout by default", () =>
test("uses grid layout in normal window", () =>
testLayoutSwitch({
expectedGridMode: "g",
windowMode: " n",
expectedLayout: "g",
}));
test("uses spotlight mode when window mode is flat", () =>
test("uses grid layout in flat window", () =>
testLayoutSwitch({
windowMode: " f",
expectedGridMode: "s",
windowMode: " f",
expectedLayout: "g",
}));
});
test("allows switching modes manually", () =>
testLayoutSwitch({
userSelection: " --sgs",
expectedGridMode: "g-sgs",
userSelection: " --sgs",
expectedLayout: "g-sgs",
}));
test("switches to spotlight mode when there is a remote screen share", () =>
testLayoutSwitch({
hasScreenShares: " n--y",
expectedGridMode: "g--s",
hasScreenShares: "n--y",
expectedLayout: " g--s",
}));
test("can manually switch to grid when there is a screenshare", () =>
testLayoutSwitch({
hasScreenShares: " n-y",
userSelection: " ---g",
expectedGridMode: "g-sg",
hasScreenShares: "n-y",
userSelection: " ---g",
expectedLayout: " g-sg",
}));
test("auto-switches after manually selecting grid", () =>
testLayoutSwitch({
// Two screenshares will happen in sequence. There is a screen share that
// forces spotlight, then the user manually switches back to grid.
hasScreenShares: " n-y-ny",
userSelection: " ---g",
expectedGridMode: "g-sg-s",
hasScreenShares: "n-y-ny",
userSelection: " ---g",
expectedLayout: " g-sg-s",
// If we did want to respect manual selection, the expectation would be: g-sg
}));
test("switches back to grid mode when the remote screen share ends", () =>
testLayoutSwitch({
hasScreenShares: " n--y--n",
expectedGridMode: "g--s--g",
hasScreenShares: "n--y--n",
expectedLayout: " g--s--g",
}));
test("auto-switches to spotlight again after first screen share ends", () =>
testLayoutSwitch({
hasScreenShares: " nyny",
expectedGridMode: "gsgs",
hasScreenShares: "nyny",
expectedLayout: " gsgs",
}));
test("switches manually to grid after screen share while manually in spotlight", () =>
testLayoutSwitch({
// Initially, no one is sharing. Then the user manually switches to spotlight.
// After a screen share starts, the user manually switches to grid.
hasScreenShares: " n-y",
userSelection: " -s-g",
expectedGridMode: "gs-g",
}));
test("auto-switches to spotlight when in flat window mode", () =>
testLayoutSwitch({
// First normal, then narrow, then flat.
windowMode: " nNf",
expectedGridMode: "g-s",
hasScreenShares: "n-y",
userSelection: " -s-g",
expectedLayout: " gs-g",
}));
test("allows switching modes manually when in flat window mode", () =>
testLayoutSwitch({
// Window becomes flat, then user switches to grid and back.
// Window becomes flat, then user switches to spotlight and back.
// Finally the window returns to a normal shape.
windowMode: " nf--n",
userSelection: " --gs",
expectedGridMode: "gsgsg",
windowMode: " nf--n",
userSelection: " --sg",
expectedLayout: "g-sg",
}));
test("stays in spotlight while there are screen shares even when window mode changes", () =>
test("switches to grid when in flat window mode even when there are screen shares", () =>
testLayoutSwitch({
windowMode: " nfn",
hasScreenShares: " y",
expectedGridMode: "s",
windowMode: " nf",
hasScreenShares: "y",
expectedLayout: " sg",
}));
test("ignores end of screen share until window mode returns to normal", () =>
test("ignores screen share until window mode returns to normal", () =>
testLayoutSwitch({
windowMode: " nf-n",
hasScreenShares: " y-n",
expectedGridMode: "s--g",
windowMode: " f-n",
hasScreenShares: "ny-n",
expectedLayout: " g-sg",
}));

99
src/state/LayoutSwitchViewModel.ts Normal file
View File

@@ -0,0 +1,99 @@
/*
Copyright 2025 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
combineLatest,
map,
Subject,
startWith,
skipWhile,
switchAll,
} from "rxjs";
import { type WindowMode } from "./CallViewModel/CallViewModel.ts";
import { constant, type Behavior } from "./Behavior.ts";
import { type ObservableScope } from "./ObservableScope.ts";
export type LayoutMode = "spotlight" | "grid";
export interface LayoutSwitchViewModel {
/**
* The layout mode of the call's media tiles.
*/
layout$: Behavior<LayoutMode>;
setLayout: (value: LayoutMode) => void;
}
/**
* Creates a layout mode switch that allows switching between grid and spotlight layouts.
* The actual layout mode might switch automatically to spotlight if there is a
* remote screen share active or if the window mode is flat.
*
* @param scope - The observable scope to manage subscriptions.
* @param windowMode$ - The current window mode.
* @param hasRemoteScreenShares$ - A behavior indicating if there are remote screen shares active.
*/
export function createLayoutSwitchViewModel(
scope: ObservableScope,
windowMode$: Behavior<WindowMode>,
hasRemoteScreenShares$: Behavior<boolean>,
): LayoutSwitchViewModel {
const userSelection$ = new Subject<LayoutMode>();
// Callback to set the layout desired by the user.
// Notice that this is only a preference, the actual layout can be overridden
// if there is a remote screen share active.
const setLayout = (value: LayoutMode): void => userSelection$.next(value);
/**
* The natural layout - the layout that the interface would prefer to be in,
* not accounting for the user's manual selections.
*/
const naturalLayout$ = scope.behavior<LayoutMode>(
combineLatest(
[hasRemoteScreenShares$, windowMode$],
(hasRemoteScreenShares, windowMode) => {
// When the window is flat (as with a phone in landscape orientation),
// grid is preferable as there's usually more than enough horizontal
// space to fit in some grid tiles on the side.
if (windowMode === "flat") return "grid";
// When there are screen shares, spotlight is a better experience. We
// want them to be big and readable.
return hasRemoteScreenShares ? "spotlight" : "grid";
},
),
);
/**
* The layout mode of the call's media tiles.
*/
const layout$ = scope.behavior<LayoutMode>(
// Whenever the user makes a selection, we enter a new mode of behavior:
userSelection$.pipe(
map((selection) => {
if (selection === "grid")
// The user has selected grid. Start by respecting their choice, but
// then follow the natural mode again as soon as it matches.
return naturalLayout$.pipe(
skipWhile((naturalMode) => naturalMode !== selection),
startWith(selection),
);
// The user has selected spotlight. If this matches the natural layout,
// then follow the natural layout going forward.
return selection === naturalLayout$.value
? naturalLayout$
: constant(selection);
}),
// Initially the mode of behavior is to just follow the natural layout.
startWith(naturalLayout$),
// Switch between each mode of behavior.
switchAll(),
),
);
return { layout$, setLayout };
}

6
src/state/MediaDevices.ts
View File

@@ -376,7 +376,11 @@ export class MediaDevices {
getUrlParams().callIntent,
window.controls,
)
: new IOSControlledAudioOutput(this.usingNames$, this.scope)
: new IOSControlledAudioOutput(
this.usingNames$,
this.scope,
getUrlParams().callIntent,
)
: new AudioOutput(this.usingNames$, this.scope);
public readonly videoInput: MediaDevice<DeviceLabel, SelectedDevice> =

12
src/state/OneOnOneLandscapeLayout.ts → src/state/OneOnOneDesktopLayout.ts
View File

@@ -10,19 +10,19 @@ import { type BehaviorSubject } from "rxjs";
import {
type Alignment,
type OneOnOneLandscapeLayout,
type OneOnOneLandscapeLayoutMedia,
type OneOnOneDesktopLayout,
type OneOnOneDesktopLayoutMedia,
} from "./layout-types";
import { type TileStore } from "./TileStore";
/**
* Produces a one-on-one landscape layout with the given media.
* Produces a one-on-one desktop layout with the given media.
*/
export function oneOnOneLandscapeLayout(
media: OneOnOneLandscapeLayoutMedia,
export function oneOnOneDesktopLayout(
media: OneOnOneDesktopLayoutMedia,
pipAlignment$: BehaviorSubject<Alignment>,
prevTiles: TileStore,
): [OneOnOneLandscapeLayout, TileStore] {
): [OneOnOneDesktopLayout, TileStore] {
const update = prevTiles.from(2);
update.registerGridTile(media.pip);
update.registerGridTile(media.spotlight);

14
src/state/OneOnOnePortraitLayout.ts → src/state/OneOnOneMobileLayout.ts
View File

@@ -10,23 +10,23 @@ import { type BehaviorSubject } from "rxjs";
import {
type Alignment,
type OneOnOnePortraitLayout,
type OneOnOnePortraitLayoutMedia,
type OneOnOneMobileLayout,
type OneOnOneMobileLayoutMedia,
} from "./layout-types";
import { type TileStore } from "./TileStore";
import { type Behavior } from "./Behavior";
/**
* Produces a one-on-one portrait layout with the given media.
* Produces a one-on-one mobile layout with the given media.
*/
export function oneOnOnePortraitLayout(
media: OneOnOnePortraitLayoutMedia,
export function oneOnOneMobileLayout(
media: OneOnOneMobileLayoutMedia,
pipSize$: Behavior<"sm" | "lg">,
pipAlignment$: BehaviorSubject<Alignment>,
prevTiles: TileStore,
): [OneOnOnePortraitLayout, TileStore] {
): [OneOnOneMobileLayout, TileStore] {
const update = prevTiles.from(media.pip === undefined ? 0 : 1);
update.registerSpotlight([media.spotlight], true);
update.registerSpotlight([media.spotlight], true, "transparent");
if (media.pip !== undefined) update.registerGridTile(media.pip);
const tiles = update.build();

1
src/state/PipLayout.ts
View File

@@ -21,6 +21,7 @@ export function pipLayout(
update.registerSpotlight(
media.spotlight,
platform === "desktop" ? false : true,
"transparent",
);
const tiles = update.build();
return [

27
src/state/ServiceInterruptionsViewModel.ts Normal file
View File

@@ -0,0 +1,27 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
export type Service =
| { name: "encryption-keys" }
| { name: "reactions" }
| { name: "screen-sharing" }
| { name: "local-media" }
| {
name: "remote-media";
/**
* The number of users affected, for the purpose of pluralising the
* strings.
*/
count: number;
};
export interface ServiceInterruptionsViewModel {
/**
* A non-empty array of services which are temporarily unavailable.
*/
unavailable: Service[];
}

2
src/state/SpotlightExpandedLayout.ts
View File

@@ -23,7 +23,7 @@ export function spotlightExpandedLayout(
prevTiles: TileStore,
): [SpotlightExpandedLayout, TileStore] {
const update = prevTiles.from(1);
update.registerSpotlight(media.spotlight, true);
update.registerSpotlight(media.spotlight, true, "transparent");
if (media.pip !== undefined) update.registerPipTile(media.pip);
const tiles = update.build();

38
src/state/TileStore.ts
View File

@@ -15,6 +15,8 @@ import { type MediaViewModel } from "./media/MediaViewModel";
import { type UserMediaViewModel } from "./media/UserMediaViewModel";
import { type RingingMediaViewModel } from "./media/RingingMediaViewModel";
type SpotlightBackground = "solid" | "transparent";
function debugEntries(entries: GridTileData[]): string[] {
return entries.map((e) => e.media.displayName$.value);
}
@@ -39,12 +41,29 @@ class SpotlightTileData {
this.maximised$.next(value);
}
private readonly background$: BehaviorSubject<SpotlightBackground>;
public get background(): SpotlightBackground {
return this.background$.value;
}
public set background(value: SpotlightBackground) {
this.background$.next(value);
}
public readonly vm: SpotlightTileViewModel;
public constructor(media: MediaViewModel[], maximised: boolean) {
public constructor(
media: MediaViewModel[],
maximised: boolean,
background: SpotlightBackground,
) {
this.media$ = new BehaviorSubject(media);
this.maximised$ = new BehaviorSubject(maximised);
this.vm = new SpotlightTileViewModel(this.media$, this.maximised$);
this.background$ = new BehaviorSubject(background);
this.vm = new SpotlightTileViewModel(
this.media$,
this.maximised$,
this.background$,
);
}
}
@@ -131,9 +150,9 @@ export class TileStoreBuilder {
private numGridEntries = 0;
// A sparse array of grid entries which should be kept in the same spots as
// which they appeared in the previous grid
private readonly stationaryGridEntries: GridTileData[] = new Array(
this.prevGrid.length,
);
private readonly stationaryGridEntries: GridTileData[] = Array.from({
length: this.prevGrid.length,
});
// Grid entries which should now enter the visible section of the grid
private readonly visibleGridEntries: GridTileData[] = [];
// Grid entries which should now enter the invisible section of the grid
@@ -157,7 +176,11 @@ export class TileStoreBuilder {
* Sets the contents of the spotlight tile. If this is never called, there
* will be no spotlight tile.
*/
public registerSpotlight(media: MediaViewModel[], maximised: boolean): void {
public registerSpotlight(
media: MediaViewModel[],
maximised: boolean,
background: SpotlightBackground = "solid",
): void {
if (DEBUG_ENABLED)
logger.debug(
`[TileStore, ${this.generation}] register spotlight: ${media.map((m) => m.displayName$.value)}`,
@@ -169,11 +192,12 @@ export class TileStoreBuilder {
// Reuse the previous spotlight tile if it exists
if (this.prevSpotlight === null) {
this.spotlight = new SpotlightTileData(media, maximised);
this.spotlight = new SpotlightTileData(media, maximised, background);
} else {
this.spotlight = this.prevSpotlight;
this.spotlight.media = media;
this.spotlight.maximised = maximised;
this.spotlight.background = background;
}
}

9
src/state/TileViewModel.ts
View File

@@ -5,6 +5,8 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { BehaviorSubject } from "rxjs";
import { type Behavior } from "./Behavior";
import { type MediaViewModel } from "./media/MediaViewModel";
import { type RingingMediaViewModel } from "./media/RingingMediaViewModel";
@@ -17,18 +19,25 @@ function createId(): string {
export class GridTileViewModel {
public readonly id = createId();
private readonly _showOutline$ = new BehaviorSubject(false);
public readonly showOutline$: Behavior<boolean> = this._showOutline$;
public constructor(
public readonly media$: Behavior<
UserMediaViewModel | RingingMediaViewModel
>,
) {}
public setShowOutline(value: boolean): void {
this._showOutline$.next(value);
}
}
export class SpotlightTileViewModel {
public constructor(
public readonly media$: Behavior<MediaViewModel[]>,
public readonly maximised$: Behavior<boolean>,
public readonly background$: Behavior<"solid" | "transparent">,
) {}
}

24
src/state/layout-types.ts
View File

@@ -45,15 +45,15 @@ export interface SpotlightExpandedLayoutMedia {
pip?: UserMediaViewModel;
}
export interface OneOnOneLandscapeLayoutMedia {
type: "one-on-one-landscape";
export interface OneOnOneDesktopLayoutMedia {
type: "one-on-one-desktop";
edgeToEdge: false;
spotlight: UserMediaViewModel;
pip: LocalUserMediaViewModel | RingingMediaViewModel;
}
export interface OneOnOnePortraitLayoutMedia {
type: "one-on-one-portrait";
export interface OneOnOneMobileLayoutMedia {
type: "one-on-one-mobile";
edgeToEdge: true;
spotlight: UserMediaViewModel | RingingMediaViewModel;
pip?: LocalUserMediaViewModel;
@@ -70,8 +70,8 @@ export type LayoutMedia =
| SpotlightLandscapeLayoutMedia
| SpotlightPortraitLayoutMedia
| SpotlightExpandedLayoutMedia
| OneOnOneLandscapeLayoutMedia
| OneOnOnePortraitLayoutMedia
| OneOnOneDesktopLayoutMedia
| OneOnOneMobileLayoutMedia
| PipLayoutMedia;
export interface Alignment {
@@ -108,15 +108,15 @@ export interface SpotlightExpandedLayout {
pipAlignment$: BehaviorSubject<Alignment>;
}
export interface OneOnOneLandscapeLayout {
type: "one-on-one-landscape";
export interface OneOnOneDesktopLayout {
type: "one-on-one-desktop";
spotlight: GridTileViewModel;
pip: GridTileViewModel;
pipAlignment$: BehaviorSubject<Alignment>;
}
export interface OneOnOnePortraitLayout {
type: "one-on-one-portrait";
export interface OneOnOneMobileLayout {
type: "one-on-one-mobile";
spotlight: SpotlightTileViewModel;
pip?: GridTileViewModel;
pipSize$: Behavior<"sm" | "lg">;
@@ -137,6 +137,6 @@ export type Layout =
| SpotlightLandscapeLayout
| SpotlightPortraitLayout
| SpotlightExpandedLayout
| OneOnOneLandscapeLayout
| OneOnOnePortraitLayout
| OneOnOneDesktopLayout
| OneOnOneMobileLayout
| PipLayout;

5
src/state/media/MemberMediaViewModel.ts
View File

@@ -92,6 +92,7 @@ export function createMemberMedia(
}: MemberMediaInputs,
): BaseMemberMediaViewModel {
const trackBehavior$ = (
scope: ObservableScope,
source: Track.Source,
): Behavior<TrackReference | undefined> =>
scope.behavior(
@@ -102,8 +103,8 @@ export function createMemberMedia(
),
);
const audio$ = trackBehavior$(audioSource);
const video$ = trackBehavior$(videoSource);
const audio$ = trackBehavior$(scope, audioSource);
const video$ = trackBehavior$(scope, videoSource);
return {
...createBaseMedia(inputs),

20
src/state/media/RingingMediaViewModel.ts
View File

@@ -5,8 +5,9 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type RTCCallIntent } from "matrix-js-sdk/lib/matrixrtc";
import { type Behavior } from "../Behavior";
import { type MuteStates } from "../MuteStates";
import {
type BaseMediaInputs,
type BaseMediaViewModel,
@@ -20,32 +21,23 @@ import {
export interface RingingMediaViewModel extends BaseMediaViewModel {
type: "ringing";
pickupState$: Behavior<"ringing" | "timeout" | "decline">;
/**
* Whether this media would be expected to have video, were it not simply a
* placeholder.
*/
videoEnabled$: Behavior<boolean>;
intent: RTCCallIntent;
}
export interface RingingMediaInputs extends BaseMediaInputs {
pickupState$: Behavior<"ringing" | "timeout" | "decline">;
/**
* The local user's own mute states.
*/
muteStates: MuteStates;
intent: RTCCallIntent;
}
export function createRingingMedia({
pickupState$,
muteStates,
intent,
...inputs
}: RingingMediaInputs): RingingMediaViewModel {
return {
...createBaseMedia(inputs),
type: "ringing",
pickupState$,
// If our own video is enabled, then this is a video call and we would
// expect remote media to have video as well
videoEnabled$: muteStates.video.enabled$,
intent,
};
}

13
src/state/media/UserMediaViewModel.ts
View File

@@ -48,6 +48,7 @@ export interface BaseUserMediaViewModel extends BaseMemberMediaViewModel {
audioEnabled$: Behavior<boolean>;
videoEnabled$: Behavior<boolean>;
videoFit$: Behavior<"cover" | "contain">;
videoOrientation$: Behavior<"landscape" | "portrait">;
toggleCropVideo: () => void;
/**
* The expected identity of the LiveKit participant. Exposed for debugging.
@@ -104,6 +105,7 @@ export function createBaseUserMedia(
{ width: number; height: number } | undefined
>(undefined);
const videoSize$ = videoSizeFromParticipant$(participant$);
return {
...createMemberMedia(scope, {
...inputs,
@@ -129,11 +131,14 @@ export function createBaseUserMedia(
videoEnabled$: scope.behavior(
media$.pipe(map((m) => m?.cameraTrack?.isMuted === false)),
),
videoFit$: videoFit$(
scope,
videoSizeFromParticipant$(participant$),
targetSize$,
videoOrientation$: scope.behavior(
videoSize$.pipe(
map((s) => (s ? s.width / s.height : 1)),
map((aspect) => (aspect > 1 ? "landscape" : "portrait")),
),
"portrait",
),
videoFit$: videoFit$(scope, videoSize$, targetSize$),
toggleCropVideo: () => toggleCropVideo$.next(),
rtcBackendIdentity,
handRaised$,

4
src/state/media/observeRtpStreamStats.ts
View File

@@ -32,7 +32,9 @@ export function observeRtpStreamStats$(
> {
return combineLatest([
observeTrackReference$(participant, source),
interval(1000).pipe(startWith(0)),
// The update frequency is high because we use this value to update the PiP orientation and the fit/fill video tile props based on that
// We want it to be responsive. For just the debug tools 1s would be sufficient.
interval(350).pipe(startWith(0)),
]).pipe(
switchMap(async ([trackReference]) => {
const track = trackReference?.publication?.track;