Matrix/element-call-Github
1
0
Fork 0
mirror of https://github.com/vector-im/element-call.git synced 2026-05-25 11:14:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'livekit' into toger5/view-model-call-footer-example

Browse Source
This commit is contained in:
Timo K
2026-05-20 14:25:25 +02:00
parent a494b64055 3adddf4b70
commit 3301d2fe03
56 changed files with 2250 additions and 650 deletions
Download Patch File Download Diff File Expand all files Collapse all files

135
src/state/CallViewModel/CallViewModel.test.ts
View File

@@ -50,6 +50,7 @@ import {
aliceParticipant,
aliceRtcMember,
aliceUserId,
bob,
bobId,
bobRtcMember,
local,
@@ -133,12 +134,19 @@ export interface SpotlightExpandedLayoutSummary {
pip?: string;
}
export interface OneOnOneLayoutSummary {
type: "one-on-one";
export interface OneOnOneLandscapeLayoutSummary {
type: "one-on-one-landscape";
spotlight: string;
pip: string;
}
export interface OneOnOnePortraitLayoutSummary {
type: "one-on-one-portrait";
spotlight: string[];
pip?: string;
pipSize: "sm" | "lg";
}
export interface PipLayoutSummary {
type: "pip";
spotlight: string[];
@@ -149,7 +157,8 @@ export type LayoutSummary =
| SpotlightLandscapeLayoutSummary
| SpotlightPortraitLayoutSummary
| SpotlightExpandedLayoutSummary
| OneOnOneLayoutSummary
| OneOnOneLandscapeLayoutSummary
| OneOnOnePortraitLayoutSummary
| PipLayoutSummary;
function summarizeLayout$(l$: Observable<Layout>): Observable<LayoutSummary> {
@@ -187,7 +196,7 @@ function summarizeLayout$(l$: Observable<Layout>): Observable<LayoutSummary> {
pip: pip?.id,
}),
);
case "one-on-one":
case "one-on-one-landscape":
return combineLatest(
[l.spotlight.media$, l.pip.media$],
(spotlight, pip) => ({
@@ -196,6 +205,20 @@ function summarizeLayout$(l$: Observable<Layout>): Observable<LayoutSummary> {
pip: pip.id,
}),
);
case "one-on-one-portrait":
return combineLatest(
[
l.spotlight.media$,
l.pip?.media$ ?? constant(undefined),
l.pipSize$,
],
(spotlight, pip, pipSize) => ({
type: l.type,
spotlight: spotlight.map((vm) => vm.id),
pip: pip?.id,
pipSize,
}),
);
case "pip":
return l.spotlight.media$.pipe(
map((spotlight) => ({
@@ -405,7 +428,7 @@ describe.each([
expectedLayoutMarbles,
{
a: {
type: "one-on-one",
type: "one-on-one-landscape",
pip: `${localId}:0`,
spotlight: `${aliceId}:0`,
},
@@ -421,6 +444,85 @@ describe.each([
});
});
test("one-on-one portrait layout shows local tile when video is enabled", () => {
withTestScheduler(({ behavior, schedule, expectObservable }) => {
// Local participant enables their video, then disables it
const videoInputMarbles = " ny--n";
// While tile is shown, tap the screen twice
const tapScreenInputMarbles = "--aa-";
// Layout should show local tile, make it small, enlarge it again, then hide it
const expectedLayoutMarbles = "abcba";
withCallViewModel(
{
remoteParticipants$: constant([aliceParticipant]),
roomMembers: [local, alice],
rtcMembers$: constant([localRtcMember, aliceRtcMember]),
videoEnabled: new Map([
[localParticipant, behavior(videoInputMarbles, yesNo)],
]),
windowSize$: constant({ width: 380, height: 700 }), // Mobile phone in portrait
},
(vm) => {
schedule(tapScreenInputMarbles, { a: () => vm.tapScreen() });
expectObservable(vm.edgeToEdge$).toBe("y", yesNo); // Edge-to-edge-layout
expectObservable(summarizeLayout$(vm.layout$)).toBe(
expectedLayoutMarbles,
{
a: {
type: "one-on-one-portrait",
spotlight: [`${aliceId}:0`],
pip: undefined,
pipSize: "lg",
},
b: {
type: "one-on-one-portrait",
spotlight: [`${aliceId}:0`],
pip: `${localId}:0`,
pipSize: "lg",
},
c: {
type: "one-on-one-portrait",
spotlight: [`${aliceId}:0`],
pip: `${localId}:0`,
pipSize: "sm",
},
},
);
},
);
});
});
test("one-on-one portrait layout shows name tags in room with 3 members", () => {
withTestScheduler(({ behavior, schedule, expectObservable }) => {
withCallViewModel(
{
remoteParticipants$: constant([aliceParticipant]),
// Both Alice and Bob are with us in the room
roomMembers: [local, alice, bob],
rtcMembers$: constant([localRtcMember, aliceRtcMember]),
windowSize$: constant({ width: 380, height: 700 }), // Mobile phone in portrait
},
(vm) => {
// Uses one-on-one portrait layout
expectObservable(summarizeLayout$(vm.layout$)).toBe("a", {
a: {
type: "one-on-one-portrait",
spotlight: [`${aliceId}:0`],
pip: undefined,
pipSize: "lg",
},
});
// It wouldn't be clear whether Alice or Bob is the remote video tile,
// so the interface must put a name tag on it
expectObservable(vm.showNameTags$).toBe("y", yesNo);
},
);
});
});
test("participants stay in the same order unless to appear/disappear", () => {
withTestScheduler(({ behavior, schedule, expectObservable }) => {
const visibilityInputMarbles = "a";
@@ -576,7 +678,7 @@ describe.each([
});
test("layout reacts to window size", () => {
withTestScheduler(({ behavior, schedule, expectObservable }) => {
withTestScheduler(({ behavior, expectObservable }) => {
const windowSizeInputMarbles = "abc";
const expectedLayoutMarbles = " abc";
withCallViewModel(
@@ -584,7 +686,7 @@ describe.each([
remoteParticipants$: constant([aliceParticipant]),
rtcMembers$: constant([localRtcMember, aliceRtcMember]),
windowSize$: behavior(windowSizeInputMarbles, {
a: { width: 300, height: 600 }, // Start very narrow, like a phone
a: { width: 380, height: 700 }, // Start very narrow, like a phone
b: { width: 1000, height: 800 }, // Go to normal desktop window size
c: { width: 200, height: 180 }, // Go to PiP size
}),
@@ -595,13 +697,14 @@ describe.each([
{
a: {
// This is the expected one-on-one layout for a narrow window
type: "spotlight-expanded",
type: "one-on-one-portrait",
spotlight: [`${aliceId}:0`],
pip: `${localId}:0`,
pip: undefined,
pipSize: "lg",
},
b: {
// In a larger window, expect the normal one-on-one layout
type: "one-on-one",
type: "one-on-one-landscape",
pip: `${localId}:0`,
spotlight: `${aliceId}:0`,
},
@@ -956,7 +1059,7 @@ describe.each([
grid: [`${localId}:0`],
},
b: {
type: "one-on-one",
type: "one-on-one-landscape",
pip: `${localId}:0`,
spotlight: `${aliceId}:0`,
},
@@ -999,7 +1102,7 @@ describe.each([
grid: [`${localId}:0`],
},
b: {
type: "one-on-one",
type: "one-on-one-landscape",
pip: `${localId}:0`,
spotlight: `${aliceId}:0`,
},
@@ -1009,7 +1112,7 @@ describe.each([
grid: [`${localId}:0`, `${aliceId}:0`, `${daveId}:0`],
},
d: {
type: "one-on-one",
type: "one-on-one-landscape",
pip: `${localId}:0`,
spotlight: `${daveId}:0`,
},
@@ -1227,7 +1330,7 @@ describe.each([
// ringing the entire time (even once timed out)
expectObservable(summarizeLayout$(vm.layout$)).toBe("a", {
a: {
type: "one-on-one",
type: "one-on-one-landscape",
spotlight: `${localId}:0`,
pip: `ringing:${aliceUserId}`,
},
@@ -1266,12 +1369,12 @@ describe.each([
// ringing the entire time
expectObservable(summarizeLayout$(vm.layout$)).toBe("a 20ms b", {
a: {
type: "one-on-one",
type: "one-on-one-landscape",
spotlight: `${localId}:0`,
pip: `ringing:${aliceUserId}`,
},
b: {
type: "one-on-one",
type: "one-on-one-landscape",
spotlight: `${aliceId}:0`,
pip: `${localId}:0`,
},

508
src/state/CallViewModel/CallViewModel.ts
View File

@@ -39,6 +39,7 @@ import {
tap,
throttleTime,
timer,
BehaviorSubject,
} from "rxjs";
import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
import {
@@ -69,7 +70,8 @@ import { setPipEnabled$ } from "../../controls";
import { TileStore } from "../TileStore";
import { gridLikeLayout } from "../GridLikeLayout";
import { spotlightExpandedLayout } from "../SpotlightExpandedLayout";
import { oneOnOneLayout } from "../OneOnOneLayout";
import { oneOnOneLandscapeLayout } from "../OneOnOneLandscapeLayout";
import { oneOnOnePortraitLayout } from "../OneOnOnePortraitLayout";
import { pipLayout } from "../PipLayout";
import { type EncryptionSystem } from "../../e2ee/sharedKeyManagement";
import {
@@ -87,10 +89,12 @@ import { getUrlParams, HeaderStyle } from "../../UrlParams";
import { type ProcessorState } from "../../livekit/TrackProcessorContext";
import { ElementWidgetActions, widget } from "../../widget";
import {
type Alignment,
type GridLayoutMedia,
type Layout,
type LayoutMedia,
type OneOnOneLayoutMedia,
type OneOnOneLandscapeLayoutMedia,
type OneOnOnePortraitLayoutMedia,
type SpotlightExpandedLayoutMedia,
type SpotlightLandscapeLayoutMedia,
type SpotlightPortraitLayoutMedia,
@@ -328,16 +332,10 @@ export interface CallViewModel {
{ sender: string; emoji: string; startX: number }[]
>;
// window/layout
/**
* The general shape of the window.
*/
windowMode$: Behavior<WindowMode>;
spotlightExpanded$: Behavior<boolean>;
toggleSpotlightExpanded$: Behavior<(() => void) | null>;
gridMode$: Behavior<GridMode>;
setGridMode: (value: GridMode) => void;
/**
* The layout of tiles in the call interface.
*/
@@ -348,10 +346,20 @@ export interface CallViewModel {
tileStoreGeneration$: Behavior<number>;
showSpotlightIndicators$: Behavior<boolean>;
showSpeakingIndicators$: Behavior<boolean>;
showNameTags$: Behavior<boolean>;
spotlightExpanded$: Behavior<boolean>;
toggleSpotlightExpanded$: Behavior<(() => void) | null>;
gridMode$: Behavior<GridMode>;
setGridMode: (value: GridMode) => void;
// header/footer visibility
showHeader$: Behavior<boolean>;
showFooter$: Behavior<boolean>;
/**
* Whether the call layout should be displayed edge-to-edge, with the footer
* and header as overlays.
*/
edgeToEdge$: Behavior<boolean>;
settingsOpen$: Behavior<boolean>;
setSettingsOpen$: Behavior<(open: boolean) => void>;
@@ -564,6 +572,7 @@ export function createCallViewModel$(
connectionManager,
matrixRTCSession,
localTransport$,
roomId: matrixRoom.roomId,
logger: logger.getChild(`[${Date.now()}]`),
});
@@ -780,6 +789,7 @@ export function createCallViewModel$(
callPickupState === "timeout" ||
callPickupState === "decline"
) {
// TODO: Respect io.element.functional_members
for (const member of roomMembers.values()) {
if (!userMedia.some((vm) => vm.userId === member.userId))
yield {
@@ -1060,6 +1070,7 @@ export function createCallViewModel$(
[grid$, spotlight$],
(grid, spotlight) => ({
type: "grid",
edgeToEdge: false,
spotlight: spotlight.some((vm) => vm.type === "screen share")
? spotlight
: undefined,
@@ -1070,6 +1081,7 @@ export function createCallViewModel$(
const spotlightLandscapeLayoutMedia$: Observable<SpotlightLandscapeLayoutMedia> =
combineLatest([grid$, spotlight$], (grid, spotlight) => ({
type: "spotlight-landscape",
edgeToEdge: false,
spotlight,
grid,
}));
@@ -1077,16 +1089,20 @@ export function createCallViewModel$(
const spotlightPortraitLayoutMedia$: Observable<SpotlightPortraitLayoutMedia> =
combineLatest([grid$, spotlight$], (grid, spotlight) => ({
type: "spotlight-portrait",
edgeToEdge: false,
spotlight,
grid,
}));
const spotlightExpandedLayoutMedia$: Observable<SpotlightExpandedLayoutMedia> =
const spotlightExpandedLayoutMedia$ = (
edgeToEdge: boolean,
): Observable<SpotlightExpandedLayoutMedia> =>
spotlightAndPip$.pipe(
switchMap(({ spotlight, pip$ }) =>
pip$.pipe(
map((pip) => ({
type: "spotlight-expanded" as const,
edgeToEdge,
spotlight,
pip: pip ?? undefined,
})),
@@ -1094,55 +1110,88 @@ export function createCallViewModel$(
),
);
const oneOnOneLayoutMedia$: Observable<OneOnOneLayoutMedia | null> =
combineLatest([userMedia$, screenShares$]).pipe(
switchMap(([userMedia, screenShares]) => {
// One-on-one layout only supports 2 user media, no screen shares
if (userMedia.length <= 2 && screenShares.length === 0) {
const local = userMedia.find(
(vm): vm is WrappedUserMediaViewModel & LocalUserMediaViewModel =>
vm.type === "user" && vm.local,
const oneOnOneLayoutMedia$: Observable<{
local: LocalUserMediaViewModel;
remote: UserMediaViewModel | RingingMediaViewModel;
} | null> = combineLatest([userMedia$, screenShares$]).pipe(
switchMap(([userMedia, screenShares]) => {
// One-on-one layout only supports 2 user media, no screen shares
if (userMedia.length <= 2 && screenShares.length === 0) {
const local = userMedia.find(
(vm): vm is WrappedUserMediaViewModel & LocalUserMediaViewModel =>
vm.type === "user" && vm.local,
);
if (local !== undefined) {
const remote = userMedia.find(
(vm): vm is WrappedUserMediaViewModel & RemoteUserMediaViewModel =>
vm.type === "user" && !vm.local,
);
if (local !== undefined) {
const remote = userMedia.find(
(
vm,
): vm is WrappedUserMediaViewModel & RemoteUserMediaViewModel =>
vm.type === "user" && !vm.local,
if (remote !== undefined) return of({ local, remote });
// If there's no other user media in the call (could still happen in
// this branch due to the duplicate tiles option), we could possibly
// show ringing media instead
if (userMedia.length === 1)
return ringingMedia$.pipe(
map((ringingMedia) => {
return ringingMedia.length === 1
? {
local,
remote: ringingMedia[0],
}
: null;
}),
);
if (remote !== undefined)
return of({
type: "one-on-one" as const,
spotlight: remote,
pip: local,
});
// If there's no other user media in the call (could still happen in
// this branch due to the duplicate tiles option), we could possibly
// show ringing media instead
if (userMedia.length === 1)
return ringingMedia$.pipe(
map((ringingMedia) => {
return ringingMedia.length === 1
? {
type: "one-on-one" as const,
spotlight: local,
pip: ringingMedia[0],
}
: null;
}),
);
}
}
}
return of(null);
return of(null);
}),
);
const oneOnOneLandscapeLayoutMedia$: Observable<OneOnOneLandscapeLayoutMedia | null> =
oneOnOneLayoutMedia$.pipe(
map((media) => {
if (media === null) return null;
return media.remote.type === "ringing"
? {
type: "one-on-one-landscape" as const,
edgeToEdge: false,
spotlight: media.local,
pip: media.remote,
}
: {
type: "one-on-one-landscape" as const,
edgeToEdge: false,
spotlight: media.remote,
pip: media.local,
};
}),
);
const oneOnOnePortraitLayoutMedia$: Observable<OneOnOnePortraitLayoutMedia | null> =
oneOnOneLayoutMedia$.pipe(
switchMap((media) => {
if (media === null) return of(null);
return media.local.videoEnabled$.pipe(
map((videoEnabled) => ({
type: "one-on-one-portrait" as const,
edgeToEdge: true as const,
spotlight: media.remote,
pip: videoEnabled ? media.local : undefined,
})),
);
}),
);
const pipLayoutMedia$: Observable<LayoutMedia> = spotlight$.pipe(
map((spotlight) => ({ type: "pip", spotlight })),
map((spotlight) => ({
type: "pip",
edgeToEdge: platform !== "desktop",
spotlight,
})),
);
/**
@@ -1157,7 +1206,7 @@ export function createCallViewModel$(
switchMap((gridMode) => {
switch (gridMode) {
case "grid":
return oneOnOneLayoutMedia$.pipe(
return oneOnOneLandscapeLayoutMedia$.pipe(
switchMap((oneOnOne) =>
oneOnOne === null ? gridLayoutMedia$ : of(oneOnOne),
),
@@ -1166,7 +1215,7 @@ export function createCallViewModel$(
return spotlightExpanded$.pipe(
switchMap((expanded) =>
expanded
? spotlightExpandedLayoutMedia$
? spotlightExpandedLayoutMedia$(false)
: spotlightLandscapeLayoutMedia$,
),
);
@@ -1174,7 +1223,7 @@ export function createCallViewModel$(
}),
);
case "narrow":
return oneOnOneLayoutMedia$.pipe(
return oneOnOnePortraitLayoutMedia$.pipe(
switchMap((oneOnOne) =>
oneOnOne === null
? combineLatest([grid$, spotlight$], (grid, spotlight) =>
@@ -1183,9 +1232,7 @@ export function createCallViewModel$(
? spotlightPortraitLayoutMedia$
: gridLayoutMedia$,
).pipe(switchAll())
: // The expanded spotlight layout makes for a better one-on-one
// experience in narrow windows
spotlightExpandedLayoutMedia$,
: of(oneOnOne),
),
);
case "flat":
@@ -1197,7 +1244,7 @@ export function createCallViewModel$(
// this window mode.
return spotlightLandscapeLayoutMedia$;
case "spotlight":
return spotlightExpandedLayoutMedia$;
return spotlightExpandedLayoutMedia$(true);
}
}),
);
@@ -1208,6 +1255,197 @@ export function createCallViewModel$(
),
);
const showSpotlightIndicators$ = scope.behavior<boolean>(
layoutMedia$.pipe(map((l) => l.type !== "grid")),
);
const showSpeakingIndicators$ = scope.behavior<boolean>(
layoutMedia$.pipe(
map((l) => {
switch (l.type) {
case "spotlight-landscape":
case "spotlight-portrait":
// If the spotlight is showing the active speaker, we can do without
// speaking indicators as they're a redundant visual cue. But if
// screen sharing feeds are in the spotlight we still need them.
return l.spotlight.some((m) => m.type === "screen share");
// In expanded spotlight layout, the active speaker is always shown in
// the picture-in-picture tile so there is no need for speaking
// indicators. And in one-on-one layout there's no question as to who is
// speaking.
case "spotlight-expanded":
case "one-on-one-landscape":
case "one-on-one-portrait":
return false;
default:
return true;
}
}),
),
);
const showNameTags$ = scope.behavior<boolean>(
layoutMedia$.pipe(
switchMap((l) =>
l.type === "pip" || l.type === "one-on-one-portrait"
? matrixRoomMembers$.pipe(
map(
(members) =>
// Hide name tags by default in these layouts. For safety we
// still need to show them in case it wouldn't be clear who
// the spotlight media belongs to.
// TODO: Respect io.element.functional_members (while still
// being careful to never show a functional member's media
// without a name tag!)
// TODO: Only hide name tags in DMs, not group chats that just
// happen to have only 2 users
members.size > 2,
),
)
: of(true),
),
),
);
const toggleSpotlightExpanded$ = scope.behavior<(() => void) | null>(
windowMode$.pipe(
switchMap((mode) =>
mode === "normal"
? layoutMedia$.pipe(
map(
(l) =>
l.type === "spotlight-landscape" ||
l.type === "spotlight-expanded",
),
)
: of(false),
),
distinctUntilChanged(),
map((enabled) =>
enabled ? (): void => spotlightExpandedToggle$.next() : null,
),
),
);
const edgeToEdge$ = scope.behavior<boolean>(
layoutMedia$.pipe(map(({ edgeToEdge }) => edgeToEdge)),
);
const screenTap$ = new Subject<void>();
const controlsTap$ = new Subject<void>();
const screenHover$ = new Subject<void>();
const screenUnhover$ = new Subject<void>();
const naturallyShowFooter$ = scope.behavior<boolean>(
edgeToEdge$.pipe(
switchMap((edgeToEdge) => {
if (!edgeToEdge) return of(true);
// Sadly Firefox has some layering glitches that prevent the footer
// from appearing properly. They happen less often if we never hide
// the footer.
if (isFirefox()) return of(true);
// Layout is edge-to-edge; show/hide the footer in response to interactions
return windowMode$.pipe(
switchMap((mode) => {
const showInitially = mode !== "flat";
const timeout$ = mode === "flat" ? timer(showFooterMs) : NEVER;
return merge(
screenTap$.pipe(map(() => "tap screen" as const)),
controlsTap$.pipe(map(() => "tap controls" as const)),
screenHover$.pipe(map(() => "hover" as const)),
).pipe(
switchScan((state, interaction) => {
switch (interaction) {
case "tap screen":
return state
? // Toggle visibility on tap
of(false)
: // Hide after a timeout
timeout$.pipe(
map(() => false),
startWith(true),
);
case "tap controls":
// The user is interacting with things, so reset the timeout
return timeout$.pipe(
map(() => false),
startWith(true),
);
case "hover":
// Show on hover and hide after a timeout
return race(timeout$, screenUnhover$.pipe(take(1))).pipe(
map(() => false),
startWith(true),
);
}
}, showInitially),
startWith(showInitially),
);
}),
);
}),
),
);
const urlParams = getUrlParams();
const showFooterUrlParams = !(
urlParams.header === HeaderStyle.None && urlParams.showControls === false
);
const showFooter$ = scope.behavior(
naturallyShowFooter$.pipe(
map((naturallyShowFooter) => naturallyShowFooter && showFooterUrlParams),
),
);
const settingsOpen$ = new BehaviorSubject(false);
const setSettingsOpen$ = constant((open: boolean) => {
settingsOpen$.next(open);
});
const showHeader$ = scope.behavior<boolean>(
windowMode$.pipe(
switchMap((mode) => {
// In small windows the header would be too obstructive
if (mode === "pip" || mode === "flat") return of(false);
// In edge-to-edge layouts, couple the visibility of the header
// to that of the footer
return edgeToEdge$.pipe(
switchMap((edgeToEdge) => (edgeToEdge ? showFooter$ : of(true))),
);
}),
),
);
/**
* The alignment of the floating spotlight tile, if present.
*/
const spotlightAlignment$ = new BehaviorSubject<Alignment>({
inline: "end",
block: "end",
});
/**
* The size of the small picture-in-picture tile, if present, when in portrait.
*/
const portraitPipSize$ = scope.behavior(
showFooter$.pipe(map((showFooter) => (showFooter ? "lg" : "sm"))),
);
/**
* The alignment of the small picture-in-picture tile, if present, when in portrait.
*/
const portraitPipAlignment$ = new BehaviorSubject<Alignment>({
inline: "end",
block: "end",
});
/**
* The alignment of the small picture-in-picture tile, if present, when in landscape.
*/
const landscapePipAlignment$ = new BehaviorSubject<Alignment>({
inline: "end",
block: "start",
});
// There is a cyclical dependency here: the layout algorithms want to know
// which tiles are on screen, but to know which tiles are on screen we have to
// first render a layout. To deal with this we assume initially that no tiles
@@ -1234,16 +1472,33 @@ export function createCallViewModel$(
case "spotlight-portrait":
[layout, newTiles] = gridLikeLayout(
media,
spotlightAlignment$,
visibleTiles,
setVisibleTiles,
prevTiles,
);
break;
case "spotlight-expanded":
[layout, newTiles] = spotlightExpandedLayout(media, prevTiles);
[layout, newTiles] = spotlightExpandedLayout(
media,
landscapePipAlignment$,
prevTiles,
);
break;
case "one-on-one":
[layout, newTiles] = oneOnOneLayout(media, prevTiles);
case "one-on-one-landscape":
[layout, newTiles] = oneOnOneLandscapeLayout(
media,
landscapePipAlignment$,
prevTiles,
);
break;
case "one-on-one-portrait":
[layout, newTiles] = oneOnOnePortraitLayout(
media,
portraitPipSize$,
portraitPipAlignment$,
prevTiles,
);
break;
case "pip":
[layout, newTiles] = pipLayout(media, prevTiles);
@@ -1271,138 +1526,6 @@ export function createCallViewModel$(
layoutInternals$.pipe(map(({ tiles }) => tiles.generation)),
);
const showSpotlightIndicators$ = scope.behavior<boolean>(
layout$.pipe(map((l) => l.type !== "grid")),
);
const showSpeakingIndicators$ = scope.behavior<boolean>(
layout$.pipe(
switchMap((l) => {
switch (l.type) {
case "spotlight-landscape":
case "spotlight-portrait":
// If the spotlight is showing the active speaker, we can do without
// speaking indicators as they're a redundant visual cue. But if
// screen sharing feeds are in the spotlight we still need them.
return l.spotlight.media$.pipe(
map((models: MediaViewModel[]) =>
models.some((m) => m.type === "screen share"),
),
);
// In expanded spotlight layout, the active speaker is always shown in
// the picture-in-picture tile so there is no need for speaking
// indicators. And in one-on-one layout there's no question as to who is
// speaking.
case "spotlight-expanded":
case "one-on-one":
return of(false);
default:
return of(true);
}
}),
),
);
const toggleSpotlightExpanded$ = scope.behavior<(() => void) | null>(
windowMode$.pipe(
switchMap((mode) =>
mode === "normal"
? layout$.pipe(
map(
(l) =>
l.type === "spotlight-landscape" ||
l.type === "spotlight-expanded",
),
)
: of(false),
),
distinctUntilChanged(),
map((enabled) =>
enabled ? (): void => spotlightExpandedToggle$.next() : null,
),
),
);
const screenTap$ = new Subject<void>();
const controlsTap$ = new Subject<void>();
const screenHover$ = new Subject<void>();
const screenUnhover$ = new Subject<void>();
const showHeader$ = scope.behavior<boolean>(
windowMode$.pipe(map((mode) => mode !== "pip" && mode !== "flat")),
);
const urlParams = getUrlParams();
const showFooterUrlParams = !(
urlParams.header === HeaderStyle.None && urlParams.showControls === false
);
// candidat to move into the FooterViewModel
const showFooterLayout$ = scope.behavior<boolean>(
windowMode$.pipe(
switchMap((mode) => {
switch (mode) {
case "pip":
return of(platform === "desktop" ? true : false);
case "normal":
case "narrow":
return of(true);
case "flat":
// Sadly Firefox has some layering glitches that prevent the footer
// from appearing properly. They happen less often if we never hide
// the footer.
if (isFirefox()) return of(true);
// Show/hide the footer in response to interactions
return merge(
screenTap$.pipe(map(() => "tap screen" as const)),
controlsTap$.pipe(map(() => "tap controls" as const)),
screenHover$.pipe(map(() => "hover" as const)),
).pipe(
switchScan((state, interaction) => {
switch (interaction) {
case "tap screen":
return state
? // Toggle visibility on tap
of(false)
: // Hide after a timeout
timer(showFooterMs).pipe(
map(() => false),
startWith(true),
);
case "tap controls":
// The user is interacting with things, so reset the timeout
return timer(showFooterMs).pipe(
map(() => false),
startWith(true),
);
case "hover":
// Show on hover and hide after a timeout
return race(
timer(showFooterMs),
screenUnhover$.pipe(take(1)),
).pipe(
map(() => false),
startWith(true),
);
}
}, false),
startWith(false),
);
}
}),
),
);
// candidat to move into the FooterViewModel
const showFooter$ = scope.behavior(
showFooterLayout$.pipe(
map((showFooter) => showFooter && showFooterUrlParams),
),
);
const settingsOpen$ = new BehaviorSubject(false);
const setSettingsOpen$ = constant((open: boolean) => {
settingsOpen$.next(open);
});
/**
* Whether audio is currently being output through the earpiece.
*/
@@ -1606,7 +1729,6 @@ export function createCallViewModel$(
audibleReactions$: audibleReactions$,
visibleReactions$: visibleReactions$,
windowMode$: windowMode$,
spotlightExpanded$: spotlightExpanded$,
toggleSpotlightExpanded$: toggleSpotlightExpanded$,
gridMode$: gridMode$,
@@ -1632,10 +1754,12 @@ export function createCallViewModel$(
tileStoreGeneration$: tileStoreGeneration$,
showSpotlightIndicators$: showSpotlightIndicators$,
showSpeakingIndicators$: showSpeakingIndicators$,
showNameTags$,
showHeader$: showHeader$,
showFooter$: showFooter$,
settingsOpen$: settingsOpen$,
setSettingsOpen$: setSettingsOpen$,
edgeToEdge$,
earpieceMode$: earpieceMode$,
audioOutputSwitcher$: audioOutputSwitcher$,
reconnecting$: localMembership.reconnecting$,

22
src/state/CallViewModel/CallViewModelTestUtils.ts
View File

@@ -8,11 +8,11 @@ Please see LICENSE in the repository root for full details.
import {
ConnectionState,
type LocalParticipant,
type Participant,
ParticipantEvent,
type RemoteParticipant,
type Room as LivekitRoom,
type TrackPublication,
} from "livekit-client";
import { SyncState } from "matrix-js-sdk/lib/sync";
import { BehaviorSubject, combineLatest, map, of } from "rxjs";
@@ -72,6 +72,7 @@ export interface CallViewModelInputs {
roomMembers: RoomMember[];
livekitConnectionState$: Behavior<ConnectionState>;
speaking: Map<Participant, Behavior<boolean>>;
videoEnabled: Map<Participant, Behavior<boolean>>;
sharingScreen: Map<Participant, Behavior<boolean>>;
mediaDevices: MediaDevices;
initialSyncState: SyncState;
@@ -98,6 +99,7 @@ export function withCallViewModel(mode: MatrixRTCMode) {
ConnectionState.Connected,
),
speaking = new Map(),
videoEnabled = new Map(),
sharingScreen = new Map(),
mediaDevices = mockMediaDevices({}),
initialSyncState = SyncState.Syncing,
@@ -151,11 +153,19 @@ export function withCallViewModel(mode: MatrixRTCMode) {
.mockReturnValue(remoteParticipants$);
const mediaSpy = vi
.spyOn(ComponentsCore, "observeParticipantMedia")
.mockImplementation((p) =>
of({ participant: p } as Partial<
ComponentsCore.ParticipantMedia<LocalParticipant>
> as ComponentsCore.ParticipantMedia<LocalParticipant>),
);
.mockImplementation((p) => {
return (videoEnabled.get(p) ?? constant(false)).pipe(
map((videoEnabled) => ({
participant: p,
isMicrophoneEnabled: false,
isCameraEnabled: videoEnabled,
isScreenShareEnabled: false,
cameraTrack: {
isMuted: !videoEnabled,
} as unknown as TrackPublication,
})),
);
});
const eventsSpy = vi
.spyOn(ComponentsCore, "observeParticipantEvents")
.mockImplementation((p, ...eventTypes) => {

147
src/state/CallViewModel/localMember/HomeserverConnected.test.ts
View File

@@ -98,108 +98,181 @@ describe("createHomeserverConnected$", () => {
// LLM generated test cases. They are a bit overkill but I improved the mocking so it is
// easy enough to read them so I think they can stay.
// Note: gracePeriodMs is set to 0 to avoid debouncing delays in tests
it("is false when sync state is not Syncing", () => {
it("reports syncing reason when sync state is not Syncing", () => {
const hsConnected = createHomeserverConnected$(scope, client, session, 0);
expect(hsConnected.combined$.value).toBe(false);
expect(hsConnected.combined$.value).toEqual([false, "sync"]);
});
it("remains false while membership status is not Connected even if sync is Syncing", () => {
it("reports membership reason when sync is Syncing but membership is not Connected", () => {
const hsConnected = createHomeserverConnected$(scope, client, session, 0);
client.setSyncState(SyncState.Syncing);
expect(hsConnected.combined$.value).toBe(false); // membership still disconnected
expect(hsConnected.combined$.value).toEqual([false, "membership"]);
});
it("is false when membership status transitions to Connected but ProbablyLeft is true", () => {
it("reports probablyLeft reason when membership transitions to Connected but ProbablyLeft is true", () => {
const hsConnected = createHomeserverConnected$(scope, client, session, 0);
// Make sync loop OK
client.setSyncState(SyncState.Syncing);
// Indicate probable leave before connection
session.setProbablyLeft(true);
session.setMembershipStatus(Status.Connected);
expect(hsConnected.combined$.value).toBe(false);
expect(hsConnected.combined$.value).toEqual([false, "probablyLeft"]);
});
it("becomes true only when all three conditions are satisfied", () => {
it("becomes null (connected) only when all three conditions are satisfied", () => {
const hsConnected = createHomeserverConnected$(scope, client, session, 0);
// 1. Sync loop connected
client.setSyncState(SyncState.Syncing);
expect(hsConnected.combined$.value).toBe(false); // not yet membership connected
expect(hsConnected.combined$.value).toEqual([false, "membership"]); // not yet membership connected
// 2. Membership connected
session.setMembershipStatus(Status.Connected);
expect(hsConnected.combined$.value).toBe(true); // probablyLeft is false
expect(hsConnected.combined$.value).toEqual([true, null]); // probablyLeft is false
});
it("drops back to false when sync loop leaves Syncing", () => {
it("returns syncing reason when sync loop leaves Syncing", () => {
const hsConnected = createHomeserverConnected$(scope, client, session, 0);
// Reach connected state
client.setSyncState(SyncState.Syncing);
session.setMembershipStatus(Status.Connected);
expect(hsConnected.combined$.value).toBe(true);
expect(hsConnected.combined$.value).toEqual([true, null]);
// Sync loop error => should flip false
// Sync loop error => should report syncing reason
client.setSyncState(SyncState.Error);
expect(hsConnected.combined$.value).toBe(false);
expect(hsConnected.combined$.value).toEqual([false, "sync"]);
});
it("drops back to false when membership status becomes disconnected", () => {
it("returns membershipConnected reason when membership status becomes disconnected", () => {
const hsConnected = createHomeserverConnected$(scope, client, session, 0);
client.setSyncState(SyncState.Syncing);
session.setMembershipStatus(Status.Connected);
expect(hsConnected.combined$.value).toBe(true);
expect(hsConnected.combined$.value).toEqual([true, null]);
session.setMembershipStatus(Status.Disconnected);
expect(hsConnected.combined$.value).toBe(false);
expect(hsConnected.combined$.value).toEqual([false, "membership"]);
});
it("drops to false when ProbablyLeft is emitted after being true", () => {
it("returns certainlyConnected reason when ProbablyLeft is emitted", () => {
const hsConnected = createHomeserverConnected$(scope, client, session, 0);
client.setSyncState(SyncState.Syncing);
session.setMembershipStatus(Status.Connected);
expect(hsConnected.combined$.value).toBe(true);
expect(hsConnected.combined$.value).toEqual([true, null]);
session.setProbablyLeft(true);
expect(hsConnected.combined$.value).toBe(false);
expect(hsConnected.combined$.value).toEqual([false, "probablyLeft"]);
});
it("recovers to true if ProbablyLeft becomes false again while other conditions remain true", () => {
it("recovers to null (connected) if ProbablyLeft becomes false again while other conditions remain true", () => {
const hsConnected = createHomeserverConnected$(scope, client, session, 0);
client.setSyncState(SyncState.Syncing);
session.setMembershipStatus(Status.Connected);
expect(hsConnected.combined$.value).toBe(true);
expect(hsConnected.combined$.value).toEqual([true, null]);
session.setProbablyLeft(true);
expect(hsConnected.combined$.value).toBe(false);
expect(hsConnected.combined$.value).toEqual([false, "probablyLeft"]);
// Simulate clearing the flag (in realistic scenario membership manager would update)
session.setProbablyLeft(false);
expect(hsConnected.combined$.value).toBe(true);
expect(hsConnected.combined$.value).toEqual([true, null]);
});
it("composite sequence reflects each individual failure reason", () => {
const hsConnected = createHomeserverConnected$(scope, client, session, 0);
// Initially false (sync error + disconnected + not probably left)
expect(hsConnected.combined$.value).toBe(false);
// Initially: sync error + membership disconnected → syncing wins (highest priority)
expect(hsConnected.combined$.value).toEqual([false, "sync"]);
// Fix sync only
// Fix sync only → membershipConnected is now the blocker
client.setSyncState(SyncState.Syncing);
expect(hsConnected.combined$.value).toBe(false);
expect(hsConnected.combined$.value).toEqual([false, "membership"]);
// Fix membership
// Fix membership → all conditions satisfied
session.setMembershipStatus(Status.Connected);
expect(hsConnected.combined$.value).toBe(true);
expect(hsConnected.combined$.value).toEqual([true, null]);
// Introduce probablyLeft -> false
// Introduce probablyLeft → certainlyConnected
session.setProbablyLeft(true);
expect(hsConnected.combined$.value).toBe(false);
expect(hsConnected.combined$.value).toEqual([false, "probablyLeft"]);
// Restore notProbablyLeft -> true again
// Restore notProbablyLeft → connected again
session.setProbablyLeft(false);
expect(hsConnected.combined$.value).toBe(true);
expect(hsConnected.combined$.value).toEqual([true, null]);
// Drop sync -> false
// Drop sync → syncing reason
client.setSyncState(SyncState.Error);
expect(hsConnected.combined$.value).toBe(false);
expect(hsConnected.combined$.value).toEqual([false, "sync"]);
});
});
describe("createHomeserverConnected$ - combined$ reason values", () => {
let scope: ObservableScope;
let client: MockMatrixClient;
let session: MockMatrixRTCSession;
beforeEach(() => {
scope = new ObservableScope();
// Start with sync failing and membership disconnected
client = new MockMatrixClient(SyncState.Error);
session = new MockMatrixRTCSession({
membershipStatus: Status.Disconnected,
probablyLeft: false,
});
});
afterEach(() => {
scope.end();
});
it("is [true, null] when all three conditions are satisfied", () => {
const { combined$ } = createHomeserverConnected$(scope, client, session, 0);
client.setSyncState(SyncState.Syncing);
session.setMembershipStatus(Status.Connected);
expect(combined$.value).toEqual([true, null]);
});
it("reports syncing when sync loop is not Syncing", () => {
const { combined$ } = createHomeserverConnected$(scope, client, session, 0);
// client starts with SyncState.Error, membership also disconnected
expect(combined$.value).toEqual([false, "sync"]);
});
it("reports membershipConnected when sync is fine but membership is not Connected", () => {
const { combined$ } = createHomeserverConnected$(scope, client, session, 0);
client.setSyncState(SyncState.Syncing);
// session still Status.Disconnected
expect(combined$.value).toEqual([false, "membership"]);
});
it("reports certainlyConnected when probablyLeft is true", () => {
const { combined$ } = createHomeserverConnected$(scope, client, session, 0);
client.setSyncState(SyncState.Syncing);
session.setMembershipStatus(Status.Connected);
session.setProbablyLeft(true);
expect(combined$.value).toEqual([false, "probablyLeft"]);
});
it("prioritises syncing over membershipConnected when both fail", () => {
const { combined$ } = createHomeserverConnected$(scope, client, session, 0);
// Both sync (Error) and membership (Disconnected) are failing
expect(combined$.value).toEqual([false, "sync"]);
});
it("updates reason as conditions change", () => {
const { combined$ } = createHomeserverConnected$(scope, client, session, 0);
// Initially: syncing fails
expect(combined$.value).toEqual([false, "sync"]);
// Fix sync → membershipConnected is now the blocker
client.setSyncState(SyncState.Syncing);
expect(combined$.value).toEqual([false, "membership"]);
// Fix membership → probablyLeft makes certainlyConnected fail
session.setProbablyLeft(true);
session.setMembershipStatus(Status.Connected);
expect(combined$.value).toEqual([false, "probablyLeft"]);
// Clear probablyLeft → all conditions satisfied
session.setProbablyLeft(false);
expect(combined$.value).toEqual([true, null]);
});
});
@@ -231,8 +304,8 @@ describe("createHomeserverConnected$ - Grace Period", () => {
GRACE_PERIOD,
);
expectObservable(hsConnected.combined$).toBe(expectedConnectedMarbles, {
y: true,
n: false,
y: [true, null],
n: [false, "sync"],
});
});
}

38
src/state/CallViewModel/localMember/HomeserverConnected.ts
View File

@@ -22,13 +22,13 @@ import {
switchMap,
of,
delay,
combineLatest,
} from "rxjs";
import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
import { Config } from "../../../config/Config";
import { type ObservableScope } from "../../ObservableScope";
import { type Behavior } from "../../Behavior";
import { and$ } from "../../../utils/observable";
import { type NodeStyleEventEmitter } from "../../../utils/test";
/**
@@ -36,8 +36,14 @@ import { type NodeStyleEventEmitter } from "../../../utils/test";
*/
const logger = rootLogger.getChild("[HomeserverConnected]");
export type HomeserverDisconnectReason = "sync" | "membership" | "probablyLeft";
export interface HomeserverConnected {
combined$: Behavior<boolean>;
/**
* Emits `[true, null]` when the homeserver connection is healthy, or
* `[false, reason]` when one of the three sub-conditions fails.
*/
combined$: Behavior<[boolean, HomeserverDisconnectReason | null]>;
rtsSession$: Behavior<Status>;
}
@@ -45,10 +51,11 @@ export interface HomeserverConnected {
* Behavior representing whether we consider ourselves connected to the Matrix homeserver
* for the purposes of a MatrixRTC session.
*
* Becomes FALSE if ANY sub-condition is fulfilled:
* 1. Sync loop is not in SyncState.Syncing (after grace period)
* 2. membershipStatus !== Status.Connected
* 3. probablyLeft === true
* `combined$` emits `null` when all conditions are satisfied, or the first failing
* reason (priority: syncing > membershipConnected > certainlyConnected):
* 1. Sync loop is not in SyncState.Syncing (after grace period) → "sync"
* 2. membershipStatus !== Status.Connected → "membership"
* 3. probablyLeft === true → "probablyLeft"
*
* @param scope - The observable scope for lifecycle management.
* @param client - The Matrix client to monitor sync state.
@@ -109,9 +116,22 @@ export function createHomeserverConnected$(
);
const combined$ = scope.behavior(
and$(syncing$, membershipConnected$, certainlyConnected$).pipe(
tap((connected) => {
logger.info(`Homeserver connected update: ${connected}`);
combineLatest([syncing$, membershipConnected$, certainlyConnected$]).pipe(
map(
([syncing, membership, certainly]): [
boolean,
HomeserverDisconnectReason | null,
] => {
if (!syncing) return [false, "sync"];
if (!membership) return [false, "membership"];
if (!certainly) return [false, "probablyLeft"];
return [true, null];
},
),
tap(([connected, reason]) => {
logger.info(
`Homeserver connected update: ${connected ? "connected" : reason}`,
);
}),
),
);

224
src/state/CallViewModel/localMember/LocalMember.test.ts
View File

@@ -11,13 +11,23 @@ import {
type LivekitTransportConfig,
type MatrixRTCSession,
} from "matrix-js-sdk/lib/matrixrtc";
import { describe, expect, it, vi } from "vitest";
import {
describe,
expect,
it,
vi,
beforeAll,
afterAll,
beforeEach,
} from "vitest";
import { AutoDiscovery } from "matrix-js-sdk/lib/autodiscovery";
import { BehaviorSubject, map, of } from "rxjs";
import { logger } from "matrix-js-sdk/lib/logger";
import { type LocalParticipant, type LocalTrack } from "livekit-client";
import { PosthogAnalytics } from "../../../analytics/PosthogAnalytics";
import { MatrixRTCMode } from "../../../settings/settings";
import { type HomeserverDisconnectReason } from "./HomeserverConnected";
import {
flushPromises,
mockConfig,
@@ -215,9 +225,13 @@ describe("LocalMembership", () => {
createPublisherFactory: vi.fn(),
joinMatrixRTC: async (): Promise<void> => {},
homeserverConnected: {
combined$: constant(true),
combined$: constant<[boolean, HomeserverDisconnectReason | null]>([
true,
null,
]),
rtsSession$: constant(RTCMemberStatus.Connected),
},
roomId: "!test-room-id:example.org",
};
it("throws error on missing RTC config error", () => {
@@ -667,4 +681,210 @@ describe("LocalMembership", () => {
// expect(publishers[0].stopTracks).toHaveBeenCalled();
});
// TODO add tests for matrix local matrix participation.
describe("reconnecting analytics", () => {
beforeAll(() => {
mockConfig();
});
beforeEach(() => {
vi.restoreAllMocks();
});
afterAll(() => {
PosthogAnalytics.resetInstance();
});
it("does not fire CallReconnecting for the initial non-connected state at startup", async () => {
const scope = new ObservableScope();
const trackSpy = vi.spyOn(
PosthogAnalytics.instance.eventCallReconnecting,
"track",
);
// Simulate startup where membership isn't established yet
const hsReason$ = new BehaviorSubject<
[boolean, HomeserverDisconnectReason | null]
>([false, "membership"]);
const connectionManagerData = new ConnectionManagerData();
connectionManagerData.add(connectionTransportAConnected, []);
createLocalMembership$({
scope,
...defaultCreateLocalMemberValues,
homeserverConnected: {
combined$: hsReason$,
rtsSession$: constant(RTCMemberStatus.Connected),
},
connectionManager: {
connectionManagerData$: constant(new Epoch(connectionManagerData)),
},
localTransport$: new BehaviorSubject({
advertised$: new BehaviorSubject(aTransport),
active$: new BehaviorSubject(aTransportWithSFUConfig),
}),
});
await flushPromises();
// Membership is established — call is now connected
hsReason$.next([true, null]);
expect(trackSpy).not.toHaveBeenCalled();
scope.end();
});
it("fires CallReconnecting with homeserver reason and duration when reconnected", async () => {
const scope = new ObservableScope();
const trackSpy = vi.spyOn(
PosthogAnalytics.instance.eventCallReconnecting,
"track",
);
const hsReason$ = new BehaviorSubject<
[boolean, HomeserverDisconnectReason | null]
>([true, null]);
const connectionManagerData = new ConnectionManagerData();
connectionManagerData.add(connectionTransportAConnected, []);
createLocalMembership$({
scope,
...defaultCreateLocalMemberValues,
homeserverConnected: {
combined$: hsReason$,
rtsSession$: constant(RTCMemberStatus.Connected),
},
connectionManager: {
connectionManagerData$: constant(new Epoch(connectionManagerData)),
},
localTransport$: new BehaviorSubject({
advertised$: new BehaviorSubject(aTransport),
active$: new BehaviorSubject(aTransportWithSFUConfig),
}),
});
await flushPromises();
hsReason$.next([false, "sync"]);
hsReason$.next([true, null]);
expect(trackSpy).toHaveBeenCalledWith(
defaultCreateLocalMemberValues.roomId,
"sync",
expect.any(Number),
);
scope.end();
});
it("reports livekit reason when livekit disconnects then reconnects", async () => {
const scope = new ObservableScope();
const trackSpy = vi.spyOn(
PosthogAnalytics.instance.eventCallReconnecting,
"track",
);
const connectionState$ = new BehaviorSubject<ConnectionState>(
ConnectionState.LivekitConnected,
);
const mutableConnection = {
...connectionTransportAConnected,
state$: connectionState$,
} as unknown as Connection;
const connectionManagerData = new ConnectionManagerData();
connectionManagerData.add(mutableConnection, []);
createLocalMembership$({
scope,
...defaultCreateLocalMemberValues,
homeserverConnected: {
combined$: new BehaviorSubject<
[boolean, HomeserverDisconnectReason | null]
>([true, null]),
rtsSession$: constant(RTCMemberStatus.Connected),
},
connectionManager: {
connectionManagerData$: constant(new Epoch(connectionManagerData)),
},
localTransport$: new BehaviorSubject({
advertised$: new BehaviorSubject(aTransport),
active$: new BehaviorSubject(aTransportWithSFUConfig),
}),
});
await flushPromises();
connectionState$.next(ConnectionState.LivekitDisconnected);
connectionState$.next(ConnectionState.LivekitConnected);
expect(trackSpy).toHaveBeenCalledWith(
defaultCreateLocalMemberValues.roomId,
"livekit",
expect.any(Number),
);
scope.end();
});
it("fires one event per completed reconnection cycle", async () => {
const scope = new ObservableScope();
const trackSpy = vi.spyOn(
PosthogAnalytics.instance.eventCallReconnecting,
"track",
);
const hsReason$ = new BehaviorSubject<
[boolean, HomeserverDisconnectReason | null]
>([true, null]);
const connectionManagerData = new ConnectionManagerData();
connectionManagerData.add(connectionTransportAConnected, []);
createLocalMembership$({
scope,
...defaultCreateLocalMemberValues,
homeserverConnected: {
combined$: hsReason$,
rtsSession$: constant(RTCMemberStatus.Connected),
},
connectionManager: {
connectionManagerData$: constant(new Epoch(connectionManagerData)),
},
localTransport$: new BehaviorSubject({
advertised$: new BehaviorSubject(aTransport),
active$: new BehaviorSubject(aTransportWithSFUConfig),
}),
});
await flushPromises();
hsReason$.next([false, "membership"]);
hsReason$.next([true, null]);
hsReason$.next([false, "probablyLeft"]);
hsReason$.next([false, "sync"]);
hsReason$.next([false, "membership"]);
hsReason$.next([true, null]);
expect(trackSpy).toHaveBeenCalledTimes(2);
expect(trackSpy).toHaveBeenNthCalledWith(
1,
defaultCreateLocalMemberValues.roomId,
"membership",
expect.any(Number),
);
expect(trackSpy).toHaveBeenNthCalledWith(
2,
defaultCreateLocalMemberValues.roomId,
"probablyLeft",
expect.any(Number),
);
scope.end();
});
});
});

58
src/state/CallViewModel/localMember/LocalMember.ts
View File

@@ -61,7 +61,6 @@ import {
type FailedToStartError,
} from "../remoteMembers/Connection.ts";
import { type HomeserverConnected } from "./HomeserverConnected.ts";
import { and$ } from "../../../utils/observable.ts";
import { type LocalTransport } from "./LocalTransport.ts";
import { areLivekitTransportsEqual } from "../remoteMembers/MatrixLivekitMembers.ts";
@@ -129,6 +128,7 @@ interface Props {
createPublisherFactory: (connection: Connection) => Publisher;
joinMatrixRTC: (transport: LivekitTransportConfig) => void;
homeserverConnected: HomeserverConnected;
roomId: string;
localTransport$: Behavior<LocalTransport>;
matrixRTCSession: Pick<
MatrixRTCSession,
@@ -152,6 +152,7 @@ interface Props {
* @param props.logger The logger to use.
* @param props.muteStates The mute states for video and audio.
* @param props.matrixRTCSession The matrix RTC session to join.
* @param props.roomId The room ID used as the call identifier in analytics events.
* @returns
* - publisher: The handle to create tracks and publish them to the room.
* - connected$: the current connection state. Including matrix server and livekit server connection. (only considering the livekit server we are using for our own media publication)
@@ -169,6 +170,7 @@ export const createLocalMembership$ = ({
logger: parentLogger,
muteStates,
matrixRTCSession,
roomId: roomId,
}: Props): {
/**
* This request to start audio and video tracks.
@@ -494,20 +496,35 @@ export const createLocalMembership$ = ({
);
/**
* Whether we are "fully" connected to the call. Accounts for both the
* connection to the MatrixRTC session and the LiveKit publish connection.
* The disconnect reason for the combined Matrix + LiveKit connection, or null
* when fully connected. Homeserver reasons take priority over livekit.
* Both connectivity state and reason come from the same combineLatest emission,
* avoiding any race between the two.
*/
const matrixAndLivekitConnected$ = scope.behavior(
and$(
const connectionDisconnectReason$ = scope.behavior(
combineLatest([
homeserverConnected.combined$,
localConnectionState$.pipe(
map((state) => state === ConnectionState.LivekitConnected),
),
).pipe(
]).pipe(
map(([[hsConnected, hsReason], livekitConnected]) => {
if (!hsConnected) return hsReason!;
if (!livekitConnected) return "livekit" as const;
return null;
}),
tap((v) => logger.debug("livekit+matrix: Connected state changed", v)),
),
);
/**
* Whether we are "fully" connected to the call. Accounts for both the
* connection to the MatrixRTC session and the LiveKit publish connection.
*/
const matrixAndLivekitConnected$ = scope.behavior(
connectionDisconnectReason$.pipe(map((reason) => reason === null)),
);
/**
* Whether we should tell the user that we're reconnecting to the call.
*/
@@ -519,6 +536,33 @@ export const createLocalMembership$ = ({
false,
);
let reconnectStart: {
time: number;
reason: NonNullable<(typeof connectionDisconnectReason$)["value"]>;
} | null = null;
connectionDisconnectReason$
.pipe(distinctUntilChanged(), pairwise(), scope.bind())
.subscribe(([prev, reason]) => {
if (reason !== null) {
// Only begin tracking when transitioning FROM connected (null → non-null).
// This prevents the initial startup phase — where we may be non-null before
// the first real connection — from being counted as a reconnect.
if (prev === null) {
reconnectStart ??= { time: Date.now(), reason };
}
} else if (reconnectStart !== null) {
PosthogAnalytics.instance.eventCallReconnecting.track(
roomId,
reconnectStart.reason,
(Date.now() - reconnectStart.time) / 1000,
);
PosthogAnalytics.instance.eventCallEnded.cacheReconnecting(
reconnectStart.reason,
);
reconnectStart = null;
}
});
// inform the widget about the connect and disconnect intent from the user.
scope
.behavior(joinAndPublishRequested$.pipe(pairwise(), scope.bind()), [
@@ -606,7 +650,7 @@ export const createLocalMembership$ = ({
// TODO refactor this based no livekitState$
combineLatest([participant$, homeserverConnected.combined$])
.pipe(scope.bind())
.subscribe(([participant, connected]) => {
.subscribe(([participant, [connected]]) => {
if (!participant) return;
const publications = participant.trackPublications.values();
if (connected) {

6
src/state/CallViewModel/localMember/Publisher.ts
View File

@@ -379,10 +379,11 @@ export class Publisher {
if (!this.shouldPublish && enable) {
await this.pauseUpstreams(lkRoom, [Track.Source.Microphone]);
}
return enable;
} catch (e) {
this.logger.error("Failed to update LiveKit audio input mute state", e);
return lkRoom.localParticipant.isMicrophoneEnabled;
}
return lkRoom.localParticipant.isMicrophoneEnabled;
});
this.muteStates.video.setHandler(async (enable) => {
try {
@@ -393,10 +394,11 @@ export class Publisher {
if (!this.shouldPublish && enable) {
await this.pauseUpstreams(lkRoom, [Track.Source.Camera]);
}
return enable;
} catch (e) {
this.logger.error("Failed to update LiveKit video input mute state", e);
return lkRoom.localParticipant.isCameraEnabled;
}
return lkRoom.localParticipant.isCameraEnabled;
});
}

10
src/state/GridLikeLayout.ts
View File

@@ -5,7 +5,13 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type Layout, type LayoutMedia } from "./layout-types.ts";
import { type BehaviorSubject } from "rxjs";
import {
type Alignment,
type Layout,
type LayoutMedia,
} from "./layout-types.ts";
import { type TileStore } from "./TileStore";
export type GridLikeLayoutType =
@@ -19,6 +25,7 @@ export type GridLikeLayoutType =
*/
export function gridLikeLayout(
media: LayoutMedia & { type: GridLikeLayoutType },
spotlightAlignment$: BehaviorSubject<Alignment>,
visibleTiles: number,
setVisibleTiles: (value: number) => void,
prevTiles: TileStore,
@@ -37,6 +44,7 @@ export function gridLikeLayout(
type: media.type,
spotlight: tiles.spotlightTile,
grid: tiles.gridTiles,
spotlightAlignment$,
setVisibleTiles,
} as Layout & { type: GridLikeLayoutType },
tiles,

20
src/state/OneOnOneLayout.ts → src/state/OneOnOneLandscapeLayout.ts
View File

@@ -1,29 +1,39 @@
/*
Copyright 2024 New Vector Ltd.
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type OneOnOneLayout, type OneOnOneLayoutMedia } from "./layout-types";
import { type BehaviorSubject } from "rxjs";
import {
type Alignment,
type OneOnOneLandscapeLayout,
type OneOnOneLandscapeLayoutMedia,
} from "./layout-types";
import { type TileStore } from "./TileStore";
/**
* Produces a one-on-one layout with the given media.
* Produces a one-on-one landscape layout with the given media.
*/
export function oneOnOneLayout(
media: OneOnOneLayoutMedia,
export function oneOnOneLandscapeLayout(
media: OneOnOneLandscapeLayoutMedia,
pipAlignment$: BehaviorSubject<Alignment>,
prevTiles: TileStore,
): [OneOnOneLayout, TileStore] {
): [OneOnOneLandscapeLayout, TileStore] {
const update = prevTiles.from(2);
update.registerGridTile(media.pip);
update.registerGridTile(media.spotlight);
const tiles = update.build();
return [
{
type: media.type,
spotlight: tiles.gridTilesByMedia.get(media.spotlight)!,
pip: tiles.gridTilesByMedia.get(media.pip)!,
pipAlignment$,
},
tiles,
];

43
src/state/OneOnOnePortraitLayout.ts Normal file
View File

@@ -0,0 +1,43 @@
/*
Copyright 2024 New Vector Ltd.
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type BehaviorSubject } from "rxjs";
import {
type Alignment,
type OneOnOnePortraitLayout,
type OneOnOnePortraitLayoutMedia,
} from "./layout-types";
import { type TileStore } from "./TileStore";
import { type Behavior } from "./Behavior";
/**
* Produces a one-on-one portrait layout with the given media.
*/
export function oneOnOnePortraitLayout(
media: OneOnOnePortraitLayoutMedia,
pipSize$: Behavior<"sm" | "lg">,
pipAlignment$: BehaviorSubject<Alignment>,
prevTiles: TileStore,
): [OneOnOnePortraitLayout, TileStore] {
const update = prevTiles.from(media.pip === undefined ? 0 : 1);
update.registerSpotlight([media.spotlight], true);
if (media.pip !== undefined) update.registerGridTile(media.pip);
const tiles = update.build();
return [
{
type: media.type,
spotlight: tiles.spotlightTile!,
pip: media.pip && tiles.gridTilesByMedia.get(media.pip),
pipSize$,
pipAlignment$,
},
tiles,
];
}

7
src/state/SpotlightExpandedLayout.ts
View File

@@ -5,7 +5,10 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type BehaviorSubject } from "rxjs";
import {
type Alignment,
type SpotlightExpandedLayout,
type SpotlightExpandedLayoutMedia,
} from "./layout-types";
@@ -16,6 +19,7 @@ import { type TileStore } from "./TileStore";
*/
export function spotlightExpandedLayout(
media: SpotlightExpandedLayoutMedia,
pipAlignment$: BehaviorSubject<Alignment>,
prevTiles: TileStore,
): [SpotlightExpandedLayout, TileStore] {
const update = prevTiles.from(1);
@@ -27,7 +31,8 @@ export function spotlightExpandedLayout(
{
type: media.type,
spotlight: tiles.spotlightTile!,
pip: tiles.gridTiles[0],
pip: tiles.gridTiles.at(0),
pipAlignment$,
},
tiles,
];

46
src/state/layout-types.ts
View File

@@ -5,6 +5,8 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type BehaviorSubject } from "rxjs";
import { type LocalUserMediaViewModel } from "./media/LocalUserMediaViewModel.ts";
import { type MediaViewModel } from "./media/MediaViewModel.ts";
import { type RingingMediaViewModel } from "./media/RingingMediaViewModel.ts";
@@ -13,39 +15,53 @@ import {
type GridTileViewModel,
type SpotlightTileViewModel,
} from "./TileViewModel.ts";
import { type Behavior } from "./Behavior.ts";
export interface GridLayoutMedia {
type: "grid";
edgeToEdge: false;
spotlight?: MediaViewModel[];
grid: UserMediaViewModel[];
}
export interface SpotlightLandscapeLayoutMedia {
type: "spotlight-landscape";
edgeToEdge: false;
spotlight: MediaViewModel[];
grid: UserMediaViewModel[];
}
export interface SpotlightPortraitLayoutMedia {
type: "spotlight-portrait";
edgeToEdge: false;
spotlight: MediaViewModel[];
grid: UserMediaViewModel[];
}
export interface SpotlightExpandedLayoutMedia {
type: "spotlight-expanded";
edgeToEdge: boolean;
spotlight: MediaViewModel[];
pip?: UserMediaViewModel;
}
export interface OneOnOneLayoutMedia {
type: "one-on-one";
export interface OneOnOneLandscapeLayoutMedia {
type: "one-on-one-landscape";
edgeToEdge: false;
spotlight: UserMediaViewModel;
pip: LocalUserMediaViewModel | RingingMediaViewModel;
}
export interface OneOnOnePortraitLayoutMedia {
type: "one-on-one-portrait";
edgeToEdge: true;
spotlight: UserMediaViewModel | RingingMediaViewModel;
pip?: LocalUserMediaViewModel;
}
export interface PipLayoutMedia {
type: "pip";
edgeToEdge: boolean;
spotlight: MediaViewModel[];
}
@@ -54,13 +70,20 @@ export type LayoutMedia =
| SpotlightLandscapeLayoutMedia
| SpotlightPortraitLayoutMedia
| SpotlightExpandedLayoutMedia
| OneOnOneLayoutMedia
| OneOnOneLandscapeLayoutMedia
| OneOnOnePortraitLayoutMedia
| PipLayoutMedia;
export interface Alignment {
inline: "start" | "end";
block: "start" | "end";
}
export interface GridLayout {
type: "grid";
spotlight?: SpotlightTileViewModel;
grid: GridTileViewModel[];
spotlightAlignment$: BehaviorSubject<Alignment>;
setVisibleTiles: (value: number) => void;
}
@@ -82,12 +105,22 @@ export interface SpotlightExpandedLayout {
type: "spotlight-expanded";
spotlight: SpotlightTileViewModel;
pip?: GridTileViewModel;
pipAlignment$: BehaviorSubject<Alignment>;
}
export interface OneOnOneLayout {
type: "one-on-one";
export interface OneOnOneLandscapeLayout {
type: "one-on-one-landscape";
spotlight: GridTileViewModel;
pip: GridTileViewModel;
pipAlignment$: BehaviorSubject<Alignment>;
}
export interface OneOnOnePortraitLayout {
type: "one-on-one-portrait";
spotlight: SpotlightTileViewModel;
pip?: GridTileViewModel;
pipSize$: Behavior<"sm" | "lg">;
pipAlignment$: BehaviorSubject<Alignment>;
}
export interface PipLayout {
@@ -104,5 +137,6 @@ export type Layout =
| SpotlightLandscapeLayout
| SpotlightPortraitLayout
| SpotlightExpandedLayout
| OneOnOneLayout
| OneOnOneLandscapeLayout
| OneOnOnePortraitLayout
| PipLayout;