Matrix/element-call-Github
1
0
Fork 0
mirror of https://github.com/vector-im/element-call.git synced 2026-07-09 18:29:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'livekit' into ringing-intent

Browse Source
This commit is contained in:
Robin
2026-06-18 06:41:17 +02:00
parent 2ac6cdeb46 3ea89e8275
commit 8cda9777a5
21 changed files with 854 additions and 1564 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
src/state/CallViewModel/CallViewModel.test.ts
View File

@@ -6,7 +6,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { test, vi, onTestFinished, it, describe } from "vitest";
import { test, vi, onTestFinished, it, describe, expect } from "vitest";
import {
BehaviorSubject,
combineLatest,
@@ -899,6 +899,41 @@ describe.each([
},
);
// TODO add media to lk mocks
test("onPipMediaOrientationUpdate is called with the spotlight media orientation", () => {
// Set the spy before creating the view model so the initial call is captured
const onPipMediaOrientationUpdate = vi.fn();
window.controls.onPipMediaOrientationUpdate = onPipMediaOrientationUpdate;
onTestFinished(() => {
window.controls.onPipMediaOrientationUpdate = undefined;
});
withTestScheduler(({ behavior }) => {
// Alice starts as a regular participant, then shares her screen, then stops
const aliceSharingInputMarbles = "nyn";
withCallViewModel(
{
remoteParticipants$: constant([aliceParticipant]),
rtcMembers$: constant([localRtcMember, aliceRtcMember]),
sharingScreen: new Map([
[aliceParticipant, behavior(aliceSharingInputMarbles, yesNo)],
]),
},
() => {},
);
});
// Should be called exactly 3 times:
// 1. Initially with "portrait" (Alice is in spotlight as a user, default portrait orientation)
// 2. With "landscape" when Alice starts screen sharing (screen shares always use landscape)
// 3. With "portrait" again when Alice stops screen sharing and returns to user tile
expect(onPipMediaOrientationUpdate).toHaveBeenCalledTimes(3);
expect(onPipMediaOrientationUpdate).toHaveBeenNthCalledWith(1, "portrait");
expect(onPipMediaOrientationUpdate).toHaveBeenNthCalledWith(2, "landscape");
expect(onPipMediaOrientationUpdate).toHaveBeenNthCalledWith(3, "portrait");
});
test("PiP tile in expanded spotlight layout switches speakers without layout shifts", () => {
withTestScheduler(({ behavior, schedule, expectObservable }) => {
// Switch to spotlight immediately

37
src/state/CallViewModel/CallViewModel.ts
View File

@@ -769,11 +769,13 @@ export function createCallViewModel$(
pretendToBeDisconnected$: localMembership.reconnecting$,
displayName$: scope.behavior(
matrixMemberMetadataStore
.createDisplayNameBehavior$(userId)
.createDisplayNameBehavior$(scope, userId)
.pipe(map((name) => name ?? userId)),
),
mxcAvatarUrl$:
matrixMemberMetadataStore.createAvatarUrlBehavior$(userId),
mxcAvatarUrl$: matrixMemberMetadataStore.createAvatarUrlBehavior$(
scope,
userId,
),
handRaised$: scope.behavior(
handsRaised$.pipe(map((v) => v[mediaId]?.time ?? null)),
),
@@ -809,7 +811,7 @@ export function createCallViewModel$(
),
),
mxcAvatarUrl$:
matrixMemberMetadataStore.createAvatarUrlBehavior$(userId),
matrixMemberMetadataStore.createAvatarUrlBehavior$(scope, userId),
pickupState$,
intent,
}),
@@ -1187,6 +1189,33 @@ export function createCallViewModel$(
})),
);
spotlight$
.pipe(
switchMap((media) => {
let layout;
const pipMedia = media[0];
if (pipMedia === undefined) return of(undefined);
switch (pipMedia.type) {
case "user":
layout = pipMedia.videoOrientation$;
break;
case "ringing":
layout = of("landscape" as const);
break;
case "screen share":
layout = of("landscape" as const);
break;
}
return layout;
}),
scope.bind(),
)
.subscribe((orientation) => {
if (orientation === undefined) return;
logger.info("controls api pip orientation updated:", orientation);
window.controls.onPipMediaOrientationUpdate?.(orientation);
});
/**
* The media to be used to produce a layout.
*/

69
src/state/CallViewModel/remoteMembers/MatrixMemberMetadata.test.ts
View File

@@ -105,7 +105,7 @@ describe("MatrixMemberMetadata", () => {
}
it("should show our own user if present in rtc session and room", () => {
withTestScheduler(({ behavior, expectObservable }) => {
withTestScheduler(({ scope, behavior, expectObservable }) => {
fakeMemberWith({
userId: "@local:example.com",
rawDisplayName: "it's a me",
@@ -118,8 +118,10 @@ describe("MatrixMemberMetadata", () => {
memberships$,
createRoomMembers$(testScope, mockMatrixRoom),
);
const dn$ =
metadataStore.createDisplayNameBehavior$("@local:example.com");
const dn$ = metadataStore.createDisplayNameBehavior$(
scope,
"@local:example.com",
);
expectObservable(dn$).toBe("a", {
a: "it's a me",
@@ -146,7 +148,7 @@ describe("MatrixMemberMetadata", () => {
it("should get displayName for users", () => {
setUpBasicRoom();
withTestScheduler(({ behavior, expectObservable }) => {
withTestScheduler(({ scope, behavior, expectObservable }) => {
const memberships$ = behavior("a", {
a: [
mockRtcMembership("@alice:example.com", "DEVICE1"),
@@ -158,8 +160,10 @@ describe("MatrixMemberMetadata", () => {
memberships$,
createRoomMembers$(testScope, mockMatrixRoom),
);
const aliceDispName$ =
metadataStore.createDisplayNameBehavior$("@alice:example.com");
const aliceDispName$ = metadataStore.createDisplayNameBehavior$(
scope,
"@alice:example.com",
);
expectObservable(aliceDispName$).toBe("a", {
a: "Alice",
@@ -322,7 +326,7 @@ describe("MatrixMemberMetadata", () => {
});
it("should track individual member id with createDisplayNameBehavior", () => {
withTestScheduler(({ behavior, schedule, expectObservable }) => {
withTestScheduler(({ scope, behavior, schedule, expectObservable }) => {
setUpBasicRoom();
const BOB = "@bob:example.com";
const CARL = "@carl:example.com";
@@ -356,8 +360,8 @@ describe("MatrixMemberMetadata", () => {
createRoomMembers$(testScope, mockMatrixRoom),
);
const bob$ = metadataStore.createDisplayNameBehavior$(BOB);
const carl$ = metadataStore.createDisplayNameBehavior$(CARL);
const bob$ = metadataStore.createDisplayNameBehavior$(scope, BOB);
const carl$ = metadataStore.createDisplayNameBehavior$(scope, CARL);
expectObservable(bob$).toBe("abc-", {
a: undefined,
@@ -378,7 +382,7 @@ describe("MatrixMemberMetadata", () => {
});
it("should disambiguate users with invisible characters", () => {
withTestScheduler(({ behavior, expectObservable }) => {
withTestScheduler(({ scope, behavior, expectObservable }) => {
const bobRtcMember = mockRtcMembership("@bob:example.org", "BBBB");
const bobZeroWidthSpaceRtcMember = mockRtcMembership(
"@bob2:example.org",
@@ -411,12 +415,18 @@ describe("MatrixMemberMetadata", () => {
createRoomMembers$(testScope, mockMatrixRoom),
);
const bob$ =
metadataStore.createDisplayNameBehavior$("@bob:example.org");
const bob2$ =
metadataStore.createDisplayNameBehavior$("@bob2:example.org");
const carol$ =
metadataStore.createDisplayNameBehavior$("@carol:example.org");
const bob$ = metadataStore.createDisplayNameBehavior$(
scope,
"@bob:example.org",
);
const bob2$ = metadataStore.createDisplayNameBehavior$(
scope,
"@bob2:example.org",
);
const carol$ = metadataStore.createDisplayNameBehavior$(
scope,
"@carol:example.org",
);
expectObservable(bob$).toBe("ab", {
a: "Bob",
b: "Bob (@bob:example.org)",
@@ -517,7 +527,7 @@ describe("MatrixMemberMetadata", () => {
}
it("should use avatar url from room members", () => {
withTestScheduler(({ behavior, expectObservable }) => {
withTestScheduler(({ scope, behavior, expectObservable }) => {
fakeMemberWith({
userId: "@local:example.com",
});
@@ -536,11 +546,15 @@ describe("MatrixMemberMetadata", () => {
memberships$,
createRoomMembers$(testScope, mockMatrixRoom),
);
const local$ =
metadataStore.createAvatarUrlBehavior$("@local:example.com");
const local$ = metadataStore.createAvatarUrlBehavior$(
scope,
"@local:example.com",
);
const alice$ =
metadataStore.createAvatarUrlBehavior$("@alice:example.com");
const alice$ = metadataStore.createAvatarUrlBehavior$(
scope,
"@alice:example.com",
);
expectObservable(local$).toBe("a", {
a: "mxc://example.com/@local:example.com",
@@ -558,7 +572,7 @@ describe("MatrixMemberMetadata", () => {
});
it("should update on avatar change and user join/leave", () => {
withTestScheduler(({ behavior, schedule, expectObservable }) => {
withTestScheduler(({ scope, behavior, schedule, expectObservable }) => {
fakeMemberWith({ userId: "@carl:example.com" });
fakeMemberWith({ userId: "@bob:example.com" });
const memberships$ = behavior("ab-d", {
@@ -585,9 +599,14 @@ describe("MatrixMemberMetadata", () => {
},
});
const bob$ = metadataStore.createAvatarUrlBehavior$("@bob:example.com");
const carl$ =
metadataStore.createAvatarUrlBehavior$("@carl:example.com");
const bob$ = metadataStore.createAvatarUrlBehavior$(
scope,
"@bob:example.com",
);
const carl$ = metadataStore.createAvatarUrlBehavior$(
scope,
"@carl:example.com",
);
expectObservable(bob$).toBe("a---", {
a: "mxc://example.com/@bob:example.com",
});

14
src/state/CallViewModel/remoteMembers/MatrixMemberMetadata.ts
View File

@@ -115,8 +115,14 @@ export const createMatrixMemberMetadata$ = (
memberships$: Behavior<Pick<CallMembership, "userId">[]>,
roomMembers$: Behavior<RoomMemberMap>,
): {
createDisplayNameBehavior$: (userId: string) => Behavior<string | undefined>;
createAvatarUrlBehavior$: (userId: string) => Behavior<string | undefined>;
createDisplayNameBehavior$: (
scope: ObservableScope,
userId: string,
) => Behavior<string | undefined>;
createAvatarUrlBehavior$: (
scope: ObservableScope,
userId: string,
) => Behavior<string | undefined>;
displaynameMap$: Behavior<Map<string, string>>;
avatarMap$: Behavior<Map<string, string | undefined>>;
} => {
@@ -136,13 +142,13 @@ export const createMatrixMemberMetadata$ = (
),
);
return {
createDisplayNameBehavior$: (userId: string) =>
createDisplayNameBehavior$: (scope: ObservableScope, userId: string) =>
scope.behavior(
displaynameMap$.pipe(
map((displaynameMap) => displaynameMap.get(userId)),
),
),
createAvatarUrlBehavior$: (userId: string) =>
createAvatarUrlBehavior$: (scope: ObservableScope, userId: string) =>
scope.behavior(
roomMembers$.pipe(
map((roomMembers) => roomMembers.get(userId)?.getMxcAvatarUrl()),

13
src/state/media/UserMediaViewModel.ts
View File

@@ -48,6 +48,7 @@ export interface BaseUserMediaViewModel extends BaseMemberMediaViewModel {
audioEnabled$: Behavior<boolean>;
videoEnabled$: Behavior<boolean>;
videoFit$: Behavior<"cover" | "contain">;
videoOrientation$: Behavior<"landscape" | "portrait">;
toggleCropVideo: () => void;
/**
* The expected identity of the LiveKit participant. Exposed for debugging.
@@ -104,6 +105,7 @@ export function createBaseUserMedia(
{ width: number; height: number } | undefined
>(undefined);
const videoSize$ = videoSizeFromParticipant$(participant$);
return {
...createMemberMedia(scope, {
...inputs,
@@ -129,11 +131,14 @@ export function createBaseUserMedia(
videoEnabled$: scope.behavior(
media$.pipe(map((m) => m?.cameraTrack?.isMuted === false)),
),
videoFit$: videoFit$(
scope,
videoSizeFromParticipant$(participant$),
targetSize$,
videoOrientation$: scope.behavior(
videoSize$.pipe(
map((s) => (s ? s.width / s.height : 1)),
map((aspect) => (aspect > 1 ? "landscape" : "portrait")),
),
"portrait",
),
videoFit$: videoFit$(scope, videoSize$, targetSize$),
toggleCropVideo: () => toggleCropVideo$.next(),
rtcBackendIdentity,
handRaised$,

4
src/state/media/observeRtpStreamStats.ts
View File

@@ -32,7 +32,9 @@ export function observeRtpStreamStats$(
> {
return combineLatest([
observeTrackReference$(participant, source),
interval(1000).pipe(startWith(0)),
// The update frequency is high because we use this value to update the PiP orientation and the fit/fill video tile props based on that
// We want it to be responsive. For just the debug tools 1s would be sufficient.
interval(350).pipe(startWith(0)),
]).pipe(
switchMap(async ([trackReference]) => {
const track = trackReference?.publication?.track;