Merge branch 'livekit' into scope-leak-lint

This commit is contained in:
Robin
2026-06-18 12:49:40 +02:00
18 changed files with 1057 additions and 2055 deletions

253
.github/labels.yml vendored Normal file
View File

@@ -0,0 +1,253 @@
- name: "A-1:1"
description: "Calls between two people"
color: "bfd4f2"
- name: "A-Big-Grid"
description: "The freedom layout system used for >12 participants"
color: "bfd4f2"
- name: "A-Developer-Experience"
description: "Workflow of developing: building, linting, debugging, profiling, etc."
color: "c5def5"
- name: "A-E2EE"
description: "End-to-end encryption"
color: "bfd4f2"
- name: "A-Embedded"
description: "Using the app embedded within other Matrix clients (as a widget)"
color: "bfd4f2"
- name: "A-Feedback-Reporting"
description: "Reporting process for bugs, debug logs (rageshakes), suggestions"
color: "bfd4f2"
- name: "A-Freedom"
description: "Freedom layout, where participants can be rearranged and resized"
color: "bfd4f2"
- name: "A-Handset"
description: "Audio playback through the earpiece of a phone. Also known as 'earpiece mode' or 'handset mode'."
color: "bfd4f2"
- name: "A-Huddle"
description: "Ad-hoc calls in a room notifying others"
color: "bfd4f2"
- name: "A-Lobby"
description: "The page before joining a call"
color: "bfd4f2"
- name: "A-Login"
color: "bfd4f2"
- name: "A-Matrix2.0"
description: "Issues relating to the Matrix 2.0 / MSC4143 work, such as sticky events and multi-sfu"
color: "bfd4f2"
- name: "A-Media-Devices"
color: "BFD4F2"
- name: "A-Media-Quality"
description: "Distortions or glitches in audio/video"
color: "bfd4f2"
- name: "A-Meeting"
description: "Scheduled call on the calendar"
color: "bfd4f2"
- name: "A-Mobile"
description: "Using the app on a mobile device"
color: "bfd4f2"
- name: "A-Moderation"
description: "Access to calls and powers within calls"
color: "bfd4f2"
- name: "A-Performance"
color: "bfd4f2"
- name: "A-Reactions"
color: "bfd4f2"
- name: "A-Registration"
color: "bfd4f2"
- name: "A-Screen-Sharing"
color: "bfd4f2"
- name: "A-SDK"
description: "SDK for building MatrixRTC + LiveKit widgets"
color: "c5def5"
- name: "A-Settings"
color: "bfd4f2"
- name: "A-SFU"
description: "Routing calls through a selective forwarding unit"
color: "bfd4f2"
- name: "A-Signaling"
description: "Call signaling"
color: "bfd4f2"
- name: "A-Simulcast"
description: "Automatic selection of variable video resolutions"
color: "bfd4f2"
- name: "A-SPA"
description: "Standalone application accessed via call links"
color: "bfd4f2"
- name: "A-Spatial-Audio"
description: "Directional audio based on where a speaker appears on screen"
color: "bfd4f2"
- name: "A-Speech-Enhancement"
description: "Techniques to enhance the intelligibility of speech in calls"
color: "c5def5"
- name: "A-Split-Grid"
description: "The freedom layout system used for ≤12 participants"
color: "bfd4f2"
- name: "A-Spotlight"
description: "Spotlight layout, where the active speaker is foregrounded"
color: "bfd4f2"
- name: "A-Telemetry-Posthog"
description: "Share opt in usage data for optimizing the app via posthog"
color: "bfd4f2"
- name: "A-Testing"
description: "Integration tests, unit tests, etc."
color: "bfd4f2"
- name: "A-Video-Rooms"
description: "Rooms reserved exclusively for calling"
color: "bfd4f2"
- name: "A-Walkie-Talkie"
description: "Walkie-talkie / PTT (push-to-talk) mode"
color: "bfd4f2"
- name: "A11y"
description: "Accessibility"
color: "4ADEC0"
- name: "backport-candidate"
description: "Something that is a candidate for backport to a particular release branch"
color: "0B8D85"
- name: "customer-retainer"
color: "F44A5F"
- name: "dependencies"
description: "Pull requests that update a dependency file"
color: "0366d6"
- name: "development build"
description: "runs yarn build process in development mode"
color: "1d76db"
- name: "Discord"
description: "Use case familiar to Discord users"
color: "3670d2"
- name: "docker build"
description: "Creates a docker image for this PR"
color: "0e8a16"
- name: "EPIC"
color: "5319E7"
- name: "good first issue"
description: "Good for newcomers"
color: "7057ff"
- name: "Help Wanted"
description: "Community contributions are welcome!"
color: "159818"
- name: "I18n"
description: "Internationalisation"
color: "d4c5f9"
- name: "O-Frequent"
description: "Affects or can be seen by most users regularly or impacts most users' first experience"
color: "0052CC"
- name: "O-Occasional"
description: "Affects or can be seen by some users regularly or most users rarely"
color: "1D76DB"
- name: "O-Uncommon"
description: "Most users are unlikely to come across this or unexpected workflow"
color: "C5DEF5"
- name: "p1"
description: "Must fix/implement before this is usable as a product"
color: "D93F0B"
- name: "p2"
description: "Should fix/implement, but not at the expense of p1s"
color: "FBCA04"
- name: "p3"
description: "Could fix/implement when time allows"
color: "0E8A16"
- name: "PR-Breaking-Change"
description: "A Pull request that changes EC in a way that is incompatible to the previous version."
color: "D93F0B"
- name: "PR-Bug-Fix"
description: "Release note category. A PR that fixes a bug."
color: "C2E0C6"
- name: "PR-Developer-Experience"
description: "Release note category. A PR that does not change EC but improves working with the repository."
color: "C2E0C6"
- name: "PR-Documentation"
description: "Release note category. A PR that improves the documentation."
color: "C2E0C6"
- name: "PR-Feature"
description: "Release note category. A PR that introduces a new user facing feature."
color: "C2E0C6"
- name: "PR-Improvement"
description: "Release note category. A PR that improves EC's performance or stability."
color: "C2E0C6"
- name: "PR-Task"
description: "Release note category. A PR that is hidden from release note."
color: "C2E0C6"
- name: "Privacy"
color: "f41192"
- name: "Roadmap"
color: "57457E"
- name: "S-Critical"
description: "Prevents work, causes data loss and/or has no workaround"
color: "bd0026"
- name: "S-Major"
description: "Severely degrades major functionality or product features, with no satisfactory workaround"
color: "fc4e2a"
- name: "S-Minor"
description: "Impairs non-critical functionality or suitable workarounds exist"
color: "feb24c"
- name: "S-Tolerable"
description: "Low/no impact on users"
color: "ffeda0"
- name: "Security"
color: "b3e5fc"
- name: "storybook build"
description: "Build and deploy the storybook frontend to netlify."
color: "45cd61"
- name: "T-Defect"
description: "Something isn't working: bugs, crashes, hangs, vulnerabilities, or other reported problems"
color: "98e6ae"
- name: "T-Enhancement"
description: "New features, changes in functionality, performance boosts, user-facing improvements"
color: "98e6ae"
- name: "T-Other"
description: "Questions, user support, anything else"
color: "98e6ae"
- name: "T-Task"
description: "Refactoring, enabling or disabling functionality, other engineering tasks"
color: "98e6ae"
- name: "X-Blocked"
description: "Cannot be merged due to external dependencies"
color: "ff7979"
- name: "X-Cannot-Reproduce"
description: "Needs reproduction steps"
color: "ff7979"
- name: "X-Needs-Design"
description: "May require input from the design team"
color: "ff7979"
- name: "X-Needs-Info"
description: "This issue is blocked awaiting information from the reporter"
color: "ff7979"
- name: "X-Needs-Investigation"
color: "ff7979"
- name: "X-Needs-Product"
description: "More input needed from the Product team"
color: "ff7979"
- name: "X-Regression"
color: "ff7979"
- name: "X-Release-Blocker"
color: "ff7979"
- name: "X-Spec-Changes"
description: "May require spec changes"
color: "ff7979"
- name: "X-Won't-Fix"
description: "This will not be worked on"
color: "ff7979"
- name: "Z-Community-Testing"
description: "Issues found during the community testing sessions"
color: "efefef"
- name: "Z-Could"
color: "ededed"
- name: "Z-Design"
color: "ededed"
- name: "Z-Flaky-Test"
color: "aaaaaa"
- name: "Z-Media-Failure"
description: "Someone's audio or video isn't coming through"
color: "ededed"
- name: "Z-Must"
color: "ededed"
- name: "Z-Platform-Specific"
color: "ededed"
- name: "Z-Power-Users"
color: "ededed"
- name: "Z-ProductPolish"
color: "aaaaaa"
- name: "Z-Should"
color: "ededed"
- name: "Z-Splitbrain"
description: "Someone who should be on the call isn't showing up"
color: "ededed"

23
.github/workflows/sync-labels.yml vendored Normal file
View File

@@ -0,0 +1,23 @@
name: Sync labels
on:
workflow_dispatch: {}
push:
branches:
- livekit
paths:
- .github/labels.yml
- .github/workflows/sync-labels.yml
permissions: {} # We use ELEMENT_BOT_TOKEN instead
jobs:
sync-labels:
uses: element-hq/element-meta/.github/workflows/sync-labels.yml@7f2f93fb9b52ece7a0998f60e64862aa203c1746
with:
LABELS: |
.github/labels.yml
DELETE: true
WET: true
secrets:
ELEMENT_BOT_TOKEN: ${{ secrets.ELEMENT_BOT_TOKEN }}

View File

@@ -359,6 +359,16 @@ Usage and other technical details about the project can be found here:
[**Docs**](./docs/README.md) [**Docs**](./docs/README.md)
## GitHub Labels
GitHub labels in this repository are maintained in the [`labels.yml`](.github/labels.yml) file and
automatically synced to GitHub using the [`sync-labels` workflow](.github/workflows/sync-labels.yml).
We do this so that we can reuse the labels between repositories.
> [!WARNING]
> Do not manually edit labels in the GitHub UI. Any manual changes will be overridden by the
> workflow on its next invocation.
## 📝 Copyright & License ## 📝 Copyright & License
Copyright 2021-2025 New Vector Ltd Copyright 2021-2025 New Vector Ltd

View File

@@ -1,20 +0,0 @@
module.exports = {
presets: [
[
"@babel/preset-env",
{
targets: {
node: "current",
},
},
],
[
"@babel/preset-react",
{
runtime: "automatic",
},
],
"@babel/preset-typescript",
],
plugins: ["babel-plugin-transform-vite-meta-env"],
};

View File

@@ -40,10 +40,6 @@
"build-storybook": "storybook build" "build-storybook": "storybook build"
}, },
"devDependencies": { "devDependencies": {
"@babel/core": "^7.16.5",
"@babel/preset-env": "^7.29.5",
"@babel/preset-react": "^7.22.15",
"@babel/preset-typescript": "^7.23.0",
"@codecov/vite-plugin": "^1.3.0", "@codecov/vite-plugin": "^1.3.0",
"@fontsource/inconsolata": "^5.1.0", "@fontsource/inconsolata": "^5.1.0",
"@fontsource/inter": "^5.1.0", "@fontsource/inter": "^5.1.0",
@@ -85,11 +81,10 @@
"@use-gesture/react": "^10.2.11", "@use-gesture/react": "^10.2.11",
"@vector-im/compound-design-tokens": "^10.0.0", "@vector-im/compound-design-tokens": "^10.0.0",
"@vector-im/compound-web": "^9.3.0", "@vector-im/compound-web": "^9.3.0",
"@vitejs/plugin-react": "^4.0.1", "@vitejs/plugin-react": "^6.0.2",
"@vitest/browser-playwright": "^4.1.5", "@vitest/browser-playwright": "^4.1.5",
"@vitest/coverage-v8": "^4.0.18", "@vitest/coverage-v8": "^4.0.18",
"@vitest/ui": "4.1.7", "@vitest/ui": "4.1.7",
"babel-plugin-transform-vite-meta-env": "^1.0.3",
"classnames": "^2.3.1", "classnames": "^2.3.1",
"copy-to-clipboard": "^3.3.3", "copy-to-clipboard": "^3.3.3",
"eslint": "^8.14.0", "eslint": "^8.14.0",
@@ -150,17 +145,5 @@
"vitest": "^4.1.5", "vitest": "^4.1.5",
"vitest-axe": "^1.0.0-pre.3" "vitest-axe": "^1.0.0-pre.3"
}, },
"pnpm": { "packageManager": "pnpm@11.6.0+sha512.9a36518224080c6fe5165afdcfe79bfa118c29be703f3f462b1e32efe1e98e47e8750b148e08286250aad4113cc7993ca413c4e2cd447752708c2ee5751bc95f"
"overrides": {
"@livekit/components-core>rxjs": "^7.8.1",
"@livekit/track-processors>@mediapipe/tasks-vision": "^0.10.18",
"minimatch": "^10.2.3",
"tar": "^7.5.11",
"glob": "^10.5.0",
"qs": "^6.14.1",
"js-yaml": "^4.1.1",
"esbuild": "^0.28.0"
}
},
"packageManager": "pnpm@10.33.0"
} }

1724
pnpm-lock.yaml generated

File diff suppressed because it is too large Load Diff

View File

@@ -1,3 +1,35 @@
# dependencies where we use branches and hashes in the package.json. But that also use a pre/post install script. allowBuilds:
onlyBuiltDependencies: "@parcel/watcher": true
- "matrix-js-sdk" "@sentry/cli": true
"@swc/core": true
"core-js": true
"esbuild": true
"matrix-js-sdk@https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/8c95727b6278fe7942c20d0b9485f984dd0694b7": true
"protobufjs": true
overrides:
# We need compatible versions of RxJS in our dependencies and LiveKit's dependencies, but
# LiveKit has pinned it to a very specific version which is now holding us back from updating.
# See livekit/components-js#1101 for a request for a proper solution.
"@livekit/components-core>rxjs": "^7.8.1"
# Dedupe Mediapipe dependencies.
"@livekit/track-processors>@mediapipe/tasks-vision": "^0.10.18"
# Security fix: https://security-tracker.debian.org/tracker/CVE-2026-31802
"tar": "^7.5.11"
# Security fixes:
# - https://github.com/advisories/GHSA-7r86-cg39-jmmj
# - https://github.com/advisories/GHSA-23c5-xmqv-rm74
"minimatch": "^10.2.3"
# Security fix: https://github.com/element-hq/element-call/security/dependabot/109
"glob": "^10.5.0"
# Security fixes:
# - https://github.com/element-hq/element-call/security/dependabot/110
# - https://github.com/element-hq/element-call/security/dependabot/122
"qs": "^6.14.1"
# Security fix: https://github.com/element-hq/element-call/security/dependabot/106
"js-yaml": "^4.1.1"
# Storybook declares support for 0.27.0 only but empirically works fine with 0.28.0.
"esbuild": "^0.28.0"
# Multiple security fixes: https://github.com/nodejs/undici/releases/tag/v6.24.0
"undici": "^6.24.0"
# Security fix: https://github.com/advisories/GHSA-rf6f-7fwh-wjgh
"flatted": "^3.4.2"

View File

@@ -272,7 +272,7 @@ test.skip("GroupCallView plays a leave sound synchronously in widget mode", asyn
expect(leaveRTCSession).toHaveBeenCalledOnce(); expect(leaveRTCSession).toHaveBeenCalledOnce();
}); });
test("Should close widget when all other left and have time to play a sound", async () => { test("Should close widget when all other left and play a sound", async () => {
const user = userEvent.setup(); const user = userEvent.setup();
let widgetClosedCalled = false; let widgetClosedCalled = false;
const { promise: widgetClosedPromise, resolve: widgetClosedResolver } = const { promise: widgetClosedPromise, resolve: widgetClosedResolver } =
@@ -310,8 +310,6 @@ test("Should close widget when all other left and have time to play a sound", as
expect(widgetClosedCalled).toBeFalsy(); expect(widgetClosedCalled).toBeFalsy();
resolvePlaySound.resolve(); resolvePlaySound.resolve();
// Expect the leave sound to be played but silent (volumeOverwrite = 0)
// The allOthersLeft effect should already play a leave sound for the last user in the call.
expect(playSound).toHaveBeenCalledWith("left", 0); expect(playSound).toHaveBeenCalledWith("left", 0);
await widgetClosedPromise; await widgetClosedPromise;
await flushPromises(); await flushPromises();
@@ -319,37 +317,6 @@ test("Should close widget when all other left and have time to play a sound", as
expect(widgetStopMock).toHaveBeenCalledOnce(); expect(widgetStopMock).toHaveBeenCalledOnce();
}, 80000); }, 80000);
test("Should close widget when all other left", async () => {
const user = userEvent.setup();
const widgetClosedCalled = Promise.withResolvers<void>();
const widgetSendMock = vi.fn().mockImplementation((action: string) => {
if (action === ElementWidgetActions.Close) {
widgetClosedCalled.resolve();
}
});
const widgetStopMock = vi.fn().mockResolvedValue(undefined);
const widget = {
api: {
setAlwaysOnScreen: vi.fn().mockResolvedValue(true),
transport: {
send: widgetSendMock,
reply: vi.fn().mockResolvedValue(undefined),
stop: widgetStopMock,
} as unknown as ITransport,
} as Partial<WidgetHelpers["api"]>,
lazyActions: new LazyEventEmitter(),
};
const { getByText } = createGroupCallView(widget as WidgetHelpers);
const leaveButton = getByText("SimulateOtherLeft");
await user.click(leaveButton);
await flushPromises();
await widgetClosedCalled.promise;
await flushPromises();
expect(widgetStopMock).toHaveBeenCalledOnce();
});
test("Should not close widget when auto leave due to error", async () => { test("Should not close widget when auto leave due to error", async () => {
const user = userEvent.setup(); const user = userEvent.setup();

View File

@@ -266,7 +266,7 @@ export const InCallView: FC<InCallViewProps> = ({
() => void toggleRaisedHand(), () => void toggleRaisedHand(),
); );
const ringing = useBehavior(vm.ringing$); const ringingIntent = useBehavior(vm.ringingIntent$);
const audioParticipants = useBehavior(vm.livekitRoomItems$); const audioParticipants = useBehavior(vm.livekitRoomItems$);
const participantCount = useBehavior(vm.participantCount$); const participantCount = useBehavior(vm.participantCount$);
const reconnecting = useBehavior(vm.reconnecting$); const reconnecting = useBehavior(vm.reconnecting$);
@@ -289,7 +289,7 @@ export const InCallView: FC<InCallViewProps> = ({
// While ringing, loop the ringtone // While ringing, loop the ringtone
useEffect((): void | (() => void) => { useEffect((): void | (() => void) => {
const audio = latestPickupPhaseAudio.current; const audio = latestPickupPhaseAudio.current;
if (ringing && audio) { if (ringingIntent !== null && audio) {
const endSound = audio.playSoundLooping( const endSound = audio.playSoundLooping(
"waiting", "waiting",
audio.soundDuration["waiting"] ?? 1, audio.soundDuration["waiting"] ?? 1,
@@ -300,7 +300,7 @@ export const InCallView: FC<InCallViewProps> = ({
}); });
}; };
} }
}, [ringing, latestPickupPhaseAudio]); }, [ringingIntent, latestPickupPhaseAudio]);
// iOS Safari doesn't reliably fire `click` on plain <div>s, so we listen // iOS Safari doesn't reliably fire `click` on plain <div>s, so we listen
// for `pointerup` instead. Scrolls end in `pointercancel`, not `pointerup`, // for `pointerup` instead. Scrolls end in `pointercancel`, not `pointerup`,

View File

@@ -5,17 +5,19 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details. Please see LICENSE in the repository root for full details.
*/ */
import { describe, it } from "vitest"; import { test } from "vitest";
import { import {
EventType, EventType,
type IEvent,
type IRoomTimelineData, type IRoomTimelineData,
MatrixEvent, MatrixEvent,
type Room, type Room,
} from "matrix-js-sdk"; } from "matrix-js-sdk";
import { type RTCCallIntent } from "matrix-js-sdk/lib/matrixrtc";
import { map, mergeMap, NEVER, type Observable, startWith } from "rxjs";
import { withTestScheduler } from "../../utils/test"; import { withTestScheduler } from "../../utils/test";
import { import {
alice,
aliceRtcMember, aliceRtcMember,
local, local,
localRtcMember, localRtcMember,
@@ -23,9 +25,10 @@ import {
import { import {
type CallNotificationWrapper, type CallNotificationWrapper,
createCallNotificationLifecycle$, createCallNotificationLifecycle$,
type Props as CallNotificationLifecycleProps, type RingAttempt,
} from "./CallNotificationLifecycle"; } from "./CallNotificationLifecycle";
import { trackEpoch } from "../ObservableScope"; import { Epoch, trackEpoch } from "../ObservableScope";
import { constant } from "../Behavior";
function mockRingEvent( function mockRingEvent(
eventId: string, eventId: string,
@@ -40,311 +43,272 @@ function mockRingEvent(
} as unknown as CallNotificationWrapper; } as unknown as CallNotificationWrapper;
} }
describe("waitForCallPickup$", () => { const defaultProps = {
it("unknown -> ringing -> timeout when notified and nobody joins", () => { memberships$: constant(new Epoch([])),
withTestScheduler(({ scope, expectObservable, behavior, hot }) => { matrixRoomMembers$: constant(new Map([[alice.userId, alice]])),
// No one ever joins (only local user) receivedDecline$: NEVER,
const props: CallNotificationLifecycleProps = { options: {
scope, waitForCallPickup: true,
memberships$: scope.behavior( autoLeaveWhenOthersLeft: false,
behavior("a", { a: [] }).pipe(trackEpoch()), },
), localUser: localRtcMember,
sentCallNotification$: hot("10ms a", { };
a: mockRingEvent("$notif1", 30),
}),
receivedDecline$: hot(""),
options: {
waitForCallPickup: true,
autoLeaveWhenOthersLeft: false,
},
localUser: localRtcMember,
};
const lifecycle = createCallNotificationLifecycle$(props); function summarizeRingAttempts$(
ringAttempts$: Observable<RingAttempt>,
): Observable<
| { intent: RTCCallIntent; recipient: string }
| { outcome: "accept" | "decline" | "timeout" }
> {
return ringAttempts$.pipe(
mergeMap(({ intent, recipient, outcome$ }) =>
outcome$.pipe(
map((outcome) => ({ outcome })),
startWith({ intent, recipient }),
),
),
);
}
expectObservable(lifecycle.callPickupState$).toBe("a 9ms b 29ms c", { test("no ring attempt when waitForCallPickup=false", () => {
a: "unknown", withTestScheduler(({ scope, expectObservable, hot }) => {
b: "ringing", const { ringAttempts$ } = createCallNotificationLifecycle$({
c: "timeout", scope,
}); ...defaultProps,
sentCallNotification$: hot("-a", {
a: mockRingEvent("$notif1", 30),
}),
options: { ...defaultProps.options, waitForCallPickup: false },
}); });
});
it("ringing -> success if someone joins before timeout is reached", () => { expectObservable(ringAttempts$).toBe("");
withTestScheduler(({ scope, hot, behavior, expectObservable }) => { });
// Someone joins at 20ms (both LiveKit participant and MatrixRTC member) });
const props: CallNotificationLifecycleProps = {
scope, test("no ring attempt when notification type is not ring", () => {
memberships$: scope.behavior( withTestScheduler(({ scope, expectObservable, hot }) => {
behavior("a 19ms b", { const { ringAttempts$ } = createCallNotificationLifecycle$({
a: [localRtcMember], scope,
b: [localRtcMember, aliceRtcMember], ...defaultProps,
}).pipe(trackEpoch()), sentCallNotification$: hot("-a", {
), a: {
sentCallNotification$: hot("5ms a", { ...mockRingEvent("$notif1", 30),
a: mockRingEvent("$notif2", 100), notification_type: "notification",
}), },
receivedDecline$: hot(""), }),
options: { });
waitForCallPickup: true,
autoLeaveWhenOthersLeft: false, expectObservable(ringAttempts$).toBe("");
}, });
localUser: localRtcMember, });
};
const lifecycle = createCallNotificationLifecycle$(props); test("no ring attempt if lifetime is missing", () => {
expectObservable(lifecycle.callPickupState$).toBe("a 4ms b 14ms c", { withTestScheduler(({ scope, expectObservable, hot }) => {
a: "unknown", const { ringAttempts$ } = createCallNotificationLifecycle$({
b: "ringing", scope,
c: "success", ...defaultProps,
}); sentCallNotification$: hot("-a", {
}); a: mockRingEvent("$notif1", undefined),
}); }),
it("success when someone joins before we notify", () => { });
withTestScheduler(({ scope, hot, behavior, expectObservable }) => {
// Someone joins at 20ms (both LiveKit participant and MatrixRTC member) expectObservable(ringAttempts$).toBe("");
const props: CallNotificationLifecycleProps = { });
scope, });
memberships$: scope.behavior(
behavior("a 9ms b", { test("ring attempt times out after nobody joins", () => {
a: [localRtcMember], withTestScheduler(({ scope, expectObservable, hot }) => {
b: [localRtcMember, aliceRtcMember], const { ringAttempts$ } = createCallNotificationLifecycle$({
}).pipe(trackEpoch()), scope,
), ...defaultProps,
sentCallNotification$: hot("20ms a", { // No one ever joins (only local user)
a: mockRingEvent("$notif2", 50), memberships$: constant(new Epoch([])),
}), sentCallNotification$: hot("-a", {
receivedDecline$: hot(""), a: mockRingEvent("$notif1", 30),
options: { }),
waitForCallPickup: true, });
autoLeaveWhenOthersLeft: false,
}, expectObservable(summarizeRingAttempts$(ringAttempts$)).toBe("-a 29ms A", {
localUser: localRtcMember, a: { intent: "audio", recipient: alice.userId },
}; A: { outcome: "timeout" },
const lifecycle = createCallNotificationLifecycle$(props); });
expectObservable(lifecycle.callPickupState$).toBe("a 9ms b", { });
a: "unknown", });
b: "success",
}); test("ring attempt is accepted once recipient joins", () => {
}); withTestScheduler(({ scope, expectObservable, hot, behavior }) => {
}); const { ringAttempts$ } = createCallNotificationLifecycle$({
it("notify without lifetime -> immediate timeout", () => { scope,
withTestScheduler(({ scope, hot, behavior, expectObservable }) => { ...defaultProps,
// Someone joins at 20ms (both LiveKit participant and MatrixRTC member) memberships$: scope.behavior(
const props: CallNotificationLifecycleProps = { behavior("a-b", { a: [], b: [aliceRtcMember] }).pipe(trackEpoch()),
scope, ),
memberships$: scope.behavior( sentCallNotification$: hot("-a", {
behavior("a", { a: mockRingEvent("$notif1", 30),
a: [localRtcMember], }),
}).pipe(trackEpoch()), });
),
sentCallNotification$: hot("10ms a", { expectObservable(summarizeRingAttempts$(ringAttempts$)).toBe("-aA", {
a: mockRingEvent("$notif2", undefined), a: { intent: "audio", recipient: alice.userId },
}), A: { outcome: "accept" },
receivedDecline$: hot(""), });
options: { });
waitForCallPickup: true, });
autoLeaveWhenOthersLeft: false,
}, test("ring attempt is immediately accepted if recipient is already joined", () => {
localUser: localRtcMember, withTestScheduler(({ scope, expectObservable, hot }) => {
}; const { ringAttempts$ } = createCallNotificationLifecycle$({
const lifecycle = createCallNotificationLifecycle$(props); scope,
expectObservable(lifecycle.callPickupState$).toBe("a 9ms b", { ...defaultProps,
a: "unknown", memberships$: constant(new Epoch([aliceRtcMember])),
b: "timeout", sentCallNotification$: hot("-a", {
}); a: mockRingEvent("$notif1", 30),
}); }),
}); });
it("stays null when waitForCallPickup=false", () => { expectObservable(summarizeRingAttempts$(ringAttempts$)).toBe("-(aA)", {
withTestScheduler(({ scope, hot, behavior, expectObservable }) => { a: { intent: "audio", recipient: alice.userId },
// Someone joins at 20ms (both LiveKit participant and MatrixRTC member) A: { outcome: "accept" },
const validProps: CallNotificationLifecycleProps = { });
scope, });
memberships$: scope.behavior( });
behavior("a--b", {
a: [localRtcMember], test("ring attempt can be declined", () => {
b: [localRtcMember, aliceRtcMember], withTestScheduler(({ scope, expectObservable, hot }) => {
}).pipe(trackEpoch()), const { ringAttempts$ } = createCallNotificationLifecycle$({
), scope,
sentCallNotification$: hot("10ms a", { ...defaultProps,
a: mockRingEvent("$notif5", 30), sentCallNotification$: hot("-a", {
}), a: mockRingEvent("$notif1", 30),
receivedDecline$: hot(""), }),
options: { receivedDecline$: hot("--d", {
waitForCallPickup: true, d: [
autoLeaveWhenOthersLeft: false, new MatrixEvent({
}, type: EventType.RTCDecline,
localUser: localRtcMember, sender: alice.userId,
}; content: {
const propsDeactivated = { "m.relates_to": {
...validProps, rel_type: "m.reference",
options: { event_id: "$notif1",
...validProps.options, },
waitForCallPickup: false, },
}, }),
}; {} as Room,
const lifecycle = createCallNotificationLifecycle$(propsDeactivated); undefined,
expectObservable(lifecycle.callPickupState$).toBe("n", { false,
n: null, {} as IRoomTimelineData,
}); ],
const lifecycleReference = createCallNotificationLifecycle$(validProps); }),
expectObservable(lifecycleReference.callPickupState$).toBe("u--s", { });
u: "unknown",
s: "success", expectObservable(summarizeRingAttempts$(ringAttempts$)).toBe("-aA", {
}); a: { intent: "audio", recipient: alice.userId },
}); A: { outcome: "decline" },
}); });
});
it("decline before timeout window ends -> decline", () => { });
withTestScheduler(({ scope, hot, behavior, expectObservable }) => {
// Someone joins at 20ms (both LiveKit participant and MatrixRTC member) test("ring attempt times out if recipient declines too late", () => {
const props: CallNotificationLifecycleProps = { withTestScheduler(({ scope, expectObservable, hot }) => {
scope, const { ringAttempts$ } = createCallNotificationLifecycle$({
memberships$: scope.behavior( scope,
behavior("a", { ...defaultProps,
a: [localRtcMember], sentCallNotification$: hot("-a", {
}).pipe(trackEpoch()), a: mockRingEvent("$notif1", 30),
), }),
sentCallNotification$: hot("10ms a", { receivedDecline$: hot("100ms d", {
a: mockRingEvent("$decl1", 50), d: [
}), new MatrixEvent({
receivedDecline$: hot("40ms d", { type: EventType.RTCDecline,
d: [ sender: alice.userId,
new MatrixEvent({ content: {
type: EventType.RTCDecline, "m.relates_to": {
content: { rel_type: "m.reference",
"m.relates_to": { event_id: "$notif1",
rel_type: "m.reference", },
event_id: "$decl1", },
}, }),
}, {} as Room,
}), undefined,
{} as Room, false,
undefined, {} as IRoomTimelineData,
false, ],
{} as IRoomTimelineData, }),
], });
}),
options: { expectObservable(summarizeRingAttempts$(ringAttempts$)).toBe("-a 29ms A", {
waitForCallPickup: true, a: { intent: "audio", recipient: alice.userId },
autoLeaveWhenOthersLeft: false, A: { outcome: "timeout" },
}, });
localUser: localRtcMember, });
}; });
const lifecycle = createCallNotificationLifecycle$(props);
expectObservable(lifecycle.callPickupState$).toBe("a 9ms b 29ms e", { test("decline event relating to wrong event is ignored (times out)", () => {
a: "unknown", withTestScheduler(({ scope, expectObservable, hot }) => {
b: "ringing", const { ringAttempts$ } = createCallNotificationLifecycle$({
e: "decline", scope,
}); ...defaultProps,
}); sentCallNotification$: hot("-a", {
}); a: mockRingEvent("$notif1", 30),
it("decline after timeout window ends -> stays timeout", () => { }),
withTestScheduler(({ scope, hot, behavior, expectObservable }) => { receivedDecline$: hot("--d", {
// Someone joins at 20ms (both LiveKit participant and MatrixRTC member) d: [
const props: CallNotificationLifecycleProps = { new MatrixEvent({
scope, type: EventType.RTCDecline,
memberships$: scope.behavior( sender: alice.userId,
behavior("a", { content: {
a: [localRtcMember], "m.relates_to": {
}).pipe(trackEpoch()), rel_type: "m.reference",
), event_id: "$other", // <---- WRONG
sentCallNotification$: hot("10ms a", { },
a: mockRingEvent("$decl", 20), },
}), }),
receivedDecline$: hot("40ms d", { {} as Room,
d: [ undefined,
new MatrixEvent({ false,
type: EventType.RTCDecline, {} as IRoomTimelineData,
content: { ],
"m.relates_to": { }),
rel_type: "m.reference", });
event_id: "$decl",
}, expectObservable(summarizeRingAttempts$(ringAttempts$)).toBe("-a 29ms A", {
}, a: { intent: "audio", recipient: alice.userId },
}), A: { outcome: "timeout" },
{} as Room, });
undefined, });
false, });
{} as IRoomTimelineData,
], test("decline event from wrong sender is ignored (times out)", () => {
}), withTestScheduler(({ scope, expectObservable, hot }) => {
options: { const { ringAttempts$ } = createCallNotificationLifecycle$({
waitForCallPickup: true, scope,
autoLeaveWhenOthersLeft: false, ...defaultProps,
}, sentCallNotification$: hot("-a", {
localUser: localRtcMember, a: mockRingEvent("$notif1", 30),
}; }),
const lifecycle = createCallNotificationLifecycle$(props); receivedDecline$: hot("--d", {
expectObservable(lifecycle.callPickupState$, "50ms !").toBe( d: [
"a 9ms b 19ms e", new MatrixEvent({
{ type: EventType.RTCDecline,
a: "unknown", sender: local.userId, // <---- WRONG
b: "ringing", content: {
e: "timeout", "m.relates_to": {
}, rel_type: "m.reference",
); event_id: "$notif1",
}); },
}); },
// }),
function testStaysRinging( {} as Room,
declineEvent: Partial<IEvent>, undefined,
expectDecline: boolean, false,
): void { {} as IRoomTimelineData,
withTestScheduler(({ scope, hot, behavior, expectObservable }) => { ],
// Someone joins at 20ms (both LiveKit participant and MatrixRTC member) }),
const props: CallNotificationLifecycleProps = { });
scope,
memberships$: scope.behavior( expectObservable(summarizeRingAttempts$(ringAttempts$)).toBe("-a 29ms A", {
behavior("a", { a: { intent: "audio", recipient: alice.userId },
a: [localRtcMember], A: { outcome: "timeout" },
}).pipe(trackEpoch()), });
),
sentCallNotification$: hot("10ms a", {
a: mockRingEvent("$right", 50),
}),
receivedDecline$: hot("20ms d", {
d: [
new MatrixEvent(declineEvent),
{} as Room,
undefined,
false,
{} as IRoomTimelineData,
],
}),
options: {
waitForCallPickup: true,
autoLeaveWhenOthersLeft: false,
},
localUser: localRtcMember,
};
const lifecycle = createCallNotificationLifecycle$(props);
const marbles = expectDecline ? "a 9ms b 9ms d" : "a 9ms b";
expectObservable(lifecycle.callPickupState$, "21ms !").toBe(marbles, {
a: "unknown",
b: "ringing",
d: "decline",
});
});
}
const reference = (refId?: string, sender?: string): Partial<IEvent> => ({
event_id: "$decline",
type: EventType.RTCDecline,
sender: sender ?? "@other:example.org",
content: {
"m.relates_to": {
rel_type: "m.reference",
event_id: refId ?? "$right",
},
},
});
it("decline reference works", () => {
testStaysRinging(reference(), true);
});
it("decline with wrong id is ignored (stays ringing)", () => {
testStaysRinging(reference("$wrong"), false);
});
it("decline with wrong id is ignored (stays ringing)", () => {
testStaysRinging(reference(undefined, local.userId), false);
}); });
}); });

View File

@@ -10,24 +10,22 @@ import {
type IRTCNotificationContent, type IRTCNotificationContent,
type MatrixRTCSession, type MatrixRTCSession,
MatrixRTCSessionEvent, MatrixRTCSessionEvent,
type RTCCallIntent,
} from "matrix-js-sdk/lib/matrixrtc"; } from "matrix-js-sdk/lib/matrixrtc";
import { import {
combineLatest,
concat,
endWith,
filter, filter,
fromEvent, fromEvent,
ignoreElements,
map, map,
merge, merge,
NEVER, NEVER,
type Observable, type Observable,
of, of,
pairwise, pairwise,
startWith,
switchMap, switchMap,
takeUntil,
timer, timer,
EMPTY,
race,
take,
} from "rxjs"; } from "rxjs";
import { import {
type EventTimelineSetHandlerMap, type EventTimelineSetHandlerMap,
@@ -35,18 +33,28 @@ import {
type Room as MatrixRoom, type Room as MatrixRoom,
RoomEvent, RoomEvent,
} from "matrix-js-sdk"; } from "matrix-js-sdk";
import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
import { type Behavior } from "../Behavior"; import { type Behavior } from "../Behavior";
import { type Epoch, mapEpoch, type ObservableScope } from "../ObservableScope"; import { type Epoch, type ObservableScope } from "../ObservableScope";
import { type RoomMemberMap } from "./remoteMembers/MatrixMemberMetadata";
const logger = rootLogger.getChild("[CallNotificationLifecycle]");
export type AutoLeaveReason = "allOthersLeft" | "timeout" | "decline"; export type AutoLeaveReason = "allOthersLeft" | "timeout" | "decline";
export type CallPickupState =
| "unknown" export interface RingAttempt {
| "ringing" intent: RTCCallIntent;
| "timeout" /**
| "decline" * The user ID of the recipient being rung.
| "success" */
| null; recipient: string;
/**
* The eventual outcome of the ringing attempt. (Emits a single value.)
*/
// TODO: Include a callback for attempting ringing again in case of a timeout
outcome$: Observable<"accept" | "decline" | "timeout">;
}
export type CallNotificationWrapper = { export type CallNotificationWrapper = {
event_id: string; event_id: string;
@@ -76,6 +84,7 @@ export function createReceivedDecline$(
export interface Props { export interface Props {
scope: ObservableScope; scope: ObservableScope;
memberships$: Behavior<Epoch<CallMembership[]>>; memberships$: Behavior<Epoch<CallMembership[]>>;
matrixRoomMembers$: Behavior<RoomMemberMap>;
sentCallNotification$: Observable<CallNotificationWrapper | null>; sentCallNotification$: Observable<CallNotificationWrapper | null>;
receivedDecline$: Observable< receivedDecline$: Observable<
Parameters<EventTimelineSetHandlerMap[RoomEvent.Timeline]> Parameters<EventTimelineSetHandlerMap[RoomEvent.Timeline]>
@@ -84,34 +93,81 @@ export interface Props {
localUser: { deviceId: string; userId: string }; localUser: { deviceId: string; userId: string };
} }
/**
* @returns two observables:
* `callPickupState$` The current call pickup state of the call.
* - "unknown": The client has not yet sent the notification event. We don't know if it will because it first needs to send its own membership.
* Then we can conclude if we were the first one to join or not.
* - "ringing": The call is ringing on other devices in this room (This client should give audiovisual feedback that this is happening).
* - "timeout": No-one picked up in the defined time this call should be ringing on others devices.
* The call failed. If desired this can be used as a trigger to exit the call.
* - "success": Someone else joined. The call is in a normal state. No audiovisual feedback.
* - null: EC is configured to never show any waiting for answer state.
*
* `autoLeave$` An observable that emits (null) when the call should be automatically left.
* - if options.autoLeaveWhenOthersLeft is set to true it emits when all others left.
* - if options.waitForCallPickup is set to true it emits if noone picked up the ring or if the ring got declined.
* - if options.autoLeaveWhenOthersLeft && options.waitForCallPickup is false it will never emit.
*
*/
export function createCallNotificationLifecycle$({ export function createCallNotificationLifecycle$({
scope, scope,
memberships$, memberships$,
matrixRoomMembers$,
sentCallNotification$, sentCallNotification$,
receivedDecline$, receivedDecline$,
options, options,
localUser, localUser,
}: Props): { }: Props): {
callPickupState$: Behavior<CallPickupState>; /**
* An observable of attempts to ring the remote participant's devices.
*/
ringAttempts$: Observable<RingAttempt>;
/**
* An observable that emits when the call should be automatically left.
* - if options.autoLeaveWhenOthersLeft is set to true it emits when all others left.
* - if options.waitForCallPickup is set to true it emits if noone picked up the ring or if the ring got declined.
* - if options.autoLeaveWhenOthersLeft && options.waitForCallPickup is false it will never emit.
*/
autoLeave$: Observable<AutoLeaveReason>; autoLeave$: Observable<AutoLeaveReason>;
} { } {
let ringAttempts$: Observable<RingAttempt> = NEVER;
if (options.waitForCallPickup)
ringAttempts$ = sentCallNotification$.pipe(
filter(
(
notificationEvent: CallNotificationWrapper | null,
): notificationEvent is CallNotificationWrapper =>
// only care about new events (legacy do not have decline pattern)
notificationEvent?.notification_type === "ring" &&
notificationEvent.lifetime > 0,
),
switchMap((notificationEvent) => {
// We assume that there is only one other user in the room when ringing
// TODO: Respect io.element.functional_members
const recipient = [...matrixRoomMembers$.value.keys()].find(
(userId) => userId !== localUser.userId,
);
if (recipient === undefined) {
logger.warn("No recipient for notification event; not ringing.");
return EMPTY;
}
// Ringing times out after lifetime ms have passed
const timeout$ = timer(notificationEvent.lifetime).pipe(
map(() => "timeout" as const),
);
// Call is accepted when the recipient joins
const accept$ = memberships$.pipe(
filter((ms) => ms.value.some((m) => m.userId === recipient)),
map(() => "accept" as const),
);
// Call is declined when we receive a decline event
const decline$ = receivedDecline$.pipe(
filter(
([event]) =>
event.getRelation()?.rel_type === "m.reference" &&
event.getRelation()?.event_id === notificationEvent.event_id &&
event.getSender() === recipient,
),
map(() => "decline" as const),
);
return of({
intent: notificationEvent["m.call.intent"] ?? "audio",
recipient,
outcome$: race(timeout$, accept$, decline$).pipe(
take(1),
scope.share,
),
});
}),
scope.share,
);
const allOthersLeft$ = memberships$.pipe( const allOthersLeft$ = memberships$.pipe(
pairwise(), pairwise(),
filter( filter(
@@ -122,87 +178,18 @@ export function createCallNotificationLifecycle$({
map(() => {}), map(() => {}),
); );
/**
* Whether some Matrix user other than ourself is joined to the call.
*/
const someoneElseJoined$ = memberships$.pipe(
mapEpoch((ms) => ms.some((m) => m.userId !== localUser.userId)),
) as Behavior<Epoch<boolean>>;
/**
* The state of the current ringing attempt, if the RTC session is indeed
* ringing the remote participant's devices. Otherwise `null`.
*/
const remoteRingState$: Behavior<"ringing" | "timeout" | "decline" | null> =
scope.behavior(
sentCallNotification$.pipe(
filter(
(notificationEventArgs: CallNotificationWrapper | null) =>
// only care about new events (legacy do not have decline pattern)
notificationEventArgs?.notification_type === "ring",
),
map((e) => e as CallNotificationWrapper),
switchMap((notificationEvent) => {
const lifetimeMs = notificationEvent?.lifetime ?? 0;
return concat(
lifetimeMs === 0
? // If no lifetime, skip the ring state
of(null)
: // Ring until lifetime ms have passed
timer(lifetimeMs).pipe(
ignoreElements(),
startWith("ringing" as const),
),
// The notification lifetime has timed out, meaning ringing has likely
// stopped on all receiving clients.
of("timeout" as const),
// This makes sure we will not drop into the `endWith("decline" as const)` state
NEVER,
).pipe(
takeUntil(
receivedDecline$.pipe(
filter(
([event]) =>
event.getRelation()?.rel_type === "m.reference" &&
event.getRelation()?.event_id ===
notificationEvent.event_id &&
event.getSender() !== localUser.userId &&
callPickupState$.value !== "timeout",
),
),
),
endWith("decline" as const),
);
}),
),
null,
);
const callPickupState$ = scope.behavior(
options.waitForCallPickup === true
? combineLatest(
[someoneElseJoined$, remoteRingState$],
(someoneElseJoined, ring) => {
if (someoneElseJoined.value === true) {
return "success" as const;
}
// Show the ringing state of the most recent ringing attempt.
// as long as we have not yet sent an RTC notification event or noone else joined,
// ring will be null -> callPickupState$ = unknown.
return ring ?? ("unknown" as const);
},
)
: NEVER,
null,
);
const autoLeave$ = merge( const autoLeave$ = merge(
options.autoLeaveWhenOthersLeft === true options.autoLeaveWhenOthersLeft === true
? allOthersLeft$.pipe(map(() => "allOthersLeft" as const)) ? allOthersLeft$.pipe(map(() => "allOthersLeft" as const))
: NEVER, : NEVER,
callPickupState$.pipe( ringAttempts$.pipe(
filter((state) => state === "timeout" || state === "decline"), switchMap(({ outcome$ }) =>
outcome$.pipe(
filter((outcome) => outcome === "timeout" || outcome === "decline"),
),
),
), ),
); );
return { autoLeave$, callPickupState$ };
return { ringAttempts$, autoLeave$ };
} }

View File

@@ -1421,7 +1421,10 @@ describe.each([
}); });
// Should ring for 30ms and then time out // Should ring for 30ms and then time out
expectObservable(vm.ringing$).toBe("(ny) 26ms n", yesNo); expectObservable(vm.ringingIntent$).toBe("(ab) 26ms a", {
a: null,
b: "audio",
});
// Layout should show placeholder media for the participant we're // Layout should show placeholder media for the participant we're
// ringing the entire time (even once timed out) // ringing the entire time (even once timed out)
expectObservable(summarizeLayout$(vm.layout$)).toBe("a", { expectObservable(summarizeLayout$(vm.layout$)).toBe("a", {
@@ -1460,7 +1463,10 @@ describe.each([
}); });
// Should ring until Alice joins // Should ring until Alice joins
expectObservable(vm.ringing$).toBe("(ny) 17ms n", yesNo); expectObservable(vm.ringingIntent$).toBe("(ab) 17ms a", {
a: null,
b: "audio",
});
// Layout should show placeholder media for the participant we're // Layout should show placeholder media for the participant we're
// ringing the entire time // ringing the entire time
expectObservable(summarizeLayout$(vm.layout$)).toBe("a 20ms b", { expectObservable(summarizeLayout$(vm.layout$)).toBe("a 20ms b", {

View File

@@ -29,7 +29,6 @@ import {
pairwise, pairwise,
race, race,
scan, scan,
skipWhile,
startWith, startWith,
Subject, Subject,
switchAll, switchAll,
@@ -39,10 +38,13 @@ import {
tap, tap,
throttleTime, throttleTime,
timer, timer,
takeUntil,
concat,
} from "rxjs"; } from "rxjs";
import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
import { import {
MembershipManagerEvent, MembershipManagerEvent,
type RTCCallIntent,
type LivekitTransportConfig, type LivekitTransportConfig,
type MatrixRTCSession, type MatrixRTCSession,
} from "matrix-js-sdk/lib/matrixrtc"; } from "matrix-js-sdk/lib/matrixrtc";
@@ -230,9 +232,9 @@ export interface CallViewModel {
// lifecycle // lifecycle
autoLeave$: Observable<AutoLeaveReason>; autoLeave$: Observable<AutoLeaveReason>;
/** /**
* Whether we are ringing a call recipient. * Whether we are ringing a call recipient. Contains the ringing intent if so.
*/ */
ringing$: Behavior<boolean>; ringingIntent$: Behavior<RTCCallIntent | null>;
/** Observable that emits when the user should leave the call (hangup pressed, widget action, error). /** Observable that emits when the user should leave the call (hangup pressed, widget action, error).
* THIS DOES NOT LEAVE THE CALL YET. The only way to leave the call (send the hangup event) is * THIS DOES NOT LEAVE THE CALL YET. The only way to leave the call (send the hangup event) is
* - by ending the scope * - by ending the scope
@@ -610,20 +612,6 @@ export function createCallViewModel$(
), ),
); );
// ------------------------------------------------------------------------
// callLifecycle
// TODO if we are in "unknown" state we need a loading rendering (or empty screen)
// Otherwise it looks like we already connected and only than the ringing starts which is weird.
const { callPickupState$, autoLeave$ } = createCallNotificationLifecycle$({
scope: scope,
memberships$: memberships$,
sentCallNotification$: createSentCallNotification$(scope, matrixRTCSession),
receivedDecline$: createReceivedDecline$(matrixRoom),
options: options,
localUser: { userId: userId, deviceId: deviceId },
});
// ------------------------------------------------------------------------ // ------------------------------------------------------------------------
// matrixMemberMetadataStore // matrixMemberMetadataStore
@@ -634,6 +622,21 @@ export function createCallViewModel$(
matrixRoomMembers$, matrixRoomMembers$,
); );
// ------------------------------------------------------------------------
// callLifecycle
// TODO if we are in "unknown" state we need a loading rendering (or empty screen)
// Otherwise it looks like we already connected and only than the ringing starts which is weird.
const { ringAttempts$, autoLeave$ } = createCallNotificationLifecycle$({
scope,
memberships$,
matrixRoomMembers$,
sentCallNotification$: createSentCallNotification$(scope, matrixRTCSession),
receivedDecline$: createReceivedDecline$(matrixRoom),
options,
localUser: { userId, deviceId },
});
const allConnections$ = scope.behavior( const allConnections$ = scope.behavior(
connectionManager.connectionManagerData$.pipe(map((d) => d.value)), connectionManager.connectionManagerData$.pipe(map((d) => d.value)),
); );
@@ -784,51 +787,42 @@ export function createCallViewModel$(
), ),
); );
const ringingMedia$ = scope.behavior<RingingMediaViewModel[]>( const ringingMedia$ = scope.behavior<RingingMediaViewModel | null>(
combineLatest([userMedia$, matrixRoomMembers$, callPickupState$]).pipe( ringAttempts$.pipe(
generateItems( switchMap(({ intent, recipient, outcome$ }) =>
"CallViewModel ringingMedia$", outcome$.pipe(
function* ([userMedia, roomMembers, callPickupState]) { startWith("ringing" as const),
if ( generateItems(
callPickupState === "ringing" || "CallViewModel ringingMedia$",
callPickupState === "timeout" || function* (pickupState) {
callPickupState === "decline" if (pickupState !== "accept")
) { yield { keys: [intent, recipient], data: pickupState };
// TODO: Respect io.element.functional_members },
for (const member of roomMembers.values()) { (scope, pickupState$, intent, userId) =>
if (!userMedia.some((vm) => vm.userId === member.userId)) createRingingMedia({
yield { id: `ringing:${userId}`,
keys: [member.userId], userId,
data: callPickupState, displayName$: scope.behavior(
}; matrixRoomMembers$.pipe(
} map(
} (members) =>
}, members.get(userId)?.rawDisplayName || userId,
(scope, pickupState$, userId) => ),
createRingingMedia({ ),
id: `ringing:${userId}`, ),
userId, mxcAvatarUrl$:
displayName$: scope.behavior( matrixMemberMetadataStore.createAvatarUrlBehavior$(
matrixRoomMembers$.pipe( scope,
map((members) => members.get(userId)?.rawDisplayName || userId), userId,
), ),
), pickupState$,
mxcAvatarUrl$: matrixMemberMetadataStore.createAvatarUrlBehavior$( intent,
scope, }),
userId, ),
), map(([media]) => media ?? null),
pickupState$, ),
muteStates,
}),
), ),
distinctUntilChanged(shallowEquals), startWith(null),
tap((ringingMedia) => {
if (ringingMedia.length > 1)
// Warn that UI may do something unexpected in this case
logger.warn(
`Ringing more than one participant is not supported (ringing ${ringingMedia.map((vm) => vm.userId).join(", ")})`,
);
}),
), ),
); );
@@ -870,11 +864,7 @@ export function createCallViewModel$(
matrixLivekitMembers$.pipe(map((ms) => ms.value.length)), matrixLivekitMembers$.pipe(map((ms) => ms.value.length)),
); );
const leaveSoundEffect$ = combineLatest([callPickupState$, userMedia$]).pipe( const leaveSoundEffect$ = userMedia$.pipe(
// Until the call is successful, do not play a leave sound.
// If callPickupState$ is null, then we always play the sound as it will not conflict with a decline sound.
skipWhile(([c]) => c !== null && c !== "success"),
map(([, userMedia]) => userMedia),
pairwise(), pairwise(),
filter( filter(
([prev, current]) => ([prev, current]) =>
@@ -883,6 +873,9 @@ export function createCallViewModel$(
), ),
map(() => {}), map(() => {}),
throttleTime(THROTTLE_SOUND_EFFECT_MS), throttleTime(THROTTLE_SOUND_EFFECT_MS),
// Avoid doubling up on any auto-leave sounds (e.g. the decline sound),
// which are handled elsewhere
takeUntil(autoLeave$),
); );
const userHangup$ = new Subject<void>(); const userHangup$ = new Subject<void>();
@@ -987,8 +980,8 @@ export function createCallViewModel$(
}>( }>(
ringingMedia$.pipe( ringingMedia$.pipe(
switchMap((ringingMedia) => { switchMap((ringingMedia) => {
if (ringingMedia.length > 0) if (ringingMedia !== null)
return of({ spotlight: ringingMedia, pip$: localUserMediaForPip$ }); return of({ spotlight: [ringingMedia], pip$: localUserMediaForPip$ });
return screenShares$.pipe( return screenShares$.pipe(
switchMap((screenShares) => { switchMap((screenShares) => {
@@ -1144,14 +1137,10 @@ export function createCallViewModel$(
// show ringing media instead // show ringing media instead
if (userMedia.length === 1) if (userMedia.length === 1)
return ringingMedia$.pipe( return ringingMedia$.pipe(
map((ringingMedia) => { map(
return ringingMedia.length === 1 (ringingMedia) =>
? { ringingMedia && { local, remote: ringingMedia },
local, ),
remote: ringingMedia[0],
}
: null;
}),
); );
} }
} }
@@ -1728,8 +1717,14 @@ export function createCallViewModel$(
return { return {
autoLeave$: autoLeave$, autoLeave$: autoLeave$,
ringing$: scope.behavior( ringingIntent$: scope.behavior(
callPickupState$.pipe(map((state) => state === "ringing")), ringAttempts$.pipe(
switchMap(({ intent, outcome$ }) =>
// Hold the intent as the value until the ring attempt completes
concat(of(intent), NEVER.pipe(takeUntil(outcome$)), of(null)),
),
startWith<RTCCallIntent | null>(null),
),
), ),
leave$: leave$, leave$: leave$,
hangup: (): void => userHangup$.next(), hangup: (): void => userHangup$.next(),

View File

@@ -5,8 +5,9 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details. Please see LICENSE in the repository root for full details.
*/ */
import { type RTCCallIntent } from "matrix-js-sdk/lib/matrixrtc";
import { type Behavior } from "../Behavior"; import { type Behavior } from "../Behavior";
import { type MuteStates } from "../MuteStates";
import { import {
type BaseMediaInputs, type BaseMediaInputs,
type BaseMediaViewModel, type BaseMediaViewModel,
@@ -20,32 +21,23 @@ import {
export interface RingingMediaViewModel extends BaseMediaViewModel { export interface RingingMediaViewModel extends BaseMediaViewModel {
type: "ringing"; type: "ringing";
pickupState$: Behavior<"ringing" | "timeout" | "decline">; pickupState$: Behavior<"ringing" | "timeout" | "decline">;
/** intent: RTCCallIntent;
* Whether this media would be expected to have video, were it not simply a
* placeholder.
*/
videoEnabled$: Behavior<boolean>;
} }
export interface RingingMediaInputs extends BaseMediaInputs { export interface RingingMediaInputs extends BaseMediaInputs {
pickupState$: Behavior<"ringing" | "timeout" | "decline">; pickupState$: Behavior<"ringing" | "timeout" | "decline">;
/** intent: RTCCallIntent;
* The local user's own mute states.
*/
muteStates: MuteStates;
} }
export function createRingingMedia({ export function createRingingMedia({
pickupState$, pickupState$,
muteStates, intent,
...inputs ...inputs
}: RingingMediaInputs): RingingMediaViewModel { }: RingingMediaInputs): RingingMediaViewModel {
return { return {
...createBaseMedia(inputs), ...createBaseMedia(inputs),
type: "ringing", type: "ringing",
pickupState$, pickupState$,
// If our own video is enabled, then this is a video call and we would intent,
// expect remote media to have video as well
videoEnabled$: muteStates.video.enabled$,
}; };
} }

View File

@@ -26,7 +26,6 @@ import {
createRingingMedia, createRingingMedia,
type RingingMediaViewModel, type RingingMediaViewModel,
} from "../state/media/RingingMediaViewModel"; } from "../state/media/RingingMediaViewModel";
import { type MuteStates } from "../state/MuteStates";
global.IntersectionObserver = class MockIntersectionObserver { global.IntersectionObserver = class MockIntersectionObserver {
public observe(): void {} public observe(): void {}
@@ -93,10 +92,8 @@ test("GridTile displays ringing media", async () => {
>("ringing"); >("ringing");
const vm = createRingingMedia({ const vm = createRingingMedia({
pickupState$, pickupState$,
muteStates: {
video: { enabled$: constant(false) },
} as unknown as MuteStates,
id: "test", id: "test",
intent: "audio",
userId: "@alice:example.org", userId: "@alice:example.org",
displayName$: constant("Alice"), displayName$: constant("Alice"),
mxcAvatarUrl$: constant(undefined), mxcAvatarUrl$: constant(undefined),

View File

@@ -77,7 +77,6 @@ const RingingMediaTile: FC<RingingMediaTileProps> = ({
}) => { }) => {
const { t } = useTranslation(); const { t } = useTranslation();
const pickupState = useBehavior(vm.pickupState$); const pickupState = useBehavior(vm.pickupState$);
const videoEnabled = useBehavior(vm.videoEnabled$);
return ( return (
<MediaView <MediaView
@@ -89,11 +88,12 @@ const RingingMediaTile: FC<RingingMediaTileProps> = ({
pickupState === "ringing" pickupState === "ringing"
? { ? {
text: t("video_tile.calling"), text: t("video_tile.calling"),
Icon: videoEnabled ? VideoCallSolidIcon : VoiceCallSolidIcon, Icon:
vm.intent === "video" ? VideoCallSolidIcon : VoiceCallSolidIcon,
} }
: { text: t("video_tile.call_ended"), Icon: EndCallIcon } : { text: t("video_tile.call_ended"), Icon: EndCallIcon }
} }
videoEnabled={videoEnabled} videoEnabled={false}
videoFit="cover" videoFit="cover"
mirror={false} mirror={false}
{...props} {...props}

View File

@@ -28,7 +28,6 @@ import {
createRingingMedia, createRingingMedia,
type RingingMediaViewModel, type RingingMediaViewModel,
} from "../state/media/RingingMediaViewModel"; } from "../state/media/RingingMediaViewModel";
import { type MuteStates } from "../state/MuteStates";
global.IntersectionObserver = class MockIntersectionObserver { global.IntersectionObserver = class MockIntersectionObserver {
public observe(): void {} public observe(): void {}
@@ -156,10 +155,8 @@ test("SpotlightTile displays ringing media", async () => {
>("ringing"); >("ringing");
const vm = createRingingMedia({ const vm = createRingingMedia({
pickupState$, pickupState$,
muteStates: {
video: { enabled$: constant(false) },
} as unknown as MuteStates,
id: "test", id: "test",
intent: "audio",
userId: "@alice:example.org", userId: "@alice:example.org",
displayName$: constant("Alice"), displayName$: constant("Alice"),
mxcAvatarUrl$: constant(undefined), mxcAvatarUrl$: constant(undefined),

View File

@@ -212,7 +212,6 @@ const SpotlightRingingMediaItem: FC<SpotlightRingingMediaItemProps> = ({
}) => { }) => {
const { t } = useTranslation(); const { t } = useTranslation();
const pickupState = useBehavior(vm.pickupState$); const pickupState = useBehavior(vm.pickupState$);
const videoEnabled = useBehavior(vm.videoEnabled$);
return ( return (
<MediaView <MediaView
@@ -222,7 +221,8 @@ const SpotlightRingingMediaItem: FC<SpotlightRingingMediaItemProps> = ({
pickupState === "ringing" pickupState === "ringing"
? { ? {
text: t("video_tile.calling"), text: t("video_tile.calling"),
Icon: videoEnabled ? VideoCallSolidIcon : VoiceCallSolidIcon, Icon:
vm.intent === "video" ? VideoCallSolidIcon : VoiceCallSolidIcon,
} }
: { text: t("video_tile.call_ended"), Icon: EndCallIcon } : { text: t("video_tile.call_ended"), Icon: EndCallIcon }
} }