mirror of
https://github.com/vector-im/element-call.git
synced 2026-08-23 20:59:21 +00:00
Compare commits
74 Commits
robin/butt
...
v0.19.1
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f880e6d45d | ||
|
|
f6d80ed7f5 | ||
|
|
3c2c189830 | ||
|
|
2c34e681a1 | ||
|
|
48226795de | ||
|
|
3503f8177d | ||
|
|
9fe35ba822 | ||
|
|
a84419c8a4 | ||
|
|
a89d231858 | ||
|
|
6d99450d81 | ||
|
|
47e14ff574 | ||
|
|
55eea7a968 | ||
|
|
121160e835 | ||
|
|
47cb01ab6f | ||
|
|
3d286e8ab4 | ||
|
|
b1f84a34a8 | ||
|
|
95eb8943af | ||
|
|
b5be123953 | ||
|
|
90bfaecd13 | ||
|
|
4cecdda955 | ||
|
|
869bab8253 | ||
|
|
23f846a308 | ||
|
|
6dcb470162 | ||
|
|
c5c154c99b | ||
|
|
fd08489afb | ||
|
|
67d7ec6703 | ||
|
|
a2005febd5 | ||
|
|
5e337a5399 | ||
|
|
e0a8e84df4 | ||
|
|
99ba23bc96 | ||
|
|
f7488a0474 | ||
|
|
bb7b2d1060 | ||
|
|
b5870c9e78 | ||
|
|
1fce220965 | ||
|
|
cd3c8f001f | ||
|
|
f316ece404 | ||
|
|
ce1cd9aefb | ||
|
|
d5ec3c342c | ||
|
|
c9242148f9 | ||
|
|
0dabb43e0f | ||
|
|
8cee4df46e | ||
|
|
6b7467ce6d | ||
|
|
3389fb5a34 | ||
|
|
421c49e7ba | ||
|
|
1eabb0f0aa | ||
|
|
8f38ecac2e | ||
|
|
40fdef89eb | ||
|
|
2ab5e07f73 | ||
|
|
09092e97c8 | ||
|
|
cb6bef3849 | ||
|
|
4d491be4a9 | ||
|
|
aad160a715 | ||
|
|
fc61a36d4a | ||
|
|
1bc2abb84f | ||
|
|
4f518819d3 | ||
|
|
b40feae7da | ||
|
|
e15761d8ee | ||
|
|
c60ed50a9d | ||
|
|
6dcd883fc3 | ||
|
|
c8b8d350d5 | ||
|
|
9c2b2d4780 | ||
|
|
396225a900 | ||
|
|
c4ec52ae15 | ||
|
|
4be2bc7560 | ||
|
|
385ab0a0ed | ||
|
|
9dfade68ee | ||
|
|
32ea8f522c | ||
|
|
997cc9bb27 | ||
|
|
fa74aaa81e | ||
|
|
af807489f9 | ||
|
|
b198721969 | ||
|
|
8ecb1b3dbf | ||
|
|
699e31f59a | ||
|
|
005b965fba |
@@ -97,7 +97,7 @@ jobs:
|
||||
run: find ${FILENAME_PREFIX} -type f -print0 | sort -z | xargs -0 sha256sum | tee ${FILENAME_PREFIX}.sha256
|
||||
- name: Upload
|
||||
if: ${{ needs.versioning.outputs.DRY_RUN == 'false' }}
|
||||
uses: softprops/action-gh-release@a06a81a03ee405af7f2048a818ed3f03bbf83c7b # v2
|
||||
uses: softprops/action-gh-release@153bb8e04406b158c6c84fc1615b65b24149a1fe # v2
|
||||
with:
|
||||
files: |
|
||||
${{ env.FILENAME_PREFIX }}.tar.gz
|
||||
@@ -297,7 +297,7 @@ jobs:
|
||||
NEEDS_PUBLISH_IOS_OUTPUTS_ARTIFACT_VERSION: ${{ needs.publish_ios.outputs.ARTIFACT_VERSION }}
|
||||
- name: Add release notes
|
||||
if: ${{ needs.versioning.outputs.DRY_RUN == 'false' }}
|
||||
uses: softprops/action-gh-release@a06a81a03ee405af7f2048a818ed3f03bbf83c7b # v2
|
||||
uses: softprops/action-gh-release@153bb8e04406b158c6c84fc1615b65b24149a1fe # v2
|
||||
with:
|
||||
append_body: true
|
||||
body: |
|
||||
|
||||
4
.github/workflows/publish.yaml
vendored
4
.github/workflows/publish.yaml
vendored
@@ -42,7 +42,7 @@ jobs:
|
||||
- name: Create Checksum
|
||||
run: find ${FILENAME_PREFIX} -type f -print0 | sort -z | xargs -0 sha256sum | tee ${FILENAME_PREFIX}.sha256
|
||||
- name: Upload
|
||||
uses: softprops/action-gh-release@a06a81a03ee405af7f2048a818ed3f03bbf83c7b # v2
|
||||
uses: softprops/action-gh-release@153bb8e04406b158c6c84fc1615b65b24149a1fe # v2
|
||||
with:
|
||||
files: |
|
||||
${{ env.FILENAME_PREFIX }}.tar.gz
|
||||
@@ -71,7 +71,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Add release note
|
||||
uses: softprops/action-gh-release@a06a81a03ee405af7f2048a818ed3f03bbf83c7b # v2
|
||||
uses: softprops/action-gh-release@153bb8e04406b158c6c84fc1615b65b24149a1fe # v2
|
||||
with:
|
||||
append_body: true
|
||||
body: |
|
||||
|
||||
2
.github/workflows/test.yaml
vendored
2
.github/workflows/test.yaml
vendored
@@ -24,7 +24,7 @@ jobs:
|
||||
- name: Vitest
|
||||
run: "yarn run test:coverage"
|
||||
- name: Upload to codecov
|
||||
uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5
|
||||
uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5
|
||||
env:
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
with:
|
||||
|
||||
@@ -249,6 +249,8 @@
|
||||
"version": "{{productName}} version: {{version}}",
|
||||
"video_tile": {
|
||||
"always_show": "Always show",
|
||||
"call_ended": "Call ended",
|
||||
"calling": "Calling…",
|
||||
"camera_starting": "Video loading...",
|
||||
"collapse": "Collapse",
|
||||
"expand": "Expand",
|
||||
|
||||
@@ -76,7 +76,7 @@
|
||||
"@typescript-eslint/eslint-plugin": "^8.31.0",
|
||||
"@typescript-eslint/parser": "^8.31.0",
|
||||
"@use-gesture/react": "^10.2.11",
|
||||
"@vector-im/compound-design-tokens": "^6.0.0",
|
||||
"@vector-im/compound-design-tokens": "^9.0.0",
|
||||
"@vector-im/compound-web": "^8.0.0",
|
||||
"@vitejs/plugin-react": "^4.0.1",
|
||||
"@vitest/coverage-v8": "^4.0.18",
|
||||
@@ -86,7 +86,7 @@
|
||||
"eslint": "^8.14.0",
|
||||
"eslint-config-google": "^0.14.0",
|
||||
"eslint-config-prettier": "^10.0.0",
|
||||
"eslint-plugin-deprecate": "^0.8.2",
|
||||
"eslint-plugin-deprecate": "^0.9.0",
|
||||
"eslint-plugin-import": "^2.26.0",
|
||||
"eslint-plugin-jsdoc": "^61.5.0",
|
||||
"eslint-plugin-jsx-a11y": "^6.5.1",
|
||||
@@ -118,7 +118,7 @@
|
||||
"qrcode": "^1.5.4",
|
||||
"react": "19",
|
||||
"react-dom": "19",
|
||||
"react-i18next": "^16.0.0 <16.6.0",
|
||||
"react-i18next": "^16.0.0 <16.7.0",
|
||||
"react-router-dom": "^7.0.0",
|
||||
"react-use-measure": "^2.1.1",
|
||||
"rxjs": "^7.8.1",
|
||||
|
||||
@@ -100,8 +100,16 @@ mobileTest(
|
||||
{ id: "earpiece", name: "Handset", isEarpiece: true },
|
||||
{ id: "headphones", name: "Headphones" },
|
||||
]);
|
||||
window.controls.setAudioDevice("earpiece");
|
||||
});
|
||||
|
||||
// Open settings to select earpiece
|
||||
await guestPage.getByRole("button", { name: "Settings" }).click();
|
||||
await guestPage.getByText("Handset", { exact: true }).click();
|
||||
|
||||
// dismiss settings
|
||||
await guestPage.locator("#root").getByLabel("Settings").press("Escape");
|
||||
|
||||
await guestPage.pause();
|
||||
await expect(
|
||||
guestPage.getByRole("heading", { name: "Handset Mode" }),
|
||||
).toBeVisible();
|
||||
|
||||
92
playwright/widget/federation-oldest-membership-bug.spec.ts
Normal file
92
playwright/widget/federation-oldest-membership-bug.spec.ts
Normal file
@@ -0,0 +1,92 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { expect, test } from "@playwright/test";
|
||||
|
||||
import { widgetTest } from "../fixtures/widget-user";
|
||||
import { HOST1, HOST2, TestHelpers } from "./test-helpers";
|
||||
|
||||
widgetTest(
|
||||
"Bug new joiner was not publishing on correct SFU",
|
||||
async ({ addUser, browserName }) => {
|
||||
test.skip(
|
||||
browserName === "firefox",
|
||||
"This is a bug in the old widget, not a browser problem.",
|
||||
);
|
||||
|
||||
test.slow();
|
||||
|
||||
// 2 users in federation
|
||||
const florian = await addUser("floriant", HOST1);
|
||||
const timo = await addUser("timo", HOST2);
|
||||
|
||||
// Florian creates a room and invites Timo to it
|
||||
const roomName = "Call Room";
|
||||
await TestHelpers.createRoom(roomName, florian.page, [timo.mxId]);
|
||||
|
||||
// Timo joins the room
|
||||
await TestHelpers.acceptRoomInvite(roomName, timo.page);
|
||||
|
||||
// Ensure we are in legacy mode (should be the default)
|
||||
await TestHelpers.openWidgetSetEmbeddedElementCallRtcModeCloseWidget(
|
||||
florian.page,
|
||||
"legacy",
|
||||
);
|
||||
await TestHelpers.openWidgetSetEmbeddedElementCallRtcModeCloseWidget(
|
||||
timo.page,
|
||||
"legacy",
|
||||
);
|
||||
|
||||
// Let timo create a call
|
||||
await TestHelpers.startCallInCurrentRoom(timo.page, false);
|
||||
await TestHelpers.joinCallFromLobby(timo.page);
|
||||
|
||||
// We want to simulate that the oldest membership authentication is way slower than
|
||||
// the preffered auth.
|
||||
// In this setup, timo advertised$ transport will be it's own, and the active will be the one from florian
|
||||
await florian.page.route(
|
||||
"**/matrix-rtc.othersite.m.localhost/livekit/jwt/**",
|
||||
async (route) => {
|
||||
await new Promise((resolve) => setTimeout(resolve, 2000)); // 5 second delay
|
||||
await route.continue();
|
||||
},
|
||||
);
|
||||
|
||||
// Florian joins the call
|
||||
await expect(florian.page.getByTestId("join-call-button")).toBeVisible();
|
||||
await florian.page.getByTestId("join-call-button").click();
|
||||
await TestHelpers.joinCallFromLobby(florian.page);
|
||||
|
||||
await florian.page.waitForTimeout(3000);
|
||||
await timo.page.waitForTimeout(3000);
|
||||
|
||||
// We should see 2 video tiles everywhere now
|
||||
for (const user of [timo, florian]) {
|
||||
const frame = user.page
|
||||
.locator('iframe[title="Element Call"]')
|
||||
.contentFrame();
|
||||
await expect(frame.getByTestId("videoTile")).toHaveCount(2);
|
||||
|
||||
// No one should be waiting for media
|
||||
await expect(frame.getByText("Waiting for media...")).not.toBeVisible();
|
||||
|
||||
// There should be 2 video elements, visible and autoplaying
|
||||
const videoElements = await frame.locator("video").all();
|
||||
expect(videoElements.length).toBe(2);
|
||||
|
||||
const blockDisplayCount = await frame
|
||||
.locator("video")
|
||||
.evaluateAll(
|
||||
(videos: Element[]) =>
|
||||
videos.filter(
|
||||
(v: Element) => window.getComputedStyle(v).display === "block",
|
||||
).length,
|
||||
);
|
||||
expect(blockDisplayCount).toBe(2);
|
||||
}
|
||||
},
|
||||
);
|
||||
@@ -66,7 +66,7 @@ widgetTest("Footer interaction in PiP", async ({ addUser, browserName }) => {
|
||||
await iFrame.getByTestId("videoTile").hover();
|
||||
|
||||
await expect(audioBtn).toHaveAccessibleName("Unmute microphone");
|
||||
await expect(audioBtn).toBeChecked();
|
||||
await expect(audioBtn).not.toBeChecked();
|
||||
await expect(videoBtn).toHaveAccessibleName("Start video");
|
||||
await expect(videoBtn).not.toBeChecked();
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ Please see LICENSE in the repository root for full details.
|
||||
import { expect, test } from "@playwright/test";
|
||||
|
||||
import { widgetTest } from "../fixtures/widget-user.ts";
|
||||
import { TestHelpers } from "./test-helpers.ts";
|
||||
|
||||
// Skip test, including Fixtures
|
||||
widgetTest.skip(
|
||||
@@ -20,19 +21,7 @@ widgetTest("Start a new call as widget", async ({ asWidget, browserName }) => {
|
||||
|
||||
const { brooks, whistler } = asWidget;
|
||||
|
||||
await expect(
|
||||
brooks.page.getByRole("button", { name: "Video call" }),
|
||||
).toBeVisible();
|
||||
await brooks.page.getByRole("button", { name: "Video call" }).click();
|
||||
|
||||
await expect(
|
||||
brooks.page.getByRole("menuitem", { name: "Legacy Call" }),
|
||||
).toBeVisible();
|
||||
await expect(
|
||||
brooks.page.getByRole("menuitem", { name: "Element Call" }),
|
||||
).toBeVisible();
|
||||
|
||||
await brooks.page.getByRole("menuitem", { name: "Element Call" }).click();
|
||||
await TestHelpers.startCallInCurrentRoom(brooks.page, false);
|
||||
|
||||
await expect(
|
||||
brooks.page
|
||||
@@ -56,11 +45,7 @@ widgetTest("Start a new call as widget", async ({ asWidget, browserName }) => {
|
||||
).toBeVisible();
|
||||
|
||||
// Join from the other side
|
||||
await expect(whistler.page.getByText("Video call started")).toBeVisible();
|
||||
await expect(
|
||||
whistler.page.getByRole("button", { name: "Join" }),
|
||||
).toBeVisible();
|
||||
await whistler.page.getByRole("button", { name: "Join" }).click();
|
||||
await TestHelpers.joinCallInCurrentRoom(whistler.page);
|
||||
|
||||
// Currently disabled due to recent Element Web is bypassing Lobby
|
||||
// await expect(
|
||||
|
||||
@@ -152,6 +152,22 @@ export class TestHelpers {
|
||||
}
|
||||
}
|
||||
|
||||
public static async maybeDismissKeyBackupToast(page: Page): Promise<void> {
|
||||
const toast = page
|
||||
.locator(".mx_Toast_toast")
|
||||
.getByText("Back up your chats");
|
||||
|
||||
try {
|
||||
await expect(toast).toBeVisible({ timeout: 700 });
|
||||
await page
|
||||
.locator(".mx_Toast_toast")
|
||||
.getByRole("button", { name: "Dismiss" })
|
||||
.click();
|
||||
} catch {
|
||||
// toast not visible, continue as normal
|
||||
}
|
||||
}
|
||||
|
||||
public static async createRoom(
|
||||
name: string,
|
||||
page: Page,
|
||||
@@ -167,6 +183,7 @@ export class TestHelpers {
|
||||
await page.getByRole("button", { name: "Create room" }).click();
|
||||
await expect(page.getByText("You created this room.")).toBeVisible();
|
||||
await expect(page.getByText("Encryption enabled")).toBeVisible();
|
||||
await TestHelpers.maybeDismissKeyBackupToast(page);
|
||||
|
||||
// Invite users if any
|
||||
if (andInvite.length > 0) {
|
||||
@@ -201,6 +218,7 @@ export class TestHelpers {
|
||||
await expect(
|
||||
page.getByRole("main").getByRole("heading", { name: roomName }),
|
||||
).toBeVisible();
|
||||
await TestHelpers.maybeDismissKeyBackupToast(page);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -217,9 +235,11 @@ export class TestHelpers {
|
||||
): Promise<void> {
|
||||
await page.getByRole("button", { name: "Video call" }).click();
|
||||
await page.getByRole("menuitem", { name: "Element Call" }).click();
|
||||
|
||||
await TestHelpers.setEmbeddedElementCallRtcMode(page, mode);
|
||||
await page.getByRole("button", { name: "Close lobby" }).click();
|
||||
}
|
||||
|
||||
/**
|
||||
* Goes to the settings to set the RTC mode.
|
||||
* then closes the settings modal.
|
||||
|
||||
@@ -34,9 +34,12 @@ widgetTest(
|
||||
.locator('iframe[title="Element Call"]')
|
||||
.contentFrame();
|
||||
|
||||
// We should show a ringing overlay, let's check for that
|
||||
// We should show a ringing tile, let's check for that
|
||||
await expect(
|
||||
brooksFrame.getByText(`Waiting for ${whistler.displayName} to join…`),
|
||||
brooksFrame
|
||||
.getByTestId("videoTile")
|
||||
.filter({ has: brooksFrame.getByText(whistler.displayName) })
|
||||
.filter({ has: brooksFrame.getByText("Calling…") }),
|
||||
).toBeVisible();
|
||||
|
||||
await expect(whistler.page.getByText("Incoming voice call")).toBeVisible();
|
||||
@@ -124,9 +127,12 @@ widgetTest(
|
||||
.locator('iframe[title="Element Call"]')
|
||||
.contentFrame();
|
||||
|
||||
// We should show a ringing overlay, let's check for that
|
||||
// We should show a ringing tile, let's check for that
|
||||
await expect(
|
||||
brooksFrame.getByText(`Waiting for ${whistler.displayName} to join…`),
|
||||
brooksFrame
|
||||
.getByTestId("videoTile")
|
||||
.filter({ has: brooksFrame.getByText(whistler.displayName) })
|
||||
.filter({ has: brooksFrame.getByText("Calling…") }),
|
||||
).toBeVisible();
|
||||
|
||||
await expect(whistler.page.getByText("Incoming video call")).toBeVisible();
|
||||
@@ -208,9 +214,12 @@ widgetTest(
|
||||
.locator('iframe[title="Element Call"]')
|
||||
.contentFrame();
|
||||
|
||||
// We should show a ringing overlay, let's check for that
|
||||
// We should show a ringing tile, let's check for that
|
||||
await expect(
|
||||
brooksFrame.getByText(`Waiting for ${whistler.displayName} to join…`),
|
||||
brooksFrame
|
||||
.getByTestId("videoTile")
|
||||
.filter({ has: brooksFrame.getByText(whistler.displayName) })
|
||||
.filter({ has: brooksFrame.getByText("Calling…") }),
|
||||
).toBeVisible();
|
||||
|
||||
await expect(whistler.page.getByText("Incoming video call")).toBeVisible();
|
||||
|
||||
@@ -9,14 +9,18 @@ import { afterEach, expect, test, vi } from "vitest";
|
||||
import { render, screen } from "@testing-library/react";
|
||||
import { type MatrixClient } from "matrix-js-sdk";
|
||||
import { type FC, type PropsWithChildren } from "react";
|
||||
import { type WidgetApi } from "matrix-widget-api";
|
||||
|
||||
import { ClientContextProvider } from "./ClientContext";
|
||||
import { Avatar } from "./Avatar";
|
||||
import { Avatar, getAvatarFromWidgetAPI } from "./Avatar";
|
||||
import { mockMatrixRoomMember, mockRtcMembership } from "./utils/test";
|
||||
import { widget } from "./widget";
|
||||
|
||||
const TestComponent: FC<
|
||||
PropsWithChildren<{ client: MatrixClient; supportsThumbnails?: boolean }>
|
||||
> = ({ client, children, supportsThumbnails }) => {
|
||||
PropsWithChildren<{
|
||||
client: MatrixClient;
|
||||
}>
|
||||
> = ({ client, children }) => {
|
||||
return (
|
||||
<ClientContextProvider
|
||||
value={{
|
||||
@@ -24,7 +28,6 @@ const TestComponent: FC<
|
||||
disconnected: false,
|
||||
supportedFeatures: {
|
||||
reactions: true,
|
||||
thumbnails: supportsThumbnails ?? true,
|
||||
},
|
||||
setClient: vi.fn(),
|
||||
authenticated: {
|
||||
@@ -40,6 +43,12 @@ const TestComponent: FC<
|
||||
);
|
||||
};
|
||||
|
||||
vi.mock("./widget", () => ({
|
||||
widget: {
|
||||
api: null, // Ideally we'd only mock this in the as a widget test so the whole module is otherwise null, but just nulling `api` by default works well enough
|
||||
},
|
||||
}));
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
@@ -73,36 +82,7 @@ test("should just render a placeholder when the user has no avatar", () => {
|
||||
expect(client.mxcUrlToHttp).toBeCalledTimes(0);
|
||||
});
|
||||
|
||||
test("should just render a placeholder when thumbnails are not supported", () => {
|
||||
const client = vi.mocked<MatrixClient>({
|
||||
getAccessToken: () => "my-access-token",
|
||||
mxcUrlToHttp: () => vi.fn(),
|
||||
} as unknown as MatrixClient);
|
||||
|
||||
vi.spyOn(client, "mxcUrlToHttp");
|
||||
const member = mockMatrixRoomMember(
|
||||
mockRtcMembership("@alice:example.org", "AAAA"),
|
||||
{
|
||||
getMxcAvatarUrl: () => "mxc://example.org/alice-avatar",
|
||||
},
|
||||
);
|
||||
const displayName = "Alice";
|
||||
render(
|
||||
<TestComponent client={client} supportsThumbnails={false}>
|
||||
<Avatar
|
||||
id={member.userId}
|
||||
name={displayName}
|
||||
size={96}
|
||||
src={member.getMxcAvatarUrl()}
|
||||
/>
|
||||
</TestComponent>,
|
||||
);
|
||||
const element = screen.getByRole("img", { name: "@alice:example.org" });
|
||||
expect(element.tagName).toEqual("SPAN");
|
||||
expect(client.mxcUrlToHttp).toBeCalledTimes(0);
|
||||
});
|
||||
|
||||
test("should attempt to fetch authenticated media", async () => {
|
||||
test("should attempt to fetch authenticated media from the server", async () => {
|
||||
const expectedAuthUrl = "http://example.org/media/alice-avatar";
|
||||
const expectedObjectURL = "my-object-url";
|
||||
const accessToken = "my-access-token";
|
||||
@@ -154,3 +134,80 @@ test("should attempt to fetch authenticated media", async () => {
|
||||
headers: { Authorization: `Bearer ${accessToken}` },
|
||||
});
|
||||
});
|
||||
|
||||
test("should attempt to use widget API if running as a widget", async () => {
|
||||
const expectedMXCUrl = "mxc://example.org/alice-avatar";
|
||||
const expectedObjectURL = "my-object-url";
|
||||
const theBlob = new Blob([]);
|
||||
|
||||
// vitest doesn't have a implementation of create/revokeObjectURL, so we need
|
||||
// to delete the property. It's a bit odd, but it works.
|
||||
Reflect.deleteProperty(global.window.URL, "createObjectURL");
|
||||
globalThis.URL.createObjectURL = vi.fn().mockReturnValue(expectedObjectURL);
|
||||
Reflect.deleteProperty(global.window.URL, "revokeObjectURL");
|
||||
globalThis.URL.revokeObjectURL = vi.fn();
|
||||
|
||||
const client = vi.mocked<MatrixClient>({
|
||||
getAccessToken: () => undefined,
|
||||
} as unknown as MatrixClient);
|
||||
|
||||
widget!.api = { downloadFile: vi.fn() } as unknown as WidgetApi;
|
||||
vi.spyOn(widget!.api, "downloadFile").mockResolvedValue({ file: theBlob });
|
||||
const member = mockMatrixRoomMember(
|
||||
mockRtcMembership("@alice:example.org", "AAAA"),
|
||||
{
|
||||
getMxcAvatarUrl: () => expectedMXCUrl,
|
||||
},
|
||||
);
|
||||
const displayName = "Alice";
|
||||
render(
|
||||
<TestComponent client={client}>
|
||||
<Avatar
|
||||
id={member.userId}
|
||||
name={displayName}
|
||||
size={96}
|
||||
src={member.getMxcAvatarUrl()}
|
||||
/>
|
||||
</TestComponent>,
|
||||
);
|
||||
|
||||
// Fetch is asynchronous, so wait for this to resolve.
|
||||
await vi.waitUntil(() =>
|
||||
document.querySelector(`img[src='${expectedObjectURL}']`),
|
||||
);
|
||||
|
||||
expect(widget!.api.downloadFile).toBeCalledWith(expectedMXCUrl);
|
||||
});
|
||||
|
||||
test("Supports download files as base64", async () => {
|
||||
const expectedMXCUrl = "mxc://example.org/alice-avatar";
|
||||
const expectedBase64 =
|
||||
"iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAIAAACQkWg2AAADIElEQVR4nAAQA+/8ApxhEfFNuwna" +
|
||||
"+DO1pFMx5YDg6gb8p1WFkbFSox9H6r5c8jp1gxlHXrDfA/oQFi4A0gTXH9YBNgwRm12xO68QP6lv" +
|
||||
"ZLKH9qW1VM6kz6zA3T1Ui8J+Xbnh2BZ7oXDe/2gajzoA6j1JGotpz99xO+T2NR634Nhx3zhuera/" +
|
||||
"UdrpMLdEpwWXLnSqZRasGsrl93FjdTwRBMaqsx6vJksnPOmV9ttbXFIOb0XDGPbVythSC2n7P/bS" +
|
||||
"Zv0U0QqbBLk/5Wu1werYzAHiz11Bj8bEylQ92Pxvo+PwF6/KbGnIHTvGZkFzDkMnqz3g7Pw3NOSP" +
|
||||
"oV+qfyJuSI0AeZmrPejFQ8kzBSDWO8D7lr4+6ePRBRmZtKCf+fNjSCOyb5jqwhBnD2cycbJtQQbR" +
|
||||
"A4qdPG2ONfTPeQgi96+zT7grBI0JwvgFBceJdLJd4BX1VQIyY+j7OYueNWqEpf8iYgMj78I95eRt" +
|
||||
"nfPLwlxhVns84iL4Yvw8jDrB9vQi8ktpsdJOMiDwKrBGD3q56COD2oIA96CCBgiro4tkvkumZSAc" +
|
||||
"ZKXRLsziUFGytWJLaPjwnzXv2hicPy6k9AXsF3QkysOZAkB3m9XPpixhq9b0OKqV/zZx3L79o6wZ" +
|
||||
"Dr40J7sj7f+ARd545CP01r5omHt94tbnjgA46HsM2OhP+qQ882LN+Bhscq2WSHGSHT4J9MQcsWZP" +
|
||||
"2+N2LdPy61MN4/1++BJHmDcDLQBUEwLvjZp1fRfzxV7yirwIiOA7Vr8z+1yvS/pSkfUzkjswybOd" +
|
||||
"M5i0I8Q69MTXAKxqtR0/tyGkfCmHfupGASp/SAT9J8f3aQV+gDbpva592v4w8Cv5EMm7CzZPwThF" +
|
||||
"kgTChNPts7F03ccxpblfIz0EiAON1DKk71rX07BvDlLHY1ItPuqZ7hjy19jrAgl+QqEE1btHVA5R" +
|
||||
"uAnRXpEWc6rjARlJY5G1wbMk12rrqpr8rhR3YpFgLgOx4BtQ0D/hGe7KANSGBMQojmObId0asCmd" +
|
||||
"XzmnQI9P8QnwsO9vtqZlgIoU4g+f2/G8Q3/nVMX7dujniwEAAP//KmiQs7P8MeIAAAAASUVORK5C" +
|
||||
"YII=";
|
||||
const mockWidgetAPI = {
|
||||
downloadFile: vi.fn().mockImplementation(async (contentUri) => {
|
||||
if (contentUri !== expectedMXCUrl) {
|
||||
return Promise.reject(new Error("Unexpected content URI"));
|
||||
}
|
||||
return { file: expectedBase64 };
|
||||
}),
|
||||
} as unknown as WidgetApi;
|
||||
|
||||
const blob = await getAvatarFromWidgetAPI(mockWidgetAPI, expectedMXCUrl);
|
||||
|
||||
expect(blob).toBeInstanceOf(Blob);
|
||||
});
|
||||
|
||||
@@ -14,8 +14,10 @@ import {
|
||||
} from "react";
|
||||
import { Avatar as CompoundAvatar } from "@vector-im/compound-web";
|
||||
import { type MatrixClient } from "matrix-js-sdk";
|
||||
import { type WidgetApi } from "matrix-widget-api";
|
||||
|
||||
import { useClientState } from "./ClientContext";
|
||||
import { widget } from "./widget";
|
||||
|
||||
export enum Size {
|
||||
XS = "xs",
|
||||
@@ -78,50 +80,54 @@ export const Avatar: FC<Props> = ({
|
||||
const sizePx = useMemo(
|
||||
() =>
|
||||
Object.values(Size).includes(size as Size)
|
||||
? sizes.get(size as Size)
|
||||
? sizes.get(size as Size)!
|
||||
: (size as number),
|
||||
[size],
|
||||
);
|
||||
|
||||
const [avatarUrl, setAvatarUrl] = useState<string | undefined>(undefined);
|
||||
|
||||
// In theory, a change in `clientState` or `sizePx` could run extra getAvatarFromWidgetAPI calls, but in practice they should be stable long before this code runs.
|
||||
useEffect(() => {
|
||||
if (clientState?.state !== "valid") {
|
||||
return;
|
||||
}
|
||||
const { authenticated, supportedFeatures } = clientState;
|
||||
const client = authenticated?.client;
|
||||
|
||||
if (!client || !src || !sizePx || !supportedFeatures.thumbnails) {
|
||||
if (!src) {
|
||||
setAvatarUrl(undefined);
|
||||
return;
|
||||
}
|
||||
|
||||
const token = client.getAccessToken();
|
||||
if (!token) {
|
||||
return;
|
||||
}
|
||||
const resolveSrc = getAvatarUrl(client, src, sizePx);
|
||||
if (!resolveSrc) {
|
||||
let blob: Promise<Blob>;
|
||||
|
||||
if (widget?.api) {
|
||||
blob = getAvatarFromWidgetAPI(widget.api, src);
|
||||
} else if (
|
||||
clientState?.state === "valid" &&
|
||||
clientState.authenticated?.client &&
|
||||
sizePx
|
||||
) {
|
||||
blob = getAvatarFromServer(clientState.authenticated.client, src, sizePx);
|
||||
} else {
|
||||
setAvatarUrl(undefined);
|
||||
return;
|
||||
}
|
||||
|
||||
let objectUrl: string | undefined;
|
||||
fetch(resolveSrc, {
|
||||
headers: {
|
||||
Authorization: `Bearer ${token}`,
|
||||
},
|
||||
})
|
||||
.then(async (req) => req.blob())
|
||||
let stale = false;
|
||||
blob
|
||||
.then((blob) => {
|
||||
if (stale) {
|
||||
return;
|
||||
}
|
||||
objectUrl = URL.createObjectURL(blob);
|
||||
setAvatarUrl(objectUrl);
|
||||
})
|
||||
.catch((ex) => {
|
||||
if (stale) {
|
||||
return;
|
||||
}
|
||||
setAvatarUrl(undefined);
|
||||
});
|
||||
|
||||
return (): void => {
|
||||
stale = true;
|
||||
if (objectUrl) {
|
||||
URL.revokeObjectURL(objectUrl);
|
||||
}
|
||||
@@ -140,3 +146,50 @@ export const Avatar: FC<Props> = ({
|
||||
/>
|
||||
);
|
||||
};
|
||||
|
||||
async function getAvatarFromServer(
|
||||
client: MatrixClient,
|
||||
src: string,
|
||||
sizePx: number,
|
||||
): Promise<Blob> {
|
||||
const httpSrc = getAvatarUrl(client, src, sizePx);
|
||||
if (!httpSrc) {
|
||||
throw new Error("Failed to get http avatar URL");
|
||||
}
|
||||
|
||||
const token = client.getAccessToken();
|
||||
if (!token) {
|
||||
throw new Error("Failed to get access token");
|
||||
}
|
||||
|
||||
const request = await fetch(httpSrc, {
|
||||
headers: {
|
||||
Authorization: `Bearer ${token}`,
|
||||
},
|
||||
});
|
||||
|
||||
const blob = await request.blob();
|
||||
|
||||
return blob;
|
||||
}
|
||||
|
||||
// export for testing
|
||||
export async function getAvatarFromWidgetAPI(
|
||||
api: WidgetApi,
|
||||
src: string,
|
||||
): Promise<Blob> {
|
||||
const response = await api.downloadFile(src);
|
||||
const file = response.file;
|
||||
|
||||
// element-web sends a Blob, and the MSC4039 is considering changing the spec to strictly Blob, so only handling that
|
||||
if (file instanceof Blob) {
|
||||
return file;
|
||||
} else if (typeof file === "string") {
|
||||
// it is a base64 string
|
||||
const bytes = Uint8Array.from(atob(file), (c) => c.charCodeAt(0));
|
||||
return new Blob([bytes]);
|
||||
}
|
||||
throw new Error(
|
||||
"Downloaded file format is not supported: " + typeof file + "",
|
||||
);
|
||||
}
|
||||
|
||||
@@ -48,7 +48,6 @@ export type ValidClientState = {
|
||||
disconnected: boolean;
|
||||
supportedFeatures: {
|
||||
reactions: boolean;
|
||||
thumbnails: boolean;
|
||||
};
|
||||
setClient: (client: MatrixClient, session: Session) => void;
|
||||
};
|
||||
@@ -249,7 +248,6 @@ export const ClientProvider: FC<Props> = ({ children }) => {
|
||||
|
||||
const [isDisconnected, setIsDisconnected] = useState(false);
|
||||
const [supportsReactions, setSupportsReactions] = useState(false);
|
||||
const [supportsThumbnails, setSupportsThumbnails] = useState(false);
|
||||
|
||||
const state: ClientState | undefined = useMemo(() => {
|
||||
if (alreadyOpenedErr) {
|
||||
@@ -275,7 +273,6 @@ export const ClientProvider: FC<Props> = ({ children }) => {
|
||||
disconnected: isDisconnected,
|
||||
supportedFeatures: {
|
||||
reactions: supportsReactions,
|
||||
thumbnails: supportsThumbnails,
|
||||
},
|
||||
};
|
||||
}, [
|
||||
@@ -286,7 +283,6 @@ export const ClientProvider: FC<Props> = ({ children }) => {
|
||||
setClient,
|
||||
isDisconnected,
|
||||
supportsReactions,
|
||||
supportsThumbnails,
|
||||
]);
|
||||
|
||||
const onSync = useCallback(
|
||||
@@ -312,8 +308,6 @@ export const ClientProvider: FC<Props> = ({ children }) => {
|
||||
}
|
||||
|
||||
if (initClientState.widgetApi) {
|
||||
// There is currently no widget API for authenticated media thumbnails.
|
||||
setSupportsThumbnails(false);
|
||||
const reactSend = initClientState.widgetApi.hasCapability(
|
||||
"org.matrix.msc2762.send.event:m.reaction",
|
||||
);
|
||||
@@ -335,7 +329,6 @@ export const ClientProvider: FC<Props> = ({ children }) => {
|
||||
}
|
||||
} else {
|
||||
setSupportsReactions(true);
|
||||
setSupportsThumbnails(true);
|
||||
}
|
||||
|
||||
return (): void => {
|
||||
|
||||
@@ -25,7 +25,7 @@ export abstract class TranslatedError extends Error {
|
||||
messageKey: ParseKeys<DefaultNamespace, TOptions>,
|
||||
translationFn: TFunction<DefaultNamespace>,
|
||||
) {
|
||||
super(translationFn(messageKey, { lng: "en" } as TOptions));
|
||||
super(translationFn(messageKey, { lng: "en" }));
|
||||
this.translatedMessage = translationFn(messageKey);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,7 +14,7 @@ exports[`AppBar > renders 1`] = `
|
||||
>
|
||||
<button
|
||||
aria-labelledby="_r_0_"
|
||||
class="_icon-button_1pz9o_8"
|
||||
class="_icon-button_1215g_8"
|
||||
data-kind="primary"
|
||||
role="button"
|
||||
style="--cpd-icon-button-size: 32px;"
|
||||
|
||||
@@ -33,12 +33,38 @@ export interface Controls {
|
||||
showNativeOutputDevicePicker?: () => void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Output Audio device when using the controlled audio output mode (mobile).
|
||||
*/
|
||||
export interface OutputDevice {
|
||||
id: string;
|
||||
name: string;
|
||||
/**
|
||||
* `forEarpiece` in an iOS only flag, that will be set on the default speaker device.
|
||||
* The default speaker device will be used for the earpiece mode by
|
||||
* using a stereo pan and reducing the volume significantly. (in combination this is similar to a dedicated earpiece mode)
|
||||
* - on iOS this is true if output is routed to speaker.
|
||||
* In that case then ElementCalls manually appends an earpiece device with id `EARPIECE_CONFIG_ID` and `{ type: "earpiece" }`
|
||||
* - on Android this is unused.
|
||||
*/
|
||||
forEarpiece?: boolean;
|
||||
/**
|
||||
* Is the device the OS earpiece audio configuration?
|
||||
* - on iOS always undefined
|
||||
* - on Android true for the `TYPE_BUILTIN_EARPIECE`
|
||||
*/
|
||||
isEarpiece?: boolean;
|
||||
/**
|
||||
* Is the device the OS default speaker:
|
||||
* - on iOS always true if output is routed to speaker. In other case iOS on declare a `dummy` id device.
|
||||
* - on Android true for the `TYPE_BUILTIN_SPEAKER`
|
||||
*/
|
||||
isSpeaker?: boolean;
|
||||
/**
|
||||
* Is the device the OS default external headset (bluetooth):
|
||||
* - on iOS always undefined.
|
||||
* - on Android true for the `TYPE_BLUETOOTH_SCO`
|
||||
*/
|
||||
isExternalHeadset?: boolean;
|
||||
}
|
||||
|
||||
@@ -47,8 +73,16 @@ export interface OutputDevice {
|
||||
*/
|
||||
export const setPipEnabled$ = new Subject<boolean>();
|
||||
|
||||
/**
|
||||
* Stores the list of available controlled audio output devices.
|
||||
* This is set when the native code calls `setAvailableAudioDevices` with the list of available audio output devices.
|
||||
*/
|
||||
export const availableOutputDevices$ = new Subject<OutputDevice[]>();
|
||||
|
||||
/**
|
||||
* Stores the current audio output device id.
|
||||
* This is set when the native code calls `setAudioDevice`
|
||||
*/
|
||||
export const outputDevice$ = new Subject<string>();
|
||||
|
||||
/**
|
||||
@@ -80,16 +114,41 @@ window.controls = {
|
||||
setPipEnabled$.next(false);
|
||||
},
|
||||
|
||||
/**
|
||||
* Reverse engineered:
|
||||
*
|
||||
* - on iOS:
|
||||
* This always a list of one thing. If current route output is speaker it returns
|
||||
* the single `{"id":"Speaker","name":"Speaker","forEarpiece":true,"isSpeaker":true}` Notice that EC will
|
||||
* also manually add a virtual earpiece device with id `EARPIECE_CONFIG_ID` and `{ type: "earpiece" }`.
|
||||
* If the route output is not speaker then it will be `{id: 'dummy', name: 'dummy'}`
|
||||
*
|
||||
*
|
||||
* - on Android:
|
||||
* This is a list of all available output audio devices. The `id` is the Android AudioDeviceInfo.getId()
|
||||
* and the `name` is based the Android AudioDeviceInfo.productName (mapped to static strings for known types)
|
||||
* The `isEarpiece`, `isSpeaker` and `isExternalHeadset` are set based on the Android AudioDeviceInfo.type
|
||||
* matching the corresponding types for earpiece, speaker and bluetooth headset.
|
||||
*/
|
||||
setAvailableAudioDevices(devices: OutputDevice[]): void {
|
||||
logger.info("setAvailableAudioDevices called from native:", devices);
|
||||
logger.info(
|
||||
"[MediaDevices controls] setAvailableAudioDevices called from native:",
|
||||
devices,
|
||||
);
|
||||
availableOutputDevices$.next(devices);
|
||||
},
|
||||
setAudioDevice(id: string): void {
|
||||
logger.info("setAudioDevice called from native", id);
|
||||
logger.info(
|
||||
"[MediaDevices controls] setAudioDevice called from native",
|
||||
id,
|
||||
);
|
||||
outputDevice$.next(id);
|
||||
},
|
||||
setAudioEnabled(enabled: boolean): void {
|
||||
logger.info("setAudioEnabled called from native:", enabled);
|
||||
logger.info(
|
||||
"[MediaDevices controls] setAudioEnabled called from native:",
|
||||
enabled,
|
||||
);
|
||||
if (!setAudioEnabled$.observed)
|
||||
throw new Error(
|
||||
"Output controls are disabled. No setAudioEnabled$ observer",
|
||||
|
||||
@@ -51,15 +51,15 @@ export const makeOneOnOneLayout: CallLayout<OneOnOneLayoutModel> = ({
|
||||
return (
|
||||
<div ref={ref} className={styles.layer}>
|
||||
<Slot
|
||||
id={model.remote.id}
|
||||
model={model.remote}
|
||||
id={model.spotlight.id}
|
||||
model={model.spotlight}
|
||||
className={styles.container}
|
||||
style={{ width: tileWidth, height: tileHeight }}
|
||||
>
|
||||
<Slot
|
||||
className={classNames(styles.slot, styles.local)}
|
||||
id={model.local.id}
|
||||
model={model.local}
|
||||
id={model.pip.id}
|
||||
model={model.pip}
|
||||
onDrag={onDragLocalTile}
|
||||
data-block-alignment={pipAlignmentValue.block}
|
||||
data-inline-alignment={pipAlignmentValue.inline}
|
||||
|
||||
@@ -19,12 +19,14 @@ import fetchMock from "fetch-mock";
|
||||
import { getSFUConfigWithOpenID, type OpenIDClientParts } from "./openIDSFU";
|
||||
import { testJWTToken } from "../utils/test-fixtures";
|
||||
import { ownMemberMock } from "../utils/test";
|
||||
import { FailToGetOpenIdToken } from "../utils/errors";
|
||||
|
||||
const sfuUrl = "https://sfu.example.org";
|
||||
|
||||
describe("getSFUConfigWithOpenID", () => {
|
||||
let matrixClient: MockedObject<OpenIDClientParts>;
|
||||
beforeEach(() => {
|
||||
fetchMock.catch(404);
|
||||
matrixClient = {
|
||||
getOpenIdToken: vitest.fn(),
|
||||
getDeviceId: vitest.fn(),
|
||||
@@ -71,9 +73,10 @@ describe("getSFUConfigWithOpenID", () => {
|
||||
"https://sfu.example.org",
|
||||
"!example_room_id",
|
||||
);
|
||||
} catch (ex) {
|
||||
expect((ex as Error).message).toEqual(
|
||||
"SFU Config fetch failed with status code 500",
|
||||
} catch (ex: unknown) {
|
||||
expect(ex).toBeInstanceOf(FailToGetOpenIdToken);
|
||||
expect((ex as FailToGetOpenIdToken).cause).toEqual(
|
||||
new Error("SFU Config fetch failed with status code 500"),
|
||||
);
|
||||
void (await fetchMock.flush());
|
||||
return;
|
||||
@@ -106,8 +109,9 @@ describe("getSFUConfigWithOpenID", () => {
|
||||
},
|
||||
);
|
||||
} catch (ex) {
|
||||
expect((ex as Error).message).toEqual(
|
||||
"SFU Config fetch failed with status code 500",
|
||||
expect(ex).toBeInstanceOf(FailToGetOpenIdToken);
|
||||
expect((ex as FailToGetOpenIdToken).cause).toEqual(
|
||||
new Error("SFU Config fetch failed with status code 500"),
|
||||
);
|
||||
void (await fetchMock.flush());
|
||||
}
|
||||
@@ -160,8 +164,9 @@ describe("getSFUConfigWithOpenID", () => {
|
||||
},
|
||||
);
|
||||
} catch (ex) {
|
||||
expect((ex as Error).message).toEqual(
|
||||
"SFU Config fetch failed with status code 500",
|
||||
expect(ex).toBeInstanceOf(FailToGetOpenIdToken);
|
||||
expect((ex as FailToGetOpenIdToken).cause).toEqual(
|
||||
new Error("SFU Config fetch failed with status code 500"),
|
||||
);
|
||||
void (await fetchMock.flush());
|
||||
}
|
||||
|
||||
@@ -5,11 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import {
|
||||
retryNetworkOperation,
|
||||
type IOpenIDToken,
|
||||
type MatrixClient,
|
||||
} from "matrix-js-sdk";
|
||||
import { type IOpenIDToken, type MatrixClient } from "matrix-js-sdk";
|
||||
import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
|
||||
import { type Logger } from "matrix-js-sdk/lib/logger";
|
||||
|
||||
@@ -70,6 +66,7 @@ export type OpenIDClientParts = Pick<
|
||||
MatrixClient,
|
||||
"getOpenIdToken" | "getDeviceId"
|
||||
>;
|
||||
|
||||
/**
|
||||
* Gets a bearer token from the homeserver and then use it to authenticate
|
||||
* to the matrix RTC backend in order to get acces to the SFU.
|
||||
@@ -113,9 +110,6 @@ export async function getSFUConfigWithOpenID(
|
||||
);
|
||||
}
|
||||
logger?.debug("Got openID token", openIdToken);
|
||||
|
||||
logger?.info(`Trying to get JWT for focus ${serviceUrl}...`);
|
||||
|
||||
let sfuConfig: { url: string; jwt: string } | undefined;
|
||||
|
||||
const tryBothJwtEndpoints = opts?.forceJwtEndpoint === undefined; // This is for SFUs where we do not publish.
|
||||
@@ -127,7 +121,10 @@ export async function getSFUConfigWithOpenID(
|
||||
// if we can use both or if we are forced to use the new one.
|
||||
if (tryBothJwtEndpoints || forceMatrix2Jwt) {
|
||||
try {
|
||||
sfuConfig = await getLiveKitJWTWithDelayDelegation(
|
||||
logger?.info(
|
||||
`Trying to get JWT with delegation for focus ${serviceUrl}...`,
|
||||
);
|
||||
const sfuConfig = await getLiveKitJWTWithDelayDelegation(
|
||||
membership,
|
||||
serviceUrl,
|
||||
roomId,
|
||||
@@ -135,32 +132,24 @@ export async function getSFUConfigWithOpenID(
|
||||
opts?.delayEndpointBaseUrl,
|
||||
opts?.delayId,
|
||||
);
|
||||
logger?.info(`Got JWT from call's active focus URL.`);
|
||||
|
||||
return extractFullConfigFromToken(sfuConfig);
|
||||
} catch (e) {
|
||||
if (e instanceof NotSupportedError) {
|
||||
logger?.warn(
|
||||
`Failed fetching jwt with matrix 2.0 endpoint (retry with legacy) Not supported`,
|
||||
e,
|
||||
);
|
||||
sfuConfig = undefined;
|
||||
} else {
|
||||
logger?.warn(
|
||||
`Failed fetching jwt with matrix 2.0 endpoint other issues ->`,
|
||||
`(not going to try with legacy endpoint: forceOldJwtEndpoint is set to false, we did not get a not supported error from the sfu)`,
|
||||
e,
|
||||
);
|
||||
// Make this throw a hard error in case we force the matrix2.0 endpoint.
|
||||
if (forceMatrix2Jwt)
|
||||
throw new NoMatrix2AuthorizationService(e as Error);
|
||||
// NEVER get bejond this point if we forceMatrix2 and it failed!
|
||||
logger?.debug(`Failed fetching jwt with matrix 2.0 endpoint:`, e);
|
||||
// Make this throw a hard error in case we force the matrix2.0 endpoint.
|
||||
if (forceMatrix2Jwt) {
|
||||
throw new NoMatrix2AuthorizationService(e as Error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// DEPRECATED
|
||||
// here we either have a sfuConfig or we alredy exited because of `if (forceMatrix2) throw ...`
|
||||
// here we either have a sfuConfig or we already exited because of `if (forceMatrix2) throw ...`
|
||||
// The only case we can get into this condition is, if `forceMatrix2` is `false`
|
||||
if (sfuConfig === undefined) {
|
||||
try {
|
||||
logger?.info(
|
||||
`Trying to get JWT with legacy endpoint for focus ${serviceUrl}...`,
|
||||
);
|
||||
sfuConfig = await getLiveKitJWT(
|
||||
membership.deviceId,
|
||||
serviceUrl,
|
||||
@@ -168,15 +157,19 @@ export async function getSFUConfigWithOpenID(
|
||||
openIdToken,
|
||||
);
|
||||
logger?.info(`Got JWT from call's active focus URL.`);
|
||||
return extractFullConfigFromToken(sfuConfig);
|
||||
} catch (ex) {
|
||||
throw new FailToGetOpenIdToken(
|
||||
ex instanceof Error ? ex : new Error(`Unknown error ${ex}`),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (!sfuConfig) {
|
||||
throw new Error("No `sfuConfig` after trying with old and new endpoints");
|
||||
}
|
||||
|
||||
// Pull the details from the JWT
|
||||
function extractFullConfigFromToken(sfuConfig: {
|
||||
url: string;
|
||||
jwt: string;
|
||||
}): SFUConfig {
|
||||
const [, payloadStr] = sfuConfig.jwt.split(".");
|
||||
// TODO: Prefer Uint8Array.fromBase64 when widely available
|
||||
const payload = JSON.parse(global.atob(payloadStr)) as SFUJWTPayload;
|
||||
return {
|
||||
jwt: sfuConfig.jwt,
|
||||
@@ -188,16 +181,15 @@ export async function getSFUConfigWithOpenID(
|
||||
livekitIdentity: payload.sub,
|
||||
};
|
||||
}
|
||||
const RETRIES = 4;
|
||||
|
||||
async function getLiveKitJWT(
|
||||
deviceId: string,
|
||||
livekitServiceURL: string,
|
||||
matrixRoomId: string,
|
||||
openIDToken: IOpenIDToken,
|
||||
): Promise<{ url: string; jwt: string }> {
|
||||
let res: Response | undefined;
|
||||
await retryNetworkOperation(RETRIES, async () => {
|
||||
res = await fetch(livekitServiceURL + "/sfu/get", {
|
||||
const res = await doNetworkOperationWithRetry(async () => {
|
||||
return await fetch(livekitServiceURL + "/sfu/get", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
@@ -210,11 +202,7 @@ async function getLiveKitJWT(
|
||||
}),
|
||||
});
|
||||
});
|
||||
if (!res) {
|
||||
throw new Error(
|
||||
`Network error while connecting to jwt service after ${RETRIES} retries`,
|
||||
);
|
||||
}
|
||||
|
||||
if (!res.ok) {
|
||||
throw new Error("SFU Config fetch failed with status code " + res.status);
|
||||
}
|
||||
@@ -261,10 +249,8 @@ export async function getLiveKitJWTWithDelayDelegation(
|
||||
};
|
||||
}
|
||||
|
||||
let res: Response | undefined;
|
||||
|
||||
await retryNetworkOperation(RETRIES, async () => {
|
||||
res = await fetch(livekitServiceURL + "/get_token", {
|
||||
const res = await doNetworkOperationWithRetry(async () => {
|
||||
return await fetch(livekitServiceURL + "/get_token", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
@@ -273,11 +259,6 @@ export async function getLiveKitJWTWithDelayDelegation(
|
||||
});
|
||||
});
|
||||
|
||||
if (!res) {
|
||||
throw new Error(
|
||||
`Network error while connecting to jwt service after ${RETRIES} retries`,
|
||||
);
|
||||
}
|
||||
if (!res.ok) {
|
||||
const msg = "SFU Config fetch failed with status code " + res.status;
|
||||
if (res.status === 404) {
|
||||
|
||||
@@ -67,6 +67,6 @@ Initializer.initBeforeReact()
|
||||
);
|
||||
})
|
||||
.catch((e) => {
|
||||
logger.error("Failed to initialize app", e);
|
||||
logger.error(`Failed to initialize app ${e.message}`, e);
|
||||
root.render(e.message);
|
||||
});
|
||||
|
||||
@@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { IconButton, Text, Tooltip } from "@vector-im/compound-web";
|
||||
import { IconButton, Tooltip } from "@vector-im/compound-web";
|
||||
import { type MatrixClient, type Room as MatrixRoom } from "matrix-js-sdk";
|
||||
import {
|
||||
type FC,
|
||||
@@ -98,8 +98,6 @@ import { useAppBarHidden, useAppBarSecondaryButton } from "../AppBar.tsx";
|
||||
import { useBehavior } from "../useBehavior.ts";
|
||||
import { Toast } from "../Toast.tsx";
|
||||
import overlayStyles from "../Overlay.module.css";
|
||||
import { Avatar, Size as AvatarSize } from "../Avatar";
|
||||
import waitingStyles from "./WaitingForJoin.module.css";
|
||||
import { prefetchSounds } from "../soundUtils";
|
||||
import { useAudioContext } from "../useAudioContext";
|
||||
import ringtoneMp3 from "../sound/ringtone.mp3?url";
|
||||
@@ -107,6 +105,7 @@ import ringtoneOgg from "../sound/ringtone.ogg?url";
|
||||
import { useTrackProcessorObservable$ } from "../livekit/TrackProcessorContext.tsx";
|
||||
import { type Layout } from "../state/layout-types.ts";
|
||||
import { ObservableScope } from "../state/ObservableScope.ts";
|
||||
import { useLatest } from "../useLatest.ts";
|
||||
|
||||
const logger = rootLogger.getChild("[InCallView]");
|
||||
|
||||
@@ -224,8 +223,6 @@ export const InCallView: FC<InCallViewProps> = ({
|
||||
const { showControls } = useUrlParams();
|
||||
|
||||
const muteAllAudio = useBehavior(muteAllAudio$);
|
||||
// Call pickup state and display names are needed for waiting overlay/sounds
|
||||
const callPickupState = useBehavior(vm.callPickupState$);
|
||||
|
||||
// Preload a waiting and decline sounds
|
||||
const pickupPhaseSoundCache = useInitial(async () => {
|
||||
@@ -239,6 +236,7 @@ export const InCallView: FC<InCallViewProps> = ({
|
||||
latencyHint: "interactive",
|
||||
muted: muteAllAudio,
|
||||
});
|
||||
const latestPickupPhaseAudio = useLatest(pickupPhaseAudio);
|
||||
|
||||
const audioEnabled = useBehavior(muteStates.audio.enabled$);
|
||||
const videoEnabled = useBehavior(muteStates.video.enabled$);
|
||||
@@ -257,6 +255,7 @@ export const InCallView: FC<InCallViewProps> = ({
|
||||
() => void toggleRaisedHand(),
|
||||
);
|
||||
|
||||
const ringing = useBehavior(vm.ringing$);
|
||||
const audioParticipants = useBehavior(vm.livekitRoomItems$);
|
||||
const participantCount = useBehavior(vm.participantCount$);
|
||||
const reconnecting = useBehavior(vm.reconnecting$);
|
||||
@@ -271,7 +270,6 @@ export const InCallView: FC<InCallViewProps> = ({
|
||||
const audioOutputSwitcher = useBehavior(vm.audioOutputSwitcher$);
|
||||
const sharingScreen = useBehavior(vm.sharingScreen$);
|
||||
|
||||
const ringOverlay = useBehavior(vm.ringOverlay$);
|
||||
const fatalCallError = useBehavior(vm.fatalError$);
|
||||
// Stop the rendering and throw for the error boundary
|
||||
if (fatalCallError) {
|
||||
@@ -279,58 +277,21 @@ export const InCallView: FC<InCallViewProps> = ({
|
||||
throw fatalCallError;
|
||||
}
|
||||
|
||||
// We need to set the proper timings on the animation based upon the sound length.
|
||||
const ringDuration = pickupPhaseAudio?.soundDuration["waiting"] ?? 1;
|
||||
useEffect((): (() => void) => {
|
||||
// The CSS animation includes the delay, so we must double the length of the sound.
|
||||
window.document.body.style.setProperty(
|
||||
"--call-ring-duration-s",
|
||||
`${ringDuration * 2}s`,
|
||||
);
|
||||
window.document.body.style.setProperty(
|
||||
"--call-ring-delay-s",
|
||||
`${ringDuration}s`,
|
||||
);
|
||||
// Remove properties when we unload.
|
||||
return () => {
|
||||
window.document.body.style.removeProperty("--call-ring-duration-s");
|
||||
window.document.body.style.removeProperty("--call-ring-delay-s");
|
||||
};
|
||||
}, [pickupPhaseAudio?.soundDuration, ringDuration]);
|
||||
|
||||
// When waiting for pickup, loop a waiting sound
|
||||
// While ringing, loop the ringtone
|
||||
useEffect((): void | (() => void) => {
|
||||
if (callPickupState !== "ringing" || !pickupPhaseAudio) return;
|
||||
const endSound = pickupPhaseAudio.playSoundLooping("waiting", ringDuration);
|
||||
return () => {
|
||||
void endSound().catch((e) => {
|
||||
logger.error("Failed to stop ringing sound", e);
|
||||
});
|
||||
};
|
||||
}, [callPickupState, pickupPhaseAudio, ringDuration]);
|
||||
|
||||
// Waiting UI overlay
|
||||
const waitingOverlay: JSX.Element | null = useMemo(() => {
|
||||
return ringOverlay ? (
|
||||
<div className={classNames(overlayStyles.bg, waitingStyles.overlay)}>
|
||||
<div
|
||||
className={classNames(overlayStyles.content, waitingStyles.content)}
|
||||
>
|
||||
<div className={waitingStyles.pulse}>
|
||||
<Avatar
|
||||
id={ringOverlay.idForAvatar}
|
||||
name={ringOverlay.name}
|
||||
src={ringOverlay.avatarMxc}
|
||||
size={AvatarSize.XL}
|
||||
/>
|
||||
</div>
|
||||
<Text size="md" className={waitingStyles.text}>
|
||||
{ringOverlay.text}
|
||||
</Text>
|
||||
</div>
|
||||
</div>
|
||||
) : null;
|
||||
}, [ringOverlay]);
|
||||
const audio = latestPickupPhaseAudio.current;
|
||||
if (ringing && audio) {
|
||||
const endSound = audio.playSoundLooping(
|
||||
"waiting",
|
||||
audio.soundDuration["waiting"] ?? 1,
|
||||
);
|
||||
return () => {
|
||||
void endSound().catch((e) => {
|
||||
logger.error("Failed to stop ringing sound", e);
|
||||
});
|
||||
};
|
||||
}
|
||||
}, [ringing, latestPickupPhaseAudio]);
|
||||
|
||||
const onViewClick = useCallback(
|
||||
(e: ReactMouseEvent) => {
|
||||
@@ -764,7 +725,6 @@ export const InCallView: FC<InCallViewProps> = ({
|
||||
{reconnectingToast}
|
||||
{earpieceOverlay}
|
||||
<ReactionsOverlay vm={vm} />
|
||||
{waitingOverlay}
|
||||
{footer}
|
||||
{layout.type !== "pip" && (
|
||||
<>
|
||||
|
||||
@@ -1,61 +0,0 @@
|
||||
.overlay {
|
||||
position: absolute;
|
||||
inset: 0;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
pointer-events: none;
|
||||
}
|
||||
|
||||
.content {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
gap: 16px;
|
||||
}
|
||||
|
||||
.pulse {
|
||||
position: relative;
|
||||
height: 90px;
|
||||
}
|
||||
|
||||
.pulse::before {
|
||||
content: "";
|
||||
position: absolute;
|
||||
inset: -12px;
|
||||
border-radius: 9999px;
|
||||
border: 12px solid rgba(255, 255, 255, 0.6);
|
||||
animation: pulse var(--call-ring-duration-s) ease-out infinite;
|
||||
animation-delay: 1s;
|
||||
opacity: 0;
|
||||
}
|
||||
|
||||
.text {
|
||||
color: var(--cpd-color-text-on-solid-primary);
|
||||
}
|
||||
|
||||
@keyframes pulse {
|
||||
0% {
|
||||
transform: scale(0.95);
|
||||
opacity: 0.7;
|
||||
transform: scale(0);
|
||||
opacity: 1;
|
||||
}
|
||||
35% {
|
||||
transform: scale(1.15);
|
||||
opacity: 0.15;
|
||||
}
|
||||
50% {
|
||||
transform: scale(1.2);
|
||||
opacity: 0;
|
||||
}
|
||||
50.01% {
|
||||
transform: scale(0);
|
||||
}
|
||||
85% {
|
||||
transform: scale(0);
|
||||
}
|
||||
100% {
|
||||
transform: scale(0);
|
||||
}
|
||||
}
|
||||
@@ -128,24 +128,11 @@ exports[`InCallView > rendering > renders 1`] = `
|
||||
width="1em"
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
>
|
||||
<g
|
||||
clip-path="url(#a)"
|
||||
>
|
||||
<path
|
||||
clip-rule="evenodd"
|
||||
d="M8.929 15.1a13.6 13.6 0 0 0 4.654 3.066q2.62 1.036 5.492.923h.008l.003-.004.003-.002-.034-3.124-3.52-.483-1.791 1.792-.645-.322a13.5 13.5 0 0 1-3.496-2.52 13.4 13.4 0 0 1-2.52-3.496l-.322-.644 1.792-1.792-.483-3.519-3.123-.034-.003.002-.003.004v.002a13.65 13.65 0 0 0 .932 5.492A13.4 13.4 0 0 0 8.93 15.1m3.92 4.926a15.6 15.6 0 0 1-5.334-3.511 15.4 15.4 0 0 1-3.505-5.346 15.6 15.6 0 0 1-1.069-6.274 1.93 1.93 0 0 1 .589-1.366c.366-.366.84-.589 1.386-.589h.01l3.163.035a1.96 1.96 0 0 1 1.958 1.694v.005l.487 3.545v.003c.043.297.025.605-.076.907a2 2 0 0 1-.485.773l-.762.762a11.4 11.4 0 0 0 3.206 3.54q.457.33.948.614l.762-.761a2 2 0 0 1 .774-.486c.302-.1.61-.118.907-.076l3.553.487a1.96 1.96 0 0 1 1.694 1.958l.034 3.174c0 .546-.223 1.02-.588 1.386-.361.36-.827.582-1.363.588a15.3 15.3 0 0 1-6.29-1.062"
|
||||
fill-rule="evenodd"
|
||||
/>
|
||||
</g>
|
||||
<defs>
|
||||
<clippath
|
||||
id="a"
|
||||
>
|
||||
<path
|
||||
d="M0 0h24v24H0z"
|
||||
/>
|
||||
</clippath>
|
||||
</defs>
|
||||
<path
|
||||
clip-rule="evenodd"
|
||||
d="M8.929 15.1a13.6 13.6 0 0 0 4.654 3.066q2.62 1.036 5.492.923h.008l.003-.004.003-.002-.034-3.124-3.52-.483-1.791 1.792-.645-.322a13.5 13.5 0 0 1-3.496-2.52 13.4 13.4 0 0 1-2.52-3.496l-.322-.645 1.792-1.791-.483-3.52-3.123-.033-.003.002-.003.004v.002a13.65 13.65 0 0 0 .932 5.492A13.4 13.4 0 0 0 8.93 15.1m3.92 4.926a15.6 15.6 0 0 1-5.334-3.511 15.4 15.4 0 0 1-3.505-5.346 15.6 15.6 0 0 1-1.069-6.274 1.93 1.93 0 0 1 .589-1.366c.366-.366.84-.589 1.386-.589h.01l3.163.035a1.96 1.96 0 0 1 1.958 1.694v.005l.487 3.545v.003c.043.297.025.605-.076.907a2 2 0 0 1-.485.773l-.762.762a11.3 11.3 0 0 0 1.806 2.348 11.4 11.4 0 0 0 2.348 1.806l.762-.762a2 2 0 0 1 .774-.485c.302-.1.61-.118.907-.076l3.553.487a1.96 1.96 0 0 1 1.694 1.958l.034 3.174c0 .546-.223 1.02-.588 1.386-.36.36-.827.582-1.363.588a15.3 15.3 0 0 1-6.29-1.062"
|
||||
fill-rule="evenodd"
|
||||
/>
|
||||
</svg>
|
||||
</div>
|
||||
<h2
|
||||
|
||||
@@ -234,12 +234,12 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
|
||||
class="_inline-field-control_19upo_44"
|
||||
>
|
||||
<div
|
||||
class="_container_1qhtc_10"
|
||||
class="_container_1ug7n_10"
|
||||
>
|
||||
<input
|
||||
aria-describedby="radix-_r_9_ radix-_r_b_ radix-_r_d_"
|
||||
checked=""
|
||||
class="_input_1qhtc_18"
|
||||
class="_input_1ug7n_18"
|
||||
id="radix-_r_8_"
|
||||
name="_r_0_"
|
||||
title=""
|
||||
@@ -247,7 +247,7 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
|
||||
value="legacy"
|
||||
/>
|
||||
<div
|
||||
class="_ui_1qhtc_19"
|
||||
class="_ui_1ug7n_19"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
@@ -275,11 +275,11 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
|
||||
class="_inline-field-control_19upo_44"
|
||||
>
|
||||
<div
|
||||
class="_container_1qhtc_10"
|
||||
class="_container_1ug7n_10"
|
||||
>
|
||||
<input
|
||||
aria-describedby="radix-_r_9_ radix-_r_b_ radix-_r_d_"
|
||||
class="_input_1qhtc_18"
|
||||
class="_input_1ug7n_18"
|
||||
id="radix-_r_a_"
|
||||
name="_r_0_"
|
||||
title=""
|
||||
@@ -287,7 +287,7 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
|
||||
value="compatibility"
|
||||
/>
|
||||
<div
|
||||
class="_ui_1qhtc_19"
|
||||
class="_ui_1ug7n_19"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
@@ -315,11 +315,11 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
|
||||
class="_inline-field-control_19upo_44"
|
||||
>
|
||||
<div
|
||||
class="_container_1qhtc_10"
|
||||
class="_container_1ug7n_10"
|
||||
>
|
||||
<input
|
||||
aria-describedby="radix-_r_9_ radix-_r_b_ radix-_r_d_"
|
||||
class="_input_1qhtc_18"
|
||||
class="_input_1ug7n_18"
|
||||
id="radix-_r_c_"
|
||||
name="_r_0_"
|
||||
title=""
|
||||
@@ -327,7 +327,7 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
|
||||
value="matrix_2_0"
|
||||
/>
|
||||
<div
|
||||
class="_ui_1qhtc_19"
|
||||
class="_ui_1ug7n_19"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -78,7 +78,6 @@ function renderWithMockClient(
|
||||
disconnected: false,
|
||||
supportedFeatures: {
|
||||
reactions: true,
|
||||
thumbnails: true,
|
||||
},
|
||||
setClient: vi.fn(),
|
||||
authenticated: {
|
||||
|
||||
563
src/state/AndroidControlledAudioOutput.test.ts
Normal file
563
src/state/AndroidControlledAudioOutput.test.ts
Normal file
@@ -0,0 +1,563 @@
|
||||
/*
|
||||
Copyright 2026 Element Corp.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { it, vi, expect, beforeEach, afterEach, describe } from "vitest";
|
||||
import { firstValueFrom, of, Subject, take, toArray } from "rxjs";
|
||||
import { type RTCCallIntent } from "matrix-js-sdk/lib/matrixrtc";
|
||||
|
||||
import { AndroidControlledAudioOutput } from "./AndroidControlledAudioOutput.ts";
|
||||
import type { Controls, OutputDevice } from "../controls";
|
||||
import { ObservableScope } from "./ObservableScope";
|
||||
import { withTestScheduler } from "../utils/test";
|
||||
|
||||
// All the following device types are real device types that have been observed in the wild on Android devices,
|
||||
// gathered from logs.
|
||||
// There are no BT Speakers because they are currently filtered out by EXA (native layer)
|
||||
|
||||
// A device type describing the speaker system (i.e. a mono speaker or stereo speakers) built in a device.
|
||||
const SPEAKER_DEVICE: OutputDevice = {
|
||||
id: "3",
|
||||
name: "Built-in speaker",
|
||||
isEarpiece: false,
|
||||
isSpeaker: true,
|
||||
isExternalHeadset: false,
|
||||
};
|
||||
|
||||
// A device type describing the attached earphone speaker.
|
||||
const EARPIECE_DEVICE: OutputDevice = {
|
||||
id: "2",
|
||||
name: "Built-in earpiece",
|
||||
isEarpiece: true,
|
||||
isSpeaker: false,
|
||||
isExternalHeadset: false,
|
||||
};
|
||||
|
||||
// A device type describing a Bluetooth device typically used for telephony
|
||||
const BT_HEADSET_DEVICE: OutputDevice = {
|
||||
id: "2226",
|
||||
name: "Bluetooth - OpenMove by Shokz",
|
||||
isEarpiece: false,
|
||||
isSpeaker: false,
|
||||
isExternalHeadset: true,
|
||||
};
|
||||
|
||||
// A device type describing a USB audio headset.
|
||||
const USB_HEADSET_DEVICE: OutputDevice = {
|
||||
id: "29440",
|
||||
name: "USB headset - USB-Audio - AB13X USB Audio",
|
||||
isEarpiece: false,
|
||||
isSpeaker: false,
|
||||
isExternalHeadset: false,
|
||||
};
|
||||
|
||||
// A device type describing a headset, which is the combination of a headphones and microphone
|
||||
const WIRED_HEADSET_DEVICE: OutputDevice = {
|
||||
id: "54509",
|
||||
name: "Wired headset - 23117RA68G",
|
||||
isEarpiece: false,
|
||||
isSpeaker: false,
|
||||
isExternalHeadset: false,
|
||||
};
|
||||
|
||||
// A device type describing a pair of wired headphones
|
||||
const WIRED_HEADPHONE_DEVICE: OutputDevice = {
|
||||
id: "679",
|
||||
name: "Wired headphones - TB02",
|
||||
isEarpiece: false,
|
||||
isSpeaker: false,
|
||||
isExternalHeadset: false,
|
||||
};
|
||||
|
||||
/**
|
||||
* The base device list that is always present on Android devices.
|
||||
* This list is ordered by the OS, the speaker is listed before the earpiece.
|
||||
*/
|
||||
const BASE_DEVICE_LIST = [SPEAKER_DEVICE, EARPIECE_DEVICE];
|
||||
|
||||
const BT_HEADSET_BASE_DEVICE_LIST = [BT_HEADSET_DEVICE, ...BASE_DEVICE_LIST];
|
||||
|
||||
const WIRED_HEADSET_BASE_DEVICE_LIST = [
|
||||
WIRED_HEADSET_DEVICE,
|
||||
...BASE_DEVICE_LIST,
|
||||
];
|
||||
|
||||
/**
|
||||
* A full device list containing all the observed device types in the wild on Android devices.
|
||||
* Ordered as they would be ordered by the OS.
|
||||
*/
|
||||
const FULL_DEVICE_LIST = [
|
||||
BT_HEADSET_DEVICE,
|
||||
USB_HEADSET_DEVICE,
|
||||
WIRED_HEADSET_DEVICE,
|
||||
WIRED_HEADPHONE_DEVICE,
|
||||
...BASE_DEVICE_LIST,
|
||||
];
|
||||
|
||||
let testScope: ObservableScope;
|
||||
let mockControls: Controls;
|
||||
|
||||
beforeEach(() => {
|
||||
testScope = new ObservableScope();
|
||||
mockControls = {
|
||||
onAudioDeviceSelect: vi.fn(),
|
||||
onOutputDeviceSelect: vi.fn(),
|
||||
} as unknown as Controls;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
testScope.end();
|
||||
});
|
||||
|
||||
describe("Default selection", () => {
|
||||
it("Default to speaker for video calls", async () => {
|
||||
const controlledAudioOutput = new AndroidControlledAudioOutput(
|
||||
of(BASE_DEVICE_LIST),
|
||||
testScope,
|
||||
"video",
|
||||
mockControls,
|
||||
);
|
||||
|
||||
const emissions = await firstValueFrom(
|
||||
controlledAudioOutput.selected$.pipe(take(1), toArray()),
|
||||
);
|
||||
|
||||
expect(emissions).toEqual([
|
||||
{ id: SPEAKER_DEVICE.id, virtualEarpiece: false },
|
||||
]);
|
||||
|
||||
[
|
||||
mockControls.onAudioDeviceSelect,
|
||||
mockControls.onOutputDeviceSelect,
|
||||
].forEach((mockFn) => {
|
||||
expect(mockFn).toHaveBeenCalledTimes(1);
|
||||
expect(mockFn).toHaveBeenCalledWith(SPEAKER_DEVICE.id);
|
||||
});
|
||||
});
|
||||
|
||||
it("Default to earpiece for audio calls for base config", async () => {
|
||||
const controlledAudioOutput = new AndroidControlledAudioOutput(
|
||||
of(BASE_DEVICE_LIST),
|
||||
testScope,
|
||||
"audio",
|
||||
mockControls,
|
||||
);
|
||||
|
||||
const emissions = await firstValueFrom(
|
||||
controlledAudioOutput.selected$.pipe(take(1), toArray()),
|
||||
);
|
||||
|
||||
expect(emissions).toEqual([
|
||||
{ id: EARPIECE_DEVICE.id, virtualEarpiece: false },
|
||||
]);
|
||||
|
||||
[
|
||||
mockControls.onAudioDeviceSelect,
|
||||
mockControls.onOutputDeviceSelect,
|
||||
].forEach((mockFn) => {
|
||||
expect(mockFn).toHaveBeenCalledTimes(1);
|
||||
expect(mockFn).toHaveBeenCalledWith(EARPIECE_DEVICE.id);
|
||||
});
|
||||
});
|
||||
|
||||
["audio", "video"].forEach((callIntent) => {
|
||||
it(`Default to BT headset for ${callIntent} calls if present`, async () => {
|
||||
const controlledAudioOutput = new AndroidControlledAudioOutput(
|
||||
of(BT_HEADSET_BASE_DEVICE_LIST),
|
||||
testScope,
|
||||
callIntent,
|
||||
mockControls,
|
||||
);
|
||||
|
||||
const emissions = await firstValueFrom(
|
||||
controlledAudioOutput.selected$.pipe(take(1), toArray()),
|
||||
);
|
||||
|
||||
expect(emissions).toEqual([
|
||||
{ id: BT_HEADSET_DEVICE.id, virtualEarpiece: false },
|
||||
]);
|
||||
|
||||
[
|
||||
mockControls.onAudioDeviceSelect,
|
||||
mockControls.onOutputDeviceSelect,
|
||||
].forEach((mockFn) => {
|
||||
expect(mockFn).toHaveBeenCalledTimes(1);
|
||||
expect(mockFn).toHaveBeenCalledWith(BT_HEADSET_DEVICE.id);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
["audio", "video"].forEach((callIntent) => {
|
||||
it(`Default to wired headset for ${callIntent} calls if present`, async () => {
|
||||
const controlledAudioOutput = new AndroidControlledAudioOutput(
|
||||
of(WIRED_HEADSET_BASE_DEVICE_LIST),
|
||||
testScope,
|
||||
callIntent,
|
||||
mockControls,
|
||||
);
|
||||
|
||||
const emissions = await firstValueFrom(
|
||||
controlledAudioOutput.selected$.pipe(take(1), toArray()),
|
||||
);
|
||||
|
||||
expect(emissions).toEqual([
|
||||
{ id: WIRED_HEADSET_DEVICE.id, virtualEarpiece: false },
|
||||
]);
|
||||
|
||||
expect(mockControls.onAudioDeviceSelect).toHaveBeenCalledExactlyOnceWith(
|
||||
WIRED_HEADSET_DEVICE.id,
|
||||
);
|
||||
expect(mockControls.onOutputDeviceSelect).toHaveBeenCalledExactlyOnceWith(
|
||||
WIRED_HEADSET_DEVICE.id,
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("Test mappings", () => {
|
||||
it("Should map output device to correct AudioDeviceLabel", async () => {
|
||||
const controlledAudioOutput = new AndroidControlledAudioOutput(
|
||||
of(FULL_DEVICE_LIST),
|
||||
testScope,
|
||||
undefined,
|
||||
mockControls,
|
||||
);
|
||||
|
||||
const availableDevices = await firstValueFrom(
|
||||
controlledAudioOutput.available$.pipe(take(1)),
|
||||
);
|
||||
|
||||
expect(availableDevices).toEqual(
|
||||
new Map([
|
||||
[BT_HEADSET_DEVICE.id, { type: "name", name: BT_HEADSET_DEVICE.name }],
|
||||
[
|
||||
USB_HEADSET_DEVICE.id,
|
||||
{ type: "name", name: USB_HEADSET_DEVICE.name },
|
||||
],
|
||||
[
|
||||
WIRED_HEADSET_DEVICE.id,
|
||||
{ type: "name", name: WIRED_HEADSET_DEVICE.name },
|
||||
],
|
||||
[
|
||||
WIRED_HEADPHONE_DEVICE.id,
|
||||
{ type: "name", name: WIRED_HEADPHONE_DEVICE.name },
|
||||
],
|
||||
[SPEAKER_DEVICE.id, { type: "speaker" }],
|
||||
[EARPIECE_DEVICE.id, { type: "earpiece" }],
|
||||
]),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("Test select a device", () => {
|
||||
it(`Switch to correct device `, () => {
|
||||
withTestScheduler(({ cold, schedule, expectObservable, flush }) => {
|
||||
const controlledAudioOutput = new AndroidControlledAudioOutput(
|
||||
cold("a", { a: FULL_DEVICE_LIST }),
|
||||
testScope,
|
||||
undefined,
|
||||
mockControls,
|
||||
);
|
||||
|
||||
schedule("-abc", {
|
||||
a: () => controlledAudioOutput.select(EARPIECE_DEVICE.id),
|
||||
b: () => controlledAudioOutput.select(USB_HEADSET_DEVICE.id),
|
||||
c: () => controlledAudioOutput.select(SPEAKER_DEVICE.id),
|
||||
});
|
||||
|
||||
expectObservable(controlledAudioOutput.selected$).toBe("abcd", {
|
||||
// virtualEarpiece is always false on android.
|
||||
// Initially the BT_HEADSET is selected.
|
||||
a: { id: BT_HEADSET_DEVICE.id, virtualEarpiece: false },
|
||||
b: { id: EARPIECE_DEVICE.id, virtualEarpiece: false },
|
||||
c: { id: USB_HEADSET_DEVICE.id, virtualEarpiece: false },
|
||||
d: { id: SPEAKER_DEVICE.id, virtualEarpiece: false },
|
||||
});
|
||||
|
||||
flush();
|
||||
|
||||
[
|
||||
mockControls.onOutputDeviceSelect,
|
||||
mockControls.onAudioDeviceSelect,
|
||||
].forEach((mockFn) => {
|
||||
expect(mockFn).toHaveBeenCalledTimes(4);
|
||||
expect(mockFn).toHaveBeenNthCalledWith(1, BT_HEADSET_DEVICE.id);
|
||||
expect(mockFn).toHaveBeenNthCalledWith(2, EARPIECE_DEVICE.id);
|
||||
expect(mockFn).toHaveBeenNthCalledWith(3, USB_HEADSET_DEVICE.id);
|
||||
expect(mockFn).toHaveBeenNthCalledWith(4, SPEAKER_DEVICE.id);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it(`manually switch then a bt headset is added`, () => {
|
||||
withTestScheduler(({ cold, schedule, expectObservable, flush }) => {
|
||||
const controlledAudioOutput = new AndroidControlledAudioOutput(
|
||||
cold("a--b", {
|
||||
a: BASE_DEVICE_LIST,
|
||||
b: BT_HEADSET_BASE_DEVICE_LIST,
|
||||
}),
|
||||
testScope,
|
||||
"audio",
|
||||
mockControls,
|
||||
);
|
||||
|
||||
// Default was earpiece (audio call), let's switch to speaker
|
||||
schedule("-a--", {
|
||||
a: () => controlledAudioOutput.select(SPEAKER_DEVICE.id),
|
||||
});
|
||||
|
||||
expectObservable(controlledAudioOutput.selected$).toBe("ab-c", {
|
||||
// virtualEarpiece is always false on android.
|
||||
// Initially the BT_HEADSET is selected.
|
||||
a: { id: EARPIECE_DEVICE.id, virtualEarpiece: false },
|
||||
b: { id: SPEAKER_DEVICE.id, virtualEarpiece: false },
|
||||
c: { id: BT_HEADSET_DEVICE.id, virtualEarpiece: false },
|
||||
});
|
||||
|
||||
flush();
|
||||
|
||||
[
|
||||
mockControls.onOutputDeviceSelect,
|
||||
mockControls.onAudioDeviceSelect,
|
||||
].forEach((mockFn) => {
|
||||
expect(mockFn).toHaveBeenCalledTimes(3);
|
||||
expect(mockFn).toHaveBeenNthCalledWith(1, EARPIECE_DEVICE.id);
|
||||
expect(mockFn).toHaveBeenNthCalledWith(2, SPEAKER_DEVICE.id);
|
||||
expect(mockFn).toHaveBeenNthCalledWith(3, BT_HEADSET_DEVICE.id);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it(`Go back to the previously selected after the auto-switch device goes away`, () => {
|
||||
withTestScheduler(({ cold, schedule, expectObservable, flush }) => {
|
||||
const controlledAudioOutput = new AndroidControlledAudioOutput(
|
||||
cold("a--b-c", {
|
||||
a: BASE_DEVICE_LIST,
|
||||
b: BT_HEADSET_BASE_DEVICE_LIST,
|
||||
c: BASE_DEVICE_LIST,
|
||||
}),
|
||||
testScope,
|
||||
"audio",
|
||||
mockControls,
|
||||
);
|
||||
|
||||
// Default was earpiece (audio call), let's switch to speaker
|
||||
schedule("-a---", {
|
||||
a: () => controlledAudioOutput.select(SPEAKER_DEVICE.id),
|
||||
});
|
||||
|
||||
expectObservable(controlledAudioOutput.selected$).toBe("ab-c-d", {
|
||||
// virtualEarpiece is always false on android.
|
||||
// Initially the BT_HEADSET is selected.
|
||||
a: { id: EARPIECE_DEVICE.id, virtualEarpiece: false },
|
||||
b: { id: SPEAKER_DEVICE.id, virtualEarpiece: false },
|
||||
c: { id: BT_HEADSET_DEVICE.id, virtualEarpiece: false },
|
||||
d: { id: SPEAKER_DEVICE.id, virtualEarpiece: false },
|
||||
});
|
||||
|
||||
flush();
|
||||
|
||||
[
|
||||
mockControls.onOutputDeviceSelect,
|
||||
mockControls.onAudioDeviceSelect,
|
||||
].forEach((mockFn) => {
|
||||
expect(mockFn).toHaveBeenCalledTimes(4);
|
||||
expect(mockFn).toHaveBeenNthCalledWith(1, EARPIECE_DEVICE.id);
|
||||
expect(mockFn).toHaveBeenNthCalledWith(2, SPEAKER_DEVICE.id);
|
||||
expect(mockFn).toHaveBeenNthCalledWith(3, BT_HEADSET_DEVICE.id);
|
||||
expect(mockFn).toHaveBeenNthCalledWith(4, SPEAKER_DEVICE.id);
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("Available device changes", () => {
|
||||
let availableSource$: Subject<OutputDevice[]>;
|
||||
|
||||
const createAudioControlledOutput = (
|
||||
intent: RTCCallIntent,
|
||||
): AndroidControlledAudioOutput => {
|
||||
return new AndroidControlledAudioOutput(
|
||||
availableSource$,
|
||||
testScope,
|
||||
intent,
|
||||
mockControls,
|
||||
);
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
availableSource$ = new Subject<OutputDevice[]>();
|
||||
});
|
||||
|
||||
it("When a BT headset is added, control should switch to use it", () => {
|
||||
createAudioControlledOutput("video");
|
||||
|
||||
// Emit the base device list, the speaker should be selected
|
||||
availableSource$.next(BASE_DEVICE_LIST);
|
||||
// Initially speaker would be selected
|
||||
[
|
||||
mockControls.onOutputDeviceSelect,
|
||||
mockControls.onAudioDeviceSelect,
|
||||
].forEach((mockFn) => {
|
||||
expect(mockFn).toHaveBeenCalledTimes(1);
|
||||
expect(mockFn).toHaveBeenCalledWith(SPEAKER_DEVICE.id);
|
||||
});
|
||||
|
||||
// Emit a new device list with a BT device, the control should switch to it
|
||||
availableSource$.next([BT_HEADSET_DEVICE, ...BASE_DEVICE_LIST]);
|
||||
[
|
||||
mockControls.onOutputDeviceSelect,
|
||||
mockControls.onAudioDeviceSelect,
|
||||
].forEach((mockFn) => {
|
||||
expect(mockFn).toHaveBeenCalledTimes(2);
|
||||
expect(mockFn).toHaveBeenLastCalledWith(BT_HEADSET_DEVICE.id);
|
||||
});
|
||||
});
|
||||
|
||||
// Android does not set `isExternalHeadset` to true for wired headphones, so we can't test this case.'
|
||||
it.skip("When a wired headset is added, control should switch to use it", async () => {
|
||||
const controlledAudioOutput = createAudioControlledOutput("video");
|
||||
|
||||
// Emit the base device list, the speaker should be selected
|
||||
availableSource$.next(BASE_DEVICE_LIST);
|
||||
|
||||
await firstValueFrom(controlledAudioOutput.selected$.pipe(take(1)));
|
||||
// Initially speaker would be selected
|
||||
[
|
||||
mockControls.onOutputDeviceSelect,
|
||||
mockControls.onAudioDeviceSelect,
|
||||
].forEach((mockFn) => {
|
||||
expect(mockFn).toHaveBeenCalledTimes(1);
|
||||
expect(mockFn).toHaveBeenCalledWith(SPEAKER_DEVICE.id);
|
||||
});
|
||||
|
||||
// Emit a new device list with a wired headset, the control should switch to it
|
||||
availableSource$.next([WIRED_HEADPHONE_DEVICE, ...BASE_DEVICE_LIST]);
|
||||
[
|
||||
mockControls.onOutputDeviceSelect,
|
||||
mockControls.onAudioDeviceSelect,
|
||||
].forEach((mockFn) => {
|
||||
expect(mockFn).toHaveBeenCalledTimes(2);
|
||||
expect(mockFn).toHaveBeenLastCalledWith(WIRED_HEADPHONE_DEVICE.id);
|
||||
});
|
||||
});
|
||||
|
||||
it("When the active bt headset is removed on audio call, control should switch to earpiece", () => {
|
||||
createAudioControlledOutput("audio");
|
||||
|
||||
// Emit the BT headset device list, the BT headset should be selected
|
||||
availableSource$.next(BT_HEADSET_BASE_DEVICE_LIST);
|
||||
// Initially speaker would be selected
|
||||
[
|
||||
mockControls.onOutputDeviceSelect,
|
||||
mockControls.onAudioDeviceSelect,
|
||||
].forEach((mockFn) => {
|
||||
expect(mockFn).toHaveBeenCalledTimes(1);
|
||||
expect(mockFn).toHaveBeenCalledWith(BT_HEADSET_DEVICE.id);
|
||||
});
|
||||
|
||||
// Emit a new device list without the BT headset, the control should switch to the earpiece for
|
||||
// audio calls
|
||||
availableSource$.next(BASE_DEVICE_LIST);
|
||||
[
|
||||
mockControls.onOutputDeviceSelect,
|
||||
mockControls.onAudioDeviceSelect,
|
||||
].forEach((mockFn) => {
|
||||
expect(mockFn).toHaveBeenCalledTimes(2);
|
||||
expect(mockFn).toHaveBeenLastCalledWith(EARPIECE_DEVICE.id);
|
||||
});
|
||||
});
|
||||
|
||||
it("When the active bt headset is removed on video call, control should switch to speaker", () => {
|
||||
createAudioControlledOutput("video");
|
||||
|
||||
availableSource$.next(BT_HEADSET_BASE_DEVICE_LIST);
|
||||
|
||||
// Initially bt headset would be selected
|
||||
[
|
||||
mockControls.onOutputDeviceSelect,
|
||||
mockControls.onAudioDeviceSelect,
|
||||
].forEach((mockFn) => {
|
||||
expect(mockFn).toHaveBeenCalledTimes(1);
|
||||
expect(mockFn).toHaveBeenCalledWith(BT_HEADSET_DEVICE.id);
|
||||
});
|
||||
|
||||
// Emit a new device list without the BT headset, the control should switch to speaker for video call
|
||||
availableSource$.next(BASE_DEVICE_LIST);
|
||||
[
|
||||
mockControls.onOutputDeviceSelect,
|
||||
mockControls.onAudioDeviceSelect,
|
||||
].forEach((mockFn) => {
|
||||
expect(mockFn).toHaveBeenCalledTimes(2);
|
||||
expect(mockFn).toHaveBeenLastCalledWith(SPEAKER_DEVICE.id);
|
||||
});
|
||||
});
|
||||
|
||||
it("Do not repeatidly set the same device", () => {
|
||||
createAudioControlledOutput("video");
|
||||
|
||||
availableSource$.next(BT_HEADSET_BASE_DEVICE_LIST);
|
||||
availableSource$.next(BT_HEADSET_BASE_DEVICE_LIST);
|
||||
availableSource$.next(BT_HEADSET_BASE_DEVICE_LIST);
|
||||
availableSource$.next(BT_HEADSET_BASE_DEVICE_LIST);
|
||||
availableSource$.next(BT_HEADSET_BASE_DEVICE_LIST);
|
||||
|
||||
// Initially bt headset would be selected
|
||||
[
|
||||
mockControls.onOutputDeviceSelect,
|
||||
mockControls.onAudioDeviceSelect,
|
||||
].forEach((mockFn) => {
|
||||
expect(mockFn).toHaveBeenCalledTimes(1);
|
||||
expect(mockFn).toHaveBeenCalledWith(BT_HEADSET_DEVICE.id);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("Scope management", () => {
|
||||
it("Should stop emitting when scope ends", () => {
|
||||
const aScope = new ObservableScope();
|
||||
const controlledAudioOutput = new AndroidControlledAudioOutput(
|
||||
of(BASE_DEVICE_LIST),
|
||||
aScope,
|
||||
undefined,
|
||||
mockControls,
|
||||
);
|
||||
|
||||
expect(mockControls.onAudioDeviceSelect).toHaveBeenCalledOnce();
|
||||
|
||||
aScope.end();
|
||||
|
||||
controlledAudioOutput.select(EARPIECE_DEVICE.id);
|
||||
|
||||
expect(mockControls.onAudioDeviceSelect).not.toHaveBeenCalledTimes(2);
|
||||
expect(mockControls.onAudioDeviceSelect).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("Should stop updating when scope ends", () => {
|
||||
const aScope = new ObservableScope();
|
||||
const availableSource$ = new Subject<OutputDevice[]>();
|
||||
new AndroidControlledAudioOutput(
|
||||
availableSource$,
|
||||
aScope,
|
||||
undefined,
|
||||
mockControls,
|
||||
);
|
||||
|
||||
availableSource$.next(BT_HEADSET_BASE_DEVICE_LIST);
|
||||
expect(mockControls.onAudioDeviceSelect).toHaveBeenCalledOnce();
|
||||
expect(mockControls.onAudioDeviceSelect).toHaveBeenCalledWith(
|
||||
BT_HEADSET_DEVICE.id,
|
||||
);
|
||||
|
||||
aScope.end();
|
||||
|
||||
availableSource$.next(BASE_DEVICE_LIST);
|
||||
|
||||
expect(mockControls.onAudioDeviceSelect).not.toHaveBeenCalledTimes(2);
|
||||
// Should have been called only once with the initial BT_HEADSET_DEVICE.id
|
||||
expect(mockControls.onAudioDeviceSelect).toHaveBeenCalledOnce();
|
||||
});
|
||||
});
|
||||
360
src/state/AndroidControlledAudioOutput.ts
Normal file
360
src/state/AndroidControlledAudioOutput.ts
Normal file
@@ -0,0 +1,360 @@
|
||||
/*
|
||||
Copyright 2026 Element Corp.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
|
||||
import {
|
||||
distinctUntilChanged,
|
||||
map,
|
||||
merge,
|
||||
type Observable,
|
||||
scan,
|
||||
startWith,
|
||||
Subject,
|
||||
tap,
|
||||
} from "rxjs";
|
||||
|
||||
import {
|
||||
type AudioOutputDeviceLabel,
|
||||
type MediaDevice,
|
||||
type SelectedAudioOutputDevice,
|
||||
} from "./MediaDevices.ts";
|
||||
import type { ObservableScope } from "./ObservableScope.ts";
|
||||
import type { RTCCallIntent } from "matrix-js-sdk/lib/matrixrtc";
|
||||
import { type Controls, type OutputDevice } from "../controls.ts";
|
||||
import { type Behavior } from "./Behavior.ts";
|
||||
|
||||
type ControllerState = {
|
||||
/**
|
||||
* The list of available output devices, ordered by preference order (most preferred first).
|
||||
*/
|
||||
devices: OutputDevice[];
|
||||
/**
|
||||
* Explicit user preference for the selected device.
|
||||
*/
|
||||
preferredDeviceId: string | undefined;
|
||||
/**
|
||||
* The effective selected device, always valid against available devices.
|
||||
*/
|
||||
selectedDeviceId: string | undefined;
|
||||
};
|
||||
|
||||
/**
|
||||
* The possible actions that can be performed on the controller,
|
||||
* either by the user or by the system.
|
||||
*/
|
||||
type ControllerAction =
|
||||
| { type: "selectDevice"; deviceId: string | undefined }
|
||||
| { type: "deviceUpdated"; devices: OutputDevice[] };
|
||||
/**
|
||||
* The implementation of the audio output media device for Android when using the controlled audio output mode.
|
||||
*
|
||||
* In this mode, the hosting application (e.g. Element Mobile) is responsible for providing the list of available audio output devices.
|
||||
* There are some android specific logic compared to others:
|
||||
* - AndroidControlledAudioOutput is the only one responsible for selecting the best output device.
|
||||
* - On android, we don't listen to the selected device from native code (control.setAudioDevice).
|
||||
* - If a new device is added or removed, this controller will determine the new selected device based
|
||||
* on the available devices (that is ordered by preference order) and the user's selection (if any).
|
||||
*
|
||||
* Given the differences in how the native code is handling the audio routing on Android compared to iOS,
|
||||
* we have this separate implementation. It allows us to have proper testing and avoid side effects
|
||||
* from platform specific logic breaking the other platform's implementation.
|
||||
*/
|
||||
export class AndroidControlledAudioOutput implements MediaDevice<
|
||||
AudioOutputDeviceLabel,
|
||||
SelectedAudioOutputDevice
|
||||
> {
|
||||
private logger = rootLogger.getChild(
|
||||
"[MediaDevices AndroidControlledAudioOutput]",
|
||||
);
|
||||
|
||||
/**
|
||||
* STATE stream: the current state of the controller, including the list of available devices and the selected device.
|
||||
*/
|
||||
private readonly controllerState$: Behavior<ControllerState>;
|
||||
|
||||
/**
|
||||
* @inheritdoc
|
||||
*/
|
||||
public readonly available$: Behavior<Map<string, AudioOutputDeviceLabel>>;
|
||||
|
||||
/**
|
||||
* Effective selected device, always valid against available devices.
|
||||
*
|
||||
* On android, we don't listen to the selected device from native code (control.setAudioDevice).
|
||||
* Instead, we determine the selected device ourselves based on the available devices and the user's selection (if any).
|
||||
*/
|
||||
public readonly selected$: Behavior<SelectedAudioOutputDevice | undefined>;
|
||||
|
||||
// COMMAND stream: user asks to select a device
|
||||
private readonly selectDeviceCommand$ = new Subject<string | undefined>();
|
||||
|
||||
public select(id: string): void {
|
||||
this.logger.info(`select device: ${id}`);
|
||||
this.selectDeviceCommand$.next(id);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates an instance of AndroidControlledAudioOutput.
|
||||
*
|
||||
* @constructor
|
||||
* @param controlledDevices$ - The list of available output devices coming from the hosting application, ordered by preference order (most preferred first).
|
||||
* @param scope - The ObservableScope to create the Behaviors in.
|
||||
* @param initialIntent - The initial call intent (e.g. "audio" or "video") that can be used to determine the default audio routing (e.g. default to earpiece for audio calls and speaker for video calls).
|
||||
* @param controls - The controls provided by the hosting application to control the audio routing and notify of user actions.
|
||||
*/
|
||||
public constructor(
|
||||
private readonly controlledDevices$: Observable<OutputDevice[]>,
|
||||
private readonly scope: ObservableScope,
|
||||
private initialIntent: RTCCallIntent | undefined = undefined,
|
||||
controls: Controls,
|
||||
) {
|
||||
this.controllerState$ = this.startObservingState$();
|
||||
|
||||
this.selected$ = this.effectiveSelectionFromState$(this.controllerState$);
|
||||
|
||||
this.available$ = scope.behavior(
|
||||
this.controllerState$.pipe(
|
||||
map((state) => {
|
||||
this.logger.info("available devices updated:", state.devices);
|
||||
|
||||
return new Map<string, AudioOutputDeviceLabel>(
|
||||
state.devices.map((outputDevice) => {
|
||||
return [outputDevice.id, mapDeviceToLabel(outputDevice)];
|
||||
}),
|
||||
);
|
||||
}),
|
||||
),
|
||||
);
|
||||
|
||||
// Effect 1: notify host when effective selection changes
|
||||
this.selected$
|
||||
// It is a behavior so it has built-in distinct until change
|
||||
.pipe(scope.bind())
|
||||
.subscribe((device) => {
|
||||
// Let the hosting application know which output device has been selected.
|
||||
if (device !== undefined) {
|
||||
this.logger.info("onAudioDeviceSelect called:", device);
|
||||
controls.onAudioDeviceSelect?.(device.id);
|
||||
// Also invoke the deprecated callback for backward compatibility
|
||||
// TODO: it appears that on Android the hosting application is only using the deprecated callback (onOutputDeviceSelect)
|
||||
// and not the new one (onAudioDeviceSelect), we should clean this up and only have one callback for audio device selection.
|
||||
controls.onOutputDeviceSelect?.(device.id);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private startObservingState$(): Behavior<ControllerState> {
|
||||
const initialState: ControllerState = {
|
||||
devices: [],
|
||||
preferredDeviceId: undefined,
|
||||
selectedDeviceId: undefined,
|
||||
};
|
||||
|
||||
// Merge the two possible inputs observable as a single
|
||||
// stream of actions that will update the state of the controller.
|
||||
const actions$: Observable<ControllerAction> = merge(
|
||||
this.controlledDevices$.pipe(
|
||||
map(
|
||||
(devices) =>
|
||||
({ type: "deviceUpdated", devices }) satisfies ControllerAction,
|
||||
),
|
||||
),
|
||||
this.selectDeviceCommand$.pipe(
|
||||
map(
|
||||
(deviceId) =>
|
||||
({ type: "selectDevice", deviceId }) satisfies ControllerAction,
|
||||
),
|
||||
),
|
||||
);
|
||||
|
||||
const initialAction: ControllerAction = {
|
||||
type: "deviceUpdated",
|
||||
devices: [],
|
||||
};
|
||||
|
||||
return this.scope.behavior(
|
||||
actions$.pipe(
|
||||
startWith(initialAction),
|
||||
scan((state, action): ControllerState => {
|
||||
switch (action.type) {
|
||||
case "deviceUpdated": {
|
||||
const chosenDevice = this.chooseEffectiveSelection({
|
||||
previousDevices: state.devices,
|
||||
availableDevices: action.devices,
|
||||
currentSelectedId: state.selectedDeviceId,
|
||||
preferredDeviceId: state.preferredDeviceId,
|
||||
});
|
||||
|
||||
return {
|
||||
...state,
|
||||
devices: action.devices,
|
||||
selectedDeviceId: chosenDevice,
|
||||
};
|
||||
}
|
||||
case "selectDevice": {
|
||||
const chosenDevice = this.chooseEffectiveSelection({
|
||||
previousDevices: state.devices,
|
||||
availableDevices: state.devices,
|
||||
currentSelectedId: state.selectedDeviceId,
|
||||
preferredDeviceId: action.deviceId,
|
||||
});
|
||||
|
||||
return {
|
||||
...state,
|
||||
preferredDeviceId: action.deviceId,
|
||||
selectedDeviceId: chosenDevice,
|
||||
};
|
||||
}
|
||||
}
|
||||
}, initialState),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
private effectiveSelectionFromState$(
|
||||
state$: Observable<ControllerState>,
|
||||
): Behavior<SelectedAudioOutputDevice | undefined> {
|
||||
return this.scope.behavior(
|
||||
state$
|
||||
.pipe(
|
||||
map((state) => {
|
||||
if (state.selectedDeviceId) {
|
||||
return {
|
||||
id: state.selectedDeviceId,
|
||||
/** This is an iOS thing, always false for android*/
|
||||
virtualEarpiece: false,
|
||||
};
|
||||
}
|
||||
return undefined;
|
||||
}),
|
||||
distinctUntilChanged((a, b) => a?.id === b?.id),
|
||||
)
|
||||
.pipe(
|
||||
tap((selected) => {
|
||||
this.logger.debug(`selected device: ${selected?.id}`);
|
||||
}),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
private chooseEffectiveSelection(args: {
|
||||
previousDevices: OutputDevice[];
|
||||
availableDevices: OutputDevice[];
|
||||
currentSelectedId: string | undefined;
|
||||
preferredDeviceId: string | undefined;
|
||||
}): string | undefined {
|
||||
const {
|
||||
previousDevices,
|
||||
availableDevices,
|
||||
currentSelectedId,
|
||||
preferredDeviceId,
|
||||
} = args;
|
||||
|
||||
this.logger.debug(`chooseEffectiveSelection with args:`, args);
|
||||
|
||||
// Take preferredDeviceId in priority or default to the last effective selection.
|
||||
const activeSelectedDeviceId = preferredDeviceId || currentSelectedId;
|
||||
const isAvailable = availableDevices.some(
|
||||
(device) => device.id === activeSelectedDeviceId,
|
||||
);
|
||||
|
||||
// If there is no current device, or it is not available anymore,
|
||||
// choose the default device selection logic.
|
||||
if (activeSelectedDeviceId === undefined || !isAvailable) {
|
||||
this.logger.debug(
|
||||
`No current device or it is not available, using default selection logic.`,
|
||||
);
|
||||
// use the default selection logic
|
||||
return this.chooseDefaultDeviceId(availableDevices);
|
||||
}
|
||||
|
||||
// Is there a new added device?
|
||||
// If a device is added, we might want to switch to it if it's more preferred than the currently selected device.
|
||||
const newDeviceWasAdded = availableDevices.some(
|
||||
(device) => !previousDevices.some((d) => d.id === device.id),
|
||||
);
|
||||
|
||||
if (newDeviceWasAdded) {
|
||||
// TODO only want to check from the added device, not all devices.?
|
||||
// check if the currently selected device is the most preferred one, if not switch to the most preferred one.
|
||||
const mostPreferredDevice = availableDevices[0];
|
||||
this.logger.debug(
|
||||
`A new device was added, checking if we should switch to it.`,
|
||||
mostPreferredDevice,
|
||||
);
|
||||
if (mostPreferredDevice.id !== activeSelectedDeviceId) {
|
||||
// Given this is automatic switching, we want to be careful and only switch to a more private device
|
||||
// (e.g. from speaker to a BT headset) but not switch from a more private device to a less private one
|
||||
// (e.g. from a BT headset to the speaker), as that can be disruptive for the user if it happens unexpectedly.
|
||||
if (mostPreferredDevice.isExternalHeadset == true) {
|
||||
this.logger.info(
|
||||
`The currently selected device ${mostPreferredDevice.id} is not the most preferred one, switching to the most preferred one ${activeSelectedDeviceId} instead.`,
|
||||
);
|
||||
// Let's switch as it is a more private device.
|
||||
return mostPreferredDevice.id;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// no changes
|
||||
return activeSelectedDeviceId;
|
||||
}
|
||||
|
||||
/**
|
||||
* The logic for the default is different based on the call type.
|
||||
* For example for a voice call we want to default to the earpiece if it's available,
|
||||
* but for a video call we want to default to the speaker.
|
||||
* If the user is using a BT headset we want to default to that, as it's likely what they want to use for both video and voice calls.
|
||||
*
|
||||
* @param available the available audio output devices to choose from, keyed by their id, sorted by likelihood of it being used for communication.
|
||||
*
|
||||
*/
|
||||
private chooseDefaultDeviceId(available: OutputDevice[]): string | undefined {
|
||||
this.logger.debug(
|
||||
`Android routing logic intent: ${this.initialIntent} finding best default...`,
|
||||
);
|
||||
if (this.initialIntent === "audio") {
|
||||
const systemProposed = available[0];
|
||||
// If no headset is connected, android will route to the speaker by default,
|
||||
// but for a voice call we want to route to the earpiece instead,
|
||||
// so override the system proposed routing in that case.
|
||||
if (systemProposed?.isSpeaker == true) {
|
||||
// search for the earpiece
|
||||
const earpieceDevice = available.find(
|
||||
(device) => device.isEarpiece == true,
|
||||
);
|
||||
if (earpieceDevice) {
|
||||
this.logger.debug(
|
||||
`Android routing: Switch to earpiece instead of speaker for voice call`,
|
||||
);
|
||||
return earpieceDevice.id;
|
||||
} else {
|
||||
this.logger.debug(
|
||||
`Android routing: no earpiece found, cannot switch, use system proposed routing`,
|
||||
);
|
||||
return systemProposed.id;
|
||||
}
|
||||
} else {
|
||||
this.logger.debug(
|
||||
`Android routing: Use system proposed routing `,
|
||||
systemProposed,
|
||||
);
|
||||
return systemProposed?.id;
|
||||
}
|
||||
} else {
|
||||
// Use the system best proposed best routing.
|
||||
return available[0]?.id;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Utilities
|
||||
function mapDeviceToLabel(device: OutputDevice): AudioOutputDeviceLabel {
|
||||
const { name, isEarpiece, isSpeaker } = device;
|
||||
if (isEarpiece) return { type: "earpiece" };
|
||||
else if (isSpeaker) return { type: "speaker" };
|
||||
else return { type: "name", name };
|
||||
}
|
||||
193
src/state/AudioOutput.test.ts
Normal file
193
src/state/AudioOutput.test.ts
Normal file
@@ -0,0 +1,193 @@
|
||||
/*
|
||||
Copyright 2026 Element Corp.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { afterEach, beforeEach, describe, vi, it } from "vitest";
|
||||
import * as ComponentsCore from "@livekit/components-core";
|
||||
|
||||
import { ObservableScope } from "./ObservableScope";
|
||||
import { AudioOutput } from "./MediaDevices";
|
||||
import { withTestScheduler } from "../utils/test";
|
||||
|
||||
const BT_SPEAKER = {
|
||||
deviceId: "f9fc8f5f94578fe3abd89e086c1e78c08477aa564dd9e917950f0e7ebb37a6a2",
|
||||
kind: "audiooutput",
|
||||
label: "JBL (Bluetooth)",
|
||||
groupId: "309a5c086cd8eb885a164046db6ec834c349be01d86448d02c1a5279456ff9e4",
|
||||
} as unknown as MediaDeviceInfo;
|
||||
|
||||
const BUILT_IN_SPEAKER = {
|
||||
deviceId: "acdbb8546ea6fa85ba2d861e9bcc0e71810d03bbaf6d1712c69e8d9c0c6c2e0a",
|
||||
kind: "audiooutput",
|
||||
label: "MacBook Speakers (Built-in)",
|
||||
groupId: "08a5a3a486473aaa898eb81cda3113f3e21053fb8b84155f4e612fe3f8db5d17",
|
||||
} as unknown as MediaDeviceInfo;
|
||||
|
||||
const BT_HEADSET = {
|
||||
deviceId: "ff8e6edb4ebb512b2b421335bfd14994a5b4c7192b3e84a8696863d83cf46d12",
|
||||
kind: "audiooutput",
|
||||
label: "OpenMove (Bluetooth)",
|
||||
groupId: "c2893c2438c44248368e0533300245c402764991506f42cd73818dc8c3ee9c88",
|
||||
} as unknown as MediaDeviceInfo;
|
||||
|
||||
const AMAC_DEVICE_LIST = [BT_SPEAKER, BUILT_IN_SPEAKER];
|
||||
|
||||
const AMAC_DEVICE_LIST_WITH_DEFAULT = [
|
||||
asDefault(BUILT_IN_SPEAKER),
|
||||
...AMAC_DEVICE_LIST,
|
||||
];
|
||||
|
||||
const AMAC_HS_DEVICE_LIST = [
|
||||
asDefault(BT_HEADSET),
|
||||
BT_SPEAKER,
|
||||
BT_HEADSET,
|
||||
BUILT_IN_SPEAKER,
|
||||
];
|
||||
|
||||
const LAPTOP_SPEAKER = {
|
||||
deviceId: "EcUxTMu8He2wz+3Y8m/u0fy6M92pUk=",
|
||||
kind: "audiooutput",
|
||||
label: "Raptor AVS Speaker",
|
||||
groupId: "kSrdanhpEDLg3vN8z6Z9MJ1EdanB8zI+Q1dxA=",
|
||||
} as unknown as MediaDeviceInfo;
|
||||
|
||||
const MONITOR_SPEAKER = {
|
||||
deviceId: "gBryZdAdC8I/rrJpr9r6R+rZzKkoIK5cpU=",
|
||||
kind: "audiooutput",
|
||||
label: "Raptor AVS HDMI / DisplayPort 1 Output",
|
||||
groupId: "kSrdanhpEDLg3vN8z6Z9MJ1EdanB8zI+Q1dxA=",
|
||||
} as unknown as MediaDeviceInfo;
|
||||
|
||||
const DEVICE_LIST_B = [LAPTOP_SPEAKER, MONITOR_SPEAKER];
|
||||
|
||||
// On chrome, there is an additional synthetic device called "Default - <device name>",
|
||||
// it represents what the OS default is now.
|
||||
function asDefault(device: MediaDeviceInfo): MediaDeviceInfo {
|
||||
return {
|
||||
...device,
|
||||
deviceId: "default",
|
||||
label: `Default - ${device.label}`,
|
||||
};
|
||||
}
|
||||
// When the authorization is not yet granted, every device is still listed
|
||||
// but only with empty/blank labels and ids.
|
||||
// This is a transition state.
|
||||
function toBlankDevice(device: MediaDeviceInfo): MediaDeviceInfo {
|
||||
return {
|
||||
...device,
|
||||
deviceId: "",
|
||||
label: "",
|
||||
groupId: "",
|
||||
};
|
||||
}
|
||||
|
||||
vi.mock("@livekit/components-core", () => ({
|
||||
createMediaDeviceObserver: vi.fn(),
|
||||
}));
|
||||
|
||||
describe("AudioOutput Tests", () => {
|
||||
let testScope: ObservableScope;
|
||||
|
||||
beforeEach(() => {
|
||||
testScope = new ObservableScope();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
testScope.end();
|
||||
});
|
||||
|
||||
it("should select the default audio output device", () => {
|
||||
// In a real life setup there would be first a blanked list
|
||||
// then the real one.
|
||||
withTestScheduler(({ behavior, cold, expectObservable }) => {
|
||||
vi.mocked(ComponentsCore.createMediaDeviceObserver).mockReturnValue(
|
||||
cold("ab", {
|
||||
// In a real life setup there would be first a blanked list
|
||||
// then the real one.
|
||||
a: AMAC_DEVICE_LIST_WITH_DEFAULT.map(toBlankDevice),
|
||||
b: AMAC_DEVICE_LIST_WITH_DEFAULT,
|
||||
}),
|
||||
);
|
||||
|
||||
const audioOutput = new AudioOutput(
|
||||
behavior("a", { a: true }),
|
||||
testScope,
|
||||
);
|
||||
|
||||
expectObservable(audioOutput.selected$).toBe("ab", {
|
||||
a: undefined,
|
||||
b: { id: "default", virtualEarpiece: false },
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it("Select the correct device when requested", () => {
|
||||
// In a real life setup there would be first a blanked list
|
||||
// then the real one.
|
||||
withTestScheduler(({ behavior, cold, schedule, expectObservable }) => {
|
||||
vi.mocked(ComponentsCore.createMediaDeviceObserver).mockReturnValue(
|
||||
cold("ab", {
|
||||
// In a real life setup there would be first a blanked list
|
||||
// then the real one.
|
||||
a: DEVICE_LIST_B.map(toBlankDevice),
|
||||
b: DEVICE_LIST_B,
|
||||
}),
|
||||
);
|
||||
|
||||
const audioOutput = new AudioOutput(
|
||||
behavior("a", { a: true }),
|
||||
testScope,
|
||||
);
|
||||
|
||||
schedule("--abc", {
|
||||
a: () => audioOutput.select(MONITOR_SPEAKER.deviceId),
|
||||
b: () => audioOutput.select(LAPTOP_SPEAKER.deviceId),
|
||||
c: () => audioOutput.select(MONITOR_SPEAKER.deviceId),
|
||||
});
|
||||
|
||||
expectObservable(audioOutput.selected$).toBe("abcde", {
|
||||
a: undefined,
|
||||
b: { id: LAPTOP_SPEAKER.deviceId, virtualEarpiece: false },
|
||||
c: { id: MONITOR_SPEAKER.deviceId, virtualEarpiece: false },
|
||||
d: { id: LAPTOP_SPEAKER.deviceId, virtualEarpiece: false },
|
||||
e: { id: MONITOR_SPEAKER.deviceId, virtualEarpiece: false },
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it("Test mappings", () => {
|
||||
// In a real life setup there would be first a blanked list
|
||||
// then the real one.
|
||||
withTestScheduler(({ behavior, cold, schedule, expectObservable }) => {
|
||||
vi.mocked(ComponentsCore.createMediaDeviceObserver).mockReturnValue(
|
||||
cold("a", {
|
||||
// In a real life setup there would be first a blanked list
|
||||
// then the real one.
|
||||
a: AMAC_HS_DEVICE_LIST,
|
||||
}),
|
||||
);
|
||||
|
||||
const audioOutput = new AudioOutput(
|
||||
behavior("a", { a: true }),
|
||||
testScope,
|
||||
);
|
||||
|
||||
const expectedMappings = new Map([
|
||||
[`default`, { type: "name", name: asDefault(BT_HEADSET).label }],
|
||||
[BT_SPEAKER.deviceId, { type: "name", name: BT_SPEAKER.label }],
|
||||
[BT_HEADSET.deviceId, { type: "name", name: BT_HEADSET.label }],
|
||||
[
|
||||
BUILT_IN_SPEAKER.deviceId,
|
||||
{ type: "name", name: BUILT_IN_SPEAKER.label },
|
||||
],
|
||||
]);
|
||||
|
||||
expectObservable(audioOutput.available$).toBe("a", {
|
||||
a: expectedMappings,
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -89,7 +89,6 @@ export interface Props {
|
||||
* `callPickupState$` The current call pickup state of the call.
|
||||
* - "unknown": The client has not yet sent the notification event. We don't know if it will because it first needs to send its own membership.
|
||||
* Then we can conclude if we were the first one to join or not.
|
||||
* This may also be set if we are disconnected.
|
||||
* - "ringing": The call is ringing on other devices in this room (This client should give audiovisual feedback that this is happening).
|
||||
* - "timeout": No-one picked up in the defined time this call should be ringing on others devices.
|
||||
* The call failed. If desired this can be used as a trigger to exit the call.
|
||||
@@ -131,15 +130,9 @@ export function createCallNotificationLifecycle$({
|
||||
) as Behavior<Epoch<boolean>>;
|
||||
|
||||
/**
|
||||
* Whenever the RTC session tells us that it intends to ring the remote
|
||||
* participant's devices, this emits an Observable tracking the current state of
|
||||
* that ringing process.
|
||||
* The state of the current ringing attempt, if the RTC session is indeed
|
||||
* ringing the remote participant's devices. Otherwise `null`.
|
||||
*/
|
||||
// This is a behavior since we need to store the latest state for when we subscribe to this after `didSendCallNotification$`
|
||||
// has already emitted but we still need the latest observable with a timeout timer that only gets created on after receiving `notificationEvent`.
|
||||
// A behavior will emit the latest observable with the running timer to new subscribers.
|
||||
// see also: callPickupState$ and in particular the line: `return this.ring$.pipe(mergeAll());` here we otherwise might get an EMPTY observable if
|
||||
// `ring$` would not be a behavior.
|
||||
const remoteRingState$: Behavior<"ringing" | "timeout" | "decline" | null> =
|
||||
scope.behavior(
|
||||
sentCallNotification$.pipe(
|
||||
|
||||
@@ -46,9 +46,11 @@ import {
|
||||
} from "../../utils/test.ts";
|
||||
import { E2eeType } from "../../e2ee/e2eeType.ts";
|
||||
import {
|
||||
alice,
|
||||
aliceId,
|
||||
aliceParticipant,
|
||||
aliceRtcMember,
|
||||
aliceUserId,
|
||||
bobId,
|
||||
bobRtcMember,
|
||||
local,
|
||||
@@ -140,8 +142,8 @@ export interface SpotlightExpandedLayoutSummary {
|
||||
|
||||
export interface OneOnOneLayoutSummary {
|
||||
type: "one-on-one";
|
||||
local: string;
|
||||
remote: string;
|
||||
spotlight: string;
|
||||
pip: string;
|
||||
}
|
||||
|
||||
export interface PipLayoutSummary {
|
||||
@@ -194,11 +196,11 @@ function summarizeLayout$(l$: Observable<Layout>): Observable<LayoutSummary> {
|
||||
);
|
||||
case "one-on-one":
|
||||
return combineLatest(
|
||||
[l.local.media$, l.remote.media$],
|
||||
(local, remote) => ({
|
||||
[l.spotlight.media$, l.pip.media$],
|
||||
(spotlight, pip) => ({
|
||||
type: l.type,
|
||||
local: local.id,
|
||||
remote: remote.id,
|
||||
spotlight: spotlight.id,
|
||||
pip: pip.id,
|
||||
}),
|
||||
);
|
||||
case "pip":
|
||||
@@ -537,8 +539,8 @@ describe.each([
|
||||
b: {
|
||||
// In a larger window, expect the normal one-on-one layout
|
||||
type: "one-on-one",
|
||||
local: `${localId}:0`,
|
||||
remote: `${aliceId}:0`,
|
||||
pip: `${localId}:0`,
|
||||
spotlight: `${aliceId}:0`,
|
||||
},
|
||||
c: {
|
||||
// In a PiP-sized window, we of course expect a PiP layout
|
||||
@@ -748,6 +750,53 @@ describe.each([
|
||||
});
|
||||
});
|
||||
|
||||
test("PiP tile in expanded spotlight layout avoids redundantly showing local user", () => {
|
||||
withTestScheduler(({ behavior, schedule, expectObservable }) => {
|
||||
// Switch to spotlight immediately
|
||||
const modeInputMarbles = " s";
|
||||
// And expand the spotlight immediately
|
||||
const expandInputMarbles = " a";
|
||||
// First no one else is in the call, then Alice joins
|
||||
const participantInputMarbles = "ab";
|
||||
// First local user should be in the spotlight, then they appear in PiP
|
||||
// only once Alice has joined
|
||||
const expectedLayoutMarbles = " ab";
|
||||
|
||||
withCallViewModel(
|
||||
{
|
||||
rtcMembers$: behavior(participantInputMarbles, {
|
||||
a: [localRtcMember],
|
||||
b: [localRtcMember, aliceRtcMember],
|
||||
}),
|
||||
},
|
||||
(vm) => {
|
||||
schedule(modeInputMarbles, {
|
||||
s: () => vm.setGridMode("spotlight"),
|
||||
});
|
||||
schedule(expandInputMarbles, {
|
||||
a: () => vm.toggleSpotlightExpanded$.value!(),
|
||||
});
|
||||
|
||||
expectObservable(summarizeLayout$(vm.layout$)).toBe(
|
||||
expectedLayoutMarbles,
|
||||
{
|
||||
a: {
|
||||
type: "spotlight-expanded",
|
||||
spotlight: [`${localId}:0`],
|
||||
pip: undefined,
|
||||
},
|
||||
b: {
|
||||
type: "spotlight-expanded",
|
||||
spotlight: [`${aliceId}:0`],
|
||||
pip: `${localId}:0`,
|
||||
},
|
||||
},
|
||||
);
|
||||
},
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
test("spotlight remembers whether it's expanded", () => {
|
||||
withTestScheduler(({ schedule, expectObservable }) => {
|
||||
// Start in spotlight mode, then switch to grid and back to spotlight a
|
||||
@@ -840,8 +889,8 @@ describe.each([
|
||||
},
|
||||
b: {
|
||||
type: "one-on-one",
|
||||
local: `${localId}:0`,
|
||||
remote: `${aliceId}:0`,
|
||||
pip: `${localId}:0`,
|
||||
spotlight: `${aliceId}:0`,
|
||||
},
|
||||
c: {
|
||||
type: "grid",
|
||||
@@ -883,8 +932,8 @@ describe.each([
|
||||
},
|
||||
b: {
|
||||
type: "one-on-one",
|
||||
local: `${localId}:0`,
|
||||
remote: `${aliceId}:0`,
|
||||
pip: `${localId}:0`,
|
||||
spotlight: `${aliceId}:0`,
|
||||
},
|
||||
c: {
|
||||
type: "grid",
|
||||
@@ -893,8 +942,8 @@ describe.each([
|
||||
},
|
||||
d: {
|
||||
type: "one-on-one",
|
||||
local: `${localId}:0`,
|
||||
remote: `${daveId}:0`,
|
||||
pip: `${localId}:0`,
|
||||
spotlight: `${daveId}:0`,
|
||||
},
|
||||
},
|
||||
);
|
||||
@@ -1087,83 +1136,81 @@ describe.each([
|
||||
});
|
||||
});
|
||||
|
||||
describe("waitForCallPickup$", () => {
|
||||
it.skip("regression test: does stop ringing in case livekitConnectionState$ emits after didSendCallNotification$ has already emitted", () => {
|
||||
withTestScheduler(({ schedule, expectObservable, behavior }) => {
|
||||
withCallViewModel(
|
||||
{
|
||||
livekitConnectionState$: behavior("d 9ms c", {
|
||||
d: ConnectionState.Disconnected,
|
||||
c: ConnectionState.Connected,
|
||||
}),
|
||||
},
|
||||
(vm, rtcSession) => {
|
||||
// Fire a call notification IMMEDIATELY (its important for this test, that this happens before the livekitConnectionState$ emits)
|
||||
schedule("n", {
|
||||
n: () => {
|
||||
rtcSession.emit(
|
||||
MatrixRTCSessionEvent.DidSendCallNotification,
|
||||
mockRingEvent("$notif1", 30),
|
||||
);
|
||||
},
|
||||
});
|
||||
test("recipient has placeholder tile while ringing or timed out", () => {
|
||||
withTestScheduler(({ schedule, expectObservable }) => {
|
||||
withCallViewModel(
|
||||
{
|
||||
roomMembers: [alice, local], // Simulate a DM
|
||||
},
|
||||
(vm, rtcSession) => {
|
||||
// Fire a ringing notification
|
||||
schedule("n", {
|
||||
n: () => {
|
||||
rtcSession.emit(
|
||||
MatrixRTCSessionEvent.DidSendCallNotification,
|
||||
mockRingEvent("$notif1", 30),
|
||||
);
|
||||
},
|
||||
});
|
||||
|
||||
expectObservable(vm.callPickupState$).toBe("a 9ms b 19ms c", {
|
||||
a: "unknown",
|
||||
b: "ringing",
|
||||
c: "timeout",
|
||||
});
|
||||
},
|
||||
{
|
||||
waitForCallPickup: true,
|
||||
encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
|
||||
},
|
||||
);
|
||||
});
|
||||
// Should ring for 30ms and then time out
|
||||
expectObservable(vm.ringing$).toBe("(ny) 26ms n", yesNo);
|
||||
// Layout should show placeholder media for the participant we're
|
||||
// ringing the entire time (even once timed out)
|
||||
expectObservable(summarizeLayout$(vm.layout$)).toBe("a", {
|
||||
a: {
|
||||
type: "one-on-one",
|
||||
spotlight: `${localId}:0`,
|
||||
pip: `ringing:${aliceUserId}`,
|
||||
},
|
||||
});
|
||||
},
|
||||
{ waitForCallPickup: true },
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
it.skip("ringing -> unknown if we get disconnected", () => {
|
||||
withTestScheduler(({ behavior, schedule, expectObservable }) => {
|
||||
const connectionState$ = new BehaviorSubject(ConnectionState.Connected);
|
||||
// Someone joins at 20ms (both LiveKit participant and MatrixRTC member)
|
||||
withCallViewModel(
|
||||
{
|
||||
remoteParticipants$: behavior("a 19ms b", {
|
||||
a: [],
|
||||
b: [aliceParticipant],
|
||||
}),
|
||||
rtcMembers$: behavior("a 19ms b", {
|
||||
a: [localRtcMember],
|
||||
b: [localRtcMember, aliceRtcMember],
|
||||
}),
|
||||
livekitConnectionState$: connectionState$,
|
||||
},
|
||||
(vm, rtcSession) => {
|
||||
// Notify at 5ms so we enter ringing, then get disconnected 5ms later
|
||||
schedule(" 5ms r 5ms d", {
|
||||
r: () => {
|
||||
rtcSession.emit(
|
||||
MatrixRTCSessionEvent.DidSendCallNotification,
|
||||
mockRingEvent("$notif2", 100),
|
||||
);
|
||||
},
|
||||
d: () => {
|
||||
connectionState$.next(ConnectionState.Disconnected);
|
||||
},
|
||||
});
|
||||
test("recipient's placeholder tile is replaced by their real tile once they answer", () => {
|
||||
withTestScheduler(({ behavior, schedule, expectObservable }) => {
|
||||
withCallViewModel(
|
||||
{
|
||||
// Alice answers after 20ms
|
||||
rtcMembers$: behavior("a 20ms b", {
|
||||
a: [localRtcMember],
|
||||
b: [localRtcMember, aliceRtcMember],
|
||||
}),
|
||||
roomMembers: [alice, local], // Simulate a DM
|
||||
},
|
||||
(vm, rtcSession) => {
|
||||
// Fire a ringing notification
|
||||
schedule("n", {
|
||||
n: () => {
|
||||
rtcSession.emit(
|
||||
MatrixRTCSessionEvent.DidSendCallNotification,
|
||||
mockRingEvent("$notif1", 30),
|
||||
);
|
||||
},
|
||||
});
|
||||
|
||||
expectObservable(vm.callPickupState$).toBe("a 4ms b 5ms c", {
|
||||
a: "unknown",
|
||||
b: "ringing",
|
||||
c: "unknown",
|
||||
});
|
||||
},
|
||||
{
|
||||
waitForCallPickup: true,
|
||||
encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
|
||||
},
|
||||
);
|
||||
});
|
||||
// Should ring until Alice joins
|
||||
expectObservable(vm.ringing$).toBe("(ny) 17ms n", yesNo);
|
||||
// Layout should show placeholder media for the participant we're
|
||||
// ringing the entire time
|
||||
expectObservable(summarizeLayout$(vm.layout$)).toBe("a 20ms b", {
|
||||
a: {
|
||||
type: "one-on-one",
|
||||
spotlight: `${localId}:0`,
|
||||
pip: `ringing:${aliceUserId}`,
|
||||
},
|
||||
b: {
|
||||
type: "one-on-one",
|
||||
spotlight: `${aliceId}:0`,
|
||||
pip: `${localId}:0`,
|
||||
},
|
||||
});
|
||||
},
|
||||
{ waitForCallPickup: true },
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -128,7 +128,6 @@ import {
|
||||
createSentCallNotification$,
|
||||
} from "./CallNotificationLifecycle.ts";
|
||||
import {
|
||||
createDMMember$,
|
||||
createMatrixMemberMetadata$,
|
||||
createRoomMembers$,
|
||||
} from "./remoteMembers/MatrixMemberMetadata.ts";
|
||||
@@ -137,12 +136,17 @@ import { type Connection } from "./remoteMembers/Connection.ts";
|
||||
import { createLayoutModeSwitch } from "./LayoutSwitch.ts";
|
||||
import {
|
||||
createWrappedUserMedia,
|
||||
type MediaItem,
|
||||
type WrappedUserMediaViewModel,
|
||||
} from "../media/MediaItem.ts";
|
||||
} from "../media/WrappedUserMediaViewModel.ts";
|
||||
import { type ScreenShareViewModel } from "../media/ScreenShareViewModel.ts";
|
||||
import { type UserMediaViewModel } from "../media/UserMediaViewModel.ts";
|
||||
import { type MediaViewModel } from "../media/MediaViewModel.ts";
|
||||
import { type LocalUserMediaViewModel } from "../media/LocalUserMediaViewModel.ts";
|
||||
import { type RemoteUserMediaViewModel } from "../media/RemoteUserMediaViewModel.ts";
|
||||
import {
|
||||
createRingingMedia,
|
||||
type RingingMediaViewModel,
|
||||
} from "../media/RingingMediaViewModel.ts";
|
||||
|
||||
const logger = rootLogger.getChild("[CallViewModel]");
|
||||
//TODO
|
||||
@@ -210,11 +214,10 @@ export type LivekitRoomItem = {
|
||||
export interface CallViewModel {
|
||||
// lifecycle
|
||||
autoLeave$: Observable<AutoLeaveReason>;
|
||||
// TODO if we are in "unknown" state we need a loading rendering (or empty screen)
|
||||
// Otherwise it looks like we already connected and only than the ringing starts which is weird.
|
||||
callPickupState$: Behavior<
|
||||
"unknown" | "ringing" | "timeout" | "decline" | "success" | null
|
||||
>;
|
||||
/**
|
||||
* Whether we are ringing a call recipient.
|
||||
*/
|
||||
ringing$: Behavior<boolean>;
|
||||
/** Observable that emits when the user should leave the call (hangup pressed, widget action, error).
|
||||
* THIS DOES NOT LEAVE THE CALL YET. The only way to leave the call (send the hangup event) is
|
||||
* - by ending the scope
|
||||
@@ -289,13 +292,6 @@ export interface CallViewModel {
|
||||
/** List of reactions. Keys are: membership.membershipId (currently predefined as: `${membershipEvent.userId}:${membershipEvent.deviceId}`)*/
|
||||
reactions$: Behavior<Record<string, ReactionOption>>;
|
||||
|
||||
ringOverlay$: Behavior<null | {
|
||||
name: string;
|
||||
/** roomId or userId for the avatar generation. */
|
||||
idForAvatar: string;
|
||||
text: string;
|
||||
avatarMxc?: string;
|
||||
}>;
|
||||
// sounds and events
|
||||
joinSoundEffect$: Observable<void>;
|
||||
leaveSoundEffect$: Observable<void>;
|
||||
@@ -550,9 +546,7 @@ export function createCallViewModel$(
|
||||
},
|
||||
connectionManager,
|
||||
matrixRTCSession,
|
||||
localTransport$: scope.behavior(
|
||||
localTransport$.pipe(switchMap((t) => t.advertised$)),
|
||||
),
|
||||
localTransport$,
|
||||
logger: logger.getChild(`[${Date.now()}]`),
|
||||
});
|
||||
|
||||
@@ -611,40 +605,6 @@ export function createCallViewModel$(
|
||||
matrixRoomMembers$,
|
||||
);
|
||||
|
||||
const dmMember$ = createDMMember$(scope, matrixRoomMembers$, matrixRoom);
|
||||
const noUserToCallInRoom$ = scope.behavior(
|
||||
matrixRoomMembers$.pipe(
|
||||
map(
|
||||
(roomMembersMap) =>
|
||||
roomMembersMap.size === 1 && roomMembersMap.get(userId) !== undefined,
|
||||
),
|
||||
),
|
||||
);
|
||||
|
||||
const ringOverlay$ = scope.behavior(
|
||||
combineLatest([noUserToCallInRoom$, dmMember$, callPickupState$]).pipe(
|
||||
map(([noUserToCallInRoom, dmMember, callPickupState]) => {
|
||||
// No overlay if not in ringing state
|
||||
if (callPickupState !== "ringing" || noUserToCallInRoom) return null;
|
||||
|
||||
const name = dmMember ? dmMember.rawDisplayName : matrixRoom.name;
|
||||
const id = dmMember ? dmMember.userId : matrixRoom.roomId;
|
||||
const text = dmMember
|
||||
? `Waiting for ${name} to join…`
|
||||
: "Waiting for other participants…";
|
||||
const avatarMxc = dmMember
|
||||
? (dmMember.getMxcAvatarUrl?.() ?? undefined)
|
||||
: (matrixRoom.getMxcAvatarUrl() ?? undefined);
|
||||
return {
|
||||
name: name ?? id,
|
||||
idForAvatar: id,
|
||||
text,
|
||||
avatarMxc,
|
||||
};
|
||||
}),
|
||||
),
|
||||
);
|
||||
|
||||
const allConnections$ = scope.behavior(
|
||||
connectionManager.connectionManagerData$.pipe(map((d) => d.value)),
|
||||
);
|
||||
@@ -720,7 +680,7 @@ export function createCallViewModel$(
|
||||
matrixLivekitMembers$,
|
||||
duplicateTiles.value$,
|
||||
]).pipe(
|
||||
// Generate a collection of MediaItems from the list of expected (whether
|
||||
// Generate a collection of user media from the list of expected (whether
|
||||
// present or missing) LiveKit participants.
|
||||
generateItems(
|
||||
"CallViewModel userMedia$",
|
||||
@@ -793,32 +753,67 @@ export function createCallViewModel$(
|
||||
),
|
||||
);
|
||||
|
||||
const ringingMedia$ = scope.behavior<RingingMediaViewModel[]>(
|
||||
combineLatest([userMedia$, matrixRoomMembers$, callPickupState$]).pipe(
|
||||
generateItems(
|
||||
"CallViewModel ringingMedia$",
|
||||
function* ([userMedia, roomMembers, callPickupState]) {
|
||||
if (
|
||||
callPickupState === "ringing" ||
|
||||
callPickupState === "timeout" ||
|
||||
callPickupState === "decline"
|
||||
) {
|
||||
for (const member of roomMembers.values()) {
|
||||
if (!userMedia.some((vm) => vm.userId === member.userId))
|
||||
yield {
|
||||
keys: [member.userId],
|
||||
data: callPickupState,
|
||||
};
|
||||
}
|
||||
}
|
||||
},
|
||||
(scope, pickupState$, userId) =>
|
||||
createRingingMedia({
|
||||
id: `ringing:${userId}`,
|
||||
userId,
|
||||
displayName$: scope.behavior(
|
||||
matrixRoomMembers$.pipe(
|
||||
map((members) => members.get(userId)?.rawDisplayName || userId),
|
||||
),
|
||||
),
|
||||
mxcAvatarUrl$:
|
||||
matrixMemberMetadataStore.createAvatarUrlBehavior$(userId),
|
||||
pickupState$,
|
||||
muteStates,
|
||||
}),
|
||||
),
|
||||
distinctUntilChanged(shallowEquals),
|
||||
tap((ringingMedia) => {
|
||||
if (ringingMedia.length > 1)
|
||||
// Warn that UI may do something unexpected in this case
|
||||
logger.warn(
|
||||
`Ringing more than one participant is not supported (ringing ${ringingMedia.map((vm) => vm.userId).join(", ")})`,
|
||||
);
|
||||
}),
|
||||
),
|
||||
);
|
||||
|
||||
/**
|
||||
* List of all media items (user media and screen share media) that we want
|
||||
* tiles for.
|
||||
* All screen share media that we want to display.
|
||||
*/
|
||||
const mediaItems$ = scope.behavior<MediaItem[]>(
|
||||
const screenShares$ = scope.behavior<ScreenShareViewModel[]>(
|
||||
userMedia$.pipe(
|
||||
switchMap((userMedia) =>
|
||||
userMedia.length === 0
|
||||
? of([])
|
||||
: combineLatest(
|
||||
userMedia.map((m) => m.screenShares$),
|
||||
(...screenShares) => [...userMedia, ...screenShares.flat(1)],
|
||||
(...screenShares) => screenShares.flat(1),
|
||||
),
|
||||
),
|
||||
),
|
||||
);
|
||||
|
||||
/**
|
||||
* List of MediaItems that we want to display, that are of type ScreenShare
|
||||
*/
|
||||
const screenShares$ = scope.behavior<ScreenShareViewModel[]>(
|
||||
mediaItems$.pipe(
|
||||
map((mediaItems) => mediaItems.filter((m) => m.type === "screen share")),
|
||||
),
|
||||
);
|
||||
|
||||
const joinSoundEffect$ = userMedia$.pipe(
|
||||
pairwise(),
|
||||
filter(
|
||||
@@ -931,40 +926,20 @@ export function createCallViewModel$(
|
||||
),
|
||||
);
|
||||
|
||||
const spotlight$ = scope.behavior<MediaViewModel[]>(
|
||||
screenShares$.pipe(
|
||||
switchMap((screenShares) => {
|
||||
if (screenShares.length > 0) return of(screenShares);
|
||||
|
||||
return spotlightSpeaker$.pipe(
|
||||
map((speaker) => (speaker ? [speaker] : [])),
|
||||
/**
|
||||
* Local user media suitable for displaying in a PiP (undefined if not found
|
||||
* or if user prefers to not see themselves).
|
||||
*/
|
||||
const localUserMediaForPip$ = scope.behavior<
|
||||
LocalUserMediaViewModel | undefined
|
||||
>(
|
||||
userMedia$.pipe(
|
||||
switchMap((userMedia) => {
|
||||
const localUserMedia = userMedia.find(
|
||||
(m): m is WrappedUserMediaViewModel & LocalUserMediaViewModel =>
|
||||
m.type === "user" && m.local,
|
||||
);
|
||||
}),
|
||||
distinctUntilChanged<MediaViewModel[]>(shallowEquals),
|
||||
),
|
||||
);
|
||||
|
||||
const pip$ = scope.behavior<UserMediaViewModel | undefined>(
|
||||
combineLatest([
|
||||
// TODO This also needs epoch logic to dedupe the screenshares and mediaItems emits
|
||||
screenShares$,
|
||||
spotlightSpeaker$,
|
||||
mediaItems$,
|
||||
]).pipe(
|
||||
switchMap(([screenShares, spotlight, mediaItems]) => {
|
||||
if (screenShares.length > 0) {
|
||||
return spotlightSpeaker$;
|
||||
}
|
||||
if (!spotlight || spotlight.local) {
|
||||
return of(undefined);
|
||||
}
|
||||
|
||||
const localUserMedia = mediaItems.find(
|
||||
(m) => m.type === "user" && m.local,
|
||||
);
|
||||
if (!localUserMedia) {
|
||||
return of(undefined);
|
||||
}
|
||||
if (!localUserMedia) return of(undefined);
|
||||
return localUserMedia.alwaysShow$.pipe(
|
||||
map((alwaysShow) => (alwaysShow ? localUserMedia : undefined)),
|
||||
);
|
||||
@@ -972,6 +947,42 @@ export function createCallViewModel$(
|
||||
),
|
||||
);
|
||||
|
||||
const spotlightAndPip$ = scope.behavior<{
|
||||
spotlight: MediaViewModel[];
|
||||
pip$: Observable<UserMediaViewModel | undefined>;
|
||||
}>(
|
||||
ringingMedia$.pipe(
|
||||
switchMap((ringingMedia) => {
|
||||
if (ringingMedia.length > 0)
|
||||
return of({ spotlight: ringingMedia, pip$: localUserMediaForPip$ });
|
||||
|
||||
return screenShares$.pipe(
|
||||
switchMap((screenShares) => {
|
||||
if (screenShares.length > 0)
|
||||
return of({ spotlight: screenShares, pip$: spotlightSpeaker$ });
|
||||
|
||||
return spotlightSpeaker$.pipe(
|
||||
map((speaker) => ({
|
||||
spotlight: speaker ? [speaker] : [],
|
||||
// Hide PiP if redundant (i.e. if local user is already in spotlight)
|
||||
pip$: localUserMediaForPip$.pipe(
|
||||
map((m) => (m === speaker ? undefined : m)),
|
||||
),
|
||||
})),
|
||||
);
|
||||
}),
|
||||
);
|
||||
}),
|
||||
),
|
||||
);
|
||||
|
||||
const spotlight$ = scope.behavior<MediaViewModel[]>(
|
||||
spotlightAndPip$.pipe(
|
||||
map(({ spotlight }) => spotlight),
|
||||
distinctUntilChanged<MediaViewModel[]>(shallowEquals),
|
||||
),
|
||||
);
|
||||
|
||||
const hasRemoteScreenShares$ = scope.behavior<boolean>(
|
||||
spotlight$.pipe(
|
||||
map((spotlight) =>
|
||||
@@ -1054,24 +1065,61 @@ export function createCallViewModel$(
|
||||
}));
|
||||
|
||||
const spotlightExpandedLayoutMedia$: Observable<SpotlightExpandedLayoutMedia> =
|
||||
combineLatest([spotlight$, pip$], (spotlight, pip) => ({
|
||||
type: "spotlight-expanded",
|
||||
spotlight,
|
||||
pip: pip ?? undefined,
|
||||
}));
|
||||
spotlightAndPip$.pipe(
|
||||
switchMap(({ spotlight, pip$ }) =>
|
||||
pip$.pipe(
|
||||
map((pip) => ({
|
||||
type: "spotlight-expanded" as const,
|
||||
spotlight,
|
||||
pip: pip ?? undefined,
|
||||
})),
|
||||
),
|
||||
),
|
||||
);
|
||||
|
||||
const oneOnOneLayoutMedia$: Observable<OneOnOneLayoutMedia | null> =
|
||||
mediaItems$.pipe(
|
||||
map((mediaItems) => {
|
||||
if (mediaItems.length !== 2) return null;
|
||||
const local = mediaItems.find((vm) => vm.type === "user" && vm.local);
|
||||
const remote = mediaItems.find((vm) => vm.type === "user" && !vm.local);
|
||||
// There might not be a remote tile if there are screen shares, or if
|
||||
// only the local user is in the call and they're using the duplicate
|
||||
// tiles option
|
||||
if (!remote || !local) return null;
|
||||
userMedia$.pipe(
|
||||
switchMap((userMedia) => {
|
||||
if (userMedia.length <= 2) {
|
||||
const local = userMedia.find(
|
||||
(vm): vm is WrappedUserMediaViewModel & LocalUserMediaViewModel =>
|
||||
vm.type === "user" && vm.local,
|
||||
);
|
||||
|
||||
return { type: "one-on-one", local, remote };
|
||||
if (local !== undefined) {
|
||||
const remote = userMedia.find(
|
||||
(
|
||||
vm,
|
||||
): vm is WrappedUserMediaViewModel & RemoteUserMediaViewModel =>
|
||||
vm.type === "user" && !vm.local,
|
||||
);
|
||||
|
||||
if (remote !== undefined)
|
||||
return of({
|
||||
type: "one-on-one" as const,
|
||||
spotlight: remote,
|
||||
pip: local,
|
||||
});
|
||||
|
||||
// If there's no other user media in the call (could still happen in
|
||||
// this branch due to the duplicate tiles option), we could possibly
|
||||
// show ringing media instead
|
||||
if (userMedia.length === 1)
|
||||
return ringingMedia$.pipe(
|
||||
map((ringingMedia) => {
|
||||
return ringingMedia.length === 1
|
||||
? {
|
||||
type: "one-on-one" as const,
|
||||
spotlight: local,
|
||||
pip: ringingMedia[0],
|
||||
}
|
||||
: null;
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return of(null);
|
||||
}),
|
||||
);
|
||||
|
||||
@@ -1482,8 +1530,9 @@ export function createCallViewModel$(
|
||||
|
||||
return {
|
||||
autoLeave$: autoLeave$,
|
||||
callPickupState$: callPickupState$,
|
||||
ringOverlay$: ringOverlay$,
|
||||
ringing$: scope.behavior(
|
||||
callPickupState$.pipe(map((state) => state === "ringing")),
|
||||
),
|
||||
leave$: leave$,
|
||||
hangup: (): void => userHangup$.next(),
|
||||
join: localMembership.requestJoinAndPublish,
|
||||
|
||||
@@ -17,7 +17,7 @@ import {
|
||||
import { SyncState } from "matrix-js-sdk/lib/sync";
|
||||
import { BehaviorSubject, type Observable, map, of } from "rxjs";
|
||||
import { onTestFinished, vi } from "vitest";
|
||||
import { ClientEvent, type MatrixClient } from "matrix-js-sdk";
|
||||
import { ClientEvent, type RoomMember, type MatrixClient } from "matrix-js-sdk";
|
||||
import EventEmitter from "events";
|
||||
import * as ComponentsCore from "@livekit/components-core";
|
||||
|
||||
@@ -63,15 +63,10 @@ const carol = local;
|
||||
|
||||
const dave = mockMatrixRoomMember(daveRTLRtcMember, { rawDisplayName: "Dave" });
|
||||
|
||||
const roomMembers = new Map(
|
||||
[alice, aliceDoppelganger, bob, bobZeroWidthSpace, carol, dave, daveRTL].map(
|
||||
(p) => [p.userId, p],
|
||||
),
|
||||
);
|
||||
|
||||
export interface CallViewModelInputs {
|
||||
remoteParticipants$: Behavior<RemoteParticipant[]>;
|
||||
rtcMembers$: Behavior<Partial<CallMembership>[]>;
|
||||
roomMembers: RoomMember[];
|
||||
livekitConnectionState$: Behavior<ConnectionState>;
|
||||
speaking: Map<Participant, Observable<boolean>>;
|
||||
mediaDevices: MediaDevices;
|
||||
@@ -86,6 +81,15 @@ export function withCallViewModel(mode: MatrixRTCMode) {
|
||||
{
|
||||
remoteParticipants$ = constant([]),
|
||||
rtcMembers$ = constant([localRtcMember]),
|
||||
roomMembers = [
|
||||
alice,
|
||||
aliceDoppelganger,
|
||||
bob,
|
||||
bobZeroWidthSpace,
|
||||
carol,
|
||||
dave,
|
||||
daveRTL,
|
||||
],
|
||||
livekitConnectionState$: connectionState$ = constant(
|
||||
ConnectionState.Connected,
|
||||
),
|
||||
@@ -127,9 +131,12 @@ export function withCallViewModel(mode: MatrixRTCMode) {
|
||||
public getSyncState(): SyncState {
|
||||
return syncState;
|
||||
}
|
||||
public getAccessToken(): string | null {
|
||||
return "a-token";
|
||||
}
|
||||
})() as Partial<MatrixClient> as MatrixClient,
|
||||
getMembers: () => Array.from(roomMembers.values()),
|
||||
getMembersWithMembership: () => Array.from(roomMembers.values()),
|
||||
getMembers: () => roomMembers,
|
||||
getMembersWithMembership: () => roomMembers,
|
||||
});
|
||||
const rtcSession = new MockRTCSession(room, []).withMemberships(
|
||||
rtcMembers$,
|
||||
|
||||
@@ -33,13 +33,20 @@ import {
|
||||
PublishState,
|
||||
TrackState,
|
||||
} from "./LocalMember";
|
||||
import { MatrixRTCTransportMissingError } from "../../../utils/errors";
|
||||
import {
|
||||
FailToGetOpenIdToken,
|
||||
MatrixRTCTransportMissingError,
|
||||
} from "../../../utils/errors";
|
||||
import { Epoch, ObservableScope } from "../../ObservableScope";
|
||||
import { constant } from "../../Behavior";
|
||||
import { ConnectionManagerData } from "../remoteMembers/ConnectionManager";
|
||||
import { ConnectionState, type Connection } from "../remoteMembers/Connection";
|
||||
import { type Publisher } from "./Publisher";
|
||||
import { initializeWidget } from "../../../widget";
|
||||
import {
|
||||
type LocalTransport,
|
||||
type LocalTransportWithSFUConfig,
|
||||
} from "./LocalTransport";
|
||||
|
||||
initializeWidget();
|
||||
|
||||
@@ -214,7 +221,7 @@ describe("LocalMembership", () => {
|
||||
};
|
||||
|
||||
it("throws error on missing RTC config error", () => {
|
||||
withTestScheduler(({ scope, hot, expectObservable }) => {
|
||||
withTestScheduler(({ scope, hot, behavior, expectObservable }) => {
|
||||
const localTransport$ = scope.behavior<null | LivekitTransportConfig>(
|
||||
hot("1ms #", {}, new MatrixRTCTransportMissingError("domain.com")),
|
||||
null,
|
||||
@@ -230,11 +237,16 @@ describe("LocalMembership", () => {
|
||||
),
|
||||
};
|
||||
|
||||
const aLocalTransport: LocalTransport = {
|
||||
advertised$: localTransport$,
|
||||
active$: behavior("a", { a: null }),
|
||||
};
|
||||
|
||||
const localMembership = createLocalMembership$({
|
||||
scope,
|
||||
...defaultCreateLocalMemberValues,
|
||||
connectionManager: mockConnectionManager,
|
||||
localTransport$,
|
||||
localTransport$: behavior("a", { a: aLocalTransport }),
|
||||
});
|
||||
localMembership.requestJoinAndPublish();
|
||||
|
||||
@@ -245,10 +257,62 @@ describe("LocalMembership", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("Should not publish to active transport if advertised has errors", () => {
|
||||
withTestScheduler(({ scope, hot, behavior, expectObservable }) => {
|
||||
const advertised$ = scope.behavior<null | LivekitTransportConfig>(
|
||||
hot("--#", {}, new FailToGetOpenIdToken(new Error("foo"))),
|
||||
null,
|
||||
);
|
||||
|
||||
// Populate a connection for active
|
||||
const connectionManagerData = new ConnectionManagerData();
|
||||
connectionManagerData.add(connectionTransportBConnected, []);
|
||||
const mockConnectionManager = {
|
||||
transports$: constant(new Epoch([bTransport])),
|
||||
connectionManagerData$: constant(new Epoch(connectionManagerData)),
|
||||
};
|
||||
|
||||
const aLocalTransport: LocalTransport = {
|
||||
advertised$,
|
||||
active$: behavior("a", { n: null, a: bTransportWithSFUConfig }),
|
||||
};
|
||||
|
||||
defaultCreateLocalMemberValues.createPublisherFactory.mockImplementation(
|
||||
() => {
|
||||
return {} as unknown as Publisher;
|
||||
},
|
||||
);
|
||||
const publisherFactory =
|
||||
defaultCreateLocalMemberValues.createPublisherFactory as ReturnType<
|
||||
typeof vi.fn
|
||||
>;
|
||||
|
||||
const localMembership = createLocalMembership$({
|
||||
scope,
|
||||
...defaultCreateLocalMemberValues,
|
||||
connectionManager: mockConnectionManager,
|
||||
localTransport$: behavior("a", { a: aLocalTransport }),
|
||||
});
|
||||
localMembership.requestJoinAndPublish();
|
||||
|
||||
expectObservable(localMembership.localMemberState$).toBe("n-e", {
|
||||
n: TransportState.Waiting,
|
||||
e: expect.toSatisfy((e) => e instanceof FailToGetOpenIdToken),
|
||||
});
|
||||
|
||||
// Should not have created any publisher
|
||||
expect(publisherFactory).toHaveBeenCalledTimes(0);
|
||||
});
|
||||
});
|
||||
|
||||
it("logs if callIntent cannot be updated", async () => {
|
||||
const scope = new ObservableScope();
|
||||
|
||||
const localTransport$ = new BehaviorSubject(aTransport);
|
||||
const aLocalTransport: LocalTransport = {
|
||||
advertised$: new BehaviorSubject(aTransport),
|
||||
active$: new BehaviorSubject(aTransportWithSFUConfig),
|
||||
};
|
||||
|
||||
const mockConnectionManager = {
|
||||
transports$: constant(new Epoch([])),
|
||||
connectionManagerData$: constant(new Epoch(new ConnectionManagerData())),
|
||||
@@ -264,7 +328,7 @@ describe("LocalMembership", () => {
|
||||
leaveRoomSession: vi.fn(),
|
||||
},
|
||||
connectionManager: mockConnectionManager,
|
||||
localTransport$,
|
||||
localTransport$: new BehaviorSubject(aLocalTransport),
|
||||
});
|
||||
const expextedLog =
|
||||
"'not connected yet' while updating the call intent (this is expected on startup)";
|
||||
@@ -279,10 +343,31 @@ describe("LocalMembership", () => {
|
||||
const aTransport = {
|
||||
livekit_service_url: "a",
|
||||
} as LivekitTransportConfig;
|
||||
|
||||
const aTransportWithSFUConfig = {
|
||||
transport: aTransport,
|
||||
sfuConfig: {
|
||||
jwt: "foo",
|
||||
livekitAlias: "bar",
|
||||
livekitIdentity: "baz",
|
||||
url: "bro",
|
||||
},
|
||||
} as LocalTransportWithSFUConfig;
|
||||
|
||||
const bTransport = {
|
||||
livekit_service_url: "b",
|
||||
} as LivekitTransportConfig;
|
||||
|
||||
const bTransportWithSFUConfig = {
|
||||
transport: bTransport,
|
||||
sfuConfig: {
|
||||
jwt: "foo2",
|
||||
livekitAlias: "bar2",
|
||||
livekitIdentity: "baz2",
|
||||
url: "bro2",
|
||||
},
|
||||
} as LocalTransportWithSFUConfig;
|
||||
|
||||
const connectionTransportAConnected = {
|
||||
livekitRoom: mockLivekitRoom({
|
||||
localParticipant: {
|
||||
@@ -307,7 +392,11 @@ describe("LocalMembership", () => {
|
||||
it("recreates publisher if new connection is used, always unpublish and end tracks", async () => {
|
||||
const scope = new ObservableScope();
|
||||
|
||||
const localTransport$ = new BehaviorSubject(aTransport);
|
||||
const activeTransport$ = new BehaviorSubject(aTransportWithSFUConfig);
|
||||
const aLocalTransport: LocalTransport = {
|
||||
advertised$: new BehaviorSubject(aTransport),
|
||||
active$: activeTransport$,
|
||||
};
|
||||
|
||||
const publishers: Publisher[] = [];
|
||||
let seed = 0;
|
||||
@@ -343,10 +432,13 @@ describe("LocalMembership", () => {
|
||||
connectionManager: {
|
||||
connectionManagerData$: constant(new Epoch(connectionManagerData)),
|
||||
},
|
||||
localTransport$,
|
||||
localTransport$: new BehaviorSubject(aLocalTransport),
|
||||
});
|
||||
await flushPromises();
|
||||
localTransport$.next(bTransport);
|
||||
activeTransport$.next({
|
||||
...aTransportWithSFUConfig,
|
||||
transport: bTransport,
|
||||
});
|
||||
await flushPromises();
|
||||
|
||||
expect(publisherFactory).toHaveBeenCalledTimes(2);
|
||||
@@ -368,8 +460,6 @@ describe("LocalMembership", () => {
|
||||
it("only start tracks if requested", async () => {
|
||||
const scope = new ObservableScope();
|
||||
|
||||
const localTransport$ = new BehaviorSubject(aTransport);
|
||||
|
||||
const publishers: Publisher[] = [];
|
||||
|
||||
const tracks$ = new BehaviorSubject<LocalTrack[]>([]);
|
||||
@@ -396,6 +486,11 @@ describe("LocalMembership", () => {
|
||||
typeof vi.fn
|
||||
>;
|
||||
|
||||
const aLocalTransport: LocalTransport = {
|
||||
advertised$: new BehaviorSubject(aTransport),
|
||||
active$: new BehaviorSubject(aTransportWithSFUConfig),
|
||||
};
|
||||
|
||||
const connectionManagerData = new ConnectionManagerData();
|
||||
connectionManagerData.add(connectionTransportAConnected, []);
|
||||
// connectionManagerData.add(connectionTransportB, []);
|
||||
@@ -405,7 +500,7 @@ describe("LocalMembership", () => {
|
||||
connectionManager: {
|
||||
connectionManagerData$: constant(new Epoch(connectionManagerData)),
|
||||
},
|
||||
localTransport$,
|
||||
localTransport$: new BehaviorSubject(aLocalTransport),
|
||||
});
|
||||
await flushPromises();
|
||||
expect(publisherFactory).toHaveBeenCalledOnce();
|
||||
@@ -428,9 +523,15 @@ describe("LocalMembership", () => {
|
||||
const scope = new ObservableScope();
|
||||
|
||||
const connectionManagerData = new ConnectionManagerData();
|
||||
const localTransport$ = new BehaviorSubject<null | LivekitTransportConfig>(
|
||||
null,
|
||||
);
|
||||
|
||||
const activeTransport$ =
|
||||
new BehaviorSubject<null | LocalTransportWithSFUConfig>(null);
|
||||
|
||||
const aLocalTransport: LocalTransport = {
|
||||
advertised$: new BehaviorSubject(aTransport),
|
||||
active$: activeTransport$,
|
||||
};
|
||||
|
||||
const connectionManagerData$ = new BehaviorSubject(
|
||||
new Epoch(connectionManagerData),
|
||||
);
|
||||
@@ -470,14 +571,14 @@ describe("LocalMembership", () => {
|
||||
connectionManager: {
|
||||
connectionManagerData$,
|
||||
},
|
||||
localTransport$,
|
||||
localTransport$: new BehaviorSubject(aLocalTransport),
|
||||
});
|
||||
|
||||
await flushPromises();
|
||||
expect(localMembership.localMemberState$.value).toStrictEqual(
|
||||
TransportState.Waiting,
|
||||
);
|
||||
localTransport$.next(aTransport);
|
||||
activeTransport$.next(aTransportWithSFUConfig);
|
||||
await flushPromises();
|
||||
expect(localMembership.localMemberState$.value).toStrictEqual({
|
||||
matrix: RTCMemberStatus.Connected,
|
||||
|
||||
@@ -62,6 +62,8 @@ import {
|
||||
} from "../remoteMembers/Connection.ts";
|
||||
import { type HomeserverConnected } from "./HomeserverConnected.ts";
|
||||
import { and$ } from "../../../utils/observable.ts";
|
||||
import { type LocalTransport } from "./LocalTransport.ts";
|
||||
import { areLivekitTransportsEqual } from "../remoteMembers/MatrixLivekitMembers.ts";
|
||||
|
||||
export enum TransportState {
|
||||
/** Not even a transport is available to the LocalMembership */
|
||||
@@ -127,7 +129,7 @@ interface Props {
|
||||
createPublisherFactory: (connection: Connection) => Publisher;
|
||||
joinMatrixRTC: (transport: LivekitTransportConfig) => void;
|
||||
homeserverConnected: HomeserverConnected;
|
||||
localTransport$: Behavior<LivekitTransportConfig | null>;
|
||||
localTransport$: Behavior<LocalTransport>;
|
||||
matrixRTCSession: Pick<
|
||||
MatrixRTCSession,
|
||||
"updateCallIntent" | "leaveRoomSession"
|
||||
@@ -160,7 +162,7 @@ interface Props {
|
||||
export const createLocalMembership$ = ({
|
||||
scope,
|
||||
connectionManager,
|
||||
localTransport$: localTransportCanThrow$,
|
||||
localTransport$,
|
||||
homeserverConnected,
|
||||
createPublisherFactory,
|
||||
joinMatrixRTC,
|
||||
@@ -205,23 +207,43 @@ export const createLocalMembership$ = ({
|
||||
const logger = parentLogger.getChild("[LocalMembership]");
|
||||
logger.debug(`Creating local membership..`);
|
||||
|
||||
// We consider error on the transport as fatal.
|
||||
// Whether it is the active transport or the preferred transport.
|
||||
const handleTransportError = (e: unknown): Observable<null> => {
|
||||
let error: ElementCallError;
|
||||
if (e instanceof ElementCallError) {
|
||||
error = e;
|
||||
} else {
|
||||
error = new UnknownCallError(
|
||||
e instanceof Error ? e : new Error("Unknown error from localTransport"),
|
||||
);
|
||||
}
|
||||
setTransportError(error);
|
||||
return of(null);
|
||||
};
|
||||
|
||||
// This is the transport that we will advertise in our membership.
|
||||
const advertisedTransport$ = localTransport$.pipe(
|
||||
switchMap((lt) => lt.advertised$),
|
||||
catchError(handleTransportError),
|
||||
distinctUntilChanged(areLivekitTransportsEqual),
|
||||
);
|
||||
|
||||
// Unwrap the local transport and set the state of the LocalMembership to error in case the transport is an error.
|
||||
const localTransport$ = scope.behavior(
|
||||
localTransportCanThrow$.pipe(
|
||||
catchError((e: unknown) => {
|
||||
let error: ElementCallError;
|
||||
if (e instanceof ElementCallError) {
|
||||
error = e;
|
||||
} else {
|
||||
error = new UnknownCallError(
|
||||
e instanceof Error
|
||||
? e
|
||||
: new Error("Unknown error from localTransport"),
|
||||
);
|
||||
}
|
||||
setTransportError(error);
|
||||
return of(null);
|
||||
const activeTransport$ = scope.behavior(
|
||||
localTransport$.pipe(
|
||||
switchMap((lt) => {
|
||||
return combineLatest([lt.active$, lt.advertised$]).pipe(
|
||||
map(([active, advertised]) => {
|
||||
// Our policy is to not publish to another transport if our prefered transport is miss-configured
|
||||
if (advertised == null) return null;
|
||||
|
||||
return active?.transport ?? null;
|
||||
}),
|
||||
);
|
||||
}),
|
||||
catchError(handleTransportError),
|
||||
distinctUntilChanged(areLivekitTransportsEqual),
|
||||
),
|
||||
);
|
||||
|
||||
@@ -229,7 +251,7 @@ export const createLocalMembership$ = ({
|
||||
const localConnection$ = scope.behavior(
|
||||
combineLatest([
|
||||
connectionManager.connectionManagerData$,
|
||||
localTransport$,
|
||||
activeTransport$,
|
||||
]).pipe(
|
||||
map(([{ value: connectionData }, localTransport]) => {
|
||||
if (localTransport === null) {
|
||||
@@ -398,7 +420,7 @@ export const createLocalMembership$ = ({
|
||||
const mediaState$: Behavior<LocalMemberMediaState> = scope.behavior(
|
||||
combineLatest([
|
||||
localConnectionState$,
|
||||
localTransport$,
|
||||
activeTransport$,
|
||||
joinAndPublishRequested$,
|
||||
from(trackStartRequested.promise).pipe(
|
||||
map(() => true),
|
||||
@@ -537,9 +559,11 @@ export const createLocalMembership$ = ({
|
||||
});
|
||||
});
|
||||
|
||||
// Keep matrix rtc session in sync with localTransport$, connectRequested$
|
||||
// Keep matrix rtc session in sync with advertisedTransport$, connectRequested$
|
||||
scope.reconcile(
|
||||
scope.behavior(combineLatest([localTransport$, joinAndPublishRequested$])),
|
||||
scope.behavior(
|
||||
combineLatest([advertisedTransport$, joinAndPublishRequested$]),
|
||||
),
|
||||
async ([transport, shouldConnect]) => {
|
||||
if (!transport) return;
|
||||
// if shouldConnect=false we will do the disconnect as the cleanup from the previous reconcile iteration.
|
||||
|
||||
@@ -19,7 +19,7 @@ import {
|
||||
type CallMembership,
|
||||
type LivekitTransportConfig,
|
||||
} from "matrix-js-sdk/lib/matrixrtc";
|
||||
import { BehaviorSubject, lastValueFrom } from "rxjs";
|
||||
import { BehaviorSubject, filter, lastValueFrom } from "rxjs";
|
||||
import fetchMock from "fetch-mock";
|
||||
|
||||
import {
|
||||
@@ -27,8 +27,13 @@ import {
|
||||
flushPromises,
|
||||
ownMemberMock,
|
||||
mockRtcMembership,
|
||||
testScope,
|
||||
} from "../../../utils/test";
|
||||
import { createLocalTransport$, JwtEndpointVersion } from "./LocalTransport";
|
||||
import {
|
||||
createLocalTransport$,
|
||||
JwtEndpointVersion,
|
||||
type LocalTransportWithSFUConfig,
|
||||
} from "./LocalTransport";
|
||||
import { constant } from "../../Behavior";
|
||||
import { Epoch, ObservableScope, trackEpoch } from "../../ObservableScope";
|
||||
import {
|
||||
@@ -47,19 +52,18 @@ describe("LocalTransport", () => {
|
||||
livekitIdentity: "@lk_user:ABCDEF",
|
||||
};
|
||||
|
||||
let scope: ObservableScope;
|
||||
beforeEach(() => (scope = new ObservableScope()));
|
||||
afterEach(() => scope.end());
|
||||
beforeEach(() => vi.clearAllMocks());
|
||||
|
||||
it("throws if config is missing", async () => {
|
||||
const { advertised$, active$ } = createLocalTransport$({
|
||||
scope,
|
||||
scope: testScope(),
|
||||
roomId: "!room:example.org",
|
||||
useOldestMember: false,
|
||||
memberships$: constant(new Epoch<CallMembership[]>([])),
|
||||
client: {
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
_unstable_getRTCTransports: async () => Promise.resolve([]),
|
||||
getAccessToken: vi.fn().mockReturnValue("access_token"),
|
||||
getDomain: () => "",
|
||||
baseUrl: "example.org",
|
||||
// These won't be called in this error path but satisfy the type
|
||||
@@ -102,6 +106,7 @@ describe("LocalTransport", () => {
|
||||
baseUrl: "https://lk.example.org",
|
||||
// Use empty domain to skip .well-known and use config directly
|
||||
getDomain: () => "",
|
||||
getAccessToken: vi.fn().mockReturnValue("access_token"),
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
_unstable_getRTCTransports: async () => Promise.resolve([]),
|
||||
getOpenIdToken: vi.fn(),
|
||||
@@ -138,7 +143,7 @@ describe("LocalTransport", () => {
|
||||
);
|
||||
|
||||
const { advertised$, active$ } = createLocalTransport$({
|
||||
scope,
|
||||
scope: testScope(),
|
||||
roomId: "!room:example.org",
|
||||
useOldestMember: false,
|
||||
memberships$: constant(new Epoch<CallMembership[]>([])),
|
||||
@@ -149,6 +154,7 @@ describe("LocalTransport", () => {
|
||||
getOpenIdToken: vi.fn(),
|
||||
getDeviceId: vi.fn(),
|
||||
baseUrl: "https://lk.example.org",
|
||||
getAccessToken: vi.fn().mockReturnValue("access_token"),
|
||||
},
|
||||
ownMembershipIdentity: ownMemberMock,
|
||||
forceJwtEndpoint: JwtEndpointVersion.Legacy,
|
||||
@@ -208,6 +214,7 @@ describe("LocalTransport", () => {
|
||||
// Initially, Alice is the only member
|
||||
const memberships$ = new BehaviorSubject([aliceMembership]);
|
||||
|
||||
const scope = testScope();
|
||||
const { advertised$, active$ } = createLocalTransport$({
|
||||
scope,
|
||||
roomId: "!example_room_id",
|
||||
@@ -217,6 +224,7 @@ describe("LocalTransport", () => {
|
||||
getDomain: () => "",
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
_unstable_getRTCTransports: async () => Promise.resolve([]),
|
||||
getAccessToken: vi.fn().mockReturnValue("access_token"),
|
||||
getOpenIdToken: vi.fn(),
|
||||
getDeviceId: vi.fn(),
|
||||
baseUrl: "https://lk.example.org",
|
||||
@@ -263,6 +271,7 @@ describe("LocalTransport", () => {
|
||||
// Initially, there are no members
|
||||
const memberships$ = new BehaviorSubject<CallMembership[]>([]);
|
||||
|
||||
const scope = testScope();
|
||||
const { advertised$, active$ } = createLocalTransport$({
|
||||
scope,
|
||||
roomId: "!example_room_id",
|
||||
@@ -273,6 +282,7 @@ describe("LocalTransport", () => {
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
_unstable_getRTCTransports: async () =>
|
||||
Promise.resolve([aliceTransport]),
|
||||
getAccessToken: vi.fn().mockReturnValue("access_token"),
|
||||
getOpenIdToken: vi.fn(),
|
||||
getDeviceId: vi.fn(),
|
||||
baseUrl: "https://lk.example.org",
|
||||
@@ -312,7 +322,7 @@ describe("LocalTransport", () => {
|
||||
customLivekitUrl.setValue(customLivekitUrl.defaultValue);
|
||||
localTransportOpts = {
|
||||
ownMembershipIdentity: ownMemberMock,
|
||||
scope,
|
||||
scope: testScope(),
|
||||
roomId: "!example_room_id",
|
||||
useOldestMember: false,
|
||||
forceJwtEndpoint: JwtEndpointVersion.Legacy,
|
||||
@@ -323,6 +333,7 @@ describe("LocalTransport", () => {
|
||||
getDomain: vi.fn().mockReturnValue(""),
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
_unstable_getRTCTransports: vi.fn().mockResolvedValue([]),
|
||||
getAccessToken: vi.fn().mockReturnValue("access_token"),
|
||||
getOpenIdToken: vi.fn(),
|
||||
getDeviceId: vi.fn(),
|
||||
},
|
||||
@@ -410,6 +421,42 @@ describe("LocalTransport", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("Should not call _unstable_getRTCTransports in widget mode but use well-known", async () => {
|
||||
mockConfig({
|
||||
livekit: { livekit_service_url: "https://do-not-use.lk.example.org" },
|
||||
});
|
||||
|
||||
localTransportOpts.client.getDomain.mockReturnValue("example.org");
|
||||
|
||||
fetchMock.getOnce("https://example.org/.well-known/matrix/client", {
|
||||
"org.matrix.msc4143.rtc_foci": [
|
||||
{
|
||||
type: "livekit",
|
||||
livekit_service_url: "https://use-me.jwt.call.example.org",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
localTransportOpts.client.getAccessToken.mockReturnValue(null);
|
||||
const { advertised$, active$ } =
|
||||
createLocalTransport$(localTransportOpts);
|
||||
openIdResolver.resolve?.(openIdResponse);
|
||||
expect(advertised$.value).toBe(null);
|
||||
expect(active$.value).toBe(null);
|
||||
await flushPromises();
|
||||
|
||||
expect(
|
||||
localTransportOpts.client._unstable_getRTCTransports,
|
||||
).not.toHaveBeenCalled();
|
||||
|
||||
const expectedTransport = {
|
||||
type: "livekit",
|
||||
livekit_service_url: "https://use-me.jwt.call.example.org",
|
||||
};
|
||||
|
||||
expect(advertised$.value).toStrictEqual(expectedTransport);
|
||||
});
|
||||
|
||||
it("fails fast if the openID request fails for backend config", async () => {
|
||||
localTransportOpts.client._unstable_getRTCTransports.mockResolvedValue([
|
||||
{ type: "livekit", livekit_service_url: "https://lk.example.org" },
|
||||
@@ -469,7 +516,7 @@ describe("LocalTransport", () => {
|
||||
|
||||
it("throws if no options are available", async () => {
|
||||
const { advertised$, active$ } = createLocalTransport$({
|
||||
scope,
|
||||
scope: testScope(),
|
||||
ownMembershipIdentity: ownMemberMock,
|
||||
roomId: "!example_room_id",
|
||||
useOldestMember: false,
|
||||
@@ -481,6 +528,7 @@ describe("LocalTransport", () => {
|
||||
baseUrl: "https://example.org",
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
_unstable_getRTCTransports: async () => Promise.resolve([]),
|
||||
getAccessToken: vi.fn().mockReturnValue("access_token"),
|
||||
// These won't be called in this error path but satisfy the type
|
||||
getOpenIdToken: vi.fn(),
|
||||
getDeviceId: vi.fn(),
|
||||
@@ -496,4 +544,86 @@ describe("LocalTransport", () => {
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
it("should not update advertised/active transport on delayID changes, but delay Id delegation should be called", async () => {
|
||||
// For simplicity, we'll just use the config livekit
|
||||
customLivekitUrl.setValue("https://lk.example.org");
|
||||
|
||||
const authCallSpy = vi
|
||||
.spyOn(openIDSFU, "getSFUConfigWithOpenID")
|
||||
.mockResolvedValue(openIdResponse);
|
||||
|
||||
const delayId$ = new BehaviorSubject<string | null>(null);
|
||||
|
||||
const { advertised$, active$ } = createLocalTransport$({
|
||||
scope: testScope(),
|
||||
ownMembershipIdentity: ownMemberMock,
|
||||
roomId: "!example_room_id",
|
||||
// We want multi-sdu
|
||||
useOldestMember: false,
|
||||
forceJwtEndpoint: JwtEndpointVersion.Legacy,
|
||||
delayId$: delayId$,
|
||||
memberships$: constant(new Epoch<CallMembership[]>([])),
|
||||
client: {
|
||||
getDomain: () => "",
|
||||
baseUrl: "https://example.org",
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
_unstable_getRTCTransports: async () => Promise.resolve([]),
|
||||
getAccessToken: vi.fn().mockReturnValue("access_token"),
|
||||
// These won't be called in this error path but satisfy the type
|
||||
getOpenIdToken: vi.fn(),
|
||||
getDeviceId: vi.fn(),
|
||||
},
|
||||
});
|
||||
|
||||
const advertisedValues: LivekitTransportConfig[] = [];
|
||||
const activeValues: LocalTransportWithSFUConfig[] = [];
|
||||
advertised$
|
||||
.pipe(filter((v) => v !== null))
|
||||
.subscribe((t) => advertisedValues.push(t));
|
||||
active$
|
||||
.pipe(filter((v) => v !== null))
|
||||
.subscribe((t) => activeValues.push(t));
|
||||
|
||||
await flushPromises();
|
||||
|
||||
// we have now an active and an advertised
|
||||
expect(advertisedValues.length).toEqual(1);
|
||||
expect(activeValues.length).toEqual(1);
|
||||
expect(advertisedValues[0]!.livekit_service_url).toEqual(
|
||||
"https://lk.example.org",
|
||||
);
|
||||
expect(activeValues[0]!.transport.livekit_service_url).toEqual(
|
||||
"https://lk.example.org",
|
||||
);
|
||||
|
||||
expect(authCallSpy).toHaveBeenCalledTimes(2);
|
||||
// Now emits 3 new delays id
|
||||
delayId$.next("delay_id_1");
|
||||
await flushPromises();
|
||||
delayId$.next("delay_id_2");
|
||||
await flushPromises();
|
||||
delayId$.next("delay_id_3");
|
||||
await flushPromises();
|
||||
|
||||
// No new emissions should've happened, it is the same transport.
|
||||
expect(advertisedValues.length).toEqual(1);
|
||||
expect(activeValues.length).toEqual(1);
|
||||
|
||||
// Still we should have updated the delayID to auth
|
||||
expect(authCallSpy).toHaveBeenCalledTimes(
|
||||
4 * 2 /* 2 calls for each delayId ?? why */,
|
||||
);
|
||||
|
||||
expect(authCallSpy).toHaveBeenLastCalledWith(
|
||||
expect.anything(),
|
||||
expect.anything(),
|
||||
expect.anything(),
|
||||
expect.anything(),
|
||||
expect.objectContaining({
|
||||
delayId: "delay_id_3",
|
||||
}),
|
||||
expect.anything(),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -8,16 +8,18 @@ Please see LICENSE in the repository root for full details.
|
||||
import {
|
||||
type CallMembership,
|
||||
isLivekitTransportConfig,
|
||||
type Transport,
|
||||
type LivekitTransportConfig,
|
||||
} from "matrix-js-sdk/lib/matrixrtc";
|
||||
import { MatrixError, type MatrixClient } from "matrix-js-sdk";
|
||||
import { type MatrixClient } from "matrix-js-sdk";
|
||||
import {
|
||||
catchError,
|
||||
combineLatest,
|
||||
distinctUntilChanged,
|
||||
first,
|
||||
from,
|
||||
map,
|
||||
merge,
|
||||
type Observable,
|
||||
of,
|
||||
startWith,
|
||||
switchMap,
|
||||
@@ -42,6 +44,7 @@ import {
|
||||
} from "../../../livekit/openIDSFU.ts";
|
||||
import { areLivekitTransportsEqual } from "../remoteMembers/MatrixLivekitMembers.ts";
|
||||
import { customLivekitUrl } from "../../../settings/settings.ts";
|
||||
import { RtcTransportAutoDiscovery } from "./RtcTransportAutoDiscovery.ts";
|
||||
|
||||
const logger = rootLogger.getChild("[LocalTransport]");
|
||||
|
||||
@@ -56,7 +59,7 @@ interface Props {
|
||||
memberships$: Behavior<Epoch<CallMembership[]>>;
|
||||
client: Pick<
|
||||
MatrixClient,
|
||||
"getDomain" | "baseUrl" | "_unstable_getRTCTransports"
|
||||
"getDomain" | "baseUrl" | "_unstable_getRTCTransports" | "getAccessToken"
|
||||
> &
|
||||
OpenIDClientParts;
|
||||
// Used by the jwt service to create the livekit room and compute the livekit alias.
|
||||
@@ -103,7 +106,7 @@ export function isLocalTransportWithSFUConfig(
|
||||
return "transport" in obj && "sfuConfig" in obj;
|
||||
}
|
||||
|
||||
interface LocalTransport {
|
||||
export interface LocalTransport {
|
||||
/**
|
||||
* The transport to be advertised in our MatrixRTC membership. `null` when not
|
||||
* yet fetched/validated.
|
||||
@@ -137,11 +140,116 @@ export const createLocalTransport$ = ({
|
||||
forceJwtEndpoint,
|
||||
delayId$,
|
||||
}: Props): LocalTransport => {
|
||||
/**
|
||||
* The LiveKit transport in use by the oldest RTC membership. `null` when the
|
||||
* oldest member has no such transport.
|
||||
*/
|
||||
const oldestMemberTransport$ = scope.behavior<LivekitTransportConfig | null>(
|
||||
// The LiveKit transport in use by the oldest RTC membership. `null` when the
|
||||
// oldest member has no such transport.
|
||||
const oldestMemberTransport$ = observerOldestMembership$(scope, memberships$);
|
||||
|
||||
const transportDiscovery = new RtcTransportAutoDiscovery({
|
||||
client: client,
|
||||
resolvedConfig: Config.get(),
|
||||
wellKnownFetcher: AutoDiscovery.getRawClientConfig.bind(AutoDiscovery),
|
||||
logger: logger,
|
||||
});
|
||||
|
||||
// Get the preferred transport from the current deployment.
|
||||
const discoveredTransport$ = from(
|
||||
transportDiscovery.discoverPreferredTransport(),
|
||||
);
|
||||
|
||||
const preferredConfig$ = customLivekitUrl.value$
|
||||
.pipe(
|
||||
switchMap((customUrl) => {
|
||||
if (customUrl) {
|
||||
return of({
|
||||
type: "livekit",
|
||||
livekit_service_url: customUrl,
|
||||
} as LivekitTransportConfig);
|
||||
} else {
|
||||
return discoveredTransport$;
|
||||
}
|
||||
}),
|
||||
)
|
||||
.pipe(
|
||||
map((config) => {
|
||||
if (!config) {
|
||||
// Bubbled up from the preferredConfig$ observable.
|
||||
throw new MatrixRTCTransportMissingError(client.getDomain() ?? "");
|
||||
}
|
||||
return config;
|
||||
}),
|
||||
distinctUntilChanged(areLivekitTransportsEqual),
|
||||
);
|
||||
|
||||
const preferredTransport$ = combineLatest([preferredConfig$, delayId$]).pipe(
|
||||
switchMap(async ([transport, delayId]) => {
|
||||
try {
|
||||
return await doOpenIdAndJWTFromUrl(
|
||||
transport,
|
||||
forceJwtEndpoint,
|
||||
ownMembershipIdentity,
|
||||
roomId,
|
||||
client,
|
||||
delayId ?? undefined,
|
||||
);
|
||||
} catch (e) {
|
||||
logger.error(
|
||||
`Failed to authenticate to transport ${transport.livekit_service_url}`,
|
||||
e,
|
||||
);
|
||||
throw mapAuthErrorToUserFriendlyError(e);
|
||||
}
|
||||
}),
|
||||
);
|
||||
|
||||
if (useOldestMember) {
|
||||
return observeLocalTransportForOldestMembership(
|
||||
scope,
|
||||
oldestMemberTransport$,
|
||||
preferredTransport$,
|
||||
client,
|
||||
ownMembershipIdentity,
|
||||
roomId,
|
||||
);
|
||||
}
|
||||
|
||||
// --- Multi-SFU mode ---
|
||||
// Always publish on and advertise the preferred transport.
|
||||
return {
|
||||
advertised$: scope.behavior(
|
||||
preferredTransport$.pipe(
|
||||
map((t) => t.transport),
|
||||
distinctUntilChanged(areLivekitTransportsEqual),
|
||||
),
|
||||
null,
|
||||
),
|
||||
active$: scope.behavior(
|
||||
preferredTransport$.pipe(
|
||||
// XXX: WORK AROUND due to a reconnection glitch.
|
||||
// To remove when we have a proper way to refresh the delegation event ID without refreshing
|
||||
// the whole credentials.
|
||||
// We deliberately hide any changes to the SFU config because we
|
||||
// do not want the app to reconnect whenever the JWT
|
||||
// token changes due to us delegating a new delayed event. The
|
||||
// initial SFU config for the transport is all the app needs.
|
||||
distinctUntilChanged((prev, next) =>
|
||||
areLivekitTransportsEqual(prev.transport, next.transport),
|
||||
),
|
||||
),
|
||||
null,
|
||||
),
|
||||
};
|
||||
};
|
||||
|
||||
/**
|
||||
* Observes the oldest member in the room and returns the transport that it uses if it is a livekit transport.
|
||||
* @param scope - The observable scope.
|
||||
* @param memberships$ - The observable of the call's memberships.'
|
||||
*/
|
||||
function observerOldestMembership$(
|
||||
scope: ObservableScope,
|
||||
memberships$: Behavior<Epoch<CallMembership[]>>,
|
||||
): Behavior<LivekitTransportConfig | null> {
|
||||
return scope.behavior<LivekitTransportConfig | null>(
|
||||
memberships$.pipe(
|
||||
map((memberships) => {
|
||||
const oldestMember = memberships.value[0];
|
||||
@@ -170,292 +278,141 @@ export const createLocalTransport$ = ({
|
||||
distinctUntilChanged(areLivekitTransportsEqual),
|
||||
),
|
||||
);
|
||||
|
||||
/**
|
||||
* The transport that we would personally prefer to publish on (if not for the
|
||||
* transport preferences of others, perhaps). `null` until fetched and
|
||||
* validated.
|
||||
*
|
||||
* @throws MatrixRTCTransportMissingError | FailToGetOpenIdToken
|
||||
*/
|
||||
const preferredTransport$ =
|
||||
scope.behavior<LocalTransportWithSFUConfig | null>(
|
||||
// preferredTransport$ (used for multi sfu) needs to know if we are using the old or new
|
||||
// jwt endpoint (`get_token` vs `sfu/get`) based on that the jwt endpoint will compute the rtcBackendIdentity
|
||||
// differently. (sha(`${userId}|${deviceId}|${memberId}`) vs `${userId}|${deviceId}|${memberId}`)
|
||||
// When using sticky events (we need to use the new endpoint).
|
||||
customLivekitUrl.value$.pipe(
|
||||
switchMap((customUrl) =>
|
||||
startWith<LocalTransportWithSFUConfig | null>(null)(
|
||||
// Fetch the SFU config, and repeat this asynchronously for every
|
||||
// change in delay ID.
|
||||
delayId$.pipe(
|
||||
switchMap(async (delayId) => {
|
||||
logger.info(
|
||||
"Creating preferred transport based on: ",
|
||||
"customUrl: ",
|
||||
customUrl,
|
||||
"delayId: ",
|
||||
delayId,
|
||||
"forceJwtEndpoint: ",
|
||||
forceJwtEndpoint,
|
||||
);
|
||||
return makeTransport(
|
||||
client,
|
||||
ownMembershipIdentity,
|
||||
roomId,
|
||||
customUrl,
|
||||
forceJwtEndpoint,
|
||||
delayId ?? undefined,
|
||||
);
|
||||
}),
|
||||
// We deliberately hide any changes to the SFU config because we
|
||||
// do not actually want the app to reconnect whenever the JWT
|
||||
// token changes due to us delegating a new delayed event. The
|
||||
// initial SFU config for the transport is all the app needs.
|
||||
distinctUntilChanged((prev, next) =>
|
||||
areLivekitTransportsEqual(prev.transport, next.transport),
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
);
|
||||
|
||||
if (useOldestMember) {
|
||||
// --- Oldest member mode ---
|
||||
return {
|
||||
// Never update the transport that we advertise in our membership. Just
|
||||
// take the first valid oldest member or preferred transport that we learn
|
||||
// about, and stick with that. This avoids unnecessary SFU hops and room
|
||||
// state changes.
|
||||
advertised$: scope.behavior(
|
||||
merge(
|
||||
oldestMemberTransport$,
|
||||
preferredTransport$.pipe(map((t) => t?.transport ?? null)),
|
||||
).pipe(
|
||||
first((t) => t !== null),
|
||||
tap((t) =>
|
||||
logger.info(`Advertise transport: ${t.livekit_service_url}`),
|
||||
),
|
||||
),
|
||||
null,
|
||||
),
|
||||
// Publish on the transport used by the oldest member.
|
||||
active$: scope.behavior(
|
||||
oldestMemberTransport$.pipe(
|
||||
switchMap((transport) => {
|
||||
// Oldest member not available (or invalid SFU config).
|
||||
if (transport === null) return of(null);
|
||||
// Oldest member available: fetch the SFU config.
|
||||
const fetchOldestMemberTransport =
|
||||
async (): Promise<LocalTransportWithSFUConfig> => ({
|
||||
transport,
|
||||
sfuConfig: await getSFUConfigWithOpenID(
|
||||
client,
|
||||
ownMembershipIdentity,
|
||||
transport.livekit_service_url,
|
||||
roomId,
|
||||
{ forceJwtEndpoint: JwtEndpointVersion.Legacy },
|
||||
logger,
|
||||
),
|
||||
});
|
||||
return from(fetchOldestMemberTransport()).pipe(startWith(null));
|
||||
}),
|
||||
tap((t) =>
|
||||
logger.info(
|
||||
`Publish on transport: ${t?.transport.livekit_service_url}`,
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
// --- Multi-SFU mode ---
|
||||
// Always publish on and advertise the preferred transport.
|
||||
return {
|
||||
advertised$: scope.behavior(
|
||||
preferredTransport$.pipe(
|
||||
map((t) => t?.transport ?? null),
|
||||
distinctUntilChanged(areLivekitTransportsEqual),
|
||||
),
|
||||
),
|
||||
active$: preferredTransport$,
|
||||
};
|
||||
};
|
||||
|
||||
const FOCI_WK_KEY = "org.matrix.msc4143.rtc_foci";
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine the correct Transport for the current session, including
|
||||
* validating auth against the service to ensure it's correct.
|
||||
* Prefers in order:
|
||||
* Utility to ensure the user can authenticate with the SFU.
|
||||
* We will call `getSFUConfigWithOpenID` once per transport here as it's our
|
||||
* only mechanism of validation. This means we will also ask the
|
||||
* homeserver for a OpenID token a few times. Since OpenID tokens are single
|
||||
* use we don't want to risk any issues by re-using a token.
|
||||
*
|
||||
|
||||
* 1. The `urlFromDevSettings` value. If this cannot be validated, the function will throw.
|
||||
* 2. The transports returned via the homeserver.
|
||||
* 3. The transports returned via .well-known.
|
||||
* 4. The transport configured in Element Call's config.
|
||||
* @param transport The transport to authenticate with.
|
||||
* @param forceJwtEndpoint Whether to force the JWT endpoint to be used.
|
||||
* @param membership The identity of the local member.
|
||||
* @param roomId The room ID to use for the JWT.
|
||||
* @param client The client to use for the OpenID token.
|
||||
* @param delayId The delayId to use for the JWT.
|
||||
*
|
||||
* @param client The authenticated Matrix client for the current user
|
||||
* @param membership The membership identity of the user.
|
||||
* @param roomId The ID of the room to be connected to.
|
||||
* @param urlFromDevSettings Override URL provided by the user's local config.
|
||||
* @param forceJwtEndpoint Whether to force a specific JWT endpoint
|
||||
* - `Legacy` / `Matrix_2_0`
|
||||
* - `get_token` / `sfu/get`
|
||||
* - not hashing / hashing the backendIdentity
|
||||
* @param delayId the delay id passed to the jwt service.
|
||||
*
|
||||
* @returns A fully validated transport config.
|
||||
* @throws MatrixRTCTransportMissingError | FailToGetOpenIdToken
|
||||
* @throws FailToGetOpenIdToken, NoMatrix2AuthorizationService
|
||||
*/
|
||||
async function makeTransport(
|
||||
client: Pick<
|
||||
MatrixClient,
|
||||
"getDomain" | "baseUrl" | "_unstable_getRTCTransports"
|
||||
> &
|
||||
OpenIDClientParts,
|
||||
async function doOpenIdAndJWTFromUrl(
|
||||
transport: LivekitTransportConfig,
|
||||
forceJwtEndpoint: JwtEndpointVersion,
|
||||
membership: CallMembershipIdentityParts,
|
||||
roomId: string,
|
||||
urlFromDevSettings: string | null,
|
||||
forceJwtEndpoint: JwtEndpointVersion,
|
||||
client: Pick<
|
||||
MatrixClient,
|
||||
"getDomain" | "baseUrl" | "_unstable_getRTCTransports" | "getAccessToken"
|
||||
> &
|
||||
OpenIDClientParts,
|
||||
delayId?: string,
|
||||
): Promise<LocalTransportWithSFUConfig> {
|
||||
logger.trace("Searching for a preferred transport");
|
||||
|
||||
async function doOpenIdAndJWTFromUrl(
|
||||
url: string,
|
||||
): Promise<LocalTransportWithSFUConfig> {
|
||||
const sfuConfig = await getSFUConfigWithOpenID(
|
||||
client,
|
||||
membership,
|
||||
url,
|
||||
roomId,
|
||||
{
|
||||
forceJwtEndpoint: forceJwtEndpoint,
|
||||
delayEndpointBaseUrl: client.baseUrl,
|
||||
delayId,
|
||||
},
|
||||
logger,
|
||||
);
|
||||
return {
|
||||
transport: {
|
||||
type: "livekit",
|
||||
livekit_service_url: url,
|
||||
},
|
||||
sfuConfig,
|
||||
};
|
||||
}
|
||||
// We will call `getSFUConfigWithOpenID` once per transport here as it's our
|
||||
// only mechanism of valiation. This means we will also ask the
|
||||
// homeserver for a OpenID token a few times. Since OpenID tokens are single
|
||||
// use we don't want to risk any issues by re-using a token.
|
||||
//
|
||||
// If the OpenID request were to fail then it's acceptable for us to fail
|
||||
// this function early, as we assume the homeserver has got some problems.
|
||||
|
||||
// DEVTOOL: Highest priority: Load from devtool setting
|
||||
if (urlFromDevSettings !== null) {
|
||||
// Validate that the SFU is up. Otherwise, we want to fail on this
|
||||
// as we don't permit other SFUs.
|
||||
// This will call the jwt/sfu/get endpoint to pre create the livekit room.
|
||||
logger.info("Using LiveKit transport from dev tools: ", urlFromDevSettings);
|
||||
return await doOpenIdAndJWTFromUrl(urlFromDevSettings);
|
||||
}
|
||||
|
||||
async function getFirstUsableTransport(
|
||||
transports: Transport[],
|
||||
): Promise<LocalTransportWithSFUConfig | null> {
|
||||
for (const potentialTransport of transports) {
|
||||
if (isLivekitTransportConfig(potentialTransport)) {
|
||||
try {
|
||||
// This will call the jwt/sfu/get endpoint to pre create the livekit room.
|
||||
return await doOpenIdAndJWTFromUrl(
|
||||
potentialTransport.livekit_service_url,
|
||||
);
|
||||
} catch (ex) {
|
||||
// Explictly throw these
|
||||
if (ex instanceof FailToGetOpenIdToken) {
|
||||
throw ex;
|
||||
}
|
||||
if (ex instanceof NoMatrix2AuthorizationService) {
|
||||
throw ex;
|
||||
}
|
||||
logger.debug(
|
||||
`Could not use SFU service "${potentialTransport.livekit_service_url}" as SFU`,
|
||||
ex,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// MSC4143: Attempt to fetch transports from backend.
|
||||
if ("_unstable_getRTCTransports" in client) {
|
||||
try {
|
||||
const transportList = await client._unstable_getRTCTransports();
|
||||
const selectedTransport = await getFirstUsableTransport(transportList);
|
||||
if (selectedTransport) {
|
||||
logger.info(
|
||||
"Using backend-configured (client.getRTCTransports) SFU",
|
||||
selectedTransport,
|
||||
);
|
||||
return selectedTransport;
|
||||
}
|
||||
} catch (ex) {
|
||||
if (ex instanceof MatrixError && ex.httpStatus === 404) {
|
||||
// Expected, this is an unstable endpoint and it's not required.
|
||||
// There will be expected 404 errors in the console. When we check if synapse supports the endpoint.
|
||||
logger.debug(
|
||||
"Matrix homeserver does not provide any RTC transports via `/rtc/transports` (will retry with well-known.)",
|
||||
);
|
||||
} else if (ex instanceof FailToGetOpenIdToken) {
|
||||
throw ex;
|
||||
} else {
|
||||
// We got an error that wasn't just missing support for the feature, so log it loudly.
|
||||
logger.error(
|
||||
"Unexpected error fetching RTC transports from backend",
|
||||
ex,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Legacy MSC4143 (to be removed) WELL_KNOWN: Prioritize the .well-known/matrix/client, if available.
|
||||
const domain = client.getDomain();
|
||||
if (domain) {
|
||||
// we use AutoDiscovery instead of relying on the MatrixClient having already
|
||||
// been fully configured and started
|
||||
const wellKnownFoci = (await AutoDiscovery.getRawClientConfig(domain))?.[
|
||||
FOCI_WK_KEY
|
||||
];
|
||||
const selectedTransport = Array.isArray(wellKnownFoci)
|
||||
? await getFirstUsableTransport(wellKnownFoci)
|
||||
: null;
|
||||
if (selectedTransport) {
|
||||
logger.info("Using .well-known SFU", selectedTransport);
|
||||
return selectedTransport;
|
||||
}
|
||||
}
|
||||
|
||||
// CONFIG: Least prioritized; Load from config file
|
||||
const urlFromConf = Config.get().livekit?.livekit_service_url;
|
||||
if (urlFromConf) {
|
||||
try {
|
||||
// This will call the jwt/sfu/get endpoint to pre create the livekit room.
|
||||
logger.info("Using config SFU", urlFromConf);
|
||||
return await doOpenIdAndJWTFromUrl(urlFromConf);
|
||||
} catch (ex) {
|
||||
if (ex instanceof FailToGetOpenIdToken) {
|
||||
throw ex;
|
||||
}
|
||||
logger.error("Failed to validate config SFU", ex);
|
||||
}
|
||||
}
|
||||
|
||||
// If we do not have returned a transport by now we throw an error
|
||||
throw new MatrixRTCTransportMissingError(domain ?? "");
|
||||
const sfuConfig = await getSFUConfigWithOpenID(
|
||||
client,
|
||||
membership,
|
||||
transport.livekit_service_url,
|
||||
roomId,
|
||||
{
|
||||
forceJwtEndpoint: forceJwtEndpoint,
|
||||
delayEndpointBaseUrl: client.baseUrl,
|
||||
delayId,
|
||||
},
|
||||
logger,
|
||||
);
|
||||
return {
|
||||
transport,
|
||||
sfuConfig,
|
||||
};
|
||||
}
|
||||
|
||||
function observeLocalTransportForOldestMembership(
|
||||
scope: ObservableScope,
|
||||
oldestMemberTransport$: Behavior<LivekitTransportConfig | null>,
|
||||
preferredTransport$: Observable<LocalTransportWithSFUConfig>,
|
||||
client: Pick<
|
||||
MatrixClient,
|
||||
"getDomain" | "baseUrl" | "_unstable_getRTCTransports" | "getAccessToken"
|
||||
> &
|
||||
OpenIDClientParts,
|
||||
ownMembershipIdentity: CallMembershipIdentityParts,
|
||||
roomId: string,
|
||||
): LocalTransport {
|
||||
// Ensure we can authenticate with the SFU.
|
||||
const authenticatedOldestMemberTransport$ = oldestMemberTransport$.pipe(
|
||||
switchMap((transport) => {
|
||||
// Oldest member not available -we are first- (or invalid SFU config).
|
||||
if (transport === null) return of(null);
|
||||
|
||||
// Whenever there is transport change we want to revert
|
||||
// to no transport while we do the authentication.
|
||||
// So do a from(promise) here to be able to startWith(null)
|
||||
return from(
|
||||
doOpenIdAndJWTFromUrl(
|
||||
transport,
|
||||
JwtEndpointVersion.Legacy,
|
||||
ownMembershipIdentity,
|
||||
roomId,
|
||||
client,
|
||||
undefined,
|
||||
),
|
||||
).pipe(
|
||||
catchError((e: unknown) => {
|
||||
logger.error(
|
||||
`Failed to authenticate to transport ${transport.livekit_service_url}`,
|
||||
e,
|
||||
);
|
||||
throw mapAuthErrorToUserFriendlyError(e);
|
||||
}),
|
||||
startWith(null),
|
||||
);
|
||||
}),
|
||||
);
|
||||
|
||||
// --- Oldest member mode ---
|
||||
return {
|
||||
// Never update the transport that we advertise in our membership. Just
|
||||
// take the first valid oldest member or preferred transport that we learn
|
||||
// about, and stick with that. This avoids unnecessary SFU hops and room
|
||||
// state changes.
|
||||
advertised$: scope.behavior(
|
||||
merge(
|
||||
authenticatedOldestMemberTransport$.pipe(
|
||||
map((t) => t?.transport ?? null),
|
||||
),
|
||||
preferredTransport$.pipe(map((t) => t.transport)),
|
||||
).pipe(
|
||||
first((t) => t !== null),
|
||||
tap((t) =>
|
||||
logger.info(`Advertise transport: ${t.livekit_service_url}`),
|
||||
),
|
||||
),
|
||||
null,
|
||||
),
|
||||
// Publish on the transport used by the oldest member.
|
||||
active$: scope.behavior(
|
||||
authenticatedOldestMemberTransport$.pipe(
|
||||
tap((t) =>
|
||||
logger.info(
|
||||
`Publish on transport: ${t?.transport.livekit_service_url}`,
|
||||
),
|
||||
),
|
||||
),
|
||||
null,
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
function mapAuthErrorToUserFriendlyError(e: unknown): Error {
|
||||
if (
|
||||
e instanceof FailToGetOpenIdToken ||
|
||||
e instanceof NoMatrix2AuthorizationService
|
||||
) {
|
||||
// rethrow as is
|
||||
return e;
|
||||
}
|
||||
// Catch others and rethrow as FailToGetOpenIdToken that has user friendly message.
|
||||
return new FailToGetOpenIdToken(
|
||||
e instanceof Error ? e : new Error(String(e)),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,233 @@
|
||||
/*
|
||||
Copyright 2025 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import {
|
||||
beforeEach,
|
||||
describe,
|
||||
expect,
|
||||
it,
|
||||
type MockedObject,
|
||||
vi,
|
||||
} from "vitest";
|
||||
import { type IClientWellKnown, MatrixError } from "matrix-js-sdk";
|
||||
import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
|
||||
import {
|
||||
type LivekitTransportConfig,
|
||||
type Transport,
|
||||
} from "matrix-js-sdk/lib/matrixrtc";
|
||||
|
||||
import type { ResolvedConfigOptions } from "../../../config/ConfigOptions.ts";
|
||||
import {
|
||||
RtcTransportAutoDiscovery,
|
||||
type RtcTransportAutoDiscoveryProps,
|
||||
} from "./RtcTransportAutoDiscovery.ts";
|
||||
|
||||
type DiscoveryClient = RtcTransportAutoDiscoveryProps["client"];
|
||||
|
||||
const backendTransport: LivekitTransportConfig = {
|
||||
type: "livekit",
|
||||
livekit_service_url: "https://backend.example.org",
|
||||
};
|
||||
|
||||
const wellKnownTransport: LivekitTransportConfig = {
|
||||
type: "livekit",
|
||||
livekit_service_url: "https://well-known.example.org",
|
||||
};
|
||||
|
||||
function makeClient(): MockedObject<DiscoveryClient> {
|
||||
return {
|
||||
getDomain: vi.fn().mockReturnValue("example.org"),
|
||||
baseUrl: "https://matrix.example.org",
|
||||
_unstable_getRTCTransports: vi.fn().mockResolvedValue([]),
|
||||
getAccessToken: vi.fn().mockReturnValue("access_token"),
|
||||
getOpenIdToken: vi.fn(),
|
||||
getDeviceId: vi.fn(),
|
||||
} as unknown as MockedObject<DiscoveryClient>;
|
||||
}
|
||||
|
||||
function makeResolvedConfig(livekitServiceUrl?: string): ResolvedConfigOptions {
|
||||
return {
|
||||
livekit: livekitServiceUrl
|
||||
? {
|
||||
livekit_service_url: livekitServiceUrl,
|
||||
}
|
||||
: undefined,
|
||||
} as ResolvedConfigOptions;
|
||||
}
|
||||
|
||||
function makeWellKnown(rtcFoci?: Transport[]): IClientWellKnown {
|
||||
return {
|
||||
"org.matrix.msc4143.rtc_foci": rtcFoci,
|
||||
} as unknown as IClientWellKnown;
|
||||
}
|
||||
|
||||
describe("RtcTransportAutoDiscovery", () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
const VALID_TEST_CASES: Array<{ transports: Transport[] }> = [
|
||||
{ transports: [backendTransport] },
|
||||
// will pick the first livekit transport in the list, even if there are other non-livekit transports
|
||||
{ transports: [{ type: "not_livekit" }, backendTransport] },
|
||||
];
|
||||
it.each(VALID_TEST_CASES)(
|
||||
"prefers backend transport over well-known and app config $transports",
|
||||
async ({ transports }) => {
|
||||
// it("prefers backend transport over well-known and app config", async () => {
|
||||
const client = makeClient();
|
||||
client._unstable_getRTCTransports.mockResolvedValue(transports);
|
||||
|
||||
const wellKnownFetcher = vi
|
||||
.fn<(domain: string) => Promise<IClientWellKnown>>()
|
||||
.mockResolvedValue(makeWellKnown([wellKnownTransport]));
|
||||
|
||||
const discovery = new RtcTransportAutoDiscovery({
|
||||
client,
|
||||
resolvedConfig: makeResolvedConfig("https://config.example.org"),
|
||||
wellKnownFetcher,
|
||||
logger: rootLogger,
|
||||
});
|
||||
|
||||
await expect(
|
||||
discovery.discoverPreferredTransport(),
|
||||
).resolves.toStrictEqual(backendTransport);
|
||||
|
||||
expect(client._unstable_getRTCTransports).toHaveBeenCalledTimes(1);
|
||||
expect(wellKnownFetcher).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
|
||||
it("Retries limit_exceeded backend transport over well-known", async () => {
|
||||
const client = makeClient();
|
||||
client._unstable_getRTCTransports
|
||||
.mockRejectedValueOnce(
|
||||
new MatrixError(
|
||||
{
|
||||
errcode: "M_LIMIT_EXCEEDED",
|
||||
error: "Too many requests",
|
||||
retry_after_ms: 100,
|
||||
},
|
||||
429,
|
||||
),
|
||||
)
|
||||
.mockResolvedValue([backendTransport]);
|
||||
|
||||
const wellKnownFetcher = vi
|
||||
.fn<(domain: string) => Promise<IClientWellKnown>>()
|
||||
.mockResolvedValue(makeWellKnown([wellKnownTransport]));
|
||||
|
||||
const discovery = new RtcTransportAutoDiscovery({
|
||||
client,
|
||||
resolvedConfig: makeResolvedConfig("https://config.example.org"),
|
||||
wellKnownFetcher,
|
||||
logger: rootLogger,
|
||||
});
|
||||
|
||||
await expect(discovery.discoverPreferredTransport()).resolves.toStrictEqual(
|
||||
backendTransport,
|
||||
);
|
||||
|
||||
expect(client._unstable_getRTCTransports).toHaveBeenCalledTimes(2);
|
||||
expect(wellKnownFetcher).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
const INVALID_TEST_CASES: Array<{ transports: Transport[] }> = [
|
||||
{ transports: [] },
|
||||
{ transports: [{ type: "not_livekit" }] },
|
||||
];
|
||||
it.each(INVALID_TEST_CASES)(
|
||||
"falls back to well-known when backend has no (valid) livekit transports $transports",
|
||||
async ({ transports }) => {
|
||||
const client = makeClient();
|
||||
client._unstable_getRTCTransports.mockResolvedValue(transports);
|
||||
|
||||
const wellKnownFetcher = vi
|
||||
.fn<(domain: string) => Promise<IClientWellKnown>>()
|
||||
.mockResolvedValue(makeWellKnown([wellKnownTransport]));
|
||||
|
||||
const discovery = new RtcTransportAutoDiscovery({
|
||||
client,
|
||||
resolvedConfig: makeResolvedConfig("https://config.example.org"),
|
||||
wellKnownFetcher,
|
||||
logger: rootLogger,
|
||||
});
|
||||
|
||||
await expect(
|
||||
discovery.discoverPreferredTransport(),
|
||||
).resolves.toStrictEqual(wellKnownTransport);
|
||||
|
||||
expect(wellKnownFetcher).toHaveBeenCalledWith("example.org");
|
||||
},
|
||||
);
|
||||
|
||||
it("skips backend discovery in widget mode and uses well-known", async () => {
|
||||
const client = makeClient();
|
||||
// widget mode is detected by the absence of an access token
|
||||
client.getAccessToken.mockReturnValue(null);
|
||||
|
||||
const wellKnownFetcher = vi
|
||||
.fn<(domain: string) => Promise<IClientWellKnown>>()
|
||||
.mockResolvedValue(makeWellKnown([wellKnownTransport]));
|
||||
|
||||
const discovery = new RtcTransportAutoDiscovery({
|
||||
client,
|
||||
resolvedConfig: makeResolvedConfig("https://config.example.org"),
|
||||
wellKnownFetcher,
|
||||
logger: rootLogger,
|
||||
});
|
||||
|
||||
await expect(discovery.discoverPreferredTransport()).resolves.toStrictEqual(
|
||||
wellKnownTransport,
|
||||
);
|
||||
|
||||
expect(client._unstable_getRTCTransports).not.toHaveBeenCalled();
|
||||
expect(wellKnownFetcher).toHaveBeenCalledWith("example.org");
|
||||
});
|
||||
|
||||
it("falls back to app config when backend fails and well-known has no rtc_foci", async () => {
|
||||
const client = makeClient();
|
||||
client._unstable_getRTCTransports.mockRejectedValue(
|
||||
new MatrixError({ errcode: "M_UNKNOWN" }, 404),
|
||||
);
|
||||
|
||||
const wellKnownFetcher = vi
|
||||
.fn<(domain: string) => Promise<IClientWellKnown>>()
|
||||
.mockResolvedValue({} as IClientWellKnown);
|
||||
|
||||
const discovery = new RtcTransportAutoDiscovery({
|
||||
client,
|
||||
resolvedConfig: makeResolvedConfig("https://config.example.org"),
|
||||
wellKnownFetcher,
|
||||
logger: rootLogger,
|
||||
});
|
||||
|
||||
await expect(discovery.discoverPreferredTransport()).resolves.toStrictEqual(
|
||||
{
|
||||
type: "livekit",
|
||||
livekit_service_url: "https://config.example.org",
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it("returns null when backend, well-known and config are all unavailable", async () => {
|
||||
const client = makeClient();
|
||||
client._unstable_getRTCTransports.mockResolvedValue([]);
|
||||
|
||||
const wellKnownFetcher = vi
|
||||
.fn<(domain: string) => Promise<IClientWellKnown>>()
|
||||
.mockResolvedValue({} as IClientWellKnown);
|
||||
|
||||
const discovery = new RtcTransportAutoDiscovery({
|
||||
client,
|
||||
resolvedConfig: makeResolvedConfig(undefined),
|
||||
wellKnownFetcher,
|
||||
logger: rootLogger,
|
||||
});
|
||||
|
||||
await expect(discovery.discoverPreferredTransport()).resolves.toBeNull();
|
||||
});
|
||||
});
|
||||
172
src/state/CallViewModel/localMember/RtcTransportAutoDiscovery.ts
Normal file
172
src/state/CallViewModel/localMember/RtcTransportAutoDiscovery.ts
Normal file
@@ -0,0 +1,172 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-IdFentifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
import {
|
||||
isLivekitTransportConfig,
|
||||
type LivekitTransportConfig,
|
||||
} from "matrix-js-sdk/lib/matrixrtc";
|
||||
import { type IClientWellKnown, type MatrixClient } from "matrix-js-sdk";
|
||||
import { type Logger } from "matrix-js-sdk/lib/logger";
|
||||
|
||||
import type { ResolvedConfigOptions } from "../../../config/ConfigOptions.ts";
|
||||
import { doNetworkOperationWithRetry } from "../../../utils/matrix.ts";
|
||||
|
||||
type TransportDiscoveryClient = Pick<
|
||||
MatrixClient,
|
||||
"getDomain" | "_unstable_getRTCTransports" | "getAccessToken"
|
||||
>;
|
||||
|
||||
export interface RtcTransportAutoDiscoveryProps {
|
||||
client: TransportDiscoveryClient;
|
||||
resolvedConfig: ResolvedConfigOptions;
|
||||
wellKnownFetcher: (domain: string) => Promise<IClientWellKnown>;
|
||||
logger: Logger;
|
||||
}
|
||||
|
||||
export class RtcTransportAutoDiscovery {
|
||||
private readonly client: TransportDiscoveryClient;
|
||||
private readonly resolvedConfig: ResolvedConfigOptions;
|
||||
private readonly wellKnownFetcher: (
|
||||
domain: string,
|
||||
) => Promise<IClientWellKnown>;
|
||||
private readonly logger: Logger;
|
||||
|
||||
public constructor({
|
||||
client,
|
||||
resolvedConfig,
|
||||
wellKnownFetcher,
|
||||
logger,
|
||||
}: RtcTransportAutoDiscoveryProps) {
|
||||
this.client = client;
|
||||
this.resolvedConfig = resolvedConfig;
|
||||
this.wellKnownFetcher = wellKnownFetcher;
|
||||
this.logger = logger.getChild("[RtcTransportAutoDiscovery]");
|
||||
}
|
||||
|
||||
public async discoverPreferredTransport(): Promise<LivekitTransportConfig | null> {
|
||||
// 1) backend transports
|
||||
const backendTransport = await this.tryBackendTransports();
|
||||
if (backendTransport) {
|
||||
this.logger.info(
|
||||
`Found backend transport: ${backendTransport.livekit_service_url}`,
|
||||
);
|
||||
return backendTransport;
|
||||
}
|
||||
|
||||
this.logger.info("No backend transport found, falling back to well-known");
|
||||
// 2) .well-known transports
|
||||
const wellKnownTransport = await this.tryWellKnownTransports();
|
||||
if (wellKnownTransport) {
|
||||
this.logger.info(
|
||||
`Found .well-known transport: ${wellKnownTransport.livekit_service_url}`,
|
||||
);
|
||||
return wellKnownTransport;
|
||||
}
|
||||
|
||||
this.logger.info(
|
||||
"No .well-known transport found, falling back to app config",
|
||||
);
|
||||
|
||||
// 3) app config URL
|
||||
const configTransport = this.tryConfigTransport();
|
||||
if (configTransport) {
|
||||
this.logger.info(
|
||||
`Found app config transport: ${configTransport.livekit_service_url}`,
|
||||
);
|
||||
return configTransport;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches the first rtc_foci from the backend.
|
||||
* This will not throw errors, but instead just log them and return null if the expected config is not found or malformed.
|
||||
* @private
|
||||
*/
|
||||
private async tryBackendTransports(): Promise<LivekitTransportConfig | null> {
|
||||
const client = this.client;
|
||||
// MSC4143: Attempt to fetch transports from backend.
|
||||
// TODO: Workaround for an issue in the js-sdk RoomWidgetClient that
|
||||
// is not yet implementing _unstable_getRTCTransports properly (via widget API new action).
|
||||
// For now we just skip this call if we are in a widget.
|
||||
// In widget mode the client is a `RoomWidgetClient` which has no access token (it is using the widget API).
|
||||
// Could be removed once the js-sdk is fixed (https://github.com/matrix-org/matrix-js-sdk/issues/5245)
|
||||
const isSPA = !!client.getAccessToken();
|
||||
if (isSPA && "_unstable_getRTCTransports" in client) {
|
||||
this.logger.info("First try to use getRTCTransports end point ...");
|
||||
try {
|
||||
const transportList = await doNetworkOperationWithRetry(async () =>
|
||||
client._unstable_getRTCTransports(),
|
||||
);
|
||||
const first = transportList.filter(isLivekitTransportConfig)[0];
|
||||
if (first) {
|
||||
return first;
|
||||
} else {
|
||||
this.logger.info(
|
||||
`No livekit transport found in getRTCTransports end point`,
|
||||
transportList,
|
||||
);
|
||||
}
|
||||
} catch (ex) {
|
||||
this.logger.info(`Failed to use getRTCTransports end point: ${ex}`);
|
||||
}
|
||||
} else {
|
||||
this.logger.debug(`getRTCTransports end point not available`);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches the first rtc_foci from the .well-known/matrix/client.
|
||||
* This will not throw errors, but instead just log them and return null if the expected config is not found or malformed.
|
||||
* @private
|
||||
*/
|
||||
private async tryWellKnownTransports(): Promise<LivekitTransportConfig | null> {
|
||||
// Legacy MSC4143 (to be removed) WELL_KNOWN: Prioritize the .well-known/matrix/client, if available.
|
||||
const client = this.client;
|
||||
const domain = client.getDomain();
|
||||
if (domain) {
|
||||
// we use AutoDiscovery instead of relying on the MatrixClient having already
|
||||
// been fully configured and started
|
||||
|
||||
const wellKnownFoci = await this.wellKnownFetcher(domain);
|
||||
|
||||
const fociConfig = wellKnownFoci["org.matrix.msc4143.rtc_foci"];
|
||||
if (fociConfig) {
|
||||
if (!Array.isArray(fociConfig)) {
|
||||
this.logger.warn(
|
||||
`org.matrix.msc4143.rtc_foci is not an array in .well-known`,
|
||||
);
|
||||
} else {
|
||||
return fociConfig[0];
|
||||
}
|
||||
} else {
|
||||
this.logger.info(
|
||||
`No .well-known "org.matrix.msc4143.rtc_foci" found for ${domain}`,
|
||||
wellKnownFoci,
|
||||
);
|
||||
}
|
||||
} else {
|
||||
// Should never happen, but just in case
|
||||
this.logger.warn(`No domain configured for client`);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private tryConfigTransport(): LivekitTransportConfig | null {
|
||||
const url = this.resolvedConfig.livekit?.livekit_service_url;
|
||||
if (url) {
|
||||
return {
|
||||
type: "livekit",
|
||||
livekit_service_url: url,
|
||||
};
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,7 @@ Please see LICENSE in the repository root for full details.
|
||||
|
||||
import {
|
||||
afterEach,
|
||||
beforeEach,
|
||||
describe,
|
||||
expect,
|
||||
it,
|
||||
@@ -151,6 +152,19 @@ afterEach(() => {
|
||||
});
|
||||
|
||||
describe("Start connection states", () => {
|
||||
beforeEach(() => {
|
||||
fetchMock.post(
|
||||
`https://matrix-rtc.example.org/livekit/jwt/get_token`,
|
||||
() => {
|
||||
return {
|
||||
// Return a non-retryable error, if not, the retry logic will
|
||||
// wait and fail the test with a timeout.
|
||||
status: 404,
|
||||
};
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it("start in initialized state", () => {
|
||||
setupTest();
|
||||
|
||||
|
||||
@@ -54,31 +54,6 @@ export function createRoomMembers$(
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* creates the member that this DM is with in case it is a DM (two members) otherwise null
|
||||
*/
|
||||
export function createDMMember$(
|
||||
scope: ObservableScope,
|
||||
roomMembers$: Behavior<RoomMemberMap>,
|
||||
matrixRoom: MatrixRoom,
|
||||
): Behavior<Pick<
|
||||
RoomMember,
|
||||
"userId" | "getMxcAvatarUrl" | "rawDisplayName"
|
||||
> | null> {
|
||||
// We cannot use the normal direct check from matrix since we do not have access to the account data.
|
||||
// use primitive member count === 2 check instead.
|
||||
return scope.behavior(
|
||||
roomMembers$.pipe(
|
||||
map((membersMap) => {
|
||||
// primitive appraoch do to no access to account data.
|
||||
const isDM = membersMap.size === 2;
|
||||
if (!isDM) return null;
|
||||
return matrixRoom.getMember(matrixRoom.guessDMUserId());
|
||||
}),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Displayname for each member of the call. This will disambiguate
|
||||
* any displayname that clashes with another member. Only members
|
||||
|
||||
132
src/state/IOSControlledAudioOutput.ts
Normal file
132
src/state/IOSControlledAudioOutput.ts
Normal file
@@ -0,0 +1,132 @@
|
||||
/*
|
||||
Copyright 2026 Element Corp.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
|
||||
import { combineLatest, merge, startWith, Subject, tap } from "rxjs";
|
||||
|
||||
import {
|
||||
availableOutputDevices$ as controlledAvailableOutputDevices$,
|
||||
outputDevice$ as controlledOutputSelection$,
|
||||
} from "../controls.ts";
|
||||
import type { Behavior } from "./Behavior.ts";
|
||||
import type { ObservableScope } from "./ObservableScope.ts";
|
||||
import {
|
||||
type AudioOutputDeviceLabel,
|
||||
availableRawDevices$,
|
||||
iosDeviceMenu$,
|
||||
type MediaDevice,
|
||||
type SelectedAudioOutputDevice,
|
||||
} from "./MediaDevices.ts";
|
||||
|
||||
// This hardcoded id is used in EX ios! It can only be changed in coordination with
|
||||
// the ios swift team.
|
||||
const EARPIECE_CONFIG_ID = "earpiece-id";
|
||||
|
||||
/**
|
||||
* A special implementation of audio output that allows the hosting application
|
||||
* to have more control over the device selection process. This is used when the
|
||||
* `controlledAudioDevices` URL parameter is set, which is currently only true on mobile.
|
||||
*/
|
||||
export class IOSControlledAudioOutput implements MediaDevice<
|
||||
AudioOutputDeviceLabel,
|
||||
SelectedAudioOutputDevice
|
||||
> {
|
||||
private logger = rootLogger.getChild("[MediaDevices ControlledAudioOutput]");
|
||||
// We need to subscribe to the raw devices so that the OS does update the input
|
||||
// back to what it was before. otherwise we will switch back to the default
|
||||
// whenever we allocate a new stream.
|
||||
public readonly availableRaw$ = availableRawDevices$(
|
||||
"audiooutput",
|
||||
this.usingNames$,
|
||||
this.scope,
|
||||
this.logger,
|
||||
);
|
||||
|
||||
public readonly available$ = this.scope.behavior(
|
||||
combineLatest(
|
||||
[controlledAvailableOutputDevices$.pipe(startWith([])), iosDeviceMenu$],
|
||||
(availableRaw, iosDeviceMenu) => {
|
||||
const available = new Map<string, AudioOutputDeviceLabel>(
|
||||
availableRaw.map(
|
||||
({ id, name, isEarpiece, isSpeaker /*,isExternalHeadset*/ }) => {
|
||||
let deviceLabel: AudioOutputDeviceLabel;
|
||||
// if (isExternalHeadset) // Do we want this?
|
||||
if (isEarpiece) deviceLabel = { type: "earpiece" };
|
||||
else if (isSpeaker) deviceLabel = { type: "speaker" };
|
||||
else deviceLabel = { type: "name", name };
|
||||
return [id, deviceLabel];
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
// Create a virtual earpiece device in case a non-earpiece device is
|
||||
// designated for this purpose
|
||||
if (iosDeviceMenu && availableRaw.some((d) => d.forEarpiece)) {
|
||||
this.logger.info(
|
||||
`IOS Add virtual earpiece device with id ${EARPIECE_CONFIG_ID}`,
|
||||
);
|
||||
available.set(EARPIECE_CONFIG_ID, { type: "earpiece" });
|
||||
}
|
||||
|
||||
return available;
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
private readonly deviceSelection$ = new Subject<string>();
|
||||
|
||||
public select(id: string): void {
|
||||
this.logger.info(`select device: ${id}`);
|
||||
this.deviceSelection$.next(id);
|
||||
}
|
||||
|
||||
public readonly selected$ = this.scope.behavior(
|
||||
combineLatest(
|
||||
[
|
||||
this.available$,
|
||||
merge(
|
||||
controlledOutputSelection$.pipe(startWith(undefined)),
|
||||
this.deviceSelection$,
|
||||
),
|
||||
],
|
||||
(available, preferredId) => {
|
||||
const id = preferredId ?? available.keys().next().value;
|
||||
return id === undefined
|
||||
? undefined
|
||||
: { id, virtualEarpiece: id === EARPIECE_CONFIG_ID };
|
||||
},
|
||||
).pipe(
|
||||
tap((selected) => {
|
||||
this.logger.debug(`selected device: ${selected?.id}`);
|
||||
}),
|
||||
),
|
||||
);
|
||||
|
||||
public constructor(
|
||||
private readonly usingNames$: Behavior<boolean>,
|
||||
private readonly scope: ObservableScope,
|
||||
) {
|
||||
this.selected$.subscribe((device) => {
|
||||
// Let the hosting application know which output device has been selected.
|
||||
// This information is probably only of interest if the earpiece mode has
|
||||
// been selected - for example, Element X iOS listens to this to determine
|
||||
// whether it should enable the proximity sensor.
|
||||
if (device !== undefined) {
|
||||
this.logger.info("onAudioDeviceSelect called:", device);
|
||||
window.controls.onAudioDeviceSelect?.(device.id);
|
||||
// Also invoke the deprecated callback for backward compatibility
|
||||
window.controls.onOutputDeviceSelect?.(device.id);
|
||||
}
|
||||
});
|
||||
this.available$.subscribe((available) => {
|
||||
this.logger.debug("available devices:", available);
|
||||
});
|
||||
this.availableRaw$.subscribe((availableRaw) => {
|
||||
this.logger.debug("available raw devices:", availableRaw);
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -9,35 +9,28 @@ import {
|
||||
combineLatest,
|
||||
filter,
|
||||
map,
|
||||
merge,
|
||||
type Observable,
|
||||
pairwise,
|
||||
startWith,
|
||||
Subject,
|
||||
switchMap,
|
||||
type Observable,
|
||||
} from "rxjs";
|
||||
import { createMediaDeviceObserver } from "@livekit/components-core";
|
||||
import { type Logger, logger as rootLogger } from "matrix-js-sdk/lib/logger";
|
||||
|
||||
import {
|
||||
alwaysShowIphoneEarpiece as alwaysShowIphoneEarpieceSetting,
|
||||
audioInput as audioInputSetting,
|
||||
audioOutput as audioOutputSetting,
|
||||
videoInput as videoInputSetting,
|
||||
alwaysShowIphoneEarpiece as alwaysShowIphoneEarpieceSetting,
|
||||
} from "../settings/settings";
|
||||
import { type ObservableScope } from "./ObservableScope";
|
||||
import {
|
||||
outputDevice$ as controlledOutputSelection$,
|
||||
availableOutputDevices$ as controlledAvailableOutputDevices$,
|
||||
} from "../controls";
|
||||
import { availableOutputDevices$ as controlledAvailableOutputDevices$ } from "../controls";
|
||||
import { getUrlParams } from "../UrlParams";
|
||||
import { platform } from "../Platform";
|
||||
import { switchWhen } from "../utils/observable";
|
||||
import { type Behavior, constant } from "./Behavior";
|
||||
|
||||
// This hardcoded id is used in EX ios! It can only be changed in coordination with
|
||||
// the ios swift team.
|
||||
const EARPIECE_CONFIG_ID = "earpiece-id";
|
||||
import { AndroidControlledAudioOutput } from "./AndroidControlledAudioOutput.ts";
|
||||
import { IOSControlledAudioOutput } from "./IOSControlledAudioOutput.ts";
|
||||
|
||||
export type DeviceLabel =
|
||||
| { type: "name"; name: string }
|
||||
@@ -49,10 +42,18 @@ export type AudioOutputDeviceLabel =
|
||||
| { type: "earpiece" }
|
||||
| { type: "default"; name: string | null };
|
||||
|
||||
/**
|
||||
* Base selected-device value shared by all media kinds.
|
||||
*
|
||||
* `id` is the effective device identifier used by browser media APIs.
|
||||
*/
|
||||
export interface SelectedDevice {
|
||||
id: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Selected audio input value with audio-input-specific metadata.
|
||||
*/
|
||||
export interface SelectedAudioInputDevice extends SelectedDevice {
|
||||
/**
|
||||
* Emits whenever we think that this audio input device has logically changed
|
||||
@@ -61,6 +62,9 @@ export interface SelectedAudioInputDevice extends SelectedDevice {
|
||||
hardwareDeviceChange$: Observable<void>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Selected audio output value with output-routing-specific metadata.
|
||||
*/
|
||||
export interface SelectedAudioOutputDevice extends SelectedDevice {
|
||||
/**
|
||||
* Whether this device is a "virtual earpiece" device. If so, we should output
|
||||
@@ -69,23 +73,42 @@ export interface SelectedAudioOutputDevice extends SelectedDevice {
|
||||
virtualEarpiece: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Common reactive contract for selectable input/output media devices (mic, speaker, camera).
|
||||
*
|
||||
* `Label` is the type used to represent a device in UI lists.
|
||||
* `Selected` is the type used to represent the active selection for a device kind.
|
||||
*/
|
||||
export interface MediaDevice<Label, Selected> {
|
||||
/**
|
||||
* A map from available device IDs to labels.
|
||||
* Reactive map of currently available devices keyed by device ID.
|
||||
*
|
||||
* `Label` defines the UI-facing label data structure for each device type.
|
||||
*/
|
||||
available$: Behavior<Map<string, Label>>;
|
||||
|
||||
/**
|
||||
* The selected device.
|
||||
* The active device selection.
|
||||
* Can be `undefined` when no device is yet selected.
|
||||
*
|
||||
* When defined, `Selected` contains the selected device ID plus any
|
||||
* type-specific metadata.
|
||||
*/
|
||||
selected$: Behavior<Selected | undefined>;
|
||||
|
||||
/**
|
||||
* Selects a new device.
|
||||
* Requests selection of a device by ID.
|
||||
*
|
||||
* Implementations typically persist this preference and let `selected$`
|
||||
* converge to the effective device (which may differ if the requested ID is
|
||||
* unavailable).
|
||||
*/
|
||||
select(id: string): void;
|
||||
}
|
||||
|
||||
/**
|
||||
* An observable that represents if we should display the devices menu for iOS.
|
||||
*
|
||||
* This implies the following
|
||||
* - hide any input devices (they do not work anyhow on ios)
|
||||
* - Show a button to show the native output picker instead.
|
||||
@@ -95,7 +118,7 @@ export interface MediaDevice<Label, Selected> {
|
||||
export const iosDeviceMenu$ =
|
||||
platform === "ios" ? constant(true) : alwaysShowIphoneEarpieceSetting.value$;
|
||||
|
||||
function availableRawDevices$(
|
||||
export function availableRawDevices$(
|
||||
kind: MediaDeviceKind,
|
||||
usingNames$: Behavior<boolean>,
|
||||
scope: ObservableScope,
|
||||
@@ -146,16 +169,23 @@ function selectDevice$<Label>(
|
||||
): Observable<string | undefined> {
|
||||
return combineLatest([available$, preferredId$], (available, preferredId) => {
|
||||
if (available.size) {
|
||||
// If the preferred device is available, use it. Or if every available
|
||||
// device ID is falsy, the browser is probably just being paranoid about
|
||||
// fingerprinting and we should still try using the preferred device.
|
||||
// Worst case it is not available and the browser will gracefully fall
|
||||
// back to some other device for us when requesting the media stream.
|
||||
// Otherwise, select the first available device.
|
||||
return (preferredId !== undefined && available.has(preferredId)) ||
|
||||
(available.size === 1 && available.has(""))
|
||||
? preferredId
|
||||
: available.keys().next().value;
|
||||
if (preferredId !== undefined && available.has(preferredId)) {
|
||||
// If the preferred device is available, use it.
|
||||
return preferredId;
|
||||
} else if (available.size === 1 && available.has("")) {
|
||||
// In some cases the enumerateDevices will list the devices with empty string details:
|
||||
// `{deviceId:'', kind:'audiooutput|audioinput|videoinput', label:'', groupId:''}`
|
||||
// This can happen when:
|
||||
// 1. The user has not yet granted permissions to microphone/devices
|
||||
// 2. The page is not running in a secure context (e.g. localhost or https)
|
||||
// 3. In embedded WebViews, restrictions are often tighter, need active capture..
|
||||
// 3. The browser is blocking access to device details for privacy reasons (?)
|
||||
// This is most likely transitional, so keep the current device selected until we get a more accurate enumerateDevices.
|
||||
return preferredId;
|
||||
} else {
|
||||
// No preferred, so pick a default.
|
||||
return available.keys().next().value;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
});
|
||||
@@ -212,7 +242,7 @@ class AudioInput implements MediaDevice<DeviceLabel, SelectedAudioInputDevice> {
|
||||
}
|
||||
}
|
||||
|
||||
class AudioOutput implements MediaDevice<
|
||||
export class AudioOutput implements MediaDevice<
|
||||
AudioOutputDeviceLabel,
|
||||
SelectedAudioOutputDevice
|
||||
> {
|
||||
@@ -251,14 +281,16 @@ class AudioOutput implements MediaDevice<
|
||||
|
||||
public readonly selected$ = this.scope.behavior(
|
||||
selectDevice$(this.available$, audioOutputSetting.value$).pipe(
|
||||
map((id) =>
|
||||
id === undefined
|
||||
? undefined
|
||||
: {
|
||||
id,
|
||||
virtualEarpiece: false,
|
||||
},
|
||||
),
|
||||
map((id) => {
|
||||
if (id === undefined) {
|
||||
return undefined;
|
||||
} else {
|
||||
return {
|
||||
id,
|
||||
virtualEarpiece: false,
|
||||
};
|
||||
}
|
||||
}),
|
||||
),
|
||||
);
|
||||
public select(id: string): void {
|
||||
@@ -275,103 +307,6 @@ class AudioOutput implements MediaDevice<
|
||||
}
|
||||
}
|
||||
|
||||
class ControlledAudioOutput implements MediaDevice<
|
||||
AudioOutputDeviceLabel,
|
||||
SelectedAudioOutputDevice
|
||||
> {
|
||||
private logger = rootLogger.getChild("[MediaDevices ControlledAudioOutput]");
|
||||
// We need to subscribe to the raw devices so that the OS does update the input
|
||||
// back to what it was before. otherwise we will switch back to the default
|
||||
// whenever we allocate a new stream.
|
||||
public readonly availableRaw$ = availableRawDevices$(
|
||||
"audiooutput",
|
||||
this.usingNames$,
|
||||
this.scope,
|
||||
this.logger,
|
||||
);
|
||||
|
||||
public readonly available$ = this.scope.behavior(
|
||||
combineLatest(
|
||||
[controlledAvailableOutputDevices$.pipe(startWith([])), iosDeviceMenu$],
|
||||
(availableRaw, iosDeviceMenu) => {
|
||||
const available = new Map<string, AudioOutputDeviceLabel>(
|
||||
availableRaw.map(
|
||||
({ id, name, isEarpiece, isSpeaker /*,isExternalHeadset*/ }) => {
|
||||
let deviceLabel: AudioOutputDeviceLabel;
|
||||
// if (isExternalHeadset) // Do we want this?
|
||||
if (isEarpiece) deviceLabel = { type: "earpiece" };
|
||||
else if (isSpeaker) deviceLabel = { type: "speaker" };
|
||||
else deviceLabel = { type: "name", name };
|
||||
return [id, deviceLabel];
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
// Create a virtual earpiece device in case a non-earpiece device is
|
||||
// designated for this purpose
|
||||
if (iosDeviceMenu && availableRaw.some((d) => d.forEarpiece))
|
||||
available.set(EARPIECE_CONFIG_ID, { type: "earpiece" });
|
||||
|
||||
return available;
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
private readonly deviceSelection$ = new Subject<string>();
|
||||
|
||||
public select(id: string): void {
|
||||
this.deviceSelection$.next(id);
|
||||
}
|
||||
|
||||
public readonly selected$ = this.scope.behavior(
|
||||
combineLatest(
|
||||
[
|
||||
this.available$,
|
||||
merge(
|
||||
controlledOutputSelection$.pipe(startWith(undefined)),
|
||||
this.deviceSelection$,
|
||||
),
|
||||
],
|
||||
(available, preferredId) => {
|
||||
const id = preferredId ?? available.keys().next().value;
|
||||
return id === undefined
|
||||
? undefined
|
||||
: { id, virtualEarpiece: id === EARPIECE_CONFIG_ID };
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
public constructor(
|
||||
private readonly usingNames$: Behavior<boolean>,
|
||||
private readonly scope: ObservableScope,
|
||||
) {
|
||||
this.selected$.subscribe((device) => {
|
||||
// Let the hosting application know which output device has been selected.
|
||||
// This information is probably only of interest if the earpiece mode has
|
||||
// been selected - for example, Element X iOS listens to this to determine
|
||||
// whether it should enable the proximity sensor.
|
||||
if (device !== undefined) {
|
||||
this.logger.info(
|
||||
"[controlled-output] onAudioDeviceSelect called:",
|
||||
device,
|
||||
);
|
||||
window.controls.onAudioDeviceSelect?.(device.id);
|
||||
// Also invoke the deprecated callback for backward compatibility
|
||||
window.controls.onOutputDeviceSelect?.(device.id);
|
||||
}
|
||||
});
|
||||
this.available$.subscribe((available) => {
|
||||
this.logger.info("[controlled-output] available devices:", available);
|
||||
});
|
||||
this.availableRaw$.subscribe((availableRaw) => {
|
||||
this.logger.info(
|
||||
"[controlled-output] available raw devices:",
|
||||
availableRaw,
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
class VideoInput implements MediaDevice<DeviceLabel, SelectedDevice> {
|
||||
private logger = rootLogger.getChild("[MediaDevices VideoInput]");
|
||||
|
||||
@@ -434,7 +369,14 @@ export class MediaDevices {
|
||||
AudioOutputDeviceLabel,
|
||||
SelectedAudioOutputDevice
|
||||
> = getUrlParams().controlledAudioDevices
|
||||
? new ControlledAudioOutput(this.usingNames$, this.scope)
|
||||
? platform == "android"
|
||||
? new AndroidControlledAudioOutput(
|
||||
controlledAvailableOutputDevices$,
|
||||
this.scope,
|
||||
getUrlParams().callIntent,
|
||||
window.controls,
|
||||
)
|
||||
: new IOSControlledAudioOutput(this.usingNames$, this.scope)
|
||||
: new AudioOutput(this.usingNames$, this.scope);
|
||||
|
||||
public readonly videoInput: MediaDevice<DeviceLabel, SelectedDevice> =
|
||||
|
||||
@@ -16,14 +16,14 @@ export function oneOnOneLayout(
|
||||
prevTiles: TileStore,
|
||||
): [OneOnOneLayout, TileStore] {
|
||||
const update = prevTiles.from(2);
|
||||
update.registerGridTile(media.local);
|
||||
update.registerGridTile(media.remote);
|
||||
update.registerGridTile(media.pip);
|
||||
update.registerGridTile(media.spotlight);
|
||||
const tiles = update.build();
|
||||
return [
|
||||
{
|
||||
type: media.type,
|
||||
local: tiles.gridTilesByMedia.get(media.local)!,
|
||||
remote: tiles.gridTilesByMedia.get(media.remote)!,
|
||||
spotlight: tiles.gridTilesByMedia.get(media.spotlight)!,
|
||||
pip: tiles.gridTilesByMedia.get(media.pip)!,
|
||||
},
|
||||
tiles,
|
||||
];
|
||||
|
||||
@@ -13,6 +13,7 @@ import { fillGaps } from "../utils/iter";
|
||||
import { debugTileLayout } from "../settings/settings";
|
||||
import { type MediaViewModel } from "./media/MediaViewModel";
|
||||
import { type UserMediaViewModel } from "./media/UserMediaViewModel";
|
||||
import { type RingingMediaViewModel } from "./media/RingingMediaViewModel";
|
||||
|
||||
function debugEntries(entries: GridTileData[]): string[] {
|
||||
return entries.map((e) => e.media.displayName$.value);
|
||||
@@ -48,8 +49,10 @@ class SpotlightTileData {
|
||||
}
|
||||
|
||||
class GridTileData {
|
||||
private readonly media$: BehaviorSubject<UserMediaViewModel>;
|
||||
public get media(): UserMediaViewModel {
|
||||
private readonly media$: BehaviorSubject<
|
||||
UserMediaViewModel | RingingMediaViewModel
|
||||
>;
|
||||
public get media(): UserMediaViewModel | RingingMediaViewModel {
|
||||
return this.media$.value;
|
||||
}
|
||||
public set media(value: UserMediaViewModel) {
|
||||
@@ -58,7 +61,7 @@ class GridTileData {
|
||||
|
||||
public readonly vm: GridTileViewModel;
|
||||
|
||||
public constructor(media: UserMediaViewModel) {
|
||||
public constructor(media: UserMediaViewModel | RingingMediaViewModel) {
|
||||
this.media$ = new BehaviorSubject(media);
|
||||
this.vm = new GridTileViewModel(this.media$);
|
||||
}
|
||||
@@ -178,7 +181,9 @@ export class TileStoreBuilder {
|
||||
* Sets up a grid tile for the given media. If this is never called for some
|
||||
* media, then that media will have no grid tile.
|
||||
*/
|
||||
public registerGridTile(media: UserMediaViewModel): void {
|
||||
public registerGridTile(
|
||||
media: UserMediaViewModel | RingingMediaViewModel,
|
||||
): void {
|
||||
if (DEBUG_ENABLED)
|
||||
logger.debug(
|
||||
`[TileStore, ${this.generation}] register grid tile: ${media.displayName$.value}`,
|
||||
@@ -187,7 +192,11 @@ export class TileStoreBuilder {
|
||||
if (this.spotlight !== null) {
|
||||
// We actually *don't* want spotlight speakers to appear in both the
|
||||
// spotlight and the grid, so they're filtered out here
|
||||
if (!media.local && this.spotlight.media.includes(media)) return;
|
||||
if (
|
||||
!(media.type === "user" && media.local) &&
|
||||
this.spotlight.media.includes(media)
|
||||
)
|
||||
return;
|
||||
// When the spotlight speaker changes, we would see one grid tile appear
|
||||
// and another grid tile disappear. This would be an undesirable layout
|
||||
// shift, so instead what we do is take the speaker's grid tile and swap
|
||||
|
||||
@@ -7,6 +7,7 @@ Please see LICENSE in the repository root for full details.
|
||||
|
||||
import { type Behavior } from "./Behavior";
|
||||
import { type MediaViewModel } from "./media/MediaViewModel";
|
||||
import { type RingingMediaViewModel } from "./media/RingingMediaViewModel";
|
||||
import { type UserMediaViewModel } from "./media/UserMediaViewModel";
|
||||
|
||||
let nextId = 0;
|
||||
@@ -17,7 +18,11 @@ function createId(): string {
|
||||
export class GridTileViewModel {
|
||||
public readonly id = createId();
|
||||
|
||||
public constructor(public readonly media$: Behavior<UserMediaViewModel>) {}
|
||||
public constructor(
|
||||
public readonly media$: Behavior<
|
||||
UserMediaViewModel | RingingMediaViewModel
|
||||
>,
|
||||
) {}
|
||||
}
|
||||
|
||||
export class SpotlightTileViewModel {
|
||||
|
||||
@@ -7,7 +7,7 @@ Please see LICENSE in the repository root for full details.
|
||||
|
||||
import { type LocalUserMediaViewModel } from "./media/LocalUserMediaViewModel.ts";
|
||||
import { type MediaViewModel } from "./media/MediaViewModel.ts";
|
||||
import { type RemoteUserMediaViewModel } from "./media/RemoteUserMediaViewModel.ts";
|
||||
import { type RingingMediaViewModel } from "./media/RingingMediaViewModel.ts";
|
||||
import { type UserMediaViewModel } from "./media/UserMediaViewModel.ts";
|
||||
import {
|
||||
type GridTileViewModel,
|
||||
@@ -40,8 +40,8 @@ export interface SpotlightExpandedLayoutMedia {
|
||||
|
||||
export interface OneOnOneLayoutMedia {
|
||||
type: "one-on-one";
|
||||
local: LocalUserMediaViewModel;
|
||||
remote: RemoteUserMediaViewModel;
|
||||
spotlight: UserMediaViewModel;
|
||||
pip: LocalUserMediaViewModel | RingingMediaViewModel;
|
||||
}
|
||||
|
||||
export interface PipLayoutMedia {
|
||||
@@ -86,8 +86,8 @@ export interface SpotlightExpandedLayout {
|
||||
|
||||
export interface OneOnOneLayout {
|
||||
type: "one-on-one";
|
||||
local: GridTileViewModel;
|
||||
remote: GridTileViewModel;
|
||||
spotlight: GridTileViewModel;
|
||||
pip: GridTileViewModel;
|
||||
}
|
||||
|
||||
export interface PipLayout {
|
||||
|
||||
@@ -7,13 +7,17 @@ Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { type Behavior } from "../Behavior";
|
||||
import { type RingingMediaViewModel } from "./RingingMediaViewModel";
|
||||
import { type ScreenShareViewModel } from "./ScreenShareViewModel";
|
||||
import { type UserMediaViewModel } from "./UserMediaViewModel";
|
||||
|
||||
/**
|
||||
* A participant's media.
|
||||
*/
|
||||
export type MediaViewModel = UserMediaViewModel | ScreenShareViewModel;
|
||||
export type MediaViewModel =
|
||||
| UserMediaViewModel
|
||||
| ScreenShareViewModel
|
||||
| RingingMediaViewModel;
|
||||
|
||||
/**
|
||||
* Properties which are common to all MediaViewModels.
|
||||
|
||||
@@ -38,6 +38,8 @@ import { type ObservableScope } from "../ObservableScope";
|
||||
import { observeTrackReference$ } from "../observeTrackReference";
|
||||
import { E2eeType } from "../../e2ee/e2eeType";
|
||||
import { observeInboundRtpStreamStats$ } from "./observeRtpStreamStats";
|
||||
import { type UserMediaViewModel } from "./UserMediaViewModel";
|
||||
import { type ScreenShareViewModel } from "./ScreenShareViewModel";
|
||||
|
||||
// TODO: Encryption status is kinda broken and thus unused right now. Remove?
|
||||
export enum EncryptionStatus {
|
||||
@@ -49,9 +51,9 @@ export enum EncryptionStatus {
|
||||
}
|
||||
|
||||
/**
|
||||
* Media belonging to an active member of the RTC session.
|
||||
* Properties common to all MemberMediaViewModels.
|
||||
*/
|
||||
export interface MemberMediaViewModel extends BaseMediaViewModel {
|
||||
export interface BaseMemberMediaViewModel extends BaseMediaViewModel {
|
||||
/**
|
||||
* The LiveKit video track for this media.
|
||||
*/
|
||||
@@ -88,7 +90,7 @@ export function createMemberMedia(
|
||||
encryptionSystem,
|
||||
...inputs
|
||||
}: MemberMediaInputs,
|
||||
): MemberMediaViewModel {
|
||||
): BaseMemberMediaViewModel {
|
||||
const trackBehavior$ = (
|
||||
source: Track.Source,
|
||||
): Behavior<TrackReference | undefined> =>
|
||||
@@ -270,3 +272,8 @@ function observeRemoteTrackReceivingOkay$(
|
||||
startWith(undefined),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Media belonging to an active member of the call.
|
||||
*/
|
||||
export type MemberMediaViewModel = UserMediaViewModel | ScreenShareViewModel;
|
||||
|
||||
51
src/state/media/RingingMediaViewModel.ts
Normal file
51
src/state/media/RingingMediaViewModel.ts
Normal file
@@ -0,0 +1,51 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { type Behavior } from "../Behavior";
|
||||
import { type MuteStates } from "../MuteStates";
|
||||
import {
|
||||
type BaseMediaInputs,
|
||||
type BaseMediaViewModel,
|
||||
createBaseMedia,
|
||||
} from "./MediaViewModel";
|
||||
|
||||
/**
|
||||
* Media representing a user who is not yet part of the call — one that we are
|
||||
* *ringing*.
|
||||
*/
|
||||
export interface RingingMediaViewModel extends BaseMediaViewModel {
|
||||
type: "ringing";
|
||||
pickupState$: Behavior<"ringing" | "timeout" | "decline">;
|
||||
/**
|
||||
* Whether this media would be expected to have video, were it not simply a
|
||||
* placeholder.
|
||||
*/
|
||||
videoEnabled$: Behavior<boolean>;
|
||||
}
|
||||
|
||||
export interface RingingMediaInputs extends BaseMediaInputs {
|
||||
pickupState$: Behavior<"ringing" | "timeout" | "decline">;
|
||||
/**
|
||||
* The local user's own mute states.
|
||||
*/
|
||||
muteStates: MuteStates;
|
||||
}
|
||||
|
||||
export function createRingingMedia({
|
||||
pickupState$,
|
||||
muteStates,
|
||||
...inputs
|
||||
}: RingingMediaInputs): RingingMediaViewModel {
|
||||
return {
|
||||
...createBaseMedia(inputs),
|
||||
type: "ringing",
|
||||
pickupState$,
|
||||
// If our own video is enabled, then this is a video call and we would
|
||||
// expect remote media to have video as well
|
||||
videoEnabled$: muteStates.video.enabled$,
|
||||
};
|
||||
}
|
||||
@@ -13,7 +13,7 @@ import { type LocalScreenShareViewModel } from "./LocalScreenShareViewModel";
|
||||
import {
|
||||
createMemberMedia,
|
||||
type MemberMediaInputs,
|
||||
type MemberMediaViewModel,
|
||||
type BaseMemberMediaViewModel,
|
||||
} from "./MemberMediaViewModel";
|
||||
import { type RemoteScreenShareViewModel } from "./RemoteScreenShareViewModel";
|
||||
|
||||
@@ -27,7 +27,7 @@ export type ScreenShareViewModel =
|
||||
/**
|
||||
* Properties which are common to all ScreenShareViewModels.
|
||||
*/
|
||||
export interface BaseScreenShareViewModel extends MemberMediaViewModel {
|
||||
export interface BaseScreenShareViewModel extends BaseMemberMediaViewModel {
|
||||
type: "screen share";
|
||||
}
|
||||
|
||||
|
||||
@@ -27,7 +27,7 @@ import { type LocalUserMediaViewModel } from "./LocalUserMediaViewModel";
|
||||
import {
|
||||
createMemberMedia,
|
||||
type MemberMediaInputs,
|
||||
type MemberMediaViewModel,
|
||||
type BaseMemberMediaViewModel,
|
||||
} from "./MemberMediaViewModel";
|
||||
import { type RemoteUserMediaViewModel } from "./RemoteUserMediaViewModel";
|
||||
import { type ObservableScope } from "../ObservableScope";
|
||||
@@ -42,7 +42,7 @@ export type UserMediaViewModel =
|
||||
| LocalUserMediaViewModel
|
||||
| RemoteUserMediaViewModel;
|
||||
|
||||
export interface BaseUserMediaViewModel extends MemberMediaViewModel {
|
||||
export interface BaseUserMediaViewModel extends BaseMemberMediaViewModel {
|
||||
type: "user";
|
||||
speaking$: Behavior<boolean>;
|
||||
audioEnabled$: Behavior<boolean>;
|
||||
|
||||
@@ -194,5 +194,3 @@ export function createWrappedUserMedia(
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
export type MediaItem = WrappedUserMediaViewModel | ScreenShareViewModel;
|
||||
@@ -7,9 +7,10 @@ Please see LICENSE in the repository root for full details.
|
||||
|
||||
import { type RemoteTrackPublication } from "livekit-client";
|
||||
import { test, expect } from "vitest";
|
||||
import { render, screen } from "@testing-library/react";
|
||||
import { act, render, screen } from "@testing-library/react";
|
||||
import { axe } from "vitest-axe";
|
||||
import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc";
|
||||
import { BehaviorSubject } from "rxjs";
|
||||
|
||||
import { GridTile } from "./GridTile";
|
||||
import {
|
||||
@@ -21,6 +22,11 @@ import { GridTileViewModel } from "../state/TileViewModel";
|
||||
import { ReactionsSenderProvider } from "../reactions/useReactionsSender";
|
||||
import type { CallViewModel } from "../state/CallViewModel/CallViewModel";
|
||||
import { constant } from "../state/Behavior";
|
||||
import {
|
||||
createRingingMedia,
|
||||
type RingingMediaViewModel,
|
||||
} from "../state/media/RingingMediaViewModel";
|
||||
import { type MuteStates } from "../state/MuteStates";
|
||||
|
||||
global.IntersectionObserver = class MockIntersectionObserver {
|
||||
public observe(): void {}
|
||||
@@ -28,6 +34,27 @@ global.IntersectionObserver = class MockIntersectionObserver {
|
||||
public disconnect(): void {}
|
||||
} as unknown as typeof IntersectionObserver;
|
||||
|
||||
const fakeRtcSession = {
|
||||
on: () => {},
|
||||
off: () => {},
|
||||
room: {
|
||||
on: () => {},
|
||||
off: () => {},
|
||||
client: {
|
||||
getUserId: () => null,
|
||||
getDeviceId: () => null,
|
||||
on: () => {},
|
||||
off: () => {},
|
||||
},
|
||||
},
|
||||
memberships: [],
|
||||
} as unknown as MatrixRTCSession;
|
||||
|
||||
const callVm = {
|
||||
reactions$: constant({}),
|
||||
handsRaised$: constant({}),
|
||||
} as Partial<CallViewModel> as CallViewModel;
|
||||
|
||||
test("GridTile is accessible", async () => {
|
||||
const vm = mockRemoteMedia(
|
||||
mockRtcMembership("@alice:example.org", "AAAA"),
|
||||
@@ -42,34 +69,15 @@ test("GridTile is accessible", async () => {
|
||||
}),
|
||||
);
|
||||
|
||||
const fakeRtcSession = {
|
||||
on: () => {},
|
||||
off: () => {},
|
||||
room: {
|
||||
on: () => {},
|
||||
off: () => {},
|
||||
client: {
|
||||
getUserId: () => null,
|
||||
getDeviceId: () => null,
|
||||
on: () => {},
|
||||
off: () => {},
|
||||
},
|
||||
},
|
||||
memberships: [],
|
||||
} as unknown as MatrixRTCSession;
|
||||
const cVm = {
|
||||
reactions$: constant({}),
|
||||
handsRaised$: constant({}),
|
||||
} as Partial<CallViewModel> as CallViewModel;
|
||||
const { container } = render(
|
||||
<ReactionsSenderProvider vm={cVm} rtcSession={fakeRtcSession}>
|
||||
<ReactionsSenderProvider vm={callVm} rtcSession={fakeRtcSession}>
|
||||
<GridTile
|
||||
vm={new GridTileViewModel(constant(vm))}
|
||||
onOpenProfile={() => {}}
|
||||
targetWidth={300}
|
||||
targetHeight={200}
|
||||
showSpeakingIndicators
|
||||
focusable={true}
|
||||
focusable
|
||||
/>
|
||||
</ReactionsSenderProvider>,
|
||||
);
|
||||
@@ -77,3 +85,40 @@ test("GridTile is accessible", async () => {
|
||||
// Name should be visible
|
||||
screen.getByText("Alice");
|
||||
});
|
||||
|
||||
test("GridTile displays ringing media", async () => {
|
||||
const pickupState$ = new BehaviorSubject<
|
||||
RingingMediaViewModel["pickupState$"]["value"]
|
||||
>("ringing");
|
||||
const vm = createRingingMedia({
|
||||
pickupState$,
|
||||
muteStates: {
|
||||
video: { enabled$: constant(false) },
|
||||
} as unknown as MuteStates,
|
||||
id: "test",
|
||||
userId: "@alice:example.org",
|
||||
displayName$: constant("Alice"),
|
||||
mxcAvatarUrl$: constant(undefined),
|
||||
});
|
||||
|
||||
const { container } = render(
|
||||
<ReactionsSenderProvider vm={callVm} rtcSession={fakeRtcSession}>
|
||||
<GridTile
|
||||
vm={new GridTileViewModel(constant(vm))}
|
||||
onOpenProfile={() => {}}
|
||||
targetWidth={300}
|
||||
targetHeight={200}
|
||||
showSpeakingIndicators
|
||||
focusable
|
||||
/>
|
||||
</ReactionsSenderProvider>,
|
||||
);
|
||||
expect(await axe(container)).toHaveNoViolations();
|
||||
// Name and status should be visible
|
||||
screen.getByText("Alice");
|
||||
screen.getByText("Calling…");
|
||||
|
||||
// Alice declines the call
|
||||
act(() => pickupState$.next("decline"));
|
||||
screen.getByText("Call ended");
|
||||
});
|
||||
|
||||
@@ -29,6 +29,9 @@ import {
|
||||
UserProfileIcon,
|
||||
VolumeOffSolidIcon,
|
||||
SwitchCameraSolidIcon,
|
||||
VideoCallSolidIcon,
|
||||
VoiceCallSolidIcon,
|
||||
EndCallIcon,
|
||||
} from "@vector-im/compound-design-tokens/assets/web/icons";
|
||||
import {
|
||||
ContextMenu,
|
||||
@@ -49,6 +52,7 @@ import { useBehavior } from "../useBehavior";
|
||||
import { type LocalUserMediaViewModel } from "../state/media/LocalUserMediaViewModel";
|
||||
import { type RemoteUserMediaViewModel } from "../state/media/RemoteUserMediaViewModel";
|
||||
import { type UserMediaViewModel } from "../state/media/UserMediaViewModel";
|
||||
import { type RingingMediaViewModel } from "../state/media/RingingMediaViewModel";
|
||||
|
||||
interface TileProps {
|
||||
ref?: Ref<HTMLDivElement>;
|
||||
@@ -56,21 +60,56 @@ interface TileProps {
|
||||
style?: ComponentProps<typeof animated.div>["style"];
|
||||
targetWidth: number;
|
||||
targetHeight: number;
|
||||
focusUrl: string | undefined;
|
||||
displayName: string;
|
||||
mxcAvatarUrl: string | undefined;
|
||||
showSpeakingIndicators: boolean;
|
||||
focusable: boolean;
|
||||
}
|
||||
|
||||
interface RingingMediaTileProps extends TileProps {
|
||||
vm: RingingMediaViewModel;
|
||||
}
|
||||
|
||||
const RingingMediaTile: FC<RingingMediaTileProps> = ({
|
||||
vm,
|
||||
className,
|
||||
...props
|
||||
}) => {
|
||||
const { t } = useTranslation();
|
||||
const pickupState = useBehavior(vm.pickupState$);
|
||||
const videoEnabled = useBehavior(vm.videoEnabled$);
|
||||
|
||||
return (
|
||||
<MediaView
|
||||
className={classNames(className, styles.tile)}
|
||||
video={undefined}
|
||||
userId={vm.userId}
|
||||
unencryptedWarning={false}
|
||||
status={
|
||||
pickupState === "ringing"
|
||||
? {
|
||||
text: t("video_tile.calling"),
|
||||
Icon: videoEnabled ? VideoCallSolidIcon : VoiceCallSolidIcon,
|
||||
}
|
||||
: { text: t("video_tile.call_ended"), Icon: EndCallIcon }
|
||||
}
|
||||
videoEnabled={videoEnabled}
|
||||
videoFit="cover"
|
||||
mirror={false}
|
||||
{...props}
|
||||
/>
|
||||
);
|
||||
};
|
||||
|
||||
interface UserMediaTileProps extends TileProps {
|
||||
vm: UserMediaViewModel;
|
||||
showSpeakingIndicators: boolean;
|
||||
mirror: boolean;
|
||||
playbackMuted: boolean;
|
||||
waitingForMedia?: boolean;
|
||||
primaryButton?: ReactNode;
|
||||
menuStart?: ReactNode;
|
||||
menuEnd?: ReactNode;
|
||||
focusUrl: string | undefined;
|
||||
}
|
||||
|
||||
const UserMediaTile: FC<UserMediaTileProps> = ({
|
||||
@@ -95,7 +134,6 @@ const UserMediaTile: FC<UserMediaTileProps> = ({
|
||||
const { t } = useTranslation();
|
||||
const video = useBehavior(vm.video$);
|
||||
const unencryptedWarning = useBehavior(vm.unencryptedWarning$);
|
||||
const encryptionStatus = useBehavior(vm.encryptionStatus$);
|
||||
const audioStreamStats = useObservableEagerState<
|
||||
RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats | undefined
|
||||
>(vm.audioStreamStats$);
|
||||
@@ -153,7 +191,6 @@ const UserMediaTile: FC<UserMediaTileProps> = ({
|
||||
video={video}
|
||||
userId={vm.userId}
|
||||
unencryptedWarning={unencryptedWarning}
|
||||
encryptionStatus={encryptionStatus}
|
||||
videoEnabled={videoEnabled}
|
||||
videoFit={videoFit}
|
||||
className={classNames(className, styles.tile, {
|
||||
@@ -218,6 +255,7 @@ UserMediaTile.displayName = "UserMediaTile";
|
||||
|
||||
interface LocalUserMediaTileProps extends TileProps {
|
||||
vm: LocalUserMediaViewModel;
|
||||
showSpeakingIndicators: boolean;
|
||||
onOpenProfile: (() => void) | null;
|
||||
}
|
||||
|
||||
@@ -232,6 +270,7 @@ const LocalUserMediaTile: FC<LocalUserMediaTileProps> = ({
|
||||
const mirror = useBehavior(vm.mirror$);
|
||||
const alwaysShow = useBehavior(vm.alwaysShow$);
|
||||
const switchCamera = useBehavior(vm.switchCamera$);
|
||||
const focusUrl = useBehavior(vm.focusUrl$);
|
||||
|
||||
const latestAlwaysShow = useLatest(alwaysShow);
|
||||
const onSelectAlwaysShow = useCallback(
|
||||
@@ -278,6 +317,7 @@ const LocalUserMediaTile: FC<LocalUserMediaTileProps> = ({
|
||||
)
|
||||
}
|
||||
focusable={focusable}
|
||||
focusUrl={focusUrl}
|
||||
{...props}
|
||||
/>
|
||||
);
|
||||
@@ -287,6 +327,7 @@ LocalUserMediaTile.displayName = "LocalUserMediaTile";
|
||||
|
||||
interface RemoteUserMediaTileProps extends TileProps {
|
||||
vm: RemoteUserMediaViewModel;
|
||||
showSpeakingIndicators: boolean;
|
||||
}
|
||||
|
||||
const RemoteUserMediaTile: FC<RemoteUserMediaTileProps> = ({
|
||||
@@ -298,6 +339,8 @@ const RemoteUserMediaTile: FC<RemoteUserMediaTileProps> = ({
|
||||
const waitingForMedia = useBehavior(vm.waitingForMedia$);
|
||||
const playbackMuted = useBehavior(vm.playbackMuted$);
|
||||
const playbackVolume = useBehavior(vm.playbackVolume$);
|
||||
const focusUrl = useBehavior(vm.focusUrl$);
|
||||
|
||||
const onSelectMute = useCallback(
|
||||
(e: Event) => {
|
||||
e.preventDefault();
|
||||
@@ -338,6 +381,7 @@ const RemoteUserMediaTile: FC<RemoteUserMediaTileProps> = ({
|
||||
</MenuItem>
|
||||
</>
|
||||
}
|
||||
focusUrl={focusUrl}
|
||||
{...props}
|
||||
/>
|
||||
);
|
||||
@@ -360,23 +404,33 @@ interface GridTileProps {
|
||||
export const GridTile: FC<GridTileProps> = ({
|
||||
ref: theirRef,
|
||||
vm,
|
||||
showSpeakingIndicators,
|
||||
onOpenProfile,
|
||||
...props
|
||||
}) => {
|
||||
const ourRef = useRef<HTMLDivElement | null>(null);
|
||||
const ref = useMergedRefs(ourRef, theirRef);
|
||||
const media = useBehavior(vm.media$);
|
||||
const focusUrl = useBehavior(media.focusUrl$);
|
||||
const displayName = useBehavior(media.displayName$);
|
||||
const mxcAvatarUrl = useBehavior(media.mxcAvatarUrl$);
|
||||
|
||||
if (media.local) {
|
||||
if (media.type === "ringing") {
|
||||
return (
|
||||
<RingingMediaTile
|
||||
ref={ref}
|
||||
vm={media}
|
||||
{...props}
|
||||
displayName={displayName}
|
||||
mxcAvatarUrl={mxcAvatarUrl}
|
||||
/>
|
||||
);
|
||||
} else if (media.local) {
|
||||
return (
|
||||
<LocalUserMediaTile
|
||||
ref={ref}
|
||||
vm={media}
|
||||
showSpeakingIndicators={showSpeakingIndicators}
|
||||
onOpenProfile={onOpenProfile}
|
||||
focusUrl={focusUrl}
|
||||
displayName={displayName}
|
||||
mxcAvatarUrl={mxcAvatarUrl}
|
||||
{...props}
|
||||
@@ -387,7 +441,7 @@ export const GridTile: FC<GridTileProps> = ({
|
||||
<RemoteUserMediaTile
|
||||
ref={ref}
|
||||
vm={media}
|
||||
focusUrl={focusUrl}
|
||||
showSpeakingIndicators={showSpeakingIndicators}
|
||||
displayName={displayName}
|
||||
mxcAvatarUrl={mxcAvatarUrl}
|
||||
{...props}
|
||||
|
||||
@@ -52,6 +52,11 @@ Please see LICENSE in the repository root for full details.
|
||||
pointer-events: none;
|
||||
}
|
||||
|
||||
.translucent {
|
||||
opacity: 50%;
|
||||
mix-blend-mode: multiply;
|
||||
}
|
||||
|
||||
/* CSS makes us put a condition here, even though all we want to do is
|
||||
unconditionally select the container so we can use cqmin units */
|
||||
@container mediaView (width > 0) {
|
||||
@@ -71,14 +76,15 @@ unconditionally select the container so we can use cqmin units */
|
||||
|
||||
.fg {
|
||||
position: absolute;
|
||||
inset: var(
|
||||
--fg-inset: var(
|
||||
--media-view-fg-inset,
|
||||
calc(var(--media-view-border-radius) - var(--cpd-space-3x))
|
||||
);
|
||||
inset: var(--fg-inset);
|
||||
display: grid;
|
||||
grid-template-columns: 30px 1fr 30px;
|
||||
grid-template-rows: 1fr auto;
|
||||
grid-template-areas: "reactions status ." "nameTag nameTag button";
|
||||
grid-template-areas: "status status reactions" "nameTag nameTag button";
|
||||
gap: var(--cpd-space-1x);
|
||||
place-items: start;
|
||||
}
|
||||
@@ -102,21 +108,19 @@ unconditionally select the container so we can use cqmin units */
|
||||
|
||||
.status {
|
||||
grid-area: status;
|
||||
justify-self: center;
|
||||
align-self: start;
|
||||
padding: var(--cpd-space-2x);
|
||||
padding-block: var(--cpd-space-2x);
|
||||
color: var(--cpd-color-text-primary);
|
||||
background-color: var(--cpd-color-bg-canvas-default);
|
||||
display: flex;
|
||||
flex-wrap: none;
|
||||
align-items: center;
|
||||
border-radius: var(--cpd-radius-pill-effect);
|
||||
gap: 3px;
|
||||
user-select: none;
|
||||
overflow: hidden;
|
||||
box-shadow: var(--small-drop-shadow);
|
||||
box-sizing: border-box;
|
||||
max-inline-size: 100%;
|
||||
text-align: center;
|
||||
margin-block-start: calc(var(--cpd-space-3x) - var(--fg-inset));
|
||||
margin-inline-start: calc(var(--cpd-space-4x) - var(--fg-inset));
|
||||
}
|
||||
|
||||
.status svg {
|
||||
color: var(--cpd-color-icon-tertiary);
|
||||
}
|
||||
|
||||
.reactions {
|
||||
|
||||
@@ -18,7 +18,6 @@ import { TrackInfo } from "@livekit/protocol";
|
||||
import { type ComponentProps } from "react";
|
||||
|
||||
import { MediaView } from "./MediaView";
|
||||
import { EncryptionStatus } from "../state/media/MemberMediaViewModel";
|
||||
import { mockLocalParticipant } from "../utils/test";
|
||||
|
||||
describe("MediaView", () => {
|
||||
@@ -41,7 +40,6 @@ describe("MediaView", () => {
|
||||
videoFit: "contain",
|
||||
targetWidth: 300,
|
||||
targetHeight: 200,
|
||||
encryptionStatus: EncryptionStatus.Connecting,
|
||||
mirror: false,
|
||||
unencryptedWarning: false,
|
||||
video: trackReference,
|
||||
|
||||
@@ -7,7 +7,13 @@ Please see LICENSE in the repository root for full details.
|
||||
|
||||
import { type TrackReferenceOrPlaceholder } from "@livekit/components-core";
|
||||
import { animated } from "@react-spring/web";
|
||||
import { type FC, type ComponentProps, type ReactNode } from "react";
|
||||
import {
|
||||
type FC,
|
||||
type ComponentProps,
|
||||
type ReactNode,
|
||||
type ComponentType,
|
||||
type SVGAttributes,
|
||||
} from "react";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import classNames from "classnames";
|
||||
import { VideoTrack } from "@livekit/components-react";
|
||||
@@ -16,7 +22,6 @@ import { ErrorSolidIcon } from "@vector-im/compound-design-tokens/assets/web/ico
|
||||
|
||||
import styles from "./MediaView.module.css";
|
||||
import { Avatar } from "../Avatar";
|
||||
import { type EncryptionStatus } from "../state/media/MemberMediaViewModel";
|
||||
import { RaisedHandIndicator } from "../reactions/RaisedHandIndicator";
|
||||
import {
|
||||
showConnectionStats as showConnectionStatsSetting,
|
||||
@@ -38,7 +43,7 @@ interface Props extends ComponentProps<typeof animated.div> {
|
||||
userId: string;
|
||||
videoEnabled: boolean;
|
||||
unencryptedWarning: boolean;
|
||||
encryptionStatus: EncryptionStatus;
|
||||
status?: { text: string; Icon: ComponentType<SVGAttributes<SVGElement>> };
|
||||
nameTagLeadingIcon?: ReactNode;
|
||||
displayName: string;
|
||||
mxcAvatarUrl: string | undefined;
|
||||
@@ -72,7 +77,7 @@ export const MediaView: FC<Props> = ({
|
||||
mxcAvatarUrl,
|
||||
focusable,
|
||||
primaryButton,
|
||||
encryptionStatus,
|
||||
status,
|
||||
raisedHandTime,
|
||||
currentReaction,
|
||||
raisedHandOnClick,
|
||||
@@ -106,7 +111,11 @@ export const MediaView: FC<Props> = ({
|
||||
name={displayName}
|
||||
size={avatarSize}
|
||||
src={mxcAvatarUrl}
|
||||
className={styles.avatar}
|
||||
className={classNames(styles.avatar, {
|
||||
// When the avatar is overlaid with a status, make it translucent
|
||||
// for readability
|
||||
[styles.translucent]: status,
|
||||
})}
|
||||
style={{ display: video && videoEnabled ? "none" : "initial" }}
|
||||
/>
|
||||
{video?.publication !== undefined && (
|
||||
@@ -152,6 +161,14 @@ export const MediaView: FC<Props> = ({
|
||||
/>
|
||||
</>
|
||||
)}
|
||||
{status && (
|
||||
<div className={styles.status}>
|
||||
<status.Icon width={16} height={16} aria-hidden />
|
||||
<Text as="span" size="sm" weight="medium">
|
||||
{status.text}
|
||||
</Text>
|
||||
</div>
|
||||
)}
|
||||
{/* TODO: Bring this back once encryption status is less broken */}
|
||||
{/*encryptionStatus !== EncryptionStatus.Okay && (
|
||||
<div className={styles.status}>
|
||||
|
||||
@@ -6,10 +6,11 @@ Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { test, expect, vi } from "vitest";
|
||||
import { isInaccessible, render, screen } from "@testing-library/react";
|
||||
import { act, isInaccessible, render, screen } from "@testing-library/react";
|
||||
import { axe } from "vitest-axe";
|
||||
import userEvent from "@testing-library/user-event";
|
||||
import { TooltipProvider } from "@vector-im/compound-web";
|
||||
import { BehaviorSubject } from "rxjs";
|
||||
|
||||
import { SpotlightTile } from "./SpotlightTile";
|
||||
import {
|
||||
@@ -23,6 +24,11 @@ import {
|
||||
} from "../utils/test";
|
||||
import { SpotlightTileViewModel } from "../state/TileViewModel";
|
||||
import { constant } from "../state/Behavior";
|
||||
import {
|
||||
createRingingMedia,
|
||||
type RingingMediaViewModel,
|
||||
} from "../state/media/RingingMediaViewModel";
|
||||
import { type MuteStates } from "../state/MuteStates";
|
||||
|
||||
global.IntersectionObserver = class MockIntersectionObserver {
|
||||
public observe(): void {}
|
||||
@@ -140,3 +146,41 @@ test("Screen share volume UI is hidden when screen share has no audio", async ()
|
||||
screen.queryByRole("button", { name: /volume/i }),
|
||||
).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
test("SpotlightTile displays ringing media", async () => {
|
||||
const pickupState$ = new BehaviorSubject<
|
||||
RingingMediaViewModel["pickupState$"]["value"]
|
||||
>("ringing");
|
||||
const vm = createRingingMedia({
|
||||
pickupState$,
|
||||
muteStates: {
|
||||
video: { enabled$: constant(false) },
|
||||
} as unknown as MuteStates,
|
||||
id: "test",
|
||||
userId: "@alice:example.org",
|
||||
displayName$: constant("Alice"),
|
||||
mxcAvatarUrl$: constant(undefined),
|
||||
});
|
||||
|
||||
const toggleExpanded = vi.fn();
|
||||
const { container } = render(
|
||||
<SpotlightTile
|
||||
vm={new SpotlightTileViewModel(constant([vm]), constant(false))}
|
||||
targetWidth={300}
|
||||
targetHeight={200}
|
||||
expanded={false}
|
||||
onToggleExpanded={toggleExpanded}
|
||||
showIndicators
|
||||
focusable={true}
|
||||
/>,
|
||||
);
|
||||
|
||||
expect(await axe(container)).toHaveNoViolations();
|
||||
// Alice should be in the spotlight with the right status
|
||||
screen.getByText("Alice");
|
||||
screen.getByText("Calling…");
|
||||
|
||||
// Now we time out ringing to Alice
|
||||
act(() => pickupState$.next("timeout"));
|
||||
screen.getByText("Call ended");
|
||||
});
|
||||
|
||||
@@ -24,6 +24,9 @@ import {
|
||||
VolumeOnIcon,
|
||||
VolumeOffSolidIcon,
|
||||
VolumeOnSolidIcon,
|
||||
VideoCallSolidIcon,
|
||||
VoiceCallSolidIcon,
|
||||
EndCallIcon,
|
||||
} from "@vector-im/compound-design-tokens/assets/web/icons";
|
||||
import { animated } from "@react-spring/web";
|
||||
import { type Observable, map } from "rxjs";
|
||||
@@ -43,7 +46,7 @@ import { useReactiveState } from "../useReactiveState";
|
||||
import { useLatest } from "../useLatest";
|
||||
import { type SpotlightTileViewModel } from "../state/TileViewModel";
|
||||
import { useBehavior } from "../useBehavior";
|
||||
import { type EncryptionStatus } from "../state/media/MemberMediaViewModel";
|
||||
import { type MemberMediaViewModel } from "../state/media/MemberMediaViewModel";
|
||||
import { type LocalUserMediaViewModel } from "../state/media/LocalUserMediaViewModel";
|
||||
import { type RemoteUserMediaViewModel } from "../state/media/RemoteUserMediaViewModel";
|
||||
import { type UserMediaViewModel } from "../state/media/UserMediaViewModel";
|
||||
@@ -52,6 +55,7 @@ import { type RemoteScreenShareViewModel } from "../state/media/RemoteScreenShar
|
||||
import { type MediaViewModel } from "../state/media/MediaViewModel";
|
||||
import { Slider } from "../Slider";
|
||||
import { platform } from "../Platform";
|
||||
import { type RingingMediaViewModel } from "../state/media/RingingMediaViewModel";
|
||||
|
||||
interface SpotlightItemBaseProps {
|
||||
ref?: Ref<HTMLDivElement>;
|
||||
@@ -59,18 +63,20 @@ interface SpotlightItemBaseProps {
|
||||
"data-id": string;
|
||||
targetWidth: number;
|
||||
targetHeight: number;
|
||||
video: TrackReferenceOrPlaceholder | undefined;
|
||||
userId: string;
|
||||
unencryptedWarning: boolean;
|
||||
encryptionStatus: EncryptionStatus;
|
||||
focusUrl: string | undefined;
|
||||
displayName: string;
|
||||
mxcAvatarUrl: string | undefined;
|
||||
focusable: boolean;
|
||||
"aria-hidden"?: boolean;
|
||||
}
|
||||
|
||||
interface SpotlightUserMediaItemBaseProps extends SpotlightItemBaseProps {
|
||||
interface SpotlightMemberMediaItemBaseProps extends SpotlightItemBaseProps {
|
||||
video: TrackReferenceOrPlaceholder | undefined;
|
||||
unencryptedWarning: boolean;
|
||||
focusUrl: string | undefined;
|
||||
}
|
||||
|
||||
interface SpotlightUserMediaItemBaseProps extends SpotlightMemberMediaItemBaseProps {
|
||||
videoFit: "contain" | "cover";
|
||||
videoEnabled: boolean;
|
||||
}
|
||||
@@ -103,21 +109,32 @@ const SpotlightRemoteUserMediaItem: FC<SpotlightRemoteUserMediaItemProps> = ({
|
||||
);
|
||||
};
|
||||
|
||||
interface SpotlightUserMediaItemProps extends SpotlightItemBaseProps {
|
||||
interface SpotlightUserMediaItemProps extends SpotlightMemberMediaItemBaseProps {
|
||||
vm: UserMediaViewModel;
|
||||
}
|
||||
|
||||
const SpotlightUserMediaItem: FC<SpotlightUserMediaItemProps> = ({
|
||||
vm,
|
||||
targetWidth,
|
||||
targetHeight,
|
||||
...props
|
||||
}) => {
|
||||
const videoFit = useBehavior(vm.videoFit$);
|
||||
const videoEnabled = useBehavior(vm.videoEnabled$);
|
||||
|
||||
// Whenever target bounds change, inform the viewModel
|
||||
useEffect(() => {
|
||||
if (targetWidth > 0 && targetHeight > 0) {
|
||||
vm.setTargetDimensions(targetWidth, targetHeight);
|
||||
}
|
||||
}, [targetWidth, targetHeight, vm]);
|
||||
|
||||
const baseProps: SpotlightUserMediaItemBaseProps &
|
||||
RefAttributes<HTMLDivElement> = {
|
||||
videoFit,
|
||||
videoEnabled,
|
||||
targetWidth,
|
||||
targetHeight,
|
||||
...props,
|
||||
};
|
||||
|
||||
@@ -130,7 +147,7 @@ const SpotlightUserMediaItem: FC<SpotlightUserMediaItemProps> = ({
|
||||
|
||||
SpotlightUserMediaItem.displayName = "SpotlightUserMediaItem";
|
||||
|
||||
interface SpotlightScreenShareItemProps extends SpotlightItemBaseProps {
|
||||
interface SpotlightScreenShareItemProps extends SpotlightMemberMediaItemBaseProps {
|
||||
vm: ScreenShareViewModel;
|
||||
videoEnabled: boolean;
|
||||
}
|
||||
@@ -142,7 +159,7 @@ const SpotlightScreenShareItem: FC<SpotlightScreenShareItemProps> = ({
|
||||
return <MediaView videoFit="contain" mirror={false} {...props} />;
|
||||
};
|
||||
|
||||
interface SpotlightRemoteScreenShareItemProps extends SpotlightItemBaseProps {
|
||||
interface SpotlightRemoteScreenShareItemProps extends SpotlightMemberMediaItemBaseProps {
|
||||
vm: RemoteScreenShareViewModel;
|
||||
}
|
||||
|
||||
@@ -155,6 +172,67 @@ const SpotlightRemoteScreenShareItem: FC<
|
||||
);
|
||||
};
|
||||
|
||||
interface SpotlightMemberMediaItemProps extends SpotlightItemBaseProps {
|
||||
vm: MemberMediaViewModel;
|
||||
}
|
||||
|
||||
const SpotlightMemberMediaItem: FC<SpotlightMemberMediaItemProps> = ({
|
||||
vm,
|
||||
...props
|
||||
}) => {
|
||||
const video = useBehavior(vm.video$);
|
||||
const unencryptedWarning = useBehavior(vm.unencryptedWarning$);
|
||||
const focusUrl = useBehavior(vm.focusUrl$);
|
||||
|
||||
const baseProps: SpotlightMemberMediaItemBaseProps &
|
||||
RefAttributes<HTMLDivElement> = {
|
||||
video: video ?? undefined,
|
||||
unencryptedWarning,
|
||||
focusUrl,
|
||||
...props,
|
||||
};
|
||||
|
||||
if (vm.type === "user")
|
||||
return <SpotlightUserMediaItem vm={vm} {...baseProps} />;
|
||||
return vm.local ? (
|
||||
<SpotlightScreenShareItem vm={vm} videoEnabled {...baseProps} />
|
||||
) : (
|
||||
<SpotlightRemoteScreenShareItem vm={vm} {...baseProps} />
|
||||
);
|
||||
};
|
||||
|
||||
interface SpotlightRingingMediaItemProps extends SpotlightItemBaseProps {
|
||||
vm: RingingMediaViewModel;
|
||||
}
|
||||
|
||||
const SpotlightRingingMediaItem: FC<SpotlightRingingMediaItemProps> = ({
|
||||
vm,
|
||||
...props
|
||||
}) => {
|
||||
const { t } = useTranslation();
|
||||
const pickupState = useBehavior(vm.pickupState$);
|
||||
const videoEnabled = useBehavior(vm.videoEnabled$);
|
||||
|
||||
return (
|
||||
<MediaView
|
||||
video={undefined}
|
||||
unencryptedWarning={false}
|
||||
status={
|
||||
pickupState === "ringing"
|
||||
? {
|
||||
text: t("video_tile.calling"),
|
||||
Icon: videoEnabled ? VideoCallSolidIcon : VoiceCallSolidIcon,
|
||||
}
|
||||
: { text: t("video_tile.call_ended"), Icon: EndCallIcon }
|
||||
}
|
||||
videoEnabled={false}
|
||||
videoFit="cover"
|
||||
mirror={false}
|
||||
{...props}
|
||||
/>
|
||||
);
|
||||
};
|
||||
|
||||
interface SpotlightItemProps {
|
||||
ref?: Ref<HTMLDivElement>;
|
||||
vm: MediaViewModel;
|
||||
@@ -187,22 +265,9 @@ const SpotlightItem: FC<SpotlightItemProps> = ({
|
||||
}) => {
|
||||
const ourRef = useRef<HTMLDivElement | null>(null);
|
||||
|
||||
// Whenever target bounds change, inform the viewModel
|
||||
useEffect(() => {
|
||||
if (targetWidth > 0 && targetHeight > 0) {
|
||||
if (vm.type != "screen share") {
|
||||
vm.setTargetDimensions(targetWidth, targetHeight);
|
||||
}
|
||||
}
|
||||
}, [targetWidth, targetHeight, vm]);
|
||||
|
||||
const ref = useMergedRefs(ourRef, theirRef);
|
||||
const focusUrl = useBehavior(vm.focusUrl$);
|
||||
const displayName = useBehavior(vm.displayName$);
|
||||
const mxcAvatarUrl = useBehavior(vm.mxcAvatarUrl$);
|
||||
const video = useBehavior(vm.video$);
|
||||
const unencryptedWarning = useBehavior(vm.unencryptedWarning$);
|
||||
const encryptionStatus = useBehavior(vm.encryptionStatus$);
|
||||
|
||||
// Hook this item up to the intersection observer
|
||||
useEffect(() => {
|
||||
@@ -225,23 +290,17 @@ const SpotlightItem: FC<SpotlightItemProps> = ({
|
||||
className: classNames(styles.item, { [styles.snap]: snap }),
|
||||
targetWidth,
|
||||
targetHeight,
|
||||
video: video ?? undefined,
|
||||
userId: vm.userId,
|
||||
unencryptedWarning,
|
||||
focusUrl,
|
||||
displayName,
|
||||
mxcAvatarUrl,
|
||||
focusable,
|
||||
encryptionStatus,
|
||||
"aria-hidden": ariaHidden,
|
||||
};
|
||||
|
||||
if (vm.type === "user")
|
||||
return <SpotlightUserMediaItem vm={vm} {...baseProps} />;
|
||||
return vm.local ? (
|
||||
<SpotlightScreenShareItem vm={vm} videoEnabled {...baseProps} />
|
||||
return vm.type === "ringing" ? (
|
||||
<SpotlightRingingMediaItem vm={vm} {...baseProps} />
|
||||
) : (
|
||||
<SpotlightRemoteScreenShareItem vm={vm} {...baseProps} />
|
||||
<SpotlightMemberMediaItem vm={vm} {...baseProps} />
|
||||
);
|
||||
};
|
||||
|
||||
|
||||
@@ -93,6 +93,7 @@ export const initializeWidget = (
|
||||
logger.info("Widget API is available");
|
||||
const api = new WidgetApi(widgetId, parentOrigin);
|
||||
api.requestCapability(MatrixCapabilities.AlwaysOnScreen);
|
||||
api.requestCapability(MatrixCapabilities.MSC4039DownloadFile);
|
||||
|
||||
// Set up the lazy action emitter, but only for select actions that we
|
||||
// intend for the app to handle
|
||||
|
||||
Reference in New Issue
Block a user