Compare commits

..

1 Commits

Author SHA1 Message Date
Robin
ee2a6718a0 WIP 2025-11-06 11:23:36 -05:00
110 changed files with 6316 additions and 10345 deletions

View File

@@ -43,29 +43,6 @@ module.exports = {
// To encourage good usage of RxJS:
"rxjs/no-exposed-subjects": "error",
"rxjs/finnish": ["error", { names: { "^this$": false } }],
"no-restricted-imports": [
"error",
{
paths: ["matrix-widget-api", "matrix-js-sdk"].flatMap((lib) =>
["src", "src/", "src/index", "lib", "lib/", "lib/index"]
.map((path) => `${lib}/${path}`)
.map((name) => ({ name, message: `Please use ${lib} instead` })),
),
patterns: [
...["matrix-widget-api"].map((lib) => ({
group: ["src", "src/", "src/**", "lib", "lib/", "lib/**"].map(
(path) => `${lib}/${path}`,
),
message: `Please use ${lib} instead`,
})),
// XXX: We use /lib in lots of places, so allow for now.
...["matrix-js-sdk"].map((lib) => ({
group: ["src", "src/", "src/**"].map((path) => `${lib}/${path}`),
message: `Please use ${lib} instead`,
})),
],
},
],
},
overrides: [
{

View File

@@ -23,7 +23,7 @@ jobs:
packages: write
steps:
- name: Check it out
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4
- name: 📥 Download artifact
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
@@ -42,7 +42,7 @@ jobs:
- name: Extract metadata (tags, labels) for Docker
id: meta
uses: docker/metadata-action@318604b99e75e41977312d83839a89be02ca4893 # v5.9.0
uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v5.8.0
with:
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
tags: ${{ inputs.docker_tags}}

View File

@@ -32,7 +32,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4
- name: Enable Corepack
run: corepack enable
- name: Yarn cache

View File

@@ -7,7 +7,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4
- name: Enable Corepack
run: corepack enable
- name: Yarn cache

View File

@@ -85,7 +85,7 @@ jobs:
run: find ${{ env.FILENAME_PREFIX }} -type f -print0 | sort -z | xargs -0 sha256sum | tee ${{ env.FILENAME_PREFIX }}.sha256
- name: Upload
if: ${{ needs.versioning.outputs.DRY_RUN == 'false' }}
uses: softprops/action-gh-release@a06a81a03ee405af7f2048a818ed3f03bbf83c7b # v2
uses: softprops/action-gh-release@aec2ec56f94eb8180ceec724245f64ef008b89f5 # v2
with:
files: |
${{ env.FILENAME_PREFIX }}.tar.gz
@@ -100,10 +100,10 @@ jobs:
ARTIFACT_VERSION: ${{ steps.artifact_version.outputs.ARTIFACT_VERSION }}
permissions:
contents: read
id-token: write # Allow npm to authenticate as a trusted publisher
id-token: write # required for the provenance flag on npm publish
steps:
- name: Checkout
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4
- name: 📥 Download built element-call artifact
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
@@ -126,6 +126,8 @@ jobs:
npm version ${{ needs.versioning.outputs.PREFIXED_VERSION }} --no-git-tag-version
echo "ARTIFACT_VERSION=$(jq '.version' --raw-output package.json)" >> "$GITHUB_ENV"
npm publish --provenance --access public --tag ${{ needs.versioning.outputs.TAG }} ${{ needs.versioning.outputs.DRY_RUN == 'true' && '--dry-run' || '' }}
env:
NODE_AUTH_TOKEN: ${{ secrets.NPM_RELEASE_TOKEN }}
- id: artifact_version
name: Output artifact version
@@ -142,7 +144,7 @@ jobs:
contents: read
steps:
- name: Checkout
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4
- name: 📥 Download built element-call artifact
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
@@ -197,7 +199,7 @@ jobs:
contents: read
steps:
- name: Checkout
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4
with:
path: element-call
@@ -210,7 +212,7 @@ jobs:
path: element-call/embedded/ios/Sources/dist
- name: Checkout element-call-swift
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4
with:
repository: element-hq/element-call-swift
path: element-call-swift
@@ -262,7 +264,7 @@ jobs:
echo "iOS: ${{ needs.publish_ios.outputs.ARTIFACT_VERSION }}"
- name: Add release notes
if: ${{ needs.versioning.outputs.DRY_RUN == 'false' }}
uses: softprops/action-gh-release@a06a81a03ee405af7f2048a818ed3f03bbf83c7b # v2
uses: softprops/action-gh-release@aec2ec56f94eb8180ceec724245f64ef008b89f5 # v2
with:
append_body: true
body: |

View File

@@ -42,7 +42,7 @@ jobs:
- name: Create Checksum
run: find ${{ env.FILENAME_PREFIX }} -type f -print0 | sort -z | xargs -0 sha256sum | tee ${{ env.FILENAME_PREFIX }}.sha256
- name: Upload
uses: softprops/action-gh-release@a06a81a03ee405af7f2048a818ed3f03bbf83c7b # v2
uses: softprops/action-gh-release@aec2ec56f94eb8180ceec724245f64ef008b89f5 # v2
with:
files: |
${{ env.FILENAME_PREFIX }}.tar.gz
@@ -68,7 +68,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Add release note
uses: softprops/action-gh-release@a06a81a03ee405af7f2048a818ed3f03bbf83c7b # v2
uses: softprops/action-gh-release@aec2ec56f94eb8180ceec724245f64ef008b89f5 # v2
with:
append_body: true
body: |

View File

@@ -9,7 +9,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4
- name: Enable Corepack
run: corepack enable
- name: Yarn cache
@@ -33,7 +33,7 @@ jobs:
timeout-minutes: 60
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4
- name: Enable Corepack
run: corepack enable
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4

View File

@@ -13,7 +13,7 @@ jobs:
steps:
- name: Checkout the code
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4
- name: Enable Corepack
run: corepack enable
@@ -42,7 +42,7 @@ jobs:
- name: Create Pull Request
id: cpr
uses: peter-evans/create-pull-request@84ae59a2cdc2258d6fa0732dd66352dddae2a412 # v7.0.9
uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8
with:
token: ${{ secrets.ELEMENT_BOT_TOKEN }}
branch: actions/localazy-download

View File

@@ -14,7 +14,7 @@ jobs:
steps:
- name: Checkout the code
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4
- name: Upload
uses: localazy/upload@27e6b5c0fddf4551596b42226b1c24124335d24a # v1

View File

@@ -1 +1 @@
24
22

View File

@@ -13,6 +13,7 @@ coverage:
informational: true
patch:
default:
# Enforce 80% coverage on all lines that a PR
# Encourage (but don't enforce) 80% coverage on all lines that a PR
# touches
target: 80%
informational: true

View File

@@ -18,7 +18,7 @@
"api_host": "https://posthog-element-call.element.io"
},
"rageshake": {
"submit_url": "https://rageshakes.element.io/api/submit"
"submit_url": "https://element.io/bugreports/submit"
},
"sentry": {
"environment": "netlify-pr-preview",

View File

@@ -136,8 +136,8 @@ handle @jwt_service {
reverse_proxy http://[::1]:8080 {
header_up Host {host}
header_up X-Forwarded-Server {host}
header_up X-Real-IP {remote_host}
header_up X-Forwarded-For {remote_host}
header_up X-Real-IP {remote_addr}
header_up X-Forwarded-For {remote_addr}
}
}
@@ -146,8 +146,8 @@ handle {
reverse_proxy http://localhost:7880 {
header_up Host {host}
header_up X-Forwarded-Server {host}
header_up X-Real-IP {remote_host}
header_up X-Forwarded-For {remote_host}
header_up X-Real-IP {remote_addr}
header_up X-Forwarded-For {remote_addr}
}
}
```

View File

@@ -96,6 +96,6 @@ These parameters are only supported in the [embedded](./embedded-standalone.md)
| -------------------- | -------------------------------------------------------------------------- | -------- | ---------------------------------------------------------------------------------------------------------------------------------- |
| `posthogApiHost` | Posthog server URL | No | e.g. `https://posthog-element-call.element.io`. Only supported in embedded package. In full package the value from config is used. |
| `posthogApiKey` | Posthog project API key | No | Only supported in embedded package. In full package the value from config is used. |
| `rageshakeSubmitUrl` | Rageshake server URL endpoint | No | e.g. `https://rageshakes.element.io/api/submit`. In full package the value from config is used. |
| `rageshakeSubmitUrl` | Rageshake server URL endpoint | No | e.g. `https://element.io/bugreports/submit`. In full package the value from config is used. |
| `sentryDsn` | Sentry [DSN](https://docs.sentry.io/concepts/key-terms/dsn-explainer/) | No | In full package the value from config is used. |
| `sentryEnvironment` | Sentry [environment](https://docs.sentry.io/concepts/key-terms/key-terms/) | No | In full package the value from config is used. |

View File

@@ -2,11 +2,11 @@
# https://docs.gradle.org/current/userguide/platforms.html#sub::toml-dependencies-format
[versions]
android_gradle_plugin = "8.13.1"
android_gradle_plugin = "8.11.1"
[libraries]
android_gradle_plugin = { module = "com.android.tools.build:gradle", version.ref = "android_gradle_plugin" }
[plugins]
android_library = { id = "com.android.library", version.ref = "android_gradle_plugin" }
maven_publish = { id = "com.vanniktech.maven.publish", version = "0.35.0" }
maven_publish = { id = "com.vanniktech.maven.publish", version = "0.34.0" }

View File

@@ -64,14 +64,6 @@
"developer_mode": {
"always_show_iphone_earpiece": "Show iPhone earpiece option on all platforms",
"crypto_version": "Crypto version: {{version}}",
"custom_livekit_url": {
"current_url": "Currently set to: ",
"from_config": "Currently, no overwrite is set. Url from well-known or config is used.",
"label": "Custom Livekit-url",
"reset": "Reset overwrite",
"save": "Save",
"saving": "Saving..."
},
"debug_tile_layout_label": "Debug tile layout",
"device_id": "Device ID: {{id}}",
"duplicate_tiles_label": "Number of additional tile copies per participant",
@@ -82,18 +74,17 @@
"matrix_id": "Matrix ID: {{id}}",
"matrixRTCMode": {
"Comptibility": {
"description": "Compatible with homeservers that do not support sticky events (but all other EC clients are v0.17.0 or later)",
"label": "Compatibility: state events & multi SFU"
"label": "Compatibility: state events & multi SFU",
"description": "Compatible with homeservers that do not support sticky events (but all other EC clients are v0.17.0 or later)"
},
"Legacy": {
"description": "Compatible with old versions of EC that do not support multi SFU",
"label": "Legacy: state events & oldest membership SFU"
"label": "Legacy: state events & oldest membership SFU",
"description": "Compatible with old versions of EC that do not support multi SFU"
},
"Matrix_2_0": {
"description": "Compatible only with homservers supporting sticky events and all EC clients v0.17.0 or later",
"label": "Matrix 2.0: sticky events & multi SFU"
},
"title": "MatrixRTC mode"
"label": "Matrix 2.0: sticky events & multi SFU",
"description": "Compatible only with homservers supporting sticky events and all EC clients v0.17.0 or later"
}
},
"mute_all_audio": "Mute all audio (participants, reactions, join sounds)",
"show_connection_stats": "Show connection statistics",
@@ -108,14 +99,11 @@
"connection_lost_description": "You were disconnected from the call.",
"e2ee_unsupported": "Incompatible browser",
"e2ee_unsupported_description": "Your web browser does not support encrypted calls. Supported browsers include Chrome, Safari, and Firefox 117+.",
"failed_to_start_livekit": "Failed to start Livekit connection",
"generic": "Something went wrong",
"generic_description": "Submitting debug logs will help us track down the problem.",
"insufficient_capacity": "Insufficient capacity",
"insufficient_capacity_description": "The server has reached its maximum capacity and you cannot join the call at this time. Try again later, or contact your server admin if the problem persists.",
"matrix_rtc_transport_missing": "The server is not configured to work with {{brand}}. Please contact your server admin (Domain: {{domain}}, Error Code: {{ errorCode }}).",
"membership_manager": "Membership Manager Error",
"membership_manager_description": "The Membership Manager had to shut down. This is caused by many consequtive failed network requests.",
"open_elsewhere": "Opened in another tab",
"open_elsewhere_description": "{{brand}} has been opened in another tab. If that doesn't sound right, try reloading the page.",
"room_creation_restricted": "Failed to create call",

View File

@@ -71,7 +71,7 @@
"@types/grecaptcha": "^3.0.9",
"@types/jsdom": "^21.1.7",
"@types/lodash-es": "^4.17.12",
"@types/node": "^24.0.0",
"@types/node": "^22.0.0",
"@types/pako": "^2.0.3",
"@types/qrcode": "^1.5.5",
"@types/react": "^19.0.0",
@@ -109,8 +109,8 @@
"livekit-client": "^2.13.0",
"lodash-es": "^4.17.21",
"loglevel": "^1.9.1",
"matrix-js-sdk": "^39.2.0",
"matrix-widget-api": "^1.14.0",
"matrix-js-sdk": "github:matrix-org/matrix-js-sdk#head=toger5/sticky-events&commit=e7f5bec51b6f70501a025b79fe5021c933385b21",
"matrix-widget-api": "^1.13.0",
"normalize.css": "^8.0.1",
"observable-hooks": "^4.2.3",
"pako": "^2.0.4",

View File

@@ -75,12 +75,7 @@ test("Should automatically retry non fatal JWT errors", async ({
test("Should show error screen if call creation is restricted", async ({
page,
browserName,
}) => {
test.skip(
browserName === "firefox",
"The is test is not working on firefox CI environment.",
);
await page.goto("/");
// We need the socket connection to fail, but this cannot be done by using the websocket route.

View File

@@ -159,11 +159,7 @@ export const widgetTest = test.extend<MyFixtures>({
} = await registerUser(browser, userB);
// Invite the second user
await ewPage1
.getByRole("navigation", { name: "Room list" })
.getByRole("button", { name: "New conversation" })
.click();
await ewPage1.getByRole("button", { name: "Add", exact: true }).click();
await ewPage1.getByRole("menuitem", { name: "New Room" }).click();
await ewPage1.getByRole("textbox", { name: "Name" }).fill("Welcome Room");
await ewPage1.getByRole("button", { name: "Create room" }).click();

View File

@@ -332,42 +332,6 @@ describe("UrlParams", () => {
expect(computeUrlParams("?intent=join_existing").skipLobby).toBe(false);
});
});
describe("noiseSuppression", () => {
it("defaults to true", () => {
expect(computeUrlParams().noiseSuppression).toBe(true);
});
it("is parsed", () => {
expect(
computeUrlParams("?intent=start_call&noiseSuppression=true")
.noiseSuppression,
).toBe(true);
expect(
computeUrlParams("?intent=start_call&noiseSuppression&bar=foo")
.noiseSuppression,
).toBe(true);
expect(computeUrlParams("?noiseSuppression=false").noiseSuppression).toBe(
false,
);
});
});
describe("echoCancellation", () => {
it("defaults to true", () => {
expect(computeUrlParams().echoCancellation).toBe(true);
});
it("is parsed", () => {
expect(computeUrlParams("?echoCancellation=true").echoCancellation).toBe(
true,
);
expect(computeUrlParams("?echoCancellation=false").echoCancellation).toBe(
false,
);
});
});
describe("header", () => {
it("uses header if provided", () => {
expect(computeUrlParams("?header=app_bar&hideHeader=true").header).toBe(

View File

@@ -233,17 +233,6 @@ export interface UrlConfiguration {
*/
waitForCallPickup: boolean;
/**
* Whether to enable echo cancellation for audio capture.
* Defaults to true.
*/
echoCancellation?: boolean;
/**
* Whether to enable noise suppression for audio capture.
* Defaults to true.
*/
noiseSuppression?: boolean;
callIntent?: RTCCallIntent;
}
interface IntentAndPlatformDerivedConfiguration {
@@ -536,8 +525,6 @@ export const computeUrlParams = (search = "", hash = ""): UrlParams => {
]),
waitForCallPickup: parser.getFlag("waitForCallPickup"),
autoLeaveWhenOthersLeft: parser.getFlag("autoLeave"),
noiseSuppression: parser.getFlagParam("noiseSuppression", true),
echoCancellation: parser.getFlagParam("echoCancellation", true),
};
// Log the final configuration for debugging purposes.

View File

@@ -13,7 +13,7 @@ import { type ReactNode } from "react";
import { ReactionToggleButton } from "./ReactionToggleButton";
import { ElementCallReactionEventType } from "../reactions";
import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
import { type CallViewModel } from "../state/CallViewModel";
import { getBasicCallViewModelEnvironment } from "../utils/test-viewmodel";
import { alice, local, localRtcMember } from "../utils/test-fixtures";
import { type MockRTCSession } from "../utils/test";

View File

@@ -33,7 +33,7 @@ import {
ReactionsRowSize,
} from "../reactions";
import { Modal } from "../Modal";
import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
import { type CallViewModel } from "../state/CallViewModel";
import { useBehavior } from "../useBehavior";
interface InnerButtonProps extends ComponentPropsWithoutRef<"button"> {

View File

@@ -122,7 +122,7 @@ export interface ConfigOptions {
delayed_leave_event_delay_ms?: number;
/**
* The time (in milliseconds) after which we consider a delayed event restart http request to have failed.
* The time (in milliseconds) after which a we consider a delayed event restart http request to have failed.
* Setting this to a lower value will result in more frequent retries but also a higher chance of failiour.
*
* In the presence of network packet loss (hurting TCP connections), the custom delayedEventRestartLocalTimeoutMs

View File

@@ -28,7 +28,6 @@ import {
} from "../settings/settings";
import { BlurBackgroundTransformer } from "./BlurBackgroundTransformer";
import { type Behavior } from "../state/Behavior";
import { type ObservableScope } from "../state/ObservableScope";
//TODO-MULTI-SFU: This is not yet fully there.
// it is a combination of exposing observable and react hooks.
@@ -64,17 +63,13 @@ export function useTrackProcessorObservable$(): Observable<ProcessorState> {
return state$;
}
/**
* Updates your video tracks to always use the given processor.
*/
export const trackProcessorSync = (
scope: ObservableScope,
videoTrack$: Behavior<LocalVideoTrack | null>,
processor$: Behavior<ProcessorState>,
): void => {
combineLatest([videoTrack$, processor$])
.pipe(scope.bind())
.subscribe(([videoTrack, processorState]) => {
// TODO-MULTI-SFU: Bind to an ObservableScope to avoid leaking resources.
combineLatest([videoTrack$, processor$]).subscribe(
([videoTrack, processorState]) => {
if (!processorState) return;
if (!videoTrack) return;
const { processor } = processorState;
@@ -84,7 +79,8 @@ export const trackProcessorSync = (
if (!processor && videoTrack.getProcessor()) {
void videoTrack.stopProcessor();
}
});
},
);
};
export const useTrackProcessorSync = (

View File

@@ -21,16 +21,7 @@ export type OpenIDClientParts = Pick<
MatrixClient,
"getOpenIdToken" | "getDeviceId"
>;
/**
* Gets a bearer token from the homeserver and then use it to authenticate
* to the matrix RTC backend in order to get acces to the SFU.
* It has built-in retry for calls to the homeserver with a backoff policy.
* @param client
* @param serviceUrl
* @param matrixRoomId
* @returns Object containing the token information
* @throws FailToGetOpenIdToken
*/
export async function getSFUConfigWithOpenID(
client: OpenIDClientParts,
serviceUrl: string,

View File

@@ -62,7 +62,7 @@ Initializer.initBeforeReact()
.then(() => {
root.render(
<StrictMode>
<App vm={new AppViewModel(globalScope)} />
<App vm={new AppViewModel(globalScope)} />,
</StrictMode>,
);
})

View File

@@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { beforeEach, describe, expect, test, vi } from "vitest";
import { describe, expect, test } from "vitest";
import { render, configure } from "@testing-library/react";
import { RaisedHandIndicator } from "./RaisedHandIndicator";
@@ -15,13 +15,6 @@ configure({
});
describe("RaisedHandIndicator", () => {
const fixedTime = new Date("2025-01-01T12:00:00.000Z");
beforeEach(() => {
vi.useFakeTimers();
vi.setSystemTime(fixedTime);
});
test("renders nothing when no hand has been raised", () => {
const { container } = render(<RaisedHandIndicator />);
expect(container.firstChild).toBeNull();

View File

@@ -266,7 +266,6 @@ export class ReactionsReader {
);
return;
}
// TODO refactor to use memer id `membershipEvent.membershipID` (needs to happen in combination with other memberId refactors)
const identifier = `${membershipEvent.userId}:${membershipEvent.deviceId}`;
if (!content.emoji) {

View File

@@ -15,7 +15,7 @@ exports[`RaisedHandIndicator > renders a smaller indicator when miniature is spe
</span>
</div>
<p>
00:00
00:01
</p>
</div>
`;
@@ -35,7 +35,7 @@ exports[`RaisedHandIndicator > renders an indicator when a hand has been raised
</span>
</div>
<p>
00:00
00:01
</p>
</div>
`;
@@ -55,7 +55,7 @@ exports[`RaisedHandIndicator > renders an indicator when a hand has been raised
</span>
</div>
<p>
01:00
01:01
</p>
</div>
`;

View File

@@ -20,7 +20,7 @@ import { logger } from "matrix-js-sdk/lib/logger";
import { useMatrixRTCSessionMemberships } from "../useMatrixRTCSessionMemberships";
import { useClientState } from "../ClientContext";
import { ElementCallReactionEventType, type ReactionOption } from ".";
import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
import { type CallViewModel } from "../state/CallViewModel";
import { useBehavior } from "../useBehavior";
interface ReactionsSenderContextType {

View File

@@ -38,7 +38,7 @@ import {
local,
localRtcMember,
} from "../utils/test-fixtures";
import { MAX_PARTICIPANT_COUNT_FOR_SOUND } from "../state/CallViewModel/CallViewModel";
import { MAX_PARTICIPANT_COUNT_FOR_SOUND } from "../state/CallViewModel";
vitest.mock("livekit-client/e2ee-worker?worker");
vitest.mock("../useAudioContext");

View File

@@ -7,7 +7,7 @@ Please see LICENSE in the repository root for full details.
import { type ReactNode, useEffect } from "react";
import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
import { type CallViewModel } from "../state/CallViewModel";
import joinCallSoundMp3 from "../sound/join_call.mp3";
import joinCallSoundOgg from "../sound/join_call.ogg";
import leftCallSoundMp3 from "../sound/left_call.mp3";

View File

@@ -22,7 +22,6 @@ import {
WebBrowserIcon,
} from "@vector-im/compound-design-tokens/assets/web/icons";
import { Button } from "@vector-im/compound-web";
import { logger } from "matrix-js-sdk/lib/logger";
import {
ConnectionLostError,
@@ -54,7 +53,7 @@ const ErrorPage: FC<ErrorPageProps> = ({
widget,
}: ErrorPageProps): ReactElement => {
const { t } = useTranslation();
logger.error("Error boundary caught:", error);
let icon: ComponentType<SVGAttributes<SVGElement>>;
switch (error.category) {
case ErrorCategory.CONFIGURATION_ISSUE:

View File

@@ -29,7 +29,6 @@ import userEvent from "@testing-library/user-event";
import { type RelationsContainer } from "matrix-js-sdk/lib/models/relations-container";
import { useState } from "react";
import { TooltipProvider } from "@vector-im/compound-web";
import { type ITransport } from "matrix-widget-api";
import { prefetchSounds } from "../soundUtils";
import { useAudioContext } from "../useAudioContext";
@@ -44,7 +43,7 @@ import {
MockRTCSession,
} from "../utils/test";
import { GroupCallView } from "./GroupCallView";
import { ElementWidgetActions, type WidgetHelpers } from "../widget";
import { type WidgetHelpers } from "../widget";
import { LazyEventEmitter } from "../LazyEventEmitter";
import { MatrixRTCTransportMissingError } from "../utils/errors";
import { ProcessorProvider } from "../livekit/TrackProcessorContext";
@@ -78,13 +77,13 @@ const leaveRTCSession = vi.hoisted(() =>
),
);
// vi.mock("../rtcSessionHelpers", async (importOriginal) => {
// // TODO: perhaps there is a more elegant way to manage the type import here?
// // eslint-disable-next-line @typescript-eslint/consistent-type-imports
// const orig = await importOriginal<typeof import("../rtcSessionHelpers")>();
// // TODO: leaveRTCSession no longer exists! Tests need adapting.
// return { ...orig, enterRTCSession, leaveRTCSession };
// });
vi.mock("../rtcSessionHelpers", async (importOriginal) => {
// TODO: perhaps there is a more elegant way to manage the type import here?
// eslint-disable-next-line @typescript-eslint/consistent-type-imports
const orig = await importOriginal<typeof import("../rtcSessionHelpers")>();
// TODO: leaveRTCSession no longer exists! Tests need adapting.
return { ...orig, enterRTCSession, leaveRTCSession };
});
let playSound: MockedFunction<
NonNullable<ReturnType<typeof useAudioContext>>["playSound"]
@@ -113,10 +112,6 @@ beforeEach(() => {
return (
<div>
<button onClick={() => onLeave("user")}>Leave</button>
<button onClick={() => onLeave("allOthersLeft")}>
SimulateOtherLeft
</button>
<button onClick={() => onLeave("error")}>SimulateErrorLeft</button>
</div>
);
},
@@ -248,112 +243,6 @@ test.skip("GroupCallView plays a leave sound synchronously in widget mode", asyn
expect(leaveRTCSession).toHaveBeenCalledOnce();
});
test.skip("Should close widget when all other left and have time to play a sound", async () => {
const user = userEvent.setup();
const widgetClosedCalled = Promise.withResolvers<void>();
const widgetSendMock = vi.fn().mockImplementation((action: string) => {
if (action === ElementWidgetActions.Close) {
widgetClosedCalled.resolve();
}
});
const widgetStopMock = vi.fn().mockResolvedValue(undefined);
const widget = {
api: {
setAlwaysOnScreen: vi.fn().mockResolvedValue(true),
transport: {
send: widgetSendMock,
reply: vi.fn().mockResolvedValue(undefined),
stop: widgetStopMock,
} as unknown as ITransport,
} as Partial<WidgetHelpers["api"]>,
lazyActions: new LazyEventEmitter(),
};
const resolvePlaySound = Promise.withResolvers<void>();
playSound = vi.fn().mockReturnValue(resolvePlaySound);
(useAudioContext as MockedFunction<typeof useAudioContext>).mockReturnValue({
playSound,
playSoundLooping: vitest.fn(),
soundDuration: {},
});
const { getByText } = createGroupCallView(widget as WidgetHelpers);
const leaveButton = getByText("SimulateOtherLeft");
await user.click(leaveButton);
await flushPromises();
expect(widgetSendMock).not.toHaveBeenCalled();
resolvePlaySound.resolve();
await flushPromises();
expect(playSound).toHaveBeenCalledWith("left");
await widgetClosedCalled.promise;
await flushPromises();
expect(widgetStopMock).toHaveBeenCalledOnce();
});
test("Should close widget when all other left", async () => {
const user = userEvent.setup();
const widgetClosedCalled = Promise.withResolvers<void>();
const widgetSendMock = vi.fn().mockImplementation((action: string) => {
if (action === ElementWidgetActions.Close) {
widgetClosedCalled.resolve();
}
});
const widgetStopMock = vi.fn().mockResolvedValue(undefined);
const widget = {
api: {
setAlwaysOnScreen: vi.fn().mockResolvedValue(true),
transport: {
send: widgetSendMock,
reply: vi.fn().mockResolvedValue(undefined),
stop: widgetStopMock,
} as unknown as ITransport,
} as Partial<WidgetHelpers["api"]>,
lazyActions: new LazyEventEmitter(),
};
const { getByText } = createGroupCallView(widget as WidgetHelpers);
const leaveButton = getByText("SimulateOtherLeft");
await user.click(leaveButton);
await flushPromises();
await widgetClosedCalled.promise;
await flushPromises();
expect(widgetStopMock).toHaveBeenCalledOnce();
});
test("Should not close widget when auto leave due to error", async () => {
const user = userEvent.setup();
const widgetStopMock = vi.fn().mockResolvedValue(undefined);
const widgetSendMock = vi.fn().mockResolvedValue(undefined);
const widget = {
api: {
setAlwaysOnScreen: vi.fn().mockResolvedValue(true),
transport: {
send: widgetSendMock,
reply: vi.fn().mockResolvedValue(undefined),
stop: widgetStopMock,
} as unknown as ITransport,
} as Partial<WidgetHelpers["api"]>,
lazyActions: new LazyEventEmitter(),
};
const alwaysOnScreenSpy = vi.spyOn(widget.api, "setAlwaysOnScreen");
const { getByText } = createGroupCallView(widget as WidgetHelpers);
const leaveButton = getByText("SimulateErrorLeft");
await user.click(leaveButton);
await flushPromises();
// When onLeft is called, we first set always on screen to false
await waitFor(() => expect(alwaysOnScreenSpy).toHaveBeenCalledWith(false));
await flushPromises();
// But then we do not close the widget automatically
expect(widgetStopMock).not.toHaveBeenCalledOnce();
expect(widgetSendMock).not.toHaveBeenCalledOnce();
});
test.skip("GroupCallView leaves the session when an error occurs", async () => {
(ActiveCall as MockedFunction<typeof ActiveCall>).mockImplementation(() => {
const [error, setError] = useState<Error | null>(null);
@@ -377,7 +266,6 @@ test.skip("GroupCallView leaves the session when an error occurs", async () => {
test.skip("GroupCallView shows errors that occur during joining", async () => {
const user = userEvent.setup();
// This should not mock this error that deep. it should only mock the CallViewModel.
enterRTCSession.mockRejectedValue(new MatrixRTCTransportMissingError(""));
onTestFinished(() => {
enterRTCSession.mockReset();

View File

@@ -159,7 +159,6 @@ export const GroupCallView: FC<Props> = ({
};
}, [rtcSession]);
// TODO move this into the callViewModel LocalMembership.ts
useTypedEventEmitter(
rtcSession,
MatrixRTCSessionEvent.MembershipManagerError,
@@ -314,9 +313,7 @@ export const GroupCallView: FC<Props> = ({
const navigate = useNavigate();
const onLeft = useCallback(
(
reason: "timeout" | "user" | "allOthersLeft" | "decline" | "error",
): void => {
(reason: "timeout" | "user" | "allOthersLeft" | "decline"): void => {
let playSound: CallEventSounds = "left";
if (reason === "timeout" || reason === "decline") playSound = reason;
@@ -369,7 +366,7 @@ export const GroupCallView: FC<Props> = ({
}
// On a normal user hangup we can shut down and close the widget. But if an
// error occurs we should keep the widget open until the user reads it.
if (reason != "error" && !getUrlParams().returnToLobby) {
if (reason === "user" && !getUrlParams().returnToLobby) {
try {
await widget.api.transport.send(ElementWidgetActions.Close, {});
} catch (e) {
@@ -521,7 +518,8 @@ export const GroupCallView: FC<Props> = ({
}}
onError={
(/**error*/) => {
if (rtcSession.isJoined()) onLeft("error");
// TODO this should not be "user". It needs a new case
if (rtcSession.isJoined()) onLeft("user");
}
}
>

View File

@@ -43,6 +43,9 @@ import { LivekitRoomAudioRenderer } from "../livekit/MatrixAudioRenderer";
import { MediaDevicesContext } from "../MediaDevicesContext";
import { HeaderStyle } from "../UrlParams";
// vi.hoisted(() => {
// localStorage = {} as unknown as Storage;
// });
vi.hoisted(
() =>
(global.ImageData = class MockImageData {
@@ -106,7 +109,6 @@ function createInCallView(): RenderResult & {
getUserId: () => localRtcMember.userId,
getDeviceId: () => localRtcMember.deviceId,
getRoom: (rId) => (rId === roomId ? room : null),
getDomain: () => "example.com",
} as Partial<MatrixClient> as MatrixClient;
const room = mockMatrixRoom({
relations: {
@@ -117,8 +119,7 @@ function createInCallView(): RenderResult & {
} as unknown as RelationsContainer,
client,
roomId,
// getMember: (userId) => roomMembers.get(userId) ?? null,
getMembers: () => Array.from(roomMembers.values()),
getMember: (userId) => roomMembers.get(userId) ?? null,
getMxcAvatarUrl: () => null,
hasEncryptionStateEvent: vi.fn().mockReturnValue(true),
getCanonicalAlias: () => null,

View File

@@ -58,11 +58,7 @@ import { type MuteStates } from "../state/MuteStates";
import { type MatrixInfo } from "./VideoPreview";
import { InviteButton } from "../button/InviteButton";
import { LayoutToggle } from "./LayoutToggle";
import {
type CallViewModel,
createCallViewModel$,
type GridMode,
} from "../state/CallViewModel/CallViewModel.ts";
import { CallViewModel, type GridMode } from "../state/CallViewModel";
import { Grid, type TileProps } from "../grid/Grid";
import { useInitial } from "../useInitial";
import { SpotlightTile } from "../tile/SpotlightTile";
@@ -115,23 +111,21 @@ export interface ActiveCallProps
extends Omit<InCallViewProps, "vm" | "livekitRoom" | "connState"> {
e2eeSystem: EncryptionSystem;
// TODO refactor those reasons into an enum
onLeft: (
reason: "user" | "timeout" | "decline" | "allOthersLeft" | "error",
) => void;
onLeft: (reason: "user" | "timeout" | "decline" | "allOthersLeft") => void;
}
export const ActiveCall: FC<ActiveCallProps> = (props) => {
const mediaDevices = useMediaDevices();
const [vm, setVm] = useState<CallViewModel | null>(null);
const urlParams = useUrlParams();
const mediaDevices = useMediaDevices();
const { autoLeaveWhenOthersLeft, waitForCallPickup, sendNotificationType } =
useUrlParams();
const trackProcessorState$ = useTrackProcessorObservable$();
useEffect(() => {
const scope = new ObservableScope();
const reactionsReader = new ReactionsReader(scope, props.rtcSession);
const { autoLeaveWhenOthersLeft, waitForCallPickup, sendNotificationType } =
urlParams;
const vm = createCallViewModel$(
const vm = new CallViewModel(
scope,
props.rtcSession,
props.matrixRoom,
@@ -146,8 +140,6 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
reactionsReader.reactions$,
scope.behavior(trackProcessorState$),
);
// TODO move this somewhere else once we use the callViewModel in the lobby as well!
vm.join();
setVm(vm);
vm.leave$.pipe(scope.bind()).subscribe(props.onLeft);
@@ -157,11 +149,13 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
}, [
props.rtcSession,
props.matrixRoom,
mediaDevices,
props.muteStates,
props.e2eeSystem,
autoLeaveWhenOthersLeft,
sendNotificationType,
waitForCallPickup,
props.onLeft,
urlParams,
mediaDevices,
trackProcessorState$,
]);
@@ -253,6 +247,7 @@ export const InCallView: FC<InCallViewProps> = ({
() => void toggleRaisedHand(),
);
// const allLivekitRooms = useBehavior(vm.allLivekitRooms$);
const audioParticipants = useBehavior(vm.audioParticipants$);
const participantCount = useBehavior(vm.participantCount$);
const reconnecting = useBehavior(vm.reconnecting$);
@@ -267,8 +262,7 @@ export const InCallView: FC<InCallViewProps> = ({
const audioOutputSwitcher = useBehavior(vm.audioOutputSwitcher$);
const sharingScreen = useBehavior(vm.sharingScreen$);
const ringOverlay = useBehavior(vm.ringOverlay$);
const fatalCallError = useBehavior(vm.fatalError$);
const fatalCallError = useBehavior(vm.configError$);
// Stop the rendering and throw for the error boundary
if (fatalCallError) throw fatalCallError;
@@ -304,26 +298,47 @@ export const InCallView: FC<InCallViewProps> = ({
// Waiting UI overlay
const waitingOverlay: JSX.Element | null = useMemo(() => {
return ringOverlay ? (
// No overlay if not in ringing state
if (callPickupState !== "ringing") return null;
// Use room state for other participants data (the one that we likely want to reach)
// TODO: this screams it wants to be a behavior in the vm.
const roomOthers = [
...matrixRoom.getMembersWithMembership("join"),
...matrixRoom.getMembersWithMembership("invite"),
].filter((m) => m.userId !== client.getUserId());
// Yield if there are not other members in the room.
if (roomOthers.length === 0) return null;
const otherMember = roomOthers.length > 0 ? roomOthers[0] : undefined;
const isOneOnOne = roomOthers.length === 1 && otherMember;
const text = isOneOnOne
? `Waiting for ${otherMember.name ?? otherMember.userId} to join…`
: "Waiting for other participants…";
const avatarMxc = isOneOnOne
? (otherMember.getMxcAvatarUrl?.() ?? undefined)
: (matrixRoom.getMxcAvatarUrl() ?? undefined);
return (
<div className={classNames(overlayStyles.bg, waitingStyles.overlay)}>
<div
className={classNames(overlayStyles.content, waitingStyles.content)}
>
<div className={waitingStyles.pulse}>
<Avatar
id={ringOverlay.idForAvatar}
name={ringOverlay.name}
src={ringOverlay.avatarMxc}
id={isOneOnOne ? otherMember.userId : matrixRoom.roomId}
name={isOneOnOne ? otherMember.name : matrixRoom.name}
src={avatarMxc}
size={AvatarSize.XL}
/>
</div>
<Text size="md" className={waitingStyles.text}>
{ringOverlay.text}
{text}
</Text>
</div>
</div>
) : null;
}, [ringOverlay]);
);
}, [callPickupState, client, matrixRoom]);
// Ideally we could detect taps by listening for click events and checking
// that the pointerType of the event is "touch", but this isn't yet supported
@@ -804,7 +819,7 @@ export const InCallView: FC<InCallViewProps> = ({
key={url}
url={url}
livekitRoom={livekitRoom}
validIdentities={participants}
validIdentities={participants.map((p) => p.identity)}
muted={muteAllAudio}
/>
))}

View File

@@ -10,9 +10,8 @@ import {
afterAll,
afterEach,
beforeEach,
describe,
expect,
it,
test,
vitest,
type MockedFunction,
type Mock,
@@ -28,7 +27,7 @@ import {
import { useAudioContext } from "../useAudioContext";
import { GenericReaction, ReactionSet } from "../reactions";
import { prefetchSounds } from "../soundUtils";
import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
import { type CallViewModel } from "../state/CallViewModel";
import { getBasicCallViewModelEnvironment } from "../utils/test-viewmodel";
import {
alice,
@@ -50,125 +49,122 @@ vitest.mock("livekit-client/e2ee-worker?worker");
vitest.mock("../useAudioContext");
vitest.mock("../soundUtils");
afterEach(() => {
vitest.resetAllMocks();
playReactionsSoundSetting.setValue(playReactionsSoundSetting.defaultValue);
soundEffectVolumeSetting.setValue(soundEffectVolumeSetting.defaultValue);
});
afterAll(() => {
vitest.restoreAllMocks();
});
let playSound: Mock<
NonNullable<ReturnType<typeof useAudioContext>>["playSound"]
>;
describe("ReactionAudioRenderer", () => {
afterEach(() => {
playReactionsSoundSetting.setValue(playReactionsSoundSetting.defaultValue);
soundEffectVolumeSetting.setValue(soundEffectVolumeSetting.defaultValue);
beforeEach(() => {
(prefetchSounds as MockedFunction<typeof prefetchSounds>).mockResolvedValue({
sound: new ArrayBuffer(0),
});
beforeEach(() => {
(prefetchSounds as MockedFunction<typeof prefetchSounds>).mockResolvedValue(
{
sound: new ArrayBuffer(0),
},
);
playSound = vitest.fn();
(useAudioContext as MockedFunction<typeof useAudioContext>).mockReturnValue(
{
playSound,
playSoundLooping: vitest.fn(),
soundDuration: {},
},
);
});
afterAll(() => {
vitest.restoreAllMocks();
});
it("preloads all audio elements", () => {
const { vm } = getBasicCallViewModelEnvironment([local, alice]);
playReactionsSoundSetting.setValue(true);
render(<TestComponent vm={vm} />);
expect(prefetchSounds).toHaveBeenCalledOnce();
});
it("will play an audio sound when there is a reaction", () => {
const { vm, reactionsSubject$ } = getBasicCallViewModelEnvironment([
local,
alice,
]);
playReactionsSoundSetting.setValue(true);
render(<TestComponent vm={vm} />);
// Find the first reaction with a sound effect
const chosenReaction = ReactionSet.find((r) => !!r.sound);
if (!chosenReaction) {
throw Error(
"No reactions have sounds configured, this test cannot succeed",
);
}
act(() => {
reactionsSubject$.next({
[aliceRtcMember.deviceId]: {
reactionOption: chosenReaction,
expireAfter: new Date(0),
},
});
});
expect(playSound).toHaveBeenCalledWith(chosenReaction.name);
});
it("will play the generic audio sound when there is soundless reaction", () => {
const { vm, reactionsSubject$ } = getBasicCallViewModelEnvironment([
local,
alice,
]);
playReactionsSoundSetting.setValue(true);
render(<TestComponent vm={vm} />);
// Find the first reaction with a sound effect
const chosenReaction = ReactionSet.find((r) => !r.sound);
if (!chosenReaction) {
throw Error(
"No reactions have sounds configured, this test cannot succeed",
);
}
act(() => {
reactionsSubject$.next({
[aliceRtcMember.deviceId]: {
reactionOption: chosenReaction,
expireAfter: new Date(0),
},
});
});
expect(playSound).toHaveBeenCalledWith(GenericReaction.name);
});
it("will play multiple audio sounds when there are multiple different reactions", () => {
const { vm, reactionsSubject$ } = getBasicCallViewModelEnvironment([
local,
alice,
]);
playReactionsSoundSetting.setValue(true);
render(<TestComponent vm={vm} />);
// Find the first reaction with a sound effect
const [reaction1, reaction2] = ReactionSet.filter((r) => !!r.sound);
if (!reaction1 || !reaction2) {
throw Error(
"No reactions have sounds configured, this test cannot succeed",
);
}
act(() => {
reactionsSubject$.next({
[aliceRtcMember.deviceId]: {
reactionOption: reaction1,
expireAfter: new Date(0),
},
[bobRtcMember.deviceId]: {
reactionOption: reaction2,
expireAfter: new Date(0),
},
[localRtcMember.deviceId]: {
reactionOption: reaction1,
expireAfter: new Date(0),
},
});
});
expect(playSound).toHaveBeenCalledWith(reaction1.name);
expect(playSound).toHaveBeenCalledWith(reaction2.name);
playSound = vitest.fn();
(useAudioContext as MockedFunction<typeof useAudioContext>).mockReturnValue({
playSound,
playSoundLooping: vitest.fn(),
soundDuration: {},
});
});
test("preloads all audio elements", () => {
const { vm } = getBasicCallViewModelEnvironment([local, alice]);
playReactionsSoundSetting.setValue(true);
render(<TestComponent vm={vm} />);
expect(prefetchSounds).toHaveBeenCalledOnce();
});
test("will play an audio sound when there is a reaction", () => {
const { vm, reactionsSubject$ } = getBasicCallViewModelEnvironment([
local,
alice,
]);
playReactionsSoundSetting.setValue(true);
render(<TestComponent vm={vm} />);
// Find the first reaction with a sound effect
const chosenReaction = ReactionSet.find((r) => !!r.sound);
if (!chosenReaction) {
throw Error(
"No reactions have sounds configured, this test cannot succeed",
);
}
act(() => {
reactionsSubject$.next({
[aliceRtcMember.deviceId]: {
reactionOption: chosenReaction,
expireAfter: new Date(0),
},
});
});
expect(playSound).toHaveBeenCalledWith(chosenReaction.name);
});
test("will play the generic audio sound when there is soundless reaction", () => {
const { vm, reactionsSubject$ } = getBasicCallViewModelEnvironment([
local,
alice,
]);
playReactionsSoundSetting.setValue(true);
render(<TestComponent vm={vm} />);
// Find the first reaction with a sound effect
const chosenReaction = ReactionSet.find((r) => !r.sound);
if (!chosenReaction) {
throw Error(
"No reactions have sounds configured, this test cannot succeed",
);
}
act(() => {
reactionsSubject$.next({
[aliceRtcMember.deviceId]: {
reactionOption: chosenReaction,
expireAfter: new Date(0),
},
});
});
expect(playSound).toHaveBeenCalledWith(GenericReaction.name);
});
test("will play multiple audio sounds when there are multiple different reactions", () => {
const { vm, reactionsSubject$ } = getBasicCallViewModelEnvironment([
local,
alice,
]);
playReactionsSoundSetting.setValue(true);
render(<TestComponent vm={vm} />);
// Find the first reaction with a sound effect
const [reaction1, reaction2] = ReactionSet.filter((r) => !!r.sound);
if (!reaction1 || !reaction2) {
throw Error(
"No reactions have sounds configured, this test cannot succeed",
);
}
act(() => {
reactionsSubject$.next({
[aliceRtcMember.deviceId]: {
reactionOption: reaction1,
expireAfter: new Date(0),
},
[bobRtcMember.deviceId]: {
reactionOption: reaction2,
expireAfter: new Date(0),
},
[localRtcMember.deviceId]: {
reactionOption: reaction1,
expireAfter: new Date(0),
},
});
});
expect(playSound).toHaveBeenCalledWith(reaction1.name);
expect(playSound).toHaveBeenCalledWith(reaction2.name);
});

View File

@@ -12,7 +12,7 @@ import { GenericReaction, ReactionSet } from "../reactions";
import { useAudioContext } from "../useAudioContext";
import { prefetchSounds } from "../soundUtils";
import { useLatest } from "../useLatest";
import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
import { type CallViewModel } from "../state/CallViewModel";
const soundMap = Object.fromEntries([
...ReactionSet.filter((v) => v.sound !== undefined).map((v) => [

View File

@@ -8,7 +8,7 @@ Please see LICENSE in the repository root for full details.
import { type ReactNode } from "react";
import styles from "./ReactionsOverlay.module.css";
import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
import { type CallViewModel } from "../state/CallViewModel";
import { useBehavior } from "../useBehavior";
export function ReactionsOverlay({ vm }: { vm: CallViewModel }): ReactNode {

View File

@@ -83,6 +83,9 @@ exports[`InCallView > rendering > renders 1`] = `
class="nav rightNav"
/>
</header>
<div>
mocked: MatrixAudioRenderer
</div>
<div
class="scrollingGrid grid"
>

View File

@@ -0,0 +1,159 @@
/*
Copyright 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc";
import { expect, test, vi } from "vitest";
import { AutoDiscovery } from "matrix-js-sdk/lib/autodiscovery";
import EventEmitter from "events";
import { enterRTCSession } from "../src/rtcSessionHelpers";
import { mockConfig } from "./utils/test";
import { MatrixRTCMode } from "./settings/settings";
const MATRIX_RTC_MODE = MatrixRTCMode.Legacy;
const getUrlParams = vi.hoisted(() => vi.fn(() => ({})));
vi.mock("./UrlParams", () => ({ getUrlParams }));
const actualWidget = await vi.hoisted(async () => vi.importActual("./widget"));
vi.mock("./widget", () => ({
...actualWidget,
widget: {
api: {
setAlwaysOnScreen: (): void => {},
transport: { send: vi.fn(), reply: vi.fn(), stop: vi.fn() },
},
lazyActions: new EventEmitter(),
},
}));
test("It joins the correct Session", async () => {
const focusFromOlderMembership = {
type: "livekit",
livekit_service_url: "http://my-oldest-member-service-url.com",
livekit_alias: "my-oldest-member-service-alias",
};
const focusConfigFromWellKnown = {
type: "livekit",
livekit_service_url: "http://my-well-known-service-url.com",
};
const focusConfigFromWellKnown2 = {
type: "livekit",
livekit_service_url: "http://my-well-known-service-url2.com",
};
const clientWellKnown = {
"org.matrix.msc4143.rtc_foci": [
focusConfigFromWellKnown,
focusConfigFromWellKnown2,
],
};
mockConfig({
livekit: { livekit_service_url: "http://my-default-service-url.com" },
});
vi.spyOn(AutoDiscovery, "getRawClientConfig").mockImplementation(
async (domain) => {
if (domain === "example.org") {
return Promise.resolve(clientWellKnown);
}
return Promise.resolve({});
},
);
const mockedSession = vi.mocked({
room: {
roomId: "roomId",
client: {
getDomain: vi.fn().mockReturnValue("example.org"),
getOpenIdToken: vi.fn().mockResolvedValue({
access_token: "ACCCESS_TOKEN",
token_type: "Bearer",
matrix_server_name: "localhost",
expires_in: 10000,
}),
},
},
memberships: [],
getFocusInUse: vi.fn().mockReturnValue(focusFromOlderMembership),
getOldestMembership: vi.fn().mockReturnValue({
getPreferredFoci: vi.fn().mockReturnValue([focusFromOlderMembership]),
}),
joinRoomSession: vi.fn(),
}) as unknown as MatrixRTCSession;
await enterRTCSession(
mockedSession,
{
livekit_alias: "roomId",
livekit_service_url: "http://my-well-known-service-url.com",
type: "livekit",
},
{
encryptMedia: true,
matrixRTCMode: MATRIX_RTC_MODE,
},
);
expect(mockedSession.joinRoomSession).toHaveBeenLastCalledWith(
[
{
livekit_alias: "roomId",
livekit_service_url: "http://my-well-known-service-url.com",
type: "livekit",
},
],
undefined,
expect.objectContaining({
manageMediaKeys: true,
useLegacyMemberEvents: false,
}),
);
});
test("It should not fail with configuration error if homeserver config has livekit url but not fallback", async () => {
mockConfig({});
vi.spyOn(AutoDiscovery, "getRawClientConfig").mockResolvedValue({
"org.matrix.msc4143.rtc_foci": [
{
type: "livekit",
livekit_service_url: "http://my-well-known-service-url.com",
},
],
});
const mockedSession = vi.mocked({
room: {
roomId: "roomId",
client: {
getDomain: vi.fn().mockReturnValue("example.org"),
getOpenIdToken: vi.fn().mockResolvedValue({
access_token: "ACCCESS_TOKEN",
token_type: "Bearer",
matrix_server_name: "localhost",
expires_in: 10000,
}),
},
},
memberships: [],
getFocusInUse: vi.fn(),
joinRoomSession: vi.fn(),
}) as unknown as MatrixRTCSession;
await enterRTCSession(
mockedSession,
{
livekit_alias: "roomId",
livekit_service_url: "http://my-well-known-service-url.com",
type: "livekit",
},
{
encryptMedia: true,
matrixRTCMode: MATRIX_RTC_MODE,
},
);
});

160
src/rtcSessionHelpers.ts Normal file
View File

@@ -0,0 +1,160 @@
/*
Copyright 2023, 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
type MatrixRTCSession,
isLivekitTransportConfig,
type LivekitTransportConfig,
type LivekitTransport,
} from "matrix-js-sdk/lib/matrixrtc";
import { logger } from "matrix-js-sdk/lib/logger";
import { AutoDiscovery } from "matrix-js-sdk/lib/autodiscovery";
import { type MatrixClient } from "matrix-js-sdk";
import { PosthogAnalytics } from "./analytics/PosthogAnalytics";
import { Config } from "./config/Config";
import { ElementWidgetActions, widget } from "./widget";
import { MatrixRTCTransportMissingError } from "./utils/errors";
import { getUrlParams } from "./UrlParams";
import { getSFUConfigWithOpenID } from "./livekit/openIDSFU.ts";
import { MatrixRTCMode } from "./settings/settings.ts";
const FOCI_WK_KEY = "org.matrix.msc4143.rtc_foci";
async function makeTransportInternal(
client: MatrixClient,
roomId: string,
): Promise<LivekitTransport> {
logger.log("Searching for a preferred transport");
//TODO refactor this to use the jwt service returned alias.
const livekitAlias = roomId;
// TODO-MULTI-SFU: Either remove this dev tool or make it more official
const urlFromStorage =
localStorage.getItem("robin-matrixrtc-auth") ??
localStorage.getItem("timo-focus-url");
if (urlFromStorage !== null) {
const transportFromStorage: LivekitTransport = {
type: "livekit",
livekit_service_url: urlFromStorage,
livekit_alias: livekitAlias,
};
logger.log(
"Using LiveKit transport from local storage: ",
transportFromStorage,
);
return transportFromStorage;
}
// Prioritize the .well-known/matrix/client, if available, over the configured SFU
const domain = client.getDomain();
if (domain) {
// we use AutoDiscovery instead of relying on the MatrixClient having already
// been fully configured and started
const wellKnownFoci = (await AutoDiscovery.getRawClientConfig(domain))?.[
FOCI_WK_KEY
];
if (Array.isArray(wellKnownFoci)) {
const transport: LivekitTransportConfig | undefined = wellKnownFoci.find(
(f) => f && isLivekitTransportConfig(f),
);
if (transport !== undefined) {
logger.log("Using LiveKit transport from .well-known: ", transport);
return { ...transport, livekit_alias: livekitAlias };
}
}
}
const urlFromConf = Config.get().livekit?.livekit_service_url;
if (urlFromConf) {
const transportFromConf: LivekitTransport = {
type: "livekit",
livekit_service_url: urlFromConf,
livekit_alias: livekitAlias,
};
logger.log("Using LiveKit transport from config: ", transportFromConf);
return transportFromConf;
}
throw new MatrixRTCTransportMissingError(domain ?? "");
}
export async function makeTransport(
client: MatrixClient,
roomId: string,
): Promise<LivekitTransport> {
const transport = await makeTransportInternal(client, roomId);
// this will call the jwt/sfu/get endpoint to pre create the livekit room.
await getSFUConfigWithOpenID(
client,
transport.livekit_service_url,
transport.livekit_alias,
);
return transport;
}
export interface EnterRTCSessionOptions {
encryptMedia: boolean;
matrixRTCMode: MatrixRTCMode;
}
/**
* TODO! document this function properly
* @param rtcSession
* @param transport
* @param options
*/
export async function enterRTCSession(
rtcSession: MatrixRTCSession,
transport: LivekitTransport,
{ encryptMedia, matrixRTCMode }: EnterRTCSessionOptions,
): Promise<void> {
PosthogAnalytics.instance.eventCallEnded.cacheStartCall(new Date());
PosthogAnalytics.instance.eventCallStarted.track(rtcSession.room.roomId);
// This must be called before we start trying to join the call, as we need to
// have started tracking by the time calls start getting created.
// groupCallOTelMembership?.onJoinCall();
const { features, matrix_rtc_session: matrixRtcSessionConfig } = Config.get();
const useDeviceSessionMemberEvents =
features?.feature_use_device_session_member_events;
const { sendNotificationType: notificationType, callIntent } = getUrlParams();
const multiSFU = matrixRTCMode !== MatrixRTCMode.Legacy;
// Multi-sfu does not need a preferred foci list. just the focus that is actually used.
rtcSession.joinRoomSession(
multiSFU ? [] : [transport],
multiSFU ? transport : undefined,
{
notificationType,
callIntent,
manageMediaKeys: encryptMedia,
...(useDeviceSessionMemberEvents !== undefined && {
useLegacyMemberEvents: !useDeviceSessionMemberEvents,
}),
delayedLeaveEventRestartMs:
matrixRtcSessionConfig?.delayed_leave_event_restart_ms,
delayedLeaveEventDelayMs:
matrixRtcSessionConfig?.delayed_leave_event_delay_ms,
delayedLeaveEventRestartLocalTimeoutMs:
matrixRtcSessionConfig?.delayed_leave_event_restart_local_timeout_ms,
networkErrorRetryMs: matrixRtcSessionConfig?.network_error_retry_ms,
makeKeyDelay: matrixRtcSessionConfig?.wait_for_key_rotation_ms,
membershipEventExpiryMs:
matrixRtcSessionConfig?.membership_event_expiry_ms,
useExperimentalToDeviceTransport: true,
unstableSendStickyEvents: matrixRTCMode === MatrixRTCMode.Matrix_2_0,
},
);
if (widget) {
try {
await widget.api.transport.send(ElementWidgetActions.JoinCall, {});
} catch (e) {
logger.error("Failed to send join action", e);
}
}
}

View File

@@ -1,95 +0,0 @@
/*
Copyright 2025 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { describe, expect, it, vi } from "vitest";
import { render, waitFor } from "@testing-library/react";
import type { MatrixClient } from "matrix-js-sdk";
import type { Room as LivekitRoom } from "livekit-client";
import { DeveloperSettingsTab } from "./DeveloperSettingsTab";
// Mock url params hook to avoid environment-dependent snapshot churn.
vi.mock("../UrlParams", () => ({
useUrlParams: (): { mocked: boolean; answer: number } => ({
mocked: true,
answer: 42,
}),
}));
// Provide a minimal mock of a Livekit Room structure used by the component.
function createMockLivekitRoom(
wsUrl: string,
serverInfo: object,
metadata: string,
): { isLocal: boolean; url: string; room: LivekitRoom } {
const mockRoom = {
serverInfo,
metadata,
engine: { client: { ws: { url: wsUrl } } },
} as unknown as LivekitRoom;
return {
isLocal: true,
url: wsUrl,
room: mockRoom,
};
}
// Minimal MatrixClient mock with only the methods used by the component.
function createMockMatrixClient(): MatrixClient {
return {
doesServerSupportUnstableFeature: vi.fn().mockResolvedValue(true), // ensure stickyEventsSupported eventually becomes true
getCrypto: (): { getVersion: () => string } | undefined => ({
getVersion: () => "crypto-1.0.0",
}),
getUserId: () => "@alice:example.org",
getDeviceId: () => "DEVICE123",
} as unknown as MatrixClient;
}
describe("DeveloperSettingsTab", () => {
it("renders and matches snapshot", async () => {
const client = createMockMatrixClient();
const livekitRooms: {
room: LivekitRoom;
url: string;
isLocal?: boolean;
}[] = [
createMockLivekitRoom(
"wss://local-sfu.example.org",
{ region: "local", version: "1.2.3" },
"local-metadata",
),
{
isLocal: false,
url: "wss://remote-sfu.example.org",
room: {
serverInfo: { region: "remote", version: "4.5.6" },
metadata: "remote-metadata",
engine: { client: { ws: { url: "wss://remote-sfu.example.org" } } },
} as unknown as LivekitRoom,
},
];
const { container } = render(
<DeveloperSettingsTab
client={client}
livekitRooms={livekitRooms}
env={{ MY_MOCK_ENV: 10, ENV: "test" } as unknown as ImportMetaEnv}
/>,
);
// Wait for the async sticky events feature check to resolve so the final UI
// (e.g. enabled Matrix_2_0 radio button) appears deterministically.
await waitFor(() =>
expect(client.doesServerSupportUnstableFeature).toHaveBeenCalled(),
);
expect(container).toMatchSnapshot();
});
});

View File

@@ -11,7 +11,6 @@ import {
useCallback,
useEffect,
useMemo,
useId,
useState,
} from "react";
import { useTranslation } from "react-i18next";
@@ -20,15 +19,6 @@ import {
type MatrixClient,
} from "matrix-js-sdk";
import { logger } from "matrix-js-sdk/lib/logger";
import {
EditInPlace,
Root as Form,
Heading,
HelpMessage,
InlineField,
Label,
RadioControl,
} from "@vector-im/compound-web";
import { FieldRow, InputField } from "../input/Input";
import {
@@ -36,10 +26,10 @@ import {
duplicateTiles as duplicateTilesSetting,
debugTileLayout as debugTileLayoutSetting,
showConnectionStats as showConnectionStatsSetting,
multiSfu as multiSfuSetting,
muteAllAudio as muteAllAudioSetting,
alwaysShowIphoneEarpiece as alwaysShowIphoneEarpieceSetting,
matrixRTCMode as matrixRTCModeSetting,
customLivekitUrl as customLivekitUrlSetting,
MatrixRTCMode,
} from "./settings";
import type { Room as LivekitRoom } from "livekit-client";
@@ -49,14 +39,9 @@ import { useUrlParams } from "../UrlParams";
interface Props {
client: MatrixClient;
livekitRooms?: { room: LivekitRoom; url: string; isLocal?: boolean }[];
env: ImportMetaEnv;
}
export const DeveloperSettingsTab: FC<Props> = ({
client,
livekitRooms,
env,
}) => {
export const DeveloperSettingsTab: FC<Props> = ({ client, livekitRooms }) => {
const { t } = useTranslation();
const [duplicateTiles, setDuplicateTiles] = useSetting(duplicateTilesSetting);
const [debugTileLayout, setDebugTileLayout] = useSetting(
@@ -76,13 +61,6 @@ export const DeveloperSettingsTab: FC<Props> = ({
}, [client]);
const [matrixRTCMode, setMatrixRTCMode] = useSetting(matrixRTCModeSetting);
const matrixRTCModeRadioGroup = useId();
const onMatrixRTCModeChange = useCallback(
(e: ChangeEvent<HTMLInputElement>) => {
setMatrixRTCMode(e.target.value as MatrixRTCMode);
},
[setMatrixRTCMode],
);
const [showConnectionStats, setShowConnectionStats] = useSetting(
showConnectionStatsSetting,
@@ -92,15 +70,6 @@ export const DeveloperSettingsTab: FC<Props> = ({
alwaysShowIphoneEarpieceSetting,
);
const [customLivekitUrl, setCustomLivekitUrl] = useSetting(
customLivekitUrlSetting,
);
const [customLivekitUrlTextBuffer, setCustomLivekitUrlTextBuffer] =
useState(customLivekitUrl);
useEffect(() => {
setCustomLivekitUrlTextBuffer(customLivekitUrl);
}, [customLivekitUrl]);
const [muteAllAudio, setMuteAllAudio] = useSetting(muteAllAudioSetting);
const urlParams = useUrlParams();
@@ -174,6 +143,52 @@ export const DeveloperSettingsTab: FC<Props> = ({
}
/>
</FieldRow>
<FieldRow>
<InputField
id="matrixRTCMode"
type="checkbox"
label={t("developer_mode.matrixRTCMode.Legacy.label")}
description={t("developer_mode.matrixRTCMode.Legacy.description")}
checked={matrixRTCMode === MatrixRTCMode.Legacy}
onChange={useCallback(
(event: ChangeEvent<HTMLInputElement>) => {
if (event.target.checked) setMatrixRTCMode(MatrixRTCMode.Legacy);
},
[setMatrixRTCMode],
)}
/>
<InputField
id="matrixRTCMode"
type="checkbox"
label={t("developer_mode.matrixRTCMode.Comptibility.label")}
description={t(
"developer_mode.matrixRTCMode.Comptibility.description",
)}
checked={matrixRTCMode === MatrixRTCMode.Compatibil}
onChange={useCallback(
(event: ChangeEvent<HTMLInputElement>) => {
if (event.target.checked)
setMatrixRTCMode(MatrixRTCMode.Compatibil);
},
[setMatrixRTCMode],
)}
/>
<InputField
id="matrixRTCMode"
type="checkbox"
label={t("developer_mode.matrixRTCMode.Matrix_2_0.label")}
description={t("developer_mode.matrixRTCMode.Matrix_2_0.description")}
disabled={!stickyEventsSupported}
checked={matrixRTCMode === MatrixRTCMode.Matrix_2_0}
onChange={useCallback(
(event: ChangeEvent<HTMLInputElement>) => {
if (event.target.checked)
setMatrixRTCMode(MatrixRTCMode.Matrix_2_0);
},
[setMatrixRTCMode],
)}
/>
</FieldRow>
<FieldRow>
<InputField
id="showConnectionStats"
@@ -216,93 +231,6 @@ export const DeveloperSettingsTab: FC<Props> = ({
)}
/>{" "}
</FieldRow>
<EditInPlace
onSubmit={(e) => e.preventDefault()}
helpLabel={
customLivekitUrl === null
? t("developer_mode.custom_livekit_url.from_config")
: t("developer_mode.custom_livekit_url.current_url") +
customLivekitUrl
}
label={t("developer_mode.custom_livekit_url.label")}
saveButtonLabel={t("developer_mode.custom_livekit_url.save")}
savingLabel={t("developer_mode.custom_livekit_url.saving")}
cancelButtonLabel={t("developer_mode.custom_livekit_url.reset")}
onSave={useCallback(
(e: React.FormEvent<HTMLFormElement>) => {
setCustomLivekitUrl(
customLivekitUrlTextBuffer === ""
? null
: customLivekitUrlTextBuffer,
);
},
[setCustomLivekitUrl, customLivekitUrlTextBuffer],
)}
value={customLivekitUrlTextBuffer ?? ""}
onChange={useCallback(
(event: ChangeEvent<HTMLInputElement>): void => {
setCustomLivekitUrlTextBuffer(event.target.value);
},
[setCustomLivekitUrlTextBuffer],
)}
onCancel={useCallback(
(e: React.FormEvent<HTMLFormElement>) => {
setCustomLivekitUrl(null);
},
[setCustomLivekitUrl],
)}
/>
<Heading as="h3" type="body" weight="semibold" size="lg">
{t("developer_mode.matrixRTCMode.title")}
</Heading>
<Form>
<InlineField
name={matrixRTCModeRadioGroup}
control={
<RadioControl
checked={matrixRTCMode === MatrixRTCMode.Legacy}
value={MatrixRTCMode.Legacy}
onChange={onMatrixRTCModeChange}
/>
}
>
<Label>{t("developer_mode.matrixRTCMode.Legacy.label")}</Label>
<HelpMessage>
{t("developer_mode.matrixRTCMode.Legacy.description")}
</HelpMessage>
</InlineField>
<InlineField
name={matrixRTCModeRadioGroup}
control={
<RadioControl
checked={matrixRTCMode === MatrixRTCMode.Compatibil}
value={MatrixRTCMode.Compatibil}
onChange={onMatrixRTCModeChange}
/>
}
>
<Label>{t("developer_mode.matrixRTCMode.Comptibility.label")}</Label>
<HelpMessage>
{t("developer_mode.matrixRTCMode.Comptibility.description")}
</HelpMessage>
</InlineField>
<InlineField
name={matrixRTCModeRadioGroup}
control={
<RadioControl
checked={matrixRTCMode === MatrixRTCMode.Matrix_2_0}
value={MatrixRTCMode.Matrix_2_0}
disabled={!stickyEventsSupported}
onChange={onMatrixRTCModeChange}
/>
}
>
<Label>{t("developer_mode.matrixRTCMode.Matrix_2_0.label")}</Label>
<HelpMessage>
{t("developer_mode.matrixRTCMode.Matrix_2_0.description")}
</HelpMessage>
</InlineField>
</Form>
{livekitRooms?.map((livekitRoom) => (
<>
<h3>
@@ -324,7 +252,7 @@ export const DeveloperSettingsTab: FC<Props> = ({
</>
))}
<p>{t("developer_mode.environment_variables")}</p>
<pre>{JSON.stringify(env, null, 2)}</pre>
<pre>{JSON.stringify(import.meta.env, null, 2)}</pre>
<p>{t("developer_mode.url_params")}</p>
<pre>{JSON.stringify(urlParams, null, 2)}</pre>
</>

View File

@@ -209,11 +209,7 @@ export const SettingsModal: FC<Props> = ({
key: "developer",
name: t("settings.developer_tab_title"),
content: (
<DeveloperSettingsTab
env={import.meta.env}
client={client}
livekitRooms={livekitRooms}
/>
<DeveloperSettingsTab client={client} livekitRooms={livekitRooms} />
),
};

View File

@@ -1,411 +0,0 @@
// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
<div>
<p>
Hostname: localhost
</p>
<p>
Element Call version: dev
</p>
<p>
Crypto version: crypto-1.0.0
</p>
<p>
Matrix ID: @alice:example.org
</p>
<p>
Device ID: DEVICE123
</p>
<div
class="fieldRow"
>
<div
class="field inputField"
>
<input
aria-describedby="«r1»"
id="duplicateTiles"
min="0"
type="number"
value="0"
/>
<label
for="duplicateTiles"
>
Number of additional tile copies per participant
</label>
</div>
</div>
<div
class="fieldRow"
>
<div
class="field checkboxField"
>
<input
aria-describedby="«r2»"
id="debugTileLayout"
type="checkbox"
/>
<label
for="debugTileLayout"
>
<div
class="checkbox"
>
<svg
fill="none"
height="24"
stroke="#000"
stroke-linecap="round"
stroke-linejoin="round"
stroke-width="2"
viewBox="0 0 24 24"
width="24"
xmlns="http://www.w3.org/2000/svg"
>
<path
d="m20 6-11 11-5-5"
/>
</svg>
</div>
Debug tile layout
</label>
</div>
</div>
<div
class="fieldRow"
>
<div
class="field checkboxField"
>
<input
aria-describedby="«r3»"
id="showConnectionStats"
type="checkbox"
/>
<label
for="showConnectionStats"
>
<div
class="checkbox"
>
<svg
fill="none"
height="24"
stroke="#000"
stroke-linecap="round"
stroke-linejoin="round"
stroke-width="2"
viewBox="0 0 24 24"
width="24"
xmlns="http://www.w3.org/2000/svg"
>
<path
d="m20 6-11 11-5-5"
/>
</svg>
</div>
Show connection statistics
</label>
</div>
</div>
<div
class="fieldRow"
>
<div
class="field checkboxField"
>
<input
aria-describedby="«r4»"
id="muteAllAudio"
type="checkbox"
/>
<label
for="muteAllAudio"
>
<div
class="checkbox"
>
<svg
fill="none"
height="24"
stroke="#000"
stroke-linecap="round"
stroke-linejoin="round"
stroke-width="2"
viewBox="0 0 24 24"
width="24"
xmlns="http://www.w3.org/2000/svg"
>
<path
d="m20 6-11 11-5-5"
/>
</svg>
</div>
Mute all audio (participants, reactions, join sounds)
</label>
</div>
</div>
<div
class="fieldRow"
>
<div
class="field checkboxField"
>
<input
aria-describedby="«r5»"
id="alwaysShowIphoneEarpiece"
type="checkbox"
/>
<label
for="alwaysShowIphoneEarpiece"
>
<div
class="checkbox"
>
<svg
fill="none"
height="24"
stroke="#000"
stroke-linecap="round"
stroke-linejoin="round"
stroke-width="2"
viewBox="0 0 24 24"
width="24"
xmlns="http://www.w3.org/2000/svg"
>
<path
d="m20 6-11 11-5-5"
/>
</svg>
</div>
Show iPhone earpiece option on all platforms
</label>
</div>
</div>
<form
class="_root_19upo_16"
>
<div
class="_field_19upo_26"
>
<label
class="_label_19upo_59"
for="radix-«r6»"
>
Custom Livekit-url
</label>
<div
class="_controls_17lij_8"
>
<input
aria-describedby="radix-«r7»"
class="_control_sqdq4_10"
id="radix-«r6»"
name="input"
title=""
value=""
/>
</div>
<span
class="_message_19upo_85 _help-message_19upo_91"
id="radix-«r7»"
>
Currently, no overwrite is set. Url from well-known or config is used.
</span>
</div>
</form>
<h3
class="_typography_6v6n8_153 _font-body-lg-semibold_6v6n8_74"
>
MatrixRTC mode
</h3>
<form
class="_root_19upo_16"
>
<div
class="_inline-field_19upo_32"
>
<div
class="_inline-field-control_19upo_44"
>
<div
class="_container_1e0uz_10"
>
<input
aria-describedby="radix-«r9» radix-«rb» radix-«rd»"
checked=""
class="_input_1e0uz_18"
id="radix-«r8»"
name="«r0»"
title=""
type="radio"
value="legacy"
/>
<div
class="_ui_1e0uz_19"
/>
</div>
</div>
<div
class="_inline-field-body_19upo_38"
>
<label
class="_label_19upo_59"
for="radix-«r8»"
>
Legacy: state events & oldest membership SFU
</label>
<span
class="_message_19upo_85 _help-message_19upo_91"
id="radix-«r9»"
>
Compatible with old versions of EC that do not support multi SFU
</span>
</div>
</div>
<div
class="_inline-field_19upo_32"
>
<div
class="_inline-field-control_19upo_44"
>
<div
class="_container_1e0uz_10"
>
<input
aria-describedby="radix-«r9» radix-«rb» radix-«rd»"
class="_input_1e0uz_18"
id="radix-«ra»"
name="«r0»"
title=""
type="radio"
value="compatibil"
/>
<div
class="_ui_1e0uz_19"
/>
</div>
</div>
<div
class="_inline-field-body_19upo_38"
>
<label
class="_label_19upo_59"
for="radix-«ra»"
>
Compatibility: state events & multi SFU
</label>
<span
class="_message_19upo_85 _help-message_19upo_91"
id="radix-«rb»"
>
Compatible with homeservers that do not support sticky events (but all other EC clients are v0.17.0 or later)
</span>
</div>
</div>
<div
class="_inline-field_19upo_32"
>
<div
class="_inline-field-control_19upo_44"
>
<div
class="_container_1e0uz_10"
>
<input
aria-describedby="radix-«r9» radix-«rb» radix-«rd»"
class="_input_1e0uz_18"
id="radix-«rc»"
name="«r0»"
title=""
type="radio"
value="matrix_2_0"
/>
<div
class="_ui_1e0uz_19"
/>
</div>
</div>
<div
class="_inline-field-body_19upo_38"
>
<label
class="_label_19upo_59"
for="radix-«rc»"
>
Matrix 2.0: sticky events & multi SFU
</label>
<span
class="_message_19upo_85 _help-message_19upo_91"
id="radix-«rd»"
>
Compatible only with homservers supporting sticky events and all EC clients v0.17.0 or later
</span>
</div>
</div>
</form>
<h3>
LiveKit SFU: wss://local-sfu.example.org
</h3>
<p>
ws-url:
wss://local-sfu.example.org/
</p>
<p>
LiveKit Server Info
(
local
)
</p>
<pre
class="pre"
>
{
"region": "local",
"version": "1.2.3"
}
local-metadata
</pre>
<h3>
LiveKit SFU: wss://remote-sfu.example.org
</h3>
<p>
LiveKit Server Info
(
remote
)
</p>
<pre
class="pre"
>
{
"region": "remote",
"version": "4.5.6"
}
remote-metadata
</pre>
<p>
Environment variables
</p>
<pre>
{
"MY_MOCK_ENV": 10,
"ENV": "test"
}
</pre>
<p>
URL parameters
</p>
<pre>
{
"mocked": true,
"answer": 42
}
</pre>
</div>
`;

View File

@@ -134,8 +134,3 @@ export const matrixRTCMode = new Setting<MatrixRTCMode>(
"matrix-rtc-mode",
MatrixRTCMode.Legacy,
);
export const customLivekitUrl = new Setting<string | null>(
"custom-livekit-url",
null,
);

59
src/state/Async.ts Normal file
View File

@@ -0,0 +1,59 @@
/*
Copyright 2025 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { catchError, from, map, type Observable, of, startWith } from "rxjs";
/**
* Data that may need to be loaded asynchronously.
*
* This type is for when you need to represent the current state of an operation
* involving Promises as **immutable data**. See the async$ function below.
*/
export type Async<A> =
| { state: "loading" }
| { state: "error"; value: Error }
| { state: "ready"; value: A };
export const loading: Async<never> = { state: "loading" };
export function error(value: Error): Async<never> {
return { state: "error", value };
}
export function ready<A>(value: A): Async<A> {
return { state: "ready", value };
}
/**
* Turn a Promise into an Observable async value. The Observable will have the
* value "loading" while the Promise is pending, "ready" when the Promise
* resolves, and "error" when the Promise rejects.
*/
export function async$<A>(promise: Promise<A>): Observable<Async<A>> {
return from(promise).pipe(
map(ready),
startWith(loading),
catchError((e: unknown) =>
of(error((e as Error) ?? new Error("Unknown error"))),
),
);
}
/**
* If the async value is ready, apply the given function to the inner value.
*/
export function mapAsync<A, B>(
async: Async<A>,
project: (value: A) => B,
): Async<B> {
return async.state === "ready" ? ready(project(async.value)) : async;
}
export function unwrapAsync<A>(fallback: A): (async: Async<A>) => A {
return (async: Async<A>) => {
return async.state === "ready" ? async.value : fallback;
};
}

View File

@@ -16,10 +16,7 @@ import { BehaviorSubject } from "rxjs";
* distinction between Behaviors and Observables, see
* https://monoid.dk/post/behaviors-and-streams-why-both/.
*/
export type Behavior<T> = Omit<
BehaviorSubject<T>,
"next" | "observers" | "error"
>;
export type Behavior<T> = Omit<BehaviorSubject<T>, "next" | "observers">;
/**
* Creates a Behavior which never changes in value.

File diff suppressed because it is too large Load Diff

1369
src/state/CallViewModel.ts Normal file

File diff suppressed because it is too large Load Diff

View File

@@ -1,354 +0,0 @@
/*
Copyright 2025 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
type ICallNotifyContent,
type IRTCNotificationContent,
} from "matrix-js-sdk/lib/matrixrtc";
import { describe, it } from "vitest";
import {
EventType,
type IEvent,
type IRoomTimelineData,
MatrixEvent,
type Room,
} from "matrix-js-sdk";
import { withTestScheduler } from "../../utils/test";
import {
aliceRtcMember,
local,
localRtcMember,
} from "../../utils/test-fixtures";
import {
createCallNotificationLifecycle$,
type Props as CallNotificationLifecycleProps,
} from "./CallNotificationLifecycle";
import { trackEpoch } from "../ObservableScope";
const mockLegacyRingEvent = {} as { event_id: string } & ICallNotifyContent;
function mockRingEvent(
eventId: string,
lifetimeMs: number | undefined,
sender = local.userId,
): { event_id: string } & IRTCNotificationContent {
return {
event_id: eventId,
...(lifetimeMs === undefined ? {} : { lifetime: lifetimeMs }),
notification_type: "ring",
sender,
} as unknown as { event_id: string } & IRTCNotificationContent;
}
describe("waitForCallPickup$", () => {
it("unknown -> ringing -> timeout when notified and nobody joins", () => {
withTestScheduler(({ scope, expectObservable, behavior, hot }) => {
// No one ever joins (only local user)
const props: CallNotificationLifecycleProps = {
scope,
memberships$: scope.behavior(
behavior("a", { a: [] }).pipe(trackEpoch()),
),
sentCallNotification$: hot("10ms a", {
a: [mockRingEvent("$notif1", 30), mockLegacyRingEvent],
}),
receivedDecline$: hot(""),
options: {
waitForCallPickup: true,
autoLeaveWhenOthersLeft: false,
},
localUser: localRtcMember,
};
const lifecycle = createCallNotificationLifecycle$(props);
expectObservable(lifecycle.callPickupState$).toBe("a 9ms b 29ms c", {
a: "unknown",
b: "ringing",
c: "timeout",
});
});
});
it("ringing -> success if someone joins before timeout is reached", () => {
withTestScheduler(({ scope, hot, behavior, expectObservable }) => {
// Someone joins at 20ms (both LiveKit participant and MatrixRTC member)
const props: CallNotificationLifecycleProps = {
scope,
memberships$: scope.behavior(
behavior("a 19ms b", {
a: [localRtcMember],
b: [localRtcMember, aliceRtcMember],
}).pipe(trackEpoch()),
),
sentCallNotification$: hot("5ms a", {
a: [mockRingEvent("$notif2", 100), mockLegacyRingEvent],
}),
receivedDecline$: hot(""),
options: {
waitForCallPickup: true,
autoLeaveWhenOthersLeft: false,
},
localUser: localRtcMember,
};
const lifecycle = createCallNotificationLifecycle$(props);
expectObservable(lifecycle.callPickupState$).toBe("a 4ms b 14ms c", {
a: "unknown",
b: "ringing",
c: "success",
});
});
});
it("success when someone joins before we notify", () => {
withTestScheduler(({ scope, hot, behavior, expectObservable }) => {
// Someone joins at 20ms (both LiveKit participant and MatrixRTC member)
const props: CallNotificationLifecycleProps = {
scope,
memberships$: scope.behavior(
behavior("a 9ms b", {
a: [localRtcMember],
b: [localRtcMember, aliceRtcMember],
}).pipe(trackEpoch()),
),
sentCallNotification$: hot("20ms a", {
a: [mockRingEvent("$notif2", 50), mockLegacyRingEvent],
}),
receivedDecline$: hot(""),
options: {
waitForCallPickup: true,
autoLeaveWhenOthersLeft: false,
},
localUser: localRtcMember,
};
const lifecycle = createCallNotificationLifecycle$(props);
expectObservable(lifecycle.callPickupState$).toBe("a 9ms b", {
a: "unknown",
b: "success",
});
});
});
it("notify without lifetime -> immediate timeout", () => {
withTestScheduler(({ scope, hot, behavior, expectObservable }) => {
// Someone joins at 20ms (both LiveKit participant and MatrixRTC member)
const props: CallNotificationLifecycleProps = {
scope,
memberships$: scope.behavior(
behavior("a", {
a: [localRtcMember],
}).pipe(trackEpoch()),
),
sentCallNotification$: hot("10ms a", {
a: [mockRingEvent("$notif2", undefined), mockLegacyRingEvent],
}),
receivedDecline$: hot(""),
options: {
waitForCallPickup: true,
autoLeaveWhenOthersLeft: false,
},
localUser: localRtcMember,
};
const lifecycle = createCallNotificationLifecycle$(props);
expectObservable(lifecycle.callPickupState$).toBe("a 9ms b", {
a: "unknown",
b: "timeout",
});
});
});
it("stays null when waitForCallPickup=false", () => {
withTestScheduler(({ scope, hot, behavior, expectObservable }) => {
// Someone joins at 20ms (both LiveKit participant and MatrixRTC member)
const validProps: CallNotificationLifecycleProps = {
scope,
memberships$: scope.behavior(
behavior("a--b", {
a: [localRtcMember],
b: [localRtcMember, aliceRtcMember],
}).pipe(trackEpoch()),
),
sentCallNotification$: hot("10ms a", {
a: [mockRingEvent("$notif5", 30), mockLegacyRingEvent],
}),
receivedDecline$: hot(""),
options: {
waitForCallPickup: true,
autoLeaveWhenOthersLeft: false,
},
localUser: localRtcMember,
};
const propsDeactivated = {
...validProps,
options: {
...validProps.options,
waitForCallPickup: false,
},
};
const lifecycle = createCallNotificationLifecycle$(propsDeactivated);
expectObservable(lifecycle.callPickupState$).toBe("n", {
n: null,
});
const lifecycleReference = createCallNotificationLifecycle$(validProps);
expectObservable(lifecycleReference.callPickupState$).toBe("u--s", {
u: "unknown",
s: "success",
});
});
});
it("decline before timeout window ends -> decline", () => {
withTestScheduler(({ scope, hot, behavior, expectObservable }) => {
// Someone joins at 20ms (both LiveKit participant and MatrixRTC member)
const props: CallNotificationLifecycleProps = {
scope,
memberships$: scope.behavior(
behavior("a", {
a: [localRtcMember],
}).pipe(trackEpoch()),
),
sentCallNotification$: hot("10ms a", {
a: [mockRingEvent("$decl1", 50), mockLegacyRingEvent],
}),
receivedDecline$: hot("40ms d", {
d: [
new MatrixEvent({
type: EventType.RTCDecline,
content: {
"m.relates_to": {
rel_type: "m.reference",
event_id: "$decl1",
},
},
}),
{} as Room,
undefined,
false,
{} as IRoomTimelineData,
],
}),
options: {
waitForCallPickup: true,
autoLeaveWhenOthersLeft: false,
},
localUser: localRtcMember,
};
const lifecycle = createCallNotificationLifecycle$(props);
expectObservable(lifecycle.callPickupState$).toBe("a 9ms b 29ms e", {
a: "unknown",
b: "ringing",
e: "decline",
});
});
});
it("decline after timeout window ends -> stays timeout", () => {
withTestScheduler(({ scope, hot, behavior, expectObservable }) => {
// Someone joins at 20ms (both LiveKit participant and MatrixRTC member)
const props: CallNotificationLifecycleProps = {
scope,
memberships$: scope.behavior(
behavior("a", {
a: [localRtcMember],
}).pipe(trackEpoch()),
),
sentCallNotification$: hot("10ms a", {
a: [mockRingEvent("$decl", 20), mockLegacyRingEvent],
}),
receivedDecline$: hot("40ms d", {
d: [
new MatrixEvent({
type: EventType.RTCDecline,
content: {
"m.relates_to": {
rel_type: "m.reference",
event_id: "$decl",
},
},
}),
{} as Room,
undefined,
false,
{} as IRoomTimelineData,
],
}),
options: {
waitForCallPickup: true,
autoLeaveWhenOthersLeft: false,
},
localUser: localRtcMember,
};
const lifecycle = createCallNotificationLifecycle$(props);
expectObservable(lifecycle.callPickupState$, "50ms !").toBe(
"a 9ms b 19ms e",
{
a: "unknown",
b: "ringing",
e: "timeout",
},
);
});
});
//
function testStaysRinging(
declineEvent: Partial<IEvent>,
expectDecline: boolean,
): void {
withTestScheduler(({ scope, hot, behavior, expectObservable }) => {
// Someone joins at 20ms (both LiveKit participant and MatrixRTC member)
const props: CallNotificationLifecycleProps = {
scope,
memberships$: scope.behavior(
behavior("a", {
a: [localRtcMember],
}).pipe(trackEpoch()),
),
sentCallNotification$: hot("10ms a", {
a: [mockRingEvent("$right", 50), mockLegacyRingEvent],
}),
receivedDecline$: hot("20ms d", {
d: [
new MatrixEvent(declineEvent),
{} as Room,
undefined,
false,
{} as IRoomTimelineData,
],
}),
options: {
waitForCallPickup: true,
autoLeaveWhenOthersLeft: false,
},
localUser: localRtcMember,
};
const lifecycle = createCallNotificationLifecycle$(props);
const marbles = expectDecline ? "a 9ms b 9ms d" : "a 9ms b";
expectObservable(lifecycle.callPickupState$, "21ms !").toBe(marbles, {
a: "unknown",
b: "ringing",
d: "decline",
});
});
}
const reference = (refId?: string, sender?: string): Partial<IEvent> => ({
event_id: "$decline",
type: EventType.RTCDecline,
sender: sender ?? "@other:example.org",
content: {
"m.relates_to": {
rel_type: "m.reference",
event_id: refId ?? "$right",
},
},
});
it("decline reference works", () => {
testStaysRinging(reference(), true);
});
it("decline with wrong id is ignored (stays ringing)", () => {
testStaysRinging(reference("$wrong"), false);
});
it("decline with wrong id is ignored (stays ringing)", () => {
testStaysRinging(reference(undefined, local.userId), false);
});
});

View File

@@ -1,211 +0,0 @@
/*
Copyright 2025 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
type CallMembership,
type MatrixRTCSession,
MatrixRTCSessionEvent,
type MatrixRTCSessionEventHandlerMap,
} from "matrix-js-sdk/lib/matrixrtc";
import {
combineLatest,
concat,
endWith,
filter,
fromEvent,
ignoreElements,
map,
merge,
NEVER,
type Observable,
of,
pairwise,
startWith,
switchMap,
takeUntil,
timer,
} from "rxjs";
import {
type EventTimelineSetHandlerMap,
EventType,
type Room as MatrixRoom,
RoomEvent,
} from "matrix-js-sdk";
import { type Behavior } from "../Behavior";
import { type Epoch, mapEpoch, type ObservableScope } from "../ObservableScope";
export type AutoLeaveReason = "allOthersLeft" | "timeout" | "decline";
export type CallPickupState =
| "unknown"
| "ringing"
| "timeout"
| "decline"
| "success"
| null;
export type CallNotificationWrapper = Parameters<
MatrixRTCSessionEventHandlerMap[MatrixRTCSessionEvent.DidSendCallNotification]
>;
export function createSentCallNotification$(
scope: ObservableScope,
matrixRTCSession: MatrixRTCSession,
): Behavior<CallNotificationWrapper | null> {
const sentCallNotification$ = scope.behavior(
fromEvent(matrixRTCSession, MatrixRTCSessionEvent.DidSendCallNotification),
null,
) as Behavior<CallNotificationWrapper | null>;
return sentCallNotification$;
}
export function createReceivedDecline$(
matrixRoom: MatrixRoom,
): Observable<Parameters<EventTimelineSetHandlerMap[RoomEvent.Timeline]>> {
return (
fromEvent(matrixRoom, RoomEvent.Timeline) as Observable<
Parameters<EventTimelineSetHandlerMap[RoomEvent.Timeline]>
>
).pipe(filter(([event]) => event.getType() === EventType.RTCDecline));
}
export interface Props {
scope: ObservableScope;
memberships$: Behavior<Epoch<CallMembership[]>>;
sentCallNotification$: Observable<CallNotificationWrapper | null>;
receivedDecline$: Observable<
Parameters<EventTimelineSetHandlerMap[RoomEvent.Timeline]>
>;
options: { waitForCallPickup?: boolean; autoLeaveWhenOthersLeft?: boolean };
localUser: { deviceId: string; userId: string };
}
/**
* @returns {callPickupState$, autoLeave$}
* `callPickupState$` The current call pickup state of the call.
* - "unknown": The client has not yet sent the notification event. We don't know if it will because it first needs to send its own membership.
* Then we can conclude if we were the first one to join or not.
* This may also be set if we are disconnected.
* - "ringing": The call is ringing on other devices in this room (This client should give audiovisual feedback that this is happening).
* - "timeout": No-one picked up in the defined time this call should be ringing on others devices.
* The call failed. If desired this can be used as a trigger to exit the call.
* - "success": Someone else joined. The call is in a normal state. No audiovisual feedback.
* - null: EC is configured to never show any waiting for answer state.
*
* `autoLeave$` An observable that emits (null) when the call should be automatically left.
* - if options.autoLeaveWhenOthersLeft is set to true it emits when all others left.
* - if options.waitForCallPickup is set to true it emits if noone picked up the ring or if the ring got declined.
* - if options.autoLeaveWhenOthersLeft && options.waitForCallPickup is false it will never emit.
*
*/
export function createCallNotificationLifecycle$({
scope,
memberships$,
sentCallNotification$,
receivedDecline$,
options,
localUser,
}: Props): {
callPickupState$: Behavior<CallPickupState>;
autoLeave$: Observable<AutoLeaveReason>;
} {
const allOthersLeft$ = memberships$.pipe(
pairwise(),
filter(
([{ value: prev }, { value: current }]) =>
current.every((m) => m.userId === localUser.userId) &&
prev.some((m) => m.userId !== localUser.userId),
),
map(() => {}),
);
/**
* Whether some Matrix user other than ourself is joined to the call.
*/
const someoneElseJoined$ = memberships$.pipe(
mapEpoch((ms) => ms.some((m) => m.userId !== localUser.userId)),
) as Behavior<Epoch<boolean>>;
/**
* Whenever the RTC session tells us that it intends to ring the remote
* participant's devices, this emits an Observable tracking the current state of
* that ringing process.
*/
// This is a behavior since we need to store the latest state for when we subscribe to this after `didSendCallNotification$`
// has already emitted but we still need the latest observable with a timeout timer that only gets created on after receiving `notificationEvent`.
// A behavior will emit the latest observable with the running timer to new subscribers.
// see also: callPickupState$ and in particular the line: `return this.ring$.pipe(mergeAll());` here we otherwise might get an EMPTY observable if
// `ring$` would not be a behavior.
const remoteRingState$: Behavior<"ringing" | "timeout" | "decline" | null> =
scope.behavior(
sentCallNotification$.pipe(
filter(
(newAndLegacyEvents) =>
// only care about new events (legacy do not have decline pattern)
newAndLegacyEvents?.[0].notification_type === "ring",
),
map((e) => e as CallNotificationWrapper),
switchMap(([notificationEvent]) => {
const lifetimeMs = notificationEvent?.lifetime ?? 0;
return concat(
lifetimeMs === 0
? // If no lifetime, skip the ring state
of(null)
: // Ring until lifetime ms have passed
timer(lifetimeMs).pipe(
ignoreElements(),
startWith("ringing" as const),
),
// The notification lifetime has timed out, meaning ringing has likely
// stopped on all receiving clients.
of("timeout" as const),
// This makes sure we will not drop into the `endWith("decline" as const)` state
NEVER,
).pipe(
takeUntil(
receivedDecline$.pipe(
filter(
([event]) =>
event.getRelation()?.rel_type === "m.reference" &&
event.getRelation()?.event_id ===
notificationEvent.event_id &&
event.getSender() !== localUser.userId &&
callPickupState$.value !== "timeout",
),
),
),
endWith("decline" as const),
);
}),
),
null,
);
const callPickupState$ = scope.behavior(
options.waitForCallPickup === true
? combineLatest(
[someoneElseJoined$, remoteRingState$],
(someoneElseJoined, ring) => {
if (someoneElseJoined.value === true) {
return "success" as const;
}
// Show the ringing state of the most recent ringing attempt.
// as long as we have not yet sent an RTC notification event or noone else joined,
// ring will be null -> callPickupState$ = unknown.
return ring ?? ("unknown" as const);
},
)
: NEVER,
null,
);
const autoLeave$ = merge(
options.autoLeaveWhenOthersLeft === true
? allOthersLeft$.pipe(map(() => "allOthersLeft" as const))
: NEVER,
callPickupState$.pipe(
filter((state) => state === "timeout" || state === "decline"),
),
);
return { autoLeave$, callPickupState$ };
}

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -1,193 +0,0 @@
/*
Copyright 2025 Element Corp.
Copyright 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
ConnectionState,
type LocalParticipant,
type Participant,
ParticipantEvent,
type RemoteParticipant,
type Room as LivekitRoom,
} from "livekit-client";
import { SyncState } from "matrix-js-sdk/lib/sync";
import { BehaviorSubject, type Observable, map, of } from "rxjs";
import { onTestFinished, vi } from "vitest";
import { ClientEvent, type MatrixClient } from "matrix-js-sdk";
import EventEmitter from "events";
import * as ComponentsCore from "@livekit/components-core";
import type { CallMembership } from "matrix-js-sdk/lib/matrixrtc";
import { E2eeType } from "../../e2ee/e2eeType";
import { type RaisedHandInfo, type ReactionInfo } from "../../reactions";
import {
type CallViewModel,
createCallViewModel$,
type CallViewModelOptions,
} from "./CallViewModel";
import {
mockConfig,
mockLivekitRoom,
mockLocalParticipant,
mockMatrixRoom,
mockMatrixRoomMember,
mockMediaDevices,
mockMuteStates,
MockRTCSession,
testScope,
} from "../../utils/test";
import {
alice,
aliceDoppelganger,
bob,
bobZeroWidthSpace,
daveRTL,
daveRTLRtcMember,
local,
localRtcMember,
} from "../../utils/test-fixtures";
import { type Behavior, constant } from "../Behavior";
import { type ProcessorState } from "../../livekit/TrackProcessorContext";
import { type MediaDevices } from "../MediaDevices";
mockConfig({
livekit: { livekit_service_url: "http://my-default-service-url.com" },
});
const carol = local;
const dave = mockMatrixRoomMember(daveRTLRtcMember, { rawDisplayName: "Dave" });
const roomMembers = new Map(
[alice, aliceDoppelganger, bob, bobZeroWidthSpace, carol, dave, daveRTL].map(
(p) => [p.userId, p],
),
);
export interface CallViewModelInputs {
remoteParticipants$: Behavior<RemoteParticipant[]>;
rtcMembers$: Behavior<Partial<CallMembership>[]>;
livekitConnectionState$: Behavior<ConnectionState>;
speaking: Map<Participant, Observable<boolean>>;
mediaDevices: MediaDevices;
initialSyncState: SyncState;
}
const localParticipant = mockLocalParticipant({ identity: "" });
export function withCallViewModel(
{
remoteParticipants$ = constant([]),
rtcMembers$ = constant([localRtcMember]),
livekitConnectionState$: connectionState$ = constant(
ConnectionState.Connected,
),
speaking = new Map(),
mediaDevices = mockMediaDevices({}),
initialSyncState = SyncState.Syncing,
}: Partial<CallViewModelInputs> = {},
continuation: (
vm: CallViewModel,
rtcSession: MockRTCSession,
subjects: { raisedHands$: BehaviorSubject<Record<string, RaisedHandInfo>> },
setSyncState: (value: SyncState) => void,
) => void,
options: CallViewModelOptions = {
encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
autoLeaveWhenOthersLeft: false,
},
): void {
let syncState = initialSyncState;
const setSyncState = (value: SyncState): void => {
const prev = syncState;
syncState = value;
room.client.emit(ClientEvent.Sync, value, prev);
};
const room = mockMatrixRoom({
client: new (class extends EventEmitter {
public getUserId(): string | undefined {
return localRtcMember.userId;
}
public getDeviceId(): string {
return localRtcMember.deviceId;
}
public getDomain(): string {
return "example.com";
}
public getSyncState(): SyncState {
return syncState;
}
})() as Partial<MatrixClient> as MatrixClient,
getMembers: () => Array.from(roomMembers.values()),
getMembersWithMembership: () => Array.from(roomMembers.values()),
});
const rtcSession = new MockRTCSession(room, []).withMemberships(rtcMembers$);
const participantsSpy = vi
.spyOn(ComponentsCore, "connectedParticipantsObserver")
.mockReturnValue(remoteParticipants$);
const mediaSpy = vi
.spyOn(ComponentsCore, "observeParticipantMedia")
.mockImplementation((p) =>
of({ participant: p } as Partial<
ComponentsCore.ParticipantMedia<LocalParticipant>
> as ComponentsCore.ParticipantMedia<LocalParticipant>),
);
const eventsSpy = vi
.spyOn(ComponentsCore, "observeParticipantEvents")
.mockImplementation((p, ...eventTypes) => {
if (eventTypes.includes(ParticipantEvent.IsSpeakingChanged)) {
return (speaking.get(p) ?? of(false)).pipe(
map((s): Participant => ({ ...p, isSpeaking: s }) as Participant),
);
} else {
return of(p);
}
});
const roomEventSelectorSpy = vi
.spyOn(ComponentsCore, "roomEventSelector")
.mockImplementation((_room, _eventType) => of());
const muteStates = mockMuteStates();
const raisedHands$ = new BehaviorSubject<Record<string, RaisedHandInfo>>({});
const reactions$ = new BehaviorSubject<Record<string, ReactionInfo>>({});
const vm = createCallViewModel$(
testScope(),
rtcSession.asMockedSession(),
room,
mediaDevices,
muteStates,
{
...options,
livekitRoomFactory: (): LivekitRoom =>
mockLivekitRoom({
localParticipant,
disconnect: async () => Promise.resolve(),
setE2EEEnabled: async () => Promise.resolve(),
}),
connectionState$,
},
raisedHands$,
reactions$,
new BehaviorSubject<ProcessorState>({
processor: undefined,
supported: undefined,
}),
);
onTestFinished(() => {
participantsSpy.mockRestore();
mediaSpy.mockRestore();
eventsSpy.mockRestore();
roomEventSelectorSpy.mockRestore();
});
continuation(vm, rtcSession, { raisedHands$: raisedHands$ }, setSyncState);
}

View File

@@ -1,202 +0,0 @@
/*
Copyright 2025 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { afterEach, beforeEach, describe, expect, test } from "vitest";
import { firstValueFrom, of } from "rxjs";
import { createLayoutModeSwitch } from "./LayoutSwitch";
import { ObservableScope } from "../ObservableScope";
import { constant } from "../Behavior";
import { withTestScheduler } from "../../utils/test";
let scope: ObservableScope;
beforeEach(() => {
scope = new ObservableScope();
});
afterEach(() => {
scope.end();
});
describe("Default mode", () => {
test("Should be in grid layout by default", async () => {
const { gridMode$ } = createLayoutModeSwitch(
scope,
constant("normal"),
of(false),
);
const mode = await firstValueFrom(gridMode$);
expect(mode).toBe("grid");
});
test("Should switch to spotlight mode when window mode is flat", async () => {
const { gridMode$ } = createLayoutModeSwitch(
scope,
constant("flat"),
of(false),
);
const mode = await firstValueFrom(gridMode$);
expect(mode).toBe("spotlight");
});
});
test("Should allow switching modes manually", () => {
withTestScheduler(({ cold, behavior, expectObservable, schedule }): void => {
const { gridMode$, setGridMode } = createLayoutModeSwitch(
scope,
behavior("n", { n: "normal" }),
cold("f", { f: false, t: true }),
);
schedule("--sgs", {
s: () => setGridMode("spotlight"),
g: () => setGridMode("grid"),
});
expectObservable(gridMode$).toBe("g-sgs", {
g: "grid",
s: "spotlight",
});
});
});
test("Should switch to spotlight mode when there is a remote screen share", () => {
withTestScheduler(({ cold, behavior, expectObservable }): void => {
const shareMarble = "f--t";
const gridsMarble = "g--s";
const { gridMode$ } = createLayoutModeSwitch(
scope,
behavior("n", { n: "normal" }),
cold(shareMarble, { f: false, t: true }),
);
expectObservable(gridMode$).toBe(gridsMarble, {
g: "grid",
s: "spotlight",
});
});
});
test("Can manually force grid when there is a screenshare", () => {
withTestScheduler(({ cold, behavior, expectObservable, schedule }): void => {
const { gridMode$, setGridMode } = createLayoutModeSwitch(
scope,
behavior("n", { n: "normal" }),
cold("-ft", { f: false, t: true }),
);
schedule("---g", {
g: () => setGridMode("grid"),
});
expectObservable(gridMode$).toBe("ggsg", {
g: "grid",
s: "spotlight",
});
});
});
test("Should auto-switch after manually selected grid", () => {
withTestScheduler(({ cold, behavior, expectObservable, schedule }): void => {
const { gridMode$, setGridMode } = createLayoutModeSwitch(
scope,
behavior("n", { n: "normal" }),
// Two screenshares will happen in sequence
cold("-ft-ft", { f: false, t: true }),
);
// There was a screen-share that forced spotlight, then
// the user manually switch back to grid
schedule("---g", {
g: () => setGridMode("grid"),
});
// If we did want to respect manual selection, the expectation would be:
// const expectation = "ggsg";
const expectation = "ggsg-s";
expectObservable(gridMode$).toBe(expectation, {
g: "grid",
s: "spotlight",
});
});
});
test("Should switch back to grid mode when the remote screen share ends", () => {
withTestScheduler(({ cold, behavior, expectObservable }): void => {
const shareMarble = "f--t--f-";
const gridsMarble = "g--s--g-";
const { gridMode$ } = createLayoutModeSwitch(
scope,
behavior("n", { n: "normal" }),
cold(shareMarble, { f: false, t: true }),
);
expectObservable(gridMode$).toBe(gridsMarble, {
g: "grid",
s: "spotlight",
});
});
});
test("can auto-switch to spotlight again after first screen share ends", () => {
withTestScheduler(({ cold, behavior, expectObservable }): void => {
const shareMarble = "ftft";
const gridsMarble = "gsgs";
const { gridMode$ } = createLayoutModeSwitch(
scope,
behavior("n", { n: "normal" }),
cold(shareMarble, { f: false, t: true }),
);
expectObservable(gridMode$).toBe(gridsMarble, {
g: "grid",
s: "spotlight",
});
});
});
test("can switch manually to grid after screen share while manually in spotlight", () => {
withTestScheduler(({ cold, behavior, schedule, expectObservable }): void => {
// Initially, no one is sharing. Then the user manually switches to
// spotlight. After a screen share starts, the user manually switches to
// grid.
const shareMarbles = " f-t-";
const setModeMarbles = "-s-g";
const expectation = " gs-g";
const { gridMode$, setGridMode } = createLayoutModeSwitch(
scope,
behavior("n", { n: "normal" }),
cold(shareMarbles, { f: false, t: true }),
);
schedule(setModeMarbles, {
g: () => setGridMode("grid"),
s: () => setGridMode("spotlight"),
});
expectObservable(gridMode$).toBe(expectation, {
g: "grid",
s: "spotlight",
});
});
});
test("Should auto-switch to spotlight when in flat window mode", () => {
withTestScheduler(({ cold, behavior, expectObservable }): void => {
const { gridMode$ } = createLayoutModeSwitch(
scope,
behavior("naf", { n: "normal", a: "narrow", f: "flat" }),
cold("f", { f: false, t: true }),
);
expectObservable(gridMode$).toBe("g-s-", {
g: "grid",
s: "spotlight",
});
});
});

View File

@@ -1,130 +0,0 @@
/*
Copyright 2025 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
BehaviorSubject,
combineLatest,
map,
type Observable,
scan,
} from "rxjs";
import { logger } from "matrix-js-sdk/lib/logger";
import { type GridMode, type WindowMode } from "./CallViewModel.ts";
import { type Behavior } from "../Behavior.ts";
import { type ObservableScope } from "../ObservableScope.ts";
/**
* Creates a layout mode switch that allows switching between grid and spotlight modes.
* The actual layout mode can be overridden to spotlight mode if there is a remote screen share active
* or if the window mode is flat.
*
* @param scope - The observable scope to manage subscriptions.
* @param windowMode$ - The current window mode observable.
* @param hasRemoteScreenShares$ - An observable indicating if there are remote screen shares active.
*/
export function createLayoutModeSwitch(
scope: ObservableScope,
windowMode$: Behavior<WindowMode>,
hasRemoteScreenShares$: Observable<boolean>,
): {
gridMode$: Behavior<GridMode>;
setGridMode: (value: GridMode) => void;
} {
const gridModeUserSelection$ = new BehaviorSubject<GridMode>("grid");
// Callback to set the grid mode desired by the user.
// Notice that this is only a preference, the actual grid mode can be overridden
// if there is a remote screen share active.
const setGridMode = (value: GridMode): void => {
gridModeUserSelection$.next(value);
};
/**
* The layout mode of the media tile grid.
*/
const gridMode$ =
// If the user hasn't selected spotlight and somebody starts screen sharing,
// automatically switch to spotlight mode and reset when screen sharing ends
scope.behavior<GridMode>(
combineLatest([
gridModeUserSelection$,
hasRemoteScreenShares$,
windowMode$,
]).pipe(
// Scan to keep track if we have auto-switched already or not.
// To allow the user to override the auto-switch by selecting grid mode again.
scan<
[GridMode, boolean, WindowMode],
{
mode: GridMode;
/** Remember if the change was user driven or not */
hasAutoSwitched: boolean;
/** To know if it is new screen share or an already handled */
hasScreenShares: boolean;
}
>(
(prev, [userSelection, hasScreenShares, windowMode]) => {
const isFlatMode = windowMode === "flat";
// Always force spotlight in flat mode, grid layout is not supported
// in that mode.
// TODO: strange that we do that for flat mode but not for other modes?
// TODO: Why is this not handled in layoutMedia$ like other window modes?
if (isFlatMode) {
logger.debug(`Forcing spotlight mode, windowMode=${windowMode}`);
return {
mode: "spotlight",
hasAutoSwitched: prev.hasAutoSwitched,
hasScreenShares,
};
}
// User explicitly chose spotlight.
// Respect that choice.
if (userSelection === "spotlight") {
return {
mode: "spotlight",
hasAutoSwitched: prev.hasAutoSwitched,
hasScreenShares,
};
}
// User has chosen grid mode. If a screen share starts, we will
// auto-switch to spotlight mode for better experience.
// But we only do it once, if the user switches back to grid mode,
// we respect that choice until they explicitly change it again.
const isNewShare = hasScreenShares && !prev.hasScreenShares;
if (isNewShare && !prev.hasAutoSwitched) {
return {
mode: "spotlight",
hasAutoSwitched: true,
hasScreenShares: true,
};
}
// Respect user's grid choice
// XXX If we want to forbid switching automatically again after we can
// return hasAutoSwitched: acc.hasAutoSwitched here instead of setting to false.
return {
mode: "grid",
hasAutoSwitched: false,
hasScreenShares,
};
},
// initial value
{ mode: "grid", hasAutoSwitched: false, hasScreenShares: false },
),
map(({ mode }) => mode),
),
"grid",
);
return {
gridMode$,
setGridMode,
};
}

View File

@@ -1,202 +0,0 @@
/*
Copyright 2025 Element Creations Ltd.
Copyright 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { describe, it, expect, beforeEach, afterEach } from "vitest";
import { EventEmitter } from "events";
import { ClientEvent, SyncState } from "matrix-js-sdk";
import { MembershipManagerEvent, Status } from "matrix-js-sdk/lib/matrixrtc";
import { ObservableScope } from "../../ObservableScope";
import { createHomeserverConnected$ } from "./HomeserverConnected";
/**
* Minimal stub of a Matrix client sufficient for our tests:
```
createHomeserverConnected$(
scope: ObservableScope,
client: NodeStyleEventEmitter & Pick<MatrixClient, "getSyncState">,
matrixRTCSession: NodeStyleEventEmitter &
Pick<MatrixRTCSession, "membershipStatus" | "probablyLeft">,
)
```
*/
class MockMatrixClient extends EventEmitter {
private syncState: SyncState;
public constructor(initial: SyncState) {
super();
this.syncState = initial;
}
public setSyncState(state: SyncState): void {
this.syncState = state;
// Matrix's Sync event in createHomeserverConnected$ expects [SyncState]
this.emit(ClientEvent.Sync, [state]);
}
public getSyncState(): SyncState {
return this.syncState;
}
}
/**
* Minimal stub of MatrixRTCSession (membership manager):
```
createHomeserverConnected$(
scope: ObservableScope,
client: NodeStyleEventEmitter & Pick<MatrixClient, "getSyncState">,
matrixRTCSession: NodeStyleEventEmitter &
Pick<MatrixRTCSession, "membershipStatus" | "probablyLeft">,
)
```
*/
class MockMatrixRTCSession extends EventEmitter {
public membershipStatus: Status;
public probablyLeft: boolean;
public constructor(props: {
membershipStatus: Status;
probablyLeft: boolean;
}) {
super();
this.membershipStatus = props.membershipStatus;
this.probablyLeft = props.probablyLeft;
}
public setMembershipStatus(status: Status): void {
this.membershipStatus = status;
this.emit(MembershipManagerEvent.StatusChanged);
}
public setProbablyLeft(flag: boolean): void {
this.probablyLeft = flag;
this.emit(MembershipManagerEvent.ProbablyLeft);
}
}
describe("createHomeserverConnected$", () => {
let scope: ObservableScope;
let client: MockMatrixClient;
let session: MockMatrixRTCSession;
beforeEach(() => {
scope = new ObservableScope();
client = new MockMatrixClient(SyncState.Error); // start disconnected
session = new MockMatrixRTCSession({
membershipStatus: Status.Disconnected,
probablyLeft: false,
});
});
afterEach(() => {
scope.end();
});
// LLM generated test cases. They are a bit overkill but I improved the mocking so it is
// easy enough to read them so I think they can stay.
it("is false when sync state is not Syncing", () => {
const hsConnected$ = createHomeserverConnected$(scope, client, session);
expect(hsConnected$.value).toBe(false);
});
it("remains false while membership status is not Connected even if sync is Syncing", () => {
const hsConnected$ = createHomeserverConnected$(scope, client, session);
client.setSyncState(SyncState.Syncing);
expect(hsConnected$.value).toBe(false); // membership still disconnected
});
it("is false when membership status transitions to Connected but ProbablyLeft is true", () => {
const hsConnected$ = createHomeserverConnected$(scope, client, session);
// Make sync loop OK
client.setSyncState(SyncState.Syncing);
// Indicate probable leave before connection
session.setProbablyLeft(true);
session.setMembershipStatus(Status.Connected);
expect(hsConnected$.value).toBe(false);
});
it("becomes true only when all three conditions are satisfied", () => {
const hsConnected$ = createHomeserverConnected$(scope, client, session);
// 1. Sync loop connected
client.setSyncState(SyncState.Syncing);
expect(hsConnected$.value).toBe(false); // not yet membership connected
// 2. Membership connected
session.setMembershipStatus(Status.Connected);
expect(hsConnected$.value).toBe(true); // probablyLeft is false
});
it("drops back to false when sync loop leaves Syncing", () => {
const hsConnected$ = createHomeserverConnected$(scope, client, session);
// Reach connected state
client.setSyncState(SyncState.Syncing);
session.setMembershipStatus(Status.Connected);
expect(hsConnected$.value).toBe(true);
// Sync loop error => should flip false
client.setSyncState(SyncState.Error);
expect(hsConnected$.value).toBe(false);
});
it("drops back to false when membership status becomes disconnected", () => {
const hsConnected$ = createHomeserverConnected$(scope, client, session);
client.setSyncState(SyncState.Syncing);
session.setMembershipStatus(Status.Connected);
expect(hsConnected$.value).toBe(true);
session.setMembershipStatus(Status.Disconnected);
expect(hsConnected$.value).toBe(false);
});
it("drops to false when ProbablyLeft is emitted after being true", () => {
const hsConnected$ = createHomeserverConnected$(scope, client, session);
client.setSyncState(SyncState.Syncing);
session.setMembershipStatus(Status.Connected);
expect(hsConnected$.value).toBe(true);
session.setProbablyLeft(true);
expect(hsConnected$.value).toBe(false);
});
it("recovers to true if ProbablyLeft becomes false again while other conditions remain true", () => {
const hsConnected$ = createHomeserverConnected$(scope, client, session);
client.setSyncState(SyncState.Syncing);
session.setMembershipStatus(Status.Connected);
expect(hsConnected$.value).toBe(true);
session.setProbablyLeft(true);
expect(hsConnected$.value).toBe(false);
// Simulate clearing the flag (in realistic scenario membership manager would update)
session.setProbablyLeft(false);
expect(hsConnected$.value).toBe(true);
});
it("composite sequence reflects each individual failure reason", () => {
const hsConnected$ = createHomeserverConnected$(scope, client, session);
// Initially false (sync error + disconnected + not probably left)
expect(hsConnected$.value).toBe(false);
// Fix sync only
client.setSyncState(SyncState.Syncing);
expect(hsConnected$.value).toBe(false);
// Fix membership
session.setMembershipStatus(Status.Connected);
expect(hsConnected$.value).toBe(true);
// Introduce probablyLeft -> false
session.setProbablyLeft(true);
expect(hsConnected$.value).toBe(false);
// Restore notProbablyLeft -> true again
session.setProbablyLeft(false);
expect(hsConnected$.value).toBe(true);
// Drop sync -> false
client.setSyncState(SyncState.Error);
expect(hsConnected$.value).toBe(false);
});
});

View File

@@ -1,85 +0,0 @@
/*
Copyright 2025 Element Creations Ltd.
Copyright 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
MembershipManagerEvent,
Status,
type MatrixRTCSession,
} from "matrix-js-sdk/lib/matrixrtc";
import { ClientEvent, type MatrixClient, SyncState } from "matrix-js-sdk";
import { fromEvent, startWith, map, tap, type Observable } from "rxjs";
import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
import { type ObservableScope } from "../../ObservableScope";
import { type Behavior } from "../../Behavior";
import { and$ } from "../../../utils/observable";
import { type NodeStyleEventEmitter } from "../../../utils/test";
/**
* Logger instance (scoped child) for homeserver connection updates.
*/
const logger = rootLogger.getChild("[HomeserverConnected]");
/**
* Behavior representing whether we consider ourselves connected to the Matrix homeserver
* for the purposes of a MatrixRTC session.
*
* Becomes FALSE if ANY sub-condition is fulfilled:
* 1. Sync loop is not in SyncState.Syncing
* 2. membershipStatus !== Status.Connected
* 3. probablyLeft === true
*/
export function createHomeserverConnected$(
scope: ObservableScope,
client: NodeStyleEventEmitter & Pick<MatrixClient, "getSyncState">,
matrixRTCSession: NodeStyleEventEmitter &
Pick<MatrixRTCSession, "membershipStatus" | "probablyLeft">,
): Behavior<boolean> {
const syncing$ = (
fromEvent(client, ClientEvent.Sync) as Observable<[SyncState]>
).pipe(
startWith([client.getSyncState()]),
map(([state]) => state === SyncState.Syncing),
);
const membershipConnected$ = fromEvent(
matrixRTCSession,
MembershipManagerEvent.StatusChanged,
).pipe(
startWith(null),
map(() => matrixRTCSession.membershipStatus === Status.Connected),
);
// This is basically notProbablyLeft$
//
// probablyLeft is computed by a local timer that mimics the server delayed event.
// If we locally predict our server event timed out. We consider ourselves as probablyLeft
// even though we might not yet have received the delayed event leave.
//
// If that is not the case we certainly still have a valid membership on the matrix network
// independet if the sync currently works.
const certainlyConnected$ = fromEvent(
matrixRTCSession,
MembershipManagerEvent.ProbablyLeft,
).pipe(
startWith(null),
map(() => matrixRTCSession.probablyLeft !== true),
);
const connectedCombined$ = and$(
syncing$,
membershipConnected$,
certainlyConnected$,
).pipe(
tap((connected) => {
logger.info(`Homeserver connected update: ${connected}`);
}),
);
return scope.behavior(connectedCombined$);
}

View File

@@ -1,492 +0,0 @@
/*
Copyright 2025 Element Creations Ltd.
Copyright 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
type LivekitTransport,
type MatrixRTCSession,
} from "matrix-js-sdk/lib/matrixrtc";
import { describe, expect, it, vi } from "vitest";
import { AutoDiscovery } from "matrix-js-sdk/lib/autodiscovery";
import { BehaviorSubject, map, of } from "rxjs";
import { logger } from "matrix-js-sdk/lib/logger";
import {
ConnectionState as LivekitConnectionState,
type LocalParticipant,
type LocalTrack,
} from "livekit-client";
import { MatrixRTCMode } from "../../../settings/settings";
import {
flushPromises,
mockConfig,
mockLivekitRoom,
mockMuteStates,
withTestScheduler,
} from "../../../utils/test";
import {
createLocalMembership$,
enterRTCSession,
RTCBackendState,
} from "./LocalMembership";
import { MatrixRTCTransportMissingError } from "../../../utils/errors";
import { Epoch, ObservableScope } from "../../ObservableScope";
import { constant } from "../../Behavior";
import { ConnectionManagerData } from "../remoteMembers/ConnectionManager";
import { type Connection } from "../remoteMembers/Connection";
import { type Publisher } from "./Publisher";
const MATRIX_RTC_MODE = MatrixRTCMode.Legacy;
const getUrlParams = vi.hoisted(() => vi.fn(() => ({})));
vi.mock("../../../UrlParams", () => ({ getUrlParams }));
vi.mock("@livekit/components-core", () => ({
observeParticipantEvents: vi
.fn()
.mockReturnValue(of({ isScreenShareEnabled: false })),
}));
describe("LocalMembership", () => {
describe("enterRTCSession", () => {
it("It joins the correct Session", async () => {
const focusFromOlderMembership = {
type: "livekit",
livekit_service_url: "http://my-oldest-member-service-url.com",
livekit_alias: "my-oldest-member-service-alias",
};
const focusConfigFromWellKnown = {
type: "livekit",
livekit_service_url: "http://my-well-known-service-url.com",
};
const focusConfigFromWellKnown2 = {
type: "livekit",
livekit_service_url: "http://my-well-known-service-url2.com",
};
const clientWellKnown = {
"org.matrix.msc4143.rtc_foci": [
focusConfigFromWellKnown,
focusConfigFromWellKnown2,
],
};
mockConfig({
livekit: { livekit_service_url: "http://my-default-service-url.com" },
});
vi.spyOn(AutoDiscovery, "getRawClientConfig").mockImplementation(
async (domain) => {
if (domain === "example.org") {
return Promise.resolve(clientWellKnown);
}
return Promise.resolve({});
},
);
const mockedSession = vi.mocked({
room: {
roomId: "roomId",
client: {
getDomain: vi.fn().mockReturnValue("example.org"),
getOpenIdToken: vi.fn().mockResolvedValue({
access_token: "ACCCESS_TOKEN",
token_type: "Bearer",
matrix_server_name: "localhost",
expires_in: 10000,
}),
},
},
memberships: [],
getFocusInUse: vi.fn().mockReturnValue(focusFromOlderMembership),
getOldestMembership: vi.fn().mockReturnValue({
getPreferredFoci: vi.fn().mockReturnValue([focusFromOlderMembership]),
}),
joinRoomSession: vi.fn(),
}) as unknown as MatrixRTCSession;
await enterRTCSession(
mockedSession,
{
livekit_alias: "roomId",
livekit_service_url: "http://my-well-known-service-url.com",
type: "livekit",
},
{
encryptMedia: true,
matrixRTCMode: MATRIX_RTC_MODE,
},
);
expect(mockedSession.joinRoomSession).toHaveBeenLastCalledWith(
[
{
livekit_alias: "roomId",
livekit_service_url: "http://my-well-known-service-url.com",
type: "livekit",
},
],
undefined,
expect.objectContaining({
manageMediaKeys: true,
useLegacyMemberEvents: false,
}),
);
});
it("It should not fail with configuration error if homeserver config has livekit url but not fallback", async () => {
mockConfig({});
vi.spyOn(AutoDiscovery, "getRawClientConfig").mockResolvedValue({
"org.matrix.msc4143.rtc_foci": [
{
type: "livekit",
livekit_service_url: "http://my-well-known-service-url.com",
},
],
});
const mockedSession = vi.mocked({
room: {
roomId: "roomId",
client: {
getDomain: vi.fn().mockReturnValue("example.org"),
getOpenIdToken: vi.fn().mockResolvedValue({
access_token: "ACCCESS_TOKEN",
token_type: "Bearer",
matrix_server_name: "localhost",
expires_in: 10000,
}),
},
},
memberships: [],
getFocusInUse: vi.fn(),
joinRoomSession: vi.fn(),
}) as unknown as MatrixRTCSession;
await enterRTCSession(
mockedSession,
{
livekit_alias: "roomId",
livekit_service_url: "http://my-well-known-service-url.com",
type: "livekit",
},
{
encryptMedia: true,
matrixRTCMode: MATRIX_RTC_MODE,
},
);
});
});
const defaultCreateLocalMemberValues = {
options: constant({
encryptMedia: false,
matrixRTCMode: MatrixRTCMode.Matrix_2_0,
}),
matrixRTCSession: {
updateCallIntent: () => {},
leaveRoomSession: () => {},
} as unknown as MatrixRTCSession,
muteStates: mockMuteStates(),
isHomeserverConnected: constant(true),
trackProcessorState$: constant({
supported: false,
processor: undefined,
}),
logger: logger,
createPublisherFactory: vi.fn(),
joinMatrixRTC: async (): Promise<void> => {},
homeserverConnected$: constant(true),
};
it("throws error on missing RTC config error", () => {
withTestScheduler(({ scope, hot, expectObservable }) => {
const goodTransport = {
livekit_service_url: "other",
} as LivekitTransport;
const localTransport$ = scope.behavior<LivekitTransport>(
hot("1ms #", {}, new MatrixRTCTransportMissingError("domain.com")),
goodTransport,
);
const mockConnectionManager = {
transports$: scope.behavior(
localTransport$.pipe(map((t) => new Epoch([t]))),
),
connectionManagerData$: constant(
new Epoch(new ConnectionManagerData()),
),
};
const localMembership = createLocalMembership$({
scope,
...defaultCreateLocalMemberValues,
connectionManager: mockConnectionManager,
localTransport$,
});
expectObservable(localMembership.connectionState.livekit$).toBe("ne", {
n: { state: RTCBackendState.WaitingForConnection },
e: {
state: RTCBackendState.Error,
error: expect.toSatisfy(
(e) => e instanceof MatrixRTCTransportMissingError,
),
},
});
});
});
const aTransport = {
livekit_service_url: "a",
} as LivekitTransport;
const bTransport = {
livekit_service_url: "b",
} as LivekitTransport;
const connectionManagerData = new ConnectionManagerData();
connectionManagerData.add(
{
livekitRoom: mockLivekitRoom({
localParticipant: {
isScreenShareEnabled: false,
trackPublications: [],
} as unknown as LocalParticipant,
}),
state$: constant({
state: "ConnectedToLkRoom",
livekitConnectionState$: constant(LivekitConnectionState.Connected),
}),
transport: aTransport,
} as unknown as Connection,
[],
);
connectionManagerData.add(
{
state$: constant({
state: "ConnectedToLkRoom",
}),
transport: bTransport,
} as unknown as Connection,
[],
);
it("recreates publisher if new connection is used and ENDS always unpublish and end tracks", async () => {
const scope = new ObservableScope();
const localTransport$ = new BehaviorSubject(aTransport);
const publishers: Publisher[] = [];
defaultCreateLocalMemberValues.createPublisherFactory.mockImplementation(
() => {
const p = {
stopPublishing: vi.fn(),
stopTracks: vi.fn(),
publishing$: constant(false),
};
publishers.push(p as unknown as Publisher);
return p;
},
);
const publisherFactory =
defaultCreateLocalMemberValues.createPublisherFactory as ReturnType<
typeof vi.fn
>;
createLocalMembership$({
scope,
...defaultCreateLocalMemberValues,
connectionManager: {
connectionManagerData$: constant(new Epoch(connectionManagerData)),
},
localTransport$,
});
await flushPromises();
localTransport$.next(bTransport);
await flushPromises();
expect(publisherFactory).toHaveBeenCalledTimes(2);
expect(publishers.length).toBe(2);
// stop the first Publisher and let the second one life.
expect(publishers[0].stopTracks).toHaveBeenCalled();
expect(publishers[1].stopTracks).not.toHaveBeenCalled();
expect(publishers[0].stopPublishing).toHaveBeenCalled();
expect(publishers[1].stopPublishing).not.toHaveBeenCalled();
expect(publisherFactory.mock.calls[0][0].transport).toBe(aTransport);
expect(publisherFactory.mock.calls[1][0].transport).toBe(bTransport);
scope.end();
await flushPromises();
// stop all tracks after ending scopes
expect(publishers[1].stopPublishing).toHaveBeenCalled();
expect(publishers[1].stopTracks).toHaveBeenCalled();
defaultCreateLocalMemberValues.createPublisherFactory.mockReset();
});
it("only start tracks if requested", async () => {
const scope = new ObservableScope();
const localTransport$ = new BehaviorSubject(aTransport);
const publishers: Publisher[] = [];
const tracks$ = new BehaviorSubject<LocalTrack[]>([]);
const publishing$ = new BehaviorSubject<boolean>(false);
defaultCreateLocalMemberValues.createPublisherFactory.mockImplementation(
() => {
const p = {
stopPublishing: vi.fn(),
stopTracks: vi.fn(),
createAndSetupTracks: vi.fn().mockImplementation(async () => {
tracks$.next([{}, {}] as LocalTrack[]);
return Promise.resolve();
}),
tracks$,
publishing$,
};
publishers.push(p as unknown as Publisher);
return p;
},
);
const publisherFactory =
defaultCreateLocalMemberValues.createPublisherFactory as ReturnType<
typeof vi.fn
>;
const localMembership = createLocalMembership$({
scope,
...defaultCreateLocalMemberValues,
connectionManager: {
connectionManagerData$: constant(new Epoch(connectionManagerData)),
},
localTransport$,
});
await flushPromises();
expect(publisherFactory).toHaveBeenCalledOnce();
expect(localMembership.tracks$.value.length).toBe(0);
localMembership.startTracks();
await flushPromises();
expect(localMembership.tracks$.value.length).toBe(2);
scope.end();
await flushPromises();
// stop all tracks after ending scopes
expect(publishers[0].stopPublishing).toHaveBeenCalled();
expect(publishers[0].stopTracks).toHaveBeenCalled();
publisherFactory.mockClear();
});
// TODO add an integration test combining publisher and localMembership
//
it("tracks livekit state correctly", async () => {
const scope = new ObservableScope();
const localTransport$ = new BehaviorSubject<null | LivekitTransport>(null);
const connectionManagerData$ = new BehaviorSubject<
Epoch<ConnectionManagerData>
>(new Epoch(new ConnectionManagerData()));
const publishers: Publisher[] = [];
const tracks$ = new BehaviorSubject<LocalTrack[]>([]);
const publishing$ = new BehaviorSubject<boolean>(false);
const createTrackResolver = Promise.withResolvers<void>();
const publishResolver = Promise.withResolvers<void>();
defaultCreateLocalMemberValues.createPublisherFactory.mockImplementation(
() => {
const p = {
stopPublishing: vi.fn(),
stopTracks: vi.fn().mockImplementation(() => {
logger.info("stopTracks");
tracks$.next([]);
}),
createAndSetupTracks: vi.fn().mockImplementation(async () => {
await createTrackResolver.promise;
tracks$.next([{}, {}] as LocalTrack[]);
}),
startPublishing: vi.fn().mockImplementation(async () => {
await publishResolver.promise;
publishing$.next(true);
}),
tracks$,
publishing$,
};
publishers.push(p as unknown as Publisher);
return p;
},
);
const publisherFactory =
defaultCreateLocalMemberValues.createPublisherFactory as ReturnType<
typeof vi.fn
>;
const localMembership = createLocalMembership$({
scope,
...defaultCreateLocalMemberValues,
connectionManager: {
connectionManagerData$,
},
localTransport$,
});
await flushPromises();
expect(localMembership.connectionState.livekit$.value).toStrictEqual({
state: RTCBackendState.WaitingForTransport,
});
localTransport$.next(aTransport);
await flushPromises();
expect(localMembership.connectionState.livekit$.value).toStrictEqual({
state: RTCBackendState.WaitingForConnection,
});
connectionManagerData$.next(new Epoch(connectionManagerData));
await flushPromises();
expect(localMembership.connectionState.livekit$.value).toStrictEqual({
state: RTCBackendState.Initialized,
});
expect(publisherFactory).toHaveBeenCalledOnce();
expect(localMembership.tracks$.value.length).toBe(0);
// -------
localMembership.startTracks();
// -------
await flushPromises();
expect(localMembership.connectionState.livekit$.value).toStrictEqual({
state: RTCBackendState.CreatingTracks,
});
createTrackResolver.resolve();
await flushPromises();
expect(localMembership.connectionState.livekit$.value).toStrictEqual({
state: RTCBackendState.ReadyToPublish,
});
// -------
localMembership.requestConnect();
// -------
expect(localMembership.connectionState.livekit$.value).toStrictEqual({
state: RTCBackendState.WaitingToPublish,
});
publishResolver.resolve();
await flushPromises();
expect(localMembership.connectionState.livekit$.value).toStrictEqual({
state: RTCBackendState.Connected,
});
expect(publishers[0].stopPublishing).not.toHaveBeenCalled();
expect(localMembership.connectionState.livekit$.isStopped).toBe(false);
scope.end();
await flushPromises();
// stays in connected state because it is stopped before the update to tracks update the state.
expect(localMembership.connectionState.livekit$.value).toStrictEqual({
state: RTCBackendState.Connected,
});
// stop all tracks after ending scopes
expect(publishers[0].stopPublishing).toHaveBeenCalled();
expect(publishers[0].stopTracks).toHaveBeenCalled();
});
// TODO add tests for matrix local matrix participation.
});

View File

@@ -1,675 +0,0 @@
/*
Copyright 2025 Element Creations Ltd.
SPDX-License-IdFentifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
type LocalTrack,
type Participant,
ParticipantEvent,
type LocalParticipant,
type ScreenShareCaptureOptions,
ConnectionState,
} from "livekit-client";
import { observeParticipantEvents } from "@livekit/components-core";
import {
type LivekitTransport,
type MatrixRTCSession,
} from "matrix-js-sdk/lib/matrixrtc";
import {
BehaviorSubject,
catchError,
combineLatest,
distinctUntilChanged,
from,
map,
type Observable,
of,
scan,
startWith,
switchMap,
tap,
} from "rxjs";
import { type Logger } from "matrix-js-sdk/lib/logger";
import { deepCompare } from "matrix-js-sdk/lib/utils";
import { constant, type Behavior } from "../../Behavior";
import { type IConnectionManager } from "../remoteMembers/ConnectionManager";
import { ObservableScope } from "../../ObservableScope";
import { type Publisher } from "./Publisher";
import { type MuteStates } from "../../MuteStates";
import { and$ } from "../../../utils/observable";
import {
ElementCallError,
MembershipManagerError,
UnknownCallError,
} from "../../../utils/errors";
import { ElementWidgetActions, widget } from "../../../widget";
import { getUrlParams } from "../../../UrlParams.ts";
import { PosthogAnalytics } from "../../../analytics/PosthogAnalytics.ts";
import { MatrixRTCMode } from "../../../settings/settings.ts";
import { Config } from "../../../config/Config.ts";
import { type Connection } from "../remoteMembers/Connection.ts";
export enum RTCBackendState {
Error = "error",
/** Not even a transport is available to the LocalMembership */
WaitingForTransport = "waiting_for_transport",
/** A connection appeared so we can initialise the publisher */
WaitingForConnection = "waiting_for_connection",
/** Connection and transport arrived, publisher Initialized */
Initialized = "Initialized",
CreatingTracks = "creating_tracks",
ReadyToPublish = "ready_to_publish",
WaitingToPublish = "waiting_to_publish",
Connected = "connected",
Disconnected = "disconnected",
Disconnecting = "disconnecting",
}
type LocalMemberRtcBackendState =
| { state: RTCBackendState.Error; error: ElementCallError }
| { state: RTCBackendState.WaitingForTransport }
| { state: RTCBackendState.WaitingForConnection }
| { state: RTCBackendState.Initialized }
| { state: RTCBackendState.CreatingTracks }
| { state: RTCBackendState.ReadyToPublish }
| { state: RTCBackendState.WaitingToPublish }
| { state: RTCBackendState.Connected }
| { state: RTCBackendState.Disconnected }
| { state: RTCBackendState.Disconnecting };
export enum MatrixState {
WaitingForTransport = "waiting_for_transport",
Ready = "ready",
Connecting = "connecting",
Connected = "connected",
Disconnected = "disconnected",
Error = "Error",
}
type LocalMemberMatrixState =
| { state: MatrixState.Connected }
| { state: MatrixState.WaitingForTransport }
| { state: MatrixState.Ready }
| { state: MatrixState.Connecting }
| { state: MatrixState.Disconnected }
| { state: MatrixState.Error; error: Error };
export interface LocalMemberConnectionState {
livekit$: Behavior<LocalMemberRtcBackendState>;
matrix$: Behavior<LocalMemberMatrixState>;
}
/*
* - get well known
* - get oldest membership
* - get transport to use
* - get openId + jwt token
* - wait for createTrack() call
* - create tracks
* - wait for join() call
* - Publisher.publishTracks()
* - send join state/sticky event
*/
interface Props {
// TODO add a comment into some code style readme or file header callviewmodel
// that the inputs for those createSomething$() functions should NOT contain any js-sdk objectes
scope: ObservableScope;
muteStates: MuteStates;
connectionManager: IConnectionManager;
createPublisherFactory: (connection: Connection) => Publisher;
joinMatrixRTC: (transport: LivekitTransport) => Promise<void>;
homeserverConnected$: Behavior<boolean>;
localTransport$: Behavior<LivekitTransport | null>;
matrixRTCSession: Pick<
MatrixRTCSession,
"updateCallIntent" | "leaveRoomSession"
>;
logger: Logger;
}
/**
* This class is responsible for managing the own membership in a room.
* We want
* - a publisher
* -
* @param param0
* @returns
* - publisher: The handle to create tracks and publish them to the room.
* - connected$: the current connection state. Including matrix server and livekit server connection. (only considering the livekit server we are using for our own media publication)
* - transport$: the transport object the ownMembership$ ended up using.
* - connectionState: the current connection state. Including matrix server and livekit server connection.
* - sharingScreen$: Whether we are sharing our screen. `undefined` if we cannot share the screen.
*/
export const createLocalMembership$ = ({
scope,
connectionManager,
localTransport$: localTransportCanThrow$,
homeserverConnected$,
createPublisherFactory,
joinMatrixRTC,
logger: parentLogger,
muteStates,
matrixRTCSession,
}: Props): {
/**
* This starts audio and video tracks. They will be reused when calling `requestConnect`.
*/
startTracks: () => Behavior<LocalTrack[]>;
/**
* This sets a inner state (shouldConnect) to true and instructs the js-sdk and livekit to keep the user
* connected to matrix and livekit.
*/
requestConnect: () => void;
requestDisconnect: () => void;
connectionState: LocalMemberConnectionState;
sharingScreen$: Behavior<boolean>;
/**
* Callback to toggle screen sharing. If null, screen sharing is not possible.
*/
toggleScreenSharing: (() => void) | null;
tracks$: Behavior<LocalTrack[]>;
participant$: Behavior<LocalParticipant | null>;
connection$: Behavior<Connection | null>;
homeserverConnected$: Behavior<boolean>;
// this needs to be discussed
/** @deprecated use state instead*/
reconnecting$: Behavior<boolean>;
} => {
const logger = parentLogger.getChild("[LocalMembership]");
logger.debug(`Creating local membership..`);
// Unwrap the local transport and set the state of the LocalMembership to error in case the transport is an error.
const localTransport$ = scope.behavior(
localTransportCanThrow$.pipe(
catchError((e: unknown) => {
let error: ElementCallError;
if (e instanceof ElementCallError) {
error = e;
} else {
error = new UnknownCallError(
e instanceof Error
? e
: new Error("Unknown error from localTransport"),
);
}
setLivekitError(error);
return of(null);
}),
),
);
// Drop Epoch data here since we will not combine this anymore
const localConnection$ = scope.behavior(
combineLatest([
connectionManager.connectionManagerData$,
localTransport$,
]).pipe(
map(([{ value: connectionData }, localTransport]) => {
if (localTransport === null) {
return null;
}
return connectionData.getConnectionForTransport(localTransport);
}),
tap((connection) => {
logger.info(
`Local connection updated: ${connection?.transport?.livekit_service_url}`,
);
}),
),
);
const localConnectionState$ = localConnection$.pipe(
switchMap((connection) => (connection ? connection.state$ : of(null))),
);
// /**
// * Whether we are "fully" connected to the call. Accounts for both the
// * connection to the MatrixRTC session and the LiveKit publish connection.
// */
const connected$ = scope.behavior(
and$(
homeserverConnected$.pipe(
tap((v) => logger.debug("matrix: Connected state changed", v)),
),
localConnectionState$.pipe(
switchMap((state) => {
logger.debug("livekit: Connected state changed", state);
if (!state) return of(false);
if (state.state === "ConnectedToLkRoom") {
logger.debug(
"livekit: Connected state changed (inner livekitConnectionState$)",
state.livekitConnectionState$.value,
);
return state.livekitConnectionState$.pipe(
map((lkState) => lkState === ConnectionState.Connected),
);
}
return of(false);
}),
),
).pipe(tap((v) => logger.debug("combined: Connected state changed", v))),
);
// MATRIX RELATED
// /**
// * Whether we should tell the user that we're reconnecting to the call.
// */
// DISCUSSION is there a better way to do this?
// sth that is more deriectly implied from the membership manager of the js sdk. (fromEvent(matrixRTCSession, Reconnecting)) ??? or similar
const reconnecting$ = scope.behavior(
connected$.pipe(
// We are reconnecting if we previously had some successful initial
// connection but are now disconnected
scan(
({ connectedPreviously }, connectedNow) => ({
connectedPreviously: connectedPreviously || connectedNow,
reconnecting: connectedPreviously && !connectedNow,
}),
{ connectedPreviously: false, reconnecting: false },
),
map(({ reconnecting }) => reconnecting),
),
);
// This should be used in a combineLatest with publisher$ to connect.
// to make it possible to call startTracks before the preferredTransport$ has resolved.
const trackStartRequested = Promise.withResolvers<void>();
// This should be used in a combineLatest with publisher$ to connect.
// to make it possible to call startTracks before the preferredTransport$ has resolved.
const connectRequested$ = new BehaviorSubject(false);
/**
* The publisher is stored in here an abstracts creating and publishing tracks.
*/
const publisher$ = new BehaviorSubject<Publisher | null>(null);
/**
* Extract the tracks from the published. Also reacts to changing publishers.
*/
const tracks$ = scope.behavior(
publisher$.pipe(switchMap((p) => (p?.tracks$ ? p.tracks$ : constant([])))),
);
const publishing$ = scope.behavior(
publisher$.pipe(switchMap((p) => p?.publishing$ ?? constant(false))),
);
const startTracks = (): Behavior<LocalTrack[]> => {
trackStartRequested.resolve();
return tracks$;
};
const requestConnect = (): void => {
trackStartRequested.resolve();
connectRequested$.next(true);
};
const requestDisconnect = (): void => {
connectRequested$.next(false);
};
// Take care of the publisher$
// create a new one as soon as a local Connection is available
//
// Recreate a new one once the local connection changes
// - stop publishing
// - destruct all current streams
// - overwrite current publisher
scope.reconcile(localConnection$, async (connection) => {
if (connection !== null) {
publisher$.next(createPublisherFactory(connection));
}
return Promise.resolve(async (): Promise<void> => {
await publisher$?.value?.stopPublishing();
publisher$?.value?.stopTracks();
});
});
// Use reconcile here to not run concurrent createAndSetupTracks calls
// `tracks$` will update once they are ready.
scope.reconcile(
scope.behavior(
combineLatest([publisher$, tracks$, from(trackStartRequested.promise)]),
null,
),
async (valueIfReady) => {
if (!valueIfReady) return;
const [publisher, tracks] = valueIfReady;
if (publisher && tracks.length === 0) {
await publisher.createAndSetupTracks().catch((e) => logger.error(e));
}
},
);
// Based on `connectRequested$` we start publishing tracks. (once they are there!)
scope.reconcile(
scope.behavior(combineLatest([publisher$, tracks$, connectRequested$])),
async ([publisher, tracks, shouldConnect]) => {
if (shouldConnect === publisher?.publishing$.value) return;
if (tracks.length !== 0 && shouldConnect) {
try {
await publisher?.startPublishing();
} catch (error) {
setLivekitError(error as ElementCallError);
}
} else if (tracks.length !== 0 && !shouldConnect) {
try {
await publisher?.stopPublishing();
} catch (error) {
setLivekitError(new UnknownCallError(error as Error));
}
}
},
);
const fatalLivekitError$ = new BehaviorSubject<ElementCallError | null>(null);
const setLivekitError = (e: ElementCallError): void => {
if (fatalLivekitError$.value !== null)
logger.error("Multiple Livkit Errors:", e);
else fatalLivekitError$.next(e);
};
const livekitState$: Behavior<LocalMemberRtcBackendState> = scope.behavior(
combineLatest([
publisher$,
localTransport$,
tracks$.pipe(
tap((t) => {
logger.info("tracks$: ", t);
}),
),
publishing$,
connectRequested$,
from(trackStartRequested.promise).pipe(
map(() => true),
startWith(false),
),
fatalLivekitError$,
]).pipe(
map(
([
publisher,
localTransport,
tracks,
publishing,
shouldConnect,
shouldStartTracks,
error,
]) => {
// read this:
// if(!<A>) return {state: ...}
// if(!<B>) return {state: <MyState>}
//
// as:
// We do have <A> but not yet <B> so we are in <MyState>
if (error !== null) return { state: RTCBackendState.Error, error };
const hasTracks = tracks.length > 0;
if (!localTransport)
return { state: RTCBackendState.WaitingForTransport };
if (!publisher)
return { state: RTCBackendState.WaitingForConnection };
if (!shouldStartTracks) return { state: RTCBackendState.Initialized };
if (!hasTracks) return { state: RTCBackendState.CreatingTracks };
if (!shouldConnect) return { state: RTCBackendState.ReadyToPublish };
if (!publishing) return { state: RTCBackendState.WaitingToPublish };
return { state: RTCBackendState.Connected };
},
),
distinctUntilChanged(deepCompare),
),
);
const fatalMatrixError$ = new BehaviorSubject<ElementCallError | null>(null);
const setMatrixError = (e: ElementCallError): void => {
if (fatalMatrixError$.value !== null)
logger.error("Multiple Matrix Errors:", e);
else fatalMatrixError$.next(e);
};
const matrixState$: Behavior<LocalMemberMatrixState> = scope.behavior(
combineLatest([
localTransport$,
connectRequested$,
homeserverConnected$,
]).pipe(
map(([localTransport, connectRequested, homeserverConnected]) => {
if (!localTransport) return { state: MatrixState.WaitingForTransport };
if (!connectRequested) return { state: MatrixState.Ready };
if (!homeserverConnected) return { state: MatrixState.Connecting };
return { state: MatrixState.Connected };
}),
),
);
// Keep matrix rtc session in sync with localTransport$, connectRequested$ and muteStates.video.enabled$
scope.reconcile(
scope.behavior(combineLatest([localTransport$, connectRequested$])),
async ([transport, shouldConnect]) => {
if (!shouldConnect) return;
if (!transport) return;
try {
await joinMatrixRTC(transport);
} catch (error) {
logger.error("Error entering RTC session", error);
if (error instanceof Error)
setMatrixError(new MembershipManagerError(error));
}
// Update our member event when our mute state changes.
const callIntentScope = new ObservableScope();
// because this uses its own scope, we can start another reconciliation for the duration of one connection.
callIntentScope.reconcile(
muteStates.video.enabled$,
async (videoEnabled) =>
matrixRTCSession.updateCallIntent(videoEnabled ? "video" : "audio"),
);
return async (): Promise<void> => {
callIntentScope.end();
try {
// Update matrixRTCSession to allow udpating the transport without leaving the session!
await matrixRTCSession.leaveRoomSession();
} catch (e) {
logger.error("Error leaving RTC session", e);
}
try {
await widget?.api.transport.send(ElementWidgetActions.HangupCall, {});
} catch (e) {
logger.error("Failed to send hangup action", e);
}
};
},
);
const participant$ = scope.behavior(
localConnection$.pipe(map((c) => c?.livekitRoom?.localParticipant ?? null)),
);
// Pause upstream of all local media tracks when we're disconnected from
// MatrixRTC, because it can be an unpleasant surprise for the app to say
// 'reconnecting' and yet still be transmitting your media to others.
// We use matrixConnected$ rather than reconnecting$ because we want to
// pause tracks during the initial joining sequence too until we're sure
// that our own media is displayed on screen.
// TODO refactor this based no livekitState$
combineLatest([participant$, homeserverConnected$])
.pipe(scope.bind())
.subscribe(([participant, connected]) => {
if (!participant) return;
const publications = participant.trackPublications.values();
if (connected) {
for (const p of publications) {
if (p.track?.isUpstreamPaused === true) {
const kind = p.track.kind;
logger.info(
`Resuming ${kind} track (MatrixRTC connection present)`,
);
p.track
.resumeUpstream()
.catch((e) =>
logger.error(
`Failed to resume ${kind} track after MatrixRTC reconnection`,
e,
),
);
}
}
} else {
for (const p of publications) {
if (p.track?.isUpstreamPaused === false) {
const kind = p.track.kind;
logger.info(
`Pausing ${kind} track (uncertain MatrixRTC connection)`,
);
p.track
.pauseUpstream()
.catch((e) =>
logger.error(
`Failed to pause ${kind} track after entering uncertain MatrixRTC connection`,
e,
),
);
}
}
}
});
/**
* Whether the user is currently sharing their screen.
*/
const sharingScreen$ = scope.behavior(
participant$.pipe(
switchMap((p) => (p !== null ? observeSharingScreen$(p) : of(false))),
),
);
let toggleScreenSharing: (() => void) | null = null;
if (
"getDisplayMedia" in (navigator.mediaDevices ?? {}) &&
!getUrlParams().hideScreensharing
) {
toggleScreenSharing = (): void => {
const screenshareSettings: ScreenShareCaptureOptions = {
audio: true,
selfBrowserSurface: "include",
surfaceSwitching: "include",
systemAudio: "include",
};
const targetScreenshareState = !sharingScreen$.value;
logger.info(
`toggleScreenSharing called. Switching ${
targetScreenshareState ? "On" : "Off"
}`,
);
// If a connection is ready, toggle screen sharing.
// We deliberately do nothing in the case of a null connection because
// it looks nice for the call control buttons to all become available
// at once upon joining the call, rather than introducing a disabled
// state. The user can just click again.
// We also allow screen sharing to be toggled even if the connection
// is still initializing or publishing tracks, because there's no
// technical reason to disallow this. LiveKit will publish if it can.
participant$.value
?.setScreenShareEnabled(targetScreenshareState, screenshareSettings)
.catch(logger.error);
};
}
return {
startTracks,
requestConnect,
requestDisconnect,
connectionState: {
livekit$: livekitState$,
matrix$: matrixState$,
},
tracks$,
participant$,
homeserverConnected$,
reconnecting$,
sharingScreen$,
toggleScreenSharing,
connection$: localConnection$,
};
};
export function observeSharingScreen$(p: Participant): Observable<boolean> {
return observeParticipantEvents(
p,
ParticipantEvent.TrackPublished,
ParticipantEvent.TrackUnpublished,
ParticipantEvent.LocalTrackPublished,
ParticipantEvent.LocalTrackUnpublished,
).pipe(map((p) => p.isScreenShareEnabled));
}
interface EnterRTCSessionOptions {
encryptMedia: boolean;
matrixRTCMode: MatrixRTCMode;
}
/**
* Does the necessary steps to enter the RTC session on the matrix side:
* - Preparing the membership info (FOCUS to use, options)
* - Sends the matrix event to join the call, and starts the membership manager:
* - Delay events management
* - Handles retries (fails only after several attempts)
*
* @param rtcSession
* @param transport
* @param options
* @throws If the widget could not send ElementWidgetActions.JoinCall action.
*/
// Exported for unit testing
export async function enterRTCSession(
rtcSession: MatrixRTCSession,
transport: LivekitTransport,
{ encryptMedia, matrixRTCMode }: EnterRTCSessionOptions,
): Promise<void> {
PosthogAnalytics.instance.eventCallEnded.cacheStartCall(new Date());
PosthogAnalytics.instance.eventCallStarted.track(rtcSession.room.roomId);
// This must be called before we start trying to join the call, as we need to
// have started tracking by the time calls start getting created.
// groupCallOTelMembership?.onJoinCall();
const { features, matrix_rtc_session: matrixRtcSessionConfig } = Config.get();
const useDeviceSessionMemberEvents =
features?.feature_use_device_session_member_events;
const { sendNotificationType: notificationType, callIntent } = getUrlParams();
const multiSFU = matrixRTCMode !== MatrixRTCMode.Legacy;
// Multi-sfu does not need a preferred foci list. just the focus that is actually used.
// TODO where/how do we track errors originating from the ongoing rtcSession?
rtcSession.joinRoomSession(
multiSFU ? [] : [transport],
multiSFU ? transport : undefined,
{
notificationType,
callIntent,
manageMediaKeys: encryptMedia,
...(useDeviceSessionMemberEvents !== undefined && {
useLegacyMemberEvents: !useDeviceSessionMemberEvents,
}),
delayedLeaveEventRestartMs:
matrixRtcSessionConfig?.delayed_leave_event_restart_ms,
delayedLeaveEventDelayMs:
matrixRtcSessionConfig?.delayed_leave_event_delay_ms,
delayedLeaveEventRestartLocalTimeoutMs:
matrixRtcSessionConfig?.delayed_leave_event_restart_local_timeout_ms,
networkErrorRetryMs: matrixRtcSessionConfig?.network_error_retry_ms,
makeKeyDelay: matrixRtcSessionConfig?.wait_for_key_rotation_ms,
membershipEventExpiryMs:
matrixRtcSessionConfig?.membership_event_expiry_ms,
useExperimentalToDeviceTransport: true,
unstableSendStickyEvents: matrixRTCMode === MatrixRTCMode.Matrix_2_0,
},
);
if (widget) {
await widget.api.transport.send(ElementWidgetActions.JoinCall, {});
}
}

View File

@@ -1,156 +0,0 @@
/*
Copyright 2025 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc";
import { BehaviorSubject } from "rxjs";
import { mockConfig, flushPromises } from "../../../utils/test";
import { createLocalTransport$ } from "./LocalTransport";
import { constant } from "../../Behavior";
import { Epoch, ObservableScope } from "../../ObservableScope";
import {
MatrixRTCTransportMissingError,
FailToGetOpenIdToken,
} from "../../../utils/errors";
import * as openIDSFU from "../../../livekit/openIDSFU";
describe("LocalTransport", () => {
let scope: ObservableScope;
beforeEach(() => (scope = new ObservableScope()));
afterEach(() => scope.end());
it("throws if config is missing", async () => {
const localTransport$ = createLocalTransport$({
scope,
roomId: "!room:example.org",
useOldestMember$: constant(false),
memberships$: constant(new Epoch<CallMembership[]>([])),
client: {
getDomain: () => "",
// These won't be called in this error path but satisfy the type
getOpenIdToken: vi.fn(),
getDeviceId: vi.fn(),
},
});
await flushPromises();
expect(() => localTransport$.value).toThrow(
new MatrixRTCTransportMissingError(""),
);
});
it("throws FailToGetOpenIdToken when OpenID fetch fails", async () => {
// Provide a valid config so makeTransportInternal resolves a transport
const scope = new ObservableScope();
mockConfig({
livekit: { livekit_service_url: "https://lk.example.org" },
});
const resolver = Promise.withResolvers<void>();
vi.spyOn(openIDSFU, "getSFUConfigWithOpenID").mockImplementation(
async () => {
await resolver.promise;
throw new FailToGetOpenIdToken(new Error("no openid"));
},
);
const observations: unknown[] = [];
const errors: Error[] = [];
const localTransport$ = createLocalTransport$({
scope,
roomId: "!room:example.org",
useOldestMember$: constant(false),
memberships$: constant(new Epoch<CallMembership[]>([])),
client: {
// Use empty domain to skip .well-known and use config directly
getDomain: () => "",
getOpenIdToken: vi.fn(),
getDeviceId: vi.fn(),
},
});
localTransport$.subscribe(
(o) => observations.push(o),
(e) => errors.push(e),
);
resolver.resolve();
await flushPromises();
const expectedError = new FailToGetOpenIdToken(new Error("no openid"));
expect(observations).toStrictEqual([null]);
expect(errors).toStrictEqual([expectedError]);
expect(() => localTransport$.value).toThrow(expectedError);
});
it("emits preferred transport after OpenID resolves", async () => {
// Use config so transport discovery succeeds, but delay OpenID JWT fetch
mockConfig({
livekit: { livekit_service_url: "https://lk.example.org" },
});
const openIdResolver = Promise.withResolvers<openIDSFU.SFUConfig>();
vi.spyOn(openIDSFU, "getSFUConfigWithOpenID").mockReturnValue(
openIdResolver.promise,
);
const localTransport$ = createLocalTransport$({
scope,
roomId: "!room:example.org",
useOldestMember$: constant(false),
memberships$: constant(new Epoch<CallMembership[]>([])),
client: {
getDomain: () => "",
getOpenIdToken: vi.fn(),
getDeviceId: vi.fn(),
},
});
openIdResolver.resolve?.({ url: "https://lk.example.org", jwt: "jwt" });
expect(localTransport$.value).toBe(null);
await flushPromises();
// final
expect(localTransport$.value).toStrictEqual({
livekit_alias: "!room:example.org",
livekit_service_url: "https://lk.example.org",
type: "livekit",
});
});
it("updates local transport when oldest member changes", async () => {
// Use config so transport discovery succeeds, but delay OpenID JWT fetch
mockConfig({
livekit: { livekit_service_url: "https://lk.example.org" },
});
const memberships$ = new BehaviorSubject(new Epoch([]));
const openIdResolver = Promise.withResolvers<openIDSFU.SFUConfig>();
vi.spyOn(openIDSFU, "getSFUConfigWithOpenID").mockReturnValue(
openIdResolver.promise,
);
const localTransport$ = createLocalTransport$({
scope,
roomId: "!room:example.org",
useOldestMember$: constant(true),
memberships$,
client: {
getDomain: () => "",
getOpenIdToken: vi.fn(),
getDeviceId: vi.fn(),
},
});
openIdResolver.resolve?.({ url: "https://lk.example.org", jwt: "jwt" });
expect(localTransport$.value).toBe(null);
await flushPromises();
// final
expect(localTransport$.value).toStrictEqual({
livekit_alias: "!room:example.org",
livekit_service_url: "https://lk.example.org",
type: "livekit",
});
});
});

View File

@@ -1,188 +0,0 @@
/*
Copyright 2025 Element Creations Ltd.
SPDX-License-IdFentifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
type CallMembership,
isLivekitTransport,
type LivekitTransportConfig,
type LivekitTransport,
isLivekitTransportConfig,
} from "matrix-js-sdk/lib/matrixrtc";
import { type MatrixClient } from "matrix-js-sdk";
import {
combineLatest,
distinctUntilChanged,
first,
from,
map,
switchMap,
} from "rxjs";
import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
import { AutoDiscovery } from "matrix-js-sdk/lib/autodiscovery";
import { type Behavior } from "../../Behavior.ts";
import { type Epoch, type ObservableScope } from "../../ObservableScope.ts";
import { Config } from "../../../config/Config.ts";
import { MatrixRTCTransportMissingError } from "../../../utils/errors.ts";
import {
getSFUConfigWithOpenID,
type OpenIDClientParts,
} from "../../../livekit/openIDSFU.ts";
import { areLivekitTransportsEqual } from "../remoteMembers/MatrixLivekitMembers.ts";
import { customLivekitUrl } from "../../../settings/settings.ts";
const logger = rootLogger.getChild("[LocalTransport]");
/*
* It figures out “which LiveKit focus URL/alias the local user should use,”
* optionally aligning with the oldest member, and ensures the SFU path is primed
* before advertising that choice.
*/
interface Props {
scope: ObservableScope;
memberships$: Behavior<Epoch<CallMembership[]>>;
client: Pick<MatrixClient, "getDomain"> & OpenIDClientParts;
roomId: string;
useOldestMember$: Behavior<boolean>;
}
/**
* This class is responsible for managing the local transport.
* "Which transport is the local member going to use"
*
* @prop useOldestMember Whether to use the same transport as the oldest member.
* This will only update once the first oldest member appears. Will not recompute if the oldest member leaves.
*
* @throws MatrixRTCTransportMissingError | FailToGetOpenIdToken
*/
export const createLocalTransport$ = ({
scope,
memberships$,
client,
roomId,
useOldestMember$,
}: Props): Behavior<LivekitTransport | null> => {
/**
* The transport over which we should be actively publishing our media.
* undefined when not joined.
*/
const oldestMemberTransport$ = scope.behavior(
memberships$.pipe(
map(
(memberships) =>
memberships.value[0]?.getTransport(memberships.value[0]) ?? null,
),
first((t) => t != null && isLivekitTransport(t)),
),
null,
);
/**
* The transport that we would personally prefer to publish on (if not for the
* transport preferences of others, perhaps).
*
* @throws
*/
const preferredTransport$: Behavior<LivekitTransport | null> = scope.behavior(
customLivekitUrl.value$.pipe(
switchMap((customUrl) => from(makeTransport(client, roomId, customUrl))),
),
null,
);
/**
* The chosen transport we should advertise in our MatrixRTC membership.
*/
return scope.behavior(
combineLatest([
useOldestMember$,
oldestMemberTransport$,
preferredTransport$,
]).pipe(
map(([useOldestMember, oldestMemberTransport, preferredTransport]) =>
useOldestMember
? (oldestMemberTransport ?? preferredTransport)
: preferredTransport,
),
distinctUntilChanged(areLivekitTransportsEqual),
),
);
};
const FOCI_WK_KEY = "org.matrix.msc4143.rtc_foci";
/**
*
* @param client
* @param roomId
* @returns
* @throws MatrixRTCTransportMissingError | FailToGetOpenIdToken
*/
async function makeTransport(
client: Pick<MatrixClient, "getDomain"> & OpenIDClientParts,
roomId: string,
urlFromDevSettings: string | null,
): Promise<LivekitTransport> {
let transport: LivekitTransport | undefined;
logger.trace("Searching for a preferred transport");
//TODO refactor this to use the jwt service returned alias.
const livekitAlias = roomId;
// DEVTOOL: Highest priority: Load from devtool setting
if (urlFromDevSettings !== null) {
const transportFromStorage: LivekitTransport = {
type: "livekit",
livekit_service_url: urlFromDevSettings,
livekit_alias: livekitAlias,
};
logger.info(
"Using LiveKit transport from dev tools: ",
transportFromStorage,
);
transport = transportFromStorage;
}
// WELL_KNOWN: Prioritize the .well-known/matrix/client, if available, over the configured SFU
const domain = client.getDomain();
if (domain && transport === undefined) {
// we use AutoDiscovery instead of relying on the MatrixClient having already
// been fully configured and started
const wellKnownFoci = (await AutoDiscovery.getRawClientConfig(domain))?.[
FOCI_WK_KEY
];
if (Array.isArray(wellKnownFoci)) {
const wellKnownTransport: LivekitTransportConfig | undefined =
wellKnownFoci.find((f) => f && isLivekitTransportConfig(f));
if (wellKnownTransport !== undefined) {
logger.info("Using LiveKit transport from .well-known: ", transport);
transport = { ...wellKnownTransport, livekit_alias: livekitAlias };
}
}
}
// CONFIG: Least prioritized; Load from config file
const urlFromConf = Config.get().livekit?.livekit_service_url;
if (urlFromConf && transport === undefined) {
const transportFromConf: LivekitTransport = {
type: "livekit",
livekit_service_url: urlFromConf,
livekit_alias: livekitAlias,
};
logger.info("Using LiveKit transport from config: ", transportFromConf);
transport = transportFromConf;
}
if (!transport) throw new MatrixRTCTransportMissingError(domain ?? ""); // this will call the jwt/sfu/get endpoint to pre create the livekit room.
await getSFUConfigWithOpenID(
client,
transport.livekit_service_url,
transport.livekit_alias,
);
return transport;
}

View File

@@ -1,140 +0,0 @@
/*
Copyright 2025 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
afterEach,
beforeEach,
describe,
expect,
it,
type Mock,
vi,
} from "vitest";
import { ConnectionState as LivekitConenctionState } from "livekit-client";
import { type BehaviorSubject } from "rxjs";
import { logger } from "matrix-js-sdk/lib/logger";
import { ObservableScope } from "../../ObservableScope";
import { constant } from "../../Behavior";
import {
mockLivekitRoom,
mockLocalParticipant,
mockMediaDevices,
} from "../../../utils/test";
import { Publisher } from "./Publisher";
import {
type Connection,
type ConnectionState,
} from "../remoteMembers/Connection";
import { type MuteStates } from "../../MuteStates";
import { FailToStartLivekitConnection } from "../../../utils/errors";
describe("Publisher", () => {
let scope: ObservableScope;
let connection: Connection;
let muteStates: MuteStates;
beforeEach(() => {
muteStates = {
audio: {
enabled$: constant(false),
unsetHandler: vi.fn(),
setHandler: vi.fn(),
},
video: {
enabled$: constant(false),
unsetHandler: vi.fn(),
setHandler: vi.fn(),
},
} as unknown as MuteStates;
scope = new ObservableScope();
connection = {
state$: constant({
state: "ConnectedToLkRoom",
livekitConnectionState$: constant(LivekitConenctionState.Connected),
}),
livekitRoom: mockLivekitRoom({
localParticipant: mockLocalParticipant({}),
}),
} as unknown as Connection;
});
afterEach(() => scope.end());
it("throws if livekit room could not publish", async () => {
const publisher = new Publisher(
scope,
connection,
mockMediaDevices({}),
muteStates,
constant({ supported: false, processor: undefined }),
logger,
);
// should do nothing if no tracks have been created yet.
await publisher.startPublishing();
expect(
connection.livekitRoom.localParticipant.publishTrack,
).not.toHaveBeenCalled();
await expect(publisher.createAndSetupTracks()).rejects.toThrow(
Error("audio and video is false"),
);
(muteStates.audio.enabled$ as BehaviorSubject<boolean>).next(true);
(
connection.livekitRoom.localParticipant.createTracks as Mock
).mockResolvedValue([{}, {}]);
await expect(publisher.createAndSetupTracks()).resolves.not.toThrow();
expect(
connection.livekitRoom.localParticipant.createTracks,
).toHaveBeenCalledOnce();
// failiour due to localParticipant.publishTrack
(
connection.livekitRoom.localParticipant.publishTrack as Mock
).mockRejectedValue(Error("testError"));
await expect(publisher.startPublishing()).rejects.toThrow(
new FailToStartLivekitConnection("testError"),
);
// does not try other conenction after the first one failed
expect(
connection.livekitRoom.localParticipant.publishTrack,
).toHaveBeenCalledTimes(1);
// failiour due to connection.state$
const beforeState = connection.state$.value;
(connection.state$ as BehaviorSubject<ConnectionState>).next({
state: "FailedToStart",
error: Error("testStartError"),
});
await expect(publisher.startPublishing()).rejects.toThrow(
new FailToStartLivekitConnection("testStartError"),
);
(connection.state$ as BehaviorSubject<ConnectionState>).next(beforeState);
// does not try other conenction after the first one failed
expect(
connection.livekitRoom.localParticipant.publishTrack,
).toHaveBeenCalledTimes(1);
// success case
(
connection.livekitRoom.localParticipant.publishTrack as Mock
).mockResolvedValue({});
await expect(publisher.startPublishing()).resolves.not.toThrow();
expect(
connection.livekitRoom.localParticipant.publishTrack,
).toHaveBeenCalledTimes(3);
});
});

View File

@@ -1,338 +0,0 @@
/*
Copyright 2025 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";
import { BehaviorSubject } from "rxjs";
import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
import { type Participant as LivekitParticipant } from "livekit-client";
import { logger } from "matrix-js-sdk/lib/logger";
import { Epoch, mapEpoch, ObservableScope } from "../../ObservableScope.ts";
import {
createConnectionManager$,
type ConnectionManagerData,
} from "./ConnectionManager.ts";
import { type ConnectionFactory } from "./ConnectionFactory.ts";
import { type Connection } from "./Connection.ts";
import { withTestScheduler } from "../../../utils/test.ts";
import { areLivekitTransportsEqual } from "./MatrixLivekitMembers.ts";
import { type Behavior } from "../../Behavior.ts";
// Some test constants
const TRANSPORT_1: LivekitTransport = {
type: "livekit",
livekit_service_url: "https://lk.example.org",
livekit_alias: "!alias:example.org",
};
const TRANSPORT_2: LivekitTransport = {
type: "livekit",
livekit_service_url: "https://lk.sample.com",
livekit_alias: "!alias:sample.com",
};
let fakeConnectionFactory: ConnectionFactory;
let testScope: ObservableScope;
// Can be useful to track all created connections in tests, even the disposed ones
let allCreatedConnections: Connection[];
beforeEach(() => {
testScope = new ObservableScope();
allCreatedConnections = [];
fakeConnectionFactory = {} as unknown as ConnectionFactory;
vi.mocked(fakeConnectionFactory).createConnection = vi
.fn()
.mockImplementation(
(transport: LivekitTransport, scope: ObservableScope) => {
const mockConnection = {
transport,
remoteParticipantsWithTracks$: new BehaviorSubject([]),
} as unknown as Connection;
vi.mocked(mockConnection).start = vi.fn();
vi.mocked(mockConnection).stop = vi.fn();
// Tie the connection's lifecycle to the scope to test scope lifecycle management
scope.onEnd(() => {
void mockConnection.stop();
});
allCreatedConnections.push(mockConnection);
return mockConnection;
},
);
});
afterEach(() => {
testScope.end();
});
describe("connections$ stream", () => {
test("Should create and start new connections for each transports", () => {
withTestScheduler(({ behavior, expectObservable }) => {
const { connectionManagerData$ } = createConnectionManager$({
scope: testScope,
connectionFactory: fakeConnectionFactory,
inputTransports$: behavior("a", {
a: new Epoch([TRANSPORT_1, TRANSPORT_2], 0),
}),
logger: logger,
});
expectObservable(
connectionManagerData$.pipe(mapEpoch((d) => d.getConnections())),
).toBe("a", {
a: expect.toSatisfy((e: Epoch<Connection[]>) => {
const connections = e.value;
expect(connections.length).toBe(2);
expect(
vi.mocked(fakeConnectionFactory).createConnection,
).toHaveBeenCalledTimes(2);
const conn1 = connections.find((c) =>
areLivekitTransportsEqual(c.transport, TRANSPORT_1),
);
expect(conn1).toBeDefined();
expect(conn1!.start).toHaveBeenCalled();
const conn2 = connections.find((c) =>
areLivekitTransportsEqual(c.transport, TRANSPORT_2),
);
expect(conn2).toBeDefined();
expect(conn2!.start).toHaveBeenCalled();
return true;
}),
});
});
});
test("Should start connection only once", () => {
withTestScheduler(({ behavior, expectObservable }) => {
const { connectionManagerData$ } = createConnectionManager$({
scope: testScope,
connectionFactory: fakeConnectionFactory,
inputTransports$: behavior("abcdef", {
a: new Epoch([TRANSPORT_1], 0),
b: new Epoch([TRANSPORT_1], 1),
c: new Epoch([TRANSPORT_1], 2),
d: new Epoch([TRANSPORT_1], 3),
e: new Epoch([TRANSPORT_1], 4),
f: new Epoch([TRANSPORT_1, TRANSPORT_2], 5),
}),
logger: logger,
});
expectObservable(
connectionManagerData$.pipe(mapEpoch((d) => d.getConnections())),
).toBe("xxxxxa", {
x: expect.anything(),
a: expect.toSatisfy((e: Epoch<Connection[]>) => {
const connections = e.value;
expect(connections.length).toBe(2);
expect(
vi.mocked(fakeConnectionFactory).createConnection,
).toHaveBeenCalledTimes(2);
const conn2 = connections.find((c) =>
areLivekitTransportsEqual(c.transport, TRANSPORT_2),
);
expect(conn2).toBeDefined();
const conn1 = connections.find((c) =>
areLivekitTransportsEqual(c.transport, TRANSPORT_1),
);
expect(conn1).toBeDefined();
expect(conn1!.start).toHaveBeenCalledOnce();
return true;
}),
});
});
});
test("Should cleanup connections when not needed anymore", () => {
withTestScheduler(({ behavior, expectObservable }) => {
const { connectionManagerData$ } = createConnectionManager$({
scope: testScope,
connectionFactory: fakeConnectionFactory,
inputTransports$: behavior("abc", {
a: new Epoch([TRANSPORT_1], 0),
b: new Epoch([TRANSPORT_1, TRANSPORT_2], 1),
c: new Epoch([TRANSPORT_1], 2),
}),
logger: logger,
});
expectObservable(
connectionManagerData$.pipe(mapEpoch((d) => d.getConnections())),
).toBe("xab", {
x: expect.anything(),
a: expect.toSatisfy((e: Epoch<Connection[]>) => {
const connections = e.value;
expect(connections.length).toBe(2);
return true;
}),
b: expect.toSatisfy((e: Epoch<Connection[]>) => {
const connections = e.value;
expect(connections.length).toBe(1);
// The second connection should have been stopped has it is no longer needed.
const connection2 = allCreatedConnections.find((c) =>
areLivekitTransportsEqual(c.transport, TRANSPORT_2),
);
expect(connection2).toBeDefined();
expect(connection2!.stop).toHaveBeenCalled();
// The first connection should still be active
const conn1 = connections[0];
expect(conn1.stop).not.toHaveBeenCalledOnce();
return true;
}),
});
});
});
});
describe("connectionManagerData$ stream", () => {
// Used in test to control fake connections' remoteParticipantsWithTracks$ streams
let fakePublishingParticipantsStreams: Map<
string,
Behavior<LivekitParticipant[]>
>;
function keyForTransport(transport: LivekitTransport): string {
return `${transport.livekit_service_url}|${transport.livekit_alias}`;
}
beforeEach(() => {
fakePublishingParticipantsStreams = new Map();
function getPublishingParticipantsFor(
transport: LivekitTransport,
): Behavior<LivekitParticipant[]> {
return (
fakePublishingParticipantsStreams.get(keyForTransport(transport)) ??
new BehaviorSubject([])
);
}
// need a more advanced fake connection factory
vi.mocked(fakeConnectionFactory).createConnection = vi
.fn()
.mockImplementation(
(transport: LivekitTransport, scope: ObservableScope) => {
const fakePublishingParticipants$ = new BehaviorSubject<
LivekitParticipant[]
>([]);
const mockConnection = {
transport,
remoteParticipantsWithTracks$:
getPublishingParticipantsFor(transport),
} as unknown as Connection;
vi.mocked(mockConnection).start = vi.fn();
vi.mocked(mockConnection).stop = vi.fn();
// Tie the connection's lifecycle to the scope to test scope lifecycle management
scope.onEnd(() => {
void mockConnection.stop();
});
fakePublishingParticipantsStreams.set(
keyForTransport(transport),
fakePublishingParticipants$,
);
return mockConnection;
},
);
});
test("Should report connections with the publishing participants", () => {
withTestScheduler(({ expectObservable, schedule, behavior }) => {
// Setup the fake participants streams behavior
// ==============================
fakePublishingParticipantsStreams.set(
keyForTransport(TRANSPORT_1),
behavior("oa-b", {
o: [],
a: [{ identity: "user1A" } as LivekitParticipant],
b: [
{ identity: "user1A" } as LivekitParticipant,
{ identity: "user1B" } as LivekitParticipant,
],
}),
);
fakePublishingParticipantsStreams.set(
keyForTransport(TRANSPORT_2),
behavior("o-a", {
o: [],
a: [{ identity: "user2A" } as LivekitParticipant],
}),
);
// ==============================
const { connectionManagerData$ } = createConnectionManager$({
scope: testScope,
connectionFactory: fakeConnectionFactory,
inputTransports$: behavior("a", {
a: new Epoch([TRANSPORT_1, TRANSPORT_2], 0),
}),
logger,
});
expectObservable(connectionManagerData$).toBe("abcd", {
a: expect.toSatisfy((e) => {
const data: ConnectionManagerData = e.value;
expect(data.getConnections().length).toBe(2);
expect(data.getParticipantForTransport(TRANSPORT_1).length).toBe(0);
expect(data.getParticipantForTransport(TRANSPORT_2).length).toBe(0);
return true;
}),
b: expect.toSatisfy((e) => {
const data: ConnectionManagerData = e.value;
expect(data.getConnections().length).toBe(2);
expect(data.getParticipantForTransport(TRANSPORT_1).length).toBe(1);
expect(data.getParticipantForTransport(TRANSPORT_2).length).toBe(0);
expect(data.getParticipantForTransport(TRANSPORT_1)[0].identity).toBe(
"user1A",
);
return true;
}),
c: expect.toSatisfy((e) => {
const data: ConnectionManagerData = e.value;
expect(data.getConnections().length).toBe(2);
expect(data.getParticipantForTransport(TRANSPORT_1).length).toBe(1);
expect(data.getParticipantForTransport(TRANSPORT_2).length).toBe(1);
expect(data.getParticipantForTransport(TRANSPORT_1)[0].identity).toBe(
"user1A",
);
expect(data.getParticipantForTransport(TRANSPORT_2)[0].identity).toBe(
"user2A",
);
return true;
}),
d: expect.toSatisfy((e) => {
const data: ConnectionManagerData = e.value;
expect(data.getConnections().length).toBe(2);
expect(data.getParticipantForTransport(TRANSPORT_1).length).toBe(2);
expect(data.getParticipantForTransport(TRANSPORT_2).length).toBe(1);
expect(data.getParticipantForTransport(TRANSPORT_1)[0].identity).toBe(
"user1A",
);
expect(data.getParticipantForTransport(TRANSPORT_1)[1].identity).toBe(
"user1B",
);
expect(data.getParticipantForTransport(TRANSPORT_2)[0].identity).toBe(
"user2A",
);
return true;
}),
});
});
});
});

View File

@@ -1,133 +0,0 @@
/*
Copyright 2025 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";
import { Room as LivekitRoom } from "livekit-client";
import { BehaviorSubject } from "rxjs";
import fetchMock from "fetch-mock";
import { logger } from "matrix-js-sdk/lib/logger";
import EventEmitter from "events";
import { ObservableScope } from "../../ObservableScope.ts";
import { ECConnectionFactory } from "./ConnectionFactory.ts";
import type { OpenIDClientParts } from "../../../livekit/openIDSFU.ts";
import { exampleTransport, mockMediaDevices } from "../../../utils/test.ts";
import type { ProcessorState } from "../../../livekit/TrackProcessorContext.tsx";
import { constant } from "../../Behavior";
// At the top of your test file, after imports
vi.mock("livekit-client", async (importOriginal) => {
return {
...(await importOriginal()),
Room: vi.fn().mockImplementation(function (this: LivekitRoom, options) {
const emitter = new EventEmitter();
return {
on: emitter.on.bind(emitter),
off: emitter.off.bind(emitter),
emit: emitter.emit.bind(emitter),
disconnect: vi.fn(),
remoteParticipants: new Map(),
} as unknown as LivekitRoom;
}),
};
});
let testScope: ObservableScope;
let mockClient: OpenIDClientParts;
beforeEach(() => {
testScope = new ObservableScope();
mockClient = {
getOpenIdToken: vi.fn().mockReturnValue(""),
getDeviceId: vi.fn().mockReturnValue("DEV000"),
};
});
describe("ECConnectionFactory - Audio inputs options", () => {
test.each([
{ echo: true, noise: true },
{ echo: true, noise: false },
{ echo: false, noise: true },
{ echo: false, noise: false },
])(
"it sets echoCancellation=$echo and noiseSuppression=$noise based on constructor parameters",
({ echo, noise }) => {
// test("it sets echoCancellation and noiseSuppression based on constructor parameters", () => {
const RoomConstructor = vi.mocked(LivekitRoom);
const ecConnectionFactory = new ECConnectionFactory(
mockClient,
mockMediaDevices({}),
new BehaviorSubject<ProcessorState>({
supported: true,
processor: undefined,
}),
undefined,
false,
undefined,
echo,
noise,
);
ecConnectionFactory.createConnection(exampleTransport, testScope, logger);
// Check if Room was constructed with expected options
expect(RoomConstructor).toHaveBeenCalledWith(
expect.objectContaining({
audioCaptureDefaults: expect.objectContaining({
echoCancellation: echo,
noiseSuppression: noise,
}),
}),
);
},
);
});
describe("ECConnectionFactory - ControlledAudioDevice", () => {
test.each([{ controlled: true }, { controlled: false }])(
"it sets controlledAudioDevice=$controlled then uses deviceId accordingly",
({ controlled }) => {
// test("it sets echoCancellation and noiseSuppression based on constructor parameters", () => {
const RoomConstructor = vi.mocked(LivekitRoom);
const ecConnectionFactory = new ECConnectionFactory(
mockClient,
mockMediaDevices({
audioOutput: {
available$: constant(new Map<never, never>()),
selected$: constant({ id: "DEV00", virtualEarpiece: false }),
select: () => {},
},
}),
new BehaviorSubject<ProcessorState>({
supported: true,
processor: undefined,
}),
undefined,
controlled,
undefined,
false,
false,
);
ecConnectionFactory.createConnection(exampleTransport, testScope, logger);
// Check if Room was constructed with expected options
expect(RoomConstructor).toHaveBeenCalledWith(
expect.objectContaining({
audioOutput: expect.objectContaining({
deviceId: controlled ? undefined : "DEV00",
}),
}),
);
},
);
});
afterEach(() => {
testScope.end();
fetchMock.reset();
});

View File

@@ -1,384 +0,0 @@
/*
Copyright 2025 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { describe, test, expect, beforeEach, afterEach } from "vitest";
import {
type CallMembership,
type LivekitTransport,
} from "matrix-js-sdk/lib/matrixrtc";
import { getParticipantId } from "matrix-js-sdk/lib/matrixrtc/utils";
import { combineLatest, map, type Observable } from "rxjs";
import { type IConnectionManager } from "./ConnectionManager.ts";
import {
type MatrixLivekitMember,
createMatrixLivekitMembers$,
} from "./MatrixLivekitMembers.ts";
import {
Epoch,
mapEpoch,
ObservableScope,
trackEpoch,
} from "../../ObservableScope.ts";
import { ConnectionManagerData } from "./ConnectionManager.ts";
import {
mockCallMembership,
mockRemoteParticipant,
withTestScheduler,
} from "../../../utils/test.ts";
import { type Connection } from "./Connection.ts";
let testScope: ObservableScope;
const transportA: LivekitTransport = {
type: "livekit",
livekit_service_url: "https://lk.example.org",
livekit_alias: "!alias:example.org",
};
const transportB: LivekitTransport = {
type: "livekit",
livekit_service_url: "https://lk.sample.com",
livekit_alias: "!alias:sample.com",
};
const bobMembership = mockCallMembership(
"@bob:example.org",
"DEV000",
transportA,
);
const carlMembership = mockCallMembership(
"@carl:sample.com",
"DEV111",
transportB,
);
beforeEach(() => {
testScope = new ObservableScope();
});
afterEach(() => {
testScope.end();
});
function epochMeWith$<T, U>(
source$: Observable<Epoch<U>>,
me$: Observable<T>,
): Observable<Epoch<T>> {
return combineLatest([source$, me$]).pipe(
map(([ep, cd]) => {
return new Epoch(cd, ep.epoch);
}),
);
}
test("should signal participant not yet connected to livekit", () => {
withTestScheduler(({ behavior, expectObservable }) => {
const { memberships$, membershipsWithTransport$ } = fromMemberships$(
behavior("a", {
a: [bobMembership],
}),
);
const connectionManagerData$ = epochMeWith$(
memberships$,
behavior("a", {
a: new ConnectionManagerData(),
}),
);
const matrixLivekitMember$ = createMatrixLivekitMembers$({
scope: testScope,
membershipsWithTransport$: testScope.behavior(membershipsWithTransport$),
connectionManager: {
connectionManagerData$: connectionManagerData$,
} as unknown as IConnectionManager,
});
expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe("a", {
a: expect.toSatisfy((data: MatrixLivekitMember[]) => {
expect(data.length).toEqual(1);
expectObservable(data[0].membership$).toBe("a", {
a: bobMembership,
});
expectObservable(data[0].participant$).toBe("a", {
a: null,
});
expectObservable(data[0].connection$).toBe("a", {
a: null,
});
return true;
}),
});
});
});
// Helper to create epoch'ed memberships$ and membershipsWithTransport$ from memberships observable.
function fromMemberships$(m$: Observable<CallMembership[]>): {
memberships$: Observable<Epoch<CallMembership[]>>;
membershipsWithTransport$: Observable<
Epoch<{ membership: CallMembership; transport?: LivekitTransport }[]>
>;
} {
const memberships$ = m$.pipe(trackEpoch());
const membershipsWithTransport$ = memberships$.pipe(
mapEpoch((members) => {
return members.map((m) => {
const tr = m.getTransport(m);
return {
membership: m,
transport:
tr?.type === "livekit" ? (tr as LivekitTransport) : undefined,
};
});
}),
);
return {
memberships$,
membershipsWithTransport$,
};
}
test("should signal participant on a connection that is publishing", () => {
withTestScheduler(({ behavior, expectObservable }) => {
const bobParticipantId = getParticipantId(
bobMembership.userId,
bobMembership.deviceId,
);
const { memberships$, membershipsWithTransport$ } = fromMemberships$(
behavior("a", {
a: [bobMembership],
}),
);
const connection = {
transport: bobMembership.getTransport(bobMembership),
} as unknown as Connection;
const dataWithPublisher = new ConnectionManagerData();
dataWithPublisher.add(connection, [
mockRemoteParticipant({ identity: bobParticipantId }),
]);
const connectionManagerData$ = epochMeWith$(
memberships$,
behavior("a", {
a: dataWithPublisher,
}),
);
const matrixLivekitMember$ = createMatrixLivekitMembers$({
scope: testScope,
membershipsWithTransport$: testScope.behavior(membershipsWithTransport$),
connectionManager: {
connectionManagerData$: connectionManagerData$,
} as unknown as IConnectionManager,
});
expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe("a", {
a: expect.toSatisfy((data: MatrixLivekitMember[]) => {
expect(data.length).toEqual(1);
expectObservable(data[0].membership$).toBe("a", {
a: bobMembership,
});
expectObservable(data[0].participant$).toBe("a", {
a: expect.toSatisfy((participant) => {
expect(participant).toBeDefined();
expect(participant!.identity).toEqual(bobParticipantId);
return true;
}),
});
expectObservable(data[0].connection$).toBe("a", {
a: connection,
});
return true;
}),
});
});
});
test("should signal participant on a connection that is not publishing", () => {
withTestScheduler(({ behavior, expectObservable }) => {
const { memberships$, membershipsWithTransport$ } = fromMemberships$(
behavior("a", {
a: [bobMembership],
}),
);
const connection = {
transport: bobMembership.getTransport(bobMembership),
} as unknown as Connection;
const dataWithPublisher = new ConnectionManagerData();
dataWithPublisher.add(connection, []);
const connectionManagerData$ = epochMeWith$(
memberships$,
behavior("a", {
a: dataWithPublisher,
}),
);
const matrixLivekitMember$ = createMatrixLivekitMembers$({
scope: testScope,
membershipsWithTransport$: testScope.behavior(membershipsWithTransport$),
connectionManager: {
connectionManagerData$: connectionManagerData$,
} as unknown as IConnectionManager,
});
expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe("a", {
a: expect.toSatisfy((data: MatrixLivekitMember[]) => {
expect(data.length).toEqual(1);
expectObservable(data[0].membership$).toBe("a", {
a: bobMembership,
});
expectObservable(data[0].participant$).toBe("a", {
a: null,
});
expectObservable(data[0].connection$).toBe("a", {
a: connection,
});
return true;
}),
});
});
});
describe("Publication edge case", () => {
test("bob is publishing in several connections", () => {
withTestScheduler(({ behavior, expectObservable }) => {
const { memberships$, membershipsWithTransport$ } = fromMemberships$(
behavior("a", {
a: [bobMembership, carlMembership],
}),
);
const connectionWithPublisher = new ConnectionManagerData();
const bobParticipantId = getParticipantId(
bobMembership.userId,
bobMembership.deviceId,
);
const connectionA = {
transport: transportA,
} as unknown as Connection;
const connectionB = {
transport: transportB,
} as unknown as Connection;
connectionWithPublisher.add(connectionA, [
mockRemoteParticipant({ identity: bobParticipantId }),
]);
connectionWithPublisher.add(connectionB, [
mockRemoteParticipant({ identity: bobParticipantId }),
]);
const connectionManagerData$ = epochMeWith$(
memberships$,
behavior("a", {
a: connectionWithPublisher,
}),
);
const matrixLivekitMember$ = createMatrixLivekitMembers$({
scope: testScope,
membershipsWithTransport$: testScope.behavior(
membershipsWithTransport$,
),
connectionManager: {
connectionManagerData$: connectionManagerData$,
} as unknown as IConnectionManager,
});
expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe(
"a",
{
a: expect.toSatisfy((data: MatrixLivekitMember[]) => {
expect(data.length).toEqual(2);
expectObservable(data[0].membership$).toBe("a", {
a: bobMembership,
});
expectObservable(data[0].connection$).toBe("a", {
// The real connection should be from transportA as per the membership
a: connectionA,
});
expectObservable(data[0].participant$).toBe("a", {
a: expect.toSatisfy((participant) => {
expect(participant).toBeDefined();
expect(participant!.identity).toEqual(bobParticipantId);
return true;
}),
});
return true;
}),
},
);
});
});
test("bob is publishing in the wrong connection", () => {
withTestScheduler(({ behavior, expectObservable }) => {
const { memberships$, membershipsWithTransport$ } = fromMemberships$(
behavior("a", {
a: [bobMembership, carlMembership],
}),
);
const connectionWithPublisher = new ConnectionManagerData();
const bobParticipantId = getParticipantId(
bobMembership.userId,
bobMembership.deviceId,
);
const connectionA = { transport: transportA } as unknown as Connection;
const connectionB = { transport: transportB } as unknown as Connection;
// Bob is not publishing on A
connectionWithPublisher.add(connectionA, []);
// Bob is publishing on B but his membership says A
connectionWithPublisher.add(connectionB, [
mockRemoteParticipant({ identity: bobParticipantId }),
]);
const connectionManagerData$ = epochMeWith$(
memberships$,
behavior("a", {
a: connectionWithPublisher,
}),
);
const matrixLivekitMember$ = createMatrixLivekitMembers$({
scope: testScope,
membershipsWithTransport$: testScope.behavior(
membershipsWithTransport$,
),
connectionManager: {
connectionManagerData$: connectionManagerData$,
} as unknown as IConnectionManager,
});
expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe(
"a",
{
a: expect.toSatisfy((data: MatrixLivekitMember[]) => {
expect(data.length).toEqual(2);
expectObservable(data[0].membership$).toBe("a", {
a: bobMembership,
});
expectObservable(data[0].connection$).toBe("a", {
// The real connection should be from transportA as per the membership
a: connectionA,
});
expectObservable(data[0].participant$).toBe("a", {
// No participant as Bob is not publishing on his membership transport
a: null,
});
return true;
}),
},
);
});
});
});

View File

@@ -1,138 +0,0 @@
/*
Copyright 2025 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
type LocalParticipant as LocalLivekitParticipant,
type RemoteParticipant as RemoteLivekitParticipant,
} from "livekit-client";
import {
type LivekitTransport,
type CallMembership,
} from "matrix-js-sdk/lib/matrixrtc";
import { combineLatest, filter, map } from "rxjs";
import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
import { type Behavior } from "../../Behavior";
import { type IConnectionManager } from "./ConnectionManager";
import { Epoch, type ObservableScope } from "../../ObservableScope";
import { type Connection } from "./Connection";
import { generateItemsWithEpoch } from "../../../utils/observable";
const logger = rootLogger.getChild("[MatrixLivekitMembers]");
/**
* Represents a Matrix call member and their associated LiveKit participation.
* `livekitParticipant` can be undefined if the member is not yet connected to the livekit room
* or if it has no livekit transport at all.
*/
export interface MatrixLivekitMember {
membership$: Behavior<CallMembership>;
participant$: Behavior<
LocalLivekitParticipant | RemoteLivekitParticipant | null
>;
connection$: Behavior<Connection | null>;
// participantId: string; We do not want a participantId here since it will be generated by the jwt
// TODO decide if we can also drop the userId. Its in the matrix membership anyways.
userId: string;
}
interface Props {
scope: ObservableScope;
membershipsWithTransport$: Behavior<
Epoch<{ membership: CallMembership; transport?: LivekitTransport }[]>
>;
connectionManager: IConnectionManager;
}
/**
* Combines MatrixRTC and Livekit worlds.
*
* It has a small public interface:
* - in (via constructor):
* - an observable of CallMembership[] to track the call members (The matrix side)
* - a `ConnectionManager` for the lk rooms (The livekit side)
* - out (via public Observable):
* - `remoteMatrixLivekitMember` an observable of MatrixLivekitMember[] to track the remote members and associated livekit data.
*/
export function createMatrixLivekitMembers$({
scope,
membershipsWithTransport$,
connectionManager,
}: Props): Behavior<Epoch<MatrixLivekitMember[]>> {
/**
* Stream of all the call members and their associated livekit data (if available).
*/
return scope.behavior(
combineLatest([
membershipsWithTransport$,
connectionManager.connectionManagerData$,
]).pipe(
filter((values) =>
values.every((value) => value.epoch === values[0].epoch),
),
map(
([
{ value: membershipsWithTransports, epoch },
{ value: managerData },
]) =>
new Epoch([membershipsWithTransports, managerData] as const, epoch),
),
generateItemsWithEpoch(
// Generator function.
// creates an array of `{key, data}[]`
// Each change in the keys (new key, missing key) will result in a call to the factory function.
function* ([membershipsWithTransports, managerData]) {
for (const { membership, transport } of membershipsWithTransports) {
// TODO! cannot use membership.membershipID yet, Currently its hardcoded by the jwt service to
const participantId = /*membership.membershipID*/ `${membership.userId}:${membership.deviceId}`;
const participants = transport
? managerData.getParticipantForTransport(transport)
: [];
const participant =
participants.find((p) => p.identity == participantId) ?? null;
const connection = transport
? managerData.getConnectionForTransport(transport)
: null;
yield {
keys: [participantId, membership.userId],
data: { membership, participant, connection },
};
}
},
// Each update where the key of the generator array do not change will result in updates to the `data$` observable in the factory.
(scope, data$, participantId, userId) => {
logger.debug(
`Updating data$ for participantId: ${participantId}, userId: ${userId}`,
);
// will only get called once per `participantId, userId` pair.
// updates to data$ and as a result to displayName$ and mxcAvatarUrl$ are more frequent.
return {
participantId,
userId,
...scope.splitBehavior(data$),
};
},
),
),
);
}
// TODO add back in the callviewmodel pauseWhen(this.pretendToBeDisconnected$)
// TODO add this to the JS-SDK
export function areLivekitTransportsEqual(
t1: LivekitTransport | null,
t2: LivekitTransport | null,
): boolean {
if (t1 && t2) return t1.livekit_service_url === t2.livekit_service_url;
// In case we have different lk rooms in the same SFU (depends on the livekit authorization service)
// It is only needed in case the livekit authorization service is not behaving as expected (or custom implementation)
if (!t1 && !t2) return true;
return false;
}

View File

@@ -1,613 +0,0 @@
/*
Copyright 2025 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { afterEach, beforeEach, describe, vi } from "vitest";
import {
type MatrixEvent,
type RoomMember,
type RoomState,
RoomStateEvent,
} from "matrix-js-sdk";
import EventEmitter from "events";
import { it } from "vitest";
import { ObservableScope } from "../../ObservableScope.ts";
import type { Room as MatrixRoom } from "matrix-js-sdk/lib/models/room";
import {
mockCallMembership,
mockMatrixRoomMember,
withTestScheduler,
} from "../../../utils/test.ts";
import {
createMatrixMemberMetadata$,
createRoomMembers$,
} from "./MatrixMemberMetadata.ts";
let testScope: ObservableScope;
let mockMatrixRoom: MatrixRoom;
describe("MatrixMemberMetadata", () => {
/*
* To be populated in the test setup.
* Maps userId to a partial/mock RoomMember object.
*/
let fakeMembersMap: Map<string, Partial<RoomMember>>;
beforeEach(() => {
testScope = new ObservableScope();
fakeMembersMap = new Map<string, Partial<RoomMember>>();
const roomEmitter = new EventEmitter();
mockMatrixRoom = {
on: roomEmitter.on.bind(roomEmitter),
off: roomEmitter.off.bind(roomEmitter),
emit: roomEmitter.emit.bind(roomEmitter),
// addListener: roomEmitter.addListener.bind(roomEmitter),
// removeListener: roomEmitter.removeListener.bind(roomEmitter),
getMember: vi.fn().mockImplementation((userId: string) => {
const member = fakeMembersMap.get(userId);
if (member) {
return member as RoomMember;
}
return null;
}),
getMembers: vi.fn().mockImplementation(() => {
const members = Array.from(fakeMembersMap.values());
return members;
}),
getMembersWithMembership: vi.fn().mockImplementation(() => {
const members = Array.from(fakeMembersMap.values());
return members;
}),
} as unknown as MatrixRoom;
});
function fakeMemberWith(data: Partial<RoomMember>): void {
const userId = data.userId || "@alice:example.com";
const member: Partial<RoomMember> = {
userId: userId,
rawDisplayName: data.rawDisplayName ?? userId,
getMxcAvatarUrl:
data.getMxcAvatarUrl ||
vi.fn().mockImplementation(() => {
return `mxc://example.com/${userId}`;
}),
...data,
} as unknown as RoomMember;
fakeMembersMap.set(userId, member);
}
afterEach(() => {
fakeMembersMap.clear();
});
describe("displayname", () => {
function updateDisplayName(
userId: `@${string}:${string}`,
newDisplayName: string,
): void {
const member = fakeMembersMap.get(userId);
if (member) {
member.rawDisplayName = newDisplayName;
// Emit the event to notify listeners
mockMatrixRoom.emit(
RoomStateEvent.Members,
{} as unknown as MatrixEvent,
{} as unknown as RoomState,
member as RoomMember,
);
} else {
throw new Error(`No member found with userId: ${userId}`);
}
}
it("should show our own user if present in rtc session and room", () => {
withTestScheduler(({ behavior, expectObservable }) => {
fakeMemberWith({
userId: "@local:example.com",
rawDisplayName: "it's a me",
});
const memberships$ = behavior("a", {
a: [mockCallMembership("@local:example.com", "DEVICE1")],
});
const metadataStore = createMatrixMemberMetadata$(
testScope,
memberships$,
createRoomMembers$(testScope, mockMatrixRoom),
);
const dn$ =
metadataStore.createDisplayNameBehavior$("@local:example.com");
expectObservable(dn$).toBe("a", {
a: "it's a me",
});
expectObservable(metadataStore.displaynameMap$).toBe("a", {
a: new Map<string, string>([["@local:example.com", "it's a me"]]),
});
});
});
function setUpBasicRoom(): void {
fakeMemberWith({
userId: "@local:example.com",
rawDisplayName: "it's a me",
});
fakeMemberWith({ userId: "@alice:example.com", rawDisplayName: "Alice" });
fakeMemberWith({ userId: "@bob:example.com", rawDisplayName: "Bob" });
fakeMemberWith({ userId: "@carl:example.com", rawDisplayName: "Carl" });
fakeMemberWith({ userId: "@evil:example.com", rawDisplayName: "Carl" });
fakeMemberWith({ userId: "@bob:foo.bar", rawDisplayName: "Bob" });
fakeMemberWith({ userId: "@no-name:foo.bar" });
}
it("should get displayName for users", () => {
setUpBasicRoom();
withTestScheduler(({ behavior, expectObservable }) => {
const memberships$ = behavior("a", {
a: [
mockCallMembership("@alice:example.com", "DEVICE1"),
mockCallMembership("@bob:example.com", "DEVICE1"),
],
});
const metadataStore = createMatrixMemberMetadata$(
testScope,
memberships$,
createRoomMembers$(testScope, mockMatrixRoom),
);
const aliceDispName$ =
metadataStore.createDisplayNameBehavior$("@alice:example.com");
expectObservable(aliceDispName$).toBe("a", {
a: "Alice",
});
expectObservable(metadataStore.displaynameMap$).toBe("a", {
a: new Map<string, string>([
["@alice:example.com", "Alice"],
["@bob:example.com", "Bob"],
]),
});
});
});
it("should use userId if no display name", () => {
withTestScheduler(({ behavior, expectObservable }) => {
setUpBasicRoom();
const memberships$ = behavior("a", {
a: [mockCallMembership("@no-name:foo.bar", "D000")],
});
const metadataStore = createMatrixMemberMetadata$(
testScope,
memberships$,
createRoomMembers$(testScope, mockMatrixRoom),
);
expectObservable(metadataStore.displaynameMap$).toBe("a", {
a: new Map<string, string>([
["@no-name:foo.bar", "@no-name:foo.bar"],
]),
});
});
});
it("should disambiguate users with same display name", () => {
withTestScheduler(({ behavior, expectObservable }) => {
setUpBasicRoom();
const memberships$ = behavior("a", {
a: [
mockCallMembership("@bob:example.com", "DEVICE1"),
mockCallMembership("@bob:example.com", "DEVICE2"),
mockCallMembership("@bob:foo.bar", "BOB000"),
mockCallMembership("@carl:example.com", "C000"),
mockCallMembership("@evil:example.com", "E000"),
],
});
const metadataStore = createMatrixMemberMetadata$(
testScope,
memberships$,
createRoomMembers$(testScope, mockMatrixRoom),
);
expectObservable(metadataStore.displaynameMap$).toBe("a", {
a: new Map<string, string>([
// ["@local:example.com", "it's a me"],
["@bob:example.com", "Bob (@bob:example.com)"],
["@bob:example.com", "Bob (@bob:example.com)"],
["@bob:foo.bar", "Bob (@bob:foo.bar)"],
["@carl:example.com", "Carl (@carl:example.com)"],
["@evil:example.com", "Carl (@evil:example.com)"],
]),
});
});
});
it("should start to disambiguate reactivly when needed", () => {
withTestScheduler(({ behavior, expectObservable }) => {
setUpBasicRoom();
const memberships$ = behavior("ab", {
a: [mockCallMembership("@bob:example.com", "DEVICE1")],
b: [
mockCallMembership("@bob:example.com", "DEVICE1"),
mockCallMembership("@bob:foo.bar", "BOB000"),
],
});
const metadataStore = createMatrixMemberMetadata$(
testScope,
memberships$,
createRoomMembers$(testScope, mockMatrixRoom),
);
expectObservable(metadataStore.displaynameMap$).toBe("ab", {
a: new Map<string, string>([["@bob:example.com", "Bob"]]),
b: new Map<string, string>([
["@bob:example.com", "Bob (@bob:example.com)"],
["@bob:foo.bar", "Bob (@bob:foo.bar)"],
]),
});
});
});
it("should keep disambiguated name when other leave", () => {
withTestScheduler(({ behavior, expectObservable }) => {
setUpBasicRoom();
const memberships$ = behavior("ab", {
a: [
mockCallMembership("@bob:example.com", "DEVICE1"),
mockCallMembership("@bob:foo.bar", "BOB000"),
],
b: [mockCallMembership("@bob:example.com", "DEVICE1")],
});
const metadataStore = createMatrixMemberMetadata$(
testScope,
memberships$,
createRoomMembers$(testScope, mockMatrixRoom),
);
expectObservable(metadataStore.displaynameMap$).toBe("ab", {
a: new Map<string, string>([
["@bob:example.com", "Bob (@bob:example.com)"],
["@bob:foo.bar", "Bob (@bob:foo.bar)"],
]),
b: new Map<string, string>([
["@bob:example.com", "Bob (@bob:example.com)"],
]),
});
});
});
it("should disambiguate on name change", () => {
withTestScheduler(({ behavior, schedule, expectObservable }) => {
setUpBasicRoom();
const memberships$ = behavior("a", {
a: [
mockCallMembership("@bob:example.com", "B000"),
mockCallMembership("@carl:example.com", "C000"),
],
});
const metadataStore = createMatrixMemberMetadata$(
testScope,
memberships$,
createRoomMembers$(testScope, mockMatrixRoom),
);
schedule("-a", {
a: () => {
updateDisplayName("@carl:example.com", "Bob");
},
});
expectObservable(metadataStore.displaynameMap$).toBe("ab", {
a: new Map<string, string>([
["@bob:example.com", "Bob"],
["@carl:example.com", "Carl"],
]),
b: new Map<string, string>([
["@bob:example.com", "Bob (@bob:example.com)"],
["@carl:example.com", "Bob (@carl:example.com)"],
]),
});
});
});
it("should track individual member id with createDisplayNameBehavior", () => {
withTestScheduler(({ behavior, schedule, expectObservable }) => {
setUpBasicRoom();
const BOB = "@bob:example.com";
const CARL = "@carl:example.com";
// for this test we build a mock environment that does all possible changes:
// - memberships join/leave
// - room join/leave
// - disambiguate
const memberships$ = behavior("ab-d", {
a: [mockCallMembership(CARL, "C000")],
b: [
mockCallMembership(CARL, "C000"),
// bob joins
mockCallMembership(BOB, "B000"),
],
// c carl gets renamed to BOB
d: [
// carl leaves
mockCallMembership(BOB, "B000"),
],
});
schedule("--a-", {
a: () => {
// carl renames
updateDisplayName(CARL, "Bob");
},
});
const metadataStore = createMatrixMemberMetadata$(
testScope,
memberships$,
createRoomMembers$(testScope, mockMatrixRoom),
);
const bob$ = metadataStore.createDisplayNameBehavior$(BOB);
const carl$ = metadataStore.createDisplayNameBehavior$(CARL);
expectObservable(bob$).toBe("abc-", {
a: undefined,
b: "Bob",
c: "Bob (@bob:example.com)",
// bob stays disambiguate even though carl left
// d: "Bob (@bob:example.com)",
});
expectObservable(carl$).toBe("a-cd", {
a: "Carl",
// b: "Carl",
// carl gets renamed and disambiguate
c: "Bob (@carl:example.com)",
d: undefined,
});
});
});
it("should disambiguate users with invisible characters", () => {
withTestScheduler(({ behavior, expectObservable }) => {
const bobRtcMember = mockCallMembership("@bob:example.org", "BBBB");
const bobZeroWidthSpaceRtcMember = mockCallMembership(
"@bob2:example.org",
"BBBB",
);
const bob = mockMatrixRoomMember(bobRtcMember, {
rawDisplayName: "Bob",
});
const bobZeroWidthSpace = mockMatrixRoomMember(
bobZeroWidthSpaceRtcMember,
{
rawDisplayName: "Bo\u200bb",
},
);
fakeMemberWith(bob);
fakeMemberWith(bobZeroWidthSpace);
fakeMemberWith({ userId: "@carol:example.org" });
const memberships$ = behavior("ab", {
a: [mockCallMembership("@carol:example.org", "1111"), bobRtcMember],
b: [
mockCallMembership("@carol:example.org", "1111"),
bobRtcMember,
bobZeroWidthSpaceRtcMember,
],
});
const metadataStore = createMatrixMemberMetadata$(
testScope,
memberships$,
createRoomMembers$(testScope, mockMatrixRoom),
);
const bob$ =
metadataStore.createDisplayNameBehavior$("@bob:example.org");
const bob2$ =
metadataStore.createDisplayNameBehavior$("@bob2:example.org");
const carol$ =
metadataStore.createDisplayNameBehavior$("@carol:example.org");
expectObservable(bob$).toBe("ab", {
a: "Bob",
b: "Bob (@bob:example.org)",
});
expectObservable(bob2$).toBe("ab", {
a: undefined,
b: "Bo\u200bb (@bob2:example.org)",
});
expectObservable(carol$).toBe("a-", {
a: "@carol:example.org",
});
expectObservable(metadataStore.displaynameMap$).toBe("ab", {
// Carol has no displayname - So userId is used.
a: new Map([
["@carol:example.org", "@carol:example.org"],
["@bob:example.org", "Bob"],
]),
// Other Bob joins, and should handle zero width hacks.
b: new Map([
["@carol:example.org", "@carol:example.org"],
[bobRtcMember.userId, `Bob (@bob:example.org)`],
[
bobZeroWidthSpace.userId,
`${bobZeroWidthSpace.rawDisplayName} (${bobZeroWidthSpace.userId})`,
],
]),
});
});
});
it("should strip RTL characters from displayname", () => {
withTestScheduler(({ behavior, expectObservable }) => {
const daveRtcMember = mockCallMembership("@dave:example.org", "DDDD");
const daveRTLRtcMember = mockCallMembership(
"@dave2:example.org",
"DDDD",
);
const dave = mockMatrixRoomMember(daveRtcMember, {
rawDisplayName: "Dave",
});
const daveRTL = mockMatrixRoomMember(daveRTLRtcMember, {
rawDisplayName: "\u202eevaD",
});
fakeMemberWith({ userId: "@carol:example.org" });
fakeMemberWith(daveRTL);
fakeMemberWith(dave);
const memberships$ = behavior("ab", {
a: [mockCallMembership("@carol:example.org", "DDDD")],
b: [
mockCallMembership("@carol:example.org", "DDDD"),
daveRtcMember,
daveRTLRtcMember,
],
});
const metadataStore = createMatrixMemberMetadata$(
testScope,
memberships$,
createRoomMembers$(testScope, mockMatrixRoom),
);
expectObservable(metadataStore.displaynameMap$).toBe("ab", {
// Carol has no displayname - So userId is used.
a: new Map([["@carol:example.org", "@carol:example.org"]]),
// Both Dave's join. Since after stripping
b: new Map([
["@carol:example.org", "@carol:example.org"],
// Not disambiguated
["@dave:example.org", "Dave"],
// This one is, since it's using RTL.
["@dave2:example.org", "evaD (@dave2:example.org)"],
]),
});
});
});
});
describe("avatarUrl", () => {
function updateAvatarUrl(
userId: `@${string}:${string}`,
avatarUrl: string,
): void {
const member = fakeMembersMap.get(userId);
if (member) {
member.getMxcAvatarUrl = vi.fn().mockReturnValue(avatarUrl);
// Emit the event to notify listeners
mockMatrixRoom.emit(
RoomStateEvent.Members,
{} as unknown as MatrixEvent,
{} as unknown as RoomState,
member as RoomMember,
);
} else {
throw new Error(`No member found with userId: ${userId}`);
}
}
it("should use avatar url from room members", () => {
withTestScheduler(({ behavior, expectObservable }) => {
fakeMemberWith({
userId: "@local:example.com",
});
fakeMemberWith({
userId: "@alice:example.com",
getMxcAvatarUrl: vi.fn().mockReturnValue("mxc://custom.url/avatar"),
});
const memberships$ = behavior("a", {
a: [
mockCallMembership("@local:example.com", "DEVICE1"),
mockCallMembership("@alice:example.com", "DEVICE1"),
],
});
const metadataStore = createMatrixMemberMetadata$(
testScope,
memberships$,
createRoomMembers$(testScope, mockMatrixRoom),
);
const local$ =
metadataStore.createAvatarUrlBehavior$("@local:example.com");
const alice$ =
metadataStore.createAvatarUrlBehavior$("@alice:example.com");
expectObservable(local$).toBe("a", {
a: "mxc://example.com/@local:example.com",
});
expectObservable(alice$).toBe("a", {
a: "mxc://custom.url/avatar",
});
expectObservable(metadataStore.avatarMap$).toBe("a", {
a: new Map<string, string>([
["@local:example.com", "mxc://example.com/@local:example.com"],
["@alice:example.com", "mxc://custom.url/avatar"],
]),
});
});
});
it("should update on avatar change and user join/leave", () => {
withTestScheduler(({ behavior, schedule, expectObservable }) => {
fakeMemberWith({ userId: "@carl:example.com" });
fakeMemberWith({ userId: "@bob:example.com" });
const memberships$ = behavior("ab-d", {
a: [mockCallMembership("@bob:example.com", "B000")],
b: [
mockCallMembership("@bob:example.com", "B000"),
mockCallMembership("@carl:example.com", "C000"),
],
d: [mockCallMembership("@carl:example.com", "C000")],
});
const metadataStore = createMatrixMemberMetadata$(
testScope,
memberships$,
createRoomMembers$(testScope, mockMatrixRoom),
);
schedule("--c-", {
c: () => {
updateAvatarUrl(
"@carl:example.com",
"mxc://updated.me/updatedAvatar",
);
},
});
const bob$ = metadataStore.createAvatarUrlBehavior$("@bob:example.com");
const carl$ =
metadataStore.createAvatarUrlBehavior$("@carl:example.com");
expectObservable(bob$).toBe("a---", {
a: "mxc://example.com/@bob:example.com",
});
expectObservable(carl$).toBe("a-c-", {
a: "mxc://example.com/@carl:example.com",
c: "mxc://updated.me/updatedAvatar",
});
expectObservable(metadataStore.avatarMap$).toBe("a-c-", {
a: new Map<string, string>([
["@bob:example.com", "mxc://example.com/@bob:example.com"],
["@carl:example.com", "mxc://example.com/@carl:example.com"],
]),
// expect an update once we update the avatar URL
c: new Map<string, string>([
["@bob:example.com", "mxc://example.com/@bob:example.com"],
["@carl:example.com", "mxc://updated.me/updatedAvatar"],
]),
});
});
});
});
});

View File

@@ -1,180 +0,0 @@
/*
Copyright 2025 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type RoomMember, RoomStateEvent } from "matrix-js-sdk";
import { combineLatest, fromEvent, map } from "rxjs";
import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc";
import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
import {
KnownMembership,
type Room as MatrixRoom,
} from "matrix-js-sdk/lib/matrix";
// eslint-disable-next-line rxjs/no-internal
import { type ObservableScope } from "../../ObservableScope";
import {
calculateDisplayName,
shouldDisambiguate,
} from "../../../utils/displayname";
import { type Behavior } from "../../Behavior";
const logger = rootLogger.getChild("[MatrixMemberMetadata]");
export type RoomMemberMap = Map<
string,
Pick<RoomMember, "userId" | "getMxcAvatarUrl" | "rawDisplayName">
>;
export function roomToMembersMap(matrixRoom: MatrixRoom): RoomMemberMap {
const members = matrixRoom
.getMembersWithMembership(KnownMembership.Join)
.concat(matrixRoom.getMembersWithMembership(KnownMembership.Invite));
return members.reduce((acc, member) => {
acc.set(member.userId, {
userId: member.userId,
getMxcAvatarUrl: member.getMxcAvatarUrl.bind(member),
rawDisplayName: member.rawDisplayName,
});
return acc;
}, new Map());
}
export function createRoomMembers$(
scope: ObservableScope,
matrixRoom: MatrixRoom,
): Behavior<RoomMemberMap> {
return scope.behavior(
fromEvent(matrixRoom, RoomStateEvent.Members).pipe(
map(() => roomToMembersMap(matrixRoom)),
),
roomToMembersMap(matrixRoom),
);
}
/**
* creates the member that this DM is with in case it is a DM (two members) otherwise null
*/
export function createDMMember$(
scope: ObservableScope,
roomMembers$: Behavior<RoomMemberMap>,
matrixRoom: MatrixRoom,
): Behavior<Pick<
RoomMember,
"userId" | "getMxcAvatarUrl" | "rawDisplayName"
> | null> {
// We cannot use the normal direct check from matrix since we do not have access to the account data.
// use primitive member count === 2 check instead.
return scope.behavior(
roomMembers$.pipe(
map((membersMap) => {
// primitive appraoch do to no access to account data.
const isDM = membersMap.size === 2;
if (!isDM) return null;
return matrixRoom.getMember(matrixRoom.guessDMUserId());
}),
),
);
}
/**
* Displayname for each member of the call. This will disambiguate
* any displayname that clashes with another member. Only members
* joined to the call are considered here.
*
* @returns Map<userId, displayname> uses the Matrix user ID as the key.
*/
// don't do this work more times than we need to. This is achieved by converting to a behavior:
export const memberDisplaynames$ = (
scope: ObservableScope,
memberships$: Behavior<Pick<CallMembership, "userId">[]>,
roomMembers$: Behavior<RoomMemberMap>,
): Behavior<Map<string, string>> => {
// This map tracks userIds that at some point needed disambiguation.
// This is a memory leak bound to the number of participants.
// A call application will always increase the memory if there have been more members in a call.
// Its capped by room member participants.
const shouldDisambiguateTrackerMap = new Set<string>();
return scope.behavior(
combineLatest([
// Handle call membership changes
memberships$,
// Additionally handle display name changes (implicitly reacting to them)
roomMembers$,
// TODO: do we need: pauseWhen(this.pretendToBeDisconnected$),
]).pipe(
map(([memberships, roomMembers]) => {
const displaynameMap = new Map<string, string>();
// We only consider RTC members for disambiguation as they are the only visible members.
for (const rtcMember of memberships) {
const member = roomMembers.get(rtcMember.userId);
if (!member) {
logger.error(`Could not find member for user ${rtcMember.userId}`);
continue;
}
const disambiguateComputed = shouldDisambiguate(
member,
memberships,
roomMembers,
);
const disambiguate =
shouldDisambiguateTrackerMap.has(rtcMember.userId) ||
disambiguateComputed;
if (disambiguate) shouldDisambiguateTrackerMap.add(rtcMember.userId);
displaynameMap.set(
rtcMember.userId,
calculateDisplayName(member, disambiguate),
);
}
return displaynameMap;
}),
),
);
};
export const createMatrixMemberMetadata$ = (
scope: ObservableScope,
memberships$: Behavior<Pick<CallMembership, "userId">[]>,
roomMembers$: Behavior<RoomMemberMap>,
): {
createDisplayNameBehavior$: (userId: string) => Behavior<string | undefined>;
createAvatarUrlBehavior$: (userId: string) => Behavior<string | undefined>;
displaynameMap$: Behavior<Map<string, string>>;
avatarMap$: Behavior<Map<string, string | undefined>>;
} => {
const displaynameMap$ = memberDisplaynames$(
scope,
memberships$,
roomMembers$,
);
const avatarMap$ = scope.behavior(
roomMembers$.pipe(
map((roomMembers) =>
Array.from(roomMembers.keys()).reduce((acc, key) => {
acc.set(key, roomMembers.get(key)?.getMxcAvatarUrl());
return acc;
}, new Map<string, string | undefined>()),
),
),
);
return {
createDisplayNameBehavior$: (userId: string) =>
scope.behavior(
displaynameMap$.pipe(
map((displaynameMap) => displaynameMap.get(userId)),
),
),
createAvatarUrlBehavior$: (userId: string) =>
scope.behavior(
roomMembers$.pipe(
map((roomMembers) => roomMembers.get(userId)?.getMxcAvatarUrl()),
),
),
// mostly for testing purposes
displaynameMap$,
avatarMap$,
};
};

View File

@@ -1,228 +0,0 @@
/*
Copyright 2025 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { test, vi, expect, beforeEach, afterEach } from "vitest";
import { BehaviorSubject } from "rxjs";
import { type Room as LivekitRoom } from "livekit-client";
import EventEmitter from "events";
import fetchMock from "fetch-mock";
import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
import { logger } from "matrix-js-sdk/lib/logger";
import {
type Epoch,
ObservableScope,
trackEpoch,
} from "../../ObservableScope.ts";
import { ECConnectionFactory } from "./ConnectionFactory.ts";
import { type OpenIDClientParts } from "../../../livekit/openIDSFU.ts";
import {
mockCallMembership,
mockMediaDevices,
withTestScheduler,
} from "../../../utils/test.ts";
import { type ProcessorState } from "../../../livekit/TrackProcessorContext.tsx";
import {
areLivekitTransportsEqual,
createMatrixLivekitMembers$,
type MatrixLivekitMember,
} from "./MatrixLivekitMembers.ts";
import { createConnectionManager$ } from "./ConnectionManager.ts";
import { membershipsAndTransports$ } from "../../SessionBehaviors.ts";
// Test the integration of ConnectionManager and MatrixLivekitMerger
let testScope: ObservableScope;
let ecConnectionFactory: ECConnectionFactory;
let mockClient: OpenIDClientParts;
let lkRoomFactory: () => LivekitRoom;
const createdMockLivekitRooms: Map<string, LivekitRoom> = new Map();
beforeEach(() => {
testScope = new ObservableScope();
mockClient = {
getOpenIdToken: vi.fn().mockReturnValue(""),
getDeviceId: vi.fn().mockReturnValue("DEV000"),
};
lkRoomFactory = vi.fn().mockImplementation(() => {
const emitter = new EventEmitter();
const base = {
on: emitter.on.bind(emitter),
off: emitter.off.bind(emitter),
emit: emitter.emit.bind(emitter),
disconnect: vi.fn(),
remoteParticipants: new Map(),
} as unknown as LivekitRoom;
vi.mocked(base).connect = vi.fn().mockImplementation(({ url }) => {
createdMockLivekitRooms.set(url, base);
});
return base;
});
ecConnectionFactory = new ECConnectionFactory(
mockClient,
mockMediaDevices({}),
new BehaviorSubject<ProcessorState>({
supported: true,
processor: undefined,
}),
undefined,
false,
lkRoomFactory,
);
//TODO a bit annoying to have to do a http mock?
fetchMock.post(`path:/sfu/get`, (url) => {
const domain = new URL(url).hostname; // Extract the domain from the URL
return {
status: 200,
body: {
url: `wss://${domain}/livekit/sfu`,
jwt: "ATOKEN",
},
};
});
});
afterEach(() => {
testScope.end();
fetchMock.reset();
});
test("bob, carl, then bob joining no tracks yet", () => {
withTestScheduler(({ expectObservable, behavior, scope }) => {
const bobMembership = mockCallMembership("@bob:example.com", "BDEV000");
const carlMembership = mockCallMembership("@carl:example.com", "CDEV000");
const daveMembership = mockCallMembership("@dave:foo.bar", "DDEV000");
const eMarble = "abc";
const vMarble = "abc";
const memberships$ = scope.behavior(
behavior(eMarble, {
a: [bobMembership],
b: [bobMembership, carlMembership],
c: [bobMembership, carlMembership, daveMembership],
}).pipe(trackEpoch()),
);
const membershipsAndTransports = membershipsAndTransports$(
testScope,
memberships$,
);
const connectionManager = createConnectionManager$({
scope: testScope,
connectionFactory: ecConnectionFactory,
inputTransports$: membershipsAndTransports.transports$,
logger: logger,
});
const matrixLivekitItems$ = createMatrixLivekitMembers$({
scope: testScope,
membershipsWithTransport$:
membershipsAndTransports.membershipsWithTransport$,
connectionManager,
});
expectObservable(matrixLivekitItems$).toBe(vMarble, {
a: expect.toSatisfy((e: Epoch<MatrixLivekitMember[]>) => {
const items = e.value;
expect(items.length).toBe(1);
const item = items[0]!;
expectObservable(item.membership$).toBe("a", {
a: bobMembership,
});
expectObservable(item.connection$).toBe("a", {
a: expect.toSatisfy((co) =>
areLivekitTransportsEqual(
co.transport,
bobMembership.transports[0]! as LivekitTransport,
),
),
});
expectObservable(item.participant$).toBe("a", {
a: null,
});
return true;
}),
b: expect.toSatisfy((e: Epoch<MatrixLivekitMember[]>) => {
const items = e.value;
expect(items.length).toBe(2);
{
const item = items[0]!;
expectObservable(item.membership$).toBe("a", {
a: bobMembership,
});
expectObservable(item.participant$).toBe("a", {
a: null,
});
}
{
const item = items[1]!;
expectObservable(item.membership$).toBe("a", {
a: carlMembership,
});
expectObservable(item.participant$).toBe("a", {
a: null,
});
expectObservable(item.connection$).toBe("a", {
a: expect.toSatisfy((connection) => {
expect(
areLivekitTransportsEqual(
connection.transport,
carlMembership.transports[0]! as LivekitTransport,
),
).toBe(true);
return true;
}),
});
}
return true;
}),
c: expect.toSatisfy((e: Epoch<MatrixLivekitMember[]>) => {
const items = e.value;
expect(items.length).toBe(3);
expectObservable(items[0].membership$).toBe("a", {
a: bobMembership,
});
expectObservable(items[1].membership$).toBe("b", {
a: carlMembership,
});
{
const item = items[2]!;
expectObservable(item.membership$).toBe("a", {
a: daveMembership,
});
expectObservable(item.connection$).toBe("a", {
a: expect.toSatisfy((connection) => {
expect(
areLivekitTransportsEqual(
connection.transport,
daveMembership.transports[0]! as LivekitTransport,
),
).toBe(true);
return true;
}),
});
expectObservable(item.participant$).toBe("a", {
a: null,
});
}
return true;
}),
x: expect.anything(),
});
});
});

View File

@@ -5,18 +5,15 @@ Copyright 2025 Element Creations Ltd.
Please see LICENSE in the repository root for full details.
*/
import { it, vi, expect } from "vitest";
import { test, vi, expect } from "vitest";
import EventEmitter from "events";
// import * as ComponentsCore from "@livekit/components-core";
import { withCallViewModel } from "./CallViewModel/CallViewModelTestUtils.ts";
import { type CallViewModel } from "./CallViewModel/CallViewModel.ts";
import { constant } from "./Behavior.ts";
import { withCallViewModel } from "./CallViewModel.test.ts";
import { aliceParticipant, localRtcMember } from "../utils/test-fixtures.ts";
import { ElementWidgetActions, widget } from "../widget.ts";
import { E2eeType } from "../e2ee/e2eeType.ts";
vi.mock("@livekit/components-core", { spy: true });
import { type CallViewModel } from "./CallViewModel.ts";
vi.mock("../widget", () => ({
ElementWidgetActions: {
@@ -34,7 +31,7 @@ vi.mock("../widget", () => ({
},
}));
it("expect leave when ElementWidgetActions.HangupCall is called", async () => {
test("expect leave when ElementWidgetActions.HangupCall is called", async () => {
const pr = Promise.withResolvers<string>();
withCallViewModel(
{

View File

@@ -27,6 +27,7 @@ import {
RoomEvent as LivekitRoomEvent,
RemoteTrack,
} from "livekit-client";
import { type RoomMember } from "matrix-js-sdk";
import { logger } from "matrix-js-sdk/lib/logger";
import {
BehaviorSubject,
@@ -43,7 +44,6 @@ import {
startWith,
switchMap,
throttleTime,
distinctUntilChanged,
} from "rxjs";
import { alwaysShowSelf } from "../settings/settings";
@@ -180,35 +180,29 @@ function observeRemoteTrackReceivingOkay$(
}
function encryptionErrorObservable$(
room$: Behavior<LivekitRoom | undefined>,
room: LivekitRoom,
participant: Participant,
encryptionSystem: EncryptionSystem,
criteria: string,
): Observable<boolean> {
return room$.pipe(
switchMap((room) => {
if (room === undefined) return of(false);
return roomEventSelector(room, LivekitRoomEvent.EncryptionError).pipe(
map((e) => {
const [err] = e;
if (encryptionSystem.kind === E2eeType.PER_PARTICIPANT) {
return (
// Ideally we would pull the participant identity from the field on the error.
// However, it gets lost in the serialization process between workers.
// So, instead we do a string match
(err?.message.includes(participant.identity) &&
err?.message.includes(criteria)) ??
false
);
} else if (encryptionSystem.kind === E2eeType.SHARED_KEY) {
return !!err?.message.includes(criteria);
}
return roomEventSelector(room, LivekitRoomEvent.EncryptionError).pipe(
map((e) => {
const [err] = e;
if (encryptionSystem.kind === E2eeType.PER_PARTICIPANT) {
return (
// Ideally we would pull the participant identity from the field on the error.
// However, it gets lost in the serialization process between workers.
// So, instead we do a string match
(err?.message.includes(participant.identity) &&
err?.message.includes(criteria)) ??
false
);
} else if (encryptionSystem.kind === E2eeType.SHARED_KEY) {
return !!err?.message.includes(criteria);
}
return false;
}),
);
return false;
}),
distinctUntilChanged(),
throttleTime(1000), // Throttle to avoid spamming the UI
startWith(false),
);
@@ -226,7 +220,7 @@ abstract class BaseMediaViewModel {
/**
* The LiveKit video track for this media.
*/
public readonly video$: Behavior<TrackReferenceOrPlaceholder | null>;
public readonly video$: Behavior<TrackReferenceOrPlaceholder | undefined>;
/**
* Whether there should be a warning that this media is unencrypted.
*/
@@ -241,10 +235,12 @@ abstract class BaseMediaViewModel {
private observeTrackReference$(
source: Track.Source,
): Behavior<TrackReferenceOrPlaceholder | null> {
): Behavior<TrackReferenceOrPlaceholder | undefined> {
return this.scope.behavior(
this.participant$.pipe(
switchMap((p) => (!p ? of(null) : observeTrackReference$(p, source))),
switchMap((p) =>
p === undefined ? of(undefined) : observeTrackReference$(p, source),
),
),
);
}
@@ -256,22 +252,23 @@ abstract class BaseMediaViewModel {
*/
public readonly id: string,
/**
* The Matrix user to which this media belongs.
* The Matrix room member to which this media belongs.
*/
public readonly userId: string,
// TODO: Fully separate the data layer from the UI layer by keeping the
// member object internal
public readonly member: RoomMember,
// We don't necessarily have a participant if a user connects via MatrixRTC but not (yet) through
// livekit.
protected readonly participant$: Observable<
LocalParticipant | RemoteParticipant | null
LocalParticipant | RemoteParticipant | undefined
>,
encryptionSystem: EncryptionSystem,
audioSource: AudioSource,
videoSource: VideoSource,
livekitRoom$: Behavior<LivekitRoom | undefined>,
public readonly focusUrl$: Behavior<string | undefined>,
livekitRoom: LivekitRoom,
public readonly focusURL: string,
public readonly displayName$: Behavior<string>,
public readonly mxcAvatarUrl$: Behavior<string | undefined>,
) {
const audio$ = this.observeTrackReference$(audioSource);
this.video$ = this.observeTrackReference$(videoSource);
@@ -299,13 +296,13 @@ abstract class BaseMediaViewModel {
} else if (encryptionSystem.kind === E2eeType.PER_PARTICIPANT) {
return combineLatest([
encryptionErrorObservable$(
livekitRoom$,
livekitRoom,
participant,
encryptionSystem,
"MissingKey",
),
encryptionErrorObservable$(
livekitRoom$,
livekitRoom,
participant,
encryptionSystem,
"InvalidKey",
@@ -325,7 +322,7 @@ abstract class BaseMediaViewModel {
} else {
return combineLatest([
encryptionErrorObservable$(
livekitRoom$,
livekitRoom,
participant,
encryptionSystem,
"InvalidKey",
@@ -407,28 +404,26 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel {
public constructor(
scope: ObservableScope,
id: string,
userId: string,
participant$: Observable<LocalParticipant | RemoteParticipant | null>,
member: RoomMember,
participant$: Observable<LocalParticipant | RemoteParticipant | undefined>,
encryptionSystem: EncryptionSystem,
livekitRoom$: Behavior<LivekitRoom | undefined>,
focusUrl$: Behavior<string | undefined>,
livekitRoom: LivekitRoom,
focusUrl: string,
displayName$: Behavior<string>,
mxcAvatarUrl$: Behavior<string | undefined>,
public readonly handRaised$: Behavior<Date | null>,
public readonly reaction$: Behavior<ReactionOption | null>,
) {
super(
scope,
id,
userId,
member,
participant$,
encryptionSystem,
Track.Source.Microphone,
Track.Source.Camera,
livekitRoom$,
focusUrl$,
livekitRoom,
focusUrl,
displayName$,
mxcAvatarUrl$,
);
const media$ = this.scope.behavior(
@@ -545,27 +540,25 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel {
public constructor(
scope: ObservableScope,
id: string,
userId: string,
participant$: Behavior<LocalParticipant | null>,
member: RoomMember,
participant$: Behavior<LocalParticipant | undefined>,
encryptionSystem: EncryptionSystem,
livekitRoom$: Behavior<LivekitRoom | undefined>,
focusUrl$: Behavior<string | undefined>,
livekitRoom: LivekitRoom,
focusURL: string,
private readonly mediaDevices: MediaDevices,
displayName$: Behavior<string>,
mxcAvatarUrl$: Behavior<string | undefined>,
handRaised$: Behavior<Date | null>,
reaction$: Behavior<ReactionOption | null>,
) {
super(
scope,
id,
userId,
member,
participant$,
encryptionSystem,
livekitRoom$,
focusUrl$,
livekitRoom,
focusURL,
displayName$,
mxcAvatarUrl$,
handRaised$,
reaction$,
);
@@ -657,27 +650,25 @@ export class RemoteUserMediaViewModel extends BaseUserMediaViewModel {
public constructor(
scope: ObservableScope,
id: string,
userId: string,
participant$: Observable<RemoteParticipant | null>,
member: RoomMember,
participant$: Observable<RemoteParticipant | undefined>,
encryptionSystem: EncryptionSystem,
livekitRoom$: Behavior<LivekitRoom | undefined>,
focusUrl$: Behavior<string | undefined>,
livekitRoom: LivekitRoom,
focusUrl: string,
private readonly pretendToBeDisconnected$: Behavior<boolean>,
displayName$: Behavior<string>,
mxcAvatarUrl$: Behavior<string | undefined>,
displayname$: Behavior<string>,
handRaised$: Behavior<Date | null>,
reaction$: Behavior<ReactionOption | null>,
) {
super(
scope,
id,
userId,
member,
participant$,
encryptionSystem,
livekitRoom$,
focusUrl$,
displayName$,
mxcAvatarUrl$,
livekitRoom,
focusUrl,
displayname$,
handRaised$,
reaction$,
);
@@ -758,28 +749,26 @@ export class ScreenShareViewModel extends BaseMediaViewModel {
public constructor(
scope: ObservableScope,
id: string,
userId: string,
member: RoomMember,
participant$: Observable<LocalParticipant | RemoteParticipant>,
encryptionSystem: EncryptionSystem,
livekitRoom$: Behavior<LivekitRoom | undefined>,
focusUrl$: Behavior<string | undefined>,
livekitRoom: LivekitRoom,
focusUrl: string,
private readonly pretendToBeDisconnected$: Behavior<boolean>,
displayName$: Behavior<string>,
mxcAvatarUrl$: Behavior<string | undefined>,
displayname$: Behavior<string>,
public readonly local: boolean,
) {
super(
scope,
id,
userId,
member,
participant$,
encryptionSystem,
Track.Source.ScreenShareAudio,
Track.Source.ScreenShare,
livekitRoom$,
focusUrl$,
displayName$,
mxcAvatarUrl$,
livekitRoom,
focusUrl,
displayname$,
);
}
}

View File

@@ -1,239 +0,0 @@
/*
Copyright 2025 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { BehaviorSubject, combineLatest, Subject } from "rxjs";
import { logger } from "matrix-js-sdk/lib/logger";
import { sleep } from "matrix-js-sdk/lib/utils";
import {
Epoch,
mapEpoch,
ObservableScope,
trackEpoch,
} from "./ObservableScope";
import { withTestScheduler } from "../utils/test";
describe("Epoch", () => {
it("should map the value correctly", () => {
const epoch = new Epoch(1);
const mappedEpoch = epoch.mapInner((v) => v + 1);
expect(mappedEpoch.value).toBe(2);
expect(mappedEpoch.epoch).toBe(0);
});
it("should be tracked from an observable", () => {
withTestScheduler(({ expectObservable, behavior }) => {
const observable$ = behavior("abc", {
a: 1,
b: 2,
c: 3,
});
const epochObservable$ = observable$.pipe(trackEpoch());
expectObservable(epochObservable$).toBe("abc", {
a: expect.toSatisfy((e) => e.epoch === 0 && e.value === 1),
b: expect.toSatisfy((e) => e.epoch === 1 && e.value === 2),
c: expect.toSatisfy((e) => e.epoch === 2 && e.value === 3),
});
});
});
it("can be mapped without loosing epoch information", () => {
withTestScheduler(({ expectObservable, behavior }) => {
const observable$ = behavior("abc", {
a: "A",
b: "B",
c: "C",
});
const epochObservable$ = observable$.pipe(trackEpoch());
const derivedEpoch$ = epochObservable$.pipe(
mapEpoch((e) => e + "-mapped"),
);
expectObservable(derivedEpoch$).toBe("abc", {
a: new Epoch("A-mapped", 0),
b: new Epoch("B-mapped", 1),
c: new Epoch("C-mapped", 2),
});
});
});
it("diamonds emits in a predictable order", () => {
const sb$ = new BehaviorSubject("initial");
const root$ = sb$.pipe(trackEpoch());
const derivedA$ = root$.pipe(mapEpoch((e) => e + "-A"));
const derivedB$ = root$.pipe(mapEpoch((e) => e + "-B"));
combineLatest([root$, derivedB$, derivedA$]).subscribe(
([root, derivedA, derivedB]) => {
logger.log(
"combined" +
root.epoch +
root.value +
"\n" +
derivedA.epoch +
derivedA.value +
"\n" +
derivedB.epoch +
derivedB.value,
);
},
);
sb$.next("updated");
sb$.next("ANOTERUPDATE");
});
it("behavior test", () => {
const scope = new ObservableScope();
const s$ = new Subject();
const behavior$ = scope.behavior(s$, 0);
behavior$.subscribe((value) => {
logger.log(`Received value: ${value}`);
});
s$.next(1);
s$.next(2);
s$.next(3);
s$.next(3);
s$.next(3);
s$.next(3);
s$.next(3);
s$.complete();
});
});
describe("Reconcile", () => {
beforeEach(() => {
vi.useFakeTimers();
});
afterEach(() => {
vi.useRealTimers();
});
it("should wait clean up before processing next", async () => {
vi.useFakeTimers();
const scope = new ObservableScope();
const behavior$ = new BehaviorSubject<number>(0);
const setup = vi.fn().mockImplementation(async () => await sleep(100));
const cleanup = vi
.fn()
.mockImplementation(async (n: number) => await sleep(100));
scope.reconcile(behavior$, async (value) => {
await setup();
return async (): Promise<void> => {
await cleanup(value);
};
});
// Let the initial setup process
await vi.advanceTimersByTimeAsync(120);
expect(setup).toHaveBeenCalledTimes(1);
expect(cleanup).toHaveBeenCalledTimes(0);
// Send next value
behavior$.next(1);
await vi.advanceTimersByTimeAsync(50);
// Should not have started setup for 1 yet
expect(setup).toHaveBeenCalledTimes(1);
expect(cleanup).toHaveBeenCalledTimes(1);
expect(cleanup).toHaveBeenCalledWith(0);
// Let cleanup finish
await vi.advanceTimersByTimeAsync(50);
// Now setup for 1 should have started
expect(setup).toHaveBeenCalledTimes(2);
});
it("should skip intermediates values that are not setup", async () => {
vi.useFakeTimers();
const scope = new ObservableScope();
const behavior$ = new BehaviorSubject<number>(0);
const setup = vi
.fn()
.mockImplementation(async (n: number) => await sleep(100));
const cleanupLock = Promise.withResolvers();
const cleanup = vi
.fn()
.mockImplementation(async (n: number) => await cleanupLock.promise);
scope.reconcile(behavior$, async (value) => {
await setup(value);
return async (): Promise<void> => {
await cleanup(value);
};
});
// Let the initial setup process (0)
await vi.advanceTimersByTimeAsync(120);
// Send 4 next values quickly
behavior$.next(1);
behavior$.next(2);
behavior$.next(3);
behavior$.next(4);
await vi.advanceTimersByTimeAsync(3000);
// should have only called cleanup for 0
expect(cleanup).toHaveBeenCalledTimes(1);
expect(cleanup).toHaveBeenCalledWith(0);
// Let cleanup finish
cleanupLock.resolve(undefined);
await vi.advanceTimersByTimeAsync(120);
// Now setup for 4 should have started, skipping 1,2,3
expect(setup).toHaveBeenCalledTimes(2);
expect(setup).toHaveBeenCalledWith(4);
expect(setup).not.toHaveBeenCalledWith(1);
expect(setup).not.toHaveBeenCalledWith(2);
expect(setup).not.toHaveBeenCalledWith(3);
});
it("should wait for setup to complete before starting cleanup", async () => {
vi.useFakeTimers();
const scope = new ObservableScope();
const behavior$ = new BehaviorSubject<number>(0);
const setup = vi
.fn()
.mockImplementation(async (n: number) => await sleep(3000));
const cleanupLock = Promise.withResolvers();
const cleanup = vi
.fn()
.mockImplementation(async (n: number) => await cleanupLock.promise);
scope.reconcile(behavior$, async (value) => {
await setup(value);
return async (): Promise<void> => {
await cleanup(value);
};
});
await vi.advanceTimersByTimeAsync(500);
// Setup for 0 should be in progress
expect(setup).toHaveBeenCalledTimes(1);
behavior$.next(1);
await vi.advanceTimersByTimeAsync(500);
// Should not have started setup for 1 yet
expect(setup).not.toHaveBeenCalledWith(1);
// Should not have called cleanup yet, because the setup for 0 is not done
expect(cleanup).toHaveBeenCalledTimes(0);
// Let setup for 0 finish
await vi.advanceTimersByTimeAsync(2500 + 100);
// Now cleanup for 0 should have started
expect(cleanup).toHaveBeenCalledTimes(1);
expect(cleanup).toHaveBeenCalledWith(0);
cleanupLock.resolve(undefined);
await vi.advanceTimersByTimeAsync(100);
// Now setup for 1 should have started
expect(setup).toHaveBeenCalledWith(1);
});
});

View File

@@ -12,9 +12,7 @@ import {
EMPTY,
endWith,
filter,
map,
type Observable,
type OperatorFunction,
share,
take,
takeUntil,
@@ -24,10 +22,6 @@ import { type Behavior } from "./Behavior";
type MonoTypeOperator = <T>(o: Observable<T>) => Observable<T>;
type SplitBehavior<T> = keyof T extends string | number
? { [K in keyof T as `${K}$`]: Behavior<T[K]> }
: never;
const nothing = Symbol("nothing");
/**
@@ -123,22 +117,8 @@ export class ObservableScope {
callback: (value: T) => Promise<(() => Promise<void>) | void>,
): void {
let latestValue: T | typeof nothing = nothing;
let reconcilePromise: Promise<void> | undefined = undefined;
let reconciledValue: T | typeof nothing = nothing;
let cleanUp: (() => Promise<void>) | void = undefined;
let prevVal: T | typeof nothing = nothing;
// While this loop runs it will process the latest from `value$` until it caught up with the updates.
// It might skip updates from `value$` and only process the newest value after callback has resolved.
const reconcileLoop = async (): Promise<void> => {
while (latestValue !== prevVal) {
await cleanUp?.(); // Call the previous value's clean-up handler
prevVal = latestValue;
if (latestValue !== nothing) cleanUp = await callback(latestValue); // Sync current value...
// `latestValue` might have gotten updated during the `await callback`. That is why we loop here
}
};
value$
.pipe(
catchError(() => EMPTY), // Ignore errors
@@ -146,143 +126,28 @@ export class ObservableScope {
endWith(nothing), // Clean up when the scope ends
)
.subscribe((value) => {
// Always track the latest value! The `reconcileLoop` will run until it "processed" the "last" `latestValue`.
latestValue = value;
// There's already an instance of the below 'reconcileLoop' loop running
// concurrently. So lets let the loop handle it. NEVER instanciate two `reconcileLoop`s.
if (reconcilePromise) return;
reconcilePromise = reconcileLoop().finally(() => {
reconcilePromise = undefined;
});
void (async (): Promise<void> => {
if (latestValue === nothing) {
latestValue = value;
while (latestValue !== reconciledValue) {
await cleanUp?.(); // Call the previous value's clean-up handler
reconciledValue = latestValue;
if (latestValue !== nothing)
cleanUp = await callback(latestValue); // Sync current value
}
// Reset to signal that reconciliation is done for now
latestValue = nothing;
} else {
// There's already an instance of the above 'while' loop running
// concurrently. Just update the latest value and let it be handled.
latestValue = value;
}
})();
});
}
/**
* Splits a Behavior of objects with static properties into an object with
* Behavior properties.
*
* For example, splitting a `Behavior<{ name: string; age: number }>` results
* in an object of type `{ name$: Behavior<string>; age$: Behavior<number> }`.
*/
public splitBehavior<T extends object>(
input$: Behavior<T>,
): SplitBehavior<T> {
return Object.fromEntries(
Object.keys(input$.value).map((key) => [
`${key}$`,
this.behavior(input$.pipe(map((input) => input[key as keyof T]))),
]),
) as SplitBehavior<T>;
}
}
/**
* The global scope, a scope which never ends.
*/
export const globalScope = new ObservableScope();
/**
* `Epoch`'s can be used to create `Behavior`s and `Observable`s which derivitives can be merged
* with `combinedLatest` without duplicated emissions.
*
* This is useful in the following example:
* ```
* const rootObs$ = of("red","green","blue");
* const derivedObs$ = rootObs$.pipe(
* map((v)=> {red:"fire", green:"grass", blue:"water"}[v])
* );
* const otherDerivedObs$ = rootObs$.pipe(
* map((v)=> {red:"tomatoes", green:"leaves", blue:"sky"}[v])
* );
* const mergedObs$ = combineLatest([rootObs$, derivedObs$, otherDerivedObs$]).pipe(
* map(([color, a,b]) => color + " like " + a + " and " + b)
* );
*
* ```
* will result in 6 emissions with mismatching items like "red like fire and leaves"
*
* # Use Epoch
* ```
* const ancestorObs$ = of(1,2,3).pipe(trackEpoch());
* const derivedObs$ = ancestorObs$.pipe(
* mapEpoch((v)=> "this number: " + v)
* );
* const otherDerivedObs$ = ancestorObs$.pipe(
* mapEpoch((v)=> "multiplied by: " + v)
* );
* const mergedObs$ = combineLatest([derivedObs$, otherDerivedObs$]).pipe(
* filter((values) => values.every((v) => v.epoch === values[0].v)),
* map(([color, a, b]) => color + " like " + a + " and " + b)
* );
*
* ```
* will result in 3 emissions all matching (e.g. "blue like water and sky")
*/
export class Epoch<T> {
public readonly epoch: number;
public readonly value: T;
public constructor(value: T, epoch?: number) {
this.value = value;
this.epoch = epoch ?? 0;
}
/**
* Maps the value inside the epoch to a new value while keeping the epoch number.
* # usage
* ```
* const myEpoch$ = myObservable$.pipe(
* map(trackEpoch()),
* // this is the preferred way using mapEpoch
* mapEpoch((v)=> v+1)
* // This is how inner map can be used:
* map((epoch) => epoch.innerMap((v)=> v+1))
* // It is equivalent to:
* map((epoch) => new Epoch(epoch.value + 1, epoch.epoch))
* )
* ```
* See also `Epoch<T>`
*/
public mapInner<U>(map: (value: T) => U): Epoch<U> {
return new Epoch<U>(map(this.value), this.epoch);
}
}
/**
* A `pipe` compatible map oparator that keeps the epoch in tact but allows mapping the value.
* # usage
* ```
* const myEpoch$ = myObservable$.pipe(
* map(trackEpoch()),
* // this is the preferred way using mapEpoch
* mapEpoch((v)=> v+1)
* // This is how inner map can be used:
* map((epoch) => epoch.innerMap((v)=> v+1))
* // It is equivalent to:
* map((epoch) => new Epoch(epoch.value + 1, epoch.epoch))
* )
* ```
* See also `Epoch<T>`
*/
export function mapEpoch<T, U>(
mapFn: (value: T) => U,
): OperatorFunction<Epoch<T>, Epoch<U>> {
return map((e) => e.mapInner(mapFn));
}
/**
* # usage
* ```
* const myEpoch$ = myObservable$.pipe(
* map(trackEpoch()),
* map((epoch) => epoch.innerMap((v)=> v+1))
* )
* const derived = myEpoch$.pipe(
* mapEpoch((v)=>v^2)
* )
* ```
* See also `Epoch<T>`
*/
export function trackEpoch<T>(): OperatorFunction<T, Epoch<T>> {
return map<T, Epoch<T>>((value, number) => new Epoch(value, number));
}

View File

@@ -4,7 +4,7 @@ Copyright 2025 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { of } from "rxjs";
import { of, type Observable } from "rxjs";
import {
type LocalParticipant,
type RemoteParticipant,
@@ -13,6 +13,7 @@ import {
import { type ObservableScope } from "./ObservableScope.ts";
import { ScreenShareViewModel } from "./MediaViewModel.ts";
import type { RoomMember } from "matrix-js-sdk";
import type { EncryptionSystem } from "../e2ee/sharedKeyManagement.ts";
import type { Behavior } from "./Behavior.ts";
@@ -27,26 +28,24 @@ export class ScreenShare {
public constructor(
private readonly scope: ObservableScope,
id: string,
userId: string,
member: RoomMember,
participant: LocalParticipant | RemoteParticipant,
encryptionSystem: EncryptionSystem,
livekitRoom$: Behavior<LivekitRoom | undefined>,
focusUrl$: Behavior<string | undefined>,
livekitRoom: LivekitRoom,
focusUrl: string,
pretendToBeDisconnected$: Behavior<boolean>,
displayName$: Behavior<string>,
mxcAvatarUrl$: Behavior<string | undefined>,
displayName$: Observable<string>,
) {
this.vm = new ScreenShareViewModel(
this.scope,
id,
userId,
member,
of(participant),
encryptionSystem,
livekitRoom$,
focusUrl$,
livekitRoom,
focusUrl,
pretendToBeDisconnected$,
displayName$,
mxcAvatarUrl$,
this.scope.behavior(displayName$),
participant.isLocal,
);
}

View File

@@ -1,5 +1,5 @@
/*
Copyright 2025 Element Creations Ltd.
Copyright 2025 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
@@ -12,25 +12,33 @@ import {
type MatrixRTCSession,
MatrixRTCSessionEvent,
} from "matrix-js-sdk/lib/matrixrtc";
import { fromEvent } from "rxjs";
import { fromEvent, map } from "rxjs";
import {
Epoch,
mapEpoch,
trackEpoch,
type ObservableScope,
} from "./ObservableScope";
import { type ObservableScope } from "./ObservableScope";
import { type Behavior } from "./Behavior";
export const membershipsAndTransports$ = (
scope: ObservableScope,
memberships$: Behavior<Epoch<CallMembership[]>>,
): {
interface Props {
scope: ObservableScope;
matrixRTCSession: MatrixRTCSession;
}
export const createSessionMembershipsAndTransports$ = ({
scope,
matrixRTCSession,
}: Props): {
memberships$: Behavior<CallMembership[]>;
membershipsWithTransport$: Behavior<
Epoch<{ membership: CallMembership; transport?: LivekitTransport }[]>
{ membership: CallMembership; transport?: LivekitTransport }[]
>;
transports$: Behavior<Epoch<LivekitTransport[]>>;
transports$: Behavior<LivekitTransport[]>;
} => {
const memberships$ = scope.behavior(
fromEvent(
matrixRTCSession,
MatrixRTCSessionEvent.MembershipsChanged,
(_, memberships: CallMembership[]) => memberships,
),
);
/**
* Lists the transports used by ourselves, plus all other MatrixRTC session
* members. For completeness this also lists the preferred transport and
@@ -39,9 +47,11 @@ export const membershipsAndTransports$ = (
* together when it might change together is what you have to do in RxJS to
* avoid reading inconsistent state or observing too many changes.)
*/
const membershipsWithTransport$ = scope.behavior(
const membershipsWithTransport$: Behavior<
{ membership: CallMembership; transport?: LivekitTransport }[]
> = scope.behavior(
memberships$.pipe(
mapEpoch((memberships) => {
map((memberships) => {
return memberships.map((membership) => {
const oldestMembership = memberships[0] ?? membership;
const transport = membership.getTransport(oldestMembership);
@@ -56,26 +66,12 @@ export const membershipsAndTransports$ = (
const transports$ = scope.behavior(
membershipsWithTransport$.pipe(
mapEpoch((mts) => mts.flatMap(({ transport: t }) => (t ? [t] : []))),
map((mts) => mts.flatMap(({ transport: t }) => (t ? [t] : []))),
),
);
return {
memberships$,
membershipsWithTransport$,
transports$,
};
};
export const createMemberships$ = (
scope: ObservableScope,
matrixRTCSession: MatrixRTCSession,
): Behavior<Epoch<CallMembership[]>> => {
return scope.behavior(
fromEvent(
matrixRTCSession,
MatrixRTCSessionEvent.MembershipsChanged,
(_, memberships: CallMembership[]) => memberships,
).pipe(trackEpoch()),
new Epoch(matrixRTCSession.memberships),
);
};

View File

@@ -14,7 +14,7 @@ import { fillGaps } from "../utils/iter";
import { debugTileLayout } from "../settings/settings";
function debugEntries(entries: GridTileData[]): string[] {
return entries.map((e) => e.media.displayName$.value);
return entries.map((e) => e.media.member?.rawDisplayName ?? "[👻]");
}
let DEBUG_ENABLED = false;
@@ -156,7 +156,7 @@ export class TileStoreBuilder {
public registerSpotlight(media: MediaViewModel[], maximised: boolean): void {
if (DEBUG_ENABLED)
logger.debug(
`[TileStore, ${this.generation}] register spotlight: ${media.map((m) => m.displayName$.value)}`,
`[TileStore, ${this.generation}] register spotlight: ${media.map((m) => m.member?.rawDisplayName ?? "[👻]")}`,
);
if (this.spotlight !== null) throw new Error("Spotlight already set");
@@ -180,7 +180,7 @@ export class TileStoreBuilder {
public registerGridTile(media: UserMediaViewModel): void {
if (DEBUG_ENABLED)
logger.debug(
`[TileStore, ${this.generation}] register grid tile: ${media.displayName$.value}`,
`[TileStore, ${this.generation}] register grid tile: ${media.member?.rawDisplayName ?? "[👻]"}`,
);
if (this.spotlight !== null) {
@@ -263,7 +263,7 @@ export class TileStoreBuilder {
public registerPipTile(media: UserMediaViewModel): void {
if (DEBUG_ENABLED)
logger.debug(
`[TileStore, ${this.generation}] register PiP tile: ${media.displayName$.value}`,
`[TileStore, ${this.generation}] register PiP tile: ${media.member?.rawDisplayName ?? "[👻]"}`,
);
// If there is a single grid tile that we can reuse

View File

@@ -5,9 +5,17 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { combineLatest, map, type Observable, of, switchMap } from "rxjs";
import {
BehaviorSubject,
combineLatest,
map,
type Observable,
of,
switchMap,
} from "rxjs";
import {
type LocalParticipant,
type Participant,
ParticipantEvent,
type RemoteParticipant,
type Room as LivekitRoom,
@@ -21,12 +29,11 @@ import {
type UserMediaViewModel,
} from "./MediaViewModel.ts";
import type { Behavior } from "./Behavior.ts";
import type { RoomMember } from "matrix-js-sdk";
import type { EncryptionSystem } from "../e2ee/sharedKeyManagement.ts";
import type { MediaDevices } from "./MediaDevices.ts";
import type { ReactionOption } from "../reactions";
import { observeSpeaker$ } from "./observeSpeaker.ts";
import { generateItems } from "../utils/observable.ts";
import { ScreenShare } from "./ScreenShare.ts";
/**
* Sorting bins defining the order in which media tiles appear in the layout.
@@ -65,35 +72,35 @@ enum SortingBin {
/**
* A user media item to be presented in a tile. This is a thin wrapper around
* UserMediaViewModel which additionally determines the media item's sorting bin
* for inclusion in the call layout and tracks associated screen shares.
* for inclusion in the call layout.
*/
export class UserMedia {
private readonly participant$ = new BehaviorSubject(this.initialParticipant);
public readonly vm: UserMediaViewModel = this.participant$.value?.isLocal
? new LocalUserMediaViewModel(
this.scope,
this.id,
this.userId,
this.participant$ as Behavior<LocalParticipant | null>,
this.member,
this.participant$ as Behavior<LocalParticipant>,
this.encryptionSystem,
this.livekitRoom$,
this.focusUrl$,
this.livekitRoom,
this.focusURL,
this.mediaDevices,
this.displayName$,
this.mxcAvatarUrl$,
this.scope.behavior(this.displayname$),
this.scope.behavior(this.handRaised$),
this.scope.behavior(this.reaction$),
)
: new RemoteUserMediaViewModel(
this.scope,
this.id,
this.userId,
this.participant$ as Behavior<RemoteParticipant | null>,
this.member,
this.participant$ as Observable<RemoteParticipant | undefined>,
this.encryptionSystem,
this.livekitRoom$,
this.focusUrl$,
this.livekitRoom,
this.focusURL,
this.pretendToBeDisconnected$,
this.displayName$,
this.mxcAvatarUrl$,
this.scope.behavior(this.displayname$),
this.scope.behavior(this.handRaised$),
this.scope.behavior(this.reaction$),
);
@@ -102,53 +109,10 @@ export class UserMedia {
observeSpeaker$(this.vm.speaking$),
);
/**
* All screen share media associated with this user media.
*/
public readonly screenShares$ = this.scope.behavior(
this.participant$.pipe(
switchMap((p) =>
p === null
? of([])
: observeParticipantEvents(
p,
ParticipantEvent.TrackPublished,
ParticipantEvent.TrackUnpublished,
ParticipantEvent.LocalTrackPublished,
ParticipantEvent.LocalTrackUnpublished,
).pipe(
// Technically more than one screen share might be possible... our
// MediaViewModels don't support it though since they look for a unique
// track for the given source. So generateItems here is a bit overkill.
generateItems(
function* (p) {
if (p.isScreenShareEnabled)
yield {
keys: ["screen-share"],
data: undefined,
};
},
(scope, _data$, key) =>
new ScreenShare(
scope,
`${this.id}:${key}`,
this.userId,
p,
this.encryptionSystem,
this.livekitRoom$,
this.focusUrl$,
this.pretendToBeDisconnected$,
this.displayName$,
this.mxcAvatarUrl$,
),
),
),
),
),
);
private readonly presenter$ = this.scope.behavior(
this.screenShares$.pipe(map((screenShares) => screenShares.length > 0)),
this.participant$.pipe(
switchMap((p) => (p === undefined ? of(false) : sharingScreen$(p))),
),
);
/**
@@ -183,18 +147,37 @@ export class UserMedia {
public constructor(
private readonly scope: ObservableScope,
public readonly id: string,
private readonly userId: string,
private readonly participant$: Behavior<
LocalParticipant | RemoteParticipant | null
>,
private readonly member: RoomMember,
private readonly initialParticipant:
| LocalParticipant
| RemoteParticipant
| undefined,
private readonly encryptionSystem: EncryptionSystem,
private readonly livekitRoom$: Behavior<LivekitRoom | undefined>,
private readonly focusUrl$: Behavior<string | undefined>,
private readonly livekitRoom: LivekitRoom,
private readonly focusURL: string,
private readonly mediaDevices: MediaDevices,
private readonly pretendToBeDisconnected$: Behavior<boolean>,
private readonly displayName$: Behavior<string>,
private readonly mxcAvatarUrl$: Behavior<string | undefined>,
private readonly displayname$: Observable<string>,
private readonly handRaised$: Observable<Date | null>,
private readonly reaction$: Observable<ReactionOption | null>,
) {}
public updateParticipant(
newParticipant: LocalParticipant | RemoteParticipant | undefined,
): void {
if (this.participant$.value !== newParticipant) {
// Update the BehaviourSubject in the UserMedia.
this.participant$.next(newParticipant);
}
}
}
export function sharingScreen$(p: Participant): Observable<boolean> {
return observeParticipantEvents(
p,
ParticipantEvent.TrackPublished,
ParticipantEvent.TrackUnpublished,
ParticipantEvent.LocalTrackPublished,
ParticipantEvent.LocalTrackUnpublished,
).pipe(map((p) => p.isScreenShareEnabled));
}

View File

@@ -0,0 +1,422 @@
/*
Copyright 2025 New Vector Ltd.
SPDX-License-IdFentifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type LocalTrack, type E2EEOptions } from "livekit-client";
import {
type LivekitTransport,
type MatrixRTCSession,
MembershipManagerEvent,
Status,
} from "matrix-js-sdk/lib/matrixrtc";
import { ClientEvent, SyncState, type Room as MatrixRoom } from "matrix-js-sdk";
import {
BehaviorSubject,
combineLatest,
fromEvent,
map,
type Observable,
of,
scan,
startWith,
switchMap,
tap,
} from "rxjs";
import { logger } from "matrix-js-sdk/lib/logger";
import { type Behavior } from "../Behavior";
import { type createConnectionManager$ } from "../remoteMembers/ConnectionManager";
import { ObservableScope } from "../ObservableScope";
import { Publisher } from "./Publisher";
import { type MuteStates } from "../MuteStates";
import { type ProcessorState } from "../../livekit/TrackProcessorContext";
import { type MediaDevices } from "../MediaDevices";
import { and$ } from "../../utils/observable";
import { areLivekitTransportsEqual } from "../remoteMembers/matrixLivekitMerger";
import {
enterRTCSession,
type EnterRTCSessionOptions,
} from "../../rtcSessionHelpers";
import { type ElementCallError } from "../../utils/errors";
import { ElementWidgetActions, type WidgetHelpers } from "../../widget";
enum LivekitState {
UNINITIALIZED = "uninitialized",
CONNECTING = "connecting",
CONNECTED = "connected",
ERROR = "error",
DISCONNECTED = "disconnected",
DISCONNECTING = "disconnecting",
}
type LocalMemberLivekitState =
| { state: LivekitState.ERROR; error: string }
| { state: LivekitState.CONNECTED }
| { state: LivekitState.CONNECTING }
| { state: LivekitState.UNINITIALIZED }
| { state: LivekitState.DISCONNECTED }
| { state: LivekitState.DISCONNECTING };
enum MatrixState {
CONNECTED = "connected",
DISCONNECTED = "disconnected",
CONNECTING = "connecting",
}
type LocalMemberMatrixState =
| { state: MatrixState.CONNECTED }
| { state: MatrixState.CONNECTING }
| { state: MatrixState.DISCONNECTED };
export interface LocalMemberState {
livekit$: BehaviorSubject<LocalMemberLivekitState>;
matrix$: BehaviorSubject<LocalMemberMatrixState>;
}
/*
* - get well known
* - get oldest membership
* - get transport to use
* - get openId + jwt token
* - wait for createTrack() call
* - create tracks
* - wait for join() call
* - Publisher.publishTracks()
* - send join state/sticky event
*/
interface Props {
options: Behavior<EnterRTCSessionOptions>;
scope: ObservableScope;
mediaDevices: MediaDevices;
muteStates: MuteStates;
connectionManager: ReturnType<typeof createConnectionManager$>;
matrixRTCSession: MatrixRTCSession;
matrixRoom: MatrixRoom;
localTransport$: Behavior<LivekitTransport | undefined>;
e2eeLivekitOptions: E2EEOptions | undefined;
trackProcessorState$: Behavior<ProcessorState>;
widget: WidgetHelpers | null;
}
/**
* This class is responsible for managing the own membership in a room.
* We want
* - a publisher
* -
* @param param0
* @returns
* - publisher: The handle to create tracks and publish them to the room.
* - connected$: the current connection state. Including matrix server and livekit server connection. (only the livekit server relevant for our own participation)
* - transport$: the transport object the ownMembership$ ended up using.
*
*/
export const createLocalMembership$ = ({
scope,
options,
muteStates,
mediaDevices,
connectionManager,
matrixRTCSession,
localTransport$,
matrixRoom,
e2eeLivekitOptions,
trackProcessorState$,
widget,
}: Props): {
// publisher: Publisher
requestConnect: () => LocalMemberState;
startTracks: () => Behavior<LocalTrack[]>;
requestDisconnect: () => Observable<LocalMemberLivekitState> | null;
state: LocalMemberState; // TODO this is probably superseeded by joinState$
homeserverConnected$: Behavior<boolean>;
connected$: Behavior<boolean>;
reconnecting$: Behavior<boolean>;
configError$: Behavior<ElementCallError | null>;
} => {
const state = {
livekit$: new BehaviorSubject<LocalMemberLivekitState>({
state: LivekitState.UNINITIALIZED,
}),
matrix$: new BehaviorSubject<LocalMemberMatrixState>({
state: MatrixState.DISCONNECTED,
}),
};
// This should be used in a combineLatest with publisher$ to connect.
// to make it possible to call startTracks before the preferredTransport$ has resolved.
const shouldStartTracks$ = new BehaviorSubject(false);
// This should be used in a combineLatest with publisher$ to connect.
const tracks$ = new BehaviorSubject<LocalTrack[]>([]);
const connection$ = scope.behavior(
combineLatest(
[connectionManager.connections$, localTransport$],
(connections, transport) => {
if (transport === undefined) return undefined;
return connections.find((connection) =>
areLivekitTransportsEqual(connection.transport, transport),
);
},
),
);
/**
* Whether we are connected to the MatrixRTC session.
*/
const homeserverConnected$ = scope.behavior(
// To consider ourselves connected to MatrixRTC, we check the following:
and$(
// The client is connected to the sync loop
(
fromEvent(matrixRoom.client, ClientEvent.Sync) as Observable<
[SyncState]
>
).pipe(
startWith([matrixRoom.client.getSyncState()]),
map(([state]) => state === SyncState.Syncing),
),
// Room state observed by session says we're connected
fromEvent(matrixRTCSession, MembershipManagerEvent.StatusChanged).pipe(
startWith(null),
map(() => matrixRTCSession.membershipStatus === Status.Connected),
),
// Also watch out for warnings that we've likely hit a timeout and our
// delayed leave event is being sent (this condition is here because it
// provides an earlier warning than the sync loop timeout, and we wouldn't
// see the actual leave event until we reconnect to the sync loop)
fromEvent(matrixRTCSession, MembershipManagerEvent.ProbablyLeft).pipe(
startWith(null),
map(() => matrixRTCSession.probablyLeft !== true),
),
),
);
// /**
// * Whether we are "fully" connected to the call. Accounts for both the
// * connection to the MatrixRTC session and the LiveKit publish connection.
// */
// // TODO use this in combination with the MemberState.
const connected$ = scope.behavior(
and$(
homeserverConnected$,
connection$.pipe(
switchMap((c) =>
c
? c.state$.pipe(map((state) => state.state === "ConnectedToLkRoom"))
: of(false),
),
),
),
);
const publisher$ = scope.behavior(
connection$.pipe(
map((connection) =>
connection
? new Publisher(
scope,
connection,
mediaDevices,
muteStates,
e2eeLivekitOptions,
trackProcessorState$,
)
: null,
),
),
);
combineLatest(
[publisher$, shouldStartTracks$],
(publisher, shouldStartTracks) => {
if (publisher && shouldStartTracks) {
publisher
.createAndSetupTracks()
.then((tracks) => {
tracks$.next(tracks);
})
.catch((error) => {
logger.error("Error creating tracks:", error);
});
}
},
);
// MATRIX RELATED
// /**
// * Whether we should tell the user that we're reconnecting to the call.
// */
// DISCUSSION is there a better way to do this?
// sth that is more deriectly implied from the membership manager of the js sdk. (fromEvent(matrixRTCSession, Reconnecting)) ??? or similar
const reconnecting$ = scope.behavior(
connected$.pipe(
// We are reconnecting if we previously had some successful initial
// connection but are now disconnected
scan(
({ connectedPreviously }, connectedNow) => ({
connectedPreviously: connectedPreviously || connectedNow,
reconnecting: connectedPreviously && !connectedNow,
}),
{ connectedPreviously: false, reconnecting: false },
),
map(({ reconnecting }) => reconnecting),
),
);
const startTracks = (): Behavior<LocalTrack[]> => {
shouldStartTracks$.next(true);
return tracks$;
};
const requestConnect = (): LocalMemberState => {
if (state.livekit$.value === null) {
startTracks();
state.livekit$.next({ state: LivekitState.CONNECTING });
combineLatest([publisher$, tracks$], (publisher, tracks) => {
publisher
?.startPublishing()
.then(() => {
state.livekit$.next({ state: LivekitState.CONNECTED });
})
.catch((error) => {
state.livekit$.next({ state: LivekitState.ERROR, error });
});
});
}
if (state.matrix$.value.state !== MatrixState.DISCONNECTED) {
state.matrix$.next({ state: MatrixState.CONNECTING });
localTransport$.pipe(
tap((transport) => {
if (transport !== undefined) {
enterRTCSession(matrixRTCSession, transport, options.value).catch(
(error) => {
logger.error(error);
},
);
} else {
logger.info("Waiting for transport to enter rtc session");
}
}),
);
}
return state;
};
const requestDisconnect = (): Behavior<LocalMemberLivekitState> | null => {
if (state.livekit$.value.state !== LivekitState.CONNECTED) return null;
state.livekit$.next({ state: LivekitState.DISCONNECTING });
combineLatest([publisher$, tracks$], (publisher, tracks) => {
publisher
?.stopPublishing()
.then(() => {
tracks.forEach((track) => track.stop());
state.livekit$.next({ state: LivekitState.DISCONNECTED });
})
.catch((error) => {
state.livekit$.next({ state: LivekitState.ERROR, error });
});
});
return state.livekit$;
};
// Pause upstream of all local media tracks when we're disconnected from
// MatrixRTC, because it can be an unpleasant surprise for the app to say
// 'reconnecting' and yet still be transmitting your media to others.
// We use matrixConnected$ rather than reconnecting$ because we want to
// pause tracks during the initial joining sequence too until we're sure
// that our own media is displayed on screen.
combineLatest([connection$, homeserverConnected$])
.pipe(scope.bind())
.subscribe(([connection, connected]) => {
if (connection?.state$.value.state !== "ConnectedToLkRoom") return;
const publications =
connection.livekitRoom.localParticipant.trackPublications.values();
if (connected) {
for (const p of publications) {
if (p.track?.isUpstreamPaused === true) {
const kind = p.track.kind;
logger.log(`Resuming ${kind} track (MatrixRTC connection present)`);
p.track
.resumeUpstream()
.catch((e) =>
logger.error(
`Failed to resume ${kind} track after MatrixRTC reconnection`,
e,
),
);
}
}
} else {
for (const p of publications) {
if (p.track?.isUpstreamPaused === false) {
const kind = p.track.kind;
logger.log(
`Pausing ${kind} track (uncertain MatrixRTC connection)`,
);
p.track
.pauseUpstream()
.catch((e) =>
logger.error(
`Failed to pause ${kind} track after entering uncertain MatrixRTC connection`,
e,
),
);
}
}
}
});
const configError$ = new BehaviorSubject<ElementCallError | null>(null);
// TODO I do not fully understand what this does.
// Is it needed?
// Is this at the right place?
// Can this be simplified?
// Start and stop session membership as needed
scope.reconcile(localTransport$, async (advertised) => {
if (advertised !== null && advertised !== undefined) {
try {
configError$.next(null);
await enterRTCSession(matrixRTCSession, advertised, options.value);
} catch (e) {
logger.error("Error entering RTC session", e);
}
// Update our member event when our mute state changes.
const intentScope = new ObservableScope();
intentScope.reconcile(muteStates.video.enabled$, async (videoEnabled) =>
matrixRTCSession.updateCallIntent(videoEnabled ? "video" : "audio"),
);
return async (): Promise<void> => {
intentScope.end();
// Only sends Matrix leave event. The LiveKit session will disconnect
// as soon as either the stopConnection$ handler above gets to it or
// the view model is destroyed.
try {
await matrixRTCSession.leaveRoomSession();
} catch (e) {
logger.error("Error leaving RTC session", e);
}
try {
await widget?.api.transport.send(ElementWidgetActions.HangupCall, {});
} catch (e) {
logger.error("Failed to send hangup action", e);
}
};
}
});
return {
startTracks,
requestConnect,
requestDisconnect,
state,
homeserverConnected$,
connected$,
reconnecting$,
configError$,
};
};

View File

@@ -0,0 +1,165 @@
/*
Copyright 2025 New Vector Ltd.
SPDX-License-IdFentifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
type CallMembership,
isLivekitTransport,
type LivekitTransportConfig,
type LivekitTransport,
isLivekitTransportConfig,
} from "matrix-js-sdk/lib/matrixrtc";
import { type MatrixClient } from "matrix-js-sdk";
import { combineLatest, distinctUntilChanged, first, from, map } from "rxjs";
import { logger } from "matrix-js-sdk/lib/logger";
import { AutoDiscovery } from "matrix-js-sdk/lib/autodiscovery";
import { deepCompare } from "matrix-js-sdk/lib/utils";
import { type Behavior } from "../Behavior.ts";
import { type ObservableScope } from "../ObservableScope.ts";
import { Config } from "../../config/Config.ts";
import { MatrixRTCTransportMissingError } from "../../utils/errors.ts";
import { getSFUConfigWithOpenID } from "../../livekit/openIDSFU.ts";
/*
* - get well known
* - get oldest membership
* - get transport to use
* - get openId + jwt token
* - wait for createTrack() call
* - create tracks
* - wait for join() call
* - Publisher.publishTracks()
* - send join state/sticky event
*/
interface Props {
scope: ObservableScope;
memberships$: Behavior<CallMembership[]>;
client: MatrixClient;
roomId: string;
useOldestMember$: Behavior<boolean>;
}
/**
* This class is responsible for managing the local transport.
* "Which transport is the local member going to use"
*
* @prop useOldestMember Whether to use the same transport as the oldest member.
* This will only update once the first oldest member appears. Will not recompute if the oldest member leaves.
*/
export const createLocalTransport$ = ({
scope,
memberships$,
client,
roomId,
useOldestMember$,
}: Props): Behavior<LivekitTransport | undefined> => {
/**
* The transport over which we should be actively publishing our media.
* undefined when not joined.
*/
const oldestMemberTransport$ = scope.behavior(
memberships$.pipe(
map((memberships) => memberships[0].getTransport(memberships[0])),
first((t) => t != undefined && isLivekitTransport(t)),
),
undefined,
);
/**
* The transport that we would personally prefer to publish on (if not for the
* transport preferences of others, perhaps).
*/
const preferredTransport$: Behavior<LivekitTransport | undefined> =
scope.behavior(from(makeTransport(client, roomId)), undefined);
/**
* The transport we should advertise in our MatrixRTC membership.
*/
const advertisedTransport$ = scope.behavior(
combineLatest(
[useOldestMember$, oldestMemberTransport$, preferredTransport$],
(useOldestMember, oldestMemberTransport, preferredTransport) =>
useOldestMember ? oldestMemberTransport : preferredTransport,
).pipe<LivekitTransport>(distinctUntilChanged(deepCompare)),
undefined,
);
return advertisedTransport$;
};
const FOCI_WK_KEY = "org.matrix.msc4143.rtc_foci";
async function makeTransportInternal(
client: MatrixClient,
roomId: string,
): Promise<LivekitTransport> {
logger.log("Searching for a preferred transport");
//TODO refactor this to use the jwt service returned alias.
const livekitAlias = roomId;
// TODO-MULTI-SFU: Either remove this dev tool or make it more official
const urlFromStorage =
localStorage.getItem("robin-matrixrtc-auth") ??
localStorage.getItem("timo-focus-url");
if (urlFromStorage !== null) {
const transportFromStorage: LivekitTransport = {
type: "livekit",
livekit_service_url: urlFromStorage,
livekit_alias: livekitAlias,
};
logger.log(
"Using LiveKit transport from local storage: ",
transportFromStorage,
);
return transportFromStorage;
}
// Prioritize the .well-known/matrix/client, if available, over the configured SFU
const domain = client.getDomain();
if (domain) {
// we use AutoDiscovery instead of relying on the MatrixClient having already
// been fully configured and started
const wellKnownFoci = (await AutoDiscovery.getRawClientConfig(domain))?.[
FOCI_WK_KEY
];
if (Array.isArray(wellKnownFoci)) {
const transport: LivekitTransportConfig | undefined = wellKnownFoci.find(
(f) => f && isLivekitTransportConfig(f),
);
if (transport !== undefined) {
logger.log("Using LiveKit transport from .well-known: ", transport);
return { ...transport, livekit_alias: livekitAlias };
}
}
}
const urlFromConf = Config.get().livekit?.livekit_service_url;
if (urlFromConf) {
const transportFromConf: LivekitTransport = {
type: "livekit",
livekit_service_url: urlFromConf,
livekit_alias: livekitAlias,
};
logger.log("Using LiveKit transport from config: ", transportFromConf);
return transportFromConf;
}
throw new MatrixRTCTransportMissingError(domain ?? "");
}
async function makeTransport(
client: MatrixClient,
roomId: string,
): Promise<LivekitTransport> {
const transport = await makeTransportInternal(client, roomId);
// this will call the jwt/sfu/get endpoint to pre create the livekit room.
await getSFUConfigWithOpenID(
client,
transport.livekit_service_url,
transport.livekit_alias,
);
return transport;
}

View File

@@ -1,11 +1,11 @@
/*
Copyright 2025 Element Creations Ltd.
Copyright 2025 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
type E2EEOptions,
LocalVideoTrack,
type Room as LivekitRoom,
Track,
@@ -14,7 +14,6 @@ import {
ConnectionState as LivekitConnectionState,
} from "livekit-client";
import {
BehaviorSubject,
map,
NEVER,
type Observable,
@@ -23,21 +22,17 @@ import {
} from "rxjs";
import { type Logger } from "matrix-js-sdk/lib/logger";
import type { Behavior } from "../../Behavior.ts";
import type { MediaDevices, SelectedDevice } from "../../MediaDevices.ts";
import type { MuteStates } from "../../MuteStates.ts";
import type { Behavior } from "../Behavior.ts";
import type { MediaDevices, SelectedDevice } from "../MediaDevices.ts";
import type { MuteStates } from "../MuteStates.ts";
import {
type ProcessorState,
trackProcessorSync,
} from "../../../livekit/TrackProcessorContext.tsx";
import { getUrlParams } from "../../../UrlParams.ts";
import { observeTrackReference$ } from "../../MediaViewModel.ts";
} from "../../livekit/TrackProcessorContext.tsx";
import { getUrlParams } from "../../UrlParams.ts";
import { observeTrackReference$ } from "../MediaViewModel.ts";
import { type Connection } from "../remoteMembers/Connection.ts";
import { type ObservableScope } from "../../ObservableScope.ts";
import {
ElementCallError,
FailToStartLivekitConnection,
} from "../../../utils/errors.ts";
import { type ObservableScope } from "../ObservableScope.ts";
/**
* A wrapper for a Connection object.
@@ -45,6 +40,7 @@ import {
* The Publisher is also responsible for creating the media tracks.
*/
export class Publisher {
public tracks: LocalTrack<Track.Kind>[] = [];
/**
* Creates a new Publisher.
* @param scope - The observable scope to use for managing the publisher.
@@ -56,19 +52,20 @@ export class Publisher {
*/
public constructor(
private scope: ObservableScope,
private connection: Pick<Connection, "livekitRoom" | "state$">, //setE2EEEnabled,
private connection: Connection,
devices: MediaDevices,
private readonly muteStates: MuteStates,
e2eeLivekitOptions: E2EEOptions | undefined,
trackerProcessorState$: Behavior<ProcessorState>,
private logger: Logger,
private logger?: Logger,
) {
this.logger.info("Create LiveKit room");
this.logger?.info("[PublishConnection] Create LiveKit room");
const { controlledAudioDevices } = getUrlParams();
const room = connection.livekitRoom;
room.setE2EEEnabled(room.options.e2ee !== undefined)?.catch((e: Error) => {
this.logger.error("Failed to set E2EE enabled on room", e);
room.setE2EEEnabled(e2eeLivekitOptions !== undefined)?.catch((e) => {
this.logger?.error("Failed to set E2EE enabled on room", e);
});
// Setup track processor syncing (blur)
@@ -77,32 +74,8 @@ export class Publisher {
this.observeMediaDevices(scope, devices, controlledAudioDevices);
this.workaroundRestartAudioInputTrackChrome(devices, scope);
this.scope.onEnd(() => {
this.logger.info("Scope ended -> stop publishing all tracks");
void this.stopPublishing();
});
// TODO move mute state handling here using reconcile (instead of inside the mute state class)
// this.scope.reconcile(
// this.scope.behavior(
// combineLatest([this.muteStates.video.enabled$, this.tracks$]),
// ),
// async ([videoEnabled, tracks]) => {
// const track = tracks.find((t) => t.kind == Track.Kind.Video);
// if (!track) return;
// if (videoEnabled) {
// await track.unmute();
// } else {
// await track.mute();
// }
// },
// );
}
private _tracks$ = new BehaviorSubject<LocalTrack<Track.Kind>[]>([]);
public tracks$ = this._tracks$ as Behavior<LocalTrack<Track.Kind>[]>;
/**
* Start the connection to LiveKit and publish local tracks.
*
@@ -116,62 +89,22 @@ export class Publisher {
* @throws {InsufficientCapacityError} if the LiveKit server indicates that it has insufficient capacity to accept the connection.
* @throws {SFURoomCreationRestrictedError} if the LiveKit server indicates that the room does not exist and cannot be created.
*/
public async createAndSetupTracks(): Promise<void> {
this.logger.debug("createAndSetupTracks called");
public async createAndSetupTracks(): Promise<LocalTrack[]> {
const lkRoom = this.connection.livekitRoom;
// Observe mute state changes and update LiveKit microphone/camera states accordingly
this.observeMuteStates(this.scope);
// TODO-MULTI-SFU: Prepublish a microphone track
const audio = this.muteStates.audio.enabled$.value;
const video = this.muteStates.video.enabled$.value;
// createTracks throws if called with audio=false and video=false
if (audio || video) {
// TODO this can still throw errors? It will also prompt for permissions if not already granted
return lkRoom.localParticipant
.createTracks({
audio,
video,
})
.then((tracks) => {
this.logger.info(
"created track",
tracks.map((t) => t.kind + ", " + t.id),
);
this._tracks$.next(tracks);
})
.catch((error) => {
this.logger.error("Failed to create tracks", error);
});
}
throw Error("audio and video is false");
}
private _publishing$ = new BehaviorSubject<boolean>(false);
public publishing$ = this.scope.behavior(this._publishing$);
/**
*
* @returns
* @throws ElementCallError
*/
public async startPublishing(): Promise<LocalTrack[]> {
this.logger.debug("startPublishing called");
const lkRoom = this.connection.livekitRoom;
// TODO: This should be an autostarted connection no need to start here. just check the connection state.
// TODO: This will fetch the JWT token. Perhaps we could keep it preloaded
// instead? This optimization would only be safe for a publish connection,
// because we don't want to leak the user's intent to perhaps join a call to
// remote servers before they actually commit to it.
const { promise, resolve, reject } = Promise.withResolvers<void>();
const sub = this.connection.state$.subscribe((s) => {
switch (s.state) {
case "ConnectedToLkRoom":
resolve();
break;
case "FailedToStart":
reject(
s.error instanceof ElementCallError
? s.error
: new FailToStartLivekitConnection(s.error.message),
);
break;
default:
this.logger.info("waiting for connection: ", s.state);
if (s.state !== "FailedToStart") {
reject(new Error("Disconnected from LiveKit server"));
} else {
resolve();
}
});
try {
@@ -181,27 +114,53 @@ export class Publisher {
} finally {
sub.unsubscribe();
}
// TODO-MULTI-SFU: Prepublish a microphone track
const audio = this.muteStates.audio.enabled$.value;
const video = this.muteStates.video.enabled$.value;
// createTracks throws if called with audio=false and video=false
if (audio || video) {
// TODO this can still throw errors? It will also prompt for permissions if not already granted
this.tracks = await lkRoom.localParticipant.createTracks({
audio,
video,
});
}
return this.tracks;
}
for (const track of this.tracks$.value) {
this.logger.info("publish ", this.tracks$.value.length, "tracks");
public async startPublishing(): Promise<LocalTrack[]> {
const lkRoom = this.connection.livekitRoom;
const { promise, resolve, reject } = Promise.withResolvers<void>();
const sub = this.connection.state$.subscribe((s) => {
switch (s.state) {
case "ConnectedToLkRoom":
resolve();
break;
case "FailedToStart":
reject(new Error("Failed to connect to LiveKit server"));
break;
default:
this.logger?.info("waiting for connection: ", s.state);
}
});
try {
await promise;
} catch (e) {
throw e;
} finally {
sub.unsubscribe();
}
for (const track of this.tracks) {
// TODO: handle errors? Needs the signaling connection to be up, but it has some retries internally
// with a timeout.
await lkRoom.localParticipant.publishTrack(track).catch((error) => {
this.logger.error("Failed to publish track", error);
throw new FailToStartLivekitConnection(
error instanceof Error ? error.message : error,
);
});
this.logger.info("published track ", track.kind, track.id);
await lkRoom.localParticipant.publishTrack(track);
// TODO: check if the connection is still active? and break the loop if not?
}
this._publishing$.next(true);
return this.tracks$.value;
return this.tracks;
}
public async stopPublishing(): Promise<void> {
this.logger.debug("stopPublishing called");
// TODO-MULTI-SFU: Move these calls back to ObservableScope.onEnd once scope
// actually has the right lifetime
this.muteStates.audio.unsetHandler();
@@ -213,28 +172,7 @@ export class Publisher {
if (p.track !== undefined) tracks.push(p.track);
};
localParticipant.trackPublications.forEach(addToTracksIfDefined);
this.logger.debug(
"list of tracks to unpublish:",
tracks.map((t) => t.kind + ", " + t.id),
"start unpublishing now",
);
await localParticipant.unpublishTracks(tracks).catch((error) => {
this.logger.error("Failed to unpublish tracks", error);
throw error;
});
this.logger.debug(
"unpublished tracks",
tracks.map((t) => t.kind + ", " + t.id),
);
this._publishing$.next(false);
}
/**
* Stops all tracks that are currently running
*/
public stopTracks(): void {
this.tracks$.value.forEach((t) => t.stop());
this._tracks$.next([]);
await localParticipant.unpublishTracks(tracks);
}
/// Private methods
@@ -271,9 +209,6 @@ export class Publisher {
// the process of being restarted.
activeMicTrack.mediaStreamTrack.readyState !== "ended"
) {
this.logger?.info(
"Restarting audio device track due to active media device changed (workaroundRestartAudioInputTrackChrome)",
);
// Restart the track, which will cause Livekit to do another
// getUserMedia() call with deviceId: default to get the *new* default device.
// Note that room.switchActiveDevice() won't work: Livekit will ignore it because
@@ -282,7 +217,7 @@ export class Publisher {
.getTrackPublication(Track.Source.Microphone)
?.audioTrack?.restartTrack()
.catch((e) => {
this.logger.error(`Failed to restart audio device track`, e);
this.logger?.error(`Failed to restart audio device track`, e);
});
}
});
@@ -302,7 +237,7 @@ export class Publisher {
selected$.pipe(scope.bind()).subscribe((device) => {
if (lkRoom.state != LivekitConnectionState.Connected) return;
// if (this.connectionState$.value !== ConnectionState.Connected) return;
this.logger.info(
this.logger?.info(
"[LivekitRoom] syncDevice room.getActiveDevice(kind) !== d.id :",
lkRoom.getActiveDevice(kind),
" !== ",
@@ -314,8 +249,8 @@ export class Publisher {
) {
lkRoom
.switchActiveDevice(kind, device.id)
.catch((e: Error) =>
this.logger.error(
.catch((e) =>
this.logger?.error(
`Failed to sync ${kind} device with LiveKit`,
e,
),
@@ -340,7 +275,10 @@ export class Publisher {
try {
await lkRoom.localParticipant.setMicrophoneEnabled(desired);
} catch (e) {
this.logger.error("Failed to update LiveKit audio input mute state", e);
this.logger?.error(
"Failed to update LiveKit audio input mute state",
e,
);
}
return lkRoom.localParticipant.isMicrophoneEnabled;
});
@@ -348,7 +286,10 @@ export class Publisher {
try {
await lkRoom.localParticipant.setCameraEnabled(desired);
} catch (e) {
this.logger.error("Failed to update LiveKit video input mute state", e);
this.logger?.error(
"Failed to update LiveKit video input mute state",
e,
);
}
return lkRoom.localParticipant.isCameraEnabled;
});
@@ -366,8 +307,7 @@ export class Publisher {
return track instanceof LocalVideoTrack ? track : null;
}),
),
null,
);
trackProcessorSync(scope, track$, trackerProcessorState$);
trackProcessorSync(track$, trackerProcessorState$);
}
}

View File

@@ -1,5 +1,4 @@
/*
Copyright 2025 Element Creations Ltd.
Copyright 2025 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
@@ -11,32 +10,38 @@ import {
describe,
expect,
it,
type Mock,
type MockedObject,
onTestFinished,
vi,
} from "vitest";
import { BehaviorSubject, of } from "rxjs";
import {
type LocalParticipant,
type RemoteParticipant,
type Room as LivekitRoom,
RoomEvent,
ConnectionState as LivekitConnectionState,
type RoomOptions,
} from "livekit-client";
import fetchMock from "fetch-mock";
import EventEmitter from "events";
import { type IOpenIDToken } from "matrix-js-sdk";
import { logger } from "matrix-js-sdk/lib/logger";
import type { LivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
import type {
CallMembership,
LivekitTransport,
} from "matrix-js-sdk/lib/matrixrtc";
import {
Connection,
type ConnectionOpts,
type ConnectionState,
type PublishingParticipant,
} from "./Connection.ts";
import { ObservableScope } from "../../ObservableScope.ts";
import { type OpenIDClientParts } from "../../../livekit/openIDSFU.ts";
import { FailToGetOpenIdToken } from "../../../utils/errors.ts";
import { ObservableScope } from "../ObservableScope.ts";
import { type OpenIDClientParts } from "../../livekit/openIDSFU.ts";
import { FailToGetOpenIdToken } from "../../utils/errors.ts";
import { mockMediaDevices, mockMuteStates } from "../../utils/test.ts";
import type { ProcessorState } from "../../livekit/TrackProcessorContext.tsx";
import { type MuteStates } from "../MuteStates.ts";
let testScope: ObservableScope;
@@ -48,9 +53,9 @@ let localParticipantEventEmiter: EventEmitter;
let fakeLocalParticipant: MockedObject<LocalParticipant>;
let fakeRoomEventEmiter: EventEmitter;
// let fakeMembershipsFocusMap$: BehaviorSubject<
// { membership: CallMembership; transport: LivekitTransport }[]
// >;
let fakeMembershipsFocusMap$: BehaviorSubject<
{ membership: CallMembership; transport: LivekitTransport }[]
>;
const livekitFocus: LivekitTransport = {
livekit_alias: "!roomID:example.org",
@@ -69,6 +74,9 @@ function setupTest(): void {
}),
getDeviceId: vi.fn().mockReturnValue("ABCDEF"),
} as unknown as OpenIDClientParts);
fakeMembershipsFocusMap$ = new BehaviorSubject<
{ membership: CallMembership; transport: LivekitTransport }[]
>([]);
localParticipantEventEmiter = new EventEmitter();
@@ -95,7 +103,7 @@ function setupTest(): void {
disconnect: vi.fn(),
remoteParticipants: new Map(),
localParticipant: fakeLocalParticipant,
state: LivekitConnectionState.Disconnected,
state: ConnectionState.Disconnected,
on: fakeRoomEventEmiter.on.bind(fakeRoomEventEmiter),
off: fakeRoomEventEmiter.off.bind(fakeRoomEventEmiter),
addListener: fakeRoomEventEmiter.addListener.bind(fakeRoomEventEmiter),
@@ -107,10 +115,11 @@ function setupTest(): void {
} as unknown as LivekitRoom);
}
function setupRemoteConnection(): Connection {
function setupRemoteConnection(): RemoteConnection {
const opts: ConnectionOpts = {
client: client,
transport: livekitFocus,
remoteTransports$: fakeMembershipsFocusMap$,
scope: testScope,
livekitRoomFactory: () => fakeLivekitRoom,
};
@@ -127,7 +136,7 @@ function setupRemoteConnection(): Connection {
fakeLivekitRoom.connect.mockResolvedValue(undefined);
return new Connection(opts, logger);
return new RemoteConnection(opts, undefined);
}
afterEach(() => {
@@ -143,10 +152,11 @@ describe("Start connection states", () => {
const opts: ConnectionOpts = {
client: client,
transport: livekitFocus,
remoteTransports$: fakeMembershipsFocusMap$,
scope: testScope,
livekitRoomFactory: () => fakeLivekitRoom,
};
const connection = new Connection(opts, logger);
const connection = new RemoteConnection(opts, undefined);
expect(connection.state$.getValue().state).toEqual("Initialized");
});
@@ -158,11 +168,12 @@ describe("Start connection states", () => {
const opts: ConnectionOpts = {
client: client,
transport: livekitFocus,
remoteTransports$: fakeMembershipsFocusMap$,
scope: testScope,
livekitRoomFactory: () => fakeLivekitRoom,
};
const connection = new Connection(opts, logger);
const connection = new RemoteConnection(opts, undefined);
const capturedStates: ConnectionState[] = [];
const s = connection.state$.subscribe((value) => {
@@ -193,7 +204,7 @@ describe("Start connection states", () => {
capturedState = capturedStates.pop();
if (capturedState!.state === "FailedToStart") {
expect(capturedState!.error.message).toEqual("Something went wrong");
expect(connection.transport.livekit_alias).toEqual(
expect(capturedState!.transport.livekit_alias).toEqual(
livekitFocus.livekit_alias,
);
} else {
@@ -210,11 +221,12 @@ describe("Start connection states", () => {
const opts: ConnectionOpts = {
client: client,
transport: livekitFocus,
remoteTransports$: fakeMembershipsFocusMap$,
scope: testScope,
livekitRoomFactory: () => fakeLivekitRoom,
};
const connection = new Connection(opts, logger);
const connection = new RemoteConnection(opts, undefined);
const capturedStates: ConnectionState[] = [];
const s = connection.state$.subscribe((value) => {
@@ -249,7 +261,7 @@ describe("Start connection states", () => {
expect(capturedState?.error.message).toContain(
"SFU Config fetch failed with exception Error",
);
expect(connection.transport.livekit_alias).toEqual(
expect(capturedState?.transport.livekit_alias).toEqual(
livekitFocus.livekit_alias,
);
} else {
@@ -266,11 +278,12 @@ describe("Start connection states", () => {
const opts: ConnectionOpts = {
client: client,
transport: livekitFocus,
remoteTransports$: fakeMembershipsFocusMap$,
scope: testScope,
livekitRoomFactory: () => fakeLivekitRoom,
};
const connection = new Connection(opts, logger);
const connection = new RemoteConnection(opts, undefined);
const capturedStates: ConnectionState[] = [];
const s = connection.state$.subscribe((value) => {
@@ -313,7 +326,7 @@ describe("Start connection states", () => {
expect(capturedState.error.message).toContain(
"Failed to connect to livekit",
);
expect(connection.transport.livekit_alias).toEqual(
expect(capturedState.transport.livekit_alias).toEqual(
livekitFocus.livekit_alias,
);
} else {
@@ -363,18 +376,21 @@ describe("Start connection states", () => {
});
});
function fakeRemoteLivekitParticipant(
id: string,
publications: number = 1,
): RemoteParticipant {
function fakeRemoteLivekitParticipant(id: string): RemoteParticipant {
return {
identity: id,
getTrackPublications: () => Array(publications),
} as unknown as RemoteParticipant;
}
function fakeRtcMemberShip(userId: string, deviceId: string): CallMembership {
return {
userId,
deviceId,
} as unknown as CallMembership;
}
describe("Publishing participants observations", () => {
it("should emit the list of publishing participants", () => {
it("should emit the list of publishing participants", async () => {
setupTest();
const connection = setupRemoteConnection();
@@ -382,53 +398,135 @@ describe("Publishing participants observations", () => {
const bobIsAPublisher = Promise.withResolvers<void>();
const danIsAPublisher = Promise.withResolvers<void>();
const observedPublishers: PublishingParticipant[][] = [];
const s = connection.remoteParticipantsWithTracks$.subscribe(
(publishers) => {
observedPublishers.push(publishers);
if (publishers.some((p) => p.identity === "@bob:example.org:DEV111")) {
bobIsAPublisher.resolve();
}
if (publishers.some((p) => p.identity === "@dan:example.org:DEV333")) {
danIsAPublisher.resolve();
}
},
);
const s = connection.allLivekitParticipants$.subscribe((publishers) => {
observedPublishers.push(publishers);
if (
publishers.some(
(p) => p.participant?.identity === "@bob:example.org:DEV111",
)
) {
bobIsAPublisher.resolve();
}
if (
publishers.some(
(p) => p.participant?.identity === "@dan:example.org:DEV333",
)
) {
danIsAPublisher.resolve();
}
});
onTestFinished(() => s.unsubscribe());
// The publishingParticipants$ observable is derived from the current members of the
// livekitRoom and the rtc membership in order to publish the members that are publishing
// on this connection.
let participants: RemoteParticipant[] = [
fakeRemoteLivekitParticipant("@alice:example.org:DEV000", 0),
fakeRemoteLivekitParticipant("@bob:example.org:DEV111", 0),
fakeRemoteLivekitParticipant("@carol:example.org:DEV222", 0),
fakeRemoteLivekitParticipant("@dan:example.org:DEV333", 0),
fakeRemoteLivekitParticipant("@alice:example.org:DEV000"),
fakeRemoteLivekitParticipant("@bob:example.org:DEV111"),
fakeRemoteLivekitParticipant("@carol:example.org:DEV222"),
fakeRemoteLivekitParticipant("@dan:example.org:DEV333"),
];
// Let's simulate 3 members on the livekitRoom
vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get").mockImplementation(
() => new Map(participants.map((p) => [p.identity, p])),
vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get").mockReturnValue(
new Map(participants.map((p) => [p.identity, p])),
);
participants.forEach((p) =>
fakeRoomEventEmiter.emit(RoomEvent.ParticipantConnected, p),
);
for (const participant of participants) {
fakeRoomEventEmiter.emit(RoomEvent.ParticipantConnected, participant);
}
// At this point there should be no publishers
expect(observedPublishers.pop()!.length).toEqual(0);
participants = [
fakeRemoteLivekitParticipant("@alice:example.org:DEV000", 1),
fakeRemoteLivekitParticipant("@bob:example.org:DEV111", 1),
fakeRemoteLivekitParticipant("@carol:example.org:DEV222", 1),
fakeRemoteLivekitParticipant("@dan:example.org:DEV333", 2),
const otherFocus: LivekitTransport = {
livekit_alias: "!roomID:example.org",
livekit_service_url: "https://other-matrix-rtc.example.org/livekit/jwt",
type: "livekit",
};
const rtcMemberships = [
// Say bob is on the same focus
{
membership: fakeRtcMemberShip("@bob:example.org", "DEV111"),
transport: livekitFocus,
},
// Alice and carol is on a different focus
{
membership: fakeRtcMemberShip("@alice:example.org", "DEV000"),
transport: otherFocus,
},
{
membership: fakeRtcMemberShip("@carol:example.org", "DEV222"),
transport: otherFocus,
},
// NO DAVE YET
];
participants.forEach((p) =>
fakeRoomEventEmiter.emit(RoomEvent.ParticipantConnected, p),
// signal this change in rtc memberships
fakeMembershipsFocusMap$.next(rtcMemberships);
// We should have bob has a publisher now
await bobIsAPublisher.promise;
const publishers = observedPublishers.pop();
expect(publishers?.length).toEqual(1);
expect(publishers?.[0].participant?.identity).toEqual(
"@bob:example.org:DEV111",
);
// At this point there should be no publishers
expect(observedPublishers.pop()!.length).toEqual(4);
// Now let's make dan join the rtc memberships
rtcMemberships.push({
membership: fakeRtcMemberShip("@dan:example.org", "DEV333"),
transport: livekitFocus,
});
fakeMembershipsFocusMap$.next(rtcMemberships);
// We should have bob and dan has publishers now
await danIsAPublisher.promise;
const twoPublishers = observedPublishers.pop();
expect(twoPublishers?.length).toEqual(2);
expect(
twoPublishers?.some(
(p) => p.participant?.identity === "@bob:example.org:DEV111",
),
).toBeTruthy();
expect(
twoPublishers?.some(
(p) => p.participant?.identity === "@dan:example.org:DEV333",
),
).toBeTruthy();
// Now let's make bob leave the livekit room
participants = participants.filter(
(p) => p.identity !== "@bob:example.org:DEV111",
);
vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get").mockReturnValue(
new Map(participants.map((p) => [p.identity, p])),
);
fakeRoomEventEmiter.emit(
RoomEvent.ParticipantDisconnected,
fakeRemoteLivekitParticipant("@bob:example.org:DEV111"),
);
const updatedPublishers = observedPublishers.pop();
// Bob is not connected to the room but he is still in the rtc memberships declaring that
// he is using that focus to publish, so he should still appear as a publisher
expect(updatedPublishers?.length).toEqual(2);
const pp = updatedPublishers?.find(
(p) => p.membership.userId == "@bob:example.org",
);
expect(pp).toBeDefined();
expect(pp!.participant).not.toBeDefined();
expect(
updatedPublishers?.some(
(p) => p.participant?.identity === "@dan:example.org:DEV333",
),
).toBeTruthy();
// Now if bob is not in the rtc memberships, he should disappear
const noBob = rtcMemberships.filter(
({ membership }) => membership.userId !== "@bob:example.org",
);
fakeMembershipsFocusMap$.next(noBob);
expect(observedPublishers.pop()?.length).toEqual(1);
});
it("should be scoped to parent scope", (): void => {
@@ -437,20 +535,18 @@ describe("Publishing participants observations", () => {
const connection = setupRemoteConnection();
let observedPublishers: PublishingParticipant[][] = [];
const s = connection.remoteParticipantsWithTracks$.subscribe(
(publishers) => {
observedPublishers.push(publishers);
},
);
const s = connection.allLivekitParticipants$.subscribe((publishers) => {
observedPublishers.push(publishers);
});
onTestFinished(() => s.unsubscribe());
let participants: RemoteParticipant[] = [
fakeRemoteLivekitParticipant("@bob:example.org:DEV111", 0),
fakeRemoteLivekitParticipant("@bob:example.org:DEV111"),
];
// Let's simulate 3 members on the livekitRoom
vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get").mockImplementation(
() => new Map(participants.map((p) => [p.identity, p])),
vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get").mockReturnValue(
new Map(participants.map((p) => [p.identity, p])),
);
for (const participant of participants) {
@@ -460,16 +556,22 @@ describe("Publishing participants observations", () => {
// At this point there should be no publishers
expect(observedPublishers.pop()!.length).toEqual(0);
participants = [fakeRemoteLivekitParticipant("@bob:example.org:DEV111", 1)];
for (const participant of participants) {
fakeRoomEventEmiter.emit(RoomEvent.ParticipantConnected, participant);
}
const rtcMemberships = [
// Say bob is on the same focus
{
membership: fakeRtcMemberShip("@bob:example.org", "DEV111"),
transport: livekitFocus,
},
];
// signal this change in rtc memberships
fakeMembershipsFocusMap$.next(rtcMemberships);
// We should have bob has a publisher now
const publishers = observedPublishers.pop();
expect(publishers?.length).toEqual(1);
expect(publishers?.[0]?.identity).toEqual("@bob:example.org:DEV111");
expect(publishers?.[0].participant?.identity).toEqual(
"@bob:example.org:DEV111",
);
// end the parent scope
testScope.end();
@@ -479,7 +581,9 @@ describe("Publishing participants observations", () => {
participants = participants.filter(
(p) => p.identity !== "@bob:example.org:DEV111",
);
vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get").mockReturnValue(
new Map(participants.map((p) => [p.identity, p])),
);
fakeRoomEventEmiter.emit(
RoomEvent.ParticipantDisconnected,
fakeRemoteLivekitParticipant("@bob:example.org:DEV111"),
@@ -489,112 +593,108 @@ describe("Publishing participants observations", () => {
});
});
//
// NOT USED ANYMORE ?
//
// This setup look like sth for the Publisher. Not a connection.
describe("PublishConnection", () => {
// let fakeBlurProcessor: ProcessorWrapper<BackgroundOptions>;
let roomFactoryMock: Mock<() => LivekitRoom>;
let muteStates: MockedObject<MuteStates>;
// describe("PublishConnection", () => {
// // let fakeBlurProcessor: ProcessorWrapper<BackgroundOptions>;
// let roomFactoryMock: Mock<() => LivekitRoom>;
// let muteStates: MockedObject<MuteStates>;
function setUpPublishConnection(): void {
setupTest();
// function setUpPublishConnection(): void {
// setupTest();
roomFactoryMock = vi.fn().mockReturnValue(fakeLivekitRoom);
// roomFactoryMock = vi.fn().mockReturnValue(fakeLivekitRoom);
muteStates = mockMuteStates();
// muteStates = mockMuteStates();
// fakeBlurProcessor = vi.mocked<ProcessorWrapper<BackgroundOptions>>({
// name: "BackgroundBlur",
// restart: vi.fn().mockResolvedValue(undefined),
// setOptions: vi.fn().mockResolvedValue(undefined),
// getOptions: vi.fn().mockReturnValue({ strength: 0.5 }),
// isRunning: vi.fn().mockReturnValue(false)
// });
}
// // fakeBlurProcessor = vi.mocked<ProcessorWrapper<BackgroundOptions>>({
// // name: "BackgroundBlur",
// // restart: vi.fn().mockResolvedValue(undefined),
// // setOptions: vi.fn().mockResolvedValue(undefined),
// // getOptions: vi.fn().mockReturnValue({ strength: 0.5 }),
// // isRunning: vi.fn().mockReturnValue(false)
// // });
// }
describe("Livekit room creation", () => {
function createSetup(): void {
setUpPublishConnection();
// describe("Livekit room creation", () => {
// function createSetup(): void {
// setUpPublishConnection();
const fakeTrackProcessorSubject$ = new BehaviorSubject<ProcessorState>({
supported: true,
processor: undefined,
});
// const fakeTrackProcessorSubject$ = new BehaviorSubject<ProcessorState>({
// supported: true,
// processor: undefined,
// });
const opts: ConnectionOpts = {
client: client,
transport: livekitFocus,
remoteTransports$: fakeMembershipsFocusMap$,
scope: testScope,
livekitRoomFactory: roomFactoryMock,
};
// const opts: ConnectionOpts = {
// client: client,
// transport: livekitFocus,
// scope: testScope,
// livekitRoomFactory: roomFactoryMock,
// };
const audioInput = {
available$: of(new Map([["mic1", { id: "mic1" }]])),
selected$: new BehaviorSubject({ id: "mic1" }),
select(): void {},
};
// const audioInput = {
// available$: of(new Map([["mic1", { id: "mic1" }]])),
// selected$: new BehaviorSubject({ id: "mic1" }),
// select(): void {},
// };
const videoInput = {
available$: of(new Map([["cam1", { id: "cam1" }]])),
selected$: new BehaviorSubject({ id: "cam1" }),
select(): void {},
};
// const videoInput = {
// available$: of(new Map([["cam1", { id: "cam1" }]])),
// selected$: new BehaviorSubject({ id: "cam1" }),
// select(): void {},
// };
const audioOutput = {
available$: of(new Map([["speaker", { id: "speaker" }]])),
selected$: new BehaviorSubject({ id: "speaker" }),
select(): void {},
};
// const audioOutput = {
// available$: of(new Map([["speaker", { id: "speaker" }]])),
// selected$: new BehaviorSubject({ id: "speaker" }),
// select(): void {},
// };
// TODO understand what is wrong with our mocking that requires ts-expect-error
const fakeDevices = mockMediaDevices({
// @ts-expect-error Mocking only
audioInput,
// @ts-expect-error Mocking only
videoInput,
// @ts-expect-error Mocking only
audioOutput,
});
// // TODO understand what is wrong with our mocking that requires ts-expect-error
// const fakeDevices = mockMediaDevices({
// // @ts-expect-error Mocking only
// audioInput,
// // @ts-expect-error Mocking only
// videoInput,
// // @ts-expect-error Mocking only
// audioOutput,
// });
new PublishConnection(
opts,
fakeDevices,
muteStates,
undefined,
fakeTrackProcessorSubject$,
);
}
// new Connection(
// opts,
// fakeDevices,
// muteStates,
// undefined,
// fakeTrackProcessorSubject$,
// );
// }
it("should create room with proper initial audio and video settings", () => {
createSetup();
// it("should create room with proper initial audio and video settings", () => {
// createSetup();
expect(roomFactoryMock).toHaveBeenCalled();
// expect(roomFactoryMock).toHaveBeenCalled();
const lastCallArgs =
roomFactoryMock.mock.calls[roomFactoryMock.mock.calls.length - 1];
// const lastCallArgs =
// roomFactoryMock.mock.calls[roomFactoryMock.mock.calls.length - 1];
const roomOptions = lastCallArgs.pop() as unknown as RoomOptions;
expect(roomOptions).toBeDefined();
// const roomOptions = lastCallArgs.pop() as unknown as RoomOptions;
// expect(roomOptions).toBeDefined();
expect(roomOptions!.videoCaptureDefaults?.deviceId).toEqual("cam1");
expect(roomOptions!.audioCaptureDefaults?.deviceId).toEqual("mic1");
expect(roomOptions!.audioOutput?.deviceId).toEqual("speaker");
});
// expect(roomOptions!.videoCaptureDefaults?.deviceId).toEqual("cam1");
// expect(roomOptions!.audioCaptureDefaults?.deviceId).toEqual("mic1");
// expect(roomOptions!.audioOutput?.deviceId).toEqual("speaker");
// });
// it("respect controlledAudioDevices", () => {
// // TODO: Refactor the code to make it testable.
// // The UrlParams module is a singleton has a cache and is very hard to test.
// // This breaks other tests as well if not handled properly.
// // vi.mock(import("./../UrlParams"), () => {
// // return {
// // getUrlParams: vi.fn().mockReturnValue({
// // controlledAudioDevices: true
// // })
// // };
// // });
// });
// });
// });
it("respect controlledAudioDevices", () => {
// TODO: Refactor the code to make it testable.
// The UrlParams module is a singleton has a cache and is very hard to test.
// This breaks other tests as well if not handled properly.
// vi.mock(import("./../UrlParams"), () => {
// return {
// getUrlParams: vi.fn().mockReturnValue({
// controlledAudioDevices: true
// })
// };
// });
});
});
});

View File

@@ -1,5 +1,4 @@
/*
Copyright 2025 Element Creations Ltd.
Copyright 2025 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
@@ -19,20 +18,20 @@ import {
RoomEvent,
} from "livekit-client";
import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
import { BehaviorSubject, map } from "rxjs";
import { BehaviorSubject, type Observable } from "rxjs";
import { type Logger } from "matrix-js-sdk/lib/logger";
import {
getSFUConfigWithOpenID,
type OpenIDClientParts,
type SFUConfig,
} from "../../../livekit/openIDSFU.ts";
import { type Behavior } from "../../Behavior.ts";
import { type ObservableScope } from "../../ObservableScope.ts";
} from "../../livekit/openIDSFU.ts";
import { type Behavior } from "../Behavior.ts";
import { type ObservableScope } from "../ObservableScope.ts";
import {
InsufficientCapacityError,
SFURoomCreationRestrictedError,
} from "../../../utils/errors.ts";
} from "../../utils/errors.ts";
export type PublishingParticipant = LocalParticipant | RemoteParticipant;
@@ -50,14 +49,16 @@ export interface ConnectionOpts {
export type ConnectionState =
| { state: "Initialized" }
| { state: "FetchingConfig" }
| { state: "ConnectingToLkRoom" }
| { state: "FetchingConfig"; transport: LivekitTransport }
| { state: "ConnectingToLkRoom"; transport: LivekitTransport }
| { state: "PublishingTracks"; transport: LivekitTransport }
| { state: "FailedToStart"; error: Error; transport: LivekitTransport }
| {
state: "ConnectedToLkRoom";
livekitConnectionState$: Behavior<LivekitConenctionState>;
livekitConnectionState$: Observable<LivekitConenctionState>;
transport: LivekitTransport;
}
| { state: "FailedToStart"; error: Error }
| { state: "Stopped" };
| { state: "Stopped"; transport: LivekitTransport };
/**
* A connection to a Matrix RTC LiveKit backend.
@@ -75,24 +76,6 @@ export class Connection {
*/
public readonly state$: Behavior<ConnectionState> = this._state$;
/**
* The media transport to connect to.
*/
public readonly transport: LivekitTransport;
public readonly livekitRoom: LivekitRoom;
private scope: ObservableScope;
/**
* An observable of the participants that are publishing on this connection. (Excluding our local participant)
* This is derived from `participantsIncludingSubscribers$` and `remoteTransports$`.
* It filters the participants to only those that are associated with a membership that claims to publish on this connection.
*/
public readonly remoteParticipantsWithTracks$: Behavior<
PublishingParticipant[]
>;
/**
* Whether the connection has been stopped.
* @see Connection.stop
@@ -112,13 +95,14 @@ export class Connection {
* @throws {InsufficientCapacityError} if the LiveKit server indicates that it has insufficient capacity to accept the connection.
* @throws {SFURoomCreationRestrictedError} if the LiveKit server indicates that the room does not exist and cannot be created.
*/
// TODO dont make this throw and instead store a connection error state in this class?
// TODO consider an autostart pattern...
public async start(): Promise<void> {
this.logger.debug("Starting Connection");
this.stopped = false;
try {
this._state$.next({
state: "FetchingConfig",
transport: this.transport,
});
const { url, jwt } = await this.getSFUConfigWithOpenID();
// If we were stopped while fetching the config, don't proceed to connect
@@ -126,6 +110,7 @@ export class Connection {
this._state$.next({
state: "ConnectingToLkRoom",
transport: this.transport,
});
try {
await this.livekitRoom.connect(url, jwt);
@@ -156,15 +141,14 @@ export class Connection {
this._state$.next({
state: "ConnectedToLkRoom",
livekitConnectionState$: this.scope.behavior(
connectionStateObserver(this.livekitRoom),
),
transport: this.transport,
livekitConnectionState$: connectionStateObserver(this.livekitRoom),
});
} catch (error) {
this.logger.debug(`Failed to connect to LiveKit room: ${error}`);
this._state$.next({
state: "FailedToStart",
error: error instanceof Error ? error : new Error(`${error}`),
transport: this.transport,
});
throw error;
}
@@ -185,61 +169,58 @@ export class Connection {
* If the connection is already stopped, this is a no-op.
*/
public async stop(): Promise<void> {
this.logger.debug(
`Stopping connection to ${this.transport.livekit_service_url}`,
);
if (this.stopped) return;
await this.livekitRoom.disconnect();
this._state$.next({
state: "Stopped",
transport: this.transport,
});
this.stopped = true;
}
/**
* An observable of the participants that are publishing on this connection.
* This is derived from `participantsIncludingSubscribers$` and `remoteTransports$`.
* It filters the participants to only those that are associated with a membership that claims to publish on this connection.
*/
public readonly participantsWithTrack$: Behavior<PublishingParticipant[]>;
/**
* The media transport to connect to.
*/
public readonly transport: LivekitTransport;
private readonly client: OpenIDClientParts;
private readonly logger: Logger;
public readonly livekitRoom: LivekitRoom;
/**
* Creates a new connection to a matrix RTC LiveKit backend.
*
* @param livekitRoom - LiveKit room instance to use.
* @param opts - Connection options {@link ConnectionOpts}.
*
* @param logger
*/
public constructor(opts: ConnectionOpts, logger: Logger) {
this.logger = logger.getChild("[Connection]");
this.logger.info(
public constructor(opts: ConnectionOpts, logger?: Logger) {
logger?.info(
`[Connection] Creating new connection to ${opts.transport.livekit_service_url} ${opts.transport.livekit_alias}`,
);
const { transport, client, scope } = opts;
this.scope = scope;
this.livekitRoom = opts.livekitRoomFactory();
this.transport = transport;
this.client = client;
// REMOTE participants with track!!!
// this.remoteParticipantsWithTracks$
this.remoteParticipantsWithTracks$ = scope.behavior(
// only tracks remote participants
this.participantsWithTrack$ = scope.behavior(
connectedParticipantsObserver(this.livekitRoom, {
additionalRoomEvents: [
RoomEvent.TrackPublished,
RoomEvent.TrackUnpublished,
],
}).pipe(
map((participants) => {
return participants.filter(
(participant) => participant.getTrackPublications().length > 0,
);
}),
),
}),
[],
);
scope.onEnd(() => {
this.logger.info(`Connection scope ended, stopping connection`);
void this.stop();
});
scope.onEnd(() => void this.stop());
}
}

View File

@@ -7,24 +7,20 @@ Please see LICENSE in the repository root for full details.
import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
import {
type E2EEOptions,
Room as LivekitRoom,
type RoomOptions,
type BaseKeyProvider,
type E2EEManagerOptions,
type BaseE2EEManager,
} from "livekit-client";
import { type Logger } from "matrix-js-sdk/lib/logger";
import E2EEWorker from "livekit-client/e2ee-worker?worker";
import { type ObservableScope } from "../../ObservableScope.ts";
import { type ObservableScope } from "../ObservableScope.ts";
import { Connection } from "./Connection.ts";
import type { OpenIDClientParts } from "../../../livekit/openIDSFU.ts";
import type { MediaDevices } from "../../MediaDevices.ts";
import type { Behavior } from "../../Behavior.ts";
import type { ProcessorState } from "../../../livekit/TrackProcessorContext.tsx";
import { defaultLiveKitOptions } from "../../../livekit/options.ts";
import type { OpenIDClientParts } from "../../livekit/openIDSFU.ts";
import type { MediaDevices } from "../MediaDevices.ts";
import type { Behavior } from "../Behavior.ts";
import type { ProcessorState } from "../../livekit/TrackProcessorContext.tsx";
import { defaultLiveKitOptions } from "../../livekit/options.ts";
// TODO evaluate if this should be done like the Publisher Factory
export interface ConnectionFactory {
createConnection(
transport: LivekitTransport,
@@ -42,37 +38,26 @@ export class ECConnectionFactory implements ConnectionFactory {
* @param client - The OpenID client parts for authentication, needed to get openID and JWT tokens.
* @param devices - Used for video/audio out/in capture options.
* @param processorState$ - Effects like background blur (only for publishing connection?)
* @param livekitKeyProvider
* @param e2eeLivekitOptions - The E2EE options to use for the LiveKit Room.
* @param controlledAudioDevices - Option to indicate whether audio output device is controlled externally (native mobile app).
* @param echoCancellation - Whether to enable echo cancellation for audio capture.
* @param noiseSuppression - Whether to enable noise suppression for audio capture.
* @param livekitRoomFactory - Optional factory function (for testing) to create LivekitRoom instances. If not provided, a default factory is used.
*/
public constructor(
private client: OpenIDClientParts,
private devices: MediaDevices,
private processorState$: Behavior<ProcessorState>,
livekitKeyProvider: BaseKeyProvider | undefined,
private e2eeLivekitOptions: E2EEOptions | undefined,
private controlledAudioDevices: boolean,
livekitRoomFactory?: () => LivekitRoom,
echoCancellation: boolean = true,
noiseSuppression: boolean = true,
) {
const defaultFactory = (): LivekitRoom =>
new LivekitRoom(
generateRoomOption({
devices: this.devices,
processorState: this.processorState$.value,
e2eeLivekitOptions: livekitKeyProvider && {
keyProvider: livekitKeyProvider,
// It's important that every room use a separate E2EE worker.
// They get confused if given streams from multiple rooms.
worker: new E2EEWorker(),
},
controlledAudioDevices: this.controlledAudioDevices,
echoCancellation,
noiseSuppression,
}),
generateRoomOption(
this.devices,
this.processorState$.value,
this.e2eeLivekitOptions,
this.controlledAudioDevices,
),
);
this.livekitRoomFactory = livekitRoomFactory ?? defaultFactory;
}
@@ -97,24 +82,12 @@ export class ECConnectionFactory implements ConnectionFactory {
/**
* Generate the initial LiveKit RoomOptions based on the current media devices and processor state.
*/
function generateRoomOption({
devices,
processorState,
e2eeLivekitOptions,
controlledAudioDevices,
echoCancellation,
noiseSuppression,
}: {
devices: MediaDevices;
processorState: ProcessorState;
e2eeLivekitOptions:
| E2EEManagerOptions
| { e2eeManager: BaseE2EEManager }
| undefined;
controlledAudioDevices: boolean;
echoCancellation: boolean;
noiseSuppression: boolean;
}): RoomOptions {
function generateRoomOption(
devices: MediaDevices,
processorState: ProcessorState,
e2eeLivekitOptions: E2EEOptions | undefined,
controlledAudioDevices: boolean,
): RoomOptions {
return {
...defaultLiveKitOptions,
videoCaptureDefaults: {
@@ -125,8 +98,6 @@ function generateRoomOption({
audioCaptureDefaults: {
...defaultLiveKitOptions.audioCaptureDefaults,
deviceId: devices.audioInput.selected$.value?.id,
echoCancellation,
noiseSuppression,
},
audioOutput: {
// When using controlled audio devices, we don't want to set the

View File

@@ -0,0 +1,294 @@
/*
Copyright 2025 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";
import { BehaviorSubject } from "rxjs";
import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
import { type Participant as LivekitParticipant } from "livekit-client";
import { ObservableScope } from "../ObservableScope.ts";
import { createConnectionManager$ } from "./ConnectionManager.ts";
import { type ConnectionFactory } from "./ConnectionFactory.ts";
import { type Connection } from "./Connection.ts";
import { areLivekitTransportsEqual } from "./matrixLivekitMerger.ts";
import { flushPromises, withTestScheduler } from "../../utils/test.ts";
// Some test constants
const TRANSPORT_1: LivekitTransport = {
type: "livekit",
livekit_service_url: "https://lk.example.org",
livekit_alias: "!alias:example.org",
};
const TRANSPORT_2: LivekitTransport = {
type: "livekit",
livekit_service_url: "https://lk.sample.com",
livekit_alias: "!alias:sample.com",
};
// const TRANSPORT_3: LivekitTransport = {
// type: "livekit",
// livekit_service_url: "https://lk-other.sample.com",
// livekit_alias: "!alias:sample.com",
// };
let fakeConnectionFactory: ConnectionFactory;
let testScope: ObservableScope;
let testTransportStream$: BehaviorSubject<LivekitTransport[]>;
let connectionManagerInputs: {
scope: ObservableScope;
connectionFactory: ConnectionFactory;
inputTransports$: BehaviorSubject<LivekitTransport[]>;
};
let manager: ReturnType<typeof createConnectionManager$>;
beforeEach(() => {
testScope = new ObservableScope();
fakeConnectionFactory = {} as unknown as ConnectionFactory;
vi.mocked(fakeConnectionFactory).createConnection = vi
.fn()
.mockImplementation(
(transport: LivekitTransport, scope: ObservableScope) => {
const mockConnection = {
transport,
} as unknown as Connection;
vi.mocked(mockConnection).start = vi.fn();
vi.mocked(mockConnection).stop = vi.fn();
// Tie the connection's lifecycle to the scope to test scope lifecycle management
scope.onEnd(() => {
void mockConnection.stop();
});
return mockConnection;
},
);
testTransportStream$ = new BehaviorSubject<LivekitTransport[]>([]);
connectionManagerInputs = {
scope: testScope,
connectionFactory: fakeConnectionFactory,
inputTransports$: testTransportStream$,
};
manager = createConnectionManager$(connectionManagerInputs);
});
afterEach(() => {
testScope.end();
});
describe("connections$ stream", () => {
test("Should create and start new connections for each transports", async () => {
const managedConnections = Promise.withResolvers<Connection[]>();
manager.connections$.subscribe((connections) => {
if (connections.length > 0) managedConnections.resolve(connections);
});
connectionManagerInputs.inputTransports$.next([TRANSPORT_1, TRANSPORT_2]);
const connections = await managedConnections.promise;
expect(connections.length).toBe(2);
expect(
vi.mocked(fakeConnectionFactory).createConnection,
).toHaveBeenCalledTimes(2);
const conn1 = connections.find((c) =>
areLivekitTransportsEqual(c.transport, TRANSPORT_1),
);
expect(conn1).toBeDefined();
expect(conn1!.start).toHaveBeenCalled();
const conn2 = connections.find((c) =>
areLivekitTransportsEqual(c.transport, TRANSPORT_2),
);
expect(conn2).toBeDefined();
expect(conn2!.start).toHaveBeenCalled();
});
test("Should start connection only once", async () => {
const observedConnections: Connection[][] = [];
manager.connections$.subscribe((connections) => {
observedConnections.push(connections);
});
testTransportStream$.next([TRANSPORT_1]);
testTransportStream$.next([TRANSPORT_1]);
testTransportStream$.next([TRANSPORT_1]);
testTransportStream$.next([TRANSPORT_1]);
testTransportStream$.next([TRANSPORT_1]);
testTransportStream$.next([TRANSPORT_1, TRANSPORT_2]);
await flushPromises();
const connections = observedConnections.pop()!;
expect(connections.length).toBe(2);
expect(
vi.mocked(fakeConnectionFactory).createConnection,
).toHaveBeenCalledTimes(2);
const conn2 = connections.find((c) =>
areLivekitTransportsEqual(c.transport, TRANSPORT_2),
);
expect(conn2).toBeDefined();
const conn1 = connections.find((c) =>
areLivekitTransportsEqual(c.transport, TRANSPORT_1),
);
expect(conn1).toBeDefined();
expect(conn1!.start).toHaveBeenCalledOnce();
});
test("Should cleanup connections when not needed anymore", async () => {
const observedConnections: Connection[][] = [];
manager.connections$.subscribe((connections) => {
observedConnections.push(connections);
});
testTransportStream$.next([TRANSPORT_1]);
testTransportStream$.next([TRANSPORT_1, TRANSPORT_2]);
await flushPromises();
const conn2 = observedConnections
.pop()!
.find((c) => areLivekitTransportsEqual(c.transport, TRANSPORT_2))!;
testTransportStream$.next([TRANSPORT_1]);
await flushPromises();
// The second connection should have been stopped has it is no longer needed
expect(conn2.stop).toHaveBeenCalled();
// The first connection should still be active
const conn1 = observedConnections.pop()![0];
expect(conn1.stop).not.toHaveBeenCalledOnce();
});
});
describe("connectionManagerData$ stream", () => {
// Used in test to control fake connections' participantsWithTrack$ streams
let fakePublishingParticipantsStreams: Map<
string,
BehaviorSubject<LivekitParticipant[]>
>;
function keyForTransport(transport: LivekitTransport): string {
return `${transport.livekit_service_url}|${transport.livekit_alias}`;
}
beforeEach(() => {
fakePublishingParticipantsStreams = new Map();
// need a more advanced fake connection factory
vi.mocked(fakeConnectionFactory).createConnection = vi
.fn()
.mockImplementation(
(transport: LivekitTransport, scope: ObservableScope) => {
const fakePublishingParticipants$ = new BehaviorSubject<
LivekitParticipant[]
>([]);
const mockConnection = {
transport,
participantsWithTrack$: fakePublishingParticipants$,
} as unknown as Connection;
vi.mocked(mockConnection).start = vi.fn();
vi.mocked(mockConnection).stop = vi.fn();
// Tie the connection's lifecycle to the scope to test scope lifecycle management
scope.onEnd(() => {
void mockConnection.stop();
});
fakePublishingParticipantsStreams.set(
keyForTransport(transport),
fakePublishingParticipants$,
);
return mockConnection;
},
);
});
test("Should report connections with the publishing participants", () => {
withTestScheduler(({ expectObservable, schedule, behavior }) => {
manager = createConnectionManager$({
...connectionManagerInputs,
inputTransports$: behavior("a", {
a: [TRANSPORT_1, TRANSPORT_2],
}),
});
const conn1Participants$ = fakePublishingParticipantsStreams.get(
keyForTransport(TRANSPORT_1),
)!;
schedule("-a-b", {
a: () => {
conn1Participants$.next([
{ identity: "user1A" } as LivekitParticipant,
]);
},
b: () => {
conn1Participants$.next([
{ identity: "user1A" } as LivekitParticipant,
{ identity: "user1B" } as LivekitParticipant,
]);
},
});
const conn2Participants$ = fakePublishingParticipantsStreams.get(
keyForTransport(TRANSPORT_2),
)!;
schedule("--a", {
a: () => {
conn2Participants$.next([
{ identity: "user2A" } as LivekitParticipant,
]);
},
});
expectObservable(manager.connectionManagerData$).toBe("abcd", {
a: expect.toSatisfy((data) => {
return (
data.getConnections().length == 2 &&
data.getParticipantForTransport(TRANSPORT_1).length == 0 &&
data.getParticipantForTransport(TRANSPORT_2).length == 0
);
}),
b: expect.toSatisfy((data) => {
return (
data.getConnections().length == 2 &&
data.getParticipantForTransport(TRANSPORT_1).length == 1 &&
data.getParticipantForTransport(TRANSPORT_2).length == 0 &&
data.getParticipantForTransport(TRANSPORT_1)[0].identity == "user1A"
);
}),
c: expect.toSatisfy((data) => {
return (
data.getConnections().length == 2 &&
data.getParticipantForTransport(TRANSPORT_1).length == 1 &&
data.getParticipantForTransport(TRANSPORT_2).length == 1 &&
data.getParticipantForTransport(TRANSPORT_1)[0].identity ==
"user1A" &&
data.getParticipantForTransport(TRANSPORT_2)[0].identity == "user2A"
);
}),
d: expect.toSatisfy((data) => {
return (
data.getConnections().length == 2 &&
data.getParticipantForTransport(TRANSPORT_1).length == 2 &&
data.getParticipantForTransport(TRANSPORT_2).length == 1 &&
data.getParticipantForTransport(TRANSPORT_1)[0].identity ==
"user1A" &&
data.getParticipantForTransport(TRANSPORT_1)[1].identity ==
"user1B" &&
data.getParticipantForTransport(TRANSPORT_2)[0].identity == "user2A"
);
}),
});
});
});
});

View File

@@ -1,3 +1,6 @@
// TODOs:
// - make ConnectionManager its own actual class
/*
Copyright 2025 Element Creations Ltd.
Copyright 2025 New Vector Ltd.
@@ -10,14 +13,14 @@ import {
type LivekitTransport,
type ParticipantId,
} from "matrix-js-sdk/lib/matrixrtc";
import { BehaviorSubject, combineLatest, map, of, switchMap, tap } from "rxjs";
import { type Logger } from "matrix-js-sdk/lib/logger";
import { BehaviorSubject, combineLatest, map, switchMap } from "rxjs";
import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
import { type LocalParticipant, type RemoteParticipant } from "livekit-client";
import { type Behavior } from "../../Behavior.ts";
import { type Connection } from "./Connection.ts";
import { Epoch, type ObservableScope } from "../../ObservableScope.ts";
import { generateItemsWithEpoch } from "../../../utils/observable.ts";
import { type Behavior } from "../Behavior";
import { type Connection } from "./Connection";
import { type ObservableScope } from "../ObservableScope";
import { generateItems$ } from "../../utils/observable";
import { areLivekitTransportsEqual } from "./MatrixLivekitMembers.ts";
import { type ConnectionFactory } from "./ConnectionFactory.ts";
@@ -52,8 +55,8 @@ export class ConnectionManagerData {
public getConnectionForTransport(
transport: LivekitTransport,
): Connection | null {
return this.store.get(this.getKey(transport))?.[0] ?? null;
): Connection | undefined {
return this.store.get(this.getKey(transport))?.[0];
}
public getParticipantForTransport(
@@ -87,13 +90,10 @@ export class ConnectionManagerData {
interface Props {
scope: ObservableScope;
connectionFactory: ConnectionFactory;
inputTransports$: Behavior<Epoch<LivekitTransport[]>>;
logger: Logger;
inputTransports$: Behavior<LivekitTransport[]>;
}
// TODO - write test for scopes (do we really need to bind scope)
export interface IConnectionManager {
connectionManagerData$: Behavior<Epoch<ConnectionManagerData>>;
}
/**
* Crete a `ConnectionManager`
* @param scope the observable scope used by this object.
@@ -112,9 +112,12 @@ export function createConnectionManager$({
scope,
connectionFactory,
inputTransports$,
logger: parentLogger,
}: Props): IConnectionManager {
const logger = parentLogger.getChild("[ConnectionManager]");
}: Props): {
transports$: Behavior<LivekitTransport[]>;
connectionManagerData$: Behavior<ConnectionManagerData>;
connections$: Behavior<Connection[]>;
} {
const logger = rootLogger.getChild("ConnectionManager");
const running$ = new BehaviorSubject(true);
scope.onEnd(() => running$.next(false));
@@ -130,15 +133,8 @@ export function createConnectionManager$({
*/
const transports$ = scope.behavior(
combineLatest([running$, inputTransports$]).pipe(
map(([running, transports]) =>
transports.mapInner((transport) => (running ? transport : [])),
),
map((transports) => transports.mapInner(removeDuplicateTransports)),
tap(({ value: transports }) => {
logger.trace(
`Managing transports: ${transports.map((t) => t.livekit_service_url).join(", ")}`,
);
}),
map(([running, transports]) => (running ? transports : [])),
map(removeDuplicateTransports),
),
);
@@ -146,76 +142,64 @@ export function createConnectionManager$({
* Connections for each transport in use by one or more session members.
*/
const connections$ = scope.behavior(
transports$.pipe(
generateItemsWithEpoch(
function* (transports) {
for (const transport of transports)
yield {
keys: [transport.livekit_service_url, transport.livekit_alias],
data: undefined,
};
},
(scope, _data$, serviceUrl, alias) => {
logger.debug(`Creating connection to ${serviceUrl} (${alias})`);
const connection = connectionFactory.createConnection(
{
type: "livekit",
livekit_service_url: serviceUrl,
livekit_alias: alias,
},
scope,
logger,
);
// Start the connection immediately
// Use connection state to track connection progress
void connection.start();
// TODO subscribe to connection state to retry or log issues?
return connection;
},
),
),
);
const connectionManagerData$ = scope.behavior(
connections$.pipe(
switchMap((connections) => {
const epoch = connections.epoch;
// Map the connections to list of {connection, participants}[]
const listOfConnectionsWithPublishingParticipants =
connections.value.map((connection) => {
return connection.remoteParticipantsWithTracks$.pipe(
map((participants) => ({
connection,
participants,
})),
);
});
// probably not required
if (listOfConnectionsWithPublishingParticipants.length === 0) {
return of(new Epoch(new ConnectionManagerData(), epoch));
}
// combineLatest the several streams into a single stream with the ConnectionManagerData
return combineLatest(listOfConnectionsWithPublishingParticipants).pipe(
map(
(lists) =>
new Epoch(
lists.reduce((data, { connection, participants }) => {
data.add(connection, participants);
return data;
}, new ConnectionManagerData()),
epoch,
),
),
generateItems$(
transports$,
function* (transports) {
for (const transport of transports)
yield {
// We need to serialize the transport to a string to properly use it
// as a Map key, but we also need the real transport object in order
// to construct the connection; pass it through as the item's data.
key: `${transport.livekit_service_url}|${transport.livekit_alias}`,
data: transport,
};
},
(scope, key, transport$) => {
const connection = connectionFactory.createConnection(
transport$.value,
scope,
logger,
);
}),
// Start the connection immediately
// Use connection state to track connection progress
void connection.start();
return connection;
},
),
new Epoch(new ConnectionManagerData()),
);
return { connectionManagerData$ };
const connectionManagerData$: Behavior<ConnectionManagerData> =
scope.behavior(
connections$.pipe(
switchMap((connections) => {
// Map the connections to list of {connection, participants}[]
const listOfConnectionsWithPublishingParticipants = connections.map(
(connection) => {
return connection.participantsWithTrack$.pipe(
map((participants) => ({
connection,
participants,
})),
);
},
);
// combineLatest the several streams into a single stream with the ConnectionManagerData
return combineLatest(
listOfConnectionsWithPublishingParticipants,
).pipe(
map((lists) =>
lists.reduce((data, { connection, participants }) => {
data.add(connection, participants);
return data;
}, new ConnectionManagerData()),
),
);
}),
),
// start empty
new ConnectionManagerData(),
);
return { transports$, connectionManagerData$, connections$ };
}
function removeDuplicateTransports(

View File

@@ -0,0 +1,201 @@
/*
Copyright 2025 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
type LocalParticipant as LocalLivekitParticipant,
type RemoteParticipant as RemoteLivekitParticipant,
} from "livekit-client";
import {
type LivekitTransport,
type CallMembership,
} from "matrix-js-sdk/lib/matrixrtc";
import { logger } from "matrix-js-sdk/lib/logger";
import {
combineLatest,
filter,
fromEvent,
map,
startWith,
switchMap,
type Observable,
} from "rxjs";
// eslint-disable-next-line rxjs/no-internal
import { type NodeStyleEventEmitter } from "rxjs/internal/observable/fromEvent";
import {
RoomStateEvent,
type Room as MatrixRoom,
type RoomMember,
} from "matrix-js-sdk";
// import type { Logger } from "matrix-js-sdk/lib/logger";
import { type Behavior } from "../Behavior";
import { type ObservableScope } from "../ObservableScope";
import { type createConnectionManager$ } from "./ConnectionManager";
import { getRoomMemberFromRtcMember, memberDisplaynames$ } from "./displayname";
import { type Connection } from "./Connection";
import { generateItems$ } from "../../utils/observable";
/**
* Represent a matrix call member and his associated livekit participation.
* `livekitParticipant` can be undefined if the member is not yet connected to the livekit room
* or if it has no livekit transport at all.
*/
export interface MatrixLivekitMember {
participantId: string;
membership$: Behavior<CallMembership>;
displayName$: Behavior<string>;
participant$:
| Behavior<LocalLivekitParticipant | undefined>
| Behavior<RemoteLivekitParticipant | undefined>;
connection$: Behavior<Connection | undefined>;
mxcAvatarUrl$: Behavior<string | undefined>;
/**
* TODO Try to remove this! Its waaay to much information.
* Just get the member's avatar
* @deprecated
*/
member$: Behavior<RoomMember>;
}
interface Props {
scope: ObservableScope;
membershipsWithTransport$: Behavior<
{ membership: CallMembership; transport?: LivekitTransport }[]
>;
connectionManager: ReturnType<typeof createConnectionManager$>;
// TODO this is too much information for that class,
// apparently needed to get a room member to later get the Avatar
// => Extract an AvatarService instead?
// Better with just `getMember`
matrixRoom: Pick<MatrixRoom, "getMember"> & NodeStyleEventEmitter;
userId: string;
deviceId: string;
}
// Alternative structure idea:
// const livekitMatrixMember$ = (callMemberships$,connectionManager,scope): Observable<MatrixLivekitMember[]> => {
/**
* Combines MatrixRTC and Livekit worlds.
*
* It has a small public interface:
* - in (via constructor):
* - an observable of CallMembership[] to track the call members (The matrix side)
* - a `ConnectionManager` for the lk rooms (The livekit side)
* - out (via public Observable):
* - `remoteMatrixLivekitMember` an observable of MatrixLivekitMember[] to track the remote members and associated livekit data.
*/
export function createMatrixLivekitMembers$({
scope,
membershipsWithTransport$,
connectionManager,
matrixRoom,
userId,
deviceId,
}: Props): Behavior<MatrixLivekitMember[]> {
/**
* Stream of all the call members and their associated livekit data (if available).
*/
function createMatrixLivekitMember$(): Observable<MatrixLivekitMember[]> {
const displayNameMap$ = memberDisplaynames$(
scope,
matrixRoom,
membershipsWithTransport$.pipe(map((v) => v.map((v) => v.membership))),
userId,
deviceId,
);
return generateItems$(
combineLatest([
membershipsWithTransport$,
connectionManager.connectionManagerData$,
]),
function* ([memberships, managerData]) {
for (const { membership, transport } of memberships) {
// TODO! cannot use membership.membershipID yet, Currently its hardcoded by the jwt service to
const participantId = /*membership.membershipID*/ `${membership.userId}:${membership.deviceId}`;
const participants = transport
? managerData.getParticipantForTransport(transport)
: [];
const participant = participants.find(
(p) => p.identity == participantId,
);
const member = getRoomMemberFromRtcMember(
membership,
matrixRoom,
)?.member;
if (member === undefined) {
logger.warn(`No room member for participant ${participantId}`);
continue;
}
const connection = transport
? managerData.getConnectionForTransport(transport)
: undefined;
yield {
key: participantId,
data: {
participant,
membership,
connection,
// This makes sense to add the the js-sdk callMembership (we only need the avatar so probably the call memberhsip just should aquire the avatar)
member,
},
};
}
},
(scope, participantId, data$): MatrixLivekitMember => ({
participantId,
membership$: scope.behavior(data$.pipe(map((data) => data.membership))),
displayName$: scope.behavior(
displayNameMap$.pipe(
map((displayNames) => displayNames.get(participantId)),
filter((name) => name !== undefined),
),
"",
),
participant$: scope.behavior(
data$.pipe(map((data) => data.participant)),
// Assert that a local participant will never become a remote
// participant or vice versa
) as
| Behavior<LocalLivekitParticipant | undefined>
| Behavior<RemoteLivekitParticipant | undefined>,
connection$: scope.behavior(data$.pipe(map((data) => data.connection))),
mxcAvatarUrl$: scope.behavior(
// React to avatar changes
fromEvent(matrixRoom, RoomStateEvent.Members).pipe(
startWith(null),
switchMap(() =>
data$.pipe(map((data) => data.member.getMxcAvatarUrl())),
),
),
),
member$: scope.behavior(data$.pipe(map((data) => data.member))),
}),
);
}
return scope.behavior(createMatrixLivekitMember$().pipe(startWith([])));
}
// TODO add back in the callviewmodel pauseWhen(this.pretendToBeDisconnected$)
// TODO add this to the JS-SDK
export function areLivekitTransportsEqual(
t1: LivekitTransport,
t2: LivekitTransport,
): boolean {
return (
t1.livekit_service_url === t2.livekit_service_url &&
// In case we have different lk rooms in the same SFU (depends on the livekit authorization service)
// It is only needed in case the livekit authorization service is not behaving as expected (or custom implementation)
t1.livekit_alias === t2.livekit_alias
);
}

View File

@@ -0,0 +1,262 @@
/*
Copyright 2025 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
describe,
test,
vi,
expect,
beforeEach,
afterEach,
type MockedObject,
} from "vitest";
import { BehaviorSubject, take } from "rxjs";
import {
type CallMembership,
type LivekitTransport,
} from "matrix-js-sdk/lib/matrixrtc";
import { type Room as MatrixRoom } from "matrix-js-sdk";
import { getParticipantId } from "matrix-js-sdk/lib/matrixrtc/utils";
import {
type MatrixLivekitMember,
matrixLivekitMerger$,
} from "./matrixLivekitMerger";
import { ObservableScope } from "../ObservableScope";
import {
type ConnectionManager,
ConnectionManagerData,
} from "./ConnectionManager";
import { aliceRtcMember } from "../../utils/test-fixtures";
import { mockRemoteParticipant } from "../../utils/test.ts";
import { type Connection } from "./Connection.ts";
let testScope: ObservableScope;
let fakeManagerData$: BehaviorSubject<ConnectionManagerData>;
let fakeMemberships$: BehaviorSubject<CallMembership[]>;
let mockConnectionManager: MockedObject<ConnectionManager>;
let mockMatrixRoom: MatrixRoom;
const userId = "@local:example.com";
const deviceId = "DEVICE000";
// The merger beeing tested
let matrixLivekitMerger: matrixLivekitMerger$;
beforeEach(() => {
testScope = new ObservableScope();
fakeMemberships$ = new BehaviorSubject<CallMembership[]>([]);
fakeManagerData$ = new BehaviorSubject<ConnectionManagerData>(
new ConnectionManagerData(),
);
mockConnectionManager = vi.mocked<ConnectionManager>({
registerTransports: vi.fn(),
connectionManagerData$: fakeManagerData$,
} as unknown as ConnectionManager);
mockMatrixRoom = vi.mocked<MatrixRoom>({
getMember: vi.fn().mockReturnValue(null),
addEventListener: vi.fn(),
removeEventListener: vi.fn(),
} as unknown as MatrixRoom);
matrixLivekitMerger = new matrixLivekitMerger$(
testScope,
fakeMemberships$,
mockConnectionManager,
mockMatrixRoom,
userId,
deviceId,
);
});
afterEach(() => {
testScope.end();
});
test("should signal participant not yet connected to livekit", () => {
fakeMemberships$.next([aliceRtcMember]);
let items: MatrixLivekitMember[] = [];
matrixLivekitMerger.matrixLivekitMember$
.pipe(take(1))
.subscribe((emitted) => {
items = emitted;
});
expect(items).toHaveLength(1);
const item = items[0];
// Assert the expected membership
expect(item.membership).toBe(aliceRtcMember);
// Assert participant & connection are absent (not just `undefined`)
expect(item.participant).not.toBeDefined();
expect(item.participant).not.toBeDefined();
});
test("should signal participant on a connection that is publishing", () => {
const fakeConnection = {
transport: aliceRtcMember.getTransport(aliceRtcMember) as LivekitTransport,
} as unknown as Connection;
fakeMemberships$.next([aliceRtcMember]);
const aliceParticipantId = getParticipantId(
aliceRtcMember.userId,
aliceRtcMember.deviceId,
);
const managerData: ConnectionManagerData = new ConnectionManagerData();
managerData.add(fakeConnection, [
mockRemoteParticipant({ identity: aliceParticipantId }),
]);
fakeManagerData$.next(managerData);
let items: MatrixLivekitMember[] = [];
matrixLivekitMerger.matrixLivekitMember$
.pipe(take(1))
.subscribe((emitted) => {
items = emitted;
});
expect(items).toHaveLength(1);
const item = items[0];
// Assert the expected membership
expect(item.membership).toBe(aliceRtcMember);
expect(item.participant?.identity).toBe(aliceParticipantId);
expect(item.connection?.transport).toEqual(fakeConnection.transport);
});
test("should signal participant on a connection that is not publishing", () => {
const fakeConnection = {
transport: aliceRtcMember.getTransport(aliceRtcMember) as LivekitTransport,
} as unknown as Connection;
fakeMemberships$.next([aliceRtcMember]);
const managerData: ConnectionManagerData = new ConnectionManagerData();
managerData.add(fakeConnection, []);
fakeManagerData$.next(managerData);
matrixLivekitMerger.matrixLivekitMember$.pipe(take(1)).subscribe((items) => {
expect(items).toHaveLength(1);
const item = items[0];
// Assert the expected membership
expect(item.membership).toBe(aliceRtcMember);
expect(item.participant).not.toBeDefined();
// We have the connection
expect(item.connection?.transport).toEqual(fakeConnection.transport);
});
});
describe("Publication edge case", () => {
const connectionA = {
transport: {
type: "livekit",
livekit_service_url: "https://lk.example.org",
livekit_alias: "!alias:example.org",
},
} as unknown as Connection;
const connectionB = {
transport: {
type: "livekit",
livekit_service_url: "https://lk.sample.com",
livekit_alias: "!alias:sample.com",
},
} as unknown as Connection;
const bobMembership = {
userId: "@bob:example.org",
deviceId: "DEV000",
transports: [connectionA.transport],
} as unknown as CallMembership;
const bobParticipantId = getParticipantId(
bobMembership.userId,
bobMembership.deviceId,
);
test("bob is publishing in several connections", () => {
let lastMatrixLkItems: MatrixLivekitMember[] = [];
matrixLivekitMerger.matrixLivekitMember$.subscribe((items) => {
lastMatrixLkItems = items;
});
vi.mocked(bobMembership).getTransport = vi
.fn()
.mockReturnValue(connectionA.transport);
fakeMemberships$.next([bobMembership]);
const lkMap = new ConnectionManagerData();
lkMap.add(connectionA, [
mockRemoteParticipant({ identity: bobParticipantId }),
]);
lkMap.add(connectionB, [
mockRemoteParticipant({ identity: bobParticipantId }),
]);
fakeManagerData$.next(lkMap);
const items = lastMatrixLkItems;
expect(items).toHaveLength(1);
const item = items[0];
// Assert the expected membership
expect(item.membership.userId).toEqual(bobMembership.userId);
expect(item.membership.deviceId).toEqual(bobMembership.deviceId);
expect(item.participant?.identity).toEqual(bobParticipantId);
// The transport info should come from the membership transports and not only from the publishing connection
expect(item.connection?.transport?.livekit_service_url).toEqual(
bobMembership.transports[0]?.livekit_service_url,
);
expect(item.connection?.transport?.livekit_alias).toEqual(
bobMembership.transports[0]?.livekit_alias,
);
});
test("bob is publishing in the wrong connection", () => {
let lastMatrixLkItems: MatrixLivekitMember[] = [];
matrixLivekitMerger.matrixLivekitMember$.subscribe((items) => {
lastMatrixLkItems = items;
});
vi.mocked(bobMembership).getTransport = vi
.fn()
.mockReturnValue(connectionA.transport);
fakeMemberships$.next([bobMembership]);
const lkMap = new ConnectionManagerData();
lkMap.add(connectionA, []);
lkMap.add(connectionB, [
mockRemoteParticipant({ identity: bobParticipantId }),
]);
fakeManagerData$.next(lkMap);
const items = lastMatrixLkItems;
expect(items).toHaveLength(1);
const item = items[0];
// Assert the expected membership
expect(item.membership.userId).toEqual(bobMembership.userId);
expect(item.membership.deviceId).toEqual(bobMembership.deviceId);
expect(item.participant).not.toBeDefined();
// The transport info should come from the membership transports and not only from the publishing connection
expect(item.connection?.transport?.livekit_service_url).toEqual(
bobMembership.transports[0]?.livekit_service_url,
);
expect(item.connection?.transport?.livekit_alias).toEqual(
bobMembership.transports[0]?.livekit_alias,
);
});
});

View File

@@ -0,0 +1,87 @@
/*
Copyright 2025 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type RoomMember, RoomStateEvent } from "matrix-js-sdk";
import { combineLatest, fromEvent, type Observable, startWith } from "rxjs";
import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc";
import { logger } from "matrix-js-sdk/lib/logger";
import { type Room as MatrixRoom } from "matrix-js-sdk/lib/matrix";
// eslint-disable-next-line rxjs/no-internal
import { type NodeStyleEventEmitter } from "rxjs/internal/observable/fromEvent";
import { type ObservableScope } from "../ObservableScope";
import {
calculateDisplayName,
shouldDisambiguate,
} from "../../utils/displayname";
import { type Behavior } from "../Behavior";
/**
* Displayname for each member of the call. This will disambiguate
* any displayname that clashes with another member. Only members
* joined to the call are considered here.
*
* @returns Map<member.id, displayname> uses the rtc member idenitfier as the key.
*/
// don't do this work more times than we need to. This is achieved by converting to a behavior:
export const memberDisplaynames$ = (
scope: ObservableScope,
matrixRoom: Pick<MatrixRoom, "getMember"> & NodeStyleEventEmitter,
memberships$: Observable<CallMembership[]>,
userId: string,
deviceId: string,
): Behavior<Map<string, string>> =>
scope.behavior(
combineLatest(
[
// Handle call membership changes
memberships$,
// Additionally handle display name changes (implicitly reacting to them)
fromEvent(matrixRoom, RoomStateEvent.Members).pipe(startWith(null)),
// TODO: do we need: pauseWhen(this.pretendToBeDisconnected$),
],
(memberships, _displaynames) => {
const displaynameMap = new Map<string, string>([
[
`${userId}:${deviceId}`,
matrixRoom.getMember(userId)?.rawDisplayName ?? userId,
],
]);
const room = matrixRoom;
// We only consider RTC members for disambiguation as they are the only visible members.
for (const rtcMember of memberships) {
const matrixIdentifier = `${rtcMember.userId}:${rtcMember.deviceId}`;
const { member } = getRoomMemberFromRtcMember(rtcMember, room);
if (!member) {
logger.error(
"Could not find member for media id:",
matrixIdentifier,
);
continue;
}
const disambiguate = shouldDisambiguate(member, memberships, room);
displaynameMap.set(
matrixIdentifier,
calculateDisplayName(member, disambiguate),
);
}
return displaynameMap;
},
),
new Map<string, string>(),
);
export function getRoomMemberFromRtcMember(
rtcMember: CallMembership,
room: Pick<MatrixRoom, "getMember">,
): { id: string; member: RoomMember | undefined } {
return {
id: rtcMember.userId + ":" + rtcMember.deviceId,
member: room.getMember(rtcMember.userId) ?? undefined,
};
}

View File

@@ -0,0 +1,157 @@
/*
Copyright 2025 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { test, vi, beforeEach, afterEach } from "vitest";
import { BehaviorSubject, type Observable } from "rxjs";
import { type Room as LivekitRoom } from "livekit-client";
import { logger } from "matrix-js-sdk/lib/logger";
import EventEmitter from "events";
import fetchMock from "fetch-mock";
import { ConnectionManager } from "./ConnectionManager.ts";
import { ObservableScope } from "../ObservableScope.ts";
import { ECConnectionFactory } from "./ConnectionFactory.ts";
import { type OpenIDClientParts } from "../../livekit/openIDSFU.ts";
import { mockMediaDevices, withTestScheduler } from "../../utils/test";
import { type ProcessorState } from "../../livekit/TrackProcessorContext.tsx";
import { matrixLivekitMerger$ } from "./matrixLivekitMerger.ts";
import type { CallMembership, Transport } from "matrix-js-sdk/lib/matrixrtc";
import { TRANSPORT_1 } from "./ConnectionManager.test.ts";
// Test the integration of ConnectionManager and MatrixLivekitMerger
let testScope: ObservableScope;
let ecConnectionFactory: ECConnectionFactory;
let mockClient: OpenIDClientParts;
let lkRoomFactory: () => LivekitRoom;
const createdMockLivekitRooms: Map<string, LivekitRoom> = new Map();
// Main test input
const memberships$ = new BehaviorSubject<CallMembership[]>([]);
// under test
let connectionManager: ConnectionManager;
function createLkMerger(
memberships$: Observable<CallMembership[]>,
): matrixLivekitMerger$ {
const mockRoomEmitter = new EventEmitter();
return new matrixLivekitMerger$(
testScope,
memberships$,
connectionManager,
{
on: mockRoomEmitter.on.bind(mockRoomEmitter),
off: mockRoomEmitter.off.bind(mockRoomEmitter),
getMember: vi.fn().mockReturnValue(undefined),
},
"@user:example.com",
"DEV000",
);
}
beforeEach(() => {
testScope = new ObservableScope();
mockClient = {
getOpenIdToken: vi.fn().mockReturnValue(""),
getDeviceId: vi.fn().mockReturnValue("DEV000"),
};
lkRoomFactory = vi.fn().mockImplementation(() => {
const emitter = new EventEmitter();
const base = {
on: emitter.on.bind(emitter),
off: emitter.off.bind(emitter),
emit: emitter.emit.bind(emitter),
disconnect: vi.fn(),
remoteParticipants: new Map(),
} as unknown as LivekitRoom;
vi.mocked(base).connect = vi.fn().mockImplementation(({ url }) => {
createdMockLivekitRooms.set(url, base);
});
return base;
});
ecConnectionFactory = new ECConnectionFactory(
mockClient,
mockMediaDevices({}),
new BehaviorSubject<ProcessorState>({
supported: true,
processor: undefined,
}),
undefined,
false,
lkRoomFactory,
);
connectionManager = new ConnectionManager(
testScope,
ecConnectionFactory,
logger,
);
//TODO a bit annoying to have to do a http mock?
fetchMock.post(`**/sfu/get`, (url) => {
const domain = new URL(url).hostname; // Extract the domain from the URL
return {
status: 200,
body: {
url: `wss://${domain}/livekit/sfu`,
jwt: "ATOKEN",
},
};
});
});
afterEach(() => {
testScope.end();
fetchMock.reset();
});
test("example test", () => {
withTestScheduler(({ schedule, expectObservable, cold }) => {
connectionManager.connections$.subscribe((connections) => {
// console.log(
// "Connections updated:",
// connections.map((c) => c.transport),
// );
});
const memberships$ = cold("-a-b-c", {
a: [mockCallmembership("@bob:example.com", "BDEV000")],
b: [
mockCallmembership("@bob:example.com", "BDEV000"),
mockCallmembership("@carl:example.com", "CDEV000"),
],
c: [
mockCallmembership("@bob:example.com", "BDEV000"),
mockCallmembership("@carl:example.com", "CDEV000"),
mockCallmembership("@dave:foo.bar", "DDEV000"),
],
});
// TODO IN PROGRESS
const merger = createLkMerger(memberships$);
});
});
function mockCallmembership(
userId: string,
deviceId: string,
transport?: Transport,
): CallMembership {
const t = transport ?? TRANSPORT_1;
return {
userId: userId,
deviceId: deviceId,
getTransport: vi.fn().mockReturnValue(t),
transports: [t],
} as unknown as CallMembership;
}

View File

@@ -15,7 +15,7 @@ import { GridTile } from "./GridTile";
import { mockRtcMembership, createRemoteMedia } from "../utils/test";
import { GridTileViewModel } from "../state/TileViewModel";
import { ReactionsSenderProvider } from "../reactions/useReactionsSender";
import type { CallViewModel } from "../state/CallViewModel/CallViewModel";
import type { CallViewModel } from "../state/CallViewModel";
import { constant } from "../state/Behavior";
global.IntersectionObserver = class MockIntersectionObserver {

View File

@@ -58,9 +58,7 @@ interface TileProps {
style?: ComponentProps<typeof animated.div>["style"];
targetWidth: number;
targetHeight: number;
focusUrl: string | undefined;
displayName: string;
mxcAvatarUrl: string | undefined;
showSpeakingIndicators: boolean;
focusable: boolean;
}
@@ -83,9 +81,7 @@ const UserMediaTile: FC<UserMediaTileProps> = ({
menuStart,
menuEnd,
className,
focusUrl,
displayName,
mxcAvatarUrl,
focusable,
...props
}) => {
@@ -148,8 +144,8 @@ const UserMediaTile: FC<UserMediaTileProps> = ({
const tile = (
<MediaView
ref={ref}
video={video ?? undefined}
userId={vm.userId}
video={video}
member={vm.member}
unencryptedWarning={unencryptedWarning}
encryptionStatus={encryptionStatus}
videoEnabled={videoEnabled}
@@ -168,7 +164,6 @@ const UserMediaTile: FC<UserMediaTileProps> = ({
/>
}
displayName={displayName}
mxcAvatarUrl={mxcAvatarUrl}
focusable={focusable}
primaryButton={
primaryButton ?? (
@@ -195,7 +190,7 @@ const UserMediaTile: FC<UserMediaTileProps> = ({
currentReaction={reaction ?? undefined}
raisedHandOnClick={raisedHandOnClick}
localParticipant={vm.local}
focusUrl={focusUrl}
focusUrl={vm.focusURL}
audioStreamStats={audioStreamStats}
videoStreamStats={videoStreamStats}
{...props}
@@ -364,9 +359,7 @@ export const GridTile: FC<GridTileProps> = ({
const ourRef = useRef<HTMLDivElement | null>(null);
const ref = useMergedRefs(ourRef, theirRef);
const media = useBehavior(vm.media$);
const focusUrl = useBehavior(media.focusUrl$);
const displayName = useBehavior(media.displayName$);
const mxcAvatarUrl = useBehavior(media.mxcAvatarUrl$);
if (media instanceof LocalUserMediaViewModel) {
return (
@@ -374,9 +367,7 @@ export const GridTile: FC<GridTileProps> = ({
ref={ref}
vm={media}
onOpenProfile={onOpenProfile}
focusUrl={focusUrl}
displayName={displayName}
mxcAvatarUrl={mxcAvatarUrl}
{...props}
/>
);
@@ -385,9 +376,7 @@ export const GridTile: FC<GridTileProps> = ({
<RemoteUserMediaTile
ref={ref}
vm={media}
focusUrl={focusUrl}
displayName={displayName}
mxcAvatarUrl={mxcAvatarUrl}
{...props}
/>
);

View File

@@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { describe, expect, it, test } from "vitest";
import { describe, expect, it, test, vi } from "vitest";
import { render, screen } from "@testing-library/react";
import { axe } from "vitest-axe";
import { TooltipProvider } from "@vector-im/compound-web";
@@ -16,6 +16,7 @@ import {
import { LocalTrackPublication, Track } from "livekit-client";
import { TrackInfo } from "@livekit/protocol";
import { type ComponentProps } from "react";
import { type RoomMember } from "matrix-js-sdk";
import { MediaView } from "./MediaView";
import { EncryptionStatus } from "../state/MediaViewModel";
@@ -45,8 +46,10 @@ describe("MediaView", () => {
mirror: false,
unencryptedWarning: false,
video: trackReference,
userId: "@alice:example.com",
mxcAvatarUrl: undefined,
member: vi.mocked<RoomMember>({
userId: "@alice:example.com",
getMxcAvatarUrl: vi.fn().mockReturnValue(undefined),
} as unknown as RoomMember),
localParticipant: false,
focusable: true,
};

View File

@@ -7,6 +7,7 @@ Please see LICENSE in the repository root for full details.
import { type TrackReferenceOrPlaceholder } from "@livekit/components-core";
import { animated } from "@react-spring/web";
import { type RoomMember } from "matrix-js-sdk";
import { type FC, type ComponentProps, type ReactNode } from "react";
import { useTranslation } from "react-i18next";
import classNames from "classnames";
@@ -31,13 +32,12 @@ interface Props extends ComponentProps<typeof animated.div> {
video: TrackReferenceOrPlaceholder | undefined;
videoFit: "cover" | "contain";
mirror: boolean;
userId: string;
member: RoomMember;
videoEnabled: boolean;
unencryptedWarning: boolean;
encryptionStatus: EncryptionStatus;
nameTagLeadingIcon?: ReactNode;
displayName: string;
mxcAvatarUrl: string | undefined;
focusable: boolean;
primaryButton?: ReactNode;
raisedHandTime?: Date;
@@ -59,12 +59,11 @@ export const MediaView: FC<Props> = ({
video,
videoFit,
mirror,
userId,
member,
videoEnabled,
unencryptedWarning,
nameTagLeadingIcon,
displayName,
mxcAvatarUrl,
focusable,
primaryButton,
encryptionStatus,
@@ -95,10 +94,10 @@ export const MediaView: FC<Props> = ({
>
<div className={styles.bg}>
<Avatar
id={userId}
id={member?.userId ?? displayName}
name={displayName}
size={avatarSize}
src={mxcAvatarUrl}
src={member?.getMxcAvatarUrl()}
className={styles.avatar}
style={{ display: video && videoEnabled ? "none" : "initial" }}
/>

View File

@@ -27,6 +27,7 @@ import { useObservableRef } from "observable-hooks";
import { useTranslation } from "react-i18next";
import classNames from "classnames";
import { type TrackReferenceOrPlaceholder } from "@livekit/components-core";
import { type RoomMember } from "matrix-js-sdk";
import FullScreenMaximiseIcon from "../icons/FullScreenMaximise.svg?react";
import FullScreenMinimiseIcon from "../icons/FullScreenMinimise.svg?react";
@@ -54,12 +55,10 @@ interface SpotlightItemBaseProps {
targetHeight: number;
video: TrackReferenceOrPlaceholder | undefined;
videoEnabled: boolean;
userId: string;
member: RoomMember;
unencryptedWarning: boolean;
encryptionStatus: EncryptionStatus;
focusUrl: string | undefined;
displayName: string;
mxcAvatarUrl: string | undefined;
focusable: boolean;
"aria-hidden"?: boolean;
localParticipant: boolean;
@@ -79,7 +78,7 @@ const SpotlightLocalUserMediaItem: FC<SpotlightLocalUserMediaItemProps> = ({
...props
}) => {
const mirror = useBehavior(vm.mirror$);
return <MediaView mirror={mirror} {...props} />;
return <MediaView mirror={mirror} focusUrl={vm.focusURL} {...props} />;
};
SpotlightLocalUserMediaItem.displayName = "SpotlightLocalUserMediaItem";
@@ -135,9 +134,7 @@ const SpotlightItem: FC<SpotlightItemProps> = ({
}) => {
const ourRef = useRef<HTMLDivElement | null>(null);
const ref = useMergedRefs(ourRef, theirRef);
const focusUrl = useBehavior(vm.focusUrl$);
const displayName = useBehavior(vm.displayName$);
const mxcAvatarUrl = useBehavior(vm.mxcAvatarUrl$);
const video = useBehavior(vm.video$);
const videoEnabled = useBehavior(vm.videoEnabled$);
const unencryptedWarning = useBehavior(vm.unencryptedWarning$);
@@ -164,13 +161,11 @@ const SpotlightItem: FC<SpotlightItemProps> = ({
className: classNames(styles.item, { [styles.snap]: snap }),
targetWidth,
targetHeight,
video: video ?? undefined,
video,
videoEnabled,
userId: vm.userId,
member: vm.member,
unencryptedWarning,
focusUrl,
displayName,
mxcAvatarUrl,
focusable,
encryptionStatus,
"aria-hidden": ariaHidden,

View File

@@ -6,10 +6,10 @@ Please see LICENSE in the repository root for full details.
*/
import { afterEach, beforeAll, describe, expect, test, vi } from "vitest";
import { type RoomMember } from "matrix-js-sdk";
import { shouldDisambiguate } from "./displayname";
import { alice } from "./test-fixtures";
import { mockMatrixRoom } from "./test";
// Ideally these tests would be in ./displayname.test.ts but I can't figure out how to
// just spy on the removeHiddenChars() function without impacting the other tests.
@@ -29,7 +29,7 @@ describe("shouldDisambiguate", () => {
});
test("should only call removeHiddenChars once for a single displayname", () => {
const room: Map<string, Pick<RoomMember, "userId">> = new Map([]);
const room = mockMatrixRoom({});
shouldDisambiguate(alice, [], room);
expect(jsUtils.removeHiddenChars).toHaveBeenCalledTimes(1);
for (let i = 0; i < 10; i++) {

Some files were not shown because too many files have changed in this diff Show More