Matrix/element-call-Github
1
0
Fork 0
mirror of https://github.com/vector-im/element-call.git synced 2026-03-10 05:57:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3600 from element-hq/toger5/lib-ec-version

Browse Source 
Element Call SDK target
This commit is contained in:
Timo
2025-12-23 12:53:57 +01:00
committed by GitHub
parent 6cfa95bcfa 9bd7888fab
commit 67d20a8f3d
22 changed files with 1547 additions and 193 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
knip.ts
View File

@@ -9,7 +9,7 @@ import { type KnipConfig } from "knip";
export default {
vite: {
config: ["vite.config.ts", "vite-embedded.config.ts"],
config: ["vite.config.ts", "vite-embedded.config.ts", "vite-sdk.config.ts"],
},
entry: ["src/main.tsx", "i18next-parser.config.ts"],
ignoreBinaries: [

4
package.json
View File

@@ -13,6 +13,8 @@
"build:embedded": "yarn build:full --config vite-embedded.config.js",
"build:embedded:production": "yarn build:embedded",
"build:embedded:development": "yarn build:embedded --mode development",
"build:sdk": "yarn build:full --config vite-sdk.config.js",
"build:sdk:development": "yarn build:sdk --mode development",
"serve": "vite preview",
"prettier:check": "prettier -c .",
"prettier:format": "prettier -w .",
@@ -111,6 +113,7 @@
"loglevel": "^1.9.1",
"matrix-js-sdk": "^39.2.0",
"matrix-widget-api": "^1.14.0",
"node-stdlib-browser": "^1.3.1",
"normalize.css": "^8.0.1",
"observable-hooks": "^4.2.3",
"pako": "^2.0.4",
@@ -133,6 +136,7 @@
"vite": "^7.0.0",
"vite-plugin-generate-file": "^0.3.0",
"vite-plugin-html": "^3.2.2",
"vite-plugin-node-stdlib-browser": "^0.2.1",
"vite-plugin-svgr": "^4.0.0",
"vitest": "^3.0.0",
"vitest-axe": "^1.0.0-pre.3"

35
sdk/README.md Normal file
View File

@@ -0,0 +1,35 @@
# SDK mode
EC can be build in sdk mode. This will result in a compiled js file that can be imported in very simple webapps.
It allows to use matrixRTC in combination with livekit without relying on element call.
This is done by instantiating the call view model and exposing some useful behaviors (observables) and methods.
This folder contains an example index.html file that showcases the sdk in use (hosted on localhost:8123 with a webserver ellowing cors (for example `npx serve -l 81234 --cors`)) as a godot engine HTML export template.
## Widgets
The sdk mode is particularly interesting to be used in widgets where you do not need to pay attention to matrix login/cs api ...
To create a widget see the example index.html file in this folder. And add it to EW via:
`/addwidget <widgetUrl>` (see **url parameters** for more details on `<widgetUrl>`)
### url parameters
```
widgetId = $matrix_widget_id
perParticipantE2EE = true
userId = $matrix_user_id
deviceId = $org.matrix.msc3819.matrix_device_id
baseUrl = $org.matrix.msc4039.matrix_base_url
```
`parentUrl = // will be inserted automatically`
Full template use as `<widgetUrl>`:
```
http://localhost:3000?widgetId=$matrix_widget_id&perParticipantE2EE=true&userId=$matrix_user_id&deviceId=$org.matrix.msc3819.matrix_device_id&baseUrl=$org.matrix.msc4039.matrix_base_url&roomId=$matrix_room_id
```
the `$` prefixed variables will be replaced by EW on widget instantiation. (e.g. `$matrix_user_id` -> `@user:example.com` (url encoding will also be applied automatically by EW) -> `%40user%3Aexample.com`)

55
sdk/helper.ts Normal file
View File

@@ -0,0 +1,55 @@
/*
Copyright 2025 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
/**
* This file contains helper functions and types for the MatrixRTC SDK.
*/
import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
import { scan } from "rxjs";
import { widget as _widget } from "../src/widget";
import { type LivekitRoomItem } from "../src/state/CallViewModel/CallViewModel";
export const logger = rootLogger.getChild("[MatrixRTCSdk]");
if (!_widget) throw Error("No widget. This webapp can only start as a widget");
export const widget = _widget;
export const tryMakeSticky = (): void => {
logger.info("try making sticky MatrixRTCSdk");
void widget.api
.setAlwaysOnScreen(true)
.then(() => {
logger.info("sticky MatrixRTCSdk");
})
.catch((error) => {
logger.error("failed to make sticky MatrixRTCSdk", error);
});
};
export const TEXT_LK_TOPIC = "matrixRTC";
/**
* simple helper operator to combine the last emitted and the current emitted value of a rxjs observable
*
* I think there should be a builtin for this but i did not find it...
*/
export const currentAndPrev = scan<
LivekitRoomItem[],
{
prev: LivekitRoomItem[];
current: LivekitRoomItem[];
}
>(
({ current: lastCurrentVal }, items) => ({
prev: lastCurrentVal,
current: items,
}),
{
prev: [],
current: [],
},
);

87
sdk/index.html Normal file
View File

@@ -0,0 +1,87 @@
<!doctype html>
<html>
<head>
<title>Godot MatrixRTC Widget</title>
<meta charset="utf-8" />
<script type="module">
// TODO use the url where the matrixrtc-sdk.js file from dist is hosted
import { createMatrixRTCSdk } from "http://localhost:8123/matrixrtc-sdk.js";
try {
window.matrixRTCSdk = await createMatrixRTCSdk(
"com.github.toger5.godot-game",
);
console.info("createMatrixRTCSdk was created!");
} catch (e) {
console.error("createMatrixRTCSdk", e);
}
const sdk = window.matrixRTCSdk;
// This is the main bridging interface to godot
window.matrixRTCSdkGodot = {
dataObs: sdk.data$,
memberObs: sdk.members$,
// join: sdk.join, // lets stick with autojoin for now
sendData: sdk.sendData,
leave: sdk.leave,
connectedObs: sdk.connected$,
};
console.info("matrixRTCSdk join ", sdk);
const connectionState = sdk.join();
console.info("matrixRTCSdk joined");
const div = document.getElementById("data");
div.innerHTML = "<h3>Data:</h3>";
sdk.data$.subscribe((data) => {
const child = document.createElement("p");
child.innerHTML = JSON.stringify(data);
div.appendChild(child);
// TODO forward to godot
});
sdk.members$.subscribe((memberObjects) => {
// reset div
const div = document.getElementById("members");
div.innerHTML = "<h3>Members:</h3>";
// create member list
const members = memberObjects.map((member) => member.membership.sender);
console.info("members changed", members);
for (const m of members) {
console.info("member", m);
const child = document.createElement("p");
child.innerHTML = m;
div.appendChild(child);
}
});
sdk.connected$.subscribe((connected) => {
console.info("connected changed", connected);
const div = document.getElementById("connect_status");
div.innerHTML = connected ? "Connected" : "Disconnected";
});
let engine = new Engine($GODOT_CONFIG);
engine.startGame();
</script>
<!--// TODO use it as godot HTML template-->
<script src="$GODOT_URL"></script>
</head>
<body>
<canvas id="canvas"></canvas>
<div
id="overlay"
style="position: absolute; top: 0; right: 0; background-color: #ffffff10"
>
<div id="connect_status"></div>
<button onclick="window.matrixRTCSdk.leave();">Leave</button>
<button onclick="window.matrixRTCSdk.sendData({prop: 'Hello, world!'});">
Send Text
</button>
<div id="members"></div>
<div id="data"></div>
</div>
</body>
</html>

308
sdk/main.ts Normal file
View File

@@ -0,0 +1,308 @@
/*
Copyright 2025 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
/**
* This file is the entrypoint for the sdk build of element call: `yarn build:sdk`
* use in widgets.
* It exposes the `createMatrixRTCSdk` which creates the `MatrixRTCSdk` interface (see below) that
* can be used to join a rtc session and exchange realtime data.
* It takes care of all the tricky bits:
* - sending delayed events
* - finding the right sfu
* - handling the media stream
* - sending join/leave state or sticky events
* - setting up encryption and scharing keys
*/
import {
combineLatest,
map,
type Observable,
of,
shareReplay,
Subject,
switchMap,
tap,
} from "rxjs";
import {
type CallMembership,
MatrixRTCSession,
MatrixRTCSessionEvent,
} from "matrix-js-sdk/lib/matrixrtc";
import {
type Room as LivekitRoom,
type TextStreamReader,
type LocalParticipant,
type RemoteParticipant,
} from "livekit-client";
// TODO how can this get fixed? to just be part of `livekit-client`
// Can this be done in the tsconfig.json
import { type TextStreamInfo } from "../node_modules/livekit-client/dist/src/room/types";
import { type Behavior, constant } from "../src/state/Behavior";
import { createCallViewModel$ } from "../src/state/CallViewModel/CallViewModel";
import { ObservableScope } from "../src/state/ObservableScope";
import { getUrlParams } from "../src/UrlParams";
import { MuteStates } from "../src/state/MuteStates";
import { MediaDevices } from "../src/state/MediaDevices";
import { E2eeType } from "../src/e2ee/e2eeType";
import {
currentAndPrev,
logger,
TEXT_LK_TOPIC,
tryMakeSticky,
widget,
} from "./helper";
import { ElementWidgetActions } from "../src/widget";
import { type Connection } from "../src/state/CallViewModel/remoteMembers/Connection";
interface MatrixRTCSdk {
/**
* observe connected$ to track the state.
* @returns
*/
join: () => void;
/** @throws on leave errors */
leave: () => void;
data$: Observable<{ sender: string; data: string }>;
/**
* flattened list of members
*/
members$: Behavior<
{
connection: Connection | null;
membership: CallMembership;
participant: LocalParticipant | RemoteParticipant | null;
}[]
>;
/** Use the LocalMemberConnectionState returned from `join` for a more detailed connection state */
connected$: Behavior<boolean>;
sendData?: (data: unknown) => Promise<void>;
}
export async function createMatrixRTCSdk(
application: string = "m.call",
id: string = "",
): Promise<MatrixRTCSdk> {
logger.info("Hello");
const client = await widget.client;
logger.info("client created");
const scope = new ObservableScope();
const { roomId } = getUrlParams();
if (roomId === null) throw Error("could not get roomId from url params");
const room = client.getRoom(roomId);
if (room === null) throw Error("could not get room from client");
const mediaDevices = new MediaDevices(scope);
const muteStates = new MuteStates(scope, mediaDevices, constant(true));
const slot = { application, id };
const rtcSession = new MatrixRTCSession(
client,
room,
MatrixRTCSession.sessionMembershipsForSlot(room, slot),
slot,
);
const callViewModel = createCallViewModel$(
scope,
rtcSession,
room,
mediaDevices,
muteStates,
{ encryptionSystem: { kind: E2eeType.PER_PARTICIPANT } },
of({}),
of({}),
constant({ supported: false, processor: undefined }),
);
logger.info("CallViewModelCreated");
// create data listener
const data$ = new Subject<{ sender: string; data: string }>();
const lkTextStreamHandlerFunction = async (
reader: TextStreamReader,
participantInfo: { identity: string },
livekitRoom: LivekitRoom,
): Promise<void> => {
const info = reader.info;
logger.info(
`Received text stream from ${participantInfo.identity}\n` +
` Topic: ${info.topic}\n` +
` Timestamp: ${info.timestamp}\n` +
` ID: ${info.id}\n` +
` Size: ${info.size}`, // Optional, only available if the stream was sent with `sendText`
);
const participants = callViewModel.livekitRoomItems$.value.find(
(i) => i.livekitRoom === livekitRoom,
)?.participants;
if (participants && participants.includes(participantInfo.identity)) {
const text = await reader.readAll();
logger.info(`Received text: ${text}`);
data$.next({ sender: participantInfo.identity, data: text });
} else {
logger.warn(
"Received text from unknown participant",
participantInfo.identity,
);
}
};
const livekitRoomItemsSub = callViewModel.livekitRoomItems$
.pipe(
tap((beforecurrentAndPrev) => {
logger.info(
`LiveKit room items updated: ${beforecurrentAndPrev.length}`,
beforecurrentAndPrev,
);
}),
currentAndPrev,
tap((aftercurrentAndPrev) => {
logger.info(
`LiveKit room items updated: ${aftercurrentAndPrev.current.length}, ${aftercurrentAndPrev.prev.length}`,
aftercurrentAndPrev,
);
}),
)
.subscribe({
next: ({ prev, current }) => {
const prevRooms = prev.map((i) => i.livekitRoom);
const currentRooms = current.map((i) => i.livekitRoom);
const addedRooms = currentRooms.filter((r) => !prevRooms.includes(r));
const removedRooms = prevRooms.filter((r) => !currentRooms.includes(r));
addedRooms.forEach((r) => {
logger.info(`Registering text stream handler for room `);
r.registerTextStreamHandler(
TEXT_LK_TOPIC,
(reader, participantInfo) =>
void lkTextStreamHandlerFunction(reader, participantInfo, r),
);
});
removedRooms.forEach((r) => {
logger.info(`Unregistering text stream handler for room `);
r.unregisterTextStreamHandler(TEXT_LK_TOPIC);
});
},
complete: () => {
logger.info("Livekit room items subscription completed");
for (const item of callViewModel.livekitRoomItems$.value) {
logger.info("unregistering room item from room", item.url);
item.livekitRoom.unregisterTextStreamHandler(TEXT_LK_TOPIC);
}
},
});
// create sendData function
const sendFn: Behavior<(data: string) => Promise<TextStreamInfo>> =
scope.behavior(
callViewModel.localMatrixLivekitMember$.pipe(
switchMap((m) => {
if (!m)
return of((data: string): never => {
throw Error("local membership not yet ready.");
});
return m.participant.value$.pipe(
map((p) => {
if (p === null) {
return (data: string): never => {
throw Error("local participant not yet ready to send data.");
};
} else {
return async (data: string): Promise<TextStreamInfo> =>
p.sendText(data, { topic: TEXT_LK_TOPIC });
}
}),
);
}),
),
);
const sendData = async (data: unknown): Promise<void> => {
const dataString = JSON.stringify(data);
logger.info("try sending: ", dataString);
try {
await Promise.resolve();
const info = await sendFn.value(dataString);
logger.info(`Sent text with stream ID: ${info.id}`);
} catch (e) {
logger.error("failed sending: ", dataString, e);
}
};
// after hangup gets called
const leaveSubs = callViewModel.leave$.subscribe(() => {
const scheduleWidgetCloseOnLeave = async (): Promise<void> => {
const leaveResolver = Promise.withResolvers<void>();
logger.info("waiting for RTC leave");
rtcSession.on(MatrixRTCSessionEvent.JoinStateChanged, (isJoined) => {
logger.info("received RTC join update: ", isJoined);
if (!isJoined) leaveResolver.resolve();
});
await leaveResolver.promise;
logger.info("send Unstick");
await widget.api
.setAlwaysOnScreen(false)
.catch((e) =>
logger.error(
"Failed to set call widget `alwaysOnScreen` to false",
e,
),
);
logger.info("send Close");
await widget.api.transport
.send(ElementWidgetActions.Close, {})
.catch((e) => logger.error("Failed to send close action", e));
};
// schedule close first and then leave (scope.end)
void scheduleWidgetCloseOnLeave();
// actual hangup (ending scope will send the leave event.. its kinda odd. since you might end up closing the widget too fast)
scope.end();
});
logger.info("createMatrixRTCSdk done");
return {
join: (): void => {
// first lets try making the widget sticky
tryMakeSticky();
callViewModel.join();
},
leave: (): void => {
callViewModel.hangup();
leaveSubs.unsubscribe();
livekitRoomItemsSub.unsubscribe();
},
data$,
connected$: callViewModel.connected$,
members$: scope.behavior(
callViewModel.matrixLivekitMembers$.pipe(
switchMap((members) => {
const listOfMemberObservables = members.map((member) =>
combineLatest([
member.connection$,
member.membership$,
member.participant.value$,
]).pipe(
map(([connection, membership, participant]) => ({
connection,
membership,
participant,
})),
// using shareReplay instead of a Behavior here because the behavior would need
// a tricky scope.end() setup.
shareReplay({ bufferSize: 1, refCount: true }),
),
);
return combineLatest(listOfMemberObservables);
}),
),
[],
),
sendData,
};
}

2
src/room/InCallView.tsx
View File

@@ -260,7 +260,7 @@ export const InCallView: FC<InCallViewProps> = ({
() => void toggleRaisedHand(),
);
const audioParticipants = useBehavior(vm.audioParticipants$);
const audioParticipants = useBehavior(vm.livekitRoomItems$);
const participantCount = useBehavior(vm.participantCount$);
const reconnecting = useBehavior(vm.reconnecting$);
const windowMode = useBehavior(vm.windowMode$);

64
src/state/CallViewModel/CallViewModel.ts
View File

@@ -80,7 +80,7 @@ import {
} from "../../reactions";
import { shallowEquals } from "../../utils/array";
import { type MediaDevices } from "../MediaDevices";
import { type Behavior } from "../Behavior";
import { constant, type Behavior } from "../Behavior";
import { E2eeType } from "../../e2ee/e2eeType";
import { MatrixKeyProvider } from "../../e2ee/matrixKeyProvider";
import { type MuteStates } from "../MuteStates";
@@ -117,6 +117,7 @@ import {
createMatrixLivekitMembers$,
type TaggedParticipant,
type LocalMatrixLivekitMember,
type RemoteMatrixLivekitMember,
} from "./remoteMembers/MatrixLivekitMembers.ts";
import {
type AutoLeaveReason,
@@ -156,7 +157,7 @@ export interface CallViewModelOptions {
/** Optional behavior overriding the computed window size, mainly for testing purposes. */
windowSize$?: Behavior<{ width: number; height: number }>;
/** The version & compatibility mode of MatrixRTC that we should use. */
matrixRTCMode$: Behavior<MatrixRTCMode>;
matrixRTCMode$?: Behavior<MatrixRTCMode>;
}
// Do not play any sounds if the participant count has exceeded this
@@ -182,7 +183,7 @@ interface LayoutScanState {
}
type MediaItem = UserMedia | ScreenShare;
type AudioLivekitItem = {
export type LivekitRoomItem = {
livekitRoom: LivekitRoom;
participants: string[];
url: string;
@@ -205,8 +206,11 @@ export interface CallViewModel {
callPickupState$: Behavior<
"unknown" | "ringing" | "timeout" | "decline" | "success" | null
>;
/** Observable that emits when the user should leave the call (hangup pressed, widget action, error).
* THIS DOES NOT LEAVE THE CALL YET. The only way to leave the call (send the hangup event) is by ending the scope.
*/
leave$: Observable<"user" | AutoLeaveReason>;
/** Call to initiate hangup. Use in conbination with connectino state track the async hangup process. */
/** Call to initiate hangup. Use in conbination with reconnectino state track the async hangup process. */
hangup: () => void;
// joining
@@ -258,7 +262,11 @@ export interface CallViewModel {
*/
participantCount$: Behavior<number>;
/** Participants sorted by livekit room so they can be used in the audio rendering */
audioParticipants$: Behavior<AudioLivekitItem[]>;
livekitRoomItems$: Behavior<LivekitRoomItem[]>;
userMedia$: Behavior<UserMedia[]>;
/** use the layout instead, this is just for the sdk export. */
matrixLivekitMembers$: Behavior<RemoteMatrixLivekitMember[]>;
localMatrixLivekitMember$: Behavior<LocalMatrixLivekitMember | null>;
/** List of participants raising their hand */
handsRaised$: Behavior<Record<string, RaisedHandInfo>>;
/** List of reactions. Keys are: membership.membershipId (currently predefined as: `${membershipEvent.userId}:${membershipEvent.deviceId}`)*/
@@ -341,17 +349,15 @@ export interface CallViewModel {
switch: () => void;
} | null>;
// connection state
/**
* Whether various media/event sources should pretend to be disconnected from
* all network input, even if their connection still technically works.
* Whether the app is currently reconnecting to the LiveKit server and/or setting the matrix rtc room state.
*/
// We do this when the app is in the 'reconnecting' state, because it might be
// that the LiveKit connection is still functional while the homeserver is
// down, for example, and we want to avoid making people worry that the app is
// in a split-brained state.
// DISCUSSION own membership manager ALSO this probably can be simplifis
reconnecting$: Behavior<boolean>;
/**
* Shortcut for not requireing to parse and combine connectionState.matrix and connectionState.livekit
*/
connected$: Behavior<boolean>;
}
/**
@@ -381,6 +387,8 @@ export function createCallViewModel$(
options.encryptionSystem,
matrixRTCSession,
);
const matrixRTCMode$ =
options.matrixRTCMode$ ?? constant(MatrixRTCMode.Legacy);
// Each hbar seperates a block of input variables required for the CallViewModel to function.
// The outputs of this block is written under the hbar.
@@ -413,7 +421,7 @@ export function createCallViewModel$(
client,
roomId: matrixRoom.roomId,
useOldestMember$: scope.behavior(
options.matrixRTCMode$.pipe(map((v) => v === MatrixRTCMode.Legacy)),
matrixRTCMode$.pipe(map((v) => v === MatrixRTCMode.Legacy)),
),
});
@@ -454,7 +462,7 @@ export function createCallViewModel$(
},
),
),
logger: logger,
logger,
});
const matrixLivekitMembers$ = createMatrixLivekitMembers$({
@@ -465,7 +473,7 @@ export function createCallViewModel$(
});
const connectOptions$ = scope.behavior(
options.matrixRTCMode$.pipe(
matrixRTCMode$.pipe(
map((mode) => ({
encryptMedia: livekitKeyProvider !== undefined,
// TODO. This might need to get called again on each change of matrixRTCMode...
@@ -497,7 +505,7 @@ export function createCallViewModel$(
muteStates,
trackProcessorState$,
logger.getChild(
"[Publisher" + connection.transport.livekit_service_url + "]",
"[Publisher " + connection.transport.livekit_service_url + "]",
),
);
},
@@ -596,8 +604,11 @@ export function createCallViewModel$(
),
);
const audioParticipants$ = scope.behavior(
const livekitRoomItems$ = scope.behavior(
matrixLivekitMembers$.pipe(
tap((val) => {
logger.debug("matrixLivekitMembers$ updated", val.value);
}),
switchMap((membersWithEpoch) => {
const members = membersWithEpoch.value;
const a$ = combineLatest(
@@ -622,7 +633,7 @@ export function createCallViewModel$(
return a$;
}),
map((members) =>
members.reduce<AudioLivekitItem[]>((acc, curr) => {
members.reduce<LivekitRoomItem[]>((acc, curr) => {
if (!curr) return acc;
const existing = acc.find((item) => item.url === curr.url);
@@ -1492,10 +1503,7 @@ export function createCallViewModel$(
),
null,
),
participantCount$: participantCount$,
audioParticipants$: audioParticipants$,
handsRaised$: handsRaised$,
reactions$: reactions$,
joinSoundEffect$: joinSoundEffect$,
@@ -1514,6 +1522,16 @@ export function createCallViewModel$(
spotlight$: spotlight$,
pip$: pip$,
layout$: layout$,
userMedia$,
localMatrixLivekitMember$,
matrixLivekitMembers$: scope.behavior(
matrixLivekitMembers$.pipe(
map((members) => members.value),
tap((v) => {
logger.debug("matrixLivekitMembers$ updated (exported)", v);
}),
),
),
tileStoreGeneration$: tileStoreGeneration$,
showSpotlightIndicators$: showSpotlightIndicators$,
showSpeakingIndicators$: showSpeakingIndicators$,
@@ -1522,6 +1540,8 @@ export function createCallViewModel$(
earpieceMode$: earpieceMode$,
audioOutputSwitcher$: audioOutputSwitcher$,
reconnecting$: localMembership.reconnecting$,
livekitRoomItems$,
connected$: localMembership.connected$,
};
}

9
src/state/CallViewModel/localMember/LocalMember.ts
View File

@@ -177,14 +177,18 @@ export const createLocalMembership$ = ({
// tracks$: Behavior<LocalTrack[]>;
participant$: Behavior<LocalParticipant | null>;
connection$: Behavior<Connection | null>;
/** Shorthand for homeserverConnected.rtcSession === Status.Reconnecting
* Direct translation to the js-sdk membership manager connection `Status`.
/**
* Tracks the homserver and livekit connected state and based on that computes reconnecting.
*/
reconnecting$: Behavior<boolean>;
/** Shorthand for homeserverConnected.rtcSession === Status.Disconnected
* Direct translation to the js-sdk membership manager connection `Status`.
*/
disconnected$: Behavior<boolean>;
/**
* Fully connected
*/
connected$: Behavior<boolean>;
} => {
const logger = parentLogger.getChild("[LocalMembership]");
logger.debug(`Creating local membership..`);
@@ -638,6 +642,7 @@ export const createLocalMembership$ = ({
localMemberState$,
participant$,
reconnecting$,
connected$: matrixAndLivekitConnected$,
disconnected$: scope.behavior(
homeserverConnected.rtsSession$.pipe(
map((state) => state === RTCSessionStatus.Disconnected),

19
src/state/CallViewModel/remoteMembers/Connection.test.ts
View File

@@ -392,7 +392,7 @@ describe("remote participants", () => {
// livekitRoom and the rtc membership in order to publish the members that are publishing
// on this connection.
const participants: RemoteParticipant[] = [
let participants: RemoteParticipant[] = [
mockRemoteParticipant({ identity: "@alice:example.org:DEV000" }),
mockRemoteParticipant({ identity: "@bob:example.org:DEV111" }),
mockRemoteParticipant({ identity: "@carol:example.org:DEV222" }),
@@ -414,7 +414,22 @@ describe("remote participants", () => {
fakeLivekitRoom.emit(RoomEvent.ParticipantConnected, p),
);
// All remote participants should be present
// At this point there should be ~~no~~ publishers
// We do have publisher now, since we do not filter for publishers anymore (to also have participants with only data tracks)
// The filtering we do is just based on the matrixRTC member events.
expect(observedParticipants.pop()!.length).toEqual(4);
participants = [
mockRemoteParticipant({ identity: "@alice:example.org:DEV000" }),
mockRemoteParticipant({ identity: "@bob:example.org:DEV111" }),
mockRemoteParticipant({ identity: "@carol:example.org:DEV222" }),
mockRemoteParticipant({ identity: "@dan:example.org:DEV333" }),
];
participants.forEach((p) =>
fakeLivekitRoom.emit(RoomEvent.ParticipantConnected, p),
);
// At this point there should be no publishers
expect(observedParticipants.pop()!.length).toEqual(4);
});

2
src/state/CallViewModel/remoteMembers/Connection.ts
View File

@@ -223,7 +223,7 @@ export class Connection {
public constructor(opts: ConnectionOpts, logger: Logger) {
this.logger = logger.getChild("[Connection]");
this.logger.info(
`[Connection] Creating new connection to ${opts.transport.livekit_service_url} ${opts.transport.livekit_alias}`,
`Creating new connection to ${opts.transport.livekit_service_url} ${opts.transport.livekit_alias}`,
);
const { transport, client, scope } = opts;

3
src/state/CallViewModel/remoteMembers/ConnectionFactory.ts
View File

@@ -14,7 +14,8 @@ import {
type BaseE2EEManager,
} from "livekit-client";
import { type Logger } from "matrix-js-sdk/lib/logger";
import E2EEWorker from "livekit-client/e2ee-worker?worker";
// imported as inline to support worker when loaded from a cdn (cross domain)
import E2EEWorker from "livekit-client/e2ee-worker?worker&inline";
import { type ObservableScope } from "../../ObservableScope.ts";
import { Connection } from "./Connection.ts";

52
src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts
View File

@@ -285,47 +285,47 @@ describe("connectionManagerData$ stream", () => {
a: expect.toSatisfy((e) => {
const data: ConnectionManagerData = e.value;
expect(data.getConnections().length).toBe(2);
expect(data.getParticipantForTransport(TRANSPORT_1).length).toBe(0);
expect(data.getParticipantForTransport(TRANSPORT_2).length).toBe(0);
expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(0);
expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(0);
return true;
}),
b: expect.toSatisfy((e) => {
const data: ConnectionManagerData = e.value;
expect(data.getConnections().length).toBe(2);
expect(data.getParticipantForTransport(TRANSPORT_1).length).toBe(1);
expect(data.getParticipantForTransport(TRANSPORT_2).length).toBe(0);
expect(data.getParticipantForTransport(TRANSPORT_1)[0].identity).toBe(
"user1A",
);
expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(1);
expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(0);
expect(
data.getParticipantsForTransport(TRANSPORT_1)[0].identity,
).toBe("user1A");
return true;
}),
c: expect.toSatisfy((e) => {
const data: ConnectionManagerData = e.value;
expect(data.getConnections().length).toBe(2);
expect(data.getParticipantForTransport(TRANSPORT_1).length).toBe(1);
expect(data.getParticipantForTransport(TRANSPORT_2).length).toBe(1);
expect(data.getParticipantForTransport(TRANSPORT_1)[0].identity).toBe(
"user1A",
);
expect(data.getParticipantForTransport(TRANSPORT_2)[0].identity).toBe(
"user2A",
);
expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(1);
expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(1);
expect(
data.getParticipantsForTransport(TRANSPORT_1)[0].identity,
).toBe("user1A");
expect(
data.getParticipantsForTransport(TRANSPORT_2)[0].identity,
).toBe("user2A");
return true;
}),
d: expect.toSatisfy((e) => {
const data: ConnectionManagerData = e.value;
expect(data.getConnections().length).toBe(2);
expect(data.getParticipantForTransport(TRANSPORT_1).length).toBe(2);
expect(data.getParticipantForTransport(TRANSPORT_2).length).toBe(1);
expect(data.getParticipantForTransport(TRANSPORT_1)[0].identity).toBe(
"user1A",
);
expect(data.getParticipantForTransport(TRANSPORT_1)[1].identity).toBe(
"user1B",
);
expect(data.getParticipantForTransport(TRANSPORT_2)[0].identity).toBe(
"user2A",
);
expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(2);
expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(1);
expect(
data.getParticipantsForTransport(TRANSPORT_1)[0].identity,
).toBe("user1A");
expect(
data.getParticipantsForTransport(TRANSPORT_1)[1].identity,
).toBe("user1B");
expect(
data.getParticipantsForTransport(TRANSPORT_2)[0].identity,
).toBe("user2A");
return true;
}),
});

23
src/state/CallViewModel/remoteMembers/ConnectionManager.ts
View File

@@ -19,8 +19,10 @@ import { areLivekitTransportsEqual } from "./MatrixLivekitMembers.ts";
import { type ConnectionFactory } from "./ConnectionFactory.ts";
export class ConnectionManagerData {
private readonly store: Map<string, [Connection, RemoteParticipant[]]> =
new Map();
private readonly store: Map<
string,
{ connection: Connection; participants: RemoteParticipant[] }
> = new Map();
public constructor() {}
@@ -28,9 +30,9 @@ export class ConnectionManagerData {
const key = this.getKey(connection.transport);
const existing = this.store.get(key);
if (!existing) {
this.store.set(key, [connection, participants]);
this.store.set(key, { connection, participants });
} else {
existing[1].push(...participants);
existing.participants.push(...participants);
}
}
@@ -39,20 +41,24 @@ export class ConnectionManagerData {
}
public getConnections(): Connection[] {
return Array.from(this.store.values()).map(([connection]) => connection);
return Array.from(this.store.values()).map(({ connection }) => connection);
}
public getConnectionForTransport(
transport: LivekitTransport,
): Connection | null {
return this.store.get(this.getKey(transport))?.[0] ?? null;
return this.store.get(this.getKey(transport))?.connection ?? null;
}
public getParticipantForTransport(
public getParticipantsForTransport(
transport: LivekitTransport,
): RemoteParticipant[] {
const key = transport.livekit_service_url + "|" + transport.livekit_alias;
return this.store.get(key)?.[1] ?? [];
const existing = this.store.get(key);
if (existing) {
return existing.participants;
}
return [];
}
}
@@ -162,6 +168,7 @@ export function createConnectionManager$({
);
// probably not required
if (listOfConnectionsWithRemoteParticipants.length === 0) {
return of(new Epoch(new ConnectionManagerData(), epoch));
}

121
src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts
View File

@@ -91,7 +91,7 @@ test("should signal participant not yet connected to livekit", () => {
}),
);
const matrixLivekitMember$ = createMatrixLivekitMembers$({
const matrixLivekitMembers$ = createMatrixLivekitMembers$({
scope: testScope,
membershipsWithTransport$: testScope.behavior(membershipsWithTransport$),
connectionManager: {
@@ -99,21 +99,24 @@ test("should signal participant not yet connected to livekit", () => {
} as unknown as IConnectionManager,
});
expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe("a", {
a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => {
expect(data.length).toEqual(1);
expectObservable(data[0].membership$).toBe("a", {
a: bobMembership,
});
expectObservable(data[0].participant.value$).toBe("a", {
a: null,
});
expectObservable(data[0].connection$).toBe("a", {
a: null,
});
return true;
}),
});
expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe(
"a",
{
a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => {
expect(data.length).toEqual(1);
expectObservable(data[0].membership$).toBe("a", {
a: bobMembership,
});
expectObservable(data[0].participant.value$).toBe("a", {
a: null,
});
expectObservable(data[0].connection$).toBe("a", {
a: null,
});
return true;
}),
},
);
});
});
@@ -171,7 +174,7 @@ test("should signal participant on a connection that is publishing", () => {
}),
);
const matrixLivekitMember$ = createMatrixLivekitMembers$({
const matrixLivekitMembers$ = createMatrixLivekitMembers$({
scope: testScope,
membershipsWithTransport$: testScope.behavior(membershipsWithTransport$),
connectionManager: {
@@ -179,25 +182,28 @@ test("should signal participant on a connection that is publishing", () => {
} as unknown as IConnectionManager,
});
expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe("a", {
a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => {
expect(data.length).toEqual(1);
expectObservable(data[0].membership$).toBe("a", {
a: bobMembership,
});
expectObservable(data[0].participant.value$).toBe("a", {
a: expect.toSatisfy((participant) => {
expect(participant).toBeDefined();
expect(participant!.identity).toEqual(bobParticipantId);
return true;
}),
});
expectObservable(data[0].connection$).toBe("a", {
a: connection,
});
return true;
}),
});
expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe(
"a",
{
a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => {
expect(data.length).toEqual(1);
expectObservable(data[0].membership$).toBe("a", {
a: bobMembership,
});
expectObservable(data[0].participant.value$).toBe("a", {
a: expect.toSatisfy((participant) => {
expect(participant).toBeDefined();
expect(participant!.identity).toEqual(bobParticipantId);
return true;
}),
});
expectObservable(data[0].connection$).toBe("a", {
a: connection,
});
return true;
}),
},
);
});
});
@@ -222,7 +228,7 @@ test("should signal participant on a connection that is not publishing", () => {
}),
);
const matrixLivekitMember$ = createMatrixLivekitMembers$({
const matrixLivekitMembers$ = createMatrixLivekitMembers$({
scope: testScope,
membershipsWithTransport$: testScope.behavior(membershipsWithTransport$),
connectionManager: {
@@ -230,21 +236,24 @@ test("should signal participant on a connection that is not publishing", () => {
} as unknown as IConnectionManager,
});
expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe("a", {
a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => {
expect(data.length).toEqual(1);
expectObservable(data[0].membership$).toBe("a", {
a: bobMembership,
});
expectObservable(data[0].participant.value$).toBe("a", {
a: null,
});
expectObservable(data[0].connection$).toBe("a", {
a: connection,
});
return true;
}),
});
expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe(
"a",
{
a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => {
expect(data.length).toEqual(1);
expectObservable(data[0].membership$).toBe("a", {
a: bobMembership,
});
expectObservable(data[0].participant.value$).toBe("a", {
a: null,
});
expectObservable(data[0].connection$).toBe("a", {
a: connection,
});
return true;
}),
},
);
});
});
@@ -283,7 +292,7 @@ describe("Publication edge case", () => {
}),
);
const matrixLivekitMember$ = createMatrixLivekitMembers$({
const matrixLivekitMembers$ = createMatrixLivekitMembers$({
scope: testScope,
membershipsWithTransport$: testScope.behavior(
membershipsWithTransport$,
@@ -293,7 +302,7 @@ describe("Publication edge case", () => {
} as unknown as IConnectionManager,
});
expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe(
expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe(
"a",
{
a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => {
@@ -349,7 +358,7 @@ describe("Publication edge case", () => {
}),
);
const matrixLivekitMember$ = createMatrixLivekitMembers$({
const matrixLivekitMembers$ = createMatrixLivekitMembers$({
scope: testScope,
membershipsWithTransport$: testScope.behavior(
membershipsWithTransport$,
@@ -359,7 +368,7 @@ describe("Publication edge case", () => {
} as unknown as IConnectionManager,
});
expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe(
expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe(
"a",
{
a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => {

2
src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts
View File

@@ -110,7 +110,7 @@ export function createMatrixLivekitMembers$({
const participantId = /*membership.membershipID*/ `${membership.userId}:${membership.deviceId}`;
const participants = transport
? managerData.getParticipantForTransport(transport)
? managerData.getParticipantsForTransport(transport)
: [];
const participant =
participants.find((p) => p.identity == participantId) ?? null;

4
src/state/CallViewModel/remoteMembers/integration.test.ts
View File

@@ -124,14 +124,14 @@ test("bob, carl, then bob joining no tracks yet", () => {
logger: logger,
});
const matrixLivekitItems$ = createMatrixLivekitMembers$({
const matrixLivekitMembers$ = createMatrixLivekitMembers$({
scope: testScope,
membershipsWithTransport$:
membershipsAndTransports.membershipsWithTransport$,
connectionManager,
});
expectObservable(matrixLivekitItems$).toBe(vMarble, {
expectObservable(matrixLivekitMembers$).toBe(vMarble, {
a: expect.toSatisfy((e: Epoch<RemoteMatrixLivekitMember[]>) => {
const items = e.value;
expect(items.length).toBe(1);

19
src/widget.ts
View File

@@ -64,6 +64,12 @@ export const widget = ((): WidgetHelpers | null => {
try {
const { widgetId, parentUrl } = getUrlParams();
const { roomId, userId, deviceId, baseUrl, e2eEnabled, allowIceFallback } =
getUrlParams();
if (!roomId) throw new Error("Room ID must be supplied");
if (!userId) throw new Error("User ID must be supplied");
if (!deviceId) throw new Error("Device ID must be supplied");
if (!baseUrl) throw new Error("Base URL must be supplied");
if (widgetId && parentUrl) {
const parentOrigin = new URL(parentUrl).origin;
logger.info("Widget API is available");
@@ -92,19 +98,6 @@ export const widget = ((): WidgetHelpers | null => {
// We need to do this now rather than later because it has capabilities to
// request, and is responsible for starting the transport (should it be?)
const {
roomId,
userId,
deviceId,
baseUrl,
e2eEnabled,
allowIceFallback,
} = getUrlParams();
if (!roomId) throw new Error("Room ID must be supplied");
if (!userId) throw new Error("User ID must be supplied");
if (!deviceId) throw new Error("Device ID must be supplied");
if (!baseUrl) throw new Error("Base URL must be supplied");
// These are all the event types the app uses
const sendEvent = [
EventType.CallNotify, // Sent as a deprecated fallback

7
tsconfig.json
View File

@@ -50,6 +50,11 @@
"plugins": [{ "name": "typescript-eslint-language-service" }]
},
"include": ["./src/**/*.ts", "./src/**/*.tsx", "./playwright/**/*.ts"],
"include": [
"./src/**/*.ts",
"./src/**/*.tsx",
"./playwright/**/*.ts",
"./sdk/**/*.ts"
],
"exclude": ["**.test.ts"]
}

28
vite-sdk.config.ts Normal file
View File

@@ -0,0 +1,28 @@
/*
Copyright 2025 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { defineConfig, mergeConfig } from "vite";
import nodePolyfills from "vite-plugin-node-stdlib-browser";
const base = "./";
// Config for embedded deployments (possibly hosted under a non-root path)
export default defineConfig(() => ({
worker: { format: "es" as const },
base, // Use relative URLs to allow the app to be hosted under any path
build: {
sourcemap: true,
manifest: true,
lib: {
formats: ["es" as const],
entry: "./sdk/main.ts",
name: "MatrixrtcSdk",
fileName: "matrixrtc-sdk",
},
},
plugins: [nodePolyfills()],
}));

27
vite.config.ts
View File

@@ -7,14 +7,17 @@ Please see LICENSE in the repository root for full details.
import {
loadEnv,
PluginOption,
searchForWorkspaceRoot,
type ConfigEnv,
type UserConfig,
} from "vite";
import svgrPlugin from "vite-plugin-svgr";
import { createHtmlPlugin } from "vite-plugin-html";
import { codecovVitePlugin } from "@codecov/vite-plugin";
import { sentryVitePlugin } from "@sentry/vite-plugin";
import react from "@vitejs/plugin-react";
import { realpathSync } from "fs";
import * as fs from "node:fs";
@@ -31,7 +34,7 @@ export default ({
// In future we might be able to do what is needed via code splitting at
// build time.
process.env.VITE_PACKAGE = packageType ?? "full";
const plugins = [
const plugins: PluginOption[] = [
react(),
svgrPlugin({
svgrOptions: {
@@ -41,16 +44,6 @@ export default ({
},
}),
createHtmlPlugin({
entry: "src/main.tsx",
inject: {
data: {
brand: env.VITE_PRODUCT_NAME || "Element Call",
packageType: process.env.VITE_PACKAGE,
},
},
}),
codecovVitePlugin({
enableBundleAnalysis: process.env.CODECOV_TOKEN !== undefined,
bundleName: "element-call",
@@ -73,6 +66,18 @@ export default ({
);
}
plugins.push(
createHtmlPlugin({
entry: "src/main.tsx",
inject: {
data: {
brand: env.VITE_PRODUCT_NAME || "Element Call",
packageType: process.env.VITE_PACKAGE,
},
},
}),
);
// The crypto WASM module is imported dynamically. Since it's common
// for developers to use a linked copy of matrix-js-sdk or Rust
// crypto (which could reside anywhere on their file system), Vite

867
yarn.lock
View File

File diff suppressed because it is too large Load Diff