Matrix/element-call-Github
1
0
Fork 0
mirror of https://github.com/vector-im/element-call.git synced 2026-01-18 02:32:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'livekit' into toger5/lib-ec-version

Browse Source
This commit is contained in:
Timo K
2025-12-22 12:43:09 +01:00
parent c1c73b0f02 6cfa95bcfa
commit e78f37a6b3
39 changed files with 1978 additions and 1496 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
package.json
View File

@@ -56,7 +56,7 @@
"@opentelemetry/sdk-trace-base": "^2.0.0",
"@opentelemetry/sdk-trace-web": "^2.0.0",
"@opentelemetry/semantic-conventions": "^1.25.1",
"@playwright/test": "^1.56.1",
"@playwright/test": "^1.57.0",
"@radix-ui/react-dialog": "^1.0.4",
"@radix-ui/react-slider": "^1.1.2",
"@radix-ui/react-visually-hidden": "^1.0.3",

28
playwright/fixtures/widget-user.ts
View File

@@ -111,19 +111,27 @@ async function registerUser(
await page.getByRole("textbox", { name: "Confirm password" }).click();
await page.getByRole("textbox", { name: "Confirm password" }).fill(PASSWORD);
await page.getByRole("button", { name: "Register" }).click();
const continueButton = page.getByRole("button", { name: "Continue" });
try {
await expect(continueButton).toBeVisible({ timeout: 5000 });
await page
.getByRole("textbox", { name: "Password", exact: true })
.fill(PASSWORD);
await continueButton.click();
} catch {
// continueButton not visible, continue as normal
}
await expect(
page.getByRole("heading", { name: `Welcome ${username}` }),
).toBeVisible();
const browserUnsupportedToast = page
.getByText("Element does not support this browser")
.locator("..")
.locator("..");
// Dismiss incompatible browser toast
const dismissButton = browserUnsupportedToast.getByRole("button", {
name: "Dismiss",
});
try {
await expect(dismissButton).toBeVisible({ timeout: 700 });
await dismissButton.click();
} catch {
// dismissButton not visible, continue as normal
}
await setDevToolElementCallDevUrl(page);
const clientHandle = await page.evaluateHandle(() =>

2
src/room/GroupCallView.tsx
View File

@@ -160,6 +160,7 @@ export const GroupCallView: FC<Props> = ({
}, [rtcSession]);
// TODO move this into the callViewModel LocalMembership.ts
// We might actually not need this at all. Since we get into fatalError on those errors already?
useTypedEventEmitter(
rtcSession,
MatrixRTCSessionEvent.MembershipManagerError,
@@ -313,6 +314,7 @@ export const GroupCallView: FC<Props> = ({
const navigate = useNavigate();
// TODO split this into leave and onDisconnect
const onLeft = useCallback(
(
reason: "timeout" | "user" | "allOthersLeft" | "decline" | "error",

14
src/room/InCallView.tsx
View File

@@ -24,7 +24,7 @@ import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc";
import classNames from "classnames";
import { BehaviorSubject, map } from "rxjs";
import { useObservable } from "observable-hooks";
import { logger } from "matrix-js-sdk/lib/logger";
import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
import {
VoiceCallSolidIcon,
VolumeOnSolidIcon,
@@ -88,6 +88,7 @@ import { ReactionsOverlay } from "./ReactionsOverlay";
import { CallEventAudioRenderer } from "./CallEventAudioRenderer";
import {
debugTileLayout as debugTileLayoutSetting,
matrixRTCMode as matrixRTCModeSetting,
useSetting,
} from "../settings/settings";
import { ReactionsReader } from "../reactions/ReactionsReader";
@@ -109,6 +110,8 @@ import { useTrackProcessorObservable$ } from "../livekit/TrackProcessorContext.t
import { type Layout } from "../state/layout-types.ts";
import { ObservableScope } from "../state/ObservableScope.ts";
const logger = rootLogger.getChild("[InCallView]");
const maxTapDurationMs = 400;
export interface ActiveCallProps
@@ -127,6 +130,7 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
const mediaDevices = useMediaDevices();
const trackProcessorState$ = useTrackProcessorObservable$();
useEffect(() => {
logger.info("START CALL VIEW SCOPE");
const scope = new ObservableScope();
const reactionsReader = new ReactionsReader(scope, props.rtcSession);
const { autoLeaveWhenOthersLeft, waitForCallPickup, sendNotificationType } =
@@ -141,6 +145,7 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
encryptionSystem: props.e2eeSystem,
autoLeaveWhenOthersLeft,
waitForCallPickup: waitForCallPickup && sendNotificationType === "ring",
matrixRTCMode$: matrixRTCModeSetting.value$,
},
reactionsReader.raisedHands$,
reactionsReader.reactions$,
@@ -151,7 +156,9 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
setVm(vm);
vm.leave$.pipe(scope.bind()).subscribe(props.onLeft);
return (): void => {
logger.info("END CALL VIEW SCOPE");
scope.end();
};
}, [
@@ -270,7 +277,10 @@ export const InCallView: FC<InCallViewProps> = ({
const ringOverlay = useBehavior(vm.ringOverlay$);
const fatalCallError = useBehavior(vm.fatalError$);
// Stop the rendering and throw for the error boundary
if (fatalCallError) throw fatalCallError;
if (fatalCallError) {
logger.debug("fatalCallError stop rendering", fatalCallError);
throw fatalCallError;
}
// We need to set the proper timings on the animation based upon the sound length.
const ringDuration = pickupPhaseAudio?.soundDuration["waiting"] ?? 1;

4
src/room/LobbyView.tsx
View File

@@ -79,9 +79,9 @@ export const LobbyView: FC<Props> = ({
waitingForInvite,
}) => {
useEffect(() => {
logger.info("[Lifecycle] GroupCallView Component mounted");
logger.info("[Lifecycle] LobbyView Component mounted");
return (): void => {
logger.info("[Lifecycle] GroupCallView Component unmounted");
logger.info("[Lifecycle] LobbyView Component unmounted");
};
}, []);

11
src/state/CallViewModel/CallViewModel.test.ts
View File

@@ -60,7 +60,8 @@ import {
import { MediaDevices } from "../MediaDevices.ts";
import { getValue } from "../../utils/observable.ts";
import { type Behavior, constant } from "../Behavior.ts";
import { withCallViewModel } from "./CallViewModelTestUtils.ts";
import { withCallViewModel as withCallViewModelInMode } from "./CallViewModelTestUtils.ts";
import { MatrixRTCMode } from "../../settings/settings.ts";
vi.mock("rxjs", async (importOriginal) => ({
...(await importOriginal()),
@@ -229,7 +230,13 @@ function mockRingEvent(
// need a value to fill in for them when emitting notifications
const mockLegacyRingEvent = {} as { event_id: string } & ICallNotifyContent;
describe("CallViewModel", () => {
describe.each([
[MatrixRTCMode.Legacy],
[MatrixRTCMode.Compatibil],
[MatrixRTCMode.Matrix_2_0],
])("CallViewModel (%s mode)", (mode) => {
const withCallViewModel = withCallViewModelInMode(mode);
test("participants are retained during a focus switch", () => {
withTestScheduler(({ behavior, expectObservable }) => {
// Participants disappear on frame 2 and come back on frame 3

179
src/state/CallViewModel/CallViewModel.ts
View File

@@ -16,6 +16,7 @@ import {
} from "livekit-client";
import { type Room as MatrixRoom } from "matrix-js-sdk";
import {
catchError,
combineLatest,
distinctUntilChanged,
filter,
@@ -53,11 +54,15 @@ import {
ScreenShareViewModel,
type UserMediaViewModel,
} from "../MediaViewModel";
import { accumulate, generateItems, pauseWhen } from "../../utils/observable";
import {
accumulate,
filterBehavior,
generateItems,
pauseWhen,
} from "../../utils/observable";
import {
duplicateTiles,
MatrixRTCMode,
matrixRTCMode,
playReactionsSound,
showReactions,
} from "../../settings/settings";
@@ -94,7 +99,7 @@ import {
type SpotlightLandscapeLayoutMedia,
type SpotlightPortraitLayoutMedia,
} from "../layout-types.ts";
import { type ElementCallError } from "../../utils/errors.ts";
import { ElementCallError } from "../../utils/errors.ts";
import { type ObservableScope } from "../ObservableScope.ts";
import { createHomeserverConnected$ } from "./localMember/HomeserverConnected.ts";
import {
@@ -112,7 +117,8 @@ import { ECConnectionFactory } from "./remoteMembers/ConnectionFactory.ts";
import { createConnectionManager$ } from "./remoteMembers/ConnectionManager.ts";
import {
createMatrixLivekitMembers$,
type MatrixLivekitMember,
type TaggedParticipant,
type LocalMatrixLivekitMember,
} from "./remoteMembers/MatrixLivekitMembers.ts";
import {
type AutoLeaveReason,
@@ -151,6 +157,8 @@ export interface CallViewModelOptions {
connectionState$?: Behavior<ConnectionState>;
/** Optional behavior overriding the computed window size, mainly for testing purposes. */
windowSize$?: Behavior<{ width: number; height: number }>;
/** The version & compatibility mode of MatrixRTC that we should use. */
matrixRTCMode$: Behavior<MatrixRTCMode>;
}
// Do not play any sounds if the participant count has exceeded this
@@ -430,7 +438,7 @@ export function createCallViewModel$(
client,
roomId: matrixRoom.roomId,
useOldestMember$: scope.behavior(
matrixRTCMode.value$.pipe(map((v) => v === MatrixRTCMode.Legacy)),
options.matrixRTCMode$.pipe(map((v) => v === MatrixRTCMode.Legacy)),
),
});
@@ -450,7 +458,18 @@ export function createCallViewModel$(
connectionFactory: connectionFactory,
inputTransports$: scope.behavior(
combineLatest(
[localTransport$, membershipsAndTransports.transports$],
[
localTransport$.pipe(
catchError((e: unknown) => {
logger.info(
"dont pass local transport to createConnectionManager$. localTransport$ threw an error",
e,
);
return of(null);
}),
),
membershipsAndTransports.transports$,
],
(localTransport, transports) => {
const localTransportAsArray = localTransport ? [localTransport] : [];
return transports.mapInner((transports) => [
@@ -471,7 +490,7 @@ export function createCallViewModel$(
});
const connectOptions$ = scope.behavior(
matrixRTCMode.value$.pipe(
options.matrixRTCMode$.pipe(
map((mode) => ({
encryptMedia: livekitKeyProvider !== undefined,
// TODO. This might need to get called again on each change of matrixRTCMode...
@@ -482,13 +501,13 @@ export function createCallViewModel$(
const localMembership = createLocalMembership$({
scope: scope,
homeserverConnected$: createHomeserverConnected$(
homeserverConnected: createHomeserverConnected$(
scope,
client,
matrixRTCSession,
),
muteStates: muteStates,
joinMatrixRTC: async (transport: LivekitTransport) => {
joinMatrixRTC: (transport: LivekitTransport) => {
return enterRTCSession(
matrixRTCSession,
transport,
@@ -525,22 +544,21 @@ export function createCallViewModel$(
),
);
const localMatrixLivekitMemberUninitialized = {
membership$: localRtcMembership$,
participant$: localMembership.participant$,
connection$: localMembership.connection$,
userId: userId,
};
const localMatrixLivekitMember$: Behavior<LocalMatrixLivekitMember | null> =
scope.behavior(
localRtcMembership$.pipe(
switchMap((membership) => {
if (!membership) return of(null);
return of(
// casting is save here since we know that localRtcMembership$ is !== null since we reached this case.
localMatrixLivekitMemberUninitialized as LocalMatrixLivekitMember,
);
filterBehavior((membership) => membership !== null),
map((membership$) => {
if (membership$ === null) return null;
return {
membership$,
participant: {
type: "local" as const,
value$: localMembership.participant$,
},
connection$: localMembership.connection$,
userId,
};
}),
),
);
@@ -612,7 +630,7 @@ export function createCallViewModel$(
const members = membersWithEpoch.value;
const a$ = combineLatest(
members.map((member) =>
combineLatest([member.connection$, member.participant$]).pipe(
combineLatest([member.connection$, member.participant.value$]).pipe(
map(([connection, participant]) => {
// do not render audio for local participant
if (!connection || !participant || participant.isLocal)
@@ -696,7 +714,7 @@ export function createCallViewModel$(
let localParticipantId: string | undefined = undefined;
// add local member if available
if (localMatrixLivekitMember) {
const { userId, participant$, connection$, membership$ } =
const { userId, participant, connection$, membership$ } =
localMatrixLivekitMember;
localParticipantId = `${userId}:${membership$.value.deviceId}`; // should be membership$.value.membershipID which is not optional
// const participantId = membership$.value.membershipID;
@@ -707,7 +725,7 @@ export function createCallViewModel$(
dup,
localParticipantId,
userId,
participant$,
participant satisfies TaggedParticipant as TaggedParticipant, // Widen the type safely
connection$,
],
data: undefined,
@@ -718,7 +736,7 @@ export function createCallViewModel$(
// add remote members that are available
for (const {
userId,
participant$,
participant,
connection$,
membership$,
} of matrixLivekitMembers) {
@@ -727,7 +745,7 @@ export function createCallViewModel$(
// const participantId = membership$.value?.identity;
for (let dup = 0; dup < 1 + duplicateTiles; dup++) {
yield {
keys: [dup, participantId, userId, participant$, connection$],
keys: [dup, participantId, userId, participant, connection$],
data: undefined,
};
}
@@ -739,7 +757,7 @@ export function createCallViewModel$(
dup,
participantId,
userId,
participant$,
participant,
connection$,
) => {
const livekitRoom$ = scope.behavior(
@@ -758,7 +776,7 @@ export function createCallViewModel$(
scope,
`${participantId}:${dup}`,
userId,
participant$,
participant,
options.encryptionSystem,
livekitRoom$,
focusUrl$,
@@ -968,11 +986,12 @@ export function createCallViewModel$(
),
);
const hasRemoteScreenShares$: Observable<boolean> = spotlight$.pipe(
map((spotlight) =>
spotlight.some((vm) => !vm.local && vm instanceof ScreenShareViewModel),
const hasRemoteScreenShares$ = scope.behavior<boolean>(
spotlight$.pipe(
map((spotlight) =>
spotlight.some((vm) => !vm.local && vm instanceof ScreenShareViewModel),
),
),
distinctUntilChanged(),
);
const pipEnabled$ = scope.behavior(setPipEnabled$, false);
@@ -1446,15 +1465,46 @@ export function createCallViewModel$(
// reassigned here to make it publicly accessible
const toggleScreenSharing = localMembership.toggleScreenSharing;
const errors$ = scope.behavior<{
transportError?: ElementCallError;
matrixError?: ElementCallError;
connectionError?: ElementCallError;
publishError?: ElementCallError;
} | null>(
localMembership.localMemberState$.pipe(
map((value) => {
const returnObject: {
transportError?: ElementCallError;
matrixError?: ElementCallError;
connectionError?: ElementCallError;
publishError?: ElementCallError;
} = {};
if (value instanceof ElementCallError) return { transportError: value };
if (value === TransportState.Waiting) return null;
if (value.matrix instanceof ElementCallError)
returnObject.matrixError = value.matrix;
if (value.media instanceof ElementCallError)
returnObject.publishError = value.media;
else if (
typeof value.media === "object" &&
value.media.connection instanceof ElementCallError
)
returnObject.connectionError = value.media.connection;
return returnObject;
}),
),
null,
);
return {
autoLeave$,
callPickupState$,
ringOverlay$,
leave$,
hangup: (): void => userHangup$.next(),
join: localMembership.requestConnect,
toggleScreenSharing,
sharingScreen$,
join: localMembership.requestJoinAndPublish,
toggleScreenSharing: toggleScreenSharing,
sharingScreen$: sharingScreen$,
tapScreen: (): void => screenTap$.next(),
tapControls: (): void => controlsTap$.next(),
@@ -1462,9 +1512,17 @@ export function createCallViewModel$(
unhoverScreen: (): void => screenUnhover$.next(),
fatalError$: scope.behavior(
localMembership.connectionState.livekit$.pipe(
filter((v) => v.state === RTCBackendState.Error),
map((s) => s.error),
errors$.pipe(
map((errors) => {
logger.debug("errors$ to compute any fatal errors:", errors);
return (
errors?.transportError ??
errors?.matrixError ??
errors?.connectionError ??
null
);
}),
filter((error) => error !== null),
),
null,
),
@@ -1480,15 +1538,24 @@ export function createCallViewModel$(
audibleReactions$,
visibleReactions$,
windowMode$,
spotlightExpanded$,
toggleSpotlightExpanded$,
gridMode$,
setGridMode,
grid$,
spotlight$,
pip$,
layout$,
handsRaised$: handsRaised$,
reactions$: reactions$,
joinSoundEffect$: joinSoundEffect$,
leaveSoundEffect$: leaveSoundEffect$,
newHandRaised$: newHandRaised$,
newScreenShare$: newScreenShare$,
audibleReactions$: audibleReactions$,
visibleReactions$: visibleReactions$,
windowMode$: windowMode$,
spotlightExpanded$: spotlightExpanded$,
toggleSpotlightExpanded$: toggleSpotlightExpanded$,
gridMode$: gridMode$,
setGridMode: setGridMode,
grid$: grid$,
spotlight$: spotlight$,
pip$: pip$,
layout$: layout$,
userMedia$,
localMatrixLivekitMember$,
matrixLivekitMembers$: scope.behavior(
@@ -1499,16 +1566,14 @@ export function createCallViewModel$(
}),
),
),
tileStoreGeneration$,
showSpotlightIndicators$,
showSpeakingIndicators$,
showHeader$,
showFooter$,
earpieceMode$,
audioOutputSwitcher$,
tileStoreGeneration$: tileStoreGeneration$,
showSpotlightIndicators$: showSpotlightIndicators$,
showSpeakingIndicators$: showSpeakingIndicators$,
showHeader$: showHeader$,
showFooter$: showFooter$,
earpieceMode$: earpieceMode$,
audioOutputSwitcher$: audioOutputSwitcher$,
reconnecting$: localMembership.reconnecting$,
connected$: localMembership.connected$,
connectionState: localMembership.connectionState,
};
}

223
src/state/CallViewModel/CallViewModelTestUtils.ts
View File

@@ -53,6 +53,7 @@ import {
import { type Behavior, constant } from "../Behavior";
import { type ProcessorState } from "../../livekit/TrackProcessorContext";
import { type MediaDevices } from "../MediaDevices";
import { type MatrixRTCMode } from "../../settings/settings";
mockConfig({
livekit: { livekit_service_url: "http://my-default-service-url.com" },
@@ -80,117 +81,125 @@ export interface CallViewModelInputs {
const localParticipant = mockLocalParticipant({ identity: "" });
export function withCallViewModel(
{
remoteParticipants$ = constant([]),
rtcMembers$ = constant([localRtcMember]),
livekitConnectionState$: connectionState$ = constant(
ConnectionState.Connected,
),
speaking = new Map(),
mediaDevices = mockMediaDevices({}),
initialSyncState = SyncState.Syncing,
windowSize$ = constant({ width: 1000, height: 800 }),
}: Partial<CallViewModelInputs> = {},
continuation: (
vm: CallViewModel,
rtcSession: MockRTCSession,
subjects: { raisedHands$: BehaviorSubject<Record<string, RaisedHandInfo>> },
setSyncState: (value: SyncState) => void,
) => void,
options: CallViewModelOptions = {
encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
autoLeaveWhenOthersLeft: false,
},
): void {
let syncState = initialSyncState;
const setSyncState = (value: SyncState): void => {
const prev = syncState;
syncState = value;
room.client.emit(ClientEvent.Sync, value, prev);
};
const room = mockMatrixRoom({
client: new (class extends EventEmitter {
public getUserId(): string | undefined {
return localRtcMember.userId;
}
export function withCallViewModel(mode: MatrixRTCMode) {
return (
{
remoteParticipants$ = constant([]),
rtcMembers$ = constant([localRtcMember]),
livekitConnectionState$: connectionState$ = constant(
ConnectionState.Connected,
),
speaking = new Map(),
mediaDevices = mockMediaDevices({}),
initialSyncState = SyncState.Syncing,
windowSize$ = constant({ width: 1000, height: 800 }),
}: Partial<CallViewModelInputs> = {},
continuation: (
vm: CallViewModel,
rtcSession: MockRTCSession,
subjects: {
raisedHands$: BehaviorSubject<Record<string, RaisedHandInfo>>;
},
setSyncState: (value: SyncState) => void,
) => void,
options: Partial<CallViewModelOptions> = {},
): void => {
let syncState = initialSyncState;
const setSyncState = (value: SyncState): void => {
const prev = syncState;
syncState = value;
room.client.emit(ClientEvent.Sync, value, prev);
};
const room = mockMatrixRoom({
client: new (class extends EventEmitter {
public getUserId(): string | undefined {
return localRtcMember.userId;
}
public getDeviceId(): string {
return localRtcMember.deviceId;
}
public getDeviceId(): string {
return localRtcMember.deviceId;
}
public getDomain(): string {
return "example.com";
}
public getDomain(): string {
return "example.com";
}
public getSyncState(): SyncState {
return syncState;
}
})() as Partial<MatrixClient> as MatrixClient,
getMembers: () => Array.from(roomMembers.values()),
getMembersWithMembership: () => Array.from(roomMembers.values()),
});
const rtcSession = new MockRTCSession(room, []).withMemberships(rtcMembers$);
const participantsSpy = vi
.spyOn(ComponentsCore, "connectedParticipantsObserver")
.mockReturnValue(remoteParticipants$);
const mediaSpy = vi
.spyOn(ComponentsCore, "observeParticipantMedia")
.mockImplementation((p) =>
of({ participant: p } as Partial<
ComponentsCore.ParticipantMedia<LocalParticipant>
> as ComponentsCore.ParticipantMedia<LocalParticipant>),
public getSyncState(): SyncState {
return syncState;
}
})() as Partial<MatrixClient> as MatrixClient,
getMembers: () => Array.from(roomMembers.values()),
getMembersWithMembership: () => Array.from(roomMembers.values()),
});
const rtcSession = new MockRTCSession(room, []).withMemberships(
rtcMembers$,
);
const eventsSpy = vi
.spyOn(ComponentsCore, "observeParticipantEvents")
.mockImplementation((p, ...eventTypes) => {
if (eventTypes.includes(ParticipantEvent.IsSpeakingChanged)) {
return (speaking.get(p) ?? of(false)).pipe(
map((s): Participant => ({ ...p, isSpeaking: s }) as Participant),
);
} else {
return of(p);
}
const participantsSpy = vi
.spyOn(ComponentsCore, "connectedParticipantsObserver")
.mockReturnValue(remoteParticipants$);
const mediaSpy = vi
.spyOn(ComponentsCore, "observeParticipantMedia")
.mockImplementation((p) =>
of({ participant: p } as Partial<
ComponentsCore.ParticipantMedia<LocalParticipant>
> as ComponentsCore.ParticipantMedia<LocalParticipant>),
);
const eventsSpy = vi
.spyOn(ComponentsCore, "observeParticipantEvents")
.mockImplementation((p, ...eventTypes) => {
if (eventTypes.includes(ParticipantEvent.IsSpeakingChanged)) {
return (speaking.get(p) ?? of(false)).pipe(
map((s): Participant => ({ ...p, isSpeaking: s }) as Participant),
);
} else {
return of(p);
}
});
const roomEventSelectorSpy = vi
.spyOn(ComponentsCore, "roomEventSelector")
.mockImplementation((_room, _eventType) => of());
const muteStates = mockMuteStates();
const raisedHands$ = new BehaviorSubject<Record<string, RaisedHandInfo>>(
{},
);
const reactions$ = new BehaviorSubject<Record<string, ReactionInfo>>({});
const vm = createCallViewModel$(
testScope(),
rtcSession.asMockedSession(),
room,
mediaDevices,
muteStates,
{
encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
autoLeaveWhenOthersLeft: false,
livekitRoomFactory: (): LivekitRoom =>
mockLivekitRoom({
localParticipant,
disconnect: async () => Promise.resolve(),
setE2EEEnabled: async () => Promise.resolve(),
}),
connectionState$,
windowSize$,
matrixRTCMode$: constant(mode),
...options,
},
raisedHands$,
reactions$,
new BehaviorSubject<ProcessorState>({
processor: undefined,
supported: undefined,
}),
);
onTestFinished(() => {
participantsSpy.mockRestore();
mediaSpy.mockRestore();
eventsSpy.mockRestore();
roomEventSelectorSpy.mockRestore();
});
const roomEventSelectorSpy = vi
.spyOn(ComponentsCore, "roomEventSelector")
.mockImplementation((_room, _eventType) => of());
const muteStates = mockMuteStates();
const raisedHands$ = new BehaviorSubject<Record<string, RaisedHandInfo>>({});
const reactions$ = new BehaviorSubject<Record<string, ReactionInfo>>({});
const vm = createCallViewModel$(
testScope(),
rtcSession.asMockedSession(),
room,
mediaDevices,
muteStates,
{
...options,
livekitRoomFactory: (): LivekitRoom =>
mockLivekitRoom({
localParticipant,
disconnect: async () => Promise.resolve(),
setE2EEEnabled: async () => Promise.resolve(),
}),
connectionState$,
windowSize$,
},
raisedHands$,
reactions$,
new BehaviorSubject<ProcessorState>({
processor: undefined,
supported: undefined,
}),
);
onTestFinished(() => {
participantsSpy.mockRestore();
mediaSpy.mockRestore();
eventsSpy.mockRestore();
roomEventSelectorSpy.mockRestore();
});
continuation(vm, rtcSession, { raisedHands$: raisedHands$ }, setSyncState);
continuation(vm, rtcSession, { raisedHands$: raisedHands$ }, setSyncState);
};
}

290
src/state/CallViewModel/LayoutSwitch.test.ts
View File

@@ -5,198 +5,128 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { afterEach, beforeEach, describe, expect, test } from "vitest";
import { firstValueFrom, of } from "rxjs";
import { describe, test } from "vitest";
import { createLayoutModeSwitch } from "./LayoutSwitch";
import { ObservableScope } from "../ObservableScope";
import { constant } from "../Behavior";
import { withTestScheduler } from "../../utils/test";
import { testScope, withTestScheduler } from "../../utils/test";
let scope: ObservableScope;
beforeEach(() => {
scope = new ObservableScope();
});
afterEach(() => {
scope.end();
});
describe("Default mode", () => {
test("Should be in grid layout by default", async () => {
const { gridMode$ } = createLayoutModeSwitch(
scope,
constant("normal"),
of(false),
);
const mode = await firstValueFrom(gridMode$);
expect(mode).toBe("grid");
});
test("Should switch to spotlight mode when window mode is flat", async () => {
const { gridMode$ } = createLayoutModeSwitch(
scope,
constant("flat"),
of(false),
);
const mode = await firstValueFrom(gridMode$);
expect(mode).toBe("spotlight");
});
});
test("Should allow switching modes manually", () => {
withTestScheduler(({ cold, behavior, expectObservable, schedule }): void => {
function testLayoutSwitch({
windowMode = "n",
hasScreenShares = "n",
userSelection = "",
expectedGridMode,
}: {
windowMode?: string;
hasScreenShares?: string;
userSelection?: string;
expectedGridMode: string;
}): void {
withTestScheduler(({ behavior, schedule, expectObservable }) => {
const { gridMode$, setGridMode } = createLayoutModeSwitch(
scope,
behavior("n", { n: "normal" }),
cold("f", { f: false, t: true }),
testScope(),
behavior(windowMode, { n: "normal", N: "narrow", f: "flat" }),
behavior(hasScreenShares, { y: true, n: false }),
);
schedule("--sgs", {
s: () => setGridMode("spotlight"),
g: () => setGridMode("grid"),
});
expectObservable(gridMode$).toBe("g-sgs", {
g: "grid",
s: "spotlight",
});
});
});
test("Should switch to spotlight mode when there is a remote screen share", () => {
withTestScheduler(({ cold, behavior, expectObservable }): void => {
const shareMarble = "f--t";
const gridsMarble = "g--s";
const { gridMode$ } = createLayoutModeSwitch(
scope,
behavior("n", { n: "normal" }),
cold(shareMarble, { f: false, t: true }),
);
expectObservable(gridMode$).toBe(gridsMarble, {
g: "grid",
s: "spotlight",
});
});
});
test("Can manually force grid when there is a screenshare", () => {
withTestScheduler(({ cold, behavior, expectObservable, schedule }): void => {
const { gridMode$, setGridMode } = createLayoutModeSwitch(
scope,
behavior("n", { n: "normal" }),
cold("-ft", { f: false, t: true }),
);
schedule("---g", {
g: () => setGridMode("grid"),
});
expectObservable(gridMode$).toBe("ggsg", {
g: "grid",
s: "spotlight",
});
});
});
test("Should auto-switch after manually selected grid", () => {
withTestScheduler(({ cold, behavior, expectObservable, schedule }): void => {
const { gridMode$, setGridMode } = createLayoutModeSwitch(
scope,
behavior("n", { n: "normal" }),
// Two screenshares will happen in sequence
cold("-ft-ft", { f: false, t: true }),
);
// There was a screen-share that forced spotlight, then
// the user manually switch back to grid
schedule("---g", {
g: () => setGridMode("grid"),
});
// If we did want to respect manual selection, the expectation would be:
// const expectation = "ggsg";
const expectation = "ggsg-s";
expectObservable(gridMode$).toBe(expectation, {
g: "grid",
s: "spotlight",
});
});
});
test("Should switch back to grid mode when the remote screen share ends", () => {
withTestScheduler(({ cold, behavior, expectObservable }): void => {
const shareMarble = "f--t--f-";
const gridsMarble = "g--s--g-";
const { gridMode$ } = createLayoutModeSwitch(
scope,
behavior("n", { n: "normal" }),
cold(shareMarble, { f: false, t: true }),
);
expectObservable(gridMode$).toBe(gridsMarble, {
g: "grid",
s: "spotlight",
});
});
});
test("can auto-switch to spotlight again after first screen share ends", () => {
withTestScheduler(({ cold, behavior, expectObservable }): void => {
const shareMarble = "ftft";
const gridsMarble = "gsgs";
const { gridMode$ } = createLayoutModeSwitch(
scope,
behavior("n", { n: "normal" }),
cold(shareMarble, { f: false, t: true }),
);
expectObservable(gridMode$).toBe(gridsMarble, {
g: "grid",
s: "spotlight",
});
});
});
test("can switch manually to grid after screen share while manually in spotlight", () => {
withTestScheduler(({ cold, behavior, schedule, expectObservable }): void => {
// Initially, no one is sharing. Then the user manually switches to
// spotlight. After a screen share starts, the user manually switches to
// grid.
const shareMarbles = " f-t-";
const setModeMarbles = "-s-g";
const expectation = " gs-g";
const { gridMode$, setGridMode } = createLayoutModeSwitch(
scope,
behavior("n", { n: "normal" }),
cold(shareMarbles, { f: false, t: true }),
);
schedule(setModeMarbles, {
schedule(userSelection, {
g: () => setGridMode("grid"),
s: () => setGridMode("spotlight"),
});
expectObservable(gridMode$).toBe(expectation, {
expectObservable(gridMode$).toBe(expectedGridMode, {
g: "grid",
s: "spotlight",
});
});
}
describe("default mode", () => {
test("uses grid layout by default", () =>
testLayoutSwitch({
expectedGridMode: "g",
}));
test("uses spotlight mode when window mode is flat", () =>
testLayoutSwitch({
windowMode: " f",
expectedGridMode: "s",
}));
});
test("Should auto-switch to spotlight when in flat window mode", () => {
withTestScheduler(({ cold, behavior, expectObservable }): void => {
const { gridMode$ } = createLayoutModeSwitch(
scope,
behavior("naf", { n: "normal", a: "narrow", f: "flat" }),
cold("f", { f: false, t: true }),
);
test("allows switching modes manually", () =>
testLayoutSwitch({
userSelection: " --sgs",
expectedGridMode: "g-sgs",
}));
expectObservable(gridMode$).toBe("g-s-", {
g: "grid",
s: "spotlight",
});
});
});
test("switches to spotlight mode when there is a remote screen share", () =>
testLayoutSwitch({
hasScreenShares: " n--y",
expectedGridMode: "g--s",
}));
test("can manually switch to grid when there is a screenshare", () =>
testLayoutSwitch({
hasScreenShares: " n-y",
userSelection: " ---g",
expectedGridMode: "g-sg",
}));
test("auto-switches after manually selecting grid", () =>
testLayoutSwitch({
// Two screenshares will happen in sequence. There is a screen share that
// forces spotlight, then the user manually switches back to grid.
hasScreenShares: " n-y-ny",
userSelection: " ---g",
expectedGridMode: "g-sg-s",
// If we did want to respect manual selection, the expectation would be: g-sg
}));
test("switches back to grid mode when the remote screen share ends", () =>
testLayoutSwitch({
hasScreenShares: " n--y--n",
expectedGridMode: "g--s--g",
}));
test("auto-switches to spotlight again after first screen share ends", () =>
testLayoutSwitch({
hasScreenShares: " nyny",
expectedGridMode: "gsgs",
}));
test("switches manually to grid after screen share while manually in spotlight", () =>
testLayoutSwitch({
// Initially, no one is sharing. Then the user manually switches to spotlight.
// After a screen share starts, the user manually switches to grid.
hasScreenShares: " n-y",
userSelection: " -s-g",
expectedGridMode: "gs-g",
}));
test("auto-switches to spotlight when in flat window mode", () =>
testLayoutSwitch({
// First normal, then narrow, then flat.
windowMode: " nNf",
expectedGridMode: "g-s",
}));
test("allows switching modes manually when in flat window mode", () =>
testLayoutSwitch({
// Window becomes flat, then user switches to grid and back.
// Finally the window returns to a normal shape.
windowMode: " nf--n",
userSelection: " --gs",
expectedGridMode: "gsgsg",
}));
test("stays in spotlight while there are screen shares even when window mode changes", () =>
testLayoutSwitch({
windowMode: " nfn",
hasScreenShares: " y",
expectedGridMode: "s",
}));
test("ignores end of screen share until window mode returns to normal", () =>
testLayoutSwitch({
windowMode: " nf-n",
hasScreenShares: " y-n",
expectedGridMode: "s--g",
}));

143
src/state/CallViewModel/LayoutSwitch.ts
View File

@@ -6,122 +6,85 @@ Please see LICENSE in the repository root for full details.
*/
import {
BehaviorSubject,
combineLatest,
map,
type Observable,
scan,
Subject,
startWith,
skipWhile,
switchMap,
} from "rxjs";
import { logger } from "matrix-js-sdk/lib/logger";
import { type GridMode, type WindowMode } from "./CallViewModel.ts";
import { type Behavior } from "../Behavior.ts";
import { constant, type Behavior } from "../Behavior.ts";
import { type ObservableScope } from "../ObservableScope.ts";
/**
* Creates a layout mode switch that allows switching between grid and spotlight modes.
* The actual layout mode can be overridden to spotlight mode if there is a remote screen share active
* or if the window mode is flat.
* The actual layout mode might switch automatically to spotlight if there is a
* remote screen share active or if the window mode is flat.
*
* @param scope - The observable scope to manage subscriptions.
* @param windowMode$ - The current window mode observable.
* @param hasRemoteScreenShares$ - An observable indicating if there are remote screen shares active.
* @param windowMode$ - The current window mode.
* @param hasRemoteScreenShares$ - A behavior indicating if there are remote screen shares active.
*/
export function createLayoutModeSwitch(
scope: ObservableScope,
windowMode$: Behavior<WindowMode>,
hasRemoteScreenShares$: Observable<boolean>,
hasRemoteScreenShares$: Behavior<boolean>,
): {
gridMode$: Behavior<GridMode>;
setGridMode: (value: GridMode) => void;
} {
const gridModeUserSelection$ = new BehaviorSubject<GridMode>("grid");
const userSelection$ = new Subject<GridMode>();
// Callback to set the grid mode desired by the user.
// Notice that this is only a preference, the actual grid mode can be overridden
// if there is a remote screen share active.
const setGridMode = (value: GridMode): void => {
gridModeUserSelection$.next(value);
};
const setGridMode = (value: GridMode): void => userSelection$.next(value);
/**
* The natural grid mode - the mode that the grid would prefer to be in,
* not accounting for the user's manual selections.
*/
const naturalGridMode$ = scope.behavior<GridMode>(
combineLatest(
[hasRemoteScreenShares$, windowMode$],
(hasRemoteScreenShares, windowMode) =>
// When there are screen shares or the window is flat (as with a phone
// in landscape orientation), spotlight is a better experience.
// We want screen shares to be big and readable, and we want flipping
// your phone into landscape to be a quick way of maximising the
// spotlight tile.
hasRemoteScreenShares || windowMode === "flat" ? "spotlight" : "grid",
),
);
/**
* The layout mode of the media tile grid.
*/
const gridMode$ =
// If the user hasn't selected spotlight and somebody starts screen sharing,
// automatically switch to spotlight mode and reset when screen sharing ends
scope.behavior<GridMode>(
combineLatest([
gridModeUserSelection$,
hasRemoteScreenShares$,
windowMode$,
]).pipe(
// Scan to keep track if we have auto-switched already or not.
// To allow the user to override the auto-switch by selecting grid mode again.
scan<
[GridMode, boolean, WindowMode],
{
mode: GridMode;
/** Remember if the change was user driven or not */
hasAutoSwitched: boolean;
/** To know if it is new screen share or an already handled */
hasScreenShares: boolean;
}
>(
(prev, [userSelection, hasScreenShares, windowMode]) => {
const isFlatMode = windowMode === "flat";
const gridMode$ = scope.behavior<GridMode>(
// Whenever the user makes a selection, we enter a new mode of behavior:
userSelection$.pipe(
map((selection) => {
if (selection === "grid")
// The user has selected grid mode. Start by respecting their choice,
// but then follow the natural mode again as soon as it matches.
return naturalGridMode$.pipe(
skipWhile((naturalMode) => naturalMode !== selection),
startWith(selection),
);
// Always force spotlight in flat mode, grid layout is not supported
// in that mode.
// TODO: strange that we do that for flat mode but not for other modes?
// TODO: Why is this not handled in layoutMedia$ like other window modes?
if (isFlatMode) {
logger.debug(`Forcing spotlight mode, windowMode=${windowMode}`);
return {
mode: "spotlight",
hasAutoSwitched: prev.hasAutoSwitched,
hasScreenShares,
};
}
// User explicitly chose spotlight.
// Respect that choice.
if (userSelection === "spotlight") {
return {
mode: "spotlight",
hasAutoSwitched: prev.hasAutoSwitched,
hasScreenShares,
};
}
// User has chosen grid mode. If a screen share starts, we will
// auto-switch to spotlight mode for better experience.
// But we only do it once, if the user switches back to grid mode,
// we respect that choice until they explicitly change it again.
const isNewShare = hasScreenShares && !prev.hasScreenShares;
if (isNewShare && !prev.hasAutoSwitched) {
return {
mode: "spotlight",
hasAutoSwitched: true,
hasScreenShares: true,
};
}
// Respect user's grid choice
// XXX If we want to forbid switching automatically again after we can
// return hasAutoSwitched: acc.hasAutoSwitched here instead of setting to false.
return {
mode: "grid",
hasAutoSwitched: false,
hasScreenShares,
};
},
// initial value
{ mode: "grid", hasAutoSwitched: false, hasScreenShares: false },
),
map(({ mode }) => mode),
),
"grid",
);
// The user has selected spotlight mode. If this matches the natural
// mode, then follow the natural mode going forward.
return selection === naturalGridMode$.value
? naturalGridMode$
: constant(selection);
}),
// Initially the mode of behavior is to just follow the natural grid mode.
startWith(naturalGridMode$),
// Switch between each mode of behavior.
switchMap((mode$) => mode$),
),
);
return {
gridMode$,

58
src/state/CallViewModel/localMember/HomeserverConnected.test.ts
View File

@@ -97,106 +97,106 @@ describe("createHomeserverConnected$", () => {
// LLM generated test cases. They are a bit overkill but I improved the mocking so it is
// easy enough to read them so I think they can stay.
it("is false when sync state is not Syncing", () => {
const hsConnected$ = createHomeserverConnected$(scope, client, session);
expect(hsConnected$.value).toBe(false);
const hsConnected = createHomeserverConnected$(scope, client, session);
expect(hsConnected.combined$.value).toBe(false);
});
it("remains false while membership status is not Connected even if sync is Syncing", () => {
const hsConnected$ = createHomeserverConnected$(scope, client, session);
const hsConnected = createHomeserverConnected$(scope, client, session);
client.setSyncState(SyncState.Syncing);
expect(hsConnected$.value).toBe(false); // membership still disconnected
expect(hsConnected.combined$.value).toBe(false); // membership still disconnected
});
it("is false when membership status transitions to Connected but ProbablyLeft is true", () => {
const hsConnected$ = createHomeserverConnected$(scope, client, session);
const hsConnected = createHomeserverConnected$(scope, client, session);
// Make sync loop OK
client.setSyncState(SyncState.Syncing);
// Indicate probable leave before connection
session.setProbablyLeft(true);
session.setMembershipStatus(Status.Connected);
expect(hsConnected$.value).toBe(false);
expect(hsConnected.combined$.value).toBe(false);
});
it("becomes true only when all three conditions are satisfied", () => {
const hsConnected$ = createHomeserverConnected$(scope, client, session);
const hsConnected = createHomeserverConnected$(scope, client, session);
// 1. Sync loop connected
client.setSyncState(SyncState.Syncing);
expect(hsConnected$.value).toBe(false); // not yet membership connected
expect(hsConnected.combined$.value).toBe(false); // not yet membership connected
// 2. Membership connected
session.setMembershipStatus(Status.Connected);
expect(hsConnected$.value).toBe(true); // probablyLeft is false
expect(hsConnected.combined$.value).toBe(true); // probablyLeft is false
});
it("drops back to false when sync loop leaves Syncing", () => {
const hsConnected$ = createHomeserverConnected$(scope, client, session);
const hsConnected = createHomeserverConnected$(scope, client, session);
// Reach connected state
client.setSyncState(SyncState.Syncing);
session.setMembershipStatus(Status.Connected);
expect(hsConnected$.value).toBe(true);
expect(hsConnected.combined$.value).toBe(true);
// Sync loop error => should flip false
client.setSyncState(SyncState.Error);
expect(hsConnected$.value).toBe(false);
expect(hsConnected.combined$.value).toBe(false);
});
it("drops back to false when membership status becomes disconnected", () => {
const hsConnected$ = createHomeserverConnected$(scope, client, session);
const hsConnected = createHomeserverConnected$(scope, client, session);
client.setSyncState(SyncState.Syncing);
session.setMembershipStatus(Status.Connected);
expect(hsConnected$.value).toBe(true);
expect(hsConnected.combined$.value).toBe(true);
session.setMembershipStatus(Status.Disconnected);
expect(hsConnected$.value).toBe(false);
expect(hsConnected.combined$.value).toBe(false);
});
it("drops to false when ProbablyLeft is emitted after being true", () => {
const hsConnected$ = createHomeserverConnected$(scope, client, session);
const hsConnected = createHomeserverConnected$(scope, client, session);
client.setSyncState(SyncState.Syncing);
session.setMembershipStatus(Status.Connected);
expect(hsConnected$.value).toBe(true);
expect(hsConnected.combined$.value).toBe(true);
session.setProbablyLeft(true);
expect(hsConnected$.value).toBe(false);
expect(hsConnected.combined$.value).toBe(false);
});
it("recovers to true if ProbablyLeft becomes false again while other conditions remain true", () => {
const hsConnected$ = createHomeserverConnected$(scope, client, session);
const hsConnected = createHomeserverConnected$(scope, client, session);
client.setSyncState(SyncState.Syncing);
session.setMembershipStatus(Status.Connected);
expect(hsConnected$.value).toBe(true);
expect(hsConnected.combined$.value).toBe(true);
session.setProbablyLeft(true);
expect(hsConnected$.value).toBe(false);
expect(hsConnected.combined$.value).toBe(false);
// Simulate clearing the flag (in realistic scenario membership manager would update)
session.setProbablyLeft(false);
expect(hsConnected$.value).toBe(true);
expect(hsConnected.combined$.value).toBe(true);
});
it("composite sequence reflects each individual failure reason", () => {
const hsConnected$ = createHomeserverConnected$(scope, client, session);
const hsConnected = createHomeserverConnected$(scope, client, session);
// Initially false (sync error + disconnected + not probably left)
expect(hsConnected$.value).toBe(false);
expect(hsConnected.combined$.value).toBe(false);
// Fix sync only
client.setSyncState(SyncState.Syncing);
expect(hsConnected$.value).toBe(false);
expect(hsConnected.combined$.value).toBe(false);
// Fix membership
session.setMembershipStatus(Status.Connected);
expect(hsConnected$.value).toBe(true);
expect(hsConnected.combined$.value).toBe(true);
// Introduce probablyLeft -> false
session.setProbablyLeft(true);
expect(hsConnected$.value).toBe(false);
expect(hsConnected.combined$.value).toBe(false);
// Restore notProbablyLeft -> true again
session.setProbablyLeft(false);
expect(hsConnected$.value).toBe(true);
expect(hsConnected.combined$.value).toBe(true);
// Drop sync -> false
client.setSyncState(SyncState.Error);
expect(hsConnected$.value).toBe(false);
expect(hsConnected.combined$.value).toBe(false);
});
});

38
src/state/CallViewModel/localMember/HomeserverConnected.ts
View File

@@ -25,6 +25,11 @@ import { type NodeStyleEventEmitter } from "../../../utils/test";
*/
const logger = rootLogger.getChild("[HomeserverConnected]");
export interface HomeserverConnected {
combined$: Behavior<boolean>;
rtsSession$: Behavior<Status>;
}
/**
* Behavior representing whether we consider ourselves connected to the Matrix homeserver
* for the purposes of a MatrixRTC session.
@@ -39,7 +44,7 @@ export function createHomeserverConnected$(
client: NodeStyleEventEmitter & Pick<MatrixClient, "getSyncState">,
matrixRTCSession: NodeStyleEventEmitter &
Pick<MatrixRTCSession, "membershipStatus" | "probablyLeft">,
): Behavior<boolean> {
): HomeserverConnected {
const syncing$ = (
fromEvent(client, ClientEvent.Sync) as Observable<[SyncState]>
).pipe(
@@ -47,12 +52,15 @@ export function createHomeserverConnected$(
map(([state]) => state === SyncState.Syncing),
);
const membershipConnected$ = fromEvent(
matrixRTCSession,
MembershipManagerEvent.StatusChanged,
).pipe(
startWith(null),
map(() => matrixRTCSession.membershipStatus === Status.Connected),
const rtsSession$ = scope.behavior<Status>(
fromEvent(matrixRTCSession, MembershipManagerEvent.StatusChanged).pipe(
map(() => matrixRTCSession.membershipStatus ?? Status.Unknown),
),
Status.Unknown,
);
const membershipConnected$ = rtsSession$.pipe(
map((status) => status === Status.Connected),
);
// This is basically notProbablyLeft$
@@ -71,15 +79,13 @@ export function createHomeserverConnected$(
map(() => matrixRTCSession.probablyLeft !== true),
);
const connectedCombined$ = and$(
syncing$,
membershipConnected$,
certainlyConnected$,
).pipe(
tap((connected) => {
logger.info(`Homeserver connected update: ${connected}`);
}),
const combined$ = scope.behavior(
and$(syncing$, membershipConnected$, certainlyConnected$).pipe(
tap((connected) => {
logger.info(`Homeserver connected update: ${connected}`);
}),
),
);
return scope.behavior(connectedCombined$);
return { combined$, rtsSession$ };
}

219
src/state/CallViewModel/localMember/LocalMembership.test.ts → src/state/CallViewModel/localMember/LocalMember.test.ts
View File

@@ -7,6 +7,7 @@ Please see LICENSE in the repository root for full details.
*/
import {
Status as RTCMemberStatus,
type LivekitTransport,
type MatrixRTCSession,
} from "matrix-js-sdk/lib/matrixrtc";
@@ -14,11 +15,7 @@ import { describe, expect, it, vi } from "vitest";
import { AutoDiscovery } from "matrix-js-sdk/lib/autodiscovery";
import { BehaviorSubject, map, of } from "rxjs";
import { logger } from "matrix-js-sdk/lib/logger";
import {
ConnectionState as LivekitConnectionState,
type LocalParticipant,
type LocalTrack,
} from "livekit-client";
import { type LocalParticipant, type LocalTrack } from "livekit-client";
import { MatrixRTCMode } from "../../../settings/settings";
import {
@@ -29,15 +26,17 @@ import {
withTestScheduler,
} from "../../../utils/test";
import {
TransportState,
createLocalMembership$,
enterRTCSession,
RTCBackendState,
} from "./LocalMembership";
PublishState,
TrackState,
} from "./LocalMember";
import { MatrixRTCTransportMissingError } from "../../../utils/errors";
import { Epoch, ObservableScope } from "../../ObservableScope";
import { constant } from "../../Behavior";
import { ConnectionManagerData } from "../remoteMembers/ConnectionManager";
import { type Connection } from "../remoteMembers/Connection";
import { ConnectionState, type Connection } from "../remoteMembers/Connection";
import { type Publisher } from "./Publisher";
const MATRIX_RTC_MODE = MatrixRTCMode.Legacy;
@@ -51,7 +50,7 @@ vi.mock("@livekit/components-core", () => ({
describe("LocalMembership", () => {
describe("enterRTCSession", () => {
it("It joins the correct Session", async () => {
it("It joins the correct Session", () => {
const focusFromOlderMembership = {
type: "livekit",
livekit_service_url: "http://my-oldest-member-service-url.com",
@@ -107,7 +106,7 @@ describe("LocalMembership", () => {
joinRoomSession: vi.fn(),
}) as unknown as MatrixRTCSession;
await enterRTCSession(
enterRTCSession(
mockedSession,
{
livekit_alias: "roomId",
@@ -136,7 +135,7 @@ describe("LocalMembership", () => {
);
});
it("It should not fail with configuration error if homeserver config has livekit url but not fallback", async () => {
it("It should not fail with configuration error if homeserver config has livekit url but not fallback", () => {
mockConfig({});
vi.spyOn(AutoDiscovery, "getRawClientConfig").mockResolvedValue({
"org.matrix.msc4143.rtc_foci": [
@@ -165,7 +164,7 @@ describe("LocalMembership", () => {
joinRoomSession: vi.fn(),
}) as unknown as MatrixRTCSession;
await enterRTCSession(
enterRTCSession(
mockedSession,
{
livekit_alias: "roomId",
@@ -190,7 +189,6 @@ describe("LocalMembership", () => {
leaveRoomSession: () => {},
} as unknown as MatrixRTCSession,
muteStates: mockMuteStates(),
isHomeserverConnected: constant(true),
trackProcessorState$: constant({
supported: false,
processor: undefined,
@@ -198,20 +196,20 @@ describe("LocalMembership", () => {
logger: logger,
createPublisherFactory: vi.fn(),
joinMatrixRTC: async (): Promise<void> => {},
homeserverConnected$: constant(true),
homeserverConnected: {
combined$: constant(true),
rtsSession$: constant(RTCMemberStatus.Connected),
},
};
it("throws error on missing RTC config error", () => {
withTestScheduler(({ scope, hot, expectObservable }) => {
const goodTransport = {
livekit_service_url: "other",
} as LivekitTransport;
const localTransport$ = scope.behavior<LivekitTransport>(
const localTransport$ = scope.behavior<null | LivekitTransport>(
hot("1ms #", {}, new MatrixRTCTransportMissingError("domain.com")),
goodTransport,
null,
);
// we do not need any connection data since we want to fail before reaching that.
const mockConnectionManager = {
transports$: scope.behavior(
localTransport$.pipe(map((t) => new Epoch([t]))),
@@ -227,15 +225,11 @@ describe("LocalMembership", () => {
connectionManager: mockConnectionManager,
localTransport$,
});
localMembership.requestJoinAndPublish();
expectObservable(localMembership.connectionState.livekit$).toBe("ne", {
n: { state: RTCBackendState.WaitingForConnection },
e: {
state: RTCBackendState.Error,
error: expect.toSatisfy(
(e) => e instanceof MatrixRTCTransportMissingError,
),
},
expectObservable(localMembership.localMemberState$).toBe("ne", {
n: TransportState.Waiting,
e: expect.toSatisfy((e) => e instanceof MatrixRTCTransportMissingError),
});
});
});
@@ -247,33 +241,26 @@ describe("LocalMembership", () => {
livekit_service_url: "b",
} as LivekitTransport;
const connectionManagerData = new ConnectionManagerData();
connectionManagerData.add(
{
livekitRoom: mockLivekitRoom({
localParticipant: {
isScreenShareEnabled: false,
trackPublications: [],
} as unknown as LocalParticipant,
}),
state$: constant({
state: "ConnectedToLkRoom",
livekitConnectionState$: constant(LivekitConnectionState.Connected),
}),
transport: aTransport,
} as unknown as Connection,
[],
);
connectionManagerData.add(
{
state$: constant({
state: "ConnectedToLkRoom",
}),
transport: bTransport,
} as unknown as Connection,
[],
);
const connectionTransportAConnected = {
livekitRoom: mockLivekitRoom({
localParticipant: {
isScreenShareEnabled: false,
trackPublications: [],
} as unknown as LocalParticipant,
}),
state$: constant(ConnectionState.LivekitConnected),
transport: aTransport,
} as unknown as Connection;
const connectionTransportAConnecting = {
...connectionTransportAConnected,
state$: constant(ConnectionState.LivekitConnecting),
livekitRoom: mockLivekitRoom({}),
} as unknown as Connection;
const connectionTransportBConnected = {
state$: constant(ConnectionState.LivekitConnected),
transport: bTransport,
livekitRoom: mockLivekitRoom({}),
} as unknown as Connection;
it("recreates publisher if new connection is used and ENDS always unpublish and end tracks", async () => {
const scope = new ObservableScope();
@@ -281,13 +268,17 @@ describe("LocalMembership", () => {
const localTransport$ = new BehaviorSubject(aTransport);
const publishers: Publisher[] = [];
let seed = 0;
defaultCreateLocalMemberValues.createPublisherFactory.mockImplementation(
() => {
const a = seed;
seed += 1;
logger.info(`creating [${a}]`);
const p = {
stopPublishing: vi.fn(),
stopPublishing: vi.fn().mockImplementation(() => {
logger.info(`stopPublishing [${a}]`);
}),
stopTracks: vi.fn(),
publishing$: constant(false),
};
publishers.push(p as unknown as Publisher);
return p;
@@ -298,6 +289,9 @@ describe("LocalMembership", () => {
typeof vi.fn
>;
const connectionManagerData = new ConnectionManagerData();
connectionManagerData.add(connectionTransportAConnected, []);
connectionManagerData.add(connectionTransportBConnected, []);
createLocalMembership$({
scope,
...defaultCreateLocalMemberValues,
@@ -322,7 +316,7 @@ describe("LocalMembership", () => {
await flushPromises();
// stop all tracks after ending scopes
expect(publishers[1].stopPublishing).toHaveBeenCalled();
expect(publishers[1].stopTracks).toHaveBeenCalled();
// expect(publishers[1].stopTracks).toHaveBeenCalled();
defaultCreateLocalMemberValues.createPublisherFactory.mockReset();
});
@@ -357,6 +351,9 @@ describe("LocalMembership", () => {
typeof vi.fn
>;
const connectionManagerData = new ConnectionManagerData();
connectionManagerData.add(connectionTransportAConnected, []);
// connectionManagerData.add(connectionTransportB, []);
const localMembership = createLocalMembership$({
scope,
...defaultCreateLocalMemberValues,
@@ -367,15 +364,17 @@ describe("LocalMembership", () => {
});
await flushPromises();
expect(publisherFactory).toHaveBeenCalledOnce();
expect(localMembership.tracks$.value.length).toBe(0);
// expect(localMembership.tracks$.value.length).toBe(0);
expect(publishers[0].createAndSetupTracks).not.toHaveBeenCalled();
localMembership.startTracks();
await flushPromises();
expect(localMembership.tracks$.value.length).toBe(2);
expect(publishers[0].createAndSetupTracks).toHaveBeenCalled();
// expect(localMembership.tracks$.value.length).toBe(2);
scope.end();
await flushPromises();
// stop all tracks after ending scopes
expect(publishers[0].stopPublishing).toHaveBeenCalled();
expect(publishers[0].stopTracks).toHaveBeenCalled();
// expect(publishers[0].stopTracks).toHaveBeenCalled();
publisherFactory.mockClear();
});
// TODO add an integration test combining publisher and localMembership
@@ -383,10 +382,11 @@ describe("LocalMembership", () => {
it("tracks livekit state correctly", async () => {
const scope = new ObservableScope();
const connectionManagerData = new ConnectionManagerData();
const localTransport$ = new BehaviorSubject<null | LivekitTransport>(null);
const connectionManagerData$ = new BehaviorSubject<
Epoch<ConnectionManagerData>
>(new Epoch(new ConnectionManagerData()));
const connectionManagerData$ = new BehaviorSubject(
new Epoch(connectionManagerData),
);
const publishers: Publisher[] = [];
const tracks$ = new BehaviorSubject<LocalTrack[]>([]);
@@ -432,61 +432,96 @@ describe("LocalMembership", () => {
});
await flushPromises();
expect(localMembership.connectionState.livekit$.value).toStrictEqual({
state: RTCBackendState.WaitingForTransport,
});
expect(localMembership.localMemberState$.value).toStrictEqual(
TransportState.Waiting,
);
localTransport$.next(aTransport);
await flushPromises();
expect(localMembership.connectionState.livekit$.value).toStrictEqual({
state: RTCBackendState.WaitingForConnection,
expect(localMembership.localMemberState$.value).toStrictEqual({
matrix: RTCMemberStatus.Connected,
media: { connection: null, tracks: TrackState.WaitingForUser },
});
connectionManagerData$.next(new Epoch(connectionManagerData));
const connectionManagerData2 = new ConnectionManagerData();
connectionManagerData2.add(
// clone because we will mutate this later.
{ ...connectionTransportAConnecting } as unknown as Connection,
[],
);
connectionManagerData$.next(new Epoch(connectionManagerData2));
await flushPromises();
expect(localMembership.connectionState.livekit$.value).toStrictEqual({
state: RTCBackendState.Initialized,
expect(localMembership.localMemberState$.value).toStrictEqual({
matrix: RTCMemberStatus.Connected,
media: {
connection: ConnectionState.LivekitConnecting,
tracks: TrackState.WaitingForUser,
},
});
(
connectionManagerData2.getConnectionForTransport(aTransport)!
.state$ as BehaviorSubject<ConnectionState>
).next(ConnectionState.LivekitConnected);
expect(localMembership.localMemberState$.value).toStrictEqual({
matrix: RTCMemberStatus.Connected,
media: {
connection: ConnectionState.LivekitConnected,
tracks: TrackState.WaitingForUser,
},
});
expect(publisherFactory).toHaveBeenCalledOnce();
expect(localMembership.tracks$.value.length).toBe(0);
// expect(localMembership.tracks$.value.length).toBe(0);
// -------
localMembership.startTracks();
// -------
await flushPromises();
expect(localMembership.connectionState.livekit$.value).toStrictEqual({
state: RTCBackendState.CreatingTracks,
});
// expect(localMembership.localMemberState$.value).toStrictEqual({
// matrix: RTCMemberStatus.Connected,
// media: {
// tracks: TrackState.Creating,
// connection: ConnectionState.LivekitConnected,
// },
// });
createTrackResolver.resolve();
await flushPromises();
expect(localMembership.connectionState.livekit$.value).toStrictEqual({
state: RTCBackendState.ReadyToPublish,
});
expect(
// eslint-disable-next-line @typescript-eslint/no-explicit-any
(localMembership.localMemberState$.value as any).media,
).toStrictEqual(PublishState.WaitingForUser);
// -------
localMembership.requestConnect();
localMembership.requestJoinAndPublish();
// -------
expect(localMembership.connectionState.livekit$.value).toStrictEqual({
state: RTCBackendState.WaitingToPublish,
});
expect(
// eslint-disable-next-line @typescript-eslint/no-explicit-any
(localMembership.localMemberState$.value as any).media,
).toStrictEqual(PublishState.Publishing);
publishResolver.resolve();
await flushPromises();
expect(localMembership.connectionState.livekit$.value).toStrictEqual({
state: RTCBackendState.Connected,
});
expect(
// eslint-disable-next-line @typescript-eslint/no-explicit-any
(localMembership.localMemberState$.value as any).media,
).toStrictEqual(PublishState.Publishing);
expect(publishers[0].stopPublishing).not.toHaveBeenCalled();
expect(localMembership.connectionState.livekit$.isStopped).toBe(false);
expect(localMembership.localMemberState$.isStopped).toBe(false);
scope.end();
await flushPromises();
// stays in connected state because it is stopped before the update to tracks update the state.
expect(localMembership.connectionState.livekit$.value).toStrictEqual({
state: RTCBackendState.Connected,
});
expect(
// eslint-disable-next-line @typescript-eslint/no-explicit-any
(localMembership.localMemberState$.value as any).media,
).toStrictEqual(PublishState.Publishing);
// stop all tracks after ending scopes
expect(publishers[0].stopPublishing).toHaveBeenCalled();
expect(publishers[0].stopTracks).toHaveBeenCalled();
// expect(publishers[0].stopTracks).toHaveBeenCalled();
});
// TODO add tests for matrix local matrix participation.
});

510
src/state/CallViewModel/localMember/LocalMembership.ts → src/state/CallViewModel/localMember/LocalMember.ts
View File

@@ -6,15 +6,16 @@ Please see LICENSE in the repository root for full details.
*/
import {
type LocalTrack,
type Participant,
ParticipantEvent,
type LocalParticipant,
type ScreenShareCaptureOptions,
ConnectionState,
RoomEvent,
MediaDeviceFailure,
} from "livekit-client";
import { observeParticipantEvents } from "@livekit/components-core";
import {
Status as RTCSessionStatus,
type LivekitTransport,
type MatrixRTCSession,
} from "matrix-js-sdk/lib/matrixrtc";
@@ -24,10 +25,11 @@ import {
combineLatest,
distinctUntilChanged,
from,
fromEvent,
map,
type Observable,
of,
scan,
pairwise,
startWith,
switchMap,
tap,
@@ -35,74 +37,73 @@ import {
import { type Logger } from "matrix-js-sdk/lib/logger";
import { deepCompare } from "matrix-js-sdk/lib/utils";
import { constant, type Behavior } from "../../Behavior";
import { type IConnectionManager } from "../remoteMembers/ConnectionManager";
import { ObservableScope } from "../../ObservableScope";
import { type Publisher } from "./Publisher";
import { type MuteStates } from "../../MuteStates";
import { and$ } from "../../../utils/observable";
import { type Behavior } from "../../Behavior.ts";
import { type IConnectionManager } from "../remoteMembers/ConnectionManager.ts";
import { type ObservableScope } from "../../ObservableScope.ts";
import { type Publisher } from "./Publisher.ts";
import { type MuteStates } from "../../MuteStates.ts";
import {
ElementCallError,
FailToStartLivekitConnection,
MembershipManagerError,
UnknownCallError,
} from "../../../utils/errors";
import { ElementWidgetActions, widget } from "../../../widget";
} from "../../../utils/errors.ts";
import { ElementWidgetActions, widget } from "../../../widget.ts";
import { getUrlParams } from "../../../UrlParams.ts";
import { PosthogAnalytics } from "../../../analytics/PosthogAnalytics.ts";
import { MatrixRTCMode } from "../../../settings/settings.ts";
import { Config } from "../../../config/Config.ts";
import { type Connection } from "../remoteMembers/Connection.ts";
import {
ConnectionState,
type Connection,
type FailedToStartError,
} from "../remoteMembers/Connection.ts";
import { type HomeserverConnected } from "./HomeserverConnected.ts";
import { and$ } from "../../../utils/observable.ts";
export enum RTCBackendState {
Error = "error",
export enum TransportState {
/** Not even a transport is available to the LocalMembership */
WaitingForTransport = "waiting_for_transport",
/** A connection appeared so we can initialise the publisher */
WaitingForConnection = "waiting_for_connection",
/** Connection and transport arrived, publisher Initialized */
Initialized = "Initialized",
CreatingTracks = "creating_tracks",
ReadyToPublish = "ready_to_publish",
WaitingToPublish = "waiting_to_publish",
Connected = "connected",
Disconnected = "disconnected",
Disconnecting = "disconnecting",
Waiting = "transport_waiting",
}
type LocalMemberRtcBackendState =
| { state: RTCBackendState.Error; error: ElementCallError }
| { state: RTCBackendState.WaitingForTransport }
| { state: RTCBackendState.WaitingForConnection }
| { state: RTCBackendState.Initialized }
| { state: RTCBackendState.CreatingTracks }
| { state: RTCBackendState.ReadyToPublish }
| { state: RTCBackendState.WaitingToPublish }
| { state: RTCBackendState.Connected }
| { state: RTCBackendState.Disconnected }
| { state: RTCBackendState.Disconnecting };
export enum MatrixState {
WaitingForTransport = "waiting_for_transport",
Ready = "ready",
Connecting = "connecting",
Connected = "connected",
Disconnected = "disconnected",
Error = "Error",
export enum PublishState {
WaitingForUser = "publish_waiting_for_user",
// XXX: This state is removed for now since we do not have full control over
// track publication anymore with the publisher abstraction, might come back in the future?
// /** Implies lk connection is connected */
// Starting = "publish_start_publishing",
/** Implies lk connection is connected */
Publishing = "publish_publishing",
}
type LocalMemberMatrixState =
| { state: MatrixState.Connected }
| { state: MatrixState.WaitingForTransport }
| { state: MatrixState.Ready }
| { state: MatrixState.Connecting }
| { state: MatrixState.Disconnected }
| { state: MatrixState.Error; error: Error };
export interface LocalMemberConnectionState {
livekit$: Behavior<LocalMemberRtcBackendState>;
matrix$: Behavior<LocalMemberMatrixState>;
// TODO not sure how to map that correctly with the
// new publisher that does not manage tracks itself anymore
export enum TrackState {
/** The track is waiting for user input to create tracks (waiting to call `startTracks()`) */
WaitingForUser = "tracks_waiting_for_user",
// XXX: This state is removed for now since we do not have full control over
// track creation anymore with the publisher abstraction, might come back in the future?
// /** Implies lk connection is connected */
// Creating = "tracks_creating",
/** Implies lk connection is connected */
Ready = "tracks_ready",
}
export type LocalMemberMediaState =
| {
tracks: TrackState;
connection: ConnectionState | FailedToStartError;
}
| PublishState
| ElementCallError;
export type LocalMemberState =
| ElementCallError
| TransportState.Waiting
| {
media: LocalMemberMediaState;
matrix: ElementCallError | RTCSessionStatus;
};
/*
* - get well known
* - get oldest membership
@@ -122,8 +123,8 @@ interface Props {
muteStates: MuteStates;
connectionManager: IConnectionManager;
createPublisherFactory: (connection: Connection) => Publisher;
joinMatrixRTC: (transport: LivekitTransport) => Promise<void>;
homeserverConnected$: Behavior<boolean>;
joinMatrixRTC: (transport: LivekitTransport) => void;
homeserverConnected: HomeserverConnected;
localTransport$: Behavior<LivekitTransport | null>;
matrixRTCSession: Pick<
MatrixRTCSession,
@@ -149,7 +150,7 @@ export const createLocalMembership$ = ({
scope,
connectionManager,
localTransport$: localTransportCanThrow$,
homeserverConnected$,
homeserverConnected,
createPublisherFactory,
joinMatrixRTC,
logger: parentLogger,
@@ -157,32 +158,33 @@ export const createLocalMembership$ = ({
matrixRTCSession,
}: Props): {
/**
* This starts audio and video tracks. They will be reused when calling `requestConnect`.
* This request to start audio and video tracks.
* Can be called early to pre-emptively get media permissions and start devices.
*/
startTracks: () => Behavior<LocalTrack[]>;
startTracks: () => void;
/**
* This sets a inner state (shouldConnect) to true and instructs the js-sdk and livekit to keep the user
* This sets a inner state (shouldPublish) to true and instructs the js-sdk and livekit to keep the user
* connected to matrix and livekit.
*/
requestConnect: () => void;
requestJoinAndPublish: () => void;
requestDisconnect: () => void;
connectionState: LocalMemberConnectionState;
localMemberState$: Behavior<LocalMemberState>;
sharingScreen$: Behavior<boolean>;
/**
* Callback to toggle screen sharing. If null, screen sharing is not possible.
*/
toggleScreenSharing: (() => void) | null;
tracks$: Behavior<LocalTrack[]>;
// tracks$: Behavior<LocalTrack[]>;
participant$: Behavior<LocalParticipant | null>;
connection$: Behavior<Connection | null>;
connected$: Behavior<boolean>;
// this needs to be discussed
/**
* Whether various media/event sources should pretend to be disconnected from
* all network input, even if their connection still technically works.
* @deprecated use state instead
/** Shorthand for homeserverConnected.rtcSession === Status.Reconnecting
* Direct translation to the js-sdk membership manager connection `Status`.
*/
reconnecting$: Behavior<boolean>;
/** Shorthand for homeserverConnected.rtcSession === Status.Disconnected
* Direct translation to the js-sdk membership manager connection `Status`.
*/
disconnected$: Behavior<boolean>;
} => {
const logger = parentLogger.getChild("[LocalMembership]");
logger.debug(`Creating local membership..`);
@@ -201,7 +203,7 @@ export const createLocalMembership$ = ({
: new Error("Unknown error from localTransport"),
);
}
setLivekitError(error);
setTransportError(error);
return of(null);
}),
),
@@ -228,94 +230,59 @@ export const createLocalMembership$ = ({
),
);
const localConnectionState$ = localConnection$.pipe(
switchMap((connection) => (connection ? connection.state$ : of(null))),
);
// /**
// * Whether we are "fully" connected to the call. Accounts for both the
// * connection to the MatrixRTC session and the LiveKit publish connection.
// */
const connected$ = scope.behavior(
and$(
homeserverConnected$.pipe(
tap((v) => logger.debug("matrix: Connected state changed", v)),
),
localConnectionState$.pipe(
switchMap((state) => {
logger.debug("livekit: Connected state changed", state);
if (!state) return of(false);
if (state.state === "ConnectedToLkRoom") {
logger.debug(
"livekit: Connected state changed (inner livekitConnectionState$)",
state.livekitConnectionState$.value,
);
return state.livekitConnectionState$.pipe(
map((lkState) => lkState === ConnectionState.Connected),
);
}
return of(false);
}),
),
).pipe(tap((v) => logger.debug("combined: Connected state changed", v))),
// Tracks error that happen when creating the local tracks.
const mediaErrors$ = localConnection$.pipe(
switchMap((connection) => {
if (!connection) {
return of(null);
} else {
return fromEvent(
connection.livekitRoom,
RoomEvent.MediaDevicesError,
(error: Error) => {
return MediaDeviceFailure.getFailure(error) ?? null;
},
);
}
}),
);
mediaErrors$.pipe(scope.bind()).subscribe((error) => {
if (error) {
logger.error(`Failed to create local tracks:`, error);
setMatrixError(
// TODO is it fatal? Do we need to create a new Specialized Error?
new UnknownCallError(new Error(`Media device error: ${error}`)),
);
}
});
// MATRIX RELATED
// /**
// * Whether we should tell the user that we're reconnecting to the call.
// */
// DISCUSSION is there a better way to do this?
// sth that is more deriectly implied from the membership manager of the js sdk. (fromEvent(matrixRTCSession, Reconnecting)) ??? or similar
const reconnecting$ = scope.behavior(
connected$.pipe(
// We are reconnecting if we previously had some successful initial
// connection but are now disconnected
scan(
({ connectedPreviously }, connectedNow) => ({
connectedPreviously: connectedPreviously || connectedNow,
reconnecting: connectedPreviously && !connectedNow,
}),
{ connectedPreviously: false, reconnecting: false },
),
map(({ reconnecting }) => reconnecting),
),
);
// This should be used in a combineLatest with publisher$ to connect.
// to make it possible to call startTracks before the preferredTransport$ has resolved.
const trackStartRequested = Promise.withResolvers<void>();
// This should be used in a combineLatest with publisher$ to connect.
// to make it possible to call startTracks before the preferredTransport$ has resolved.
const connectRequested$ = new BehaviorSubject(false);
const joinAndPublishRequested$ = new BehaviorSubject(false);
/**
* The publisher is stored in here an abstracts creating and publishing tracks.
*/
const publisher$ = new BehaviorSubject<Publisher | null>(null);
/**
* Extract the tracks from the published. Also reacts to changing publishers.
*/
const tracks$ = scope.behavior(
publisher$.pipe(switchMap((p) => (p?.tracks$ ? p.tracks$ : constant([])))),
);
const publishing$ = scope.behavior(
publisher$.pipe(switchMap((p) => p?.publishing$ ?? constant(false))),
);
const startTracks = (): Behavior<LocalTrack[]> => {
const startTracks = (): void => {
trackStartRequested.resolve();
return tracks$;
// This used to return the tracks, but now they are only accessible via the publisher.
};
const requestConnect = (): void => {
const requestJoinAndPublish = (): void => {
trackStartRequested.resolve();
connectRequested$.next(true);
joinAndPublishRequested$.next(true);
};
const requestDisconnect = (): void => {
connectRequested$.next(false);
joinAndPublishRequested$.next(false);
};
// Take care of the publisher$
@@ -332,7 +299,7 @@ export const createLocalMembership$ = ({
// Clean-up callback
return Promise.resolve(async (): Promise<void> => {
await publisher.stopPublishing();
publisher.stopTracks();
await publisher.stopTracks();
});
}
});
@@ -341,13 +308,16 @@ export const createLocalMembership$ = ({
// `tracks$` will update once they are ready.
scope.reconcile(
scope.behavior(
combineLatest([publisher$, tracks$, from(trackStartRequested.promise)]),
combineLatest([
publisher$ /*, tracks$*/,
from(trackStartRequested.promise),
]),
null,
),
async (valueIfReady) => {
if (!valueIfReady) return;
const [publisher, tracks] = valueIfReady;
if (publisher && tracks.length === 0) {
const [publisher] = valueIfReady;
if (publisher) {
await publisher.createAndSetupTracks().catch((e) => logger.error(e));
}
},
@@ -355,140 +325,215 @@ export const createLocalMembership$ = ({
// Based on `connectRequested$` we start publishing tracks. (once they are there!)
scope.reconcile(
scope.behavior(combineLatest([publisher$, tracks$, connectRequested$])),
async ([publisher, tracks, shouldConnect]) => {
if (shouldConnect === publisher?.publishing$.value) return;
if (tracks.length !== 0 && shouldConnect) {
scope.behavior(combineLatest([publisher$, joinAndPublishRequested$])),
async ([publisher, shouldJoinAndPublish]) => {
// Get the current publishing state to avoid redundant calls.
const isPublishing = publisher?.shouldPublish === true;
if (shouldJoinAndPublish && !isPublishing) {
try {
await publisher?.startPublishing();
} catch (error) {
setLivekitError(error as ElementCallError);
const message =
error instanceof Error ? error.message : String(error);
setPublishError(new FailToStartLivekitConnection(message));
}
} else if (tracks.length !== 0 && !shouldConnect) {
} else if (isPublishing) {
try {
await publisher?.stopPublishing();
} catch (error) {
setLivekitError(new UnknownCallError(error as Error));
setPublishError(new UnknownCallError(error as Error));
}
}
},
);
const fatalLivekitError$ = new BehaviorSubject<ElementCallError | null>(null);
const setLivekitError = (e: ElementCallError): void => {
if (fatalLivekitError$.value !== null)
logger.error("Multiple Livkit Errors:", e);
else fatalLivekitError$.next(e);
// STATE COMPUTATION
// These are non fatal since we can join a room and concume media even though publishing failed.
const publishError$ = new BehaviorSubject<ElementCallError | null>(null);
const setPublishError = (e: ElementCallError): void => {
if (publishError$.value !== null) {
logger.error("Multiple Media Errors:", e);
} else {
publishError$.next(e);
}
};
const livekitState$: Behavior<LocalMemberRtcBackendState> = scope.behavior(
const fatalTransportError$ = new BehaviorSubject<ElementCallError | null>(
null,
);
const setTransportError = (e: ElementCallError): void => {
if (fatalTransportError$.value !== null) {
logger.error("Multiple Transport Errors:", e);
} else {
fatalTransportError$.next(e);
}
};
const localConnectionState$ = localConnection$.pipe(
switchMap((connection) => (connection ? connection.state$ : of(null))),
);
const mediaState$: Behavior<LocalMemberMediaState> = scope.behavior(
combineLatest([
publisher$,
localConnectionState$,
localTransport$,
tracks$.pipe(
tap((t) => {
logger.info("tracks$: ", t);
}),
),
publishing$,
connectRequested$,
joinAndPublishRequested$,
from(trackStartRequested.promise).pipe(
map(() => true),
startWith(false),
),
fatalLivekitError$,
]).pipe(
map(
([
publisher,
localConnectionState,
localTransport,
tracks,
publishing,
shouldConnect,
shouldPublish,
shouldStartTracks,
error,
]) => {
// read this:
// if(!<A>) return {state: ...}
// if(!<B>) return {state: <MyState>}
//
// as:
// We do have <A> but not yet <B> so we are in <MyState>
if (error !== null) return { state: RTCBackendState.Error, error };
const hasTracks = tracks.length > 0;
if (!localTransport)
return { state: RTCBackendState.WaitingForTransport };
if (!publisher)
return { state: RTCBackendState.WaitingForConnection };
if (!shouldStartTracks) return { state: RTCBackendState.Initialized };
if (!hasTracks) return { state: RTCBackendState.CreatingTracks };
if (!shouldConnect) return { state: RTCBackendState.ReadyToPublish };
if (!publishing) return { state: RTCBackendState.WaitingToPublish };
return { state: RTCBackendState.Connected };
if (!localTransport) return null;
const trackState: TrackState = shouldStartTracks
? TrackState.Ready
: TrackState.WaitingForUser;
if (
localConnectionState !== ConnectionState.LivekitConnected ||
trackState !== TrackState.Ready
)
return {
connection: localConnectionState,
tracks: trackState,
};
if (!shouldPublish) return PublishState.WaitingForUser;
// if (!publishing) return PublishState.Starting;
return PublishState.Publishing;
},
),
distinctUntilChanged(deepCompare),
),
);
const fatalMatrixError$ = new BehaviorSubject<ElementCallError | null>(null);
const setMatrixError = (e: ElementCallError): void => {
if (fatalMatrixError$.value !== null)
if (fatalMatrixError$.value !== null) {
logger.error("Multiple Matrix Errors:", e);
else fatalMatrixError$.next(e);
} else {
fatalMatrixError$.next(e);
}
};
const matrixState$: Behavior<LocalMemberMatrixState> = scope.behavior(
const localMemberState$ = scope.behavior<LocalMemberState>(
combineLatest([
localTransport$,
connectRequested$,
homeserverConnected$,
mediaState$,
homeserverConnected.rtsSession$,
fatalMatrixError$,
fatalTransportError$,
publishError$,
]).pipe(
map(([localTransport, connectRequested, homeserverConnected]) => {
if (!localTransport) return { state: MatrixState.WaitingForTransport };
if (!connectRequested) return { state: MatrixState.Ready };
if (!homeserverConnected) return { state: MatrixState.Connecting };
return { state: MatrixState.Connected };
}),
map(
([
mediaState,
rtcSessionStatus,
fatalMatrixError,
fatalTransportError,
publishError,
]) => {
if (fatalTransportError !== null) return fatalTransportError;
// `mediaState` will be 'null' until the transport/connection appears.
if (mediaState && rtcSessionStatus)
return {
matrix: fatalMatrixError ?? rtcSessionStatus,
media: publishError ?? mediaState,
};
return TransportState.Waiting;
},
),
),
);
// Keep matrix rtc session in sync with localTransport$, connectRequested$ and muteStates.video.enabled$
/**
* Whether we are "fully" connected to the call. Accounts for both the
* connection to the MatrixRTC session and the LiveKit publish connection.
*/
const matrixAndLivekitConnected$ = scope.behavior(
and$(
homeserverConnected.combined$,
localConnectionState$.pipe(
map((state) => state === ConnectionState.LivekitConnected),
),
).pipe(
tap((v) => logger.debug("livekit+matrix: Connected state changed", v)),
),
);
/**
* Whether we should tell the user that we're reconnecting to the call.
*/
const reconnecting$ = scope.behavior(
matrixAndLivekitConnected$.pipe(
pairwise(),
map(([prev, current]) => prev === true && current === false),
),
false,
);
// inform the widget about the connect and disconnect intent from the user.
scope
.behavior(joinAndPublishRequested$.pipe(pairwise(), scope.bind()), [
undefined,
joinAndPublishRequested$.value,
])
.subscribe(([prev, current]) => {
if (!widget) return;
// JOIN prev=false (was left) => current-true (now joiend)
if (!prev && current) {
widget.api.transport
.send(ElementWidgetActions.JoinCall, {})
.catch((e) => {
logger.error("Failed to send join action", e);
});
}
// LEAVE prev=false (was joined) => current-true (now left)
if (prev && !current) {
widget.api.transport
.send(ElementWidgetActions.HangupCall, {})
.catch((e) => {
logger.error("Failed to send hangup action", e);
});
}
});
combineLatest([muteStates.video.enabled$, homeserverConnected.combined$])
.pipe(scope.bind())
.subscribe(([videoEnabled, connected]) => {
if (!connected) return;
void matrixRTCSession.updateCallIntent(videoEnabled ? "video" : "audio");
});
// Keep matrix rtc session in sync with localTransport$, connectRequested$
scope.reconcile(
scope.behavior(combineLatest([localTransport$, connectRequested$])),
scope.behavior(combineLatest([localTransport$, joinAndPublishRequested$])),
async ([transport, shouldConnect]) => {
if (!transport) return;
// if shouldConnect=false we will do the disconnect as the cleanup from the previous reconcile iteration.
if (!shouldConnect) return;
if (!transport) return;
try {
await joinMatrixRTC(transport);
joinMatrixRTC(transport);
} catch (error) {
logger.error("Error entering RTC session", error);
if (error instanceof Error)
setMatrixError(new MembershipManagerError(error));
}
// Update our member event when our mute state changes.
const callIntentScope = new ObservableScope();
// because this uses its own scope, we can start another reconciliation for the duration of one connection.
callIntentScope.reconcile(
muteStates.video.enabled$,
async (videoEnabled) =>
matrixRTCSession.updateCallIntent(videoEnabled ? "video" : "audio"),
);
return async (): Promise<void> => {
callIntentScope.end();
return Promise.resolve(async (): Promise<void> => {
try {
// Update matrixRTCSession to allow udpating the transport without leaving the session!
await matrixRTCSession.leaveRoomSession();
// TODO Update matrixRTCSession to allow udpating the transport without leaving the session!
await matrixRTCSession.leaveRoomSession(1000);
} catch (e) {
logger.error("Error leaving RTC session", e);
}
try {
await widget?.api.transport.send(ElementWidgetActions.HangupCall, {});
} catch (e) {
logger.error("Failed to send hangup action", e);
}
};
});
},
);
@@ -503,7 +548,7 @@ export const createLocalMembership$ = ({
// pause tracks during the initial joining sequence too until we're sure
// that our own media is displayed on screen.
// TODO refactor this based no livekitState$
combineLatest([participant$, homeserverConnected$])
combineLatest([participant$, homeserverConnected.combined$])
.pipe(scope.bind())
.subscribe(([participant, connected]) => {
if (!participant) return;
@@ -588,16 +633,20 @@ export const createLocalMembership$ = ({
return {
startTracks,
requestConnect,
requestJoinAndPublish,
requestDisconnect,
connectionState: {
livekit$: livekitState$,
matrix$: matrixState$,
},
tracks$,
localMemberState$,
participant$,
<<<<<<< HEAD:src/state/CallViewModel/localMember/LocalMembership.ts
connected$,
=======
>>>>>>> livekit:src/state/CallViewModel/localMember/LocalMember.ts
reconnecting$,
disconnected$: scope.behavior(
homeserverConnected.rtsSession$.pipe(
map((state) => state === RTCSessionStatus.Disconnected),
),
),
sharingScreen$,
toggleScreenSharing,
connection$: localConnection$,
@@ -632,11 +681,11 @@ interface EnterRTCSessionOptions {
* @throws If the widget could not send ElementWidgetActions.JoinCall action.
*/
// Exported for unit testing
export async function enterRTCSession(
export function enterRTCSession(
rtcSession: MatrixRTCSession,
transport: LivekitTransport,
{ encryptMedia, matrixRTCMode }: EnterRTCSessionOptions,
): Promise<void> {
): void {
PosthogAnalytics.instance.eventCallEnded.cacheStartCall(new Date());
PosthogAnalytics.instance.eventCallStarted.track(rtcSession.room.roomId);
@@ -675,7 +724,4 @@ export async function enterRTCSession(
unstableSendStickyEvents: matrixRTCMode === MatrixRTCMode.Matrix_2_0,
},
);
if (widget) {
await widget.api.transport.send(ElementWidgetActions.JoinCall, {});
}
}

2
src/state/CallViewModel/localMember/LocalTransport.ts
View File

@@ -85,7 +85,7 @@ export const createLocalTransport$ = ({
* The transport that we would personally prefer to publish on (if not for the
* transport preferences of others, perhaps).
*
* @throws
* @throws MatrixRTCTransportMissingError | FailToGetOpenIdToken
*/
const preferredTransport$: Behavior<LivekitTransport | null> = scope.behavior(
customLivekitUrl.value$.pipe(

424
src/state/CallViewModel/localMember/Publisher.test.ts
View File

@@ -5,66 +5,320 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { afterEach, beforeEach, describe, expect, it, test, vi } from "vitest";
import {
afterEach,
beforeEach,
describe,
expect,
it,
type Mock,
vi,
} from "vitest";
import { ConnectionState as LivekitConenctionState } from "livekit-client";
import { type BehaviorSubject } from "rxjs";
ConnectionState as LivekitConnectionState,
LocalParticipant,
type LocalTrack,
type LocalTrackPublication,
ParticipantEvent,
Track,
} from "livekit-client";
import { BehaviorSubject } from "rxjs";
import { logger } from "matrix-js-sdk/lib/logger";
import { ObservableScope } from "../../ObservableScope";
import { constant } from "../../Behavior";
import {
flushPromises,
mockLivekitRoom,
mockLocalParticipant,
mockMediaDevices,
} from "../../../utils/test";
import { Publisher } from "./Publisher";
import {
type Connection,
type ConnectionState,
} from "../remoteMembers/Connection";
import { type Connection } from "../remoteMembers/Connection";
import { type MuteStates } from "../../MuteStates";
import { FailToStartLivekitConnection } from "../../../utils/errors";
describe("Publisher", () => {
let scope: ObservableScope;
let connection: Connection;
let muteStates: MuteStates;
beforeEach(() => {
muteStates = {
audio: {
enabled$: constant(false),
unsetHandler: vi.fn(),
setHandler: vi.fn(),
},
video: {
enabled$: constant(false),
unsetHandler: vi.fn(),
setHandler: vi.fn(),
},
} as unknown as MuteStates;
scope = new ObservableScope();
connection = {
state$: constant({
state: "ConnectedToLkRoom",
livekitConnectionState$: constant(LivekitConenctionState.Connected),
}),
livekitRoom: mockLivekitRoom({
localParticipant: mockLocalParticipant({}),
}),
} as unknown as Connection;
let scope: ObservableScope;
beforeEach(() => {
scope = new ObservableScope();
});
afterEach(() => scope.end());
function createMockLocalTrack(source: Track.Source): LocalTrack {
const track = {
source,
isMuted: false,
isUpstreamPaused: false,
} as Partial<LocalTrack> as LocalTrack;
vi.mocked(track).mute = vi.fn().mockImplementation(() => {
track.isMuted = true;
});
vi.mocked(track).unmute = vi.fn().mockImplementation(() => {
track.isMuted = false;
});
vi.mocked(track).pauseUpstream = vi.fn().mockImplementation(() => {
// @ts-expect-error - for that test we want to set isUpstreamPaused directly
track.isUpstreamPaused = true;
});
vi.mocked(track).resumeUpstream = vi.fn().mockImplementation(() => {
// @ts-expect-error - for that test we want to set isUpstreamPaused directly
track.isUpstreamPaused = false;
});
afterEach(() => scope.end());
return track;
}
it("throws if livekit room could not publish", async () => {
function createMockMuteState(enabled$: BehaviorSubject<boolean>): {
enabled$: BehaviorSubject<boolean>;
setHandler: (h: (enabled: boolean) => void) => void;
unsetHandler: () => void;
} {
let currentHandler = (enabled: boolean): void => {};
const ms = {
enabled$,
setHandler: vi.fn().mockImplementation((h: (enabled: boolean) => void) => {
currentHandler = h;
}),
unsetHandler: vi.fn().mockImplementation(() => {
currentHandler = (enabled: boolean): void => {};
}),
};
// forward enabled$ emissions to the current handler
enabled$.subscribe((enabled) => {
logger.info(`MockMuteState: enabled changed to ${enabled}`);
currentHandler(enabled);
});
return ms;
}
let connection: Connection;
let muteStates: MuteStates;
let localParticipant: LocalParticipant;
let audioEnabled$: BehaviorSubject<boolean>;
let videoEnabled$: BehaviorSubject<boolean>;
let trackPublications: LocalTrackPublication[];
// use it to control when track creation resolves, default to resolved
let createTrackLock: Promise<void>;
beforeEach(() => {
trackPublications = [];
audioEnabled$ = new BehaviorSubject(false);
videoEnabled$ = new BehaviorSubject(false);
createTrackLock = Promise.resolve();
muteStates = {
audio: createMockMuteState(audioEnabled$),
video: createMockMuteState(videoEnabled$),
} as unknown as MuteStates;
const mockSendDataPacket = vi.fn();
const mockEngine = {
client: {
sendUpdateLocalMetadata: vi.fn(),
},
on: vi.fn().mockReturnThis(),
sendDataPacket: mockSendDataPacket,
};
localParticipant = new LocalParticipant(
"local-sid",
"local-identity",
// @ts-expect-error - for that test we want a real LocalParticipant to have the pending publications logic
mockEngine,
{
adaptiveStream: true,
dynacase: false,
audioCaptureDefaults: {},
videoCaptureDefaults: {},
stopLocalTrackOnUnpublish: true,
reconnectPolicy: "always",
disconnectOnPageLeave: true,
},
new Map(),
{},
);
vi.mocked(localParticipant).createTracks = vi
.fn()
.mockImplementation(async (opts) => {
const tracks: LocalTrack[] = [];
if (opts.audio) {
tracks.push(createMockLocalTrack(Track.Source.Microphone));
}
if (opts.video) {
tracks.push(createMockLocalTrack(Track.Source.Camera));
}
await createTrackLock;
return tracks;
});
vi.mocked(localParticipant).publishTrack = vi
.fn()
.mockImplementation(async (track: LocalTrack) => {
const pub = {
track,
source: track.source,
mute: track.mute,
unmute: track.unmute,
} as Partial<LocalTrackPublication> as LocalTrackPublication;
trackPublications.push(pub);
localParticipant.emit(ParticipantEvent.LocalTrackPublished, pub);
return Promise.resolve(pub);
});
vi.mocked(localParticipant).getTrackPublication = vi
.fn()
.mockImplementation((source: Track.Source) => {
return trackPublications.find((pub) => pub.track?.source === source);
});
connection = {
state$: constant({
state: "ConnectedToLkRoom",
livekitConnectionState$: constant(LivekitConnectionState.Connected),
}),
livekitRoom: mockLivekitRoom({
localParticipant: localParticipant,
}),
} as unknown as Connection;
});
describe("Publisher", () => {
let publisher: Publisher;
beforeEach(() => {
publisher = new Publisher(
scope,
connection,
mockMediaDevices({}),
muteStates,
constant({ supported: false, processor: undefined }),
logger,
);
});
afterEach(() => {});
it("Should not create tracks if started muted to avoid unneeded permission requests", async () => {
const createTracksSpy = vi.spyOn(
connection.livekitRoom.localParticipant,
"createTracks",
);
audioEnabled$.next(false);
videoEnabled$.next(false);
await publisher.createAndSetupTracks();
expect(createTracksSpy).not.toHaveBeenCalled();
});
it("Should minimize permission request by querying create at once", async () => {
const enableCameraAndMicrophoneSpy = vi.spyOn(
localParticipant,
"enableCameraAndMicrophone",
);
const createTracksSpy = vi.spyOn(localParticipant, "createTracks");
audioEnabled$.next(true);
videoEnabled$.next(true);
await publisher.createAndSetupTracks();
await flushPromises();
expect(enableCameraAndMicrophoneSpy).toHaveBeenCalled();
// It should create both at once
expect(createTracksSpy).toHaveBeenCalledWith({
audio: true,
video: true,
});
});
it("Ensure no data is streamed until publish has been called", async () => {
audioEnabled$.next(true);
await publisher.createAndSetupTracks();
// The track should be created and paused
expect(localParticipant.createTracks).toHaveBeenCalledWith({
audio: true,
video: undefined,
});
await flushPromises();
expect(localParticipant.publishTrack).toHaveBeenCalled();
await flushPromises();
const track = localParticipant.getTrackPublication(
Track.Source.Microphone,
)?.track;
expect(track).toBeDefined();
expect(track!.pauseUpstream).toHaveBeenCalled();
expect(track!.isUpstreamPaused).toBe(true);
});
it("Ensure resume upstream when published is called", async () => {
videoEnabled$.next(true);
await publisher.createAndSetupTracks();
// await flushPromises();
await publisher.startPublishing();
const track = localParticipant.getTrackPublication(
Track.Source.Camera,
)?.track;
expect(track).toBeDefined();
// expect(track.pauseUpstream).toHaveBeenCalled();
expect(track!.isUpstreamPaused).toBe(false);
});
describe("Mute states", () => {
let publisher: Publisher;
beforeEach(() => {
publisher = new Publisher(
scope,
connection,
mockMediaDevices({}),
muteStates,
constant({ supported: false, processor: undefined }),
logger,
);
});
test.each([
{ mutes: { audioEnabled: true, videoEnabled: false } },
{ mutes: { audioEnabled: true, videoEnabled: false } },
])("only create the tracks that are unmuted $mutes", async ({ mutes }) => {
// Ensure all muted
audioEnabled$.next(mutes.audioEnabled);
videoEnabled$.next(mutes.videoEnabled);
vi.mocked(connection.livekitRoom.localParticipant).createTracks = vi
.fn()
.mockResolvedValue([]);
await publisher.createAndSetupTracks();
expect(
connection.livekitRoom.localParticipant.createTracks,
).toHaveBeenCalledOnce();
expect(
connection.livekitRoom.localParticipant.createTracks,
).toHaveBeenCalledWith({
audio: mutes.audioEnabled ? true : undefined,
video: mutes.videoEnabled ? true : undefined,
});
});
});
it("does mute unmute audio", async () => {});
});
describe("Bug fix", () => {
// There is a race condition when creating and publishing tracks while the mute state changes.
// This race condition could cause tracks to be published even though they are muted at the
// beginning of a call coming from lobby.
// This is caused by our stack using manually the low level API to create and publish tracks,
// but also using the higher level setMicrophoneEnabled and setCameraEnabled functions that also create
// and publish tracks, and managing pending publications.
// Race is as follow, on creation of the Publisher we create the tracks then publish them.
// If in the middle of that process the mute state changes:
// - the `setMicrophoneEnabled` will be no-op because it is not aware of our created track and can't see any pending publication
// - If start publication is requested it will publish the track even though there was a mute request.
it("wrongly publish tracks while muted", async () => {
// setLogLevel(`debug`);
const publisher = new Publisher(
scope,
connection,
@@ -73,68 +327,34 @@ describe("Publisher", () => {
constant({ supported: false, processor: undefined }),
logger,
);
audioEnabled$.next(true);
// should do nothing if no tracks have been created yet.
await publisher.startPublishing();
expect(
connection.livekitRoom.localParticipant.publishTrack,
).not.toHaveBeenCalled();
const resolvers = Promise.withResolvers<void>();
createTrackLock = resolvers.promise;
await expect(publisher.createAndSetupTracks()).rejects.toThrow(
Error("audio and video is false"),
);
(muteStates.audio.enabled$ as BehaviorSubject<boolean>).next(true);
(
connection.livekitRoom.localParticipant.createTracks as Mock
).mockResolvedValue([{}, {}]);
await expect(publisher.createAndSetupTracks()).resolves.not.toThrow();
expect(
connection.livekitRoom.localParticipant.createTracks,
).toHaveBeenCalledOnce();
// failiour due to localParticipant.publishTrack
(
connection.livekitRoom.localParticipant.publishTrack as Mock
).mockRejectedValue(Error("testError"));
await expect(publisher.startPublishing()).rejects.toThrow(
new FailToStartLivekitConnection("testError"),
);
// does not try other conenction after the first one failed
expect(
connection.livekitRoom.localParticipant.publishTrack,
).toHaveBeenCalledTimes(1);
// failiour due to connection.state$
const beforeState = connection.state$.value;
(connection.state$ as BehaviorSubject<ConnectionState>).next({
state: "FailedToStart",
error: Error("testStartError"),
// Initially the audio is unmuted, so creating tracks should publish the audio track
const createTracks = publisher.createAndSetupTracks();
void publisher.startPublishing();
void createTracks.then(() => {
void publisher.startPublishing();
});
// now mute the audio before allowing track creation to complete
audioEnabled$.next(false);
resolvers.resolve(undefined);
await createTracks;
await expect(publisher.startPublishing()).rejects.toThrow(
new FailToStartLivekitConnection("testStartError"),
);
(connection.state$ as BehaviorSubject<ConnectionState>).next(beforeState);
await flushPromises();
// does not try other conenction after the first one failed
expect(
connection.livekitRoom.localParticipant.publishTrack,
).toHaveBeenCalledTimes(1);
const track = localParticipant.getTrackPublication(
Track.Source.Microphone,
)?.track;
expect(track).toBeDefined();
// success case
(
connection.livekitRoom.localParticipant.publishTrack as Mock
).mockResolvedValue({});
await expect(publisher.startPublishing()).resolves.not.toThrow();
expect(
connection.livekitRoom.localParticipant.publishTrack,
).toHaveBeenCalledTimes(3);
try {
expect(localParticipant.publishTrack).not.toHaveBeenCalled();
} catch {
expect(track!.mute).toHaveBeenCalled();
expect(track!.isMuted).toBe(true);
}
});
});

311
src/state/CallViewModel/localMember/Publisher.ts
View File

@@ -6,15 +6,14 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
ConnectionState as LivekitConnectionState,
type LocalTrackPublication,
LocalVideoTrack,
ParticipantEvent,
type Room as LivekitRoom,
Track,
type LocalTrack,
type LocalTrackPublication,
ConnectionState as LivekitConnectionState,
} from "livekit-client";
import {
BehaviorSubject,
map,
NEVER,
type Observable,
@@ -34,10 +33,6 @@ import { getUrlParams } from "../../../UrlParams.ts";
import { observeTrackReference$ } from "../../MediaViewModel.ts";
import { type Connection } from "../remoteMembers/Connection.ts";
import { type ObservableScope } from "../../ObservableScope.ts";
import {
ElementCallError,
FailToStartLivekitConnection,
} from "../../../utils/errors.ts";
/**
* A wrapper for a Connection object.
@@ -45,14 +40,21 @@ import {
* The Publisher is also responsible for creating the media tracks.
*/
export class Publisher {
/**
* By default, livekit will start publishing tracks as soon as they are created.
* In the matrix RTC world, we want to control when tracks are published based
* on whether the user is part of the RTC session or not.
*/
public shouldPublish = false;
/**
* Creates a new Publisher.
* @param scope - The observable scope to use for managing the publisher.
* @param connection - The connection to use for publishing.
* @param devices - The media devices to use for audio and video input.
* @param muteStates - The mute states for audio and video.
* @param e2eeLivekitOptions - The E2EE options to use for the LiveKit room. Use to share the same key provider across connections!.
* @param trackerProcessorState$ - The processor state for the video track processor (e.g. background blur).
* @param logger - The logger to use for logging :D.
*/
public constructor(
private scope: ObservableScope,
@@ -62,7 +64,6 @@ export class Publisher {
trackerProcessorState$: Behavior<ProcessorState>,
private logger: Logger,
) {
this.logger.info("Create LiveKit room");
const { controlledAudioDevices } = getUrlParams();
const room = connection.livekitRoom;
@@ -80,41 +81,63 @@ export class Publisher {
this.scope.onEnd(() => {
this.logger.info("Scope ended -> stop publishing all tracks");
void this.stopPublishing();
muteStates.audio.unsetHandler();
muteStates.video.unsetHandler();
});
// TODO move mute state handling here using reconcile (instead of inside the mute state class)
// this.scope.reconcile(
// this.scope.behavior(
// combineLatest([this.muteStates.video.enabled$, this.tracks$]),
// ),
// async ([videoEnabled, tracks]) => {
// const track = tracks.find((t) => t.kind == Track.Kind.Video);
// if (!track) return;
// if (videoEnabled) {
// await track.unmute();
// } else {
// await track.mute();
// }
// },
// );
this.connection.livekitRoom.localParticipant.on(
ParticipantEvent.LocalTrackPublished,
this.onLocalTrackPublished.bind(this),
);
}
private _tracks$ = new BehaviorSubject<LocalTrack<Track.Kind>[]>([]);
public tracks$ = this._tracks$ as Behavior<LocalTrack<Track.Kind>[]>;
// LiveKit will publish the tracks as soon as they are created
// but we want to control when tracks are published.
// We cannot just mute the tracks, even if this will effectively stop the publishing,
// it would also prevent the user from seeing their own video/audio preview.
// So for that we use pauseUpStream(): Stops sending media to the server by replacing
// the sender track with null, but keeps the local MediaStreamTrack active.
// The user can still see/hear themselves locally, but remote participants see nothing.
private onLocalTrackPublished(
localTrackPublication: LocalTrackPublication,
): void {
this.logger.info("Local track published", localTrackPublication);
const lkRoom = this.connection.livekitRoom;
if (!this.shouldPublish) {
this.pauseUpstreams(lkRoom, [localTrackPublication.source]).catch((e) => {
this.logger.error(`Failed to pause upstreams`, e);
});
}
// also check the mute state and apply it
if (localTrackPublication.source === Track.Source.Microphone) {
const enabled = this.muteStates.audio.enabled$.value;
lkRoom.localParticipant.setMicrophoneEnabled(enabled).catch((e) => {
this.logger.error(
`Failed to enable microphone track, enabled:${enabled}`,
e,
);
});
} else if (localTrackPublication.source === Track.Source.Camera) {
const enabled = this.muteStates.video.enabled$.value;
lkRoom.localParticipant.setCameraEnabled(enabled).catch((e) => {
this.logger.error(
`Failed to enable camera track, enabled:${enabled}`,
e,
);
});
}
}
/**
* Start the connection to LiveKit and publish local tracks.
* Create and setup local audio and video tracks based on the current mute states.
* It creates the tracks only if audio and/or video is enabled, to avoid unnecessary
* permission prompts.
*
* This will:
* wait for the connection to be ready.
// * 1. Request an OpenId token `request_token` (allows matrix users to verify their identity with a third-party service.)
// * 2. Use this token to request the SFU config to the MatrixRtc authentication service.
// * 3. Connect to the configured LiveKit room.
// * 4. Create local audio and video tracks based on the current mute states and publish them to the room.
* It also observes mute state changes to update LiveKit microphone/camera states accordingly.
* If a track is not created initially because disabled, it will be created when unmuting.
*
* This call is not blocking anymore, instead callers can listen to the
* `RoomEvent.MediaDevicesError` event in the LiveKit room to be notified of any errors.
*
* @throws {InsufficientCapacityError} if the LiveKit server indicates that it has insufficient capacity to accept the connection.
* @throws {SFURoomCreationRestrictedError} if the LiveKit server indicates that the room does not exist and cannot be created.
*/
public async createAndSetupTracks(): Promise<void> {
this.logger.debug("createAndSetupTracks called");
@@ -122,119 +145,121 @@ export class Publisher {
// Observe mute state changes and update LiveKit microphone/camera states accordingly
this.observeMuteStates(this.scope);
// TODO-MULTI-SFU: Prepublish a microphone track
// Check if audio and/or video is enabled. We only create tracks if enabled,
// because it could prompt for permission, and we don't want to do that unnecessarily.
const audio = this.muteStates.audio.enabled$.value;
const video = this.muteStates.video.enabled$.value;
// createTracks throws if called with audio=false and video=false
if (audio || video) {
// TODO this can still throw errors? It will also prompt for permissions if not already granted
return lkRoom.localParticipant
.createTracks({
audio,
video,
})
.then((tracks) => {
this.logger.info(
"created track",
tracks.map((t) => t.kind + ", " + t.id),
);
this._tracks$.next(tracks);
})
.catch((error) => {
this.logger.error("Failed to create tracks", error);
});
// We don't await the creation, because livekit could block until the tracks
// are fully published, and not only that they are created.
// We don't have control on that, localParticipant creates and publishes the tracks
// asap.
// We are using the `ParticipantEvent.LocalTrackPublished` to be notified
// when tracks are actually published, and at that point
// we can pause upstream if needed (depending on if startPublishing has been called).
if (audio && video) {
// Enable both at once in order to have a single permission prompt!
void lkRoom.localParticipant.enableCameraAndMicrophone();
} else if (audio) {
void lkRoom.localParticipant.setMicrophoneEnabled(true);
} else if (video) {
void lkRoom.localParticipant.setCameraEnabled(true);
}
return Promise.resolve();
}
private async pauseUpstreams(
lkRoom: LivekitRoom,
sources: Track.Source[],
): Promise<void> {
for (const source of sources) {
const track = lkRoom.localParticipant.getTrackPublication(source)?.track;
if (track) {
await track.pauseUpstream();
} else {
this.logger.warn(
`No track found for source ${source} to pause upstream`,
);
}
}
}
private async resumeUpstreams(
lkRoom: LivekitRoom,
sources: Track.Source[],
): Promise<void> {
for (const source of sources) {
const track = lkRoom.localParticipant.getTrackPublication(source)?.track;
if (track) {
await track.resumeUpstream();
} else {
this.logger.warn(
`No track found for source ${source} to resume upstream`,
);
}
}
throw Error("audio and video is false");
}
private _publishing$ = new BehaviorSubject<boolean>(false);
public publishing$ = this.scope.behavior(this._publishing$);
/**
*
* Request to publish local tracks to the LiveKit room.
* This will wait for the connection to be ready before publishing.
* Livekit also have some local retry logic for publishing tracks.
* Can be called multiple times, localparticipant manages the state of published tracks (or pending publications).
*
* @returns
* @throws ElementCallError
*/
public async startPublishing(): Promise<LocalTrack[]> {
public async startPublishing(): Promise<void> {
if (this.shouldPublish) {
this.logger.debug(`Already publishing, ignoring startPublishing call`);
return;
}
this.shouldPublish = true;
this.logger.debug("startPublishing called");
const lkRoom = this.connection.livekitRoom;
const { promise, resolve, reject } = Promise.withResolvers<void>();
const sub = this.connection.state$.subscribe((s) => {
switch (s.state) {
case "ConnectedToLkRoom":
resolve();
break;
case "FailedToStart":
reject(
s.error instanceof ElementCallError
? s.error
: new FailToStartLivekitConnection(s.error.message),
);
break;
default:
this.logger.info("waiting for connection: ", s.state);
}
});
// Resume upstream for both audio and video tracks
// We need to call it explicitly because call setTrackEnabled does not always
// resume upstream. It will only if you switch the track from disabled to enabled,
// but if the track is already enabled but upstream is paused, it won't resume it.
// TODO what about screen share?
try {
await promise;
await this.resumeUpstreams(lkRoom, [
Track.Source.Microphone,
Track.Source.Camera,
]);
} catch (e) {
throw e;
} finally {
sub.unsubscribe();
this.logger.error(`Failed to resume upstreams`, e);
}
for (const track of this.tracks$.value) {
this.logger.info("publish ", this.tracks$.value.length, "tracks");
// TODO: handle errors? Needs the signaling connection to be up, but it has some retries internally
// with a timeout.
await lkRoom.localParticipant.publishTrack(track).catch((error) => {
this.logger.error("Failed to publish track", error);
throw new FailToStartLivekitConnection(
error instanceof Error ? error.message : error,
);
});
this.logger.info("published track ", track.kind, track.id);
// TODO: check if the connection is still active? and break the loop if not?
}
this._publishing$.next(true);
return this.tracks$.value;
}
public async stopPublishing(): Promise<void> {
this.logger.debug("stopPublishing called");
// TODO-MULTI-SFU: Move these calls back to ObservableScope.onEnd once scope
// actually has the right lifetime
this.muteStates.audio.unsetHandler();
this.muteStates.video.unsetHandler();
const localParticipant = this.connection.livekitRoom.localParticipant;
const tracks: LocalTrack[] = [];
const addToTracksIfDefined = (p: LocalTrackPublication): void => {
if (p.track !== undefined) tracks.push(p.track);
};
localParticipant.trackPublications.forEach(addToTracksIfDefined);
this.logger.debug(
"list of tracks to unpublish:",
tracks.map((t) => t.kind + ", " + t.id),
"start unpublishing now",
);
await localParticipant.unpublishTracks(tracks).catch((error) => {
this.logger.error("Failed to unpublish tracks", error);
throw error;
});
this.logger.debug(
"unpublished tracks",
tracks.map((t) => t.kind + ", " + t.id),
);
this._publishing$.next(false);
this.shouldPublish = false;
// Pause upstream will stop sending media to the server, while keeping
// the local MediaStreamTrack active, so the user can still see themselves.
await this.pauseUpstreams(this.connection.livekitRoom, [
Track.Source.Microphone,
Track.Source.Camera,
Track.Source.ScreenShare,
]);
}
/**
* Stops all tracks that are currently running
*/
public stopTracks(): void {
this.tracks$.value.forEach((t) => t.stop());
this._tracks$.next([]);
public async stopTracks(): Promise<void> {
const lkRoom = this.connection.livekitRoom;
for (const source of [
Track.Source.Microphone,
Track.Source.Camera,
Track.Source.ScreenShare,
]) {
const localPub = lkRoom.localParticipant.getTrackPublication(source);
if (localPub?.track) {
// stops and unpublishes the track
await lkRoom.localParticipant.unpublishTrack(localPub!.track, true);
}
}
}
/// Private methods
@@ -336,17 +361,31 @@ export class Publisher {
*/
private observeMuteStates(scope: ObservableScope): void {
const lkRoom = this.connection.livekitRoom;
this.muteStates.audio.setHandler(async (desired) => {
this.muteStates.audio.setHandler(async (enable) => {
try {
await lkRoom.localParticipant.setMicrophoneEnabled(desired);
this.logger.debug(
`handler: Setting LiveKit microphone enabled: ${enable}`,
);
await lkRoom.localParticipant.setMicrophoneEnabled(enable);
// Unmute will restart the track if it was paused upstream,
// but until explicitly requested, we want to keep it paused.
if (!this.shouldPublish && enable) {
await this.pauseUpstreams(lkRoom, [Track.Source.Microphone]);
}
} catch (e) {
this.logger.error("Failed to update LiveKit audio input mute state", e);
}
return lkRoom.localParticipant.isMicrophoneEnabled;
});
this.muteStates.video.setHandler(async (desired) => {
this.muteStates.video.setHandler(async (enable) => {
try {
await lkRoom.localParticipant.setCameraEnabled(desired);
this.logger.debug(`handler: Setting LiveKit camera enabled: ${enable}`);
await lkRoom.localParticipant.setCameraEnabled(enable);
// Unmute will restart the track if it was paused upstream,
// but until explicitly requested, we want to keep it paused.
if (!this.shouldPublish && enable) {
await this.pauseUpstreams(lkRoom, [Track.Source.Camera]);
}
} catch (e) {
this.logger.error("Failed to update LiveKit video input mute state", e);
}
@@ -362,7 +401,7 @@ export class Publisher {
const track$ = scope.behavior(
observeTrackReference$(room.localParticipant, Track.Source.Camera).pipe(
map((trackRef) => {
const track = trackRef?.publication?.track;
const track = trackRef?.publication.track;
return track instanceof LocalVideoTrack ? track : null;
}),
),

172
src/state/CallViewModel/remoteMembers/Connection.test.ts
View File

@@ -30,13 +30,16 @@ import { logger } from "matrix-js-sdk/lib/logger";
import type { LivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
import {
Connection,
ConnectionState,
type ConnectionOpts,
type ConnectionState,
type PublishingParticipant,
} from "./Connection.ts";
import { ObservableScope } from "../../ObservableScope.ts";
import { type OpenIDClientParts } from "../../../livekit/openIDSFU.ts";
import { FailToGetOpenIdToken } from "../../../utils/errors.ts";
import {
ElementCallError,
FailToGetOpenIdToken,
} from "../../../utils/errors.ts";
import { mockRemoteParticipant } from "../../../utils/test.ts";
let testScope: ObservableScope;
@@ -47,11 +50,6 @@ let fakeLivekitRoom: MockedObject<LivekitRoom>;
let localParticipantEventEmiter: EventEmitter;
let fakeLocalParticipant: MockedObject<LocalParticipant>;
let fakeRoomEventEmiter: EventEmitter;
// let fakeMembershipsFocusMap$: BehaviorSubject<
// { membership: CallMembership; transport: LivekitTransport }[]
// >;
const livekitFocus: LivekitTransport = {
livekit_alias: "!roomID:example.org",
livekit_service_url: "https://matrix-rtc.example.org/livekit/jwt",
@@ -88,22 +86,25 @@ function setupTest(): void {
localParticipantEventEmiter,
),
} as unknown as LocalParticipant);
fakeRoomEventEmiter = new EventEmitter();
const fakeRoomEventEmitter = new EventEmitter();
fakeLivekitRoom = vi.mocked<LivekitRoom>({
connect: vi.fn(),
disconnect: vi.fn(),
remoteParticipants: new Map(),
localParticipant: fakeLocalParticipant,
state: LivekitConnectionState.Disconnected,
on: fakeRoomEventEmiter.on.bind(fakeRoomEventEmiter),
off: fakeRoomEventEmiter.off.bind(fakeRoomEventEmiter),
addListener: fakeRoomEventEmiter.addListener.bind(fakeRoomEventEmiter),
on: fakeRoomEventEmitter.on.bind(fakeRoomEventEmitter),
off: fakeRoomEventEmitter.off.bind(fakeRoomEventEmitter),
addListener: fakeRoomEventEmitter.addListener.bind(fakeRoomEventEmitter),
removeListener:
fakeRoomEventEmiter.removeListener.bind(fakeRoomEventEmiter),
fakeRoomEventEmitter.removeListener.bind(fakeRoomEventEmitter),
removeAllListeners:
fakeRoomEventEmiter.removeAllListeners.bind(fakeRoomEventEmiter),
fakeRoomEventEmitter.removeAllListeners.bind(fakeRoomEventEmitter),
setE2EEEnabled: vi.fn().mockResolvedValue(undefined),
emit: (eventName: string | symbol, ...args: unknown[]) => {
fakeRoomEventEmitter.emit(eventName, ...args);
},
} as unknown as LivekitRoom);
}
@@ -125,7 +126,16 @@ function setupRemoteConnection(): Connection {
};
});
fakeLivekitRoom.connect.mockResolvedValue(undefined);
fakeLivekitRoom.connect.mockImplementation(async (): Promise<void> => {
const changeEv = RoomEvent.ConnectionStateChanged;
fakeLivekitRoom.state = LivekitConnectionState.Connecting;
fakeLivekitRoom.emit(changeEv, fakeLivekitRoom.state);
fakeLivekitRoom.state = LivekitConnectionState.Connected;
fakeLivekitRoom.emit(changeEv, fakeLivekitRoom.state);
return Promise.resolve();
});
return new Connection(opts, logger);
}
@@ -148,7 +158,7 @@ describe("Start connection states", () => {
};
const connection = new Connection(opts, logger);
expect(connection.state$.getValue().state).toEqual("Initialized");
expect(connection.state$.getValue()).toEqual("Initialized");
});
it("fail to getOpenId token then error state", async () => {
@@ -164,7 +174,7 @@ describe("Start connection states", () => {
const connection = new Connection(opts, logger);
const capturedStates: ConnectionState[] = [];
const capturedStates: (ConnectionState | Error)[] = [];
const s = connection.state$.subscribe((value) => {
capturedStates.push(value);
});
@@ -184,22 +194,20 @@ describe("Start connection states", () => {
let capturedState = capturedStates.pop();
expect(capturedState).toBeDefined();
expect(capturedState!.state).toEqual("FetchingConfig");
expect(capturedState!).toEqual("FetchingConfig");
deferred.reject(new FailToGetOpenIdToken(new Error("Failed to get token")));
await vi.runAllTimersAsync();
capturedState = capturedStates.pop();
if (capturedState!.state === "FailedToStart") {
expect(capturedState!.error.message).toEqual("Something went wrong");
if (capturedState instanceof Error) {
expect(capturedState.message).toEqual("Something went wrong");
expect(connection.transport.livekit_alias).toEqual(
livekitFocus.livekit_alias,
);
} else {
expect.fail(
"Expected FailedToStart state but got " + capturedState?.state,
);
expect.fail("Expected FailedToStart state but got " + capturedState);
}
});
@@ -216,7 +224,7 @@ describe("Start connection states", () => {
const connection = new Connection(opts, logger);
const capturedStates: ConnectionState[] = [];
const capturedStates: (ConnectionState | Error)[] = [];
const s = connection.state$.subscribe((value) => {
capturedStates.push(value);
});
@@ -238,24 +246,25 @@ describe("Start connection states", () => {
let capturedState = capturedStates.pop();
expect(capturedState).toBeDefined();
expect(capturedState?.state).toEqual("FetchingConfig");
expect(capturedState).toEqual(ConnectionState.FetchingConfig);
deferredSFU.resolve();
await vi.runAllTimersAsync();
capturedState = capturedStates.pop();
if (capturedState?.state === "FailedToStart") {
expect(capturedState?.error.message).toContain(
if (
capturedState instanceof ElementCallError &&
capturedState.cause instanceof Error
) {
expect(capturedState.cause.message).toContain(
"SFU Config fetch failed with exception Error",
);
expect(connection.transport.livekit_alias).toEqual(
livekitFocus.livekit_alias,
);
} else {
expect.fail(
"Expected FailedToStart state but got " + capturedState?.state,
);
expect.fail("Expected FailedToStart state but got " + capturedState);
}
});
@@ -272,7 +281,7 @@ describe("Start connection states", () => {
const connection = new Connection(opts, logger);
const capturedStates: ConnectionState[] = [];
const capturedStates: (ConnectionState | Error)[] = [];
const s = connection.state$.subscribe((value) => {
capturedStates.push(value);
});
@@ -302,15 +311,18 @@ describe("Start connection states", () => {
let capturedState = capturedStates.pop();
expect(capturedState).toBeDefined();
expect(capturedState?.state).toEqual("FetchingConfig");
expect(capturedState).toEqual(ConnectionState.FetchingConfig);
deferredSFU.resolve();
await vi.runAllTimersAsync();
capturedState = capturedStates.pop();
if (capturedState && capturedState?.state === "FailedToStart") {
expect(capturedState.error.message).toContain(
if (
capturedState instanceof ElementCallError &&
capturedState.cause instanceof Error
) {
expect(capturedState.cause.message).toContain(
"Failed to connect to livekit",
);
expect(connection.transport.livekit_alias).toEqual(
@@ -329,7 +341,7 @@ describe("Start connection states", () => {
const connection = setupRemoteConnection();
const capturedStates: ConnectionState[] = [];
const capturedStates: (ConnectionState | Error)[] = [];
const s = connection.state$.subscribe((value) => {
capturedStates.push(value);
});
@@ -339,13 +351,15 @@ describe("Start connection states", () => {
await vi.runAllTimersAsync();
const initialState = capturedStates.shift();
expect(initialState?.state).toEqual("Initialized");
expect(initialState).toEqual(ConnectionState.Initialized);
const fetchingState = capturedStates.shift();
expect(fetchingState?.state).toEqual("FetchingConfig");
expect(fetchingState).toEqual(ConnectionState.FetchingConfig);
const disconnectedState = capturedStates.shift();
expect(disconnectedState).toEqual(ConnectionState.LivekitDisconnected);
const connectingState = capturedStates.shift();
expect(connectingState?.state).toEqual("ConnectingToLkRoom");
expect(connectingState).toEqual(ConnectionState.LivekitConnecting);
const connectedState = capturedStates.shift();
expect(connectedState?.state).toEqual("ConnectedToLkRoom");
expect(connectedState).toEqual(ConnectionState.LivekitConnected);
});
it("shutting down the scope should stop the connection", async () => {
@@ -363,44 +377,32 @@ describe("Start connection states", () => {
});
});
function fakeRemoteLivekitParticipant(
id: string,
publications: number = 1,
): RemoteParticipant {
return {
identity: id,
getTrackPublications: () => Array(publications),
} as unknown as RemoteParticipant;
}
describe("Publishing participants observations", () => {
it("should emit the list of publishing participants", () => {
describe("remote participants", () => {
it("emits the list of remote participants", () => {
setupTest();
const connection = setupRemoteConnection();
const bobIsAPublisher = Promise.withResolvers<void>();
const danIsAPublisher = Promise.withResolvers<void>();
const observedPublishers: PublishingParticipant[][] = [];
const s = connection.remoteParticipants$.subscribe((publishers) => {
observedPublishers.push(publishers);
if (publishers.some((p) => p.identity === "@bob:example.org:DEV111")) {
bobIsAPublisher.resolve();
}
if (publishers.some((p) => p.identity === "@dan:example.org:DEV333")) {
danIsAPublisher.resolve();
}
const observedParticipants: RemoteParticipant[][] = [];
const s = connection.remoteParticipants$.subscribe((participants) => {
observedParticipants.push(participants);
});
onTestFinished(() => s.unsubscribe());
// The publishingParticipants$ observable is derived from the current members of the
// The remoteParticipants$ observable is derived from the current members of the
// livekitRoom and the rtc membership in order to publish the members that are publishing
// on this connection.
let participants: RemoteParticipant[] = [
fakeRemoteLivekitParticipant("@alice:example.org:DEV000", 0),
fakeRemoteLivekitParticipant("@bob:example.org:DEV111", 0),
fakeRemoteLivekitParticipant("@carol:example.org:DEV222", 0),
fakeRemoteLivekitParticipant("@dan:example.org:DEV333", 0),
const participants: RemoteParticipant[] = [
mockRemoteParticipant({ identity: "@alice:example.org:DEV000" }),
mockRemoteParticipant({ identity: "@bob:example.org:DEV111" }),
mockRemoteParticipant({ identity: "@carol:example.org:DEV222" }),
// Mock Dan to have no published tracks. We want him to still show show up
// in the participants list.
mockRemoteParticipant({
identity: "@dan:example.org:DEV333",
getTrackPublication: () => undefined,
getTrackPublications: () => [],
}),
];
// Let's simulate 3 members on the livekitRoom
@@ -409,9 +411,10 @@ describe("Publishing participants observations", () => {
);
participants.forEach((p) =>
fakeRoomEventEmiter.emit(RoomEvent.ParticipantConnected, p),
fakeLivekitRoom.emit(RoomEvent.ParticipantConnected, p),
);
<<<<<<< HEAD
// At this point there should be ~~no~~ publishers
// We do have publisher now, since we do not filter for publishers anymore (to also have participants with only data tracks)
// The filtering we do is just based on the matrixRTC member events.
@@ -429,6 +432,10 @@ describe("Publishing participants observations", () => {
// At this point there should be no publishers
expect(observedPublishers.pop()!.length).toEqual(4);
=======
// All remote participants should be present
expect(observedParticipants.pop()!.length).toEqual(4);
>>>>>>> livekit
});
it("should be scoped to parent scope", (): void => {
@@ -436,14 +443,20 @@ describe("Publishing participants observations", () => {
const connection = setupRemoteConnection();
<<<<<<< HEAD
let observedPublishers: PublishingParticipant[][] = [];
const s = connection.remoteParticipants$.subscribe((publishers) => {
observedPublishers.push(publishers);
=======
let observedParticipants: RemoteParticipant[][] = [];
const s = connection.remoteParticipants$.subscribe((participants) => {
observedParticipants.push(participants);
>>>>>>> livekit
});
onTestFinished(() => s.unsubscribe());
let participants: RemoteParticipant[] = [
fakeRemoteLivekitParticipant("@bob:example.org:DEV111", 0),
mockRemoteParticipant({ identity: "@bob:example.org:DEV111" }),
];
// Let's simulate 3 members on the livekitRoom
@@ -452,9 +465,10 @@ describe("Publishing participants observations", () => {
);
for (const participant of participants) {
fakeRoomEventEmiter.emit(RoomEvent.ParticipantConnected, participant);
fakeLivekitRoom.emit(RoomEvent.ParticipantConnected, participant);
}
<<<<<<< HEAD
// At this point there should be ~~no~~ publishers
// We do have publisher now, since we do not filter for publishers anymore (to also have participants with only data tracks)
// The filtering we do is just based on the matrixRTC member events.
@@ -470,22 +484,28 @@ describe("Publishing participants observations", () => {
const publishers = observedPublishers.pop();
expect(publishers?.length).toEqual(1);
expect(publishers?.[0]?.identity).toEqual("@bob:example.org:DEV111");
=======
// We should have bob as a participant now
const ps = observedParticipants.pop();
expect(ps?.length).toEqual(1);
expect(ps?.[0]?.identity).toEqual("@bob:example.org:DEV111");
>>>>>>> livekit
// end the parent scope
testScope.end();
observedPublishers = [];
observedParticipants = [];
// SHOULD NOT emit any more publishers as the scope is ended
// SHOULD NOT emit any more participants as the scope is ended
participants = participants.filter(
(p) => p.identity !== "@bob:example.org:DEV111",
);
fakeRoomEventEmiter.emit(
fakeLivekitRoom.emit(
RoomEvent.ParticipantDisconnected,
fakeRemoteLivekitParticipant("@bob:example.org:DEV111"),
mockRemoteParticipant({ identity: "@bob:example.org:DEV111" }),
);
expect(observedPublishers.length).toEqual(0);
expect(observedParticipants.length).toEqual(0);
});
});

127
src/state/CallViewModel/remoteMembers/Connection.ts
View File

@@ -12,11 +12,8 @@ import {
} from "@livekit/components-core";
import {
ConnectionError,
type ConnectionState as LivekitConenctionState,
type Room as LivekitRoom,
type LocalParticipant,
type RemoteParticipant,
RoomEvent,
} from "livekit-client";
import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
import { BehaviorSubject } from "rxjs";
@@ -30,12 +27,12 @@ import {
import { type Behavior } from "../../Behavior.ts";
import { type ObservableScope } from "../../ObservableScope.ts";
import {
ElementCallError,
InsufficientCapacityError,
SFURoomCreationRestrictedError,
UnknownCallError,
} from "../../../utils/errors.ts";
export type PublishingParticipant = LocalParticipant | RemoteParticipant;
export interface ConnectionOpts {
/** The media transport to connect to. */
transport: LivekitTransport;
@@ -47,17 +44,30 @@ export interface ConnectionOpts {
/** Optional factory to create the LiveKit room, mainly for testing purposes. */
livekitRoomFactory: () => LivekitRoom;
}
export class FailedToStartError extends Error {
public constructor(message: string) {
super(message);
this.name = "FailedToStartError";
}
}
export type ConnectionState =
| { state: "Initialized" }
| { state: "FetchingConfig" }
| { state: "ConnectingToLkRoom" }
| {
state: "ConnectedToLkRoom";
livekitConnectionState$: Behavior<LivekitConenctionState>;
}
| { state: "FailedToStart"; error: Error }
| { state: "Stopped" };
export enum ConnectionState {
/** The start state of a connection. It has been created but nothing has loaded yet. */
Initialized = "Initialized",
/** `start` has been called on the connection. It aquires the jwt info to conenct to the LK Room */
FetchingConfig = "FetchingConfig",
Stopped = "Stopped",
/** The same as ConnectionState.Disconnected from `livekit-client` */
LivekitDisconnected = "disconnected",
/** The same as ConnectionState.Connecting from `livekit-client` */
LivekitConnecting = "connecting",
/** The same as ConnectionState.Connected from `livekit-client` */
LivekitConnected = "connected",
/** The same as ConnectionState.Reconnecting from `livekit-client` */
LivekitReconnecting = "reconnecting",
/** The same as ConnectionState.SignalReconnecting from `livekit-client` */
LivekitSignalReconnecting = "signalReconnecting",
}
/**
* A connection to a Matrix RTC LiveKit backend.
@@ -66,14 +76,14 @@ export type ConnectionState =
*/
export class Connection {
// Private Behavior
private readonly _state$ = new BehaviorSubject<ConnectionState>({
state: "Initialized",
});
private readonly _state$ = new BehaviorSubject<
ConnectionState | ElementCallError
>(ConnectionState.Initialized);
/**
* The current state of the connection to the media transport.
*/
public readonly state$: Behavior<ConnectionState> = this._state$;
public readonly state$: Behavior<ConnectionState | Error> = this._state$;
/**
* The media transport to connect to.
@@ -85,11 +95,13 @@ export class Connection {
private scope: ObservableScope;
/**
* An observable of the participants that are publishing on this connection. (Excluding our local participant)
* This is derived from `participantsIncludingSubscribers$` and `remoteTransports$`.
* It filters the participants to only those that are associated with a membership that claims to publish on this connection.
* The remote LiveKit participants that are visible on this connection.
*
* Note that this may include participants that are connected only to
* subscribe, or publishers that are otherwise unattested in MatrixRTC state.
* It is therefore more low-level than what should be presented to the user.
*/
public readonly remoteParticipants$: Behavior<PublishingParticipant[]>;
public readonly remoteParticipants$: Behavior<RemoteParticipant[]>;
/**
* Whether the connection has been stopped.
@@ -115,16 +127,24 @@ export class Connection {
this.logger.debug("Starting Connection");
this.stopped = false;
try {
this._state$.next({
state: "FetchingConfig",
});
this._state$.next(ConnectionState.FetchingConfig);
// We should already have this information after creating the localTransport.
// It would probably be better to forward this here.
const { url, jwt } = await this.getSFUConfigWithOpenID();
// If we were stopped while fetching the config, don't proceed to connect
if (this.stopped) return;
this._state$.next({
state: "ConnectingToLkRoom",
});
// Setup observer once we are done with getSFUConfigWithOpenID
connectionStateObserver(this.livekitRoom)
.pipe(
this.scope.bind(),
map((s) => s as unknown as ConnectionState),
)
.subscribe((lkState) => {
// It is save to cast lkState to ConnectionState as they are fully overlapping.
this._state$.next(lkState);
});
try {
await this.livekitRoom.connect(url, jwt);
} catch (e) {
@@ -139,7 +159,8 @@ export class Connection {
throw new InsufficientCapacityError();
}
if (e.status === 404) {
// error msg is "Could not establish signal connection: requested room does not exist"
// error msg is "Failed to create call"
// error description is "Call creation might be restricted to authorized users only. Try again later, or contact your server admin if the problem persists."
// The room does not exist. There are two different modes of operation for the SFU:
// - the room is created on the fly when connecting (livekit `auto_create` option)
// - Only authorized users can create rooms, so the room must exist before connecting (done by the auth jwt service)
@@ -151,23 +172,16 @@ export class Connection {
}
// If we were stopped while connecting, don't proceed to update state.
if (this.stopped) return;
this._state$.next({
state: "ConnectedToLkRoom",
livekitConnectionState$: this.scope.behavior(
connectionStateObserver(this.livekitRoom),
),
});
this.logger.info(
"Connected to LiveKit room",
this.transport.livekit_service_url,
);
} catch (error) {
this.logger.debug(`Failed to connect to LiveKit room: ${error}`);
this._state$.next({
state: "FailedToStart",
error: error instanceof Error ? error : new Error(`${error}`),
});
this._state$.next(
error instanceof ElementCallError
? error
: error instanceof Error
? new UnknownCallError(error)
: new UnknownCallError(new Error(`${error}`)),
);
// Its okay to ignore the throw. The error is part of the state.
throw error;
}
}
@@ -192,9 +206,7 @@ export class Connection {
);
if (this.stopped) return;
await this.livekitRoom.disconnect();
this._state$.next({
state: "Stopped",
});
this._state$.next(ConnectionState.Stopped);
this.stopped = true;
}
@@ -220,24 +232,9 @@ export class Connection {
this.transport = transport;
this.client = client;
// REMOTE participants with track!!!
// this.remoteParticipants$
this.remoteParticipants$ = scope.behavior(
// only tracks remote participants
connectedParticipantsObserver(this.livekitRoom, {
additionalRoomEvents: [
RoomEvent.TrackPublished,
RoomEvent.TrackUnpublished,
],
}),
// .pipe(
// map((participants) => {
// return participants.filter(
// (participant) => participant.getTrackPublications().length > 0,
// );
// }),
// )
[],
// Only tracks remote participants
connectedParticipantsObserver(this.livekitRoom),
);
scope.onEnd(() => {

39
src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts
View File

@@ -8,7 +8,7 @@ Please see LICENSE in the repository root for full details.
import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";
import { BehaviorSubject } from "rxjs";
import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
import { type Participant as LivekitParticipant } from "livekit-client";
import { type RemoteParticipant } from "livekit-client";
import { logger } from "matrix-js-sdk/lib/logger";
import { Epoch, mapEpoch, ObservableScope } from "../../ObservableScope.ts";
@@ -201,23 +201,20 @@ describe("connections$ stream", () => {
describe("connectionManagerData$ stream", () => {
// Used in test to control fake connections' remoteParticipants$ streams
let fakePublishingParticipantsStreams: Map<
string,
Behavior<LivekitParticipant[]>
>;
let fakeRemoteParticipantsStreams: Map<string, Behavior<RemoteParticipant[]>>;
function keyForTransport(transport: LivekitTransport): string {
return `${transport.livekit_service_url}|${transport.livekit_alias}`;
}
beforeEach(() => {
fakePublishingParticipantsStreams = new Map();
fakeRemoteParticipantsStreams = new Map();
function getPublishingParticipantsFor(
function getRemoteParticipantsFor(
transport: LivekitTransport,
): Behavior<LivekitParticipant[]> {
): Behavior<RemoteParticipant[]> {
return (
fakePublishingParticipantsStreams.get(keyForTransport(transport)) ??
fakeRemoteParticipantsStreams.get(keyForTransport(transport)) ??
new BehaviorSubject([])
);
}
@@ -227,12 +224,12 @@ describe("connectionManagerData$ stream", () => {
.fn()
.mockImplementation(
(transport: LivekitTransport, scope: ObservableScope) => {
const fakePublishingParticipants$ = new BehaviorSubject<
LivekitParticipant[]
const fakeRemoteParticipants$ = new BehaviorSubject<
RemoteParticipant[]
>([]);
const mockConnection = {
transport,
remoteParticipants$: getPublishingParticipantsFor(transport),
remoteParticipants$: getRemoteParticipantsFor(transport),
} as unknown as Connection;
vi.mocked(mockConnection).start = vi.fn();
vi.mocked(mockConnection).stop = vi.fn();
@@ -241,36 +238,36 @@ describe("connectionManagerData$ stream", () => {
void mockConnection.stop();
});
fakePublishingParticipantsStreams.set(
fakeRemoteParticipantsStreams.set(
keyForTransport(transport),
fakePublishingParticipants$,
fakeRemoteParticipants$,
);
return mockConnection;
},
);
});
test("Should report connections with the publishing participants", () => {
test("Should report connections with the remote participants", () => {
withTestScheduler(({ expectObservable, schedule, behavior }) => {
// Setup the fake participants streams behavior
// ==============================
fakePublishingParticipantsStreams.set(
fakeRemoteParticipantsStreams.set(
keyForTransport(TRANSPORT_1),
behavior("oa-b", {
o: [],
a: [{ identity: "user1A" } as LivekitParticipant],
a: [{ identity: "user1A" } as RemoteParticipant],
b: [
{ identity: "user1A" } as LivekitParticipant,
{ identity: "user1B" } as LivekitParticipant,
{ identity: "user1A" } as RemoteParticipant,
{ identity: "user1B" } as RemoteParticipant,
],
}),
);
fakePublishingParticipantsStreams.set(
fakeRemoteParticipantsStreams.set(
keyForTransport(TRANSPORT_2),
behavior("o-a", {
o: [],
a: [{ identity: "user2A" } as LivekitParticipant],
a: [{ identity: "user2A" } as RemoteParticipant],
}),
);
// ==============================

56
src/state/CallViewModel/remoteMembers/ConnectionManager.ts
View File

@@ -6,13 +6,10 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
type LivekitTransport,
type ParticipantId,
} from "matrix-js-sdk/lib/matrixrtc";
import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
import { combineLatest, map, of, switchMap, tap } from "rxjs";
import { type Logger } from "matrix-js-sdk/lib/logger";
import { type LocalParticipant, type RemoteParticipant } from "livekit-client";
import { type RemoteParticipant } from "livekit-client";
import { type Behavior } from "../../Behavior.ts";
import { type Connection } from "./Connection.ts";
@@ -22,20 +19,12 @@ import { areLivekitTransportsEqual } from "./MatrixLivekitMembers.ts";
import { type ConnectionFactory } from "./ConnectionFactory.ts";
export class ConnectionManagerData {
private readonly store: Map<
string,
{
connection: Connection;
participants: (LocalParticipant | RemoteParticipant)[];
}
> = new Map();
private readonly store: Map<string, [Connection, RemoteParticipant[]]> =
new Map();
public constructor() {}
public add(
connection: Connection,
participants: (LocalParticipant | RemoteParticipant)[],
): void {
public add(connection: Connection, participants: RemoteParticipant[]): void {
const key = this.getKey(connection.transport);
const existing = this.store.get(key);
if (!existing) {
@@ -61,7 +50,7 @@ export class ConnectionManagerData {
public getParticipantsForTransport(
transport: LivekitTransport,
): (LocalParticipant | RemoteParticipant)[] {
): RemoteParticipant[] {
const key = transport.livekit_service_url + "|" + transport.livekit_alias;
const existing = this.store.get(key);
if (existing) {
@@ -69,39 +58,20 @@ export class ConnectionManagerData {
}
return [];
}
/**
* Get all connections where the given participant is publishing.
* In theory, there could be several connections where the same participant is publishing but with
* only well behaving clients a participant should only be publishing on a single connection.
* @param participantId
*/
public getConnectionsForParticipant(
participantId: ParticipantId,
): Connection[] {
const connections: Connection[] = [];
for (const { connection, participants } of this.store.values()) {
if (
participants.some(
(participant) => participant?.identity === participantId,
)
) {
connections.push(connection);
}
}
return connections;
}
}
interface Props {
scope: ObservableScope;
connectionFactory: ConnectionFactory;
inputTransports$: Behavior<Epoch<LivekitTransport[]>>;
logger: Logger;
}
// TODO - write test for scopes (do we really need to bind scope)
export interface IConnectionManager {
connectionManagerData$: Behavior<Epoch<ConnectionManagerData>>;
}
/**
* Crete a `ConnectionManager`
* @param scope the observable scope used by this object.
@@ -184,7 +154,7 @@ export function createConnectionManager$({
const epoch = connections.epoch;
// Map the connections to list of {connection, participants}[]
const listOfConnectionsWithParticipants = connections.value.map(
const listOfConnectionsWithRemoteParticipants = connections.value.map(
(connection) => {
return connection.remoteParticipants$.pipe(
map((participants) => ({
@@ -196,12 +166,16 @@ export function createConnectionManager$({
);
// probably not required
<<<<<<< HEAD
if (listOfConnectionsWithParticipants.length === 0) {
=======
if (listOfConnectionsWithRemoteParticipants.length === 0) {
>>>>>>> livekit
return of(new Epoch(new ConnectionManagerData(), epoch));
}
// combineLatest the several streams into a single stream with the ConnectionManagerData
return combineLatest(listOfConnectionsWithParticipants).pipe(
return combineLatest(listOfConnectionsWithRemoteParticipants).pipe(
map(
(lists) =>
new Epoch(

117
src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts
View File

@@ -15,7 +15,7 @@ import { combineLatest, map, type Observable } from "rxjs";
import { type IConnectionManager } from "./ConnectionManager.ts";
import {
type MatrixLivekitMember,
type RemoteMatrixLivekitMember,
createMatrixLivekitMembers$,
} from "./MatrixLivekitMembers.ts";
import {
@@ -99,24 +99,21 @@ test("should signal participant not yet connected to livekit", () => {
} as unknown as IConnectionManager,
});
expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe(
"a",
{
a: expect.toSatisfy((data: MatrixLivekitMember[]) => {
expect(data.length).toEqual(1);
expectObservable(data[0].membership$).toBe("a", {
a: bobMembership,
});
expectObservable(data[0].participant$).toBe("a", {
a: null,
});
expectObservable(data[0].connection$).toBe("a", {
a: null,
});
return true;
}),
},
);
expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe("a", {
a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => {
expect(data.length).toEqual(1);
expectObservable(data[0].membership$).toBe("a", {
a: bobMembership,
});
expectObservable(data[0].participant.value$).toBe("a", {
a: null,
});
expectObservable(data[0].connection$).toBe("a", {
a: null,
});
return true;
}),
});
});
});
@@ -182,28 +179,25 @@ test("should signal participant on a connection that is publishing", () => {
} as unknown as IConnectionManager,
});
expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe(
"a",
{
a: expect.toSatisfy((data: MatrixLivekitMember[]) => {
expect(data.length).toEqual(1);
expectObservable(data[0].membership$).toBe("a", {
a: bobMembership,
});
expectObservable(data[0].participant$).toBe("a", {
a: expect.toSatisfy((participant) => {
expect(participant).toBeDefined();
expect(participant!.identity).toEqual(bobParticipantId);
return true;
}),
});
expectObservable(data[0].connection$).toBe("a", {
a: connection,
});
return true;
}),
},
);
expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe("a", {
a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => {
expect(data.length).toEqual(1);
expectObservable(data[0].membership$).toBe("a", {
a: bobMembership,
});
expectObservable(data[0].participant.value$).toBe("a", {
a: expect.toSatisfy((participant) => {
expect(participant).toBeDefined();
expect(participant!.identity).toEqual(bobParticipantId);
return true;
}),
});
expectObservable(data[0].connection$).toBe("a", {
a: connection,
});
return true;
}),
});
});
});
@@ -236,24 +230,21 @@ test("should signal participant on a connection that is not publishing", () => {
} as unknown as IConnectionManager,
});
expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe(
"a",
{
a: expect.toSatisfy((data: MatrixLivekitMember[]) => {
expect(data.length).toEqual(1);
expectObservable(data[0].membership$).toBe("a", {
a: bobMembership,
});
expectObservable(data[0].participant$).toBe("a", {
a: null,
});
expectObservable(data[0].connection$).toBe("a", {
a: connection,
});
return true;
}),
},
);
expectObservable(matrixLivekitMember$.pipe(map((e) => e.value))).toBe("a", {
a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => {
expect(data.length).toEqual(1);
expectObservable(data[0].membership$).toBe("a", {
a: bobMembership,
});
expectObservable(data[0].participant.value$).toBe("a", {
a: null,
});
expectObservable(data[0].connection$).toBe("a", {
a: connection,
});
return true;
}),
});
});
});
@@ -305,7 +296,7 @@ describe("Publication edge case", () => {
expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe(
"a",
{
a: expect.toSatisfy((data: MatrixLivekitMember[]) => {
a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => {
expect(data.length).toEqual(2);
expectObservable(data[0].membership$).toBe("a", {
a: bobMembership,
@@ -314,7 +305,7 @@ describe("Publication edge case", () => {
// The real connection should be from transportA as per the membership
a: connectionA,
});
expectObservable(data[0].participant$).toBe("a", {
expectObservable(data[0].participant.value$).toBe("a", {
a: expect.toSatisfy((participant) => {
expect(participant).toBeDefined();
expect(participant!.identity).toEqual(bobParticipantId);
@@ -371,7 +362,7 @@ describe("Publication edge case", () => {
expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe(
"a",
{
a: expect.toSatisfy((data: MatrixLivekitMember[]) => {
a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => {
expect(data.length).toEqual(2);
expectObservable(data[0].membership$).toBe("a", {
a: bobMembership,
@@ -380,7 +371,7 @@ describe("Publication edge case", () => {
// The real connection should be from transportA as per the membership
a: connectionA,
});
expectObservable(data[0].participant$).toBe("a", {
expectObservable(data[0].participant.value$).toBe("a", {
// No participant as Bob is not publishing on his membership transport
a: null,
});

51
src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts
View File

@@ -5,10 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
type LocalParticipant as LocalLivekitParticipant,
type RemoteParticipant as RemoteLivekitParticipant,
} from "livekit-client";
import { type LocalParticipant, type RemoteParticipant } from "livekit-client";
import {
type LivekitTransport,
type CallMembership,
@@ -24,22 +21,44 @@ import { generateItemsWithEpoch } from "../../../utils/observable";
const logger = rootLogger.getChild("[MatrixLivekitMembers]");
/**
* Represents a Matrix call member and their associated LiveKit participation.
* `livekitParticipant` can be undefined if the member is not yet connected to the livekit room
* or if it has no livekit transport at all.
*/
export interface MatrixLivekitMember {
interface LocalTaggedParticipant {
type: "local";
value$: Behavior<LocalParticipant | null>;
}
interface RemoteTaggedParticipant {
type: "remote";
value$: Behavior<RemoteParticipant | null>;
}
export type TaggedParticipant =
| LocalTaggedParticipant
| RemoteTaggedParticipant;
interface MatrixLivekitMember {
membership$: Behavior<CallMembership>;
participant$: Behavior<
LocalLivekitParticipant | RemoteLivekitParticipant | null
>;
connection$: Behavior<Connection | null>;
// participantId: string; We do not want a participantId here since it will be generated by the jwt
// TODO decide if we can also drop the userId. Its in the matrix membership anyways.
userId: string;
}
/**
* Represents the local Matrix call member and their associated LiveKit participation.
* `livekitParticipant` can be null if the member is not yet connected to the livekit room
* or if it has no livekit transport at all.
*/
export interface LocalMatrixLivekitMember extends MatrixLivekitMember {
participant: LocalTaggedParticipant;
}
/**
* Represents a remote Matrix call member and their associated LiveKit participation.
* `livekitParticipant` can be null if the member is not yet connected to the livekit room
* or if it has no livekit transport at all.
*/
export interface RemoteMatrixLivekitMember extends MatrixLivekitMember {
participant: RemoteTaggedParticipant;
}
interface Props {
scope: ObservableScope;
membershipsWithTransport$: Behavior<
@@ -61,7 +80,7 @@ export function createMatrixLivekitMembers$({
scope,
membershipsWithTransport$,
connectionManager,
}: Props): { matrixLivekitMembers$: Behavior<Epoch<MatrixLivekitMember[]>> } {
}: Props): Behavior<Epoch<RemoteMatrixLivekitMember[]>> {
/**
* Stream of all the call members and their associated livekit data (if available).
*/
@@ -110,12 +129,14 @@ export function createMatrixLivekitMembers$({
logger.debug(
`Generating member for participantId: ${participantId}, userId: ${userId}`,
);
const { participant$, ...rest } = scope.splitBehavior(data$);
// will only get called once per `participantId, userId` pair.
// updates to data$ and as a result to displayName$ and mxcAvatarUrl$ are more frequent.
return {
participantId,
userId,
...scope.splitBehavior(data$),
participant: { type: "remote" as const, value$: participant$ },
...rest,
};
},
),

18
src/state/CallViewModel/remoteMembers/integration.test.ts
View File

@@ -29,7 +29,7 @@ import { type ProcessorState } from "../../../livekit/TrackProcessorContext.tsx"
import {
areLivekitTransportsEqual,
createMatrixLivekitMembers$,
type MatrixLivekitMember,
type RemoteMatrixLivekitMember,
} from "./MatrixLivekitMembers.ts";
import { createConnectionManager$ } from "./ConnectionManager.ts";
import { membershipsAndTransports$ } from "../../SessionBehaviors.ts";
@@ -131,8 +131,8 @@ test("bob, carl, then bob joining no tracks yet", () => {
connectionManager,
});
expectObservable(matrixLivekitMembers$).toBe(vMarble, {
a: expect.toSatisfy((e: Epoch<MatrixLivekitMember[]>) => {
expectObservable(matrixLivekitItems$).toBe(vMarble, {
a: expect.toSatisfy((e: Epoch<RemoteMatrixLivekitMember[]>) => {
const items = e.value;
expect(items.length).toBe(1);
const item = items[0]!;
@@ -147,12 +147,12 @@ test("bob, carl, then bob joining no tracks yet", () => {
),
),
});
expectObservable(item.participant$).toBe("a", {
expectObservable(item.participant.value$).toBe("a", {
a: null,
});
return true;
}),
b: expect.toSatisfy((e: Epoch<MatrixLivekitMember[]>) => {
b: expect.toSatisfy((e: Epoch<RemoteMatrixLivekitMember[]>) => {
const items = e.value;
expect(items.length).toBe(2);
@@ -161,7 +161,7 @@ test("bob, carl, then bob joining no tracks yet", () => {
expectObservable(item.membership$).toBe("a", {
a: bobMembership,
});
expectObservable(item.participant$).toBe("a", {
expectObservable(item.participant.value$).toBe("a", {
a: null,
});
}
@@ -172,7 +172,7 @@ test("bob, carl, then bob joining no tracks yet", () => {
expectObservable(item.membership$).toBe("a", {
a: carlMembership,
});
expectObservable(item.participant$).toBe("a", {
expectObservable(item.participant.value$).toBe("a", {
a: null,
});
expectObservable(item.connection$).toBe("a", {
@@ -189,7 +189,7 @@ test("bob, carl, then bob joining no tracks yet", () => {
}
return true;
}),
c: expect.toSatisfy((e: Epoch<MatrixLivekitMember[]>) => {
c: expect.toSatisfy((e: Epoch<RemoteMatrixLivekitMember[]>) => {
const items = e.value;
expect(items.length).toBe(3);
@@ -216,7 +216,7 @@ test("bob, carl, then bob joining no tracks yet", () => {
return true;
}),
});
expectObservable(item.participant$).toBe("a", {
expectObservable(item.participant.value$).toBe("a", {
a: null,
});
}

70
src/state/CallViewModelWidget.test.ts
View File

@@ -15,6 +15,7 @@ import { constant } from "./Behavior.ts";
import { aliceParticipant, localRtcMember } from "../utils/test-fixtures.ts";
import { ElementWidgetActions, widget } from "../widget.ts";
import { E2eeType } from "../e2ee/e2eeType.ts";
import { MatrixRTCMode } from "../settings/settings.ts";
vi.mock("@livekit/components-core", { spy: true });
@@ -34,36 +35,43 @@ vi.mock("../widget", () => ({
},
}));
it("expect leave when ElementWidgetActions.HangupCall is called", async () => {
const pr = Promise.withResolvers<string>();
withCallViewModel(
{
remoteParticipants$: constant([aliceParticipant]),
rtcMembers$: constant([localRtcMember]),
},
(vm: CallViewModel) => {
vm.leave$.subscribe((s: string) => {
pr.resolve(s);
});
it.each([
[MatrixRTCMode.Legacy],
[MatrixRTCMode.Compatibil],
[MatrixRTCMode.Matrix_2_0],
])(
"expect leave when ElementWidgetActions.HangupCall is called (%s mode)",
async (mode) => {
const pr = Promise.withResolvers<string>();
withCallViewModel(mode)(
{
remoteParticipants$: constant([aliceParticipant]),
rtcMembers$: constant([localRtcMember]),
},
(vm: CallViewModel) => {
vm.leave$.subscribe((s: string) => {
pr.resolve(s);
});
widget!.lazyActions!.emit(
ElementWidgetActions.HangupCall,
new CustomEvent(ElementWidgetActions.HangupCall, {
detail: {
action: "im.vector.hangup",
api: "toWidget",
data: {},
requestId: "widgetapi-1761237395918",
widgetId: "mrUjS9T6uKUOWHMxXvLbSv0F",
},
}),
);
},
{
encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
},
);
widget!.lazyActions!.emit(
ElementWidgetActions.HangupCall,
new CustomEvent(ElementWidgetActions.HangupCall, {
detail: {
action: "im.vector.hangup",
api: "toWidget",
data: {},
requestId: "widgetapi-1761237395918",
widgetId: "mrUjS9T6uKUOWHMxXvLbSv0F",
},
}),
);
},
{
encryptionSystem: { kind: E2eeType.PER_PARTICIPANT },
},
);
const source = await pr.promise;
expect(source).toBe("user");
});
const source = await pr.promise;
expect(source).toBe("user");
},
);

41
src/state/MediaViewModel.test.ts
View File

@@ -20,6 +20,7 @@ import {
createLocalMedia,
createRemoteMedia,
withTestScheduler,
mockRemoteParticipant,
} from "../utils/test";
import { getValue } from "../utils/observable";
import { constant } from "./Behavior";
@@ -44,7 +45,11 @@ const rtcMembership = mockRtcMembership("@alice:example.org", "AAAA");
test("control a participant's volume", () => {
const setVolumeSpy = vi.fn();
const vm = createRemoteMedia(rtcMembership, {}, { setVolume: setVolumeSpy });
const vm = createRemoteMedia(
rtcMembership,
{},
mockRemoteParticipant({ setVolume: setVolumeSpy }),
);
withTestScheduler(({ expectObservable, schedule }) => {
schedule("-ab---c---d|", {
a() {
@@ -88,7 +93,7 @@ test("control a participant's volume", () => {
});
test("toggle fit/contain for a participant's video", () => {
const vm = createRemoteMedia(rtcMembership, {}, {});
const vm = createRemoteMedia(rtcMembership, {}, mockRemoteParticipant({}));
withTestScheduler(({ expectObservable, schedule }) => {
schedule("-ab|", {
a: () => vm.toggleFitContain(),
@@ -199,3 +204,35 @@ test("switch cameras", async () => {
});
expect(deviceId).toBe("front camera");
});
test("remote media is in waiting state when participant has not yet connected", () => {
const vm = createRemoteMedia(rtcMembership, {}, null); // null participant
expect(vm.waitingForMedia$.value).toBe(true);
});
test("remote media is not in waiting state when participant is connected", () => {
const vm = createRemoteMedia(rtcMembership, {}, mockRemoteParticipant({}));
expect(vm.waitingForMedia$.value).toBe(false);
});
test("remote media is not in waiting state when participant is connected with no publications", () => {
const vm = createRemoteMedia(
rtcMembership,
{},
mockRemoteParticipant({
getTrackPublication: () => undefined,
getTrackPublications: () => [],
}),
);
expect(vm.waitingForMedia$.value).toBe(false);
});
test("remote media is not in waiting state when user does not intend to publish anywhere", () => {
const vm = createRemoteMedia(
rtcMembership,
{},
mockRemoteParticipant({}),
undefined, // No room (no advertised transport)
);
expect(vm.waitingForMedia$.value).toBe(false);
});

45
src/state/MediaViewModel.ts
View File

@@ -7,8 +7,8 @@ Please see LICENSE in the repository root for full details.
import {
type AudioSource,
type TrackReferenceOrPlaceholder,
type VideoSource,
type TrackReference,
observeParticipantEvents,
observeParticipantMedia,
roomEventSelector,
@@ -33,7 +33,6 @@ import {
type Observable,
Subject,
combineLatest,
distinctUntilKeyChanged,
filter,
fromEvent,
interval,
@@ -60,14 +59,11 @@ import { type ObservableScope } from "./ObservableScope";
export function observeTrackReference$(
participant: Participant,
source: Track.Source,
): Observable<TrackReferenceOrPlaceholder> {
): Observable<TrackReference | undefined> {
return observeParticipantMedia(participant).pipe(
map(() => ({
participant: participant,
publication: participant.getTrackPublication(source),
source,
})),
distinctUntilKeyChanged("publication"),
map(() => participant.getTrackPublication(source)),
distinctUntilChanged(),
map((publication) => publication && { participant, publication, source }),
);
}
@@ -226,7 +222,7 @@ abstract class BaseMediaViewModel {
/**
* The LiveKit video track for this media.
*/
public readonly video$: Behavior<TrackReferenceOrPlaceholder | null>;
public readonly video$: Behavior<TrackReference | undefined>;
/**
* Whether there should be a warning that this media is unencrypted.
*/
@@ -241,10 +237,12 @@ abstract class BaseMediaViewModel {
private observeTrackReference$(
source: Track.Source,
): Behavior<TrackReferenceOrPlaceholder | null> {
): Behavior<TrackReference | undefined> {
return this.scope.behavior(
this.participant$.pipe(
switchMap((p) => (!p ? of(null) : observeTrackReference$(p, source))),
switchMap((p) =>
!p ? of(undefined) : observeTrackReference$(p, source),
),
),
);
}
@@ -268,7 +266,7 @@ abstract class BaseMediaViewModel {
encryptionSystem: EncryptionSystem,
audioSource: AudioSource,
videoSource: VideoSource,
livekitRoom$: Behavior<LivekitRoom | undefined>,
protected readonly livekitRoom$: Behavior<LivekitRoom | undefined>,
public readonly focusUrl$: Behavior<string | undefined>,
public readonly displayName$: Behavior<string>,
public readonly mxcAvatarUrl$: Behavior<string | undefined>,
@@ -281,8 +279,8 @@ abstract class BaseMediaViewModel {
[audio$, this.video$],
(a, v) =>
encryptionSystem.kind !== E2eeType.NONE &&
(a?.publication?.isEncrypted === false ||
v?.publication?.isEncrypted === false),
(a?.publication.isEncrypted === false ||
v?.publication.isEncrypted === false),
),
);
@@ -471,7 +469,7 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel {
private readonly videoTrack$: Observable<LocalVideoTrack | null> =
this.video$.pipe(
switchMap((v) => {
const track = v?.publication?.track;
const track = v?.publication.track;
if (!(track instanceof LocalVideoTrack)) return of(null);
return merge(
// Watch for track restarts because they indicate a camera switch.
@@ -596,6 +594,21 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel {
* A remote participant's user media.
*/
export class RemoteUserMediaViewModel extends BaseUserMediaViewModel {
/**
* Whether we are waiting for this user's LiveKit participant to exist. This
* could be because either we or the remote party are still connecting.
*/
public readonly waitingForMedia$ = this.scope.behavior<boolean>(
combineLatest(
[this.livekitRoom$, this.participant$],
(livekitRoom, participant) =>
// If livekitRoom is undefined, the user is not attempting to publish on
// any transport and so we shouldn't expect a participant. (They might
// be a subscribe-only bot for example.)
livekitRoom !== undefined && participant === null,
),
);
// This private field is used to override the value from the superclass
private __speaking$: Behavior<boolean>;
public get speaking$(): Behavior<boolean> {

69
src/state/UserMedia.ts
View File

@@ -27,6 +27,7 @@ import type { ReactionOption } from "../reactions";
import { observeSpeaker$ } from "./observeSpeaker.ts";
import { generateItems } from "../utils/observable.ts";
import { ScreenShare } from "./ScreenShare.ts";
import { type TaggedParticipant } from "./CallViewModel/remoteMembers/MatrixLivekitMembers.ts";
/**
* Sorting bins defining the order in which media tiles appear in the layout.
@@ -68,40 +69,46 @@ enum SortingBin {
* for inclusion in the call layout and tracks associated screen shares.
*/
export class UserMedia {
public readonly vm: UserMediaViewModel = this.participant$.value?.isLocal
? new LocalUserMediaViewModel(
this.scope,
this.id,
this.userId,
this.participant$ as Behavior<LocalParticipant | null>,
this.encryptionSystem,
this.livekitRoom$,
this.focusUrl$,
this.mediaDevices,
this.displayName$,
this.mxcAvatarUrl$,
this.scope.behavior(this.handRaised$),
this.scope.behavior(this.reaction$),
)
: new RemoteUserMediaViewModel(
this.scope,
this.id,
this.userId,
this.participant$ as Behavior<RemoteParticipant | null>,
this.encryptionSystem,
this.livekitRoom$,
this.focusUrl$,
this.pretendToBeDisconnected$,
this.displayName$,
this.mxcAvatarUrl$,
this.scope.behavior(this.handRaised$),
this.scope.behavior(this.reaction$),
);
public readonly vm: UserMediaViewModel =
this.participant.type === "local"
? new LocalUserMediaViewModel(
this.scope,
this.id,
this.userId,
this.participant.value$,
this.encryptionSystem,
this.livekitRoom$,
this.focusUrl$,
this.mediaDevices,
this.displayName$,
this.mxcAvatarUrl$,
this.scope.behavior(this.handRaised$),
this.scope.behavior(this.reaction$),
)
: new RemoteUserMediaViewModel(
this.scope,
this.id,
this.userId,
this.participant.value$,
this.encryptionSystem,
this.livekitRoom$,
this.focusUrl$,
this.pretendToBeDisconnected$,
this.displayName$,
this.mxcAvatarUrl$,
this.scope.behavior(this.handRaised$),
this.scope.behavior(this.reaction$),
);
private readonly speaker$ = this.scope.behavior(
observeSpeaker$(this.vm.speaking$),
);
// TypeScript needs this widening of the type to happen in a separate statement
private readonly participant$: Behavior<
LocalParticipant | RemoteParticipant | null
> = this.participant.value$;
/**
* All screen share media associated with this user media.
*/
@@ -184,9 +191,7 @@ export class UserMedia {
private readonly scope: ObservableScope,
public readonly id: string,
private readonly userId: string,
private readonly participant$: Behavior<
LocalParticipant | RemoteParticipant | null
>,
private readonly participant: TaggedParticipant,
private readonly encryptionSystem: EncryptionSystem,
private readonly livekitRoom$: Behavior<LivekitRoom | undefined>,
private readonly focusUrl$: Behavior<string | undefined>,

10
src/tile/GridTile.test.tsx
View File

@@ -12,7 +12,11 @@ import { axe } from "vitest-axe";
import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc";
import { GridTile } from "./GridTile";
import { mockRtcMembership, createRemoteMedia } from "../utils/test";
import {
mockRtcMembership,
createRemoteMedia,
mockRemoteParticipant,
} from "../utils/test";
import { GridTileViewModel } from "../state/TileViewModel";
import { ReactionsSenderProvider } from "../reactions/useReactionsSender";
import type { CallViewModel } from "../state/CallViewModel/CallViewModel";
@@ -31,11 +35,11 @@ test("GridTile is accessible", async () => {
rawDisplayName: "Alice",
getMxcAvatarUrl: () => "mxc://adfsg",
},
{
mockRemoteParticipant({
setVolume() {},
getTrackPublication: () =>
({}) as Partial<RemoteTrackPublication> as RemoteTrackPublication,
},
}),
);
const fakeRtcSession = {

8
src/tile/GridTile.tsx
View File

@@ -69,6 +69,7 @@ interface UserMediaTileProps extends TileProps {
vm: UserMediaViewModel;
mirror: boolean;
locallyMuted: boolean;
waitingForMedia?: boolean;
primaryButton?: ReactNode;
menuStart?: ReactNode;
menuEnd?: ReactNode;
@@ -79,6 +80,7 @@ const UserMediaTile: FC<UserMediaTileProps> = ({
vm,
showSpeakingIndicators,
locallyMuted,
waitingForMedia,
primaryButton,
menuStart,
menuEnd,
@@ -148,7 +150,7 @@ const UserMediaTile: FC<UserMediaTileProps> = ({
const tile = (
<MediaView
ref={ref}
video={video ?? undefined}
video={video}
userId={vm.userId}
unencryptedWarning={unencryptedWarning}
encryptionStatus={encryptionStatus}
@@ -194,7 +196,7 @@ const UserMediaTile: FC<UserMediaTileProps> = ({
raisedHandTime={handRaised ?? undefined}
currentReaction={reaction ?? undefined}
raisedHandOnClick={raisedHandOnClick}
localParticipant={vm.local}
waitingForMedia={waitingForMedia}
focusUrl={focusUrl}
audioStreamStats={audioStreamStats}
videoStreamStats={videoStreamStats}
@@ -290,6 +292,7 @@ const RemoteUserMediaTile: FC<RemoteUserMediaTileProps> = ({
...props
}) => {
const { t } = useTranslation();
const waitingForMedia = useBehavior(vm.waitingForMedia$);
const locallyMuted = useBehavior(vm.locallyMuted$);
const localVolume = useBehavior(vm.localVolume$);
const onSelectMute = useCallback(
@@ -311,6 +314,7 @@ const RemoteUserMediaTile: FC<RemoteUserMediaTileProps> = ({
<UserMediaTile
ref={ref}
vm={vm}
waitingForMedia={waitingForMedia}
locallyMuted={locallyMuted}
mirror={false}
menuStart={

36
src/tile/MediaView.test.tsx
View File

@@ -47,7 +47,6 @@ describe("MediaView", () => {
video: trackReference,
userId: "@alice:example.com",
mxcAvatarUrl: undefined,
localParticipant: false,
focusable: true,
};
@@ -66,24 +65,13 @@ describe("MediaView", () => {
});
});
describe("with no participant", () => {
it("shows avatar for local user", () => {
render(
<MediaView {...baseProps} video={undefined} localParticipant={true} />,
);
describe("with no video", () => {
it("shows avatar", () => {
render(<MediaView {...baseProps} video={undefined} />);
expect(
screen.getByRole("img", { name: "@alice:example.com" }),
).toBeVisible();
expect(screen.queryAllByText("Waiting for media...").length).toBe(0);
});
it("shows avatar and label for remote user", () => {
render(
<MediaView {...baseProps} video={undefined} localParticipant={false} />,
);
expect(
screen.getByRole("img", { name: "@alice:example.com" }),
).toBeVisible();
expect(screen.getByText("Waiting for media...")).toBeVisible();
expect(screen.queryByTestId("video")).toBe(null);
});
});
@@ -94,6 +82,22 @@ describe("MediaView", () => {
});
});
describe("waitingForMedia", () => {
test("defaults to false", () => {
render(<MediaView {...baseProps} />);
expect(screen.queryAllByText("Waiting for media...").length).toBe(0);
});
test("shows and is accessible", async () => {
const { container } = render(
<TooltipProvider>
<MediaView {...baseProps} waitingForMedia={true} />
</TooltipProvider>,
);
expect(await axe(container)).toHaveNoViolations();
expect(screen.getByText("Waiting for media...")).toBeVisible();
});
});
describe("unencryptedWarning", () => {
test("is shown and accessible", async () => {
const { container } = render(

6
src/tile/MediaView.tsx
View File

@@ -43,7 +43,7 @@ interface Props extends ComponentProps<typeof animated.div> {
raisedHandTime?: Date;
currentReaction?: ReactionOption;
raisedHandOnClick?: () => void;
localParticipant: boolean;
waitingForMedia?: boolean;
audioStreamStats?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats;
videoStreamStats?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats;
// The focus url, mainly for debugging purposes
@@ -71,7 +71,7 @@ export const MediaView: FC<Props> = ({
raisedHandTime,
currentReaction,
raisedHandOnClick,
localParticipant,
waitingForMedia,
audioStreamStats,
videoStreamStats,
focusUrl,
@@ -129,7 +129,7 @@ export const MediaView: FC<Props> = ({
/>
)}
</div>
{!video && !localParticipant && (
{waitingForMedia && (
<div className={styles.status}>
{t("video_tile.waiting_for_media")}
</div>

3
src/tile/SpotlightTile.test.tsx
View File

@@ -17,6 +17,7 @@ import {
mockRtcMembership,
createLocalMedia,
createRemoteMedia,
mockRemoteParticipant,
} from "../utils/test";
import { SpotlightTileViewModel } from "../state/TileViewModel";
import { constant } from "../state/Behavior";
@@ -33,7 +34,7 @@ test("SpotlightTile is accessible", async () => {
rawDisplayName: "Alice",
getMxcAvatarUrl: () => "mxc://adfsg",
},
{},
mockRemoteParticipant({}),
);
const vm2 = createLocalMedia(

18
src/tile/SpotlightTile.tsx
View File

@@ -38,6 +38,7 @@ import {
type MediaViewModel,
ScreenShareViewModel,
type UserMediaViewModel,
type RemoteUserMediaViewModel,
} from "../state/MediaViewModel";
import { useInitial } from "../useInitial";
import { useMergedRefs } from "../useMergedRefs";
@@ -84,6 +85,21 @@ const SpotlightLocalUserMediaItem: FC<SpotlightLocalUserMediaItemProps> = ({
SpotlightLocalUserMediaItem.displayName = "SpotlightLocalUserMediaItem";
interface SpotlightRemoteUserMediaItemProps
extends SpotlightUserMediaItemBaseProps {
vm: RemoteUserMediaViewModel;
}
const SpotlightRemoteUserMediaItem: FC<SpotlightRemoteUserMediaItemProps> = ({
vm,
...props
}) => {
const waitingForMedia = useBehavior(vm.waitingForMedia$);
return (
<MediaView waitingForMedia={waitingForMedia} mirror={false} {...props} />
);
};
interface SpotlightUserMediaItemProps extends SpotlightItemBaseProps {
vm: UserMediaViewModel;
}
@@ -103,7 +119,7 @@ const SpotlightUserMediaItem: FC<SpotlightUserMediaItemProps> = ({
return vm instanceof LocalUserMediaViewModel ? (
<SpotlightLocalUserMediaItem vm={vm} {...baseProps} />
) : (
<MediaView mirror={false} {...baseProps} />
<SpotlightRemoteUserMediaItem vm={vm} {...baseProps} />
);
};

35
src/utils/observable.test.ts
View File

@@ -5,11 +5,12 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { test } from "vitest";
import { Subject } from "rxjs";
import { expect, test } from "vitest";
import { type Observable, of, Subject, switchMap } from "rxjs";
import { withTestScheduler } from "./test";
import { generateItems, pauseWhen } from "./observable";
import { filterBehavior, generateItems, pauseWhen } from "./observable";
import { type Behavior } from "../state/Behavior";
test("pauseWhen", () => {
withTestScheduler(({ behavior, expectObservable }) => {
@@ -72,3 +73,31 @@ test("generateItems", () => {
expectObservable(scope4$).toBe(scope4Marbles);
});
});
test("filterBehavior", () => {
withTestScheduler(({ behavior, expectObservable }) => {
// Filtering the input should segment it into 2 modes of non-null behavior.
const inputMarbles = " abcxabx";
const filteredMarbles = "a--xa-x";
const input$ = behavior(inputMarbles, {
a: "a",
b: "b",
c: "c",
x: null,
});
const filtered$: Observable<Behavior<string> | null> = input$.pipe(
filterBehavior((value) => typeof value === "string"),
);
expectObservable(filtered$).toBe(filteredMarbles, {
a: expect.any(Object),
x: null,
});
expectObservable(
filtered$.pipe(
switchMap((value$) => (value$ === null ? of(null) : value$)),
),
).toBe(inputMarbles, { a: "a", b: "b", c: "c", x: null });
});
});

23
src/utils/observable.ts
View File

@@ -22,6 +22,7 @@ import {
withLatestFrom,
BehaviorSubject,
type OperatorFunction,
distinctUntilChanged,
} from "rxjs";
import { type Behavior } from "../state/Behavior";
@@ -185,6 +186,28 @@ export function generateItemsWithEpoch<
);
}
/**
* Segments a behavior into periods during which its value matches the filter
* (outputting a behavior with a narrowed type) and periods during which it does
* not match (outputting null).
*/
export function filterBehavior<T, S extends T>(
predicate: (value: T) => value is S,
): OperatorFunction<T, Behavior<S> | null> {
return (input$) =>
input$.pipe(
scan<T, BehaviorSubject<S> | null>((acc$, input) => {
if (predicate(input)) {
const output$ = acc$ ?? new BehaviorSubject(input);
output$.next(input);
return output$;
}
return null;
}, null),
distinctUntilChanged(),
);
}
function generateItemsInternal<
Input,
Keys extends [unknown, ...unknown[]],

2
src/utils/test-viewmodel.ts
View File

@@ -37,6 +37,7 @@ import {
import { aliceRtcMember, localRtcMember } from "./test-fixtures";
import { type RaisedHandInfo, type ReactionInfo } from "../reactions";
import { constant } from "../state/Behavior";
import { MatrixRTCMode } from "../settings/settings";
mockConfig({ livekit: { livekit_service_url: "https://example.com" } });
@@ -162,6 +163,7 @@ export function getBasicCallViewModelEnvironment(
setE2EEEnabled: async () => Promise.resolve(),
}),
connectionState$: constant(ConnectionState.Connected),
matrixRTCMode$: constant(MatrixRTCMode.Legacy),
...callViewModelOptions,
},
handRaisedSubject$,

25
src/utils/test.ts
View File

@@ -311,6 +311,8 @@ export function mockLocalParticipant(
publishTrack: vi.fn(),
unpublishTracks: vi.fn().mockResolvedValue([]),
createTracks: vi.fn(),
setMicrophoneEnabled: vi.fn(),
setCameraEnabled: vi.fn(),
getTrackPublication: () =>
({}) as Partial<LocalTrackPublication> as LocalTrackPublication,
...mockEmitter(),
@@ -319,12 +321,12 @@ export function mockLocalParticipant(
}
export function createLocalMedia(
localRtcMember: CallMembership,
rtcMember: CallMembership,
roomMember: Partial<RoomMember>,
localParticipant: LocalParticipant,
mediaDevices: MediaDevices,
): LocalUserMediaViewModel {
const member = mockMatrixRoomMember(localRtcMember, roomMember);
const member = mockMatrixRoomMember(rtcMember, roomMember);
return new LocalUserMediaViewModel(
testScope(),
"local",
@@ -359,23 +361,26 @@ export function mockRemoteParticipant(
}
export function createRemoteMedia(
localRtcMember: CallMembership,
rtcMember: CallMembership,
roomMember: Partial<RoomMember>,
participant: Partial<RemoteParticipant>,
participant: RemoteParticipant | null,
livekitRoom: LivekitRoom | undefined = mockLivekitRoom(
{},
{
remoteParticipants$: of(participant ? [participant] : []),
},
),
): RemoteUserMediaViewModel {
const member = mockMatrixRoomMember(localRtcMember, roomMember);
const remoteParticipant = mockRemoteParticipant(participant);
const member = mockMatrixRoomMember(rtcMember, roomMember);
return new RemoteUserMediaViewModel(
testScope(),
"remote",
member.userId,
of(remoteParticipant),
constant(participant),
{
kind: E2eeType.PER_PARTICIPANT,
},
constant(
mockLivekitRoom({}, { remoteParticipants$: of([remoteParticipant]) }),
),
constant(livekitRoom),
constant("https://rtc-example.org"),
constant(false),
constant(member.rawDisplayName ?? "nodisplayname"),

45
yarn.lock
View File

@@ -3380,14 +3380,14 @@ __metadata:
languageName: node
linkType: hard
"@playwright/test@npm:^1.56.1":
version: 1.56.1
resolution: "@playwright/test@npm:1.56.1"
"@playwright/test@npm:^1.57.0":
version: 1.57.0
resolution: "@playwright/test@npm:1.57.0"
dependencies:
playwright: "npm:1.56.1"
playwright: "npm:1.57.0"
bin:
playwright: cli.js
checksum: 10c0/2b5b0e1f2e6a18f6e5ce6897c7440ca78f64e0b004834e9808e93ad2b78b96366b562ae4366602669cf8ad793a43d85481b58541e74be71e905e732d833dd691
checksum: 10c0/35ba4b28be72bf0a53e33dbb11c6cff848fb9a37f49e893ce63a90675b5291ec29a1ba82c8a3b043abaead129400f0589623e9ace2e6a1c8eaa409721ecc3774
languageName: node
linkType: hard
@@ -7786,7 +7786,7 @@ __metadata:
"@opentelemetry/sdk-trace-base": "npm:^2.0.0"
"@opentelemetry/sdk-trace-web": "npm:^2.0.0"
"@opentelemetry/semantic-conventions": "npm:^1.25.1"
"@playwright/test": "npm:^1.56.1"
"@playwright/test": "npm:^1.57.0"
"@radix-ui/react-dialog": "npm:^1.0.4"
"@radix-ui/react-slider": "npm:^1.1.2"
"@radix-ui/react-visually-hidden": "npm:^1.0.3"
@@ -10781,8 +10781,8 @@ __metadata:
linkType: hard
"matrix-js-sdk@npm:^39.2.0":
version: 39.2.0
resolution: "matrix-js-sdk@npm:39.2.0"
version: 39.3.0
resolution: "matrix-js-sdk@npm:39.3.0"
dependencies:
"@babel/runtime": "npm:^7.12.5"
"@matrix-org/matrix-sdk-crypto-wasm": "npm:^15.3.0"
@@ -10798,7 +10798,7 @@ __metadata:
sdp-transform: "npm:^3.0.0"
unhomoglyph: "npm:^1.0.6"
uuid: "npm:13"
checksum: 10c0/f8b5261de2744305330ba3952821ca9303698170bfd3a0ff8a767b9286d4e8d4ed5aaf6fbaf8a1e8ff9dbd859102a2a47d882787e2da3b3078965bec00157959
checksum: 10c0/031c9ec042e00c32dc531f82fc59c64cc25fb665abfc642b1f0765c530d60684f8bd63daf0cdd0dbe96b4f87ea3f4148f9d3f024a59d57eceaec1ce5d0164755
languageName: node
linkType: hard
@@ -11735,36 +11735,27 @@ __metadata:
languageName: node
linkType: hard
"pkg-dir@npm:^5.0.0":
version: 5.0.0
resolution: "pkg-dir@npm:5.0.0"
dependencies:
find-up: "npm:^5.0.0"
checksum: 10c0/793a496d685dc55bbbdbbb22d884535c3b29241e48e3e8d37e448113a71b9e42f5481a61fdc672d7322de12fbb2c584dd3a68bf89b18fffce5c48a390f911bc5
languageName: node
linkType: hard
"playwright-core@npm:1.56.1":
version: 1.56.1
resolution: "playwright-core@npm:1.56.1"
"playwright-core@npm:1.57.0":
version: 1.57.0
resolution: "playwright-core@npm:1.57.0"
bin:
playwright-core: cli.js
checksum: 10c0/ffd40142b99c68678b387445d5b42f1fee4ab0b65d983058c37f342e5629f9cdbdac0506ea80a0dfd41a8f9f13345bad54e9a8c35826ef66dc765f4eb3db8da7
checksum: 10c0/798e35d83bf48419a8c73de20bb94d68be5dde68de23f95d80a0ebe401e3b83e29e3e84aea7894d67fa6c79d2d3d40cc5bcde3e166f657ce50987aaa2421b6a9
languageName: node
linkType: hard
"playwright@npm:1.56.1":
version: 1.56.1
resolution: "playwright@npm:1.56.1"
"playwright@npm:1.57.0":
version: 1.57.0
resolution: "playwright@npm:1.57.0"
dependencies:
fsevents: "npm:2.3.2"
playwright-core: "npm:1.56.1"
playwright-core: "npm:1.57.0"
dependenciesMeta:
fsevents:
optional: true
bin:
playwright: cli.js
checksum: 10c0/8e9965aede86df0f4722063385748498977b219630a40a10d1b82b8bd8d4d4e9b6b65ecbfa024331a30800163161aca292fb6dd7446c531a1ad25f4155625ab4
checksum: 10c0/ab03c99a67b835bdea9059f516ad3b6e42c21025f9adaa161a4ef6bc7ca716dcba476d287140bb240d06126eb23f889a8933b8f5f1f1a56b80659d92d1358899
languageName: node
linkType: hard