Matrix/element-call-Github
1
0
Fork 0
mirror of https://github.com/vector-im/element-call.git synced 2026-02-02 04:05:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

pull out all screen share related logic.

Browse Source
This commit is contained in:
Timo K
2025-11-07 08:44:44 +01:00
parent 7c41aef801
commit 92fdce33ea
17 changed files with 461 additions and 310 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/room/InCallView.tsx
View File

@@ -58,7 +58,10 @@ import { type MuteStates } from "../state/MuteStates";
import { type MatrixInfo } from "./VideoPreview";
import { InviteButton } from "../button/InviteButton";
import { LayoutToggle } from "./LayoutToggle";
import { CallViewModel, type GridMode } from "../state/CallViewModel";
import {
CallViewModel,
type GridMode,
} from "../state/CallViewModel/CallViewModel.ts";
import { Grid, type TileProps } from "../grid/Grid";
import { useInitial } from "../useInitial";
import { SpotlightTile } from "../tile/SpotlightTile";

210
src/state/CallViewModel/CallNotificationLifecycle.ts Normal file
View File

@@ -0,0 +1,210 @@
/*
Copyright 2025 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
type CallMembership,
type MatrixRTCSession,
MatrixRTCSessionEvent,
type MatrixRTCSessionEventHandlerMap,
} from "matrix-js-sdk/lib/matrixrtc";
import {
combineLatest,
concat,
endWith,
filter,
fromEvent,
ignoreElements,
map,
merge,
NEVER,
type Observable,
of,
pairwise,
startWith,
switchMap,
takeUntil,
timer,
} from "rxjs";
import {
type EventTimelineSetHandlerMap,
EventType,
type Room as MatrixRoom,
RoomEvent,
} from "matrix-js-sdk";
import { type Behavior } from "../Behavior";
import { type Epoch, mapEpoch, type ObservableScope } from "../ObservableScope";
export type AutoLeaveReason = "allOthersLeft" | "timeout" | "decline";
export type CallPickupState =
| "unknown"
| "ringing"
| "timeout"
| "decline"
| "success"
| null;
export type CallNotificationWrapper = Parameters<
MatrixRTCSessionEventHandlerMap[MatrixRTCSessionEvent.DidSendCallNotification]
>;
export function createSentCallNotification$(
scope: ObservableScope,
matrixRTCSession: MatrixRTCSession,
): Behavior<CallNotificationWrapper | null> {
const sentCallNotification$ = scope.behavior(
fromEvent(matrixRTCSession, MatrixRTCSessionEvent.DidSendCallNotification),
null,
) as Behavior<CallNotificationWrapper | null>;
return sentCallNotification$;
}
export function createReceivedDecline$(
matrixRoom: MatrixRoom,
): Observable<Parameters<EventTimelineSetHandlerMap[RoomEvent.Timeline]>> {
return (
fromEvent(matrixRoom, RoomEvent.Timeline) as Observable<
Parameters<EventTimelineSetHandlerMap[RoomEvent.Timeline]>
>
).pipe(filter(([event]) => event.getType() === EventType.RTCDecline));
}
interface Props {
scope: ObservableScope;
memberships$: Behavior<Epoch<CallMembership[]>>;
sentCallNotification$: Observable<CallNotificationWrapper | null>;
receivedDecline$: Observable<
Parameters<EventTimelineSetHandlerMap[RoomEvent.Timeline]>
>;
options: { waitForCallPickup?: boolean; autoLeaveWhenOthersLeft?: boolean };
localUser: { deviceId: string; userId: string };
}
/**
* @returns {callPickupState$, autoLeave$}
* `callPickupState$` The current call pickup state of the call.
* - "unknown": The client has not yet sent the notification event. We don't know if it will because it first needs to send its own membership.
* Then we can conclude if we were the first one to join or not.
* This may also be set if we are disconnected.
* - "ringing": The call is ringing on other devices in this room (This client should give audiovisual feedback that this is happening).
* - "timeout": No-one picked up in the defined time this call should be ringing on others devices.
* The call failed. If desired this can be used as a trigger to exit the call.
* - "success": Someone else joined. The call is in a normal state. No audiovisual feedback.
* - null: EC is configured to never show any waiting for answer state.
*
* `autoLeave$` An observable that emits (null) when the call should be automatically left.
* - if options.autoLeaveWhenOthersLeft is set to true it emits when all others left.
* - if options.waitForCallPickup is set to true it emits if noone picked up the ring or if the ring got declined.
* - if options.autoLeaveWhenOthersLeft && options.waitForCallPickup is false it will never emit.
*
*/
export function createCallNotificationLifecycle$({
scope,
memberships$,
sentCallNotification$,
receivedDecline$,
options,
localUser,
}: Props): {
callPickupState$: Behavior<CallPickupState>;
autoLeave$: Observable<AutoLeaveReason>;
} {
// TODO convert all ring and all others left logic into one callLifecycleTracker$(didSendCallNotification$,matrixLivekitItem$): {autoLeave$,callPickupState$}
const allOthersLeft$ = memberships$.pipe(
pairwise(),
filter(
([{ value: prev }, { value: current }]) =>
current.every((m) => m.userId === localUser.userId) &&
prev.some((m) => m.userId !== localUser.userId),
),
map(() => {}),
);
/**
* Whether some Matrix user other than ourself is joined to the call.
*/
const someoneElseJoined$ = memberships$.pipe(
mapEpoch((ms) => ms.some((m) => m.userId !== localUser.userId)),
) as Behavior<Epoch<boolean>>;
/**
* Whenever the RTC session tells us that it intends to ring the remote
* participant's devices, this emits an Observable tracking the current state of
* that ringing process.
*/
// This is a behavior since we need to store the latest state for when we subscribe to this after `didSendCallNotification$`
// has already emitted but we still need the latest observable with a timeout timer that only gets created on after receiving `notificationEvent`.
// A behavior will emit the latest observable with the running timer to new subscribers.
// see also: callPickupState$ and in particular the line: `return this.ring$.pipe(mergeAll());` here we otherwise might get an EMPTY observable if
// `ring$` would not be a behavior.
const remoteRingState$: Behavior<"ringing" | "timeout" | "decline" | null> =
scope.behavior(
sentCallNotification$.pipe(
filter(
(newAndLegacyEvents) =>
// only care about new events (legacy do not have decline pattern)
newAndLegacyEvents?.[0].notification_type === "ring",
),
map((e) => e as CallNotificationWrapper),
switchMap(([notificastionEvent]) => {
const lifetimeMs = notificationEvent?.lifetime ?? 0;
return concat(
lifetimeMs === 0
? // If no lifetime, skip the ring state
of(null)
: // Ring until lifetime ms have passed
timer(lifetimeMs).pipe(
ignoreElements(),
startWith("ringing" as const),
),
// The notification lifetime has timed out, meaning ringing has likely
// stopped on all receiving clients.
of("timeout" as const),
// This makes sure we will not drop into the `endWith("decline" as const)` state
NEVER,
).pipe(
takeUntil(
receivedDecline$.pipe(
filter(
([event]) =>
event.getRelation()?.rel_type === "m.reference" &&
event.getRelation()?.event_id ===
notificationEvent.event_id &&
event.getSender() !== localUser.userId,
),
),
),
endWith("decline" as const),
);
}),
),
null,
);
const callPickupState$ = scope.behavior(
options.waitForCallPickup === true
? combineLatest(
[someoneElseJoined$, remoteRingState$],
(someoneElseJoined, ring) => {
if (someoneElseJoined) {
return "success" as const;
}
// Show the ringing state of the most recent ringing attempt.
// as long as we have not yet sent an RTC notification event, ring will be null -> callPickupState$ = unknown.
return ring ?? ("unknown" as const);
},
)
: NEVER,
null,
);
const autoLeave$ = merge(
options.autoLeaveWhenOthersLeft === true
? allOthersLeft$.pipe(map(() => "allOthersLeft" as const))
: NEVER,
callPickupState$.pipe(
filter((state) => state === "timeout" || state === "decline"),
),
);
return { autoLeave$, callPickupState$ };
}

275
src/state/CallViewModel.ts → src/state/CallViewModel/CallViewModel.ts
View File

@@ -7,29 +7,20 @@ Please see LICENSE in the repository root for full details.
import {
type BaseKeyProvider,
ConnectionState,
type ConnectionState,
type E2EEOptions,
ExternalE2EEKeyProvider,
type Room as LivekitRoom,
type RoomOptions,
} from "livekit-client";
import E2EEWorker from "livekit-client/e2ee-worker?worker";
import {
type EventTimelineSetHandlerMap,
EventType,
type Room as MatrixRoom,
RoomEvent,
} from "matrix-js-sdk";
import { type Room as MatrixRoom } from "matrix-js-sdk";
import {
combineLatest,
concat,
distinctUntilChanged,
EMPTY,
endWith,
filter,
from,
fromEvent,
ignoreElements,
map,
merge,
NEVER,
@@ -46,18 +37,12 @@ import {
switchMap,
switchScan,
take,
takeUntil,
takeWhile,
tap,
throttleTime,
timer,
} from "rxjs";
import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
import {
type MatrixRTCSession,
MatrixRTCSessionEvent,
type MatrixRTCSessionEventHandlerMap,
} from "matrix-js-sdk/lib/matrixrtc";
import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc";
import { type IWidgetApiRequest } from "matrix-widget-api";
import {
@@ -66,39 +51,39 @@ import {
type RemoteUserMediaViewModel,
ScreenShareViewModel,
type UserMediaViewModel,
} from "./MediaViewModel";
import { accumulate, generateKeyed$, pauseWhen } from "../utils/observable";
} from "../MediaViewModel";
import { accumulate, generateKeyed$, pauseWhen } from "../../utils/observable";
import {
duplicateTiles,
MatrixRTCMode,
matrixRTCMode,
playReactionsSound,
showReactions,
} from "../settings/settings";
import { isFirefox } from "../Platform";
import { setPipEnabled$ } from "../controls";
import { TileStore } from "./TileStore";
import { gridLikeLayout } from "./GridLikeLayout";
import { spotlightExpandedLayout } from "./SpotlightExpandedLayout";
import { oneOnOneLayout } from "./OneOnOneLayout";
import { pipLayout } from "./PipLayout";
import { type EncryptionSystem } from "../e2ee/sharedKeyManagement";
} from "../../settings/settings";
import { isFirefox } from "../../Platform";
import { setPipEnabled$ } from "../../controls";
import { TileStore } from "../TileStore";
import { gridLikeLayout } from "../GridLikeLayout";
import { spotlightExpandedLayout } from "../SpotlightExpandedLayout";
import { oneOnOneLayout } from "../OneOnOneLayout";
import { pipLayout } from "../PipLayout";
import { type EncryptionSystem } from "../../e2ee/sharedKeyManagement";
import {
type RaisedHandInfo,
type ReactionInfo,
type ReactionOption,
} from "../reactions";
import { shallowEquals } from "../utils/array";
import { type MediaDevices } from "./MediaDevices";
import { type Behavior, constant } from "./Behavior";
import { E2eeType } from "../e2ee/e2eeType";
import { MatrixKeyProvider } from "../e2ee/matrixKeyProvider";
import { type MuteStates } from "./MuteStates";
import { getUrlParams } from "../UrlParams";
import { type ProcessorState } from "../livekit/TrackProcessorContext";
import { ElementWidgetActions, widget } from "../widget";
import { sharingScreen$, UserMedia } from "./UserMedia.ts";
import { ScreenShare } from "./ScreenShare.ts";
} from "../../reactions";
import { shallowEquals } from "../../utils/array";
import { type MediaDevices } from "../MediaDevices";
import { type Behavior, constant } from "../Behavior";
import { E2eeType } from "../../e2ee/e2eeType";
import { MatrixKeyProvider } from "../../e2ee/matrixKeyProvider";
import { type MuteStates } from "../MuteStates";
import { getUrlParams } from "../../UrlParams";
import { type ProcessorState } from "../../livekit/TrackProcessorContext";
import { ElementWidgetActions, widget } from "../../widget";
import { UserMedia } from "../UserMedia.ts";
import { ScreenShare } from "../ScreenShare.ts";
import {
type GridLayoutMedia,
type Layout,
@@ -107,18 +92,23 @@ import {
type SpotlightExpandedLayoutMedia,
type SpotlightLandscapeLayoutMedia,
type SpotlightPortraitLayoutMedia,
} from "./layout-types.ts";
import { type ElementCallError } from "../utils/errors.ts";
import { type ObservableScope } from "./ObservableScope.ts";
} from "../layout-types.ts";
import { type ElementCallError } from "../../utils/errors.ts";
import { type ObservableScope } from "../ObservableScope.ts";
import { createLocalMembership$ } from "./localMember/LocalMembership.ts";
import { createLocalTransport$ } from "./localMember/LocalTransport.ts";
import {
createMemberships$,
membershipsAndTransports$,
} from "./SessionBehaviors.ts";
} from "../SessionBehaviors.ts";
import { ECConnectionFactory } from "./remoteMembers/ConnectionFactory.ts";
import { createConnectionManager$ } from "./remoteMembers/ConnectionManager.ts";
import { createMatrixLivekitMembers$ } from "./remoteMembers/MatrixLivekitMembers.ts";
import {
createCallNotificationLifecycle$,
createReceivedDecline$,
createSentCallNotification$,
} from "./CallNotificationLifecycle.ts";
const logger = rootLogger.getChild("[CallViewModel]");
//TODO
@@ -274,6 +264,23 @@ export class CallViewModel {
options: this.connectOptions$,
});
// ------------------------------------------------------------------------
// CallNotificationLifecycle
private sentCallNotification$ = createSentCallNotification$(
this.scope,
this.matrixRTCSession,
);
private receivedDecline$ = createReceivedDecline$(this.matrixRoom);
private callLifecycle = createCallNotificationLifecycle$({
scope: this.scope,
memberships$: this.memberships$,
sentCallNotification$: this.sentCallNotification$,
receivedDecline$: this.receivedDecline$,
options: this.options,
localUser: { userId: this.userId, deviceId: this.deviceId },
});
/**
* If there is a configuration error with the call (e.g. misconfigured E2EE).
* This is a fatal error that prevents the call from being created/joined.
@@ -315,7 +322,7 @@ export class CallViewModel {
public readonly audioParticipants$ = this.scope.behavior(
this.matrixLivekitMembers$.pipe(
map((members) => members.map((m) => m.participant)),
map((members) => members.value.map((m) => m.participant)),
),
);
@@ -350,7 +357,7 @@ export class CallViewModel {
// Generate a collection of MediaItems from the list of expected (whether
// present or missing) LiveKit participants.
combineLatest([this.matrixLivekitMembers$, duplicateTiles.value$]),
([matrixLivekitMembers, duplicateTiles], createOrGet) => {
([{ value: matrixLivekitMembers }, duplicateTiles], createOrGet) => {
const items: MediaItem[] = [];
for (const {
@@ -455,129 +462,11 @@ export class CallViewModel {
*/
// TODO KEEP THIS!! and adapt it to what our membershipManger returns
public readonly participantCount$ = this.scope.behavior(
this.memberships$.pipe(map((ms) => ms.length)),
this.memberships$.pipe(map((ms) => ms.value.length)),
);
// TODO convert all ring and all others left logic into one callLifecycleTracker$(didSendCallNotification$,matrixLivekitItem$): {autoLeave$,callPickupState$}
private readonly allOthersLeft$ = this.memberships$.pipe(
pairwise(),
filter(
([prev, current]) =>
current.every((m) => m.userId === this.userId) &&
prev.some((m) => m.userId !== this.userId),
),
map(() => {}),
);
private readonly didSendCallNotification$ = fromEvent(
this.matrixRTCSession,
MatrixRTCSessionEvent.DidSendCallNotification,
) as Observable<
Parameters<
MatrixRTCSessionEventHandlerMap[MatrixRTCSessionEvent.DidSendCallNotification]
>
>;
/**
* Whenever the RTC session tells us that it intends to ring the remote
* participant's devices, this emits an Observable tracking the current state of
* that ringing process.
*/
// This is a behavior since we need to store the latest state for when we subscribe to this after `didSendCallNotification$`
// has already emitted but we still need the latest observable with a timeout timer that only gets created on after receiving `notificationEvent`.
// A behavior will emit the latest observable with the running timer to new subscribers.
// see also: callPickupState$ and in particular the line: `return this.ring$.pipe(mergeAll());` here we otherwise might get an EMPTY observable if
// `ring$` would not be a behavior.
private readonly ring$: Behavior<"ringing" | "timeout" | "decline" | null> =
this.scope.behavior(
this.didSendCallNotification$.pipe(
filter(
([notificationEvent]) =>
notificationEvent.notification_type === "ring",
),
switchMap(([notificationEvent]) => {
const lifetimeMs = notificationEvent?.lifetime ?? 0;
return concat(
lifetimeMs === 0
? // If no lifetime, skip the ring state
of(null)
: // Ring until lifetime ms have passed
timer(lifetimeMs).pipe(
ignoreElements(),
startWith("ringing" as const),
),
// The notification lifetime has timed out, meaning ringing has likely
// stopped on all receiving clients.
of("timeout" as const),
// This makes sure we will not drop into the `endWith("decline" as const)` state
NEVER,
).pipe(
takeUntil(
(
fromEvent(this.matrixRoom, RoomEvent.Timeline) as Observable<
Parameters<EventTimelineSetHandlerMap[RoomEvent.Timeline]>
>
).pipe(
filter(
([event]) =>
event.getType() === EventType.RTCDecline &&
event.getRelation()?.rel_type === "m.reference" &&
event.getRelation()?.event_id ===
notificationEvent.event_id &&
event.getSender() !== this.userId,
),
),
),
endWith("decline" as const),
);
}),
),
null,
);
/**
* Whether some Matrix user other than ourself is joined to the call.
*/
private readonly someoneElseJoined$ = this.memberships$.pipe(
map((ms) => ms.some((m) => m.userId !== this.userId)),
) as Behavior<boolean>;
/**
* The current call pickup state of the call.
* - "unknown": The client has not yet sent the notification event. We don't know if it will because it first needs to send its own membership.
* Then we can conclude if we were the first one to join or not.
* This may also be set if we are disconnected.
* - "ringing": The call is ringing on other devices in this room (This client should give audiovisual feedback that this is happening).
* - "timeout": No-one picked up in the defined time this call should be ringing on others devices.
* The call failed. If desired this can be used as a trigger to exit the call.
* - "success": Someone else joined. The call is in a normal state. No audiovisual feedback.
* - null: EC is configured to never show any waiting for answer state.
*/
public readonly callPickupState$: Behavior<
"unknown" | "ringing" | "timeout" | "decline" | "success" | null
> = this.options.waitForCallPickup
? this.scope.behavior<
"unknown" | "ringing" | "timeout" | "decline" | "success"
>(
combineLatest(
[this.livekitConnectionState$, this.someoneElseJoined$, this.ring$],
(livekitConnectionState, someoneElseJoined, ring) => {
if (livekitConnectionState === ConnectionState.Disconnected) {
// Do not ring until we're connected.
return "unknown" as const;
} else if (someoneElseJoined) {
return "success" as const;
}
// Show the ringing state of the most recent ringing attempt.
// as long as we have not yet sent an RTC notification event, ring will be null -> callPickupState$ = unknown.
return ring ?? ("unknown" as const);
},
),
)
: constant(null);
public readonly leaveSoundEffect$ = combineLatest([
this.callPickupState$,
this.callLifecycle.callPickupState$,
this.userMedia$,
]).pipe(
// Until the call is successful, do not play a leave sound.
@@ -594,16 +483,6 @@ export class CallViewModel {
throttleTime(THROTTLE_SOUND_EFFECT_MS),
);
// Public for testing
public readonly autoLeave$ = merge(
this.options.autoLeaveWhenOthersLeft
? this.allOthersLeft$.pipe(map(() => "allOthersLeft" as const))
: NEVER,
this.callPickupState$.pipe(
filter((state) => state === "timeout" || state === "decline"),
),
);
private readonly userHangup$ = new Subject<void>();
public hangup(): void {
this.userHangup$.next();
@@ -626,7 +505,7 @@ export class CallViewModel {
public readonly leave$: Observable<
"user" | "timeout" | "decline" | "allOthersLeft"
> = merge(
this.autoLeave$,
this.callLifecycle.autoLeave$,
merge(this.userHangup$, this.widgetHangup$).pipe(
map(() => "user" as const),
),
@@ -717,6 +596,7 @@ export class CallViewModel {
private readonly pip$ = this.scope.behavior<UserMediaViewModel | null>(
combineLatest([
// TODO This also needs epoch logic to dedupe the screenshares and mediaItems emits
this.screenShares$,
this.spotlightSpeaker$,
this.mediaItems$,
@@ -1298,47 +1178,16 @@ export class CallViewModel {
/**
* Whether we are sharing our screen.
*/
// TODO move to LocalMembership
public readonly sharingScreen$ = this.scope.behavior(
from(this.localConnection$).pipe(
switchMap((c) =>
c?.state === "ready"
? sharingScreen$(c.value.livekitRoom.localParticipant)
: of(false),
),
),
);
// reassigned here to make it publicly accessible
public readonly sharingScreen$ = this.localMembership.sharingScreen$;
/**
* Callback for toggling screen sharing. If null, screen sharing is not
* available.
*/
// TODO move to LocalMembership
// reassigned here to make it publicly accessible
public readonly toggleScreenSharing =
"getDisplayMedia" in (navigator.mediaDevices ?? {}) &&
!this.urlParams.hideScreensharing
? (): void =>
// Once a connection is ready...
void this.localConnection$
.pipe(
takeWhile((c) => c !== null && c.state !== "error"),
switchMap((c) => (c.state === "ready" ? of(c.value) : NEVER)),
take(1),
this.scope.bind(),
)
// ...toggle screen sharing.
.subscribe(
(c) =>
void c.livekitRoom.localParticipant
.setScreenShareEnabled(!this.sharingScreen$.value, {
audio: true,
selfBrowserSurface: "include",
surfaceSwitching: "include",
systemAudio: "include",
})
.catch(logger.error),
)
: null;
this.localMembership.toggleScreenSharing;
public constructor(
private readonly scope: ObservableScope,

0
src/state/CallViewModel/callPickupState$.ts Normal file
View File

179
src/state/localMember/LocalMembership.ts → src/state/CallViewModel/localMember/LocalMembership.ts
View File

@@ -18,58 +18,63 @@ import {
combineLatest,
fromEvent,
map,
NEVER,
type Observable,
of,
scan,
startWith,
switchMap,
take,
takeWhile,
tap,
} from "rxjs";
import { logger } from "matrix-js-sdk/lib/logger";
import { type Behavior } from "../Behavior";
import { sharingScreen$ as observeSharingScreen$ } from "../../UserMedia.ts";
import { type Behavior } from "../../Behavior";
import { type IConnectionManager } from "../remoteMembers/ConnectionManager";
import { ObservableScope } from "../ObservableScope";
import { ObservableScope } from "../../ObservableScope";
import { Publisher } from "./Publisher";
import { type MuteStates } from "../MuteStates";
import { type ProcessorState } from "../../livekit/TrackProcessorContext";
import { type MediaDevices } from "../MediaDevices";
import { and$ } from "../../utils/observable";
import { type MuteStates } from "../../MuteStates";
import { type ProcessorState } from "../../../livekit/TrackProcessorContext";
import { type MediaDevices } from "../../MediaDevices";
import { and$ } from "../../../utils/observable";
import {
enterRTCSession,
type EnterRTCSessionOptions,
} from "../../rtcSessionHelpers";
import { type ElementCallError } from "../../utils/errors";
import { ElementWidgetActions, type WidgetHelpers } from "../../widget";
} from "../../../rtcSessionHelpers";
import { type ElementCallError } from "../../../utils/errors";
import { ElementWidgetActions, type WidgetHelpers } from "../../../widget";
import { areLivekitTransportsEqual } from "../remoteMembers/MatrixLivekitMembers";
import { getUrlParams } from "../../../UrlParams.ts";
enum LivekitState {
UNINITIALIZED = "uninitialized",
CONNECTING = "connecting",
CONNECTED = "connected",
ERROR = "error",
DISCONNECTED = "disconnected",
DISCONNECTING = "disconnecting",
export enum LivekitState {
Uninitialized = "uninitialized",
Connecting = "connecting",
Connected = "connected",
Error = "error",
Disconnected = "disconnected",
Disconnecting = "disconnecting",
}
type LocalMemberLivekitState =
| { state: LivekitState.ERROR; error: string }
| { state: LivekitState.CONNECTED }
| { state: LivekitState.CONNECTING }
| { state: LivekitState.UNINITIALIZED }
| { state: LivekitState.DISCONNECTED }
| { state: LivekitState.DISCONNECTING };
| { state: LivekitState.Error; error: string }
| { state: LivekitState.Connected }
| { state: LivekitState.Connecting }
| { state: LivekitState.Uninitialized }
| { state: LivekitState.Disconnected }
| { state: LivekitState.Disconnecting };
enum MatrixState {
CONNECTED = "connected",
DISCONNECTED = "disconnected",
CONNECTING = "connecting",
export enum MatrixState {
Connected = "connected",
Disconnected = "disconnected",
Connecting = "connecting",
}
type LocalMemberMatrixState =
| { state: MatrixState.CONNECTED }
| { state: MatrixState.CONNECTING }
| { state: MatrixState.DISCONNECTED };
| { state: MatrixState.Connected }
| { state: MatrixState.Connecting }
| { state: MatrixState.Disconnected };
export interface LocalMemberState {
export interface LocalMemberConnectionState {
livekit$: BehaviorSubject<LocalMemberLivekitState>;
matrix$: BehaviorSubject<LocalMemberMatrixState>;
}
@@ -107,9 +112,10 @@ interface Props {
* @param param0
* @returns
* - publisher: The handle to create tracks and publish them to the room.
* - connected$: the current connection state. Including matrix server and livekit server connection. (only the livekit server relevant for our own participation)
* - connected$: the current connection state. Including matrix server and livekit server connection. (only considering the livekit server we are using for our own media publication)
* - transport$: the transport object the ownMembership$ ended up using.
*
* - connectionState: the current connection state. Including matrix server and livekit server connection.
* - sharingScreen$: Whether we are sharing our screen. `undefined` if we cannot share the screen.
*/
export const createLocalMembership$ = ({
scope,
@@ -125,21 +131,31 @@ export const createLocalMembership$ = ({
widget,
}: Props): {
// publisher: Publisher
requestConnect: () => LocalMemberState;
requestConnect: () => LocalMemberConnectionState;
startTracks: () => Behavior<LocalTrack[]>;
requestDisconnect: () => Observable<LocalMemberLivekitState> | null;
state: LocalMemberState; // TODO this is probably superseeded by joinState$
connectionState: LocalMemberConnectionState;
sharingScreen$: Behavior<boolean | undefined>;
toggleScreenSharing: (() => void) | null;
// deprecated fields
/** @deprecated use state instead*/
homeserverConnected$: Behavior<boolean>;
/** @deprecated use state instead*/
connected$: Behavior<boolean>;
// this needs to be discussed
/** @deprecated use state instead*/
reconnecting$: Behavior<boolean>;
// also needs to be disccues
/** @deprecated use state instead*/
configError$: Behavior<ElementCallError | null>;
} => {
const state = {
livekit$: new BehaviorSubject<LocalMemberLivekitState>({
state: LivekitState.UNINITIALIZED,
state: LivekitState.Uninitialized,
}),
matrix$: new BehaviorSubject<LocalMemberMatrixState>({
state: MatrixState.DISCONNECTED,
state: MatrixState.Disconnected,
}),
};
@@ -271,23 +287,23 @@ export const createLocalMembership$ = ({
return tracks$;
};
const requestConnect = (): LocalMemberState => {
const requestConnect = (): LocalMemberConnectionState => {
if (state.livekit$.value === null) {
startTracks();
state.livekit$.next({ state: LivekitState.CONNECTING });
state.livekit$.next({ state: LivekitState.Connecting });
combineLatest([publisher$, tracks$], (publisher, tracks) => {
publisher
?.startPublishing()
.then(() => {
state.livekit$.next({ state: LivekitState.CONNECTED });
state.livekit$.next({ state: LivekitState.Connected });
})
.catch((error) => {
state.livekit$.next({ state: LivekitState.ERROR, error });
state.livekit$.next({ state: LivekitState.Error, error });
});
});
}
if (state.matrix$.value.state !== MatrixState.DISCONNECTED) {
state.matrix$.next({ state: MatrixState.CONNECTING });
if (state.matrix$.value.state !== MatrixState.Disconnected) {
state.matrix$.next({ state: MatrixState.Connecting });
localTransport$.pipe(
tap((transport) => {
if (transport !== undefined) {
@@ -306,17 +322,17 @@ export const createLocalMembership$ = ({
};
const requestDisconnect = (): Behavior<LocalMemberLivekitState> | null => {
if (state.livekit$.value.state !== LivekitState.CONNECTED) return null;
state.livekit$.next({ state: LivekitState.DISCONNECTING });
if (state.livekit$.value.state !== LivekitState.Connected) return null;
state.livekit$.next({ state: LivekitState.Disconnecting });
combineLatest([publisher$, tracks$], (publisher, tracks) => {
publisher
?.stopPublishing()
.then(() => {
tracks.forEach((track) => track.stop());
state.livekit$.next({ state: LivekitState.DISCONNECTED });
state.livekit$.next({ state: LivekitState.Disconnected });
})
.catch((error) => {
state.livekit$.next({ state: LivekitState.ERROR, error });
state.livekit$.next({ state: LivekitState.Error, error });
});
});
@@ -410,14 +426,83 @@ export const createLocalMembership$ = ({
}
});
/**
* Returns undefined if scrennSharing is not yet ready.
*/
const sharingScreen$ = scope.behavior(
connection$.pipe(
switchMap((c) => {
if (!c) return of(undefined);
if (c.state$.value.state === "ConnectedToLkRoom")
return observeSharingScreen$(c.livekitRoom.localParticipant);
return of(false);
}),
),
);
const toggleScreenSharing =
"getDisplayMedia" in (navigator.mediaDevices ?? {}) &&
!getUrlParams().hideScreensharing
? (): void =>
// If a connection is ready...
void connection$
.pipe(
// I dont see why we need this. isnt the check later on superseeding it?
takeWhile(
(c) =>
c !== undefined && c.state$.value.state !== "FailedToStart",
),
switchMap((c) =>
c?.state$.value.state === "ConnectedToLkRoom" ? of(c) : NEVER,
),
take(1),
scope.bind(),
)
// ...toggle screen sharing.
.subscribe(
(c) =>
void c.livekitRoom.localParticipant
.setScreenShareEnabled(!sharingScreen$.value, {
audio: true,
selfBrowserSurface: "include",
surfaceSwitching: "include",
systemAudio: "include",
})
.catch(logger.error),
)
: null;
// we do not need all the auto waiting since we can just check via sharingScreen$.value !== undefined
let alternativeScreenshareToggle: (() => void) | null = null;
if (
"getDisplayMedia" in (navigator.mediaDevices ?? {}) &&
!getUrlParams().hideScreensharing
) {
alternativeScreenshareToggle = (): void =>
void connection$.value?.livekitRoom.localParticipant
.setScreenShareEnabled(!sharingScreen$.value, {
audio: true,
selfBrowserSurface: "include",
surfaceSwitching: "include",
systemAudio: "include",
})
.catch(logger.error);
}
logger.log(
"alternativeScreenshareToggle so that it is used",
alternativeScreenshareToggle,
);
return {
startTracks,
requestConnect,
requestDisconnect,
state,
connectionState: state,
homeserverConnected$,
connected$,
reconnecting$,
configError$,
sharingScreen$,
toggleScreenSharing,
};
};

10
src/state/localMember/LocalTransport.ts → src/state/CallViewModel/localMember/LocalTransport.ts
View File

@@ -18,15 +18,15 @@ import { logger } from "matrix-js-sdk/lib/logger";
import { AutoDiscovery } from "matrix-js-sdk/lib/autodiscovery";
import { deepCompare } from "matrix-js-sdk/lib/utils";
import { type Behavior } from "../Behavior.ts";
import { type Behavior } from "../../Behavior.ts";
import {
type Epoch,
mapEpoch,
type ObservableScope,
} from "../ObservableScope.ts";
import { Config } from "../../config/Config.ts";
import { MatrixRTCTransportMissingError } from "../../utils/errors.ts";
import { getSFUConfigWithOpenID } from "../../livekit/openIDSFU.ts";
} from "../../ObservableScope.ts";
import { Config } from "../../../config/Config.ts";
import { MatrixRTCTransportMissingError } from "../../../utils/errors.ts";
import { getSFUConfigWithOpenID } from "../../../livekit/openIDSFU.ts";
/*
* - get well known

16
src/state/localMember/Publisher.ts → src/state/CallViewModel/localMember/Publisher.ts
View File

@@ -22,17 +22,17 @@ import {
} from "rxjs";
import { type Logger } from "matrix-js-sdk/lib/logger";
import type { Behavior } from "../Behavior.ts";
import type { MediaDevices, SelectedDevice } from "../MediaDevices.ts";
import type { MuteStates } from "../MuteStates.ts";
import type { Behavior } from "../../Behavior.ts";
import type { MediaDevices, SelectedDevice } from "../../MediaDevices.ts";
import type { MuteStates } from "../../MuteStates.ts";
import {
type ProcessorState,
trackProcessorSync,
} from "../../livekit/TrackProcessorContext.tsx";
import { getUrlParams } from "../../UrlParams.ts";
import { observeTrackReference$ } from "../MediaViewModel.ts";
import { type Connection } from "../remoteMembers/Connection.ts";
import { type ObservableScope } from "../ObservableScope.ts";
} from "../../../livekit/TrackProcessorContext.tsx";
import { getUrlParams } from "../../../UrlParams.ts";
import { observeTrackReference$ } from "../../MediaViewModel.ts";
import { type Connection } from "../CallViewModel/remoteMembers/Connection.ts";
import { type ObservableScope } from "../../ObservableScope.ts";
/**
* A wrapper for a Connection object.

12
src/state/remoteMembers/Connection.test.ts → src/state/CallViewModel/remoteMembers/Connection.test.ts
View File

@@ -36,12 +36,12 @@ import {
type ConnectionState,
type PublishingParticipant,
} from "./Connection.ts";
import { ObservableScope } from "../ObservableScope.ts";
import { type OpenIDClientParts } from "../../livekit/openIDSFU.ts";
import { FailToGetOpenIdToken } from "../../utils/errors.ts";
import { mockMediaDevices, mockMuteStates } from "../../utils/test.ts";
import type { ProcessorState } from "../../livekit/TrackProcessorContext.tsx";
import { type MuteStates } from "../MuteStates.ts";
import { ObservableScope } from "../../ObservableScope.ts";
import { type OpenIDClientParts } from "../../../livekit/openIDSFU.ts";
import { FailToGetOpenIdToken } from "../../../utils/errors.ts";
import { mockMediaDevices, mockMuteStates } from "../../../utils/test.ts";
import type { ProcessorState } from "../../../livekit/TrackProcessorContext.tsx";
import { type MuteStates } from "../../MuteStates.ts";
let testScope: ObservableScope;

8
src/state/remoteMembers/Connection.ts → src/state/CallViewModel/remoteMembers/Connection.ts
View File

@@ -25,13 +25,13 @@ import {
getSFUConfigWithOpenID,
type OpenIDClientParts,
type SFUConfig,
} from "../../livekit/openIDSFU.ts";
import { type Behavior } from "../Behavior.ts";
import { type ObservableScope } from "../ObservableScope.ts";
} from "../../../livekit/openIDSFU.ts";
import { type Behavior } from "../../Behavior.ts";
import { type ObservableScope } from "../../ObservableScope.ts";
import {
InsufficientCapacityError,
SFURoomCreationRestrictedError,
} from "../../utils/errors.ts";
} from "../../../utils/errors.ts";
export type PublishingParticipant = LocalParticipant | RemoteParticipant;

12
src/state/remoteMembers/ConnectionFactory.ts → src/state/CallViewModel/remoteMembers/ConnectionFactory.ts
View File

@@ -13,13 +13,13 @@ import {
} from "livekit-client";
import { type Logger } from "matrix-js-sdk/lib/logger";
import { type ObservableScope } from "../ObservableScope.ts";
import { type ObservableScope } from "../../ObservableScope.ts";
import { Connection } from "./Connection.ts";
import type { OpenIDClientParts } from "../../livekit/openIDSFU.ts";
import type { MediaDevices } from "../MediaDevices.ts";
import type { Behavior } from "../Behavior.ts";
import type { ProcessorState } from "../../livekit/TrackProcessorContext.tsx";
import { defaultLiveKitOptions } from "../../livekit/options.ts";
import type { OpenIDClientParts } from "../../../livekit/openIDSFU.ts";
import type { MediaDevices } from "../../MediaDevices.ts";
import type { Behavior } from "../../Behavior.ts";
import type { ProcessorState } from "../../../livekit/TrackProcessorContext.tsx";
import { defaultLiveKitOptions } from "../../../livekit/options.ts";
export interface ConnectionFactory {
createConnection(

4
src/state/remoteMembers/ConnectionManager.test.ts → src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts
View File

@@ -10,14 +10,14 @@ import { BehaviorSubject } from "rxjs";
import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
import { type Participant as LivekitParticipant } from "livekit-client";
import { ObservableScope } from "../ObservableScope.ts";
import { ObservableScope } from "../../ObservableScope.ts";
import {
type IConnectionManager,
createConnectionManager$,
} from "./ConnectionManager.ts";
import { type ConnectionFactory } from "./ConnectionFactory.ts";
import { type Connection } from "./Connection.ts";
import { flushPromises, withTestScheduler } from "../../utils/test.ts";
import { flushPromises, withTestScheduler } from "../../../utils/test.ts";
import { areLivekitTransportsEqual } from "./MatrixLivekitMembers.ts";
// Some test constants

8
src/state/remoteMembers/ConnectionManager.ts → src/state/CallViewModel/remoteMembers/ConnectionManager.ts
View File

@@ -17,10 +17,10 @@ import { BehaviorSubject, combineLatest, map, switchMap } from "rxjs";
import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
import { type LocalParticipant, type RemoteParticipant } from "livekit-client";
import { type Behavior } from "../Behavior";
import { type Connection } from "./Connection";
import { Epoch, type ObservableScope } from "../ObservableScope";
import { generateKeyed$ } from "../../utils/observable";
import { type Behavior } from "../../Behavior.ts";
import { type Connection } from "./Connection.ts";
import { Epoch, type ObservableScope } from "../../ObservableScope.ts";
import { generateKeyed$ } from "../../../utils/observable.ts";
import { areLivekitTransportsEqual } from "./MatrixLivekitMembers.ts";
import { type ConnectionFactory } from "./ConnectionFactory.ts";

4
src/state/remoteMembers/MatrixLivekitMembers.test.ts → src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts
View File

@@ -19,14 +19,14 @@ import {
createMatrixLivekitMembers$,
areLivekitTransportsEqual,
} from "./MatrixLivekitMembers.ts";
import { ObservableScope } from "../ObservableScope.ts";
import { ObservableScope } from "../../ObservableScope.ts";
import { ConnectionManagerData } from "./ConnectionManager.ts";
import {
mockCallMembership,
mockRemoteParticipant,
type OurRunHelpers,
withTestScheduler,
} from "../../utils/test.ts";
} from "../../../utils/test.ts";
import { type Connection } from "./Connection.ts";
let testScope: ObservableScope;

4
src/state/remoteMembers/MatrixLivekitMembers.ts → src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts
View File

@@ -19,9 +19,9 @@ import { type NodeStyleEventEmitter } from "rxjs/internal/observable/fromEvent";
import { type Room as MatrixRoom, type RoomMember } from "matrix-js-sdk";
import { logger } from "matrix-js-sdk/lib/logger";
import { type Behavior } from "../Behavior";
import { type Behavior } from "../../Behavior";
import { type IConnectionManager } from "./ConnectionManager";
import { Epoch, mapEpoch, type ObservableScope } from "../ObservableScope";
import { Epoch, mapEpoch, type ObservableScope } from "../../ObservableScope";
import { getRoomMemberFromRtcMember, memberDisplaynames$ } from "./displayname";
import { type Connection } from "./Connection";

4
src/state/remoteMembers/displayname.test.ts → src/state/CallViewModel/remoteMembers/displayname.test.ts
View File

@@ -14,9 +14,9 @@ import {
} from "matrix-js-sdk";
import EventEmitter from "events";
import { ObservableScope } from "../ObservableScope.ts";
import { ObservableScope } from "../../ObservableScope.ts";
import type { Room as MatrixRoom } from "matrix-js-sdk/lib/models/room";
import { mockCallMembership, withTestScheduler } from "../../utils/test.ts";
import { mockCallMembership, withTestScheduler } from "../../../utils/test.ts";
import { memberDisplaynames$ } from "./displayname.ts";
let testScope: ObservableScope;

6
src/state/remoteMembers/displayname.ts → src/state/CallViewModel/remoteMembers/displayname.ts
View File

@@ -19,12 +19,12 @@ import { type Room as MatrixRoom } from "matrix-js-sdk/lib/matrix";
// eslint-disable-next-line rxjs/no-internal
import { type NodeStyleEventEmitter } from "rxjs/internal/observable/fromEvent";
import { Epoch, type ObservableScope } from "../ObservableScope";
import { Epoch, type ObservableScope } from "../../ObservableScope";
import {
calculateDisplayName,
shouldDisambiguate,
} from "../../utils/displayname";
import { type Behavior } from "../Behavior";
} from "../../../utils/displayname";
import { type Behavior } from "../../Behavior";
/**
* Displayname for each member of the call. This will disambiguate

14
src/state/remoteMembers/integration.test.ts → src/state/CallViewModel/remoteMembers/integration.test.ts
View File

@@ -14,22 +14,26 @@ import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
import { type Room as MatrixRoom, type RoomMember } from "matrix-js-sdk";
import { logger } from "matrix-js-sdk/lib/logger";
import { type Epoch, ObservableScope, trackEpoch } from "../ObservableScope.ts";
import {
type Epoch,
ObservableScope,
trackEpoch,
} from "../../ObservableScope.ts";
import { ECConnectionFactory } from "./ConnectionFactory.ts";
import { type OpenIDClientParts } from "../../livekit/openIDSFU.ts";
import { type OpenIDClientParts } from "../../../livekit/openIDSFU.ts";
import {
mockCallMembership,
mockMediaDevices,
withTestScheduler,
} from "../../utils/test";
import { type ProcessorState } from "../../livekit/TrackProcessorContext.tsx";
} from "../../../utils/test.ts";
import { type ProcessorState } from "../../../livekit/TrackProcessorContext.tsx";
import {
areLivekitTransportsEqual,
createMatrixLivekitMembers$,
type MatrixLivekitMember,
} from "./MatrixLivekitMembers.ts";
import { createConnectionManager$ } from "./ConnectionManager.ts";
import { membershipsAndTransports$ } from "../SessionBehaviors.ts";
import { membershipsAndTransports$ } from "../../SessionBehaviors.ts";
// Test the integration of ConnectionManager and MatrixLivekitMerger