mirror of
https://github.com/vector-im/element-call.git
synced 2026-10-16 23:45:50 +00:00
use the matrixRtcSdk
This commit is contained in:
Timo K.
@@ -7,18 +7,16 @@ Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import {
|
||||
type Room as LivekitRoom,
|
||||
RoomEvent as LivekitRoomEvent,
|
||||
type Participant,
|
||||
type Track,
|
||||
} from "livekit-client";
|
||||
import {
|
||||
type AudioSource,
|
||||
roomEventSelector,
|
||||
type TrackReference,
|
||||
type VideoSource,
|
||||
} from "@livekit/components-core";
|
||||
import { type LocalParticipant, type RemoteParticipant } from "livekit-client";
|
||||
type AudioMediaTrack,
|
||||
type Behavior,
|
||||
E2eeType,
|
||||
type EncryptionError,
|
||||
type EncryptionSystem,
|
||||
type MediaTrack,
|
||||
type MemberMedia,
|
||||
type ObservableScope,
|
||||
type VideoMediaTrack,
|
||||
} from "@element-hq/matrixrtc-sdk";
|
||||
import {
|
||||
combineLatest,
|
||||
distinctUntilChanged,
|
||||
@@ -31,13 +29,7 @@ import {
|
||||
throttleTime,
|
||||
} from "rxjs";
|
||||
|
||||
import { type Behavior } from "../Behavior";
|
||||
import { type BaseMediaViewModel, createBaseMedia } from "./MediaViewModel";
|
||||
import { type EncryptionSystem } from "../../e2ee/sharedKeyManagement";
|
||||
import { type ObservableScope } from "../ObservableScope";
|
||||
import { observeTrackReference$ } from "../observeTrackReference";
|
||||
import { E2eeType } from "../../e2ee/e2eeType";
|
||||
import { observeInboundRtpStreamStats$ } from "./observeRtpStreamStats";
|
||||
import { type UserMediaViewModel } from "./UserMediaViewModel";
|
||||
import { type ScreenShareViewModel } from "./ScreenShareViewModel";
|
||||
import { type UnstableLivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
|
||||
@@ -56,12 +48,12 @@ export enum EncryptionStatus {
|
||||
*/
|
||||
export interface BaseMemberMediaViewModel extends BaseMediaViewModel {
|
||||
/**
|
||||
* The LiveKit video track for this media.
|
||||
* The video track for this media.
|
||||
*/
|
||||
video$: Behavior<TrackReference | undefined>;
|
||||
video$: Behavior<VideoMediaTrack | undefined>;
|
||||
/**
|
||||
* The LiveKit transport on which this member should be publishing. Exposed
|
||||
* for debugging.
|
||||
* The URL of the transport on which this member should be publishing.
|
||||
* Exposed for debugging.
|
||||
*/
|
||||
transport$: Behavior<UnstableLivekitTransport | undefined>;
|
||||
/**
|
||||
@@ -72,19 +64,31 @@ export interface BaseMemberMediaViewModel extends BaseMediaViewModel {
|
||||
}
|
||||
|
||||
export interface MemberMediaInputs extends BaseMediaViewModel {
|
||||
participant$: Behavior<LocalParticipant | RemoteParticipant | null>;
|
||||
livekitRoom$: Behavior<LivekitRoom | undefined>;
|
||||
audioSource: AudioSource;
|
||||
videoSource: VideoSource;
|
||||
transport$: Behavior<UnstableLivekitTransport | undefined>;
|
||||
media$: Behavior<MemberMedia | null>;
|
||||
audioSource: "microphone" | "screenShareAudio";
|
||||
videoSource: "camera" | "screenShare";
|
||||
focusUrl$: Behavior<string | undefined>;
|
||||
encryptionSystem: EncryptionSystem;
|
||||
}
|
||||
|
||||
/** The member's track for a source, undefined while there is none. */
|
||||
export function memberTrack$<T extends MediaTrack>(
|
||||
media$: Behavior<MemberMedia | null>,
|
||||
source: "microphone" | "camera" | "screenShare" | "screenShareAudio",
|
||||
): Observable<T | undefined> {
|
||||
return media$.pipe(
|
||||
switchMap((media) =>
|
||||
media === null
|
||||
? of(undefined)
|
||||
: (media[`${source}$`] as Behavior<T | undefined>),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
export function createMemberMedia(
|
||||
scope: ObservableScope,
|
||||
{
|
||||
participant$,
|
||||
livekitRoom$,
|
||||
media$,
|
||||
audioSource,
|
||||
videoSource,
|
||||
transport$,
|
||||
@@ -92,183 +96,104 @@ export function createMemberMedia(
|
||||
...inputs
|
||||
}: MemberMediaInputs,
|
||||
): BaseMemberMediaViewModel {
|
||||
const trackBehavior$ = (
|
||||
scope: ObservableScope,
|
||||
source: Track.Source,
|
||||
): Behavior<TrackReference | undefined> =>
|
||||
scope.behavior(
|
||||
participant$.pipe(
|
||||
switchMap((p) =>
|
||||
!p ? of(undefined) : observeTrackReference$(p, source),
|
||||
),
|
||||
const audio$ = scope.behavior(
|
||||
memberTrack$<AudioMediaTrack>(media$, audioSource),
|
||||
);
|
||||
const video$ = scope.behavior(
|
||||
memberTrack$<VideoMediaTrack>(media$, videoSource),
|
||||
);
|
||||
const unencrypted$ = (
|
||||
track$: Behavior<MediaTrack | undefined>,
|
||||
): Observable<boolean> =>
|
||||
track$.pipe(
|
||||
switchMap((track) =>
|
||||
track === undefined
|
||||
? of(false)
|
||||
: track.encrypted$.pipe(map((encrypted) => !encrypted)),
|
||||
),
|
||||
);
|
||||
|
||||
const audio$ = trackBehavior$(scope, audioSource);
|
||||
const video$ = trackBehavior$(scope, videoSource);
|
||||
|
||||
return {
|
||||
...createBaseMedia(inputs),
|
||||
video$,
|
||||
transport$,
|
||||
unencryptedWarning$: scope.behavior(
|
||||
combineLatest(
|
||||
[audio$, video$],
|
||||
(a, v) =>
|
||||
encryptionSystem.kind !== E2eeType.NONE &&
|
||||
(a?.publication.isEncrypted === false ||
|
||||
v?.publication.isEncrypted === false),
|
||||
[unencrypted$(audio$), unencrypted$(video$)],
|
||||
(a, v) => encryptionSystem.kind !== E2eeType.NONE && (a || v),
|
||||
),
|
||||
),
|
||||
encryptionStatus$: scope.behavior(
|
||||
participant$.pipe(
|
||||
switchMap((participant): Observable<EncryptionStatus> => {
|
||||
if (!participant) {
|
||||
return of(EncryptionStatus.Connecting);
|
||||
} else if (
|
||||
participant.isLocal ||
|
||||
encryptionSystem.kind === E2eeType.NONE
|
||||
) {
|
||||
media$.pipe(
|
||||
switchMap((media): Observable<EncryptionStatus> => {
|
||||
if (media === null) return of(EncryptionStatus.Connecting);
|
||||
if (media.local || encryptionSystem.kind === E2eeType.NONE)
|
||||
return of(EncryptionStatus.Okay);
|
||||
} else if (encryptionSystem.kind === E2eeType.PER_PARTICIPANT) {
|
||||
return combineLatest([
|
||||
encryptionErrorObservable$(
|
||||
livekitRoom$,
|
||||
participant,
|
||||
encryptionSystem,
|
||||
"MissingKey",
|
||||
),
|
||||
encryptionErrorObservable$(
|
||||
livekitRoom$,
|
||||
participant,
|
||||
encryptionSystem,
|
||||
"InvalidKey",
|
||||
),
|
||||
observeRemoteTrackReceivingOkay$(participant, audioSource),
|
||||
observeRemoteTrackReceivingOkay$(participant, videoSource),
|
||||
]).pipe(
|
||||
map(([keyMissing, keyInvalid, audioOkay, videoOkay]) => {
|
||||
if (keyMissing) return EncryptionStatus.KeyMissing;
|
||||
if (keyInvalid) return EncryptionStatus.KeyInvalid;
|
||||
if (audioOkay || videoOkay) return EncryptionStatus.Okay;
|
||||
return undefined; // no change
|
||||
}),
|
||||
filter((x) => !!x),
|
||||
startWith(EncryptionStatus.Connecting),
|
||||
);
|
||||
} else {
|
||||
return combineLatest([
|
||||
encryptionErrorObservable$(
|
||||
livekitRoom$,
|
||||
participant,
|
||||
encryptionSystem,
|
||||
"InvalidKey",
|
||||
),
|
||||
observeRemoteTrackReceivingOkay$(participant, audioSource),
|
||||
observeRemoteTrackReceivingOkay$(participant, videoSource),
|
||||
]).pipe(
|
||||
map(
|
||||
([keyInvalid, audioOkay, videoOkay]):
|
||||
| EncryptionStatus
|
||||
| undefined => {
|
||||
if (keyInvalid) return EncryptionStatus.PasswordInvalid;
|
||||
if (audioOkay || videoOkay) return EncryptionStatus.Okay;
|
||||
return undefined; // no change
|
||||
},
|
||||
),
|
||||
filter((x) => !!x),
|
||||
startWith(EncryptionStatus.Connecting),
|
||||
);
|
||||
}
|
||||
const perParticipant =
|
||||
encryptionSystem.kind === E2eeType.PER_PARTICIPANT;
|
||||
return combineLatest([
|
||||
encryptionError$(media, "MissingKey"),
|
||||
encryptionError$(media, "InvalidKey"),
|
||||
receivingOkay$(audio$),
|
||||
receivingOkay$(video$),
|
||||
]).pipe(
|
||||
map(([keyMissing, keyInvalid, audioOkay, videoOkay]) => {
|
||||
if (perParticipant && keyMissing)
|
||||
return EncryptionStatus.KeyMissing;
|
||||
if (keyInvalid)
|
||||
return perParticipant
|
||||
? EncryptionStatus.KeyInvalid
|
||||
: EncryptionStatus.PasswordInvalid;
|
||||
if (audioOkay || videoOkay) return EncryptionStatus.Okay;
|
||||
return undefined; // no change
|
||||
}),
|
||||
filter((x) => x !== undefined),
|
||||
startWith(EncryptionStatus.Connecting),
|
||||
);
|
||||
}),
|
||||
),
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
function encryptionErrorObservable$(
|
||||
room$: Behavior<LivekitRoom | undefined>,
|
||||
participant: Participant,
|
||||
encryptionSystem: EncryptionSystem,
|
||||
criteria: string,
|
||||
function encryptionError$(
|
||||
media: MemberMedia,
|
||||
criteria: EncryptionError,
|
||||
): Observable<boolean> {
|
||||
return room$.pipe(
|
||||
switchMap((room) => {
|
||||
if (room === undefined) return of(false);
|
||||
return roomEventSelector(room, LivekitRoomEvent.EncryptionError).pipe(
|
||||
map((e) => {
|
||||
const [err] = e;
|
||||
if (encryptionSystem.kind === E2eeType.PER_PARTICIPANT) {
|
||||
return (
|
||||
// Ideally we would pull the participant identity from the field on the error.
|
||||
// However, it gets lost in the serialization process between workers.
|
||||
// So, instead we do a string match
|
||||
(err?.message.includes(participant.identity) &&
|
||||
err?.message.includes(criteria)) ??
|
||||
false
|
||||
);
|
||||
} else if (encryptionSystem.kind === E2eeType.SHARED_KEY) {
|
||||
return !!err?.message.includes(criteria);
|
||||
}
|
||||
|
||||
return false;
|
||||
}),
|
||||
);
|
||||
}),
|
||||
return media.encryptionError$.pipe(
|
||||
map((error) => error === criteria),
|
||||
distinctUntilChanged(),
|
||||
throttleTime(1000), // Throttle to avoid spamming the UI
|
||||
startWith(false),
|
||||
);
|
||||
}
|
||||
|
||||
function observeRemoteTrackReceivingOkay$(
|
||||
participant: Participant,
|
||||
source: Track.Source,
|
||||
/**
|
||||
* Whether frames arrive and decode, which is the only sign that the key in
|
||||
* use is the right one.
|
||||
*/
|
||||
function receivingOkay$(
|
||||
track$: Behavior<MediaTrack | undefined>,
|
||||
): Observable<boolean | undefined> {
|
||||
let lastStats: {
|
||||
framesDecoded: number | undefined;
|
||||
framesDropped: number | undefined;
|
||||
framesReceived: number | undefined;
|
||||
} = {
|
||||
framesDecoded: undefined,
|
||||
framesDropped: undefined,
|
||||
framesReceived: undefined,
|
||||
};
|
||||
|
||||
return observeInboundRtpStreamStats$(participant, source).pipe(
|
||||
map((stats) => {
|
||||
if (!stats) return undefined;
|
||||
const { framesDecoded, framesDropped, framesReceived } = stats;
|
||||
return {
|
||||
framesDecoded,
|
||||
framesDropped,
|
||||
framesReceived,
|
||||
};
|
||||
}),
|
||||
filter((newStats) => !!newStats),
|
||||
map((newStats): boolean | undefined => {
|
||||
const oldStats = lastStats;
|
||||
lastStats = newStats;
|
||||
let last: { framesDecoded?: number; framesReceived?: number } = {};
|
||||
return track$.pipe(
|
||||
switchMap((track) => track?.stats$ ?? of(undefined)),
|
||||
map((stats): boolean | undefined => {
|
||||
if (stats === undefined || stats.type !== "inbound-rtp") return undefined;
|
||||
const { framesDecoded, framesReceived } =
|
||||
stats as RTCInboundRtpStreamStats;
|
||||
const previous = last;
|
||||
last = { framesDecoded, framesReceived };
|
||||
if (
|
||||
typeof newStats.framesReceived === "number" &&
|
||||
typeof oldStats.framesReceived === "number" &&
|
||||
typeof newStats.framesDecoded === "number" &&
|
||||
typeof oldStats.framesDecoded === "number"
|
||||
) {
|
||||
const framesReceivedDelta =
|
||||
newStats.framesReceived - oldStats.framesReceived;
|
||||
const framesDecodedDelta =
|
||||
newStats.framesDecoded - oldStats.framesDecoded;
|
||||
|
||||
// if we received >0 frames and managed to decode >0 frames then we treat that as success
|
||||
|
||||
if (framesReceivedDelta > 0) {
|
||||
return framesDecodedDelta > 0;
|
||||
}
|
||||
}
|
||||
|
||||
// no change
|
||||
return undefined;
|
||||
framesReceived === undefined ||
|
||||
previous.framesReceived === undefined ||
|
||||
framesDecoded === undefined ||
|
||||
previous.framesDecoded === undefined
|
||||
)
|
||||
return undefined;
|
||||
const received = framesReceived - previous.framesReceived;
|
||||
if (received > 0) return framesDecoded - previous.framesDecoded > 0;
|
||||
return undefined; // no change
|
||||
}),
|
||||
filter((x) => typeof x === "boolean"),
|
||||
startWith(undefined),
|
||||
|
||||
Reference in New Issue
Block a user