diff --git a/locales/en/app.json b/locales/en/app.json index 0e71fd4e..f35c3579 100644 --- a/locales/en/app.json +++ b/locales/en/app.json @@ -73,6 +73,7 @@ "duplicate_tiles_label": "Number of additional tile copies per participant", "hostname": "Hostname: {{hostname}}", "matrix_id": "Matrix ID: {{id}}", + "show_connection_stats": "Show connection statistics", "show_non_member_tiles": "Show tiles for non-member media" }, "disconnected_banner": "Connectivity to the server has been lost.", diff --git a/src/RTCConnectionStats.module.css b/src/RTCConnectionStats.module.css new file mode 100644 index 00000000..0e29eaa9 --- /dev/null +++ b/src/RTCConnectionStats.module.css @@ -0,0 +1,20 @@ +/* +Copyright 2024 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only +Please see LICENSE in the repository root for full details. +*/ + +.modal pre { + font-size: var(--font-size-micro); +} + +.statsPill { + border-radius: var(--media-view-border-radius); + grid-area: none; + position: absolute; + top: 0; + left: 0; + flex-direction: column; + align-items: flex-start; +} diff --git a/src/RTCConnectionStats.tsx b/src/RTCConnectionStats.tsx new file mode 100644 index 00000000..d092b677 --- /dev/null +++ b/src/RTCConnectionStats.tsx @@ -0,0 +1,112 @@ +/* +Copyright 2023, 2024 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only +Please see LICENSE in the repository root for full details. +*/ + +import { useState, type FC } from "react"; +import { Button, Text } from "@vector-im/compound-web"; +import { + MicOnSolidIcon, + VideoCallSolidIcon, +} from "@vector-im/compound-design-tokens/assets/web/icons"; +import classNames from "classnames"; + +import { Modal } from "./Modal"; +import styles from "./RTCConnectionStats.module.css"; +import mediaViewStyles from "../src/tile/MediaView.module.css"; +interface Props { + audio?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats; + video?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats; +} + +// This is only used in developer mode for debugging purposes, so we don't need full localization +export const RTCConnectionStats: FC = ({ audio, video, ...rest }) => { + const [showModal, setShowModal] = useState(false); + const [modalContents, setModalContents] = useState< + "video" | "audio" | "none" + >("none"); + + const showFullModal = (contents: "video" | "audio"): void => { + setShowModal(true); + setModalContents(contents); + }; + + const onDismissModal = (): void => { + setShowModal(false); + setModalContents("none"); + }; + return ( +
+ +
+ 
+            {modalContents !== "none" &&
+              JSON.stringify(
+                modalContents === "video" ? video : audio,
+                null,
+                2,
+              )}
+
+
+ + {audio && ( +
+ +
+ )} + {video && ( +
+ +
+ )} +
+ ); +}; diff --git a/src/settings/DeveloperSettingsTab.tsx b/src/settings/DeveloperSettingsTab.tsx index 057b0b0c..96ab262f 100644 --- a/src/settings/DeveloperSettingsTab.tsx +++ b/src/settings/DeveloperSettingsTab.tsx @@ -14,6 +14,7 @@ import { duplicateTiles as duplicateTilesSetting, debugTileLayout as debugTileLayoutSetting, showNonMemberTiles as showNonMemberTilesSetting, + showConnectionStats as showConnectionStatsSetting, } from "./settings"; import type { MatrixClient } from "matrix-js-sdk/src/client"; @@ -31,6 +32,10 @@ export const DeveloperSettingsTab: FC = ({ client }) => { showNonMemberTilesSetting, ); + const [showConnectionStats, setShowConnectionStats] = useSetting( + showConnectionStatsSetting, + ); + return ( <>

@@ -103,6 +108,20 @@ export const DeveloperSettingsTab: FC = ({ client }) => { )} /> + + ): void => { + setShowConnectionStats(event.target.checked); + }, + [setShowConnectionStats], + )} + /> + ); }; diff --git a/src/settings/settings.ts b/src/settings/settings.ts index ebb5dffc..24657ac4 100644 --- a/src/settings/settings.ts +++ b/src/settings/settings.ts @@ -78,6 +78,11 @@ export const showNonMemberTiles = new Setting( ); export const debugTileLayout = new Setting("debug-tile-layout", false); +export const showConnectionStats = new Setting( + "show-connection-stats", + false, +); + export const audioInput = new Setting( "audio-input", undefined, diff --git a/src/state/MediaViewModel.ts b/src/state/MediaViewModel.ts index b57b6f15..19a71786 100644 --- a/src/state/MediaViewModel.ts +++ b/src/state/MediaViewModel.ts @@ -47,7 +47,7 @@ import { useEffect } from "react"; import { ViewModel } from "./ViewModel"; import { useReactiveState } from "../useReactiveState"; -import { alwaysShowSelf } from "../settings/settings"; +import { alwaysShowSelf, showConnectionStats } from "../settings/settings"; import { accumulate } from "../utils/observable"; import { type EncryptionSystem } from "../e2ee/sharedKeyManagement"; import { E2eeType } from "../e2ee/e2eeType"; @@ -97,6 +97,60 @@ export function observeTrackReference$( ); } +export function observeRtpStreamStats$( + participant: Participant, + source: Track.Source, + type: "inbound-rtp" | "outbound-rtp", +): Observable< + RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats | undefined +> { + return combineLatest([ + observeTrackReference$(of(participant), source), + interval(1000).pipe(startWith(0)), + ]).pipe( + switchMap(async ([trackReference]) => { + const track = trackReference?.publication?.track; + if ( + !track || + !(track instanceof RemoteTrack || track instanceof LocalTrack) + ) { + return undefined; + } + const report = await track.getRTCStatsReport(); + if (!report) { + return undefined; + } + + for (const v of report.values()) { + if (v.type === type) { + return v; + } + } + + return undefined; + }), + startWith(undefined), + ); +} + +export function observeInboundRtpStreamStats$( + participant: Participant, + source: Track.Source, +): Observable { + return observeRtpStreamStats$(participant, source, "inbound-rtp").pipe( + map((x) => x as RTCInboundRtpStreamStats | undefined), + ); +} + +export function observeOutboundRtpStreamStats$( + participant: Participant, + source: Track.Source, +): Observable { + return observeRtpStreamStats$(participant, source, "outbound-rtp").pipe( + map((x) => x as RTCOutboundRtpStreamStats | undefined), + ); +} + function observeRemoteTrackReceivingOkay$( participant: Participant, source: Track.Source, @@ -111,33 +165,15 @@ function observeRemoteTrackReceivingOkay$( framesReceived: undefined, }; - return combineLatest([ - observeTrackReference$(of(participant), source), - interval(1000).pipe(startWith(0)), - ]).pipe( - switchMap(async ([trackReference]) => { - const track = trackReference?.publication?.track; - if (!track || !(track instanceof RemoteTrack)) { - return undefined; - } - const report = await track.getRTCStatsReport(); - if (!report) { - return undefined; - } - - for (const v of report.values()) { - if (v.type === "inbound-rtp") { - const { framesDecoded, framesDropped, framesReceived } = - v as RTCInboundRtpStreamStats; - return { - framesDecoded, - framesDropped, - framesReceived, - }; - } - } - - return undefined; + return observeInboundRtpStreamStats$(participant, source).pipe( + map((stats) => { + if (!stats) return undefined; + const { framesDecoded, framesDropped, framesReceived } = stats; + return { + framesDecoded, + framesDropped, + framesReceived, + }; }), filter((newStats) => !!newStats), map((newStats): boolean | undefined => { @@ -404,6 +440,13 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel { public get local(): boolean { return this instanceof LocalUserMediaViewModel; } + + public abstract get audioStreamStats$(): Observable< + RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats | undefined + >; + public abstract get videoStreamStats$(): Observable< + RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats | undefined + >; } /** @@ -453,6 +496,26 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel { reaction$, ); } + + public audioStreamStats$ = combineLatest([ + this.participant$, + showConnectionStats.value$, + ]).pipe( + switchMap(([p, showConnectionStats]) => { + if (!p || !showConnectionStats) return of(undefined); + return observeOutboundRtpStreamStats$(p, Track.Source.Microphone); + }), + ); + + public videoStreamStats$ = combineLatest([ + this.participant$, + showConnectionStats.value$, + ]).pipe( + switchMap(([p, showConnectionStats]) => { + if (!p || !showConnectionStats) return of(undefined); + return observeOutboundRtpStreamStats$(p, Track.Source.Camera); + }), + ); } /** @@ -542,6 +605,26 @@ export class RemoteUserMediaViewModel extends BaseUserMediaViewModel { public commitLocalVolume(): void { this.localVolumeCommit$.next(); } + + public audioStreamStats$ = combineLatest([ + this.participant$, + showConnectionStats.value$, + ]).pipe( + switchMap(([p, showConnectionStats]) => { + if (!p || !showConnectionStats) return of(undefined); + return observeInboundRtpStreamStats$(p, Track.Source.Microphone); + }), + ); + + public videoStreamStats$ = combineLatest([ + this.participant$, + showConnectionStats.value$, + ]).pipe( + switchMap(([p, showConnectionStats]) => { + if (!p || !showConnectionStats) return of(undefined); + return observeInboundRtpStreamStats$(p, Track.Source.Camera); + }), + ); } /** diff --git a/src/tile/GridTile.tsx b/src/tile/GridTile.tsx index 9eb775d0..0d33ccd6 100644 --- a/src/tile/GridTile.tsx +++ b/src/tile/GridTile.tsx @@ -86,6 +86,12 @@ const UserMediaTile = forwardRef( const video = useObservableEagerState(vm.video$); const unencryptedWarning = useObservableEagerState(vm.unencryptedWarning$); const encryptionStatus = useObservableEagerState(vm.encryptionStatus$); + const audioStreamStats = useObservableEagerState< + RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats | undefined + >(vm.audioStreamStats$); + const videoStreamStats = useObservableEagerState< + RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats | undefined + >(vm.videoStreamStats$); const audioEnabled = useObservableEagerState(vm.audioEnabled$); const videoEnabled = useObservableEagerState(vm.videoEnabled$); const speaking = useObservableEagerState(vm.speaking$); @@ -174,6 +180,8 @@ const UserMediaTile = forwardRef( currentReaction={reaction ?? undefined} raisedHandOnClick={raisedHandOnClick} localParticipant={vm.local} + audioStreamStats={audioStreamStats} + videoStreamStats={videoStreamStats} {...props} /> ); diff --git a/src/tile/MediaView.tsx b/src/tile/MediaView.tsx index 0d5341a8..abc3904b 100644 --- a/src/tile/MediaView.tsx +++ b/src/tile/MediaView.tsx @@ -22,6 +22,7 @@ import { RaisedHandIndicator } from "../reactions/RaisedHandIndicator"; import { showHandRaisedTimer, useSetting } from "../settings/settings"; import { type ReactionOption } from "../reactions"; import { ReactionIndicator } from "../reactions/ReactionIndicator"; +import { RTCConnectionStats } from "../RTCConnectionStats"; interface Props extends ComponentProps { className?: string; @@ -42,6 +43,8 @@ interface Props extends ComponentProps { currentReaction?: ReactionOption; raisedHandOnClick?: () => void; localParticipant: boolean; + audioStreamStats?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats; + videoStreamStats?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats; } export const MediaView = forwardRef( @@ -65,6 +68,8 @@ export const MediaView = forwardRef( currentReaction, raisedHandOnClick, localParticipant, + audioStreamStats, + videoStreamStats, ...props }, ref, @@ -125,6 +130,12 @@ export const MediaView = forwardRef( {t("video_tile.waiting_for_media")} )} + {(audioStreamStats || videoStreamStats) && ( + + )} {/* TODO: Bring this back once encryption status is less broken */} {/*encryptionStatus !== EncryptionStatus.Okay && (