Add developer mode option to show RTC connection statistics

locales/en/app.json

@@ -73,6 +73,7 @@
     "duplicate_tiles_label": "Number of additional tile copies per participant",
     "hostname": "Hostname: {{hostname}}",
     "matrix_id": "Matrix ID: {{id}}",
+    "show_connection_stats": "Show connection statistics",
     "show_non_member_tiles": "Show tiles for non-member media"
   },
   "disconnected_banner": "Connectivity to the server has been lost.",

src/RTCConnectionStats.tsx

@@ -0,0 +1,73 @@
+/*
+Copyright 2023, 2024 New Vector Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only
+Please see LICENSE in the repository root for full details.
+*/
+
+import { type FC } from "react";
+import { Text, Tooltip, TooltipProvider } from "@vector-im/compound-web";
+import {
+  MicOnSolidIcon,
+  VideoCallSolidIcon,
+} from "@vector-im/compound-design-tokens/assets/web/icons";
+
+interface Props {
+  audio?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats;
+  video?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats;
+}
+
+export const RTCConnectionStats: FC<Props> = ({ audio, video, ...rest }) => {
+  return (
+    <div>
+      <TooltipProvider>
+        {audio && (
+          <div>
+            <Tooltip label={JSON.stringify(audio, null, 2)}>
+              <MicOnSolidIcon />
+            </Tooltip>
+            {"jitter" in audio && typeof audio.jitter === "number" && (
+              <Text as="span" size="xs">
+                &nbsp;{(audio.jitter * 1000).toFixed(0)}ms
+              </Text>
+            )}
+          </div>
+        )}
+        {video && (
+          <div>
+            {video && (
+              <Tooltip label={JSON.stringify(video, null, 2)}>
+                <VideoCallSolidIcon />
+              </Tooltip>
+            )}
+            {video?.framesPerSecond && (
+              <Text as="span" size="xs">
+                &nbsp;{video.framesPerSecond.toFixed(0)}fps
+              </Text>
+            )}
+            {"jitter" in video && typeof video.jitter === "number" && (
+              <Text as="span" size="xs">
+                &nbsp;{(video.jitter * 1000).toFixed(0)}ms
+              </Text>
+            )}
+            {"frameHeight" in video &&
+              typeof video.frameHeight === "number" &&
+              "frameWidth" in video &&
+              typeof video.frameWidth === "number" && (
+                <Text as="span" size="xs">
+                  &nbsp;{video.frameWidth}x{video.frameHeight}
+                </Text>
+              )}
+            {"qualityLimitationReason" in video &&
+              typeof video.qualityLimitationReason === "string" &&
+              video.qualityLimitationReason !== "none" && (
+                <Text as="span" size="xs">
+                  &nbsp;{video.qualityLimitationReason}
+                </Text>
+              )}
+          </div>
+        )}
+      </TooltipProvider>
+    </div>
+  );
+};

src/settings/DeveloperSettingsTab.tsx

@@ -14,6 +14,7 @@ import {
   duplicateTiles as duplicateTilesSetting,
   debugTileLayout as debugTileLayoutSetting,
   showNonMemberTiles as showNonMemberTilesSetting,
+  showConnectionStats as showConnectionStatsSetting,
 } from "./settings";
 import type { MatrixClient } from "matrix-js-sdk/src/client";
 
@@ -31,6 +32,10 @@ export const DeveloperSettingsTab: FC<Props> = ({ client }) => {
     showNonMemberTilesSetting,
   );
 
+  const [showConnectionStats, setShowConnectionStats] = useSetting(
+    showConnectionStatsSetting,
+  );
+
   return (
     <>
       <p>
@@ -103,6 +108,20 @@ export const DeveloperSettingsTab: FC<Props> = ({ client }) => {
           )}
         />
       </FieldRow>
+      <FieldRow>
+        <InputField
+          id="showConnectionStats"
+          type="checkbox"
+          label={t("developer_mode.show_connection_stats")}
+          checked={!!showConnectionStats}
+          onChange={useCallback(
+            (event: ChangeEvent<HTMLInputElement>): void => {
+              setShowConnectionStats(event.target.checked);
+            },
+            [setShowConnectionStats],
+          )}
+        />
+      </FieldRow>
     </>
   );
 };

src/settings/settings.ts

@@ -78,6 +78,11 @@ export const showNonMemberTiles = new Setting<boolean>(
 );
 export const debugTileLayout = new Setting("debug-tile-layout", false);
 
+export const showConnectionStats = new Setting<boolean>(
+  "show-connection-stats",
+  false,
+);
+
 export const audioInput = new Setting<string | undefined>(
   "audio-input",
   undefined,

src/state/MediaViewModel.ts

@@ -47,7 +47,7 @@ import { useEffect } from "react";
 
 import { ViewModel } from "./ViewModel";
 import { useReactiveState } from "../useReactiveState";
-import { alwaysShowSelf } from "../settings/settings";
+import { alwaysShowSelf, showConnectionStats } from "../settings/settings";
 import { accumulate } from "../utils/observable";
 import { type EncryptionSystem } from "../e2ee/sharedKeyManagement";
 import { E2eeType } from "../e2ee/e2eeType";
@@ -96,6 +96,60 @@ export function observeTrackReference(
   );
 }
 
+export function observeRtpStreamStats(
+  participant: Participant,
+  source: Track.Source,
+  type: "inbound-rtp" | "outbound-rtp",
+): Observable<
+  RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats | undefined
+> {
+  return combineLatest([
+    observeTrackReference(of(participant), source),
+    interval(1000).pipe(startWith(0)),
+  ]).pipe(
+    switchMap(async ([trackReference]) => {
+      const track = trackReference?.publication?.track;
+      if (
+        !track ||
+        !(track instanceof RemoteTrack || track instanceof LocalTrack)
+      ) {
+        return undefined;
+      }
+      const report = await track.getRTCStatsReport();
+      if (!report) {
+        return undefined;
+      }
+
+      for (const v of report.values()) {
+        if (v.type === type) {
+          return v;
+        }
+      }
+
+      return undefined;
+    }),
+    startWith(undefined),
+  );
+}
+
+export function observeInboundRtpStreamStats(
+  participant: Participant,
+  source: Track.Source,
+): Observable<RTCInboundRtpStreamStats | undefined> {
+  return observeRtpStreamStats(participant, source, "inbound-rtp").pipe(
+    map((x) => x as RTCInboundRtpStreamStats | undefined),
+  );
+}
+
+export function observeOutboundRtpStreamStats(
+  participant: Participant,
+  source: Track.Source,
+): Observable<RTCOutboundRtpStreamStats | undefined> {
+  return observeRtpStreamStats(participant, source, "outbound-rtp").pipe(
+    map((x) => x as RTCOutboundRtpStreamStats | undefined),
+  );
+}
+
 function observeRemoteTrackReceivingOkay(
   participant: Participant,
   source: Track.Source,
@@ -110,33 +164,15 @@ function observeRemoteTrackReceivingOkay(
     framesReceived: undefined,
   };
 
-  return combineLatest([
+  return observeInboundRtpStreamStats(participant, source).pipe(
-    observeTrackReference(of(participant), source),
+    map((stats) => {
-    interval(1000).pipe(startWith(0)),
+      if (!stats) return undefined;
-  ]).pipe(
+      const { framesDecoded, framesDropped, framesReceived } = stats;
-    switchMap(async ([trackReference]) => {
+      return {
-      const track = trackReference?.publication?.track;
+        framesDecoded,
-      if (!track || !(track instanceof RemoteTrack)) {
+        framesDropped,
-        return undefined;
+        framesReceived,
-      }
+      };
-      const report = await track.getRTCStatsReport();
-      if (!report) {
-        return undefined;
-      }
-
-      for (const v of report.values()) {
-        if (v.type === "inbound-rtp") {
-          const { framesDecoded, framesDropped, framesReceived } =
-            v as RTCInboundRtpStreamStats;
-          return {
-            framesDecoded,
-            framesDropped,
-            framesReceived,
-          };
-        }
-      }
-
-      return undefined;
     }),
     filter((newStats) => !!newStats),
     map((newStats): boolean | undefined => {
@@ -401,6 +437,13 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel {
   public get local(): boolean {
     return this instanceof LocalUserMediaViewModel;
   }
+
+  public abstract get audioStreamStats(): Observable<
+    RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats | undefined
+  >;
+  public abstract get videoStreamStats(): Observable<
+    RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats | undefined
+  >;
 }
 
 /**
@@ -440,6 +483,26 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel {
   ) {
     super(id, member, participant, encryptionSystem, livekitRoom);
   }
+
+  public audioStreamStats = combineLatest([
+    this.participant,
+    showConnectionStats.value,
+  ]).pipe(
+    switchMap(([p, enabled]) => {
+      if (!p || !enabled) return of(undefined);
+      return observeOutboundRtpStreamStats(p, Track.Source.Microphone);
+    }),
+  );
+
+  public videoStreamStats = combineLatest([
+    this.participant,
+    showConnectionStats.value,
+  ]).pipe(
+    switchMap(([p, enabled]) => {
+      if (!p || !enabled) return of(undefined);
+      return observeOutboundRtpStreamStats(p, Track.Source.Camera);
+    }),
+  );
 }
 
 /**
@@ -519,6 +582,26 @@ export class RemoteUserMediaViewModel extends BaseUserMediaViewModel {
   public commitLocalVolume(): void {
     this.localVolumeCommit.next();
   }
+
+  public audioStreamStats = combineLatest([
+    this.participant,
+    showConnectionStats.value,
+  ]).pipe(
+    switchMap(([p, enabled]) => {
+      if (!p || !enabled) return of(undefined);
+      return observeInboundRtpStreamStats(p, Track.Source.Microphone);
+    }),
+  );
+
+  public videoStreamStats = combineLatest([
+    this.participant,
+    showConnectionStats.value,
+  ]).pipe(
+    switchMap(([p, enabled]) => {
+      if (!p || !enabled) return of(undefined);
+      return observeInboundRtpStreamStats(p, Track.Source.Camera);
+    }),
+  );
 }
 
 /**

src/tile/GridTile.tsx

@@ -86,6 +86,12 @@ const UserMediaTile = forwardRef<HTMLDivElement, UserMediaTileProps>(
     const video = useObservableEagerState(vm.video);
     const unencryptedWarning = useObservableEagerState(vm.unencryptedWarning);
     const encryptionStatus = useObservableEagerState(vm.encryptionStatus);
+    const audioStreamStats = useObservableEagerState<
+      RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats | undefined
+    >(vm.audioStreamStats);
+    const videoStreamStats = useObservableEagerState<
+      RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats | undefined
+    >(vm.videoStreamStats);
     const audioEnabled = useObservableEagerState(vm.audioEnabled);
     const videoEnabled = useObservableEagerState(vm.videoEnabled);
     const speaking = useObservableEagerState(vm.speaking);
@@ -176,6 +182,8 @@ const UserMediaTile = forwardRef<HTMLDivElement, UserMediaTileProps>(
         currentReaction={currentReaction}
         raisedHandOnClick={raisedHandOnClick}
         localParticipant={vm.local}
+        audioStreamStats={audioStreamStats}
+        videoStreamStats={videoStreamStats}
         {...props}
       />
     );

src/tile/MediaView.tsx

@@ -22,6 +22,7 @@ import { RaisedHandIndicator } from "../reactions/RaisedHandIndicator";
 import { showHandRaisedTimer, useSetting } from "../settings/settings";
 import { type ReactionOption } from "../reactions";
 import { ReactionIndicator } from "../reactions/ReactionIndicator";
+import { RTCConnectionStats } from "../RTCConnectionStats";
 
 interface Props extends ComponentProps<typeof animated.div> {
   className?: string;
@@ -42,6 +43,8 @@ interface Props extends ComponentProps<typeof animated.div> {
   currentReaction?: ReactionOption;
   raisedHandOnClick?: () => void;
   localParticipant: boolean;
+  audioStreamStats?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats;
+  videoStreamStats?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats;
 }
 
 export const MediaView = forwardRef<HTMLDivElement, Props>(
@@ -65,6 +68,8 @@ export const MediaView = forwardRef<HTMLDivElement, Props>(
       currentReaction,
       raisedHandOnClick,
       localParticipant,
+      audioStreamStats,
+      videoStreamStats,
       ...props
     },
     ref,
@@ -125,6 +130,12 @@ export const MediaView = forwardRef<HTMLDivElement, Props>(
               {t("video_tile.waiting_for_media")}
             </div>
           )}
+          {(audioStreamStats || videoStreamStats) && (
+            <RTCConnectionStats
+              audio={audioStreamStats}
+              video={videoStreamStats}
+            />
+          )}
           {/* TODO: Bring this back once encryption status is less broken */}
           {/*encryptionStatus !== EncryptionStatus.Okay && (
             <div className={styles.status}>