Developer setting to show non-member tiles

This is based on top of https://github.com/element-hq/element-call/pull/2701

locales/en-GB/app.json

@@ -73,7 +73,8 @@
     "device_id": "Device ID: {{id}}",
     "duplicate_tiles_label": "Number of additional tile copies per participant",
     "hostname": "Hostname: {{hostname}}",
-    "matrix_id": "Matrix ID: {{id}}"
+    "matrix_id": "Matrix ID: {{id}}",
+    "show_non_member_tiles": "Show tiles for non-member media"
   },
   "disconnected_banner": "Connectivity to the server has been lost.",
   "full_screen_view_description": "<0>Submitting debug logs will help us track down the problem.</0>",

src/settings/SettingsModal.tsx

@@ -26,6 +26,7 @@ import {
   useSetting,
   developerSettingsTab as developerSettingsTabSetting,
   duplicateTiles as duplicateTilesSetting,
+  showNonMemberTiles as showNonMemberTilesSetting,
   useOptInAnalytics,
   soundEffectVolumeSetting,
 } from "./settings";
@@ -70,6 +71,10 @@ export const SettingsModal: FC<Props> = ({
   );
   const [duplicateTiles, setDuplicateTiles] = useSetting(duplicateTilesSetting);
 
+  const [showNonMemberTiles, setShowNonMemberTiles] = useSetting(
+    showNonMemberTilesSetting,
+  );
+
   const optInDescription = (
     <Text size="sm">
       <Trans i18nKey="settings.opt_in_description">
@@ -240,6 +245,20 @@ export const SettingsModal: FC<Props> = ({
             )}
           />
         </FieldRow>
+        <FieldRow>
+          <InputField
+            id="showNonMemberTiles"
+            type="checkbox"
+            label={t("developer_mode.show_non_member_tiles")}
+            checked={!!showNonMemberTiles}
+            onChange={useCallback(
+              (event: ChangeEvent<HTMLInputElement>): void => {
+                setShowNonMemberTiles(event.target.checked);
+              },
+              [setShowNonMemberTiles],
+            )}
+          />
+        </FieldRow>
       </>
     ),
   };

src/settings/settings.ts

@@ -40,9 +40,11 @@ export class Setting<T> {
   private readonly _value: BehaviorSubject<T>;
   public readonly value: Observable<T>;
 
-  public readonly setValue = (value: T): void => {
+  public readonly setValue = (value: T, persist = true): void => {
     this._value.next(value);
-    localStorage.setItem(this.key, JSON.stringify(value));
+    if (persist) {
+      localStorage.setItem(this.key, JSON.stringify(value));
+    }
   };
 }
 
@@ -75,6 +77,8 @@ export const developerSettingsTab = new Setting(
 
 export const duplicateTiles = new Setting("duplicate-tiles", 0);
 
+export const showNonMemberTiles = new Setting<boolean>("show-non-member-tiles", false);
+
 export const audioInput = new Setting<string | undefined>(
   "audio-input",
   undefined,

src/state/CallViewModel.test.ts

@@ -43,6 +43,7 @@ import {
   ECConnectionState,
 } from "../livekit/useECConnectionState";
 import { E2eeType } from "../e2ee/e2eeType";
+import { showNonMemberTiles } from "../settings/settings";
 
 vi.mock("@livekit/components-core");
 
@@ -636,6 +637,49 @@ test("participants must have a MatrixRTCSession to be visible", () => {
   });
 });
 
+test("shows participants without MatrixRTCSession when enabled in settings", () => {
+  // enable the setting:
+  showNonMemberTiles.setValue(true);
+  withTestScheduler(({ hot, expectObservable }) => {
+    const scenarioInputMarbles = " abc";
+    const expectedLayoutMarbles = "abc";
+
+    withCallViewModel(
+      hot(scenarioInputMarbles, {
+        a: [],
+        b: [aliceParticipant],
+        c: [aliceParticipant, bobParticipant],
+      }),
+      of([]),
+      of(ConnectionState.Connected),
+      new Map(),
+      (vm) => {
+        vm.setGridMode("grid");
+        expectObservable(summarizeLayout(vm.layout)).toBe(
+          expectedLayoutMarbles,
+          {
+            a: {
+              type: "grid",
+              spotlight: undefined,
+              grid: ["local:0"],
+            },
+            b: {
+              type: "one-on-one",
+              local: "local:0",
+              remote: `${aliceId}:0`,
+            },
+            c: {
+              type: "grid",
+              spotlight: undefined,
+              grid: ["local:0", `${aliceId}:0`, `${bobId}:0`],
+            },
+          },
+        );
+      },
+    );
+  });
+});
+
 it("should show at least one tile per MatrixRTCSession", () => {
   withTestScheduler(({ hot, expectObservable }) => {
     // iterate through some combinations of MatrixRTC memberships

src/state/CallViewModel.ts

@@ -66,7 +66,7 @@ import {
 } from "./MediaViewModel";
 import { accumulate, finalizeValue } from "../utils/observable";
 import { ObservableScope } from "./ObservableScope";
-import { duplicateTiles } from "../settings/settings";
+import { duplicateTiles, showNonMemberTiles } from "../settings/settings";
 import { isFirefox } from "../Platform";
 import { setPipEnabled } from "../controls";
 import { GridTileViewModel, SpotlightTileViewModel } from "./TileViewModel";
@@ -427,6 +427,8 @@ export class CallViewModel extends ViewModel {
       },
     );
 
+  private readonly nonMemberItemCount = new BehaviorSubject<number>(0);
+
   /**
    * List of MediaItems that we want to display
    */
@@ -441,6 +443,7 @@ export class CallViewModel extends ViewModel {
       this.matrixRTCSession,
       MatrixRTCSessionEvent.MembershipsChanged,
     ).pipe(startWith(null)),
+    showNonMemberTiles.value,
   ]).pipe(
     scan(
       (
@@ -450,6 +453,7 @@ export class CallViewModel extends ViewModel {
           { participant: localParticipant },
           duplicateTiles,
           _membershipsChanged,
+          showNonMemberTiles,
         ],
       ) => {
         const newItems = new Map(
@@ -487,9 +491,17 @@ export class CallViewModel extends ViewModel {
               }
               for (let i = 0; i < 1 + duplicateTiles; i++) {
                 const indexedMediaId = `${livekitParticipantId}:${i}`;
-                const prevMedia = prevItems.get(indexedMediaId);
+                let prevMedia = prevItems.get(indexedMediaId);
                 if (prevMedia && prevMedia instanceof UserMedia) {
                   prevMedia.updateParticipant(participant);
+                  if (prevMedia.vm.member === undefined) {
+                    // We have a previous media created because of the `debugShowNonMember` flag.
+                    // In this case we actually replace the media item.
+                    // This "hack" never occurs if we do not use the `debugShowNonMember` debugging
+                    // option and if we always find a room member for each rtc member (which also
+                    // only fails if we have a fundamental problem)
+                    prevMedia = undefined;
+                  }
                 }
                 yield [
                   indexedMediaId,
@@ -525,7 +537,63 @@ export class CallViewModel extends ViewModel {
           }.bind(this)(),
         );
 
-        return newItems;
+        // Generate non member items (items without a corresponding MatrixRTC member)
+        // Those items should not be rendered, they are participants in livekit that do not have a corresponding
+        // matrix rtc members. This cannot be any good:
+        //  - A malicious user impersonates someone
+        //  - Someone injects abusive content
+        //  - The user cannot have encryption keys so it makes no sense to participate
+        // We can only trust users that have a matrixRTC member event.
+        //
+        // This is still available as a debug option. This can be useful
+        //  - If one wants to test scalability using the livekit cli.
+        //  - If an experimental project does not yet do the matrixRTC bits.
+        //  - If someone wants to debug if the LK connection works but matrixRTC room state failed to arrive.
+        const newNonMemberItems = showNonMemberTiles
+          ? new Map(
+              function* (this: CallViewModel): Iterable<[string, MediaItem]> {
+                for (const participant of remoteParticipants) {
+                  for (let i = 0; i < 1 + duplicateTiles; i++) {
+                    const maybeNonMemberParticipantId =
+                      participant.identity + ":" + i;
+                    if (!newItems.has(maybeNonMemberParticipantId)) {
+                      const nonMemberId = maybeNonMemberParticipantId;
+                      yield [
+                        nonMemberId,
+                        // We create UserMedia with or without a participant.
+                        // This will be the initial value of a BehaviourSubject.
+                        // Once a participant appears we will update the BehaviourSubject. (see above)
+                        prevItems.get(nonMemberId) ??
+                          new UserMedia(
+                            nonMemberId,
+                            undefined,
+                            participant,
+                            this.encryptionSystem,
+                            this.livekitRoom,
+                          ),
+                      ];
+                    }
+                  }
+                }
+              }.bind(this)(),
+            )
+          : new Map();
+        if (newNonMemberItems.size > 0) {
+          logger.debug("Added NonMember items: ", newNonMemberItems);
+        }
+
+        const newNonMemberItemCount =
+          newNonMemberItems.size / (1 + duplicateTiles);
+        if (this.nonMemberItemCount.value !== newNonMemberItemCount)
+          this.nonMemberItemCount.next(newNonMemberItemCount);
+
+        const combinedNew = new Map([
+          ...newNonMemberItems.entries(),
+          ...newItems.entries(),
+        ]);
+
+        for (const [id, t] of prevItems) if (!combinedNew.has(id)) t.destroy();
+        return combinedNew;
       },
       new Map<string, MediaItem>(),
     ),