From 40526cb7c154c9fe7fd1b1facaf107bfa6f46161 Mon Sep 17 00:00:00 2001 From: Hugh Nimmo-Smith Date: Wed, 13 Nov 2024 10:21:38 +0000 Subject: [PATCH] Developer setting to show non-member tiles This is based on top of https://github.com/element-hq/element-call/pull/2701 --- locales/en-GB/app.json | 3 +- src/settings/SettingsModal.tsx | 19 +++++++++ src/settings/settings.ts | 8 +++- src/state/CallViewModel.test.ts | 44 ++++++++++++++++++++ src/state/CallViewModel.ts | 74 +++++++++++++++++++++++++++++++-- 5 files changed, 142 insertions(+), 6 deletions(-) diff --git a/locales/en-GB/app.json b/locales/en-GB/app.json index bc37dd7b..acc92741 100644 --- a/locales/en-GB/app.json +++ b/locales/en-GB/app.json @@ -73,7 +73,8 @@ "device_id": "Device ID: {{id}}", "duplicate_tiles_label": "Number of additional tile copies per participant", "hostname": "Hostname: {{hostname}}", - "matrix_id": "Matrix ID: {{id}}" + "matrix_id": "Matrix ID: {{id}}", + "show_non_member_tiles": "Show tiles for non-member media" }, "disconnected_banner": "Connectivity to the server has been lost.", "full_screen_view_description": "<0>Submitting debug logs will help us track down the problem.", diff --git a/src/settings/SettingsModal.tsx b/src/settings/SettingsModal.tsx index 4ffcecf5..54c3fc60 100644 --- a/src/settings/SettingsModal.tsx +++ b/src/settings/SettingsModal.tsx @@ -26,6 +26,7 @@ import { useSetting, developerSettingsTab as developerSettingsTabSetting, duplicateTiles as duplicateTilesSetting, + showNonMemberTiles as showNonMemberTilesSetting, useOptInAnalytics, soundEffectVolumeSetting, } from "./settings"; @@ -70,6 +71,10 @@ export const SettingsModal: FC = ({ ); const [duplicateTiles, setDuplicateTiles] = useSetting(duplicateTilesSetting); + const [showNonMemberTiles, setShowNonMemberTiles] = useSetting( + showNonMemberTilesSetting, + ); + const optInDescription = ( @@ -240,6 +245,20 @@ export const SettingsModal: FC = ({ )} /> + + ): void => { + setShowNonMemberTiles(event.target.checked); + }, + [setShowNonMemberTiles], + )} + /> + ), }; diff --git a/src/settings/settings.ts b/src/settings/settings.ts index b2da1674..8ced3b4f 100644 --- a/src/settings/settings.ts +++ b/src/settings/settings.ts @@ -40,9 +40,11 @@ export class Setting { private readonly _value: BehaviorSubject; public readonly value: Observable; - public readonly setValue = (value: T): void => { + public readonly setValue = (value: T, persist = true): void => { this._value.next(value); - localStorage.setItem(this.key, JSON.stringify(value)); + if (persist) { + localStorage.setItem(this.key, JSON.stringify(value)); + } }; } @@ -75,6 +77,8 @@ export const developerSettingsTab = new Setting( export const duplicateTiles = new Setting("duplicate-tiles", 0); +export const showNonMemberTiles = new Setting("show-non-member-tiles", false); + export const audioInput = new Setting( "audio-input", undefined, diff --git a/src/state/CallViewModel.test.ts b/src/state/CallViewModel.test.ts index 5dbfb1ca..b10fc51d 100644 --- a/src/state/CallViewModel.test.ts +++ b/src/state/CallViewModel.test.ts @@ -43,6 +43,7 @@ import { ECConnectionState, } from "../livekit/useECConnectionState"; import { E2eeType } from "../e2ee/e2eeType"; +import { showNonMemberTiles } from "../settings/settings"; vi.mock("@livekit/components-core"); @@ -636,6 +637,49 @@ test("participants must have a MatrixRTCSession to be visible", () => { }); }); +test("shows participants without MatrixRTCSession when enabled in settings", () => { + // enable the setting: + showNonMemberTiles.setValue(true); + withTestScheduler(({ hot, expectObservable }) => { + const scenarioInputMarbles = " abc"; + const expectedLayoutMarbles = "abc"; + + withCallViewModel( + hot(scenarioInputMarbles, { + a: [], + b: [aliceParticipant], + c: [aliceParticipant, bobParticipant], + }), + of([]), + of(ConnectionState.Connected), + new Map(), + (vm) => { + vm.setGridMode("grid"); + expectObservable(summarizeLayout(vm.layout)).toBe( + expectedLayoutMarbles, + { + a: { + type: "grid", + spotlight: undefined, + grid: ["local:0"], + }, + b: { + type: "one-on-one", + local: "local:0", + remote: `${aliceId}:0`, + }, + c: { + type: "grid", + spotlight: undefined, + grid: ["local:0", `${aliceId}:0`, `${bobId}:0`], + }, + }, + ); + }, + ); + }); +}); + it("should show at least one tile per MatrixRTCSession", () => { withTestScheduler(({ hot, expectObservable }) => { // iterate through some combinations of MatrixRTC memberships diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 210689b7..393a5893 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -66,7 +66,7 @@ import { } from "./MediaViewModel"; import { accumulate, finalizeValue } from "../utils/observable"; import { ObservableScope } from "./ObservableScope"; -import { duplicateTiles } from "../settings/settings"; +import { duplicateTiles, showNonMemberTiles } from "../settings/settings"; import { isFirefox } from "../Platform"; import { setPipEnabled } from "../controls"; import { GridTileViewModel, SpotlightTileViewModel } from "./TileViewModel"; @@ -427,6 +427,8 @@ export class CallViewModel extends ViewModel { }, ); + private readonly nonMemberItemCount = new BehaviorSubject(0); + /** * List of MediaItems that we want to display */ @@ -441,6 +443,7 @@ export class CallViewModel extends ViewModel { this.matrixRTCSession, MatrixRTCSessionEvent.MembershipsChanged, ).pipe(startWith(null)), + showNonMemberTiles.value, ]).pipe( scan( ( @@ -450,6 +453,7 @@ export class CallViewModel extends ViewModel { { participant: localParticipant }, duplicateTiles, _membershipsChanged, + showNonMemberTiles, ], ) => { const newItems = new Map( @@ -487,9 +491,17 @@ export class CallViewModel extends ViewModel { } for (let i = 0; i < 1 + duplicateTiles; i++) { const indexedMediaId = `${livekitParticipantId}:${i}`; - const prevMedia = prevItems.get(indexedMediaId); + let prevMedia = prevItems.get(indexedMediaId); if (prevMedia && prevMedia instanceof UserMedia) { prevMedia.updateParticipant(participant); + if (prevMedia.vm.member === undefined) { + // We have a previous media created because of the `debugShowNonMember` flag. + // In this case we actually replace the media item. + // This "hack" never occurs if we do not use the `debugShowNonMember` debugging + // option and if we always find a room member for each rtc member (which also + // only fails if we have a fundamental problem) + prevMedia = undefined; + } } yield [ indexedMediaId, @@ -525,7 +537,63 @@ export class CallViewModel extends ViewModel { }.bind(this)(), ); - return newItems; + // Generate non member items (items without a corresponding MatrixRTC member) + // Those items should not be rendered, they are participants in livekit that do not have a corresponding + // matrix rtc members. This cannot be any good: + // - A malicious user impersonates someone + // - Someone injects abusive content + // - The user cannot have encryption keys so it makes no sense to participate + // We can only trust users that have a matrixRTC member event. + // + // This is still available as a debug option. This can be useful + // - If one wants to test scalability using the livekit cli. + // - If an experimental project does not yet do the matrixRTC bits. + // - If someone wants to debug if the LK connection works but matrixRTC room state failed to arrive. + const newNonMemberItems = showNonMemberTiles + ? new Map( + function* (this: CallViewModel): Iterable<[string, MediaItem]> { + for (const participant of remoteParticipants) { + for (let i = 0; i < 1 + duplicateTiles; i++) { + const maybeNonMemberParticipantId = + participant.identity + ":" + i; + if (!newItems.has(maybeNonMemberParticipantId)) { + const nonMemberId = maybeNonMemberParticipantId; + yield [ + nonMemberId, + // We create UserMedia with or without a participant. + // This will be the initial value of a BehaviourSubject. + // Once a participant appears we will update the BehaviourSubject. (see above) + prevItems.get(nonMemberId) ?? + new UserMedia( + nonMemberId, + undefined, + participant, + this.encryptionSystem, + this.livekitRoom, + ), + ]; + } + } + } + }.bind(this)(), + ) + : new Map(); + if (newNonMemberItems.size > 0) { + logger.debug("Added NonMember items: ", newNonMemberItems); + } + + const newNonMemberItemCount = + newNonMemberItems.size / (1 + duplicateTiles); + if (this.nonMemberItemCount.value !== newNonMemberItemCount) + this.nonMemberItemCount.next(newNonMemberItemCount); + + const combinedNew = new Map([ + ...newNonMemberItems.entries(), + ...newItems.entries(), + ]); + + for (const [id, t] of prevItems) if (!combinedNew.has(id)) t.destroy(); + return combinedNew; }, new Map(), ),