diff --git a/locales/en/app.json b/locales/en/app.json index 84038c2b..33f8f921 100644 --- a/locales/en/app.json +++ b/locales/en/app.json @@ -74,7 +74,8 @@ "device_id": "Device ID: {{id}}", "duplicate_tiles_label": "Number of additional tile copies per participant", "hostname": "Hostname: {{hostname}}", - "matrix_id": "Matrix ID: {{id}}" + "matrix_id": "Matrix ID: {{id}}", + "show_non_member_tiles": "Show tiles for non-member media" }, "disconnected_banner": "Connectivity to the server has been lost.", "full_screen_view_description": "<0>Submitting debug logs will help us track down the problem.", diff --git a/src/settings/DeveloperSettingsTab.tsx b/src/settings/DeveloperSettingsTab.tsx index 209bc41e..057b0b0c 100644 --- a/src/settings/DeveloperSettingsTab.tsx +++ b/src/settings/DeveloperSettingsTab.tsx @@ -13,6 +13,7 @@ import { useSetting, duplicateTiles as duplicateTilesSetting, debugTileLayout as debugTileLayoutSetting, + showNonMemberTiles as showNonMemberTilesSetting, } from "./settings"; import type { MatrixClient } from "matrix-js-sdk/src/client"; @@ -26,6 +27,9 @@ export const DeveloperSettingsTab: FC = ({ client }) => { const [debugTileLayout, setDebugTileLayout] = useSetting( debugTileLayoutSetting, ); + const [showNonMemberTiles, setShowNonMemberTiles] = useSetting( + showNonMemberTilesSetting, + ); return ( <> @@ -85,6 +89,20 @@ export const DeveloperSettingsTab: FC = ({ client }) => { } /> + + ): void => { + setShowNonMemberTiles(event.target.checked); + }, + [setShowNonMemberTiles], + )} + /> + ); }; diff --git a/src/settings/settings.ts b/src/settings/settings.ts index e8dda379..a902f9ab 100644 --- a/src/settings/settings.ts +++ b/src/settings/settings.ts @@ -72,6 +72,10 @@ export const developerMode = new Setting("developer-settings-tab", false); export const duplicateTiles = new Setting("duplicate-tiles", 0); +export const showNonMemberTiles = new Setting( + "show-non-member-tiles", + false, +); export const debugTileLayout = new Setting("debug-tile-layout", false); export const audioInput = new Setting( diff --git a/src/state/CallViewModel.test.ts b/src/state/CallViewModel.test.ts index 0e45faa1..d5b84d49 100644 --- a/src/state/CallViewModel.test.ts +++ b/src/state/CallViewModel.test.ts @@ -46,6 +46,7 @@ import { type ECConnectionState, } from "../livekit/useECConnectionState"; import { E2eeType } from "../e2ee/e2eeType"; +import { showNonMemberTiles } from "../settings/settings"; vi.mock("@livekit/components-core"); @@ -686,6 +687,53 @@ test("participants must have a MatrixRTCSession to be visible", () => { }); }); +test("shows participants without MatrixRTCSession when enabled in settings", () => { + try { + // enable the setting: + showNonMemberTiles.setValue(true); + withTestScheduler(({ hot, expectObservable }) => { + const scenarioInputMarbles = " abc"; + const expectedLayoutMarbles = "abc"; + + withCallViewModel( + hot(scenarioInputMarbles, { + a: [], + b: [aliceParticipant], + c: [aliceParticipant, bobParticipant], + }), + of([]), // No one joins the MatrixRTC session + of(ConnectionState.Connected), + new Map(), + (vm) => { + vm.setGridMode("grid"); + expectObservable(summarizeLayout(vm.layout)).toBe( + expectedLayoutMarbles, + { + a: { + type: "grid", + spotlight: undefined, + grid: ["local:0"], + }, + b: { + type: "one-on-one", + local: "local:0", + remote: `${aliceId}:0`, + }, + c: { + type: "grid", + spotlight: undefined, + grid: ["local:0", `${aliceId}:0`, `${bobId}:0`], + }, + }, + ); + }, + ); + }); + } finally { + showNonMemberTiles.setValue(showNonMemberTiles.defaultValue); + } +}); + it("should show at least one tile per MatrixRTCSession", () => { withTestScheduler(({ hot, expectObservable }) => { // iterate through some combinations of MatrixRTC memberships diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 108a44c9..c701519b 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -69,7 +69,7 @@ import { } from "./MediaViewModel"; import { accumulate, finalizeValue } from "../utils/observable"; import { ObservableScope } from "./ObservableScope"; -import { duplicateTiles } from "../settings/settings"; +import { duplicateTiles, showNonMemberTiles } from "../settings/settings"; import { isFirefox } from "../Platform"; import { setPipEnabled } from "../controls"; import { @@ -449,6 +449,7 @@ export class CallViewModel extends ViewModel { this.matrixRTCSession, MatrixRTCSessionEvent.MembershipsChanged, ).pipe(startWith(null)), + showNonMemberTiles.value, ]).pipe( scan( ( @@ -458,6 +459,7 @@ export class CallViewModel extends ViewModel { { participant: localParticipant }, duplicateTiles, _membershipsChanged, + showNonMemberTiles, ], ) => { const newItems = new Map( @@ -495,9 +497,17 @@ export class CallViewModel extends ViewModel { } for (let i = 0; i < 1 + duplicateTiles; i++) { const indexedMediaId = `${livekitParticipantId}:${i}`; - const prevMedia = prevItems.get(indexedMediaId); + let prevMedia = prevItems.get(indexedMediaId); if (prevMedia && prevMedia instanceof UserMedia) { prevMedia.updateParticipant(participant); + if (prevMedia.vm.member === undefined) { + // We have a previous media created because of the `debugShowNonMember` flag. + // In this case we actually replace the media item. + // This "hack" never occurs if we do not use the `debugShowNonMember` debugging + // option and if we always find a room member for each rtc member (which also + // only fails if we have a fundamental problem) + prevMedia = undefined; + } } yield [ indexedMediaId, @@ -533,7 +543,55 @@ export class CallViewModel extends ViewModel { }.bind(this)(), ); - return newItems; + // Generate non member items (items without a corresponding MatrixRTC member) + // Those items should not be rendered, they are participants in LiveKit that do not have a corresponding + // MatrixRTC members. This cannot be any good: + // - A malicious user impersonates someone + // - Someone injects abusive content + // - The user cannot have encryption keys so it makes no sense to participate + // We can only trust users that have a MatrixRTC member event. + // + // This is still available as a debug option. This can be useful + // - If one wants to test scalability using the LiveKit CLI. + // - If an experimental project does not yet do the MatrixRTC bits. + // - If someone wants to debug if the LiveKit connection works but MatrixRTC room state failed to arrive. + const newNonMemberItems = showNonMemberTiles + ? new Map( + function* (this: CallViewModel): Iterable<[string, MediaItem]> { + for (const participant of remoteParticipants) { + for (let i = 0; i < 1 + duplicateTiles; i++) { + const maybeNonMemberParticipantId = + participant.identity + ":" + i; + if (!newItems.has(maybeNonMemberParticipantId)) { + const nonMemberId = maybeNonMemberParticipantId; + yield [ + nonMemberId, + prevItems.get(nonMemberId) ?? + new UserMedia( + nonMemberId, + undefined, + participant, + this.encryptionSystem, + this.livekitRoom, + ), + ]; + } + } + } + }.bind(this)(), + ) + : new Map(); + if (newNonMemberItems.size > 0) { + logger.debug("Added NonMember items: ", newNonMemberItems); + } + + const combinedNew = new Map([ + ...newNonMemberItems.entries(), + ...newItems.entries(), + ]); + + for (const [id, t] of prevItems) if (!combinedNew.has(id)) t.destroy(); + return combinedNew; }, new Map(), ),