From e1e202d7c8d2b93df5d5a5a7dae1bf625761ef67 Mon Sep 17 00:00:00 2001 From: Timo Date: Mon, 4 Nov 2024 12:23:51 +0100 Subject: [PATCH] per member tiles --- public/locales/en-GB/app.json | 1 + src/App.tsx | 9 +++++++ src/Header.tsx | 5 +++- src/config/ConfigOptions.ts | 4 ++-- src/room/InCallView.tsx | 2 ++ src/settings/SettingsModal.tsx | 17 +++++++++++++ src/settings/settings.ts | 2 ++ src/state/CallViewModel.ts | 44 ++++++++++++++++++++++------------ src/state/MediaViewModel.ts | 29 +++++++++++++++++++--- src/tile/GridTile.tsx | 2 ++ src/tile/MediaView.tsx | 9 +++++++ 11 files changed, 103 insertions(+), 21 deletions(-) diff --git a/public/locales/en-GB/app.json b/public/locales/en-GB/app.json index 02dd7740..62274c21 100644 --- a/public/locales/en-GB/app.json +++ b/public/locales/en-GB/app.json @@ -146,6 +146,7 @@ "feedback_tab_thank_you": "Thanks, we received your feedback!", "feedback_tab_title": "Feedback", "more_tab_title": "More", + "non_member_tiles": "Show non member tiles", "opt_in_description": "<0><1>You may withdraw consent by unchecking this box. If you are currently in a call, this setting will take effect at the end of the call.", "preferences_tab_body": "Here you can configure extra options for an improved experience", "preferences_tab_h4": "Preferences", diff --git a/src/App.tsx b/src/App.tsx index 8d841dba..1bc23be8 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -28,6 +28,8 @@ import { Initializer } from "./initializer"; import { MediaDevicesProvider } from "./livekit/MediaDevicesContext"; import { widget } from "./widget"; import { useTheme } from "./useTheme"; +import { nonMemberTiles } from "./settings/settings"; +import { Config } from "./config/Config"; const SentryRoute = Sentry.withSentryRouting(Route); @@ -71,6 +73,13 @@ export const App: FC = ({ history }) => { .catch(logger.error); }); + // Update settings to use the non member tile information from the config if set + useEffect(() => { + if (loaded && Config.get().show_non_member_tiles) { + nonMemberTiles.setValue(true); + } + }); + const errorPage = ; return ( diff --git a/src/Header.tsx b/src/Header.tsx index 69e77935..c17a0288 100644 --- a/src/Header.tsx +++ b/src/Header.tsx @@ -117,6 +117,7 @@ interface RoomHeaderInfoProps { avatarUrl: string | null; encrypted: boolean; participantCount: number | null; + nonMemberItemCount: number | null; } export const RoomHeaderInfo: FC = ({ @@ -125,6 +126,7 @@ export const RoomHeaderInfo: FC = ({ avatarUrl, encrypted, participantCount, + nonMemberItemCount, }) => { const { t } = useTranslation(); const size = useMediaQuery("(max-width: 550px)") ? "sm" : "lg"; @@ -157,7 +159,8 @@ export const RoomHeaderInfo: FC = ({ aria-label={t("header_participants_label")} /> - {t("participant_count", { count: participantCount ?? 0 })} + {t("participant_count", { count: participantCount ?? 0 })}{" "} + {(nonMemberItemCount ?? 0) > 0 && <>(+ {nonMemberItemCount})} )} diff --git a/src/config/ConfigOptions.ts b/src/config/ConfigOptions.ts index 6718fc34..c8f6d3e9 100644 --- a/src/config/ConfigOptions.ts +++ b/src/config/ConfigOptions.ts @@ -112,7 +112,7 @@ export interface ResolvedConfigOptions extends ConfigOptions { enable_video: boolean; }; app_prompt: boolean; - show_non_member_participants: boolean; + show_non_member_tiles: boolean; } export const DEFAULT_CONFIG: ResolvedConfigOptions = { @@ -128,5 +128,5 @@ export const DEFAULT_CONFIG: ResolvedConfigOptions = { enable_video: true, }, app_prompt: true, - show_non_member_participants: false, + show_non_member_tiles: false, }; diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index a5847f0e..66ef263d 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -194,6 +194,7 @@ export const InCallView: FC = ({ } }, [connState, onLeave]); + const nonMemberItemCount = useObservableEagerState(vm.nonMemberItemCount); const containerRef1 = useRef(null); const [containerRef2, bounds] = useMeasure(); const boundsValid = bounds.height > 0; @@ -633,6 +634,7 @@ export const InCallView: FC = ({ avatarUrl={matrixInfo.roomAvatar} encrypted={matrixInfo.e2eeSystem.kind !== E2eeType.NONE} participantCount={participantCount} + nonMemberItemCount={nonMemberItemCount} /> diff --git a/src/settings/SettingsModal.tsx b/src/settings/SettingsModal.tsx index db702ef8..6ef40c89 100644 --- a/src/settings/SettingsModal.tsx +++ b/src/settings/SettingsModal.tsx @@ -27,6 +27,7 @@ import { useSetting, developerSettingsTab as developerSettingsTabSetting, duplicateTiles as duplicateTilesSetting, + nonMemberTiles as nonMemberTilesSetting, useOptInAnalytics, } from "./settings"; import { isFirefox } from "../Platform"; @@ -68,6 +69,8 @@ export const SettingsModal: FC = ({ ); const [duplicateTiles, setDuplicateTiles] = useSetting(duplicateTilesSetting); + const [nonMemberTiles, setNonMemberTiles] = useSetting(nonMemberTilesSetting); + // Generate a `SelectInput` with a list of devices for a given device kind. const generateDeviceSelection = ( devices: MediaDevice, @@ -236,6 +239,20 @@ export const SettingsModal: FC = ({ )} /> + + ): void => { + setNonMemberTiles(event.target.checked); + }, + [setNonMemberTiles], + )} + /> + ), }; diff --git a/src/settings/settings.ts b/src/settings/settings.ts index 109a882b..293d5d59 100644 --- a/src/settings/settings.ts +++ b/src/settings/settings.ts @@ -72,6 +72,8 @@ export const developerSettingsTab = new Setting( export const duplicateTiles = new Setting("duplicate-tiles", 0); +export const nonMemberTiles = new Setting("non-member-tiles", true); + export const audioInput = new Setting( "audio-input", undefined, diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 125e473c..cc5f0544 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -68,7 +68,7 @@ import { } from "./MediaViewModel"; import { accumulate, finalizeValue } from "../utils/observable"; import { ObservableScope } from "./ObservableScope"; -import { duplicateTiles } from "../settings/settings"; +import { duplicateTiles, nonMemberTiles } from "../settings/settings"; import { isFirefox } from "../Platform"; import { setPipEnabled } from "../controls"; import { EncryptionSystem } from "../e2ee/sharedKeyManagement"; @@ -177,6 +177,7 @@ class UserMedia { member: RoomMember | undefined, participant: LocalParticipant | RemoteParticipant | undefined, encryptionSystem: EncryptionSystem, + rtcSession: MatrixRTCSession, ) { this.participant = new BehaviorSubject(participant); @@ -186,6 +187,7 @@ class UserMedia { member, this.participant.asObservable() as Observable, encryptionSystem, + rtcSession, ); } else { this.vm = new RemoteUserMediaViewModel( @@ -195,6 +197,7 @@ class UserMedia { RemoteParticipant | undefined >, encryptionSystem, + rtcSession, ); } @@ -362,6 +365,7 @@ export class CallViewModel extends ViewModel { }, ); + public readonly nonMemberItemCount = new BehaviorSubject(0); private readonly mediaItems: Observable = combineLatest([ this.remoteParticipants, observeParticipantMedia(this.livekitRoom.localParticipant), @@ -371,15 +375,18 @@ export class CallViewModel extends ViewModel { this.matrixRTCSession, MatrixRTCSessionEvent.MembershipsChanged, ).pipe(startWith(null)), - // fromEvent( - // this.matrixRTCSession, - // MatrixRTCSessionEvent.EncryptionKeyChanged, - // ).pipe(startWith(null)), + nonMemberTiles.value, ]).pipe( scan( ( prevItems, - [remoteParticipants, { participant: localParticipant }, duplicateTiles], + [ + remoteParticipants, + { participant: localParticipant }, + duplicateTiles, + _participantChange, + nonMemberTiles, + ], ) => { const newItems = new Map( function* (this: CallViewModel): Iterable<[string, MediaItem]> { @@ -423,6 +430,7 @@ export class CallViewModel extends ViewModel { member, participant, this.encryptionSystem, + this.matrixRTCSession, ), ]; } @@ -455,27 +463,28 @@ export class CallViewModel extends ViewModel { // - If one wants to test scalability using the livekit cli. // - If an experimental project does not yet do the matrixRTC bits. // - If someone wants to debug if the LK connection works but matrixRTC room state failed to arrive. - const debugShowNonMember = Config.get().show_non_member_participants; + const debugShowNonMember = nonMemberTiles; //Config.get().show_non_member_tiles; const newNonMemberItems = debugShowNonMember ? new Map( function* (this: CallViewModel): Iterable<[string, MediaItem]> { - for (let p = 0; p < remoteParticipants.length; p++) { + for (const participant of remoteParticipants) { for (let i = 0; i < 1 + duplicateTiles; i++) { - const participant = remoteParticipants[p]; - const maybeNoMemberParticipantId = + const maybeNonMemberParticipantId = participant.identity + ":" + i; - if (!newItems.has(maybeNoMemberParticipantId)) { + if (!newItems.has(maybeNonMemberParticipantId)) { + const nonMemberId = maybeNonMemberParticipantId; yield [ - maybeNoMemberParticipantId, + nonMemberId, // We create UserMedia with or without a participant. // This will be the initial value of a BehaviourSubject. // Once a participant appears we will update the BehaviourSubject. (see above) - prevItems.get(maybeNoMemberParticipantId) ?? + prevItems.get(nonMemberId) ?? new UserMedia( - maybeNoMemberParticipantId, + nonMemberId, undefined, participant, this.encrypted, + this.matrixRTCSession, ), ]; } @@ -487,13 +496,18 @@ export class CallViewModel extends ViewModel { if (newNonMemberItems.size > 0) { logger.debug("Added NonMember items: ", newNonMemberItems); } + const newNonMemberItemCount = + newNonMemberItems.size / (1 + duplicateTiles); + if (this.nonMemberItemCount.value !== newNonMemberItemCount) + this.nonMemberItemCount.next(newNonMemberItemCount); + const combinedNew = new Map([ ...newNonMemberItems.entries(), ...newItems.entries(), ]); for (const [id, t] of prevItems) if (!combinedNew.has(id)) t.destroy(); - return newItems; + return combinedNew; }, new Map(), ), diff --git a/src/state/MediaViewModel.ts b/src/state/MediaViewModel.ts index 32c09257..c83d6c08 100644 --- a/src/state/MediaViewModel.ts +++ b/src/state/MediaViewModel.ts @@ -37,6 +37,11 @@ import { switchMap, } from "rxjs"; import { useEffect } from "react"; +import { + MatrixRTCSession, + MatrixRTCSessionEvent, +} from "matrix-js-sdk/src/matrixrtc"; +import { logger } from "matrix-js-sdk/src/logger"; import { ViewModel } from "./ViewModel"; import { useReactiveState } from "../useReactiveState"; @@ -196,11 +201,16 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel { */ public readonly cropVideo: Observable = this._cropVideo; + public readonly keys = new BehaviorSubject( + [] as { index: number; key: Uint8Array }[], + ); + public constructor( id: string, member: RoomMember | undefined, participant: Observable, encryptionSystem: EncryptionSystem, + rtcSession: MatrixRTCSession, ) { super( id, @@ -211,7 +221,18 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel { Track.Source.Camera, ); - // const media = observeParticipantMedia(participant).pipe(this.scope.state()); + // rtcSession.on( + // MatrixRTCSessionEvent.EncryptionKeyChanged, + // (key, index, participantId) => { + // if (id.startsWith(participantId)) + // logger.info("got new keys: ", participant, { index, key }); + // logger.info("All keys for participant ", participant, " - ", [ + // ...this.keys.value, + // { index, key }, + // ]); + // this.keys.next([...this.keys.value, { index, key }]); + // }, + // ); const media = participant.pipe( switchMap((p) => (p && observeParticipantMedia(p)) ?? of(undefined)), @@ -263,8 +284,9 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel { member: RoomMember | undefined, participant: Observable, encryptionSystem: EncryptionSystem, + rtcSession: MatrixRTCSession, ) { - super(id, member, participant, encryptionSystem); + super(id, member, participant, encryptionSystem, rtcSession); } } @@ -323,8 +345,9 @@ export class RemoteUserMediaViewModel extends BaseUserMediaViewModel { member: RoomMember | undefined, participant: Observable, encryptionSystem: EncryptionSystem, + rtcSession: MatrixRTCSession, ) { - super(id, member, participant, encryptionSystem); + super(id, member, participant, encryptionSystem, rtcSession); // Sync the local volume with LiveKit combineLatest([ diff --git a/src/tile/GridTile.tsx b/src/tile/GridTile.tsx index 0cbe1d25..980cb4f2 100644 --- a/src/tile/GridTile.tsx +++ b/src/tile/GridTile.tsx @@ -84,6 +84,7 @@ const UserMediaTile = forwardRef( const videoEnabled = useObservableEagerState(vm.videoEnabled); const speaking = useObservableEagerState(vm.speaking); const cropVideo = useObservableEagerState(vm.cropVideo); + const keys = useObservableEagerState(vm.keys); const isRTCParticipantAvailable = useObservableEagerState( vm.isRTCParticipantAvailable, ); @@ -121,6 +122,7 @@ const UserMediaTile = forwardRef( ref={ref} video={video} member={vm.member} + keys={keys} unencryptedWarning={unencryptedWarning} videoEnabled={videoEnabled && showVideo} videoFit={cropVideo ? "cover" : "contain"} diff --git a/src/tile/MediaView.tsx b/src/tile/MediaView.tsx index 6cc9086f..a8ec58c9 100644 --- a/src/tile/MediaView.tsx +++ b/src/tile/MediaView.tsx @@ -29,6 +29,7 @@ interface Props extends ComponentProps { videoFit: "cover" | "contain"; mirror: boolean; member: RoomMember | undefined; + keys: { index: number; key: Uint8Array }[]; videoEnabled: boolean; unencryptedWarning: boolean; nameTagLeadingIcon?: ReactNode; @@ -48,6 +49,7 @@ export const MediaView = forwardRef( videoFit, mirror, member, + keys, videoEnabled, unencryptedWarning, nameTagLeadingIcon, @@ -98,11 +100,18 @@ export const MediaView = forwardRef( minature={avatarSize < 96} showTimer={handRaiseTimerVisible} /> + {/* {keys && + keys.map(({ index, key }) => ( + + index:{index}, key:{key} + + ))} */}
{nameTagLeadingIcon} {displayName} + {unencryptedWarning && (