Merge branch 'livekit' into toger5/tiles_based_on_rtc_member

src/UrlParams.test.ts

@@ -7,7 +7,7 @@ Please see LICENSE in the repository root for full details.
 
 import { describe, expect, it } from "vitest";
 
-import { getRoomIdentifierFromUrl } from "../src/UrlParams";
+import { getRoomIdentifierFromUrl, getUrlParams } from "../src/UrlParams";
 
 const ROOM_NAME = "roomNameHere";
 const ROOM_ID = "!d45f138fsd";
@@ -86,4 +86,18 @@ describe("UrlParams", () => {
         .roomAlias,
     ).toBeFalsy();
   });
+
+  describe("preload", () => {
+    it("defaults to false", () => {
+      expect(getUrlParams().preload).toBe(false);
+    });
+
+    it("ignored in SPA mode", () => {
+      expect(getUrlParams("?preload=true").preload).toBe(false);
+    });
+
+    it("respected in widget mode", () => {
+      expect(getUrlParams("?preload=true&widgetId=12345").preload).toBe(true);
+    });
+  });
 });

src/UrlParams.ts

@@ -211,8 +211,11 @@ export const getUrlParams = (
 
   const fontScale = parseFloat(parser.getParam("fontScale") ?? "");
 
+  const widgetId = parser.getParam("widgetId");
+  const isWidget = !!widgetId;
+
   return {
-    widgetId: parser.getParam("widgetId"),
+    widgetId,
     parentUrl: parser.getParam("parentUrl"),
 
     // NB. we don't validate roomId here as we do in getRoomIdentifierFromUrl:
@@ -224,7 +227,7 @@ export const getUrlParams = (
     confineToRoom:
       parser.getFlagParam("confineToRoom") || parser.getFlagParam("embed"),
     appPrompt: parser.getFlagParam("appPrompt", true),
-    preload: parser.getFlagParam("preload"),
+    preload: isWidget ? parser.getFlagParam("preload") : false,
     hideHeader: parser.getFlagParam("hideHeader"),
     showControls: parser.getFlagParam("showControls", true),
     hideScreensharing: parser.getFlagParam("hideScreensharing"),

src/livekit/MediaDevicesContext.tsx

@@ -24,6 +24,7 @@ import {
   audioInput as audioInputSetting,
   audioOutput as audioOutputSetting,
   videoInput as videoInputSetting,
+  Setting,
 } from "../settings/settings";
 import { isFirefox } from "../Platform";
 
@@ -58,7 +59,7 @@ function useObservableState<T>(
 
 function useMediaDevice(
   kind: MediaDeviceKind,
-  fallbackDevice: string | undefined,
+  setting: Setting<string | undefined>,
   usingNames: boolean,
   alwaysDefault: boolean = false,
 ): MediaDevice {
@@ -84,15 +85,21 @@ function useMediaDevice(
     [kind, requestPermissions],
   );
   const available = useObservableState(deviceObserver, []);
-  const [selectedId, select] = useState(fallbackDevice);
+  const [preferredId, select] = useSetting(setting);
 
   return useMemo(() => {
-    let devId;
-    if (available) {
-      devId = available.some((d) => d.deviceId === selectedId)
-        ? selectedId
-        : available.some((d) => d.deviceId === fallbackDevice)
-          ? fallbackDevice
+    let selectedId: string | undefined = undefined;
+    if (!alwaysDefault && available) {
+      // If the preferred device is available, use it. Or if every available
+      // device ID is falsy, the browser is probably just being paranoid about
+      // fingerprinting and we should still try using the preferred device.
+      // Worst case it is not available and the browser will gracefully fall
+      // back to some other device for us when requesting the media stream.
+      // Otherwise, select the first available device.
+      selectedId =
+        available.some((d) => d.deviceId === preferredId) ||
+        available.every((d) => d.deviceId === "")
+          ? preferredId
           : available.at(0)?.deviceId;
     }
 
@@ -102,10 +109,10 @@ function useMediaDevice(
           // device entries for the exact same device ID; deduplicate them
           [...new Map(available.map((d) => [d.deviceId, d])).values()]
         : [],
-      selectedId: alwaysDefault ? undefined : devId,
+      selectedId,
       select,
     };
-  }, [available, selectedId, fallbackDevice, select, alwaysDefault]);
+  }, [available, preferredId, select, alwaysDefault]);
 }
 
 const deviceStub: MediaDevice = {
@@ -141,36 +148,22 @@ export const MediaDevicesProvider: FC<Props> = ({ children }) => {
   // for ouput devices because the selector wont be shown on FF.
   const useOutputNames = usingNames && !isFirefox();
 
-  const [storedAudioInput, setStoredAudioInput] = useSetting(audioInputSetting);
-  const [storedAudioOutput, setStoredAudioOutput] =
-    useSetting(audioOutputSetting);
-  const [storedVideoInput, setStoredVideoInput] = useSetting(videoInputSetting);
-
-  const audioInput = useMediaDevice("audioinput", storedAudioInput, usingNames);
+  const audioInput = useMediaDevice(
+    "audioinput",
+    audioInputSetting,
+    usingNames,
+  );
   const audioOutput = useMediaDevice(
     "audiooutput",
-    storedAudioOutput,
+    audioOutputSetting,
     useOutputNames,
     alwaysUseDefaultAudio,
   );
-  const videoInput = useMediaDevice("videoinput", storedVideoInput, usingNames);
-
-  useEffect(() => {
-    if (audioInput.selectedId !== undefined)
-      setStoredAudioInput(audioInput.selectedId);
-  }, [setStoredAudioInput, audioInput.selectedId]);
-
-  useEffect(() => {
-    // Skip setting state for ff output. Redundent since it is set to always return 'undefined'
-    // but makes it clear while debugging that this is not happening on FF. + perf ;)
-    if (audioOutput.selectedId !== undefined && !isFirefox())
-      setStoredAudioOutput(audioOutput.selectedId);
-  }, [setStoredAudioOutput, audioOutput.selectedId]);
-
-  useEffect(() => {
-    if (videoInput.selectedId !== undefined)
-      setStoredVideoInput(videoInput.selectedId);
-  }, [setStoredVideoInput, videoInput.selectedId]);
+  const videoInput = useMediaDevice(
+    "videoinput",
+    videoInputSetting,
+    usingNames,
+  );
 
   const startUsingDeviceNames = useCallback(
     () => setNumCallersUsingNames((n) => n + 1),

src/room/useSwitchCamera.ts

@@ -20,7 +20,6 @@ import {
   TrackEvent,
 } from "livekit-client";
 import { useObservable, useObservableEagerState } from "observable-hooks";
-import { useEffect } from "react";
 import { logger } from "matrix-js-sdk/src/logger";
 
 import { useMediaDevices } from "../livekit/MediaDevicesContext";
@@ -35,6 +34,7 @@ export function useSwitchCamera(
   video: Observable<LocalVideoTrack | null>,
 ): (() => void) | null {
   const mediaDevices = useMediaDevices();
+  const setVideoInput = useLatest(mediaDevices.videoInput.select);
 
   // Produce an observable like the input 'video' observable, except make it
   // emit whenever the track is muted or the device changes
@@ -75,6 +75,12 @@ export function useSwitchCamera(
                   .restartTrack({
                     facingMode: facingMode === "user" ? "environment" : "user",
                   })
+                  .then(() => {
+                    // Inform the MediaDeviceContext which camera was chosen
+                    const deviceId =
+                      track.mediaStreamTrack.getSettings().deviceId;
+                    if (deviceId !== undefined) setVideoInput.current(deviceId);
+                  })
                   .catch((e) =>
                     logger.error("Failed to switch camera", facingMode, e),
                   );
@@ -83,16 +89,5 @@ export function useSwitchCamera(
     [videoTrack],
   );
 
-  const setVideoInput = useLatest(mediaDevices.videoInput.select);
-  useEffect(() => {
-    // Watch for device changes due to switching the camera and feed them back
-    // into the MediaDeviceContext
-    const subscription = videoTrack.subscribe((track) => {
-      const deviceId = track?.mediaStreamTrack.getSettings().deviceId;
-      if (deviceId !== undefined) setVideoInput.current(deviceId);
-    });
-    return (): void => subscription.unsubscribe();
-  }, [videoTrack, setVideoInput]);
-
   return useObservableEagerState(switchCamera);
 }

src/settings/DeviceSelection.module.css

@@ -0,0 +1,18 @@
+.selection {
+  gap: 0;
+}
+
+.title {
+  color: var(--cpd-color-text-secondary);
+  margin-block: var(--cpd-space-3x) 0;
+}
+
+.separator {
+  margin-block: 6px var(--cpd-space-4x);
+}
+
+.options {
+  display: flex;
+  flex-direction: column;
+  gap: var(--cpd-space-4x);
+}

src/settings/DeviceSelection.tsx

@@ -0,0 +1,71 @@
+/*
+Copyright 2024 New Vector Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only
+Please see LICENSE in the repository root for full details.
+*/
+
+import { ChangeEvent, FC, useCallback, useId } from "react";
+import {
+  Heading,
+  InlineField,
+  Label,
+  RadioControl,
+  Separator,
+} from "@vector-im/compound-web";
+
+import { MediaDevice } from "../livekit/MediaDevicesContext";
+import styles from "./DeviceSelection.module.css";
+
+interface Props {
+  devices: MediaDevice;
+  caption: string;
+}
+
+export const DeviceSelection: FC<Props> = ({ devices, caption }) => {
+  const groupId = useId();
+  const onChange = useCallback(
+    (e: ChangeEvent<HTMLInputElement>) => {
+      devices.select(e.target.value);
+    },
+    [devices],
+  );
+
+  if (devices.available.length == 0) return null;
+
+  return (
+    <div className={styles.selection}>
+      <Heading
+        type="body"
+        weight="semibold"
+        size="sm"
+        as="h4"
+        className={styles.title}
+      >
+        {caption}
+      </Heading>
+      <Separator className={styles.separator} />
+      <div className={styles.options}>
+        {devices.available.map(({ deviceId, label }, index) => (
+          <InlineField
+            key={deviceId}
+            name={groupId}
+            control={
+              <RadioControl
+                checked={deviceId === devices.selectedId}
+                onChange={onChange}
+                value={deviceId}
+              />
+            }
+          >
+            <Label>
+              {!!label && label.trim().length > 0
+                ? label
+                : `${caption} ${index + 1}`}
+            </Label>
+          </InlineField>
+        ))}
+      </div>
+    </div>
+  );
+};

src/settings/SettingsModal.tsx

@@ -5,10 +5,10 @@ SPDX-License-Identifier: AGPL-3.0-only
 Please see LICENSE in the repository root for full details.
 */
 
-import { ChangeEvent, FC, ReactNode, useCallback } from "react";
+import { ChangeEvent, FC, useCallback } from "react";
 import { Trans, useTranslation } from "react-i18next";
 import { MatrixClient } from "matrix-js-sdk/src/matrix";
-import { Dropdown, Separator, Text } from "@vector-im/compound-web";
+import { Root as Form, Text } from "@vector-im/compound-web";
 
 import { Modal } from "../Modal";
 import styles from "./SettingsModal.module.css";
@@ -19,7 +19,6 @@ import { ProfileSettingsTab } from "./ProfileSettingsTab";
 import { FeedbackSettingsTab } from "./FeedbackSettingsTab";
 import {
   useMediaDevices,
-  MediaDevice,
   useMediaDeviceNames,
 } from "../livekit/MediaDevicesContext";
 import { widget } from "../widget";
@@ -33,6 +32,7 @@ import {
 import { isFirefox } from "../Platform";
 import { PreferencesSettingsTab } from "./PreferencesSettingsTab";
 import { Slider } from "../Slider";
+import { DeviceSelection } from "./DeviceSelection";
 
 type SettingsTab =
   | "audio"
@@ -70,40 +70,6 @@ export const SettingsModal: FC<Props> = ({
   );
   const [duplicateTiles, setDuplicateTiles] = useSetting(duplicateTilesSetting);
 
-  // Generate a `SelectInput` with a list of devices for a given device kind.
-  const generateDeviceSelection = (
-    devices: MediaDevice,
-    caption: string,
-  ): ReactNode => {
-    if (devices.available.length == 0) return null;
-
-    const values = devices.available.map(
-      ({ deviceId, label }, index) =>
-        [
-          deviceId,
-          !!label && label.trim().length > 0
-            ? label
-            : `${caption} ${index + 1}`,
-        ] as [string, string],
-    );
-
-    return (
-      <Dropdown
-        label={caption}
-        defaultValue={
-          devices.selectedId === "" || !devices.selectedId
-            ? "default"
-            : devices.selectedId
-        }
-        onValueChange={(id): void => devices.select(id)}
-        values={values}
-        // XXX This is unused because we set a defaultValue. The component
-        // shouldn't require this prop.
-        placeholder=""
-      />
-    );
-  };
-
   const optInDescription = (
     <Text size="sm">
       <Trans i18nKey="settings.opt_in_description">
@@ -125,25 +91,30 @@ export const SettingsModal: FC<Props> = ({
     name: t("common.audio"),
     content: (
       <>
-        {generateDeviceSelection(devices.audioInput, t("common.microphone"))}
-        {!isFirefox() &&
-          generateDeviceSelection(
-            devices.audioOutput,
-            t("settings.speaker_device_selection_label"),
-          )}
-        <Separator />
-        <div className={styles.volumeSlider}>
-          <label>{t("settings.audio_tab.effect_volume_label")}</label>
-          <p>{t("settings.audio_tab.effect_volume_description")}</p>
-          <Slider
-            label={t("video_tile.volume")}
-            value={soundVolume}
-            onValueChange={setSoundVolume}
-            min={0}
-            max={1}
-            step={0.01}
+        <Form>
+          <DeviceSelection
+            devices={devices.audioInput}
+            caption={t("common.microphone")}
           />
-        </div>
+          {!isFirefox() && (
+            <DeviceSelection
+              devices={devices.audioOutput}
+              caption={t("settings.speaker_device_selection_label")}
+            />
+          )}
+          <div className={styles.volumeSlider}>
+            <label>{t("settings.audio_tab.effect_volume_label")}</label>
+            <p>{t("settings.audio_tab.effect_volume_description")}</p>
+            <Slider
+              label={t("video_tile.volume")}
+              value={soundVolume}
+              onValueChange={setSoundVolume}
+              min={0}
+              max={1}
+              step={0.01}
+            />
+          </div>
+        </Form>
       </>
     ),
   };
@@ -151,7 +122,14 @@ export const SettingsModal: FC<Props> = ({
   const videoTab: Tab<SettingsTab> = {
     key: "video",
     name: t("common.video"),
-    content: generateDeviceSelection(devices.videoInput, t("common.camera")),
+    content: (
+      <Form>
+        <DeviceSelection
+          devices={devices.videoInput}
+          caption={t("common.camera")}
+        />
+      </Form>
+    ),
   };
 
   const preferencesTab: Tab<SettingsTab> = {

src/state/CallViewModel.test.ts

@@ -20,6 +20,7 @@ import {
   ConnectionState,
   LocalParticipant,
   Participant,
+  ParticipantEvent,
   RemoteParticipant,
 } from "livekit-client";
 import * as ComponentsCore from "@livekit/components-core";
@@ -211,11 +212,15 @@ function withCallViewModel(
     );
   const eventsSpy = vi
     .spyOn(ComponentsCore, "observeParticipantEvents")
-    .mockImplementation((p) =>
-      (speaking.get(p) ?? of(false)).pipe(
-        map((s) => ({ ...p, isSpeaking: s }) as Participant),
-      ),
-    );
+    .mockImplementation((p, ...eventTypes) => {
+      if (eventTypes.includes(ParticipantEvent.IsSpeakingChanged)) {
+        return (speaking.get(p) ?? of(false)).pipe(
+          map((s) => ({ ...p, isSpeaking: s }) as Participant),
+        );
+      } else {
+        return of(p);
+      }
+    });
 
   const roomEventSelectorSpy = vi
     .spyOn(ComponentsCore, "roomEventSelector")

src/state/CallViewModel.ts

@@ -24,7 +24,6 @@ import {
   EMPTY,
   Observable,
   Subject,
-  audit,
   combineLatest,
   concat,
   distinctUntilChanged,
@@ -77,6 +76,7 @@ import { spotlightExpandedLayout } from "./SpotlightExpandedLayout";
 import { oneOnOneLayout } from "./OneOnOneLayout";
 import { pipLayout } from "./PipLayout";
 import { EncryptionSystem } from "../e2ee/sharedKeyManagement";
+import { observeSpeaker } from "./observeSpeaker";
 
 // How long we wait after a focus switch before showing the real participant
 // list again
@@ -259,22 +259,7 @@ class UserMedia {
       );
     }
 
-    this.speaker = this.vm.speaking.pipe(
-      // Require 1 s of continuous speaking to become a speaker, and 60 s of
-      // continuous silence to stop being considered a speaker
-      audit((s) =>
-        merge(
-          timer(s ? 1000 : 60000),
-          // If the speaking flag resets to its original value during this time,
-          // end the silencing window to stick with that original value
-          this.vm.speaking.pipe(filter((s1) => s1 !== s)),
-        ),
-      ),
-      startWith(false),
-      // Make this Observable hot so that the timers don't reset when you
-      // resubscribe
-      this.scope.state(),
-    );
+    this.speaker = observeSpeaker(this.vm.speaking).pipe(this.scope.state());
 
     this.presenter = this.participant.pipe(
       switchMap(

src/state/observeSpeaker.test.ts

@@ -0,0 +1,119 @@
+/*
+Copyright 2024 New Vector Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only
+Please see LICENSE in the repository root for full details.
+*/
+
+import { describe, test } from "vitest";
+
+import { withTestScheduler } from "../utils/test";
+import { observeSpeaker } from "./observeSpeaker";
+
+const yesNo = {
+  y: true,
+  n: false,
+};
+
+describe("observeSpeaker", () => {
+  describe("does not activate", () => {
+    const expectedOutputMarbles = "n";
+    test("starts correctly", () => {
+      // should default to false when no input is given
+      const speakingInputMarbles = "";
+      withTestScheduler(({ hot, expectObservable }) => {
+        expectObservable(observeSpeaker(hot(speakingInputMarbles, yesNo))).toBe(
+          expectedOutputMarbles,
+          yesNo,
+        );
+      });
+    });
+
+    test("after no speaking", () => {
+      const speakingInputMarbles = "n";
+      withTestScheduler(({ hot, expectObservable }) => {
+        expectObservable(observeSpeaker(hot(speakingInputMarbles, yesNo))).toBe(
+          expectedOutputMarbles,
+          yesNo,
+        );
+      });
+    });
+
+    test("with speaking for 1ms", () => {
+      const speakingInputMarbles = "y n";
+      withTestScheduler(({ hot, expectObservable }) => {
+        expectObservable(observeSpeaker(hot(speakingInputMarbles, yesNo))).toBe(
+          expectedOutputMarbles,
+          yesNo,
+        );
+      });
+    });
+
+    test("with speaking for 999ms", () => {
+      const speakingInputMarbles = "y 999ms n";
+      withTestScheduler(({ hot, expectObservable }) => {
+        expectObservable(observeSpeaker(hot(speakingInputMarbles, yesNo))).toBe(
+          expectedOutputMarbles,
+          yesNo,
+        );
+      });
+    });
+
+    test("with speaking intermittently", () => {
+      const speakingInputMarbles =
+        "y 199ms n 199ms y 199ms n 199ms y 199ms n 199ms y 199ms n 199ms y 199ms n 199ms y 199ms n 199ms y 199ms n 199ms y 199ms n";
+      withTestScheduler(({ hot, expectObservable }) => {
+        expectObservable(observeSpeaker(hot(speakingInputMarbles, yesNo))).toBe(
+          expectedOutputMarbles,
+          yesNo,
+        );
+      });
+    });
+
+    test("with consecutive speaking then stops speaking", () => {
+      const speakingInputMarbles = "y y y y y y y y y y n";
+      withTestScheduler(({ hot, expectObservable }) => {
+        expectObservable(observeSpeaker(hot(speakingInputMarbles, yesNo))).toBe(
+          expectedOutputMarbles,
+          yesNo,
+        );
+      });
+    });
+  });
+
+  describe("activates", () => {
+    test("after 1s", () => {
+      // this will active after 1s as no `n` follows it:
+      const speakingInputMarbles = " y";
+      const expectedOutputMarbles = "n 999ms y";
+      withTestScheduler(({ hot, expectObservable }) => {
+        expectObservable(observeSpeaker(hot(speakingInputMarbles, yesNo))).toBe(
+          expectedOutputMarbles,
+          yesNo,
+        );
+      });
+    });
+
+    test("speaking for 1001ms activates for 60s", () => {
+      const speakingInputMarbles = " y 1s    n      ";
+      const expectedOutputMarbles = "n 999ms y 60s n";
+      withTestScheduler(({ hot, expectObservable }) => {
+        expectObservable(observeSpeaker(hot(speakingInputMarbles, yesNo))).toBe(
+          expectedOutputMarbles,
+          yesNo,
+        );
+      });
+    });
+
+    test("speaking for 5s activates for 64s", () => {
+      const speakingInputMarbles = " y 5s    n      ";
+      const expectedOutputMarbles = "n 999ms y 64s n";
+      withTestScheduler(({ hot, expectObservable }) => {
+        expectObservable(observeSpeaker(hot(speakingInputMarbles, yesNo))).toBe(
+          expectedOutputMarbles,
+          yesNo,
+        );
+      });
+    });
+  });
+});

src/state/observeSpeaker.ts

@@ -0,0 +1,36 @@
+/*
+Copyright 2024 New Vector Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only
+Please see LICENSE in the repository root for full details.
+*/
+import {
+  Observable,
+  audit,
+  merge,
+  timer,
+  filter,
+  startWith,
+  distinctUntilChanged,
+} from "rxjs";
+
+/**
+ * Require 1 second of continuous speaking to become a speaker, and 60 second of
+ * continuous silence to stop being considered a speaker
+ */
+export function observeSpeaker(
+  isSpeakingObservable: Observable<boolean>,
+): Observable<boolean> {
+  const distinct = isSpeakingObservable.pipe(distinctUntilChanged());
+
+  return distinct.pipe(
+    // Either change to the new value after the timer or re-emit the same value if it toggles back
+    // (audit will return the latest (toggled back) value) before the timeout.
+    audit((s) =>
+      merge(timer(s ? 1000 : 60000), distinct.pipe(filter((s1) => s1 !== s))),
+    ),
+    // Filter the re-emissions (marked as: | ) that happen if we toggle quickly (<1s) from false->true->false|->..
+    startWith(false),
+    distinctUntilChanged(),
+  );
+}

src/useCallViewKeyboardShortcuts.test.tsx

@@ -93,6 +93,16 @@ test("reactions can be sent via keyboard presses", async () => {
   }
 });
 
+test("reaction is not sent when modifier key is held", async () => {
+  const user = userEvent.setup();
+
+  const sendReaction = vi.fn();
+  render(<TestComponent sendReaction={sendReaction} />);
+
+  await user.keyboard("{Meta>}1{/Meta}");
+  expect(sendReaction).not.toHaveBeenCalled();
+});
+
 test("raised hand can be sent via keyboard presses", async () => {
   const user = userEvent.setup();
 

src/useCallViewKeyboardShortcuts.ts

@@ -43,6 +43,8 @@ export function useCallViewKeyboardShortcuts(
       (event: KeyboardEvent) => {
         if (focusElement.current === null) return;
         if (!mayReceiveKeyEvents(focusElement.current)) return;
+        if (event.altKey || event.ctrlKey || event.metaKey || event.shiftKey)
+          return;
 
         if (event.key === "m") {
           event.preventDefault();