Merge branch 'livekit' into raise-hand-button

This commit is contained in:
Will Hunt
2024-11-04 08:56:39 +00:00
committed by GitHub
22 changed files with 280 additions and 90 deletions

View File

@@ -23,7 +23,7 @@ jobs:
packages: write
steps:
- name: Check it out
uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
- name: 📥 Download artifact
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4

View File

@@ -12,7 +12,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Check out test private repo
uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
with:
repository: element-hq/static-call-participant
ref: refs/heads/main

View File

@@ -21,9 +21,9 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
- name: Yarn cache
uses: actions/setup-node@0a44ba7841725637a19e28fa30b79a866c81b0a6 # v4
uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4
with:
cache: "yarn"
node-version: "lts/*"

View File

@@ -7,9 +7,9 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
- name: Yarn cache
uses: actions/setup-node@0a44ba7841725637a19e28fa30b79a866c81b0a6 # v4
uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4
with:
cache: "yarn"
node-version: "lts/*"

View File

@@ -9,9 +9,9 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
- name: Yarn cache
uses: actions/setup-node@0a44ba7841725637a19e28fa30b79a866c81b0a6 # v4
uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4
with:
cache: "yarn"
node-version: "lts/*"

View File

@@ -13,9 +13,9 @@ jobs:
steps:
- name: Checkout the code
uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
- uses: actions/setup-node@0a44ba7841725637a19e28fa30b79a866c81b0a6 # v4
- uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4
with:
cache: "yarn"
node-version: "lts/*"

View File

@@ -14,7 +14,7 @@ jobs:
steps:
- name: Checkout the code
uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
- name: Upload
uses: localazy/upload@27e6b5c0fddf4551596b42226b1c24124335d24a # v1

View File

@@ -98,6 +98,8 @@ confineToRoom: boolean; (default: false)
Whether upon entering a room, the user should be prompted to launch the
native mobile app. (Affects only Android and iOS.)
The app prompt must also be enabled in the config for this to take effect.
```ts
appPrompt: boolean; (default: true)
```

View File

@@ -160,6 +160,7 @@
"stop_screenshare_button_label": "Sharing screen",
"stop_video_button_label": "Stop video",
"submitting": "Submitting…",
"switch_camera": "Switch camera",
"unauthenticated_view_body": "Not registered yet? <2>Create an account</2>",
"unauthenticated_view_eula_caption": "By clicking \"Go\", you agree to our <2>End User Licensing Agreement (EULA)</2>",
"unauthenticated_view_login_button": "Login to your account",

View File

@@ -43,6 +43,8 @@ export interface UrlParams {
/**
* Whether upon entering a room, the user should be prompted to launch the
* native mobile app. (Affects only Android and iOS.)
*
* The app prompt must also be enabled in the config for this to take effect.
*/
appPrompt: boolean;
/**

View File

@@ -16,6 +16,7 @@ import {
EndCallIcon,
ShareScreenSolidIcon,
SettingsSolidIcon,
SwitchCameraSolidIcon,
} from "@vector-im/compound-design-tokens/assets/web/icons";
import styles from "./Button.module.css";
@@ -66,6 +67,23 @@ export const VideoButton: FC<VideoButtonProps> = ({ muted, ...props }) => {
);
};
export const SwitchCameraButton: FC<ComponentPropsWithoutRef<"button">> = (
props,
) => {
const { t } = useTranslation();
return (
<Tooltip label={t("switch_camera")}>
<CpdButton
iconOnly
Icon={SwitchCameraSolidIcon}
kind="secondary"
{...props}
/>
</Tooltip>
);
};
interface ShareScreenButtonProps extends ComponentPropsWithoutRef<"button"> {
enabled: boolean;
}

View File

@@ -88,6 +88,14 @@ export interface ConfigOptions {
*/
enable_video?: boolean;
};
/**
* Whether upon entering a room, the user should be prompted to launch the
* native mobile app. (Affects only Android and iOS.)
*
* Note that this can additionally be disabled by the app's URL parameters.
*/
app_prompt?: boolean;
}
// Overrides members from ConfigOptions that are always provided by the
@@ -103,6 +111,7 @@ export interface ResolvedConfigOptions extends ConfigOptions {
enable_audio: boolean;
enable_video: boolean;
};
app_prompt: boolean;
}
export const DEFAULT_CONFIG: ResolvedConfigOptions = {
@@ -117,4 +126,5 @@ export const DEFAULT_CONFIG: ResolvedConfigOptions = {
enable_audio: true,
enable_video: true,
},
app_prompt: true,
};

View File

@@ -42,6 +42,7 @@ import {
ShareScreenButton,
SettingsButton,
RaiseHandToggleButton,
SwitchCameraButton,
} from "../button";
import { Header, LeftNav, RightNav, RoomHeaderInfo } from "../Header";
import { useUrlParams } from "../UrlParams";
@@ -82,7 +83,8 @@ import { makeSpotlightLandscapeLayout } from "../grid/SpotlightLandscapeLayout";
import { makeSpotlightPortraitLayout } from "../grid/SpotlightPortraitLayout";
import { ReactionsProvider, useReactions } from "../useReactions";
import handSoundOgg from "../sound/raise_hand.ogg?url";
import handSoundMp3 from "../sound/raise_hand.mp3?url";
import handSoundMp3 from "../sound/raise_hand.mp3?url"
import { useSwitchCamera } from "./useSwitchCamera";
const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {});
@@ -231,6 +233,7 @@ export const InCallView: FC<InCallViewProps> = ({
const gridMode = useObservableEagerState(vm.gridMode);
const showHeader = useObservableEagerState(vm.showHeader);
const showFooter = useObservableEagerState(vm.showFooter);
const switchCamera = useSwitchCamera(vm.localVideo);
// Ideally we could detect taps by listening for click events and checking
// that the pointerType of the event is "touch", but this isn't yet supported
@@ -274,10 +277,17 @@ export const InCallView: FC<InCallViewProps> = ({
[setSettingsModalOpen],
);
const openProfile = useCallback(() => {
setSettingsTab("profile");
setSettingsModalOpen(true);
}, [setSettingsTab, setSettingsModalOpen]);
const openProfile = useMemo(
() =>
// Profile settings are unavailable in widget mode
widget === null
? (): void => {
setSettingsTab("profile");
setSettingsModalOpen(true);
}
: null,
[setSettingsTab, setSettingsModalOpen],
);
const [headerRef, headerBounds] = useMeasure();
const [footerRef, footerBounds] = useMeasure();
@@ -515,14 +525,14 @@ export const InCallView: FC<InCallViewProps> = ({
buttons.push(
<MicButton
key="1"
key="audio"
muted={!muteStates.audio.enabled}
onClick={toggleMicrophone}
disabled={muteStates.audio.setEnabled === null}
data-testid="incall_mute"
/>,
<VideoButton
key="2"
key="video"
muted={!muteStates.video.enabled}
onClick={toggleCamera}
disabled={muteStates.video.setEnabled === null}
@@ -530,10 +540,14 @@ export const InCallView: FC<InCallViewProps> = ({
/>,
);
if (!reducedControls) {
if (switchCamera !== null)
buttons.push(
<SwitchCameraButton key="switch_camera" onClick={switchCamera} />,
);
if (canScreenshare && !hideScreensharing) {
buttons.push(
<ShareScreenButton
key="3"
key="share_screen"
enabled={isScreenShareEnabled}
onClick={toggleScreensharing}
data-testid="incall_screenshare"
@@ -549,12 +563,12 @@ export const InCallView: FC<InCallViewProps> = ({
/>,
);
}
buttons.push(<SettingsButton key="5" onClick={openSettings} />);
buttons.push(<SettingsButton key="settings" onClick={openSettings} />);
}
buttons.push(
<EndCallButton
key="6"
key="end_call"
onClick={function (): void {
onLeave();
}}

View File

@@ -5,12 +5,17 @@ SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { FC, useCallback, useState } from "react";
import { FC, useCallback, useMemo, useState } from "react";
import { useTranslation } from "react-i18next";
import { MatrixClient } from "matrix-js-sdk/src/matrix";
import { Button } from "@vector-im/compound-web";
import classNames from "classnames";
import { useHistory } from "react-router-dom";
import { logger } from "matrix-js-sdk/src/logger";
import { usePreviewTracks } from "@livekit/components-react";
import { LocalVideoTrack, Track } from "livekit-client";
import { useObservable } from "observable-hooks";
import { map } from "rxjs";
import inCallStyles from "./InCallView.module.css";
import styles from "./LobbyView.module.css";
@@ -23,12 +28,16 @@ import {
EndCallButton,
MicButton,
SettingsButton,
SwitchCameraButton,
VideoButton,
} from "../button/Button";
import { SettingsModal, defaultSettingsTab } from "../settings/SettingsModal";
import { useMediaQuery } from "../useMediaQuery";
import { E2eeType } from "../e2ee/e2eeType";
import { Link } from "../button/Link";
import { useMediaDevices } from "../livekit/MediaDevicesContext";
import { useInitial } from "../useInitial";
import { useSwitchCamera } from "./useSwitchCamera";
interface Props {
client: MatrixClient;
@@ -89,6 +98,61 @@ export const LobbyView: FC<Props> = ({
</Link>
);
const devices = useMediaDevices();
// Capture the audio options as they were when we first mounted, because
// we're not doing anything with the audio anyway so we don't need to
// re-open the devices when they change (see below).
const initialAudioOptions = useInitial(
() =>
muteStates.audio.enabled && { deviceId: devices.audioInput.selectedId },
);
const localTrackOptions = useMemo(
() => ({
// The only reason we request audio here is to get the audio permission
// request over with at the same time. But changing the audio settings
// shouldn't cause this hook to recreate the track, which is why we
// reference the initial values here.
// We also pass in a clone because livekit mutates the object passed in,
// which would cause the devices to be re-opened on the next render.
audio: Object.assign({}, initialAudioOptions),
video: muteStates.video.enabled && {
deviceId: devices.videoInput.selectedId,
},
}),
[
initialAudioOptions,
devices.videoInput.selectedId,
muteStates.video.enabled,
],
);
const onError = useCallback(
(error: Error) => {
logger.error("Error while creating preview Tracks:", error);
muteStates.audio.setEnabled?.(false);
muteStates.video.setEnabled?.(false);
},
[muteStates.audio, muteStates.video],
);
const tracks = usePreviewTracks(localTrackOptions, onError);
const videoTrack = useMemo(
() =>
(tracks?.find((t) => t.kind === Track.Kind.Video) ??
null) as LocalVideoTrack | null,
[tracks],
);
const switchCamera = useSwitchCamera(
useObservable(
(inputs) => inputs.pipe(map(([video]) => video)),
[videoTrack],
),
);
// TODO: Unify this component with InCallView, so we can get slick joining
// animations and don't have to feel bad about reusing its CSS
return (
@@ -111,7 +175,11 @@ export const LobbyView: FC<Props> = ({
</Header>
)}
<div className={styles.content}>
<VideoPreview matrixInfo={matrixInfo} muteStates={muteStates}>
<VideoPreview
matrixInfo={matrixInfo}
muteStates={muteStates}
videoTrack={videoTrack}
>
<Button
className={classNames(styles.join, {
[styles.wait]: waitingForInvite,
@@ -140,6 +208,7 @@ export const LobbyView: FC<Props> = ({
onClick={onVideoPress}
disabled={muteStates.video.setEnabled === null}
/>
{switchCamera && <SwitchCameraButton onClick={switchCamera} />}
<SettingsButton onClick={openSettings} />
{!confineToRoom && <EndCallButton onClick={onLeaveClick} />}
</div>

View File

@@ -27,6 +27,7 @@ import { E2eeType } from "../e2ee/e2eeType";
import { useProfile } from "../profile/useProfile";
import { useMuteStates } from "./MuteStates";
import { useOptInAnalytics } from "../settings/settings";
import { Config } from "../config/Config";
export const RoomPage: FC = () => {
const {
@@ -185,6 +186,7 @@ export const RoomPage: FC = () => {
{content}
{/* On Android and iOS, show a prompt to launch the mobile app. */}
{appPrompt &&
Config.get().app_prompt &&
(platform === "android" || platform === "ios") &&
roomId && <AppSelectionModal roomId={roomId} />}
</>

View File

@@ -19,10 +19,13 @@ Please see LICENSE in the repository root for full details.
height: 100%;
object-fit: cover;
background-color: black;
transform: scaleX(-1);
background-color: var(--cpd-color-bg-subtle-primary);
}
video.mirror {
transform: scaleX(-1);
}
.avatarContainer {
position: absolute;
top: 0;

View File

@@ -5,18 +5,14 @@ SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { useEffect, useMemo, useRef, FC, ReactNode, useCallback } from "react";
import { useEffect, useRef, FC, ReactNode } from "react";
import useMeasure from "react-use-measure";
import { usePreviewTracks } from "@livekit/components-react";
import { LocalVideoTrack, Track } from "livekit-client";
import { facingModeFromLocalTrack, LocalVideoTrack } from "livekit-client";
import classNames from "classnames";
import { logger } from "matrix-js-sdk/src/logger";
import { Avatar } from "../Avatar";
import styles from "./VideoPreview.module.css";
import { useMediaDevices } from "../livekit/MediaDevicesContext";
import { MuteStates } from "./MuteStates";
import { useInitial } from "../useInitial";
import { EncryptionSystem } from "../e2ee/sharedKeyManagement";
export type MatrixInfo = {
@@ -33,65 +29,18 @@ export type MatrixInfo = {
interface Props {
matrixInfo: MatrixInfo;
muteStates: MuteStates;
videoTrack: LocalVideoTrack | null;
children: ReactNode;
}
export const VideoPreview: FC<Props> = ({
matrixInfo,
muteStates,
videoTrack,
children,
}) => {
const [previewRef, previewBounds] = useMeasure();
const devices = useMediaDevices();
// Capture the audio options as they were when we first mounted, because
// we're not doing anything with the audio anyway so we don't need to
// re-open the devices when they change (see below).
const initialAudioOptions = useInitial(
() =>
muteStates.audio.enabled && { deviceId: devices.audioInput.selectedId },
);
const localTrackOptions = useMemo(
() => ({
// The only reason we request audio here is to get the audio permission
// request over with at the same time. But changing the audio settings
// shouldn't cause this hook to recreate the track, which is why we
// reference the initial values here.
// We also pass in a clone because livekit mutates the object passed in,
// which would cause the devices to be re-opened on the next render.
audio: Object.assign({}, initialAudioOptions),
video: muteStates.video.enabled && {
deviceId: devices.videoInput.selectedId,
},
}),
[
initialAudioOptions,
devices.videoInput.selectedId,
muteStates.video.enabled,
],
);
const onError = useCallback(
(error: Error) => {
logger.error("Error while creating preview Tracks:", error);
muteStates.audio.setEnabled?.(false);
muteStates.video.setEnabled?.(false);
},
[muteStates.audio, muteStates.video],
);
const tracks = usePreviewTracks(localTrackOptions, onError);
const videoTrack = useMemo(
() =>
tracks?.find((t) => t.kind === Track.Kind.Video) as
| LocalVideoTrack
| undefined,
[tracks],
);
const videoEl = useRef<HTMLVideoElement | null>(null);
useEffect(() => {
@@ -107,6 +56,12 @@ export const VideoPreview: FC<Props> = ({
return (
<div className={classNames(styles.preview)} ref={previewRef}>
<video
className={
videoTrack &&
facingModeFromLocalTrack(videoTrack).facingMode === "user"
? styles.mirror
: undefined
}
ref={videoEl}
muted
playsInline

View File

@@ -0,0 +1,98 @@
/*
Copyright 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import {
fromEvent,
map,
merge,
Observable,
of,
startWith,
switchMap,
} from "rxjs";
import {
facingModeFromLocalTrack,
LocalVideoTrack,
TrackEvent,
} from "livekit-client";
import { useObservable, useObservableEagerState } from "observable-hooks";
import { useEffect } from "react";
import { logger } from "matrix-js-sdk/src/logger";
import { useMediaDevices } from "../livekit/MediaDevicesContext";
import { platform } from "../Platform";
import { useLatest } from "../useLatest";
/**
* Determines whether the user should be shown a button to switch their camera,
* producing a callback if so.
*/
export function useSwitchCamera(
video: Observable<LocalVideoTrack | null>,
): (() => void) | null {
const mediaDevices = useMediaDevices();
// Produce an observable like the input 'video' observable, except make it
// emit whenever the track is muted or the device changes
const videoTrack: Observable<LocalVideoTrack | null> = useObservable(
(inputs) =>
inputs.pipe(
switchMap(([video]) => video),
switchMap((video) => {
if (video === null) return of(null);
return merge(
fromEvent(video, TrackEvent.Restarted).pipe(
startWith(null),
map(() => video),
),
fromEvent(video, TrackEvent.Muted).pipe(map(() => null)),
);
}),
),
[video],
);
const switchCamera: Observable<(() => void) | null> = useObservable(
(inputs) =>
platform === "desktop"
? of(null)
: inputs.pipe(
switchMap(([track]) => track),
map((track) => {
if (track === null) return null;
const facingMode = facingModeFromLocalTrack(track).facingMode;
// If the camera isn't front or back-facing, don't provide a switch
// camera shortcut at all
if (facingMode !== "user" && facingMode !== "environment")
return null;
// Restart the track with a camera facing the opposite direction
return (): void =>
void track
.restartTrack({
facingMode: facingMode === "user" ? "environment" : "user",
})
.catch((e) =>
logger.error("Failed to switch camera", facingMode, e),
);
}),
),
[videoTrack],
);
const setVideoInput = useLatest(mediaDevices.videoInput.select);
useEffect(() => {
// Watch for device changes due to switching the camera and feed them back
// into the MediaDeviceContext
const subscription = videoTrack.subscribe((track) => {
const deviceId = track?.mediaStreamTrack.getSettings().deviceId;
if (deviceId !== undefined) setVideoInput.current(deviceId);
});
return (): void => subscription.unsubscribe();
}, [videoTrack, setVideoInput]);
return useObservableEagerState(switchCamera);
}

View File

@@ -13,8 +13,10 @@ import {
import {
Room as LivekitRoom,
LocalParticipant,
LocalVideoTrack,
ParticipantEvent,
RemoteParticipant,
Track,
} from "livekit-client";
import {
Room as MatrixRoom,
@@ -58,6 +60,7 @@ import {
import {
LocalUserMediaViewModel,
MediaViewModel,
observeTrackReference,
RemoteUserMediaViewModel,
ScreenShareViewModel,
UserMediaViewModel,
@@ -259,6 +262,17 @@ function findMatrixMember(
// TODO: Move wayyyy more business logic from the call and lobby views into here
export class CallViewModel extends ViewModel {
public readonly localVideo: Observable<LocalVideoTrack | null> =
observeTrackReference(
this.livekitRoom.localParticipant,
Track.Source.Camera,
).pipe(
map((trackRef) => {
const track = trackRef.publication?.track;
return track instanceof LocalVideoTrack ? track : null;
}),
);
private readonly rawRemoteParticipants = connectedParticipantsObserver(
this.livekitRoom,
).pipe(this.scope.state());

View File

@@ -65,7 +65,7 @@ export function useDisplayName(vm: MediaViewModel): string {
return displayName;
}
function observeTrackReference(
export function observeTrackReference(
participant: Participant,
source: Track.Source,
): Observable<TrackReferenceOrPlaceholder> {

View File

@@ -168,7 +168,7 @@ UserMediaTile.displayName = "UserMediaTile";
interface LocalUserMediaTileProps extends TileProps {
vm: LocalUserMediaViewModel;
onOpenProfile: () => void;
onOpenProfile: (() => void) | null;
}
const LocalUserMediaTile = forwardRef<HTMLDivElement, LocalUserMediaTileProps>(
@@ -199,11 +199,13 @@ const LocalUserMediaTile = forwardRef<HTMLDivElement, LocalUserMediaTileProps>(
/>
}
menuEnd={
<MenuItem
Icon={UserProfileIcon}
label={t("common.profile")}
onSelect={onOpenProfile}
/>
onOpenProfile && (
<MenuItem
Icon={UserProfileIcon}
label={t("common.profile")}
onSelect={onOpenProfile}
/>
)
}
{...props}
/>
@@ -276,7 +278,7 @@ RemoteUserMediaTile.displayName = "RemoteUserMediaTile";
interface GridTileProps {
vm: UserMediaViewModel;
onOpenProfile: () => void;
onOpenProfile: (() => void) | null;
targetWidth: number;
targetHeight: number;
className?: string;

View File

@@ -5676,9 +5676,9 @@ lines-and-columns@^1.1.6:
integrity sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==
livekit-client@^2.5.7:
version "2.5.10"
resolved "https://registry.yarnpkg.com/livekit-client/-/livekit-client-2.5.10.tgz#ce0b70fe9b199760a159ffb99536bc3b6492ab64"
integrity sha512-H7EeIb19LAH8ejlvhh0JWtWkvXDan6Yf3bpFGlDMb54uPmyRgBY+McfgQsFgJCB9WJL0X+GYUoV1Cmnn8iAoIQ==
version "2.6.0"
resolved "https://registry.yarnpkg.com/livekit-client/-/livekit-client-2.6.0.tgz#7506838fb59937150208b8e593701cde19f3fb75"
integrity sha512-hpxNBtyWIFCefoHjHoSjqPCw3m7AfSJVcVZw6rMsqds4u+dSpWLfYkglWP8JuPGUIssyOsZm/+bV3gBWfuOGGQ==
dependencies:
"@livekit/mutex" "1.0.0"
"@livekit/protocol" "1.24.0"