mirror of
https://github.com/vector-im/element-call.git
synced 2026-10-16 23:45:50 +00:00
Show the wait for the first background effect on its tile
- The first effect chosen in a session shows a spinner on its own tile, in place of the tick, until a frame carrying it is drawn. Later changes show nothing, having nothing to prepare. - Ended by the frame, not the promises, which resolve while the segmenter is still being built. The transformer reports the first frame that went through the segmenter; one passed through untouched doesn't count. - The pipeline publishes the wait with its state, and both footer view models take it as an input. - The lobby tests' provider mock gains the observable hook, built once: a fresh observable per render rebuilt the footer on every render. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -136,6 +136,7 @@ export const Default: Story = {
|
|||||||
})),
|
})),
|
||||||
],
|
],
|
||||||
backgroundEffectNotice: undefined,
|
backgroundEffectNotice: undefined,
|
||||||
|
backgroundEffectSettling: false,
|
||||||
hangup: fn(),
|
hangup: fn(),
|
||||||
buttonSize: "lg",
|
buttonSize: "lg",
|
||||||
showFooter: true,
|
showFooter: true,
|
||||||
|
|||||||
@@ -87,6 +87,8 @@ export interface FooterState {
|
|||||||
backgroundEffects: BackgroundEffectChoice[];
|
backgroundEffects: BackgroundEffectChoice[];
|
||||||
/** What the user is told about effects here, if anything. */
|
/** What the user is told about effects here, if anything. */
|
||||||
backgroundEffectNotice: "unavailable" | "slow" | undefined;
|
backgroundEffectNotice: "unavailable" | "slow" | undefined;
|
||||||
|
/** Whether the first effect chosen is still being prepared. */
|
||||||
|
backgroundEffectSettling: boolean;
|
||||||
showFooter: boolean;
|
showFooter: boolean;
|
||||||
|
|
||||||
/* This is needed for WindowMode = "flat" */
|
/* This is needed for WindowMode = "flat" */
|
||||||
@@ -173,6 +175,7 @@ export const CallFooter: FC<FooterProps> = ({
|
|||||||
const backgroundEffect = useBehavior(vm.backgroundEffect$);
|
const backgroundEffect = useBehavior(vm.backgroundEffect$);
|
||||||
const selectBackgroundEffect = useBehavior(vm.selectBackgroundEffect$);
|
const selectBackgroundEffect = useBehavior(vm.selectBackgroundEffect$);
|
||||||
const backgroundEffectNotice = useBehavior(vm.backgroundEffectNotice$);
|
const backgroundEffectNotice = useBehavior(vm.backgroundEffectNotice$);
|
||||||
|
const backgroundEffectSettling = useBehavior(vm.backgroundEffectSettling$);
|
||||||
const backgroundEffects = useBackgroundEffectLabels(
|
const backgroundEffects = useBackgroundEffectLabels(
|
||||||
useBehavior(vm.backgroundEffects$),
|
useBehavior(vm.backgroundEffects$),
|
||||||
);
|
);
|
||||||
@@ -240,6 +243,7 @@ export const CallFooter: FC<FooterProps> = ({
|
|||||||
backgroundEffects={backgroundEffects}
|
backgroundEffects={backgroundEffects}
|
||||||
selectedBackgroundEffect={backgroundEffect}
|
selectedBackgroundEffect={backgroundEffect}
|
||||||
onSelectBackgroundEffect={selectBackgroundEffect}
|
onSelectBackgroundEffect={selectBackgroundEffect}
|
||||||
|
backgroundEffectSettling={backgroundEffectSettling}
|
||||||
backgroundEffectNotice={
|
backgroundEffectNotice={
|
||||||
backgroundEffectNotice === "unavailable"
|
backgroundEffectNotice === "unavailable"
|
||||||
? t("background_effects.unavailable")
|
? t("background_effects.unavailable")
|
||||||
|
|||||||
@@ -116,6 +116,7 @@ describe("createCallFooterViewModel", () => {
|
|||||||
buildMinimalCallViewModel(gridLayout),
|
buildMinimalCallViewModel(gridLayout),
|
||||||
mockMuteStates(),
|
mockMuteStates(),
|
||||||
twoMicsAndOneCamMediaDevices,
|
twoMicsAndOneCamMediaDevices,
|
||||||
|
constant(false),
|
||||||
/* reactionIdentifier */ undefined,
|
/* reactionIdentifier */ undefined,
|
||||||
{ showControls: true, header: HeaderStyle.Standard },
|
{ showControls: true, header: HeaderStyle.Standard },
|
||||||
);
|
);
|
||||||
@@ -158,6 +159,7 @@ describe("createCallFooterViewModel", () => {
|
|||||||
select: vi.fn(),
|
select: vi.fn(),
|
||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
|
constant(false),
|
||||||
/* reactionIdentifier */ undefined,
|
/* reactionIdentifier */ undefined,
|
||||||
{ showControls: true, header: HeaderStyle.Standard },
|
{ showControls: true, header: HeaderStyle.Standard },
|
||||||
);
|
);
|
||||||
@@ -176,6 +178,7 @@ describe("createCallFooterViewModel", () => {
|
|||||||
buildMinimalCallViewModel(layout),
|
buildMinimalCallViewModel(layout),
|
||||||
mockMuteStates(),
|
mockMuteStates(),
|
||||||
twoMicsAndOneCamMediaDevices,
|
twoMicsAndOneCamMediaDevices,
|
||||||
|
constant(false),
|
||||||
/* reactionIdentifier */ undefined,
|
/* reactionIdentifier */ undefined,
|
||||||
{ showControls: true, header: HeaderStyle.Standard },
|
{ showControls: true, header: HeaderStyle.Standard },
|
||||||
);
|
);
|
||||||
@@ -198,6 +201,7 @@ describe("createCallFooterViewModel", () => {
|
|||||||
buildMinimalCallViewModel(gridLayout),
|
buildMinimalCallViewModel(gridLayout),
|
||||||
mockMuteStates(),
|
mockMuteStates(),
|
||||||
twoMicsAndOneCamMediaDevices,
|
twoMicsAndOneCamMediaDevices,
|
||||||
|
constant(false),
|
||||||
/* reactionIdentifier */ undefined,
|
/* reactionIdentifier */ undefined,
|
||||||
{ showControls: true, header: HeaderStyle.Standard },
|
{ showControls: true, header: HeaderStyle.Standard },
|
||||||
);
|
);
|
||||||
@@ -243,6 +247,7 @@ describe("createCallFooterViewModel", () => {
|
|||||||
testScope(),
|
testScope(),
|
||||||
mockMuteStates(),
|
mockMuteStates(),
|
||||||
twoMicsAndOneCamMediaDevices,
|
twoMicsAndOneCamMediaDevices,
|
||||||
|
constant(false),
|
||||||
/* openSettings */ undefined,
|
/* openSettings */ undefined,
|
||||||
/* hangup */ undefined,
|
/* hangup */ undefined,
|
||||||
/* showLogo */ false,
|
/* showLogo */ false,
|
||||||
@@ -312,6 +317,23 @@ describe("createCallFooterViewModel", () => {
|
|||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("passes on the wait for the first effect", () => {
|
||||||
|
platformMock.mockReturnValue("desktop");
|
||||||
|
const settling$ = new BehaviorSubject(true);
|
||||||
|
const vm = createLobbyFooterViewModel(
|
||||||
|
testScope(),
|
||||||
|
mockMuteStates(),
|
||||||
|
twoMicsAndOneCamMediaDevices,
|
||||||
|
settling$,
|
||||||
|
/* openSettings */ undefined,
|
||||||
|
/* hangup */ undefined,
|
||||||
|
/* showLogo */ false,
|
||||||
|
);
|
||||||
|
expect(vm.backgroundEffectSettling$.value).toBe(true);
|
||||||
|
settling$.next(false);
|
||||||
|
expect(vm.backgroundEffectSettling$.value).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
it("availability is the same before and during a call", () => {
|
it("availability is the same before and during a call", () => {
|
||||||
for (const supported of [true, false]) {
|
for (const supported of [true, false]) {
|
||||||
sdkSupportMock.mockReturnValue(supported);
|
sdkSupportMock.mockReturnValue(supported);
|
||||||
@@ -321,6 +343,7 @@ describe("createCallFooterViewModel", () => {
|
|||||||
buildMinimalCallViewModel(gridLayout),
|
buildMinimalCallViewModel(gridLayout),
|
||||||
mockMuteStates(),
|
mockMuteStates(),
|
||||||
twoMicsAndOneCamMediaDevices,
|
twoMicsAndOneCamMediaDevices,
|
||||||
|
constant(false),
|
||||||
/* reactionIdentifier */ undefined,
|
/* reactionIdentifier */ undefined,
|
||||||
{ showControls: true, header: HeaderStyle.Standard },
|
{ showControls: true, header: HeaderStyle.Standard },
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -5,7 +5,13 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
|||||||
Please see LICENSE in the repository root for full details.
|
Please see LICENSE in the repository root for full details.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { combineLatest, map, type Observable, switchMap } from "rxjs";
|
import {
|
||||||
|
combineLatest,
|
||||||
|
distinctUntilChanged,
|
||||||
|
map,
|
||||||
|
type Observable,
|
||||||
|
switchMap,
|
||||||
|
} from "rxjs";
|
||||||
import { supportsAudioOutputSelection } from "livekit-client";
|
import { supportsAudioOutputSelection } from "livekit-client";
|
||||||
|
|
||||||
import {
|
import {
|
||||||
@@ -72,6 +78,7 @@ function buildDeviceBehaviors(
|
|||||||
mediaDevices: MediaDevices,
|
mediaDevices: MediaDevices,
|
||||||
/** return empty arrays for audioOptions and videoOptions*/
|
/** return empty arrays for audioOptions and videoOptions*/
|
||||||
disableSwitcher$: Behavior<boolean>,
|
disableSwitcher$: Behavior<boolean>,
|
||||||
|
backgroundEffectSettling$: Observable<boolean>,
|
||||||
): Pick<
|
): Pick<
|
||||||
ViewModel<FooterSnapshot>,
|
ViewModel<FooterSnapshot>,
|
||||||
| "audioOptions$"
|
| "audioOptions$"
|
||||||
@@ -87,6 +94,7 @@ function buildDeviceBehaviors(
|
|||||||
| "selectBackgroundEffect$"
|
| "selectBackgroundEffect$"
|
||||||
| "backgroundEffects$"
|
| "backgroundEffects$"
|
||||||
| "backgroundEffectNotice$"
|
| "backgroundEffectNotice$"
|
||||||
|
| "backgroundEffectSettling$"
|
||||||
> {
|
> {
|
||||||
const options$ = (
|
const options$ = (
|
||||||
available$: Behavior<Map<string, MenuOptions["label"]>>,
|
available$: Behavior<Map<string, MenuOptions["label"]>>,
|
||||||
@@ -157,6 +165,10 @@ function buildDeviceBehaviors(
|
|||||||
),
|
),
|
||||||
),
|
),
|
||||||
),
|
),
|
||||||
|
backgroundEffectSettling$: scope.behavior(
|
||||||
|
backgroundEffectSettling$.pipe(distinctUntilChanged()),
|
||||||
|
false,
|
||||||
|
),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -167,6 +179,8 @@ function buildDeviceBehaviors(
|
|||||||
* @param callModel - The root CallViewModel; provides layout, grid mode, reactions, etc.
|
* @param callModel - The root CallViewModel; provides layout, grid mode, reactions, etc.
|
||||||
* @param muteStates - Audio and video mute state + toggles.
|
* @param muteStates - Audio and video mute state + toggles.
|
||||||
* @param mediaDevices - Available and selected input devices.
|
* @param mediaDevices - Available and selected input devices.
|
||||||
|
* @param backgroundEffectSettling$ - Whether the first effect chosen is still
|
||||||
|
* being prepared.
|
||||||
* @param reactionIdentifier - The local user's reaction identifier string, or
|
* @param reactionIdentifier - The local user's reaction identifier string, or
|
||||||
* undefined when reactions are not supported (hides the reaction button).
|
* undefined when reactions are not supported (hides the reaction button).
|
||||||
* @param options - `showControls`: whether the call controls should be shown.
|
* @param options - `showControls`: whether the call controls should be shown.
|
||||||
@@ -177,6 +191,7 @@ export function createCallFooterViewModel(
|
|||||||
callModel: CallViewModel,
|
callModel: CallViewModel,
|
||||||
muteStates: MuteStates,
|
muteStates: MuteStates,
|
||||||
mediaDevices: MediaDevices,
|
mediaDevices: MediaDevices,
|
||||||
|
backgroundEffectSettling$: Observable<boolean>,
|
||||||
reactionIdentifier: string | undefined,
|
reactionIdentifier: string | undefined,
|
||||||
options: { showControls: boolean; header: HeaderStyle },
|
options: { showControls: boolean; header: HeaderStyle },
|
||||||
): ViewModel<FooterSnapshot> {
|
): ViewModel<FooterSnapshot> {
|
||||||
@@ -191,7 +206,12 @@ export function createCallFooterViewModel(
|
|||||||
);
|
);
|
||||||
return {
|
return {
|
||||||
...buildMuteBehaviors(scope, muteStates),
|
...buildMuteBehaviors(scope, muteStates),
|
||||||
...buildDeviceBehaviors(scope, mediaDevices, disableDeviceSwitcher$),
|
...buildDeviceBehaviors(
|
||||||
|
scope,
|
||||||
|
mediaDevices,
|
||||||
|
disableDeviceSwitcher$,
|
||||||
|
backgroundEffectSettling$,
|
||||||
|
),
|
||||||
// candidat to move into the FooterViewModel
|
// candidat to move into the FooterViewModel
|
||||||
showFooter$: callModel.showFooter$,
|
showFooter$: callModel.showFooter$,
|
||||||
hideControls$: constant(!showControls),
|
hideControls$: constant(!showControls),
|
||||||
@@ -253,6 +273,8 @@ export function createCallFooterViewModel(
|
|||||||
* @param scope - ObservableScope that bounds the lifetime of derived behaviors.
|
* @param scope - ObservableScope that bounds the lifetime of derived behaviors.
|
||||||
* @param muteStates - Audio and video mute state + toggles.
|
* @param muteStates - Audio and video mute state + toggles.
|
||||||
* @param mediaDevices - Available and selected input devices.
|
* @param mediaDevices - Available and selected input devices.
|
||||||
|
* @param backgroundEffectSettling$ - Whether the first effect chosen is still
|
||||||
|
* being prepared.
|
||||||
* @param openSettings - Callback to open the settings modal, or undefined.
|
* @param openSettings - Callback to open the settings modal, or undefined.
|
||||||
* @param hangup - Callback to leave/cancel, or undefined (hides the button).
|
* @param hangup - Callback to leave/cancel, or undefined (hides the button).
|
||||||
* @param showLogo - Whether to show the Element Call logo.
|
* @param showLogo - Whether to show the Element Call logo.
|
||||||
@@ -261,6 +283,7 @@ export function createLobbyFooterViewModel(
|
|||||||
scope: ObservableScope,
|
scope: ObservableScope,
|
||||||
muteStates: MuteStates,
|
muteStates: MuteStates,
|
||||||
mediaDevices: MediaDevices,
|
mediaDevices: MediaDevices,
|
||||||
|
backgroundEffectSettling$: Observable<boolean>,
|
||||||
openSettings: (() => void) | undefined,
|
openSettings: (() => void) | undefined,
|
||||||
hangup: (() => void) | undefined,
|
hangup: (() => void) | undefined,
|
||||||
showLogo: boolean,
|
showLogo: boolean,
|
||||||
@@ -304,7 +327,12 @@ export function createLobbyFooterViewModel(
|
|||||||
selectVideoButtonOption: undefined,
|
selectVideoButtonOption: undefined,
|
||||||
}),
|
}),
|
||||||
...buildMuteBehaviors(scope, muteStates),
|
...buildMuteBehaviors(scope, muteStates),
|
||||||
...buildDeviceBehaviors(scope, mediaDevices, constant(false)),
|
...buildDeviceBehaviors(
|
||||||
|
scope,
|
||||||
|
mediaDevices,
|
||||||
|
constant(false),
|
||||||
|
backgroundEffectSettling$,
|
||||||
|
),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -305,6 +305,33 @@ export const BackgroundEffectsSlowInThisBrowser: Story = {
|
|||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/** The first effect of a session shows its wait on its own tile. */
|
||||||
|
export const BackgroundEffectsSettling: Story = {
|
||||||
|
args: {
|
||||||
|
...VideoUnmute.args,
|
||||||
|
selectedBackgroundEffect: "blur",
|
||||||
|
backgroundEffectSettling: true,
|
||||||
|
},
|
||||||
|
play: async ({ canvasElement }) => {
|
||||||
|
const canvas = within(canvasElement);
|
||||||
|
await userEvent.click(canvas.getByRole("button", { name: "Camera" }));
|
||||||
|
const menu = within(document.body);
|
||||||
|
const chosen = await menu.findByRole("menuitemradio", {
|
||||||
|
name: "Blur",
|
||||||
|
checked: true,
|
||||||
|
});
|
||||||
|
await expect(chosen).toHaveAttribute("aria-busy", "true");
|
||||||
|
const marks = chosen.querySelectorAll("svg");
|
||||||
|
await expect(marks).toHaveLength(1);
|
||||||
|
await expect(getComputedStyle(marks[0]).animationName).not.toBe("none");
|
||||||
|
const section = menu.getByRole("group", { name: "Background effects" });
|
||||||
|
for (const other of within(section).getAllByRole("menuitemradio", {
|
||||||
|
checked: false,
|
||||||
|
}))
|
||||||
|
await expect(other).toHaveAttribute("aria-busy", "false");
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
/** In a short call the effects scroll into view with the list. */
|
/** In a short call the effects scroll into view with the list. */
|
||||||
export const BackgroundEffectsScrollWhenTheyDoNotFit: Story = {
|
export const BackgroundEffectsScrollWhenTheyDoNotFit: Story = {
|
||||||
args: {
|
args: {
|
||||||
|
|||||||
@@ -77,6 +77,8 @@ export interface MediaMuteAndSwitchButtonProps {
|
|||||||
onSelectBackgroundEffect?: (id: string) => void;
|
onSelectBackgroundEffect?: (id: string) => void;
|
||||||
/** What the user should know before choosing an effect, if anything. */
|
/** What the user should know before choosing an effect, if anything. */
|
||||||
backgroundEffectNotice?: string;
|
backgroundEffectNotice?: string;
|
||||||
|
/** Whether the first effect chosen is still being prepared. */
|
||||||
|
backgroundEffectSettling?: boolean;
|
||||||
/**
|
/**
|
||||||
* For any toggle and option this method will be called.
|
* For any toggle and option this method will be called.
|
||||||
* So toggles need to be implemented by listening here and setting the right toggle item to `enabled`
|
* So toggles need to be implemented by listening here and setting the right toggle item to `enabled`
|
||||||
@@ -110,6 +112,7 @@ export const MediaMuteAndSwitchButton: FC<MediaMuteAndSwitchButtonProps> = ({
|
|||||||
selectedBackgroundEffect,
|
selectedBackgroundEffect,
|
||||||
onSelectBackgroundEffect,
|
onSelectBackgroundEffect,
|
||||||
backgroundEffectNotice,
|
backgroundEffectNotice,
|
||||||
|
backgroundEffectSettling,
|
||||||
onSelect,
|
onSelect,
|
||||||
}) => {
|
}) => {
|
||||||
// Requested but not yet selected. Keyed by kind too, since Chrome uses
|
// Requested but not yet selected. Keyed by kind too, since Chrome uses
|
||||||
@@ -422,6 +425,7 @@ export const MediaMuteAndSwitchButton: FC<MediaMuteAndSwitchButtonProps> = ({
|
|||||||
effects={backgroundEffects ?? []}
|
effects={backgroundEffects ?? []}
|
||||||
selected={selectedBackgroundEffect}
|
selected={selectedBackgroundEffect}
|
||||||
onSelect={onSelectBackgroundEffect}
|
onSelect={onSelectBackgroundEffect}
|
||||||
|
settling={backgroundEffectSettling}
|
||||||
describedBy={
|
describedBy={
|
||||||
backgroundEffectNotice === undefined ? undefined : noticeId
|
backgroundEffectNotice === undefined ? undefined : noticeId
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -15,6 +15,9 @@ import { ImageSegmenter } from "@mediapipe/tasks-vision";
|
|||||||
|
|
||||||
import { BackgroundEffectTransformer } from "./BackgroundEffectTransformer";
|
import { BackgroundEffectTransformer } from "./BackgroundEffectTransformer";
|
||||||
|
|
||||||
|
const frame = {} as VideoFrame;
|
||||||
|
const controller = {} as TransformStreamDefaultController<VideoFrame>;
|
||||||
|
|
||||||
describe("BackgroundEffectTransformer", () => {
|
describe("BackgroundEffectTransformer", () => {
|
||||||
afterEach(() => vi.restoreAllMocks());
|
afterEach(() => vi.restoreAllMocks());
|
||||||
|
|
||||||
@@ -38,6 +41,32 @@ describe("BackgroundEffectTransformer", () => {
|
|||||||
expect(load).toHaveBeenCalledWith("/background.jpg");
|
expect(load).toHaveBeenCalledWith("/background.jpg");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("reports the first frame carrying an effect, once", async () => {
|
||||||
|
vi.spyOn(BackgroundTransformer.prototype, "transform").mockImplementation(
|
||||||
|
async function (this: BackgroundTransformer) {
|
||||||
|
this.isFirstFrame = false;
|
||||||
|
return Promise.resolve();
|
||||||
|
},
|
||||||
|
);
|
||||||
|
const transformer = new BackgroundEffectTransformer({});
|
||||||
|
transformer.onFirstFrame = vi.fn();
|
||||||
|
|
||||||
|
await transformer.transform(frame, controller);
|
||||||
|
await transformer.transform(frame, controller);
|
||||||
|
expect(transformer.onFirstFrame).toHaveBeenCalledTimes(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not report a frame passed through untouched", async () => {
|
||||||
|
vi.spyOn(BackgroundTransformer.prototype, "transform").mockResolvedValue();
|
||||||
|
const transformer = new BackgroundEffectTransformer({
|
||||||
|
backgroundDisabled: true,
|
||||||
|
});
|
||||||
|
transformer.onFirstFrame = vi.fn();
|
||||||
|
|
||||||
|
await transformer.transform(frame, controller);
|
||||||
|
expect(transformer.onFirstFrame).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
it("passes frames on untouched once disabled, even after blurring", async () => {
|
it("passes frames on untouched once disabled, even after blurring", async () => {
|
||||||
const transformer = new BackgroundEffectTransformer({});
|
const transformer = new BackgroundEffectTransformer({});
|
||||||
await transformer.update({ blurRadius: 15, backgroundDisabled: false });
|
await transformer.update({ blurRadius: 15, backgroundDisabled: false });
|
||||||
|
|||||||
@@ -53,6 +53,20 @@ const wasmFileset: WasmFileset = {
|
|||||||
* resource fetched from the public internet.
|
* resource fetched from the public internet.
|
||||||
*/
|
*/
|
||||||
export class BackgroundEffectTransformer extends BackgroundTransformer {
|
export class BackgroundEffectTransformer extends BackgroundTransformer {
|
||||||
|
/** Called once, when the first frame carrying an effect has been drawn. */
|
||||||
|
public onFirstFrame: (() => void) | undefined;
|
||||||
|
|
||||||
|
public override async transform(
|
||||||
|
frame: VideoFrame,
|
||||||
|
controller: TransformStreamDefaultController<VideoFrame>,
|
||||||
|
): Promise<void> {
|
||||||
|
const priming = this.isFirstFrame;
|
||||||
|
await super.transform(frame, controller);
|
||||||
|
// Cleared only once a frame has been through the segmenter; a frame
|
||||||
|
// passed through untouched leaves it set.
|
||||||
|
if (priming && !this.isFirstFrame) this.onFirstFrame?.();
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* As the library's, except that disabling also drops the blur radius: kept,
|
* As the library's, except that disabling also drops the blur radius: kept,
|
||||||
* it has every frame segmented and the result thrown away, where with
|
* it has every frame segmented and the result thrown away, where with
|
||||||
|
|||||||
@@ -34,6 +34,8 @@ import { useBehavior } from "../useBehavior";
|
|||||||
export type BackgroundEffectsState = {
|
export type BackgroundEffectsState = {
|
||||||
supported: boolean | undefined;
|
supported: boolean | undefined;
|
||||||
processor: undefined | ProcessorWrapper<BackgroundOptions>;
|
processor: undefined | ProcessorWrapper<BackgroundOptions>;
|
||||||
|
/** From the first effect chosen until a frame carrying it is drawn. */
|
||||||
|
settling?: boolean;
|
||||||
};
|
};
|
||||||
|
|
||||||
const BackgroundEffectsContext = createContext<BackgroundEffects | undefined>(
|
const BackgroundEffectsContext = createContext<BackgroundEffects | undefined>(
|
||||||
@@ -115,13 +117,14 @@ export const useSyncBackgroundEffects = (
|
|||||||
export function createAppBackgroundEffects(
|
export function createAppBackgroundEffects(
|
||||||
scope: ObservableScope,
|
scope: ObservableScope,
|
||||||
): BackgroundEffects {
|
): BackgroundEffects {
|
||||||
|
const transformer = new BackgroundEffectTransformer({
|
||||||
|
backgroundDisabled: true,
|
||||||
|
});
|
||||||
return createBackgroundEffects(scope, {
|
return createBackgroundEffects(scope, {
|
||||||
supported: supportsBackgroundProcessors(),
|
supported: supportsBackgroundProcessors(),
|
||||||
effect$: backgroundEffectSetting.value$,
|
effect$: backgroundEffectSetting.value$,
|
||||||
pipeline: new OneStepPipeline(
|
pipeline: new OneStepPipeline(transformer, "background-effect"),
|
||||||
new BackgroundEffectTransformer({ backgroundDisabled: true }),
|
transformer,
|
||||||
"background-effect",
|
|
||||||
),
|
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -22,6 +22,7 @@ import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc";
|
|||||||
import classNames from "classnames";
|
import classNames from "classnames";
|
||||||
import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
|
import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
|
||||||
import { useTranslation } from "react-i18next";
|
import { useTranslation } from "react-i18next";
|
||||||
|
import { map } from "rxjs";
|
||||||
|
|
||||||
import { Header, LeftNav, RightNav, RoomHeaderInfo } from "../Header";
|
import { Header, LeftNav, RightNav, RoomHeaderInfo } from "../Header";
|
||||||
import { HeaderStyle, useUrlParams } from "../UrlParams";
|
import { HeaderStyle, useUrlParams } from "../UrlParams";
|
||||||
@@ -184,6 +185,7 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
|
|||||||
vm,
|
vm,
|
||||||
props.muteStates,
|
props.muteStates,
|
||||||
mediaDevices,
|
mediaDevices,
|
||||||
|
backgroundEffectsState$.pipe(map(({ settling }) => settling ?? false)),
|
||||||
`${props.client.getUserId()}:${props.client.getDeviceId()}`,
|
`${props.client.getUserId()}:${props.client.getDeviceId()}`,
|
||||||
{ showControls: urlParams.showControls, header: urlParams.header },
|
{ showControls: urlParams.showControls, header: urlParams.header },
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ import { render, screen } from "@testing-library/react";
|
|||||||
import userEvent from "@testing-library/user-event";
|
import userEvent from "@testing-library/user-event";
|
||||||
import { TooltipProvider } from "@vector-im/compound-web";
|
import { TooltipProvider } from "@vector-im/compound-web";
|
||||||
import { type MatrixClient, type RoomSummary } from "matrix-js-sdk";
|
import { type MatrixClient, type RoomSummary } from "matrix-js-sdk";
|
||||||
|
import { type Observable } from "rxjs";
|
||||||
|
|
||||||
import { KnockLobbyView } from "./KnockLobbyView";
|
import { KnockLobbyView } from "./KnockLobbyView";
|
||||||
import { LeaveToHomeProvider } from "../LeaveToHomeContext";
|
import { LeaveToHomeProvider } from "../LeaveToHomeContext";
|
||||||
@@ -21,13 +22,21 @@ vi.mock("@livekit/components-react", () => ({
|
|||||||
usePreviewTracks: (): unknown[] => [],
|
usePreviewTracks: (): unknown[] => [],
|
||||||
}));
|
}));
|
||||||
|
|
||||||
vi.mock("../livekit/BackgroundEffectsContext", () => ({
|
vi.mock("../livekit/BackgroundEffectsContext", async () => {
|
||||||
useBackgroundEffects: (): BackgroundEffectsState => ({
|
const { of } = await import("rxjs");
|
||||||
|
const none: BackgroundEffectsState = {
|
||||||
supported: false,
|
supported: false,
|
||||||
processor: undefined,
|
processor: undefined,
|
||||||
}),
|
};
|
||||||
useSyncBackgroundEffects: (): void => {},
|
// One observable, as the real hook keeps: a fresh one each render would
|
||||||
}));
|
// rebuild the footer on every render.
|
||||||
|
const none$ = of(none);
|
||||||
|
return {
|
||||||
|
useBackgroundEffects: (): BackgroundEffectsState => none,
|
||||||
|
useBackgroundEffectsState$: (): Observable<BackgroundEffectsState> => none$,
|
||||||
|
useSyncBackgroundEffects: (): void => {},
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
vi.mock("react-use-measure", () => ({
|
vi.mock("react-use-measure", () => ({
|
||||||
default: (): [() => void, object] => [(): void => {}, {}],
|
default: (): [() => void, object] => [(): void => {}, {}],
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ Please see LICENSE in the repository root for full details.
|
|||||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||||
import { render, screen } from "@testing-library/react";
|
import { render, screen } from "@testing-library/react";
|
||||||
import userEvent from "@testing-library/user-event";
|
import userEvent from "@testing-library/user-event";
|
||||||
import { of } from "rxjs";
|
import { type Observable, of } from "rxjs";
|
||||||
import { LeaveToHomeProvider } from "../LeaveToHomeContext";
|
import { LeaveToHomeProvider } from "../LeaveToHomeContext";
|
||||||
import { TooltipProvider } from "@vector-im/compound-web";
|
import { TooltipProvider } from "@vector-im/compound-web";
|
||||||
import { type MatrixClient } from "matrix-js-sdk";
|
import { type MatrixClient } from "matrix-js-sdk";
|
||||||
@@ -41,13 +41,21 @@ vi.mock("@livekit/components-react", () => ({
|
|||||||
usePreviewTracks: (): unknown[] => [],
|
usePreviewTracks: (): unknown[] => [],
|
||||||
}));
|
}));
|
||||||
|
|
||||||
vi.mock("../livekit/BackgroundEffectsContext", () => ({
|
vi.mock("../livekit/BackgroundEffectsContext", async () => {
|
||||||
useBackgroundEffects: (): BackgroundEffectsState => ({
|
const { of } = await import("rxjs");
|
||||||
|
const none: BackgroundEffectsState = {
|
||||||
supported: false,
|
supported: false,
|
||||||
processor: undefined,
|
processor: undefined,
|
||||||
}),
|
};
|
||||||
useSyncBackgroundEffects: (): void => {},
|
// One observable, as the real hook keeps: a fresh one each render would
|
||||||
}));
|
// rebuild the footer on every render.
|
||||||
|
const none$ = of(none);
|
||||||
|
return {
|
||||||
|
useBackgroundEffects: (): BackgroundEffectsState => none,
|
||||||
|
useBackgroundEffectsState$: (): Observable<BackgroundEffectsState> => none$,
|
||||||
|
useSyncBackgroundEffects: (): void => {},
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
vi.mock("react-use-measure", () => ({
|
vi.mock("react-use-measure", () => ({
|
||||||
default: (): [() => void, object] => [(): void => {}, {}],
|
default: (): [() => void, object] => [(): void => {}, {}],
|
||||||
|
|||||||
+12
-1
@@ -24,6 +24,7 @@ import {
|
|||||||
type LocalVideoTrack,
|
type LocalVideoTrack,
|
||||||
Track,
|
Track,
|
||||||
} from "livekit-client";
|
} from "livekit-client";
|
||||||
|
import { map } from "rxjs";
|
||||||
|
|
||||||
import inCallStyles from "./InCallView.module.css";
|
import inCallStyles from "./InCallView.module.css";
|
||||||
import styles from "./LobbyView.module.css";
|
import styles from "./LobbyView.module.css";
|
||||||
@@ -41,6 +42,7 @@ import { ObservableScope } from "../state/ObservableScope";
|
|||||||
import { useInitial } from "../useInitial";
|
import { useInitial } from "../useInitial";
|
||||||
import {
|
import {
|
||||||
useBackgroundEffects,
|
useBackgroundEffects,
|
||||||
|
useBackgroundEffectsState$,
|
||||||
useSyncBackgroundEffects,
|
useSyncBackgroundEffects,
|
||||||
} from "../livekit/BackgroundEffectsContext";
|
} from "../livekit/BackgroundEffectsContext";
|
||||||
import { getValue } from "../utils/observable";
|
import { getValue } from "../utils/observable";
|
||||||
@@ -137,6 +139,7 @@ export const LobbyView: FC<Props> = ({
|
|||||||
);
|
);
|
||||||
|
|
||||||
const { processor } = useBackgroundEffects();
|
const { processor } = useBackgroundEffects();
|
||||||
|
const backgroundEffectsState$ = useBackgroundEffectsState$();
|
||||||
|
|
||||||
const initialProcessor = useInitial(() => processor);
|
const initialProcessor = useInitial(() => processor);
|
||||||
const localTrackOptions = useMemo<CreateLocalTracksOptions>(
|
const localTrackOptions = useMemo<CreateLocalTracksOptions>(
|
||||||
@@ -195,6 +198,7 @@ export const LobbyView: FC<Props> = ({
|
|||||||
footerScope,
|
footerScope,
|
||||||
muteStates,
|
muteStates,
|
||||||
devices,
|
devices,
|
||||||
|
backgroundEffectsState$.pipe(map(({ settling }) => settling ?? false)),
|
||||||
openSettings,
|
openSettings,
|
||||||
hangup,
|
hangup,
|
||||||
// Logo and header are connected: only show the logo in SPA with header.
|
// Logo and header are connected: only show the logo in SPA with header.
|
||||||
@@ -204,7 +208,14 @@ export const LobbyView: FC<Props> = ({
|
|||||||
return (): void => {
|
return (): void => {
|
||||||
footerScope.end();
|
footerScope.end();
|
||||||
};
|
};
|
||||||
}, [devices, hangup, hideHeader, muteStates, openSettings]);
|
}, [
|
||||||
|
devices,
|
||||||
|
backgroundEffectsState$,
|
||||||
|
hangup,
|
||||||
|
hideHeader,
|
||||||
|
muteStates,
|
||||||
|
openSettings,
|
||||||
|
]);
|
||||||
|
|
||||||
// TODO: Unify this component with InCallView, so we can get slick joining
|
// TODO: Unify this component with InCallView, so we can get slick joining
|
||||||
// animations and don't have to feel bad about reusing its CSS
|
// animations and don't have to feel bad about reusing its CSS
|
||||||
|
|||||||
@@ -46,25 +46,33 @@ function fakePipeline(): {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
/** One letter per state: idle, attached. */
|
/** One letter per state: idle, waiting for a frame, attached. */
|
||||||
function letter(state: BackgroundEffectsState): string {
|
function letter(state: BackgroundEffectsState): string {
|
||||||
return state.processor === undefined ? "i" : "a";
|
if (state.processor === undefined) return "i";
|
||||||
|
return state.settling ? "w" : "a";
|
||||||
}
|
}
|
||||||
|
|
||||||
describe("the pipeline's state", () => {
|
describe("the pipeline's state", () => {
|
||||||
function testState({
|
function testState({
|
||||||
effect,
|
effect,
|
||||||
|
firstFrame = "",
|
||||||
expected,
|
expected,
|
||||||
}: {
|
}: {
|
||||||
effect: string;
|
effect: string;
|
||||||
|
firstFrame?: string;
|
||||||
expected: string;
|
expected: string;
|
||||||
}): void {
|
}): void {
|
||||||
withTestScheduler(({ behavior, expectObservable }) => {
|
withTestScheduler(({ behavior, schedule, expectObservable }) => {
|
||||||
|
const transformer = {
|
||||||
|
onFirstFrame: undefined as (() => void) | undefined,
|
||||||
|
};
|
||||||
const effects = createBackgroundEffects(testScope(), {
|
const effects = createBackgroundEffects(testScope(), {
|
||||||
supported: true,
|
supported: true,
|
||||||
effect$: behavior(effect, { n: "none", b: "blur" }),
|
effect$: behavior(effect, { n: "none", b: "blur" }),
|
||||||
pipeline: fakePipeline().pipeline,
|
pipeline: fakePipeline().pipeline,
|
||||||
|
transformer,
|
||||||
});
|
});
|
||||||
|
schedule(firstFrame, { f: () => transformer.onFirstFrame?.() });
|
||||||
expectObservable(
|
expectObservable(
|
||||||
effects.state$.pipe(map(letter), distinctUntilChanged()),
|
effects.state$.pipe(map(letter), distinctUntilChanged()),
|
||||||
).toBe(expected);
|
).toBe(expected);
|
||||||
@@ -73,8 +81,15 @@ describe("the pipeline's state", () => {
|
|||||||
|
|
||||||
it("defaults to no effect", () => testState({ effect: "n", expected: "i" }));
|
it("defaults to no effect", () => testState({ effect: "n", expected: "i" }));
|
||||||
|
|
||||||
|
it("waits from the first effect until a frame carries it", () =>
|
||||||
|
testState({
|
||||||
|
effect: " nb-n-b",
|
||||||
|
firstFrame: " --f",
|
||||||
|
expected: " iwa",
|
||||||
|
}));
|
||||||
|
|
||||||
it("attaches on first use and stays attached", () =>
|
it("attaches on first use and stays attached", () =>
|
||||||
testState({ effect: "nbn", expected: "ia" }));
|
testState({ effect: "nb-n", firstFrame: "--f", expected: "iwa" }));
|
||||||
});
|
});
|
||||||
|
|
||||||
describe("background effects", () => {
|
describe("background effects", () => {
|
||||||
@@ -88,6 +103,7 @@ describe("background effects", () => {
|
|||||||
supported: true,
|
supported: true,
|
||||||
effect$,
|
effect$,
|
||||||
pipeline: fake.pipeline,
|
pipeline: fake.pipeline,
|
||||||
|
transformer: { onFirstFrame: undefined },
|
||||||
...options,
|
...options,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,7 +5,13 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
|||||||
Please see LICENSE in the repository root for full details.
|
Please see LICENSE in the repository root for full details.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { combineLatest, distinctUntilChanged, map, scan } from "rxjs";
|
import {
|
||||||
|
combineLatest,
|
||||||
|
distinctUntilChanged,
|
||||||
|
map,
|
||||||
|
Observable,
|
||||||
|
scan,
|
||||||
|
} from "rxjs";
|
||||||
import {
|
import {
|
||||||
type BackgroundProcessorWrapper,
|
type BackgroundProcessorWrapper,
|
||||||
type SwitchBackgroundProcessorOptions,
|
type SwitchBackgroundProcessorOptions,
|
||||||
@@ -30,6 +36,8 @@ export interface BackgroundEffectsOptions {
|
|||||||
effect$: Behavior<string>;
|
effect$: Behavior<string>;
|
||||||
/** The background processor pipeline to be switched between effects. */
|
/** The background processor pipeline to be switched between effects. */
|
||||||
pipeline: BackgroundProcessorWrapper;
|
pipeline: BackgroundProcessorWrapper;
|
||||||
|
/** Tells, once, that a frame carrying an effect has been drawn. */
|
||||||
|
transformer: { onFirstFrame: (() => void) | undefined };
|
||||||
}
|
}
|
||||||
|
|
||||||
/** The background effect pipeline, as the camera tracks and the menus see it. */
|
/** The background effect pipeline, as the camera tracks and the menus see it. */
|
||||||
@@ -40,7 +48,7 @@ export interface BackgroundEffects {
|
|||||||
/** Switches the pipeline as the choice changes, for as long as the scope lasts. */
|
/** Switches the pipeline as the choice changes, for as long as the scope lasts. */
|
||||||
export function createBackgroundEffects(
|
export function createBackgroundEffects(
|
||||||
scope: ObservableScope,
|
scope: ObservableScope,
|
||||||
{ supported, effect$, pipeline }: BackgroundEffectsOptions,
|
{ supported, effect$, pipeline, transformer }: BackgroundEffectsOptions,
|
||||||
): BackgroundEffects {
|
): BackgroundEffects {
|
||||||
const choice$ = effect$.pipe(map(parseEffect));
|
const choice$ = effect$.pipe(map(parseEffect));
|
||||||
const wanted$ = choice$.pipe(
|
const wanted$ = choice$.pipe(
|
||||||
@@ -48,15 +56,29 @@ export function createBackgroundEffects(
|
|||||||
distinctUntilChanged(),
|
distinctUntilChanged(),
|
||||||
);
|
);
|
||||||
|
|
||||||
|
const drewAFrame$ = scope.behavior(
|
||||||
|
new Observable<boolean>((subscriber) => {
|
||||||
|
subscriber.next(false);
|
||||||
|
transformer.onFirstFrame = (): void => subscriber.next(true);
|
||||||
|
return (): void => {
|
||||||
|
transformer.onFirstFrame = undefined;
|
||||||
|
};
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
const state$ = scope.behavior(
|
const state$ = scope.behavior(
|
||||||
wanted$.pipe(
|
combineLatest([wanted$, drewAFrame$]).pipe(
|
||||||
scan<boolean, BackgroundEffectsState>(
|
scan<[boolean, boolean], BackgroundEffectsState>(
|
||||||
(previous, wanted) => {
|
(previous, [wanted, drewAFrame]) => {
|
||||||
// Attached the first time an effect is wanted and never detached
|
// Attached the first time an effect is wanted and never detached
|
||||||
// after, so someone who never turns one on pays for none of it.
|
// after, so someone who never turns one on pays for none of it.
|
||||||
const enable =
|
const enable =
|
||||||
previous.processor !== undefined || (supported && wanted);
|
previous.processor !== undefined || (supported && wanted);
|
||||||
return { supported, processor: enable ? pipeline : undefined };
|
return {
|
||||||
|
supported,
|
||||||
|
processor: enable ? pipeline : undefined,
|
||||||
|
settling: enable && !drewAFrame,
|
||||||
|
};
|
||||||
},
|
},
|
||||||
{ supported, processor: undefined },
|
{ supported, processor: undefined },
|
||||||
),
|
),
|
||||||
|
|||||||
@@ -188,6 +188,7 @@ export function getBasicCallViewModelEnvironment(
|
|||||||
vm,
|
vm,
|
||||||
muteStates,
|
muteStates,
|
||||||
mediaDevices,
|
mediaDevices,
|
||||||
|
constant(false),
|
||||||
"reactionId",
|
"reactionId",
|
||||||
{ showControls: true, header: HeaderStyle.Standard },
|
{ showControls: true, header: HeaderStyle.Standard },
|
||||||
);
|
);
|
||||||
|
|||||||
Reference in New Issue
Block a user