mirror of
https://github.com/vector-im/element-call.git
synced 2026-10-16 23:45:50 +00:00
Attach background effects through one switchable pipeline
- One pipeline, shared by the pre-join preview and the call, attached the first time an effect is wanted and never detached after; turning blur off switches it to a disabled mode in place. Main dropped the processor whenever blur went off, and a pipeline taken off the camera is destroyed and re-primed, so each time blur came back one unprocessed frame went out. Built once, the priming frame is spent once. - Still driven by the existing blur setting: nothing a user sees changes. - Its logic is BackgroundEffects in src/state/, which the provider only builds, so the tests drive it without rendering anything. - Disabled drops the blur radius too. The SDK's disabled mode keeps it, and with a radius every frame is still segmented and thrown away; with neither a radius nor a picture, frames pass untouched. - Switches run one at a time and skip any overtaken, so an earlier, slower switch can't land after a later choice. - Builds, rebuilds and teardowns of the pipeline run one at a time (OneStepPipeline). The preview's and the call's tracks attach and stop it independently, and the SDK's wrapper holds one track's streams: two builds at once overwrite each other's, and a stop during a build does nothing. - Every camera track starts with the pipeline on: the room's capture defaults follow it, and a camera turned off before the first effect is replaced on turning it on, since a stopped track can't take one. One attached after publishing lets the room through while it is built. - BlurBackgroundTransformer becomes BackgroundEffectTransformer: it also sits disabled, and its init applies that state as the SDK's would. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,93 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { combineLatest, distinctUntilChanged, filter, map, scan } from "rxjs";
|
||||
import {
|
||||
type BackgroundProcessorWrapper,
|
||||
type SwitchBackgroundProcessorOptions,
|
||||
} from "@livekit/track-processors";
|
||||
import { logger } from "matrix-js-sdk/lib/logger";
|
||||
|
||||
import { type Behavior } from "./Behavior";
|
||||
import { type ObservableScope } from "./ObservableScope";
|
||||
import { type ProcessorState } from "../livekit/TrackProcessorContext";
|
||||
|
||||
const blurRadius = 15;
|
||||
|
||||
export interface BackgroundEffectsOptions {
|
||||
/** Whether this browser can run a pipeline at all. */
|
||||
supported: boolean;
|
||||
/** Whether blur is chosen. */
|
||||
blur$: Behavior<boolean>;
|
||||
/**
|
||||
* Shared by the pre-join preview and the call. Building or destroying it is
|
||||
* what primes it, and a primed pipeline lets its next frame through
|
||||
* untouched, so it is switched rather than rebuilt.
|
||||
*/
|
||||
pipeline: BackgroundProcessorWrapper;
|
||||
}
|
||||
|
||||
/** The background effect pipeline, as the camera tracks and the menus see it. */
|
||||
export class BackgroundEffects {
|
||||
public readonly state$: Behavior<ProcessorState>;
|
||||
|
||||
public constructor(
|
||||
scope: ObservableScope,
|
||||
{ supported, blur$, pipeline }: BackgroundEffectsOptions,
|
||||
) {
|
||||
this.state$ = scope.behavior(
|
||||
blur$.pipe(
|
||||
scan<boolean, ProcessorState>(
|
||||
(previous, wanted) => {
|
||||
// Attached the first time an effect is wanted and never detached
|
||||
// after, so someone who never turns one on pays for none of it.
|
||||
const attached =
|
||||
previous.processor !== undefined || (supported && wanted);
|
||||
return { supported, processor: attached ? pipeline : undefined };
|
||||
},
|
||||
{ supported, processor: undefined },
|
||||
),
|
||||
),
|
||||
);
|
||||
|
||||
const switchTo = oneSwitchAtATime(pipeline);
|
||||
combineLatest([this.state$, blur$])
|
||||
.pipe(
|
||||
filter(([{ processor }]) => processor !== undefined),
|
||||
map(([, blur]): SwitchBackgroundProcessorOptions =>
|
||||
blur ? { mode: "background-blur", blurRadius } : { mode: "disabled" },
|
||||
),
|
||||
distinctUntilChanged((a, b) => JSON.stringify(a) === JSON.stringify(b)),
|
||||
scope.bind(),
|
||||
)
|
||||
.subscribe((options) => {
|
||||
switchTo(options).catch((e) =>
|
||||
logger.warn("Failed to switch background effect", e),
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Switches the pipeline one choice at a time, skipping those overtaken while
|
||||
* they waited. A switch to a picture ends only once it has loaded, so a
|
||||
* slower, earlier choice would otherwise land after a later one.
|
||||
*/
|
||||
function oneSwitchAtATime(
|
||||
pipeline: BackgroundProcessorWrapper,
|
||||
): (options: SwitchBackgroundProcessorOptions) => Promise<void> {
|
||||
let latest: SwitchBackgroundProcessorOptions | undefined;
|
||||
let queue = Promise.resolve();
|
||||
return async (options) => {
|
||||
latest = options;
|
||||
const turn = queue.then(async () => {
|
||||
if (options === latest) await pipeline.switchTo(options);
|
||||
});
|
||||
queue = turn.catch(() => {});
|
||||
return turn;
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user