mirror of
https://github.com/vector-im/element-call.git
synced 2026-11-16 01:22:29 +00:00
- The effects state's timeline through preparing, settling and attached, and that a failed trial build withholds the effects for the session while keeping the choice. - The footer offers nothing once the pipeline fails to build, and passes on the wait as the state's single wait field. - The call leaves the camera out of the join while the pipeline is prepared, publishes it once that is known, and turns on a camera turned off before the processor was chosen; the lobby opens no camera meanwhile. Both view models are driven by one state subject. - The transformer's trial build answers whether a segmenter can be made. - End to end, a browser whose pipeline cannot be built sees the effects withheld. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
117 lines
4.0 KiB
TypeScript
117 lines
4.0 KiB
TypeScript
/*
|
|
Copyright 2026 Element Creations Ltd.
|
|
|
|
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
|
Please see LICENSE in the repository root for full details.
|
|
*/
|
|
|
|
import { afterEach, describe, expect, it, vi } from "vitest";
|
|
import {
|
|
BackgroundTransformer,
|
|
VideoTransformer,
|
|
type VideoTransformerInitOptions,
|
|
} from "@livekit/track-processors";
|
|
import { ImageSegmenter } from "@mediapipe/tasks-vision";
|
|
|
|
import {
|
|
BackgroundEffectTransformer,
|
|
canSegment,
|
|
} from "./BackgroundEffectTransformer";
|
|
|
|
const frame = {} as VideoFrame;
|
|
const controller = {} as TransformStreamDefaultController<VideoFrame>;
|
|
|
|
describe("BackgroundEffectTransformer", () => {
|
|
afterEach(() => {
|
|
vi.restoreAllMocks();
|
|
vi.unstubAllGlobals();
|
|
});
|
|
|
|
it("loads a background chosen before it was built", async () => {
|
|
vi.spyOn(VideoTransformer.prototype, "init").mockResolvedValue();
|
|
vi.spyOn(ImageSegmenter, "createFromOptions").mockResolvedValue(
|
|
{} as ImageSegmenter,
|
|
);
|
|
const load = vi
|
|
.spyOn(BackgroundTransformer.prototype, "loadAndSetBackground")
|
|
.mockResolvedValue();
|
|
const transformer = new BackgroundEffectTransformer({
|
|
backgroundDisabled: true,
|
|
});
|
|
// Chosen before the camera is processed, as the first effect always is:
|
|
// loaded then, there is nothing yet to draw it with.
|
|
await transformer.update({ imagePath: "/background.jpg" });
|
|
load.mockClear();
|
|
|
|
await transformer.init({} as VideoTransformerInitOptions);
|
|
expect(load).toHaveBeenCalledWith("/background.jpg");
|
|
});
|
|
|
|
it("reports the first frame carrying an effect, once", async () => {
|
|
vi.spyOn(BackgroundTransformer.prototype, "transform").mockImplementation(
|
|
async function (this: BackgroundTransformer) {
|
|
this.isFirstFrame = false;
|
|
return Promise.resolve();
|
|
},
|
|
);
|
|
const transformer = new BackgroundEffectTransformer({});
|
|
transformer.onFirstFrame = vi.fn();
|
|
|
|
await transformer.transform(frame, controller);
|
|
await transformer.transform(frame, controller);
|
|
expect(transformer.onFirstFrame).toHaveBeenCalledTimes(1);
|
|
});
|
|
|
|
it("does not report a frame passed through untouched", async () => {
|
|
vi.spyOn(BackgroundTransformer.prototype, "transform").mockResolvedValue();
|
|
const transformer = new BackgroundEffectTransformer({
|
|
backgroundDisabled: true,
|
|
});
|
|
transformer.onFirstFrame = vi.fn();
|
|
|
|
await transformer.transform(frame, controller);
|
|
expect(transformer.onFirstFrame).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it("passes frames on untouched once disabled, even after blurring", async () => {
|
|
const transformer = new BackgroundEffectTransformer({});
|
|
await transformer.update({ blurRadius: 15, backgroundDisabled: false });
|
|
await transformer.update({
|
|
imagePath: undefined,
|
|
backgroundDisabled: true,
|
|
});
|
|
// With neither, the library passes a frame on without segmenting it.
|
|
expect(transformer.options.blurRadius).toBeUndefined();
|
|
expect(transformer.options.imagePath).toBeUndefined();
|
|
});
|
|
|
|
it("segments on the CPU where the GPU refuses", async () => {
|
|
vi.spyOn(VideoTransformer.prototype, "init").mockResolvedValue();
|
|
const build = vi
|
|
.spyOn(ImageSegmenter, "createFromOptions")
|
|
.mockRejectedValueOnce(new Error("blocklisted"))
|
|
.mockResolvedValue({} as ImageSegmenter);
|
|
|
|
await new BackgroundEffectTransformer({}).init(
|
|
{} as VideoTransformerInitOptions,
|
|
);
|
|
expect(build.mock.calls.map(([, o]) => o.baseOptions?.delegate)).toEqual([
|
|
"GPU",
|
|
"CPU",
|
|
]);
|
|
});
|
|
|
|
it("says whether a segmenter can be built at all", async () => {
|
|
vi.stubGlobal("OffscreenCanvas", class {});
|
|
const close = vi.fn();
|
|
const build = vi
|
|
.spyOn(ImageSegmenter, "createFromOptions")
|
|
.mockResolvedValue({ close } as unknown as ImageSegmenter);
|
|
expect(await canSegment()).toBe(true);
|
|
expect(close).toHaveBeenCalled();
|
|
|
|
build.mockRejectedValue(new Error("no model"));
|
|
expect(await canSegment()).toBe(false);
|
|
});
|
|
});
|