mirror of
https://github.com/vector-im/element-call.git
synced 2026-11-19 09:32:29 +08:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5fa2c2bec7 | ||
|
|
b04237fada | ||
|
|
2d9b76ac1b | ||
|
|
26e9980b7f | ||
|
|
839995a0c4 | ||
|
|
d3eefdbcf2 |
+11
-12
@@ -72,9 +72,9 @@ import {
|
||||
useUrlParams,
|
||||
} from "../src/UrlParams";
|
||||
import { MediaDevicesContext } from "../src/MediaDevicesContext";
|
||||
import { MediaDevices } from "../src/state/MediaDevices";
|
||||
import { AppViewModel } from "../src/state/AppViewModel";
|
||||
import { ObservableScope } from "../src/state/ObservableScope";
|
||||
import { ProcessorProvider } from "../src/tracks/TrackProcessorContext";
|
||||
import { BackgroundEffectsProvider } from "../src/tracks/BackgroundEffectsContext";
|
||||
import { Config } from "../src/config/Config";
|
||||
import { type ConfigOptions } from "../src/config/ConfigOptions";
|
||||
import { i18n } from "../src/utils/i18n";
|
||||
@@ -194,14 +194,12 @@ export const ElementCall: FC<ElementCallProps> = ({
|
||||
// observers running for the rest of the page's life. Null until then, which
|
||||
// is one render.
|
||||
const { controlledAudioDevices, callIntent } = params;
|
||||
const [mediaDevices, setMediaDevices] = useState<MediaDevices | null>(null);
|
||||
const [vm, setVm] = useState<AppViewModel | null>(null);
|
||||
useEffect(() => {
|
||||
const scope = new ObservableScope();
|
||||
setMediaDevices(
|
||||
new MediaDevices(scope, { controlledAudioDevices, callIntent }),
|
||||
);
|
||||
setVm(new AppViewModel(scope, { controlledAudioDevices, callIntent }));
|
||||
return (): void => {
|
||||
setMediaDevices(null);
|
||||
setVm(null);
|
||||
scope.end();
|
||||
};
|
||||
}, [controlledAudioDevices, callIntent]);
|
||||
@@ -222,8 +220,7 @@ export const ElementCall: FC<ElementCallProps> = ({
|
||||
// is told that Element Call has loaded, as the widget tells its client once
|
||||
// its own initialisation is over. Once per mount, however often the pieces
|
||||
// are later swapped out.
|
||||
const ready =
|
||||
container !== null && rtcSession !== null && mediaDevices !== null;
|
||||
const ready = container !== null && rtcSession !== null && vm !== null;
|
||||
const announcedLoaded = useRef(false);
|
||||
useEffect(() => {
|
||||
if (!ready || announcedLoaded.current) return;
|
||||
@@ -254,8 +251,10 @@ export const ElementCall: FC<ElementCallProps> = ({
|
||||
<Decoration>
|
||||
<TooltipProvider>
|
||||
<ClientProvider client={client}>
|
||||
<MediaDevicesContext value={mediaDevices}>
|
||||
<ProcessorProvider>
|
||||
<MediaDevicesContext value={vm.mediaDevices}>
|
||||
<BackgroundEffectsProvider
|
||||
effects={vm.backgroundEffects}
|
||||
>
|
||||
<CallView
|
||||
client={client}
|
||||
rtcSession={rtcSession}
|
||||
@@ -264,7 +263,7 @@ export const ElementCall: FC<ElementCallProps> = ({
|
||||
preload={params.preload}
|
||||
skipLobby={params.skipLobby}
|
||||
/>
|
||||
</ProcessorProvider>
|
||||
</BackgroundEffectsProvider>
|
||||
</MediaDevicesContext>
|
||||
</ClientProvider>
|
||||
</TooltipProvider>
|
||||
|
||||
@@ -63,6 +63,11 @@ export {
|
||||
} from "./src/media-backend/livekit/LivekitBackend";
|
||||
// Tracks for a lobby, captured ahead of the join and handed to it
|
||||
export { createLocalMedia } from "./src/LocalMedia";
|
||||
// For a host whose processor is a livekit-client one, such as @livekit/track-processors
|
||||
export {
|
||||
fromLivekitProcessor,
|
||||
toLivekitProcessor,
|
||||
} from "./src/media-backend/livekit/videoProcessor";
|
||||
// For a developer panel to check a transport before using it
|
||||
export {
|
||||
getSFUConfig as authenticateWithTransport,
|
||||
|
||||
@@ -8,6 +8,7 @@ Please see LICENSE in the repository root for full details.
|
||||
import { EventEmitter } from "events";
|
||||
import {
|
||||
type LocalParticipant,
|
||||
LocalVideoTrack,
|
||||
ParticipantEvent,
|
||||
RemoteAudioTrack,
|
||||
type RemoteParticipant,
|
||||
@@ -17,10 +18,22 @@ import {
|
||||
TrackEvent,
|
||||
type TrackPublication,
|
||||
} from "livekit-client";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
afterEach,
|
||||
beforeEach,
|
||||
describe,
|
||||
expect,
|
||||
it,
|
||||
onTestFinished,
|
||||
vi,
|
||||
} from "vitest";
|
||||
|
||||
import { ObservableScope } from "../../reactive/ObservableScope";
|
||||
import { type AudioMediaTrack } from "../../media-api";
|
||||
import {
|
||||
type AudioMediaTrack,
|
||||
type LocalVideoMediaTrack,
|
||||
type VideoProcessor,
|
||||
} from "../../media-api";
|
||||
import {
|
||||
createLivekitMediaTrack,
|
||||
createLocalLivekitMediaTrack,
|
||||
@@ -211,6 +224,49 @@ describe("createLivekitMediaTrack", () => {
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
// LiveKit stops a camera that is turned off; the processor is built when it
|
||||
// is turned on again
|
||||
it.each([
|
||||
{ state: "is off", isMuted: true, takes: true },
|
||||
{ state: "has stopped for good", isMuted: false, takes: false },
|
||||
])(
|
||||
"gives a processor to a camera that $state: $takes",
|
||||
async ({ isMuted, takes }) => {
|
||||
// jsdom has none; LiveKit's facing mode checks against it
|
||||
vi.stubGlobal("MediaStreamTrack", class {});
|
||||
onTestFinished(() => {
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
const camera = Object.assign(Object.create(LocalVideoTrack.prototype), {
|
||||
isMuted,
|
||||
_mediaStreamTrack: {
|
||||
readyState: "ended",
|
||||
label: "",
|
||||
getSettings: () => ({}),
|
||||
},
|
||||
getProcessor: () => undefined,
|
||||
setProcessor: vi.fn().mockResolvedValue(undefined),
|
||||
}) as LocalVideoTrack;
|
||||
const { participant, publication, room } = fakes({
|
||||
track: camera as unknown as FakeTrack,
|
||||
});
|
||||
const track = createLocalLivekitMediaTrack(
|
||||
scope,
|
||||
Object.assign(participant, {
|
||||
isLocal: true,
|
||||
}) as unknown as LocalParticipant,
|
||||
publication as unknown as TrackPublication,
|
||||
room,
|
||||
vi.fn(),
|
||||
) as LocalVideoMediaTrack;
|
||||
|
||||
await track.setProcessor({ name: "blur" } as VideoProcessor);
|
||||
expect(vi.mocked(camera.setProcessor).mock.calls.length).toBe(
|
||||
takes ? 1 : 0,
|
||||
);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
type FakeTrack = Track & {
|
||||
|
||||
@@ -50,10 +50,7 @@ import {
|
||||
type VideoMediaTrack,
|
||||
} from "../../media-api";
|
||||
import { MatrixRTCError } from "../../errors";
|
||||
import {
|
||||
convertToLivekitProcessor,
|
||||
type LivekitProcessor,
|
||||
} from "./videoProcessor";
|
||||
import { toLivekitProcessor, type LivekitProcessor } from "./videoProcessor";
|
||||
|
||||
export const mediaSources: Record<Track.Source, MediaSource> = {
|
||||
[Track.Source.Microphone]: "microphone",
|
||||
@@ -241,12 +238,16 @@ export function createLocalLivekitMediaTrack(
|
||||
},
|
||||
setProcessor: async (processor) => {
|
||||
const track = videoTrack();
|
||||
// A processor cannot be built on a track that has already ended
|
||||
if (!track || track.mediaStreamTrack.readyState === "ended") return;
|
||||
// A camera that is off has a stopped track and takes the processor on
|
||||
// turning on again; one stopped for good takes none
|
||||
if (
|
||||
!track ||
|
||||
(track.mediaStreamTrack.readyState === "ended" && !track.isMuted)
|
||||
)
|
||||
return;
|
||||
const current = track.getProcessor() as LivekitProcessor | undefined;
|
||||
if (processor === current?.source) return;
|
||||
if (processor)
|
||||
await track.setProcessor(convertToLivekitProcessor(processor));
|
||||
if (processor) await track.setProcessor(toLivekitProcessor(processor));
|
||||
else if (current) await track.stopProcessor();
|
||||
},
|
||||
};
|
||||
|
||||
@@ -120,6 +120,24 @@ describe("Publisher", () => {
|
||||
expect(participant.createTracks).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("creates both with the options of requests made since the join", async () => {
|
||||
const processor: VideoProcessor = {
|
||||
name: "effect",
|
||||
init: vi.fn(),
|
||||
restart: vi.fn(),
|
||||
destroy: vi.fn(),
|
||||
};
|
||||
want(microphone, { source: "camera", deviceId: "front", processor });
|
||||
|
||||
publisher.start();
|
||||
await flushPromises();
|
||||
|
||||
expect(room.options.videoCaptureDefaults).toMatchObject({
|
||||
deviceId: "front",
|
||||
processor: expect.objectContaining({ source: processor }),
|
||||
});
|
||||
});
|
||||
|
||||
it("asks for the camera and the microphone in one call", async () => {
|
||||
const enableBoth = vi.spyOn(participant, "enableCameraAndMicrophone");
|
||||
want(microphone, camera);
|
||||
|
||||
@@ -22,7 +22,7 @@ import {
|
||||
type PublishRequest,
|
||||
} from "../../media-api";
|
||||
import { livekitSources, mediaSources } from "./LivekitMediaTrack";
|
||||
import { convertToLivekitProcessor } from "./videoProcessor";
|
||||
import { toLivekitProcessor } from "./videoProcessor";
|
||||
import {
|
||||
audioCaptureOptions,
|
||||
screenShareCaptureOptions,
|
||||
@@ -120,8 +120,10 @@ export class Publisher {
|
||||
// resolves these once the track is published, which may block on the
|
||||
// connection; LocalTrackPublished is what tells us a track exists.
|
||||
const captureBoth = toCapture.has("microphone") && toCapture.has("camera");
|
||||
if (captureBoth)
|
||||
if (captureBoth) {
|
||||
this.takeCaptureDefaults(requests);
|
||||
void this.room.localParticipant.enableCameraAndMicrophone();
|
||||
}
|
||||
for (const request of requests)
|
||||
if (
|
||||
request.source === "screenShare" ||
|
||||
@@ -133,6 +135,27 @@ export class Publisher {
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Makes the requests the room's capture defaults. The room took its defaults
|
||||
* from the requests at the join, but one made since, such as a camera turned
|
||||
* on before the start, has options of its own: a video processor dropped
|
||||
* here would let the room see the camera before it is applied.
|
||||
*/
|
||||
private takeCaptureDefaults(requests: PublishRequest[]): void {
|
||||
const options = this.room.options;
|
||||
for (const request of requests)
|
||||
if (request.source === "microphone")
|
||||
options.audioCaptureDefaults = {
|
||||
...options.audioCaptureDefaults,
|
||||
...audioCaptureOptions(request),
|
||||
};
|
||||
else if (request.source === "camera")
|
||||
options.videoCaptureDefaults = {
|
||||
...options.videoCaptureDefaults,
|
||||
...videoCaptureOptions(request),
|
||||
};
|
||||
}
|
||||
|
||||
private hasCaptured(source: MediaSource): boolean {
|
||||
return (
|
||||
(source === "microphone" || source === "camera") &&
|
||||
@@ -159,10 +182,7 @@ export class Publisher {
|
||||
if (muted) await local.mute();
|
||||
if (processor && local instanceof LocalVideoTrack)
|
||||
await local.setProcessor(
|
||||
convertToLivekitProcessor(
|
||||
processor.processor,
|
||||
processor.processedTrack,
|
||||
),
|
||||
toLivekitProcessor(processor.processor, processor.processedTrack),
|
||||
);
|
||||
return this.room.localParticipant.publishTrack(local, {
|
||||
source: livekitSources[request.source],
|
||||
|
||||
@@ -16,7 +16,7 @@ import {
|
||||
type PublishRequest,
|
||||
type VideoCaptureSettings,
|
||||
} from "../../media-api";
|
||||
import { convertToLivekitProcessor } from "./videoProcessor";
|
||||
import { toLivekitProcessor } from "./videoProcessor";
|
||||
|
||||
export type PublishRequestFor<S extends PublishRequest["source"]> = Extract<
|
||||
PublishRequest,
|
||||
@@ -40,7 +40,7 @@ export function videoCaptureOptions({
|
||||
if (!deviceId && !processor && !capture?.resolution) return undefined;
|
||||
return {
|
||||
...(deviceId && { deviceId }),
|
||||
...(processor && { processor: convertToLivekitProcessor(processor) }),
|
||||
...(processor && { processor: toLivekitProcessor(processor) }),
|
||||
...(capture?.resolution && { resolution: capture.resolution }),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -5,13 +5,18 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { Track, type VideoProcessorOptions } from "livekit-client";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
LocalVideoTrack,
|
||||
Track,
|
||||
type TrackProcessor,
|
||||
type VideoProcessorOptions,
|
||||
} from "livekit-client";
|
||||
import { beforeAll, describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { type VideoProcessor } from "../../media-api";
|
||||
import { convertToLivekitProcessor } from "./videoProcessor";
|
||||
import { fromLivekitProcessor, toLivekitProcessor } from "./videoProcessor";
|
||||
|
||||
describe("convertToLivekitProcessor", () => {
|
||||
describe("toLivekitProcessor", () => {
|
||||
const captured = { id: "captured" } as MediaStreamTrack;
|
||||
const processed = { id: "processed" } as MediaStreamTrack;
|
||||
const restarted = { id: "restarted" } as MediaStreamTrack;
|
||||
@@ -33,7 +38,7 @@ describe("convertToLivekitProcessor", () => {
|
||||
|
||||
it("holds the processed track where LiveKit reads it", async () => {
|
||||
const processor = fakeProcessor();
|
||||
const adapter = convertToLivekitProcessor(processor);
|
||||
const adapter = toLivekitProcessor(processor);
|
||||
expect(adapter.name).toBe("blur");
|
||||
expect(adapter.processedTrack).toBeUndefined();
|
||||
|
||||
@@ -59,8 +64,10 @@ describe("convertToLivekitProcessor", () => {
|
||||
finishDestroy = resolve;
|
||||
}),
|
||||
);
|
||||
const previous = convertToLivekitProcessor(processor);
|
||||
const next = convertToLivekitProcessor(processor);
|
||||
const previous = toLivekitProcessor(processor);
|
||||
const next = toLivekitProcessor(processor);
|
||||
await previous.init(options);
|
||||
vi.mocked(processor.init).mockClear();
|
||||
|
||||
const destroying = previous.destroy();
|
||||
const initialising = next.init(options);
|
||||
@@ -73,3 +80,174 @@ describe("convertToLivekitProcessor", () => {
|
||||
expect(next.processedTrack).toBe(processed);
|
||||
});
|
||||
});
|
||||
|
||||
describe("a processor set on a camera that is off", () => {
|
||||
const ended = { id: "ended", readyState: "ended" } as MediaStreamTrack;
|
||||
|
||||
function fakeProcessor(): VideoProcessor {
|
||||
return {
|
||||
name: "blur",
|
||||
init: vi.fn().mockResolvedValue({ id: "processed" }),
|
||||
restart: vi.fn().mockResolvedValue({ id: "restarted" }),
|
||||
destroy: vi.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
}
|
||||
|
||||
it("builds on the restart that turns the camera on, not before", async () => {
|
||||
const processor = fakeProcessor();
|
||||
const adapter = toLivekitProcessor(processor);
|
||||
|
||||
await adapter.init({ kind: Track.Kind.Video, track: ended });
|
||||
expect(processor.init).not.toHaveBeenCalled();
|
||||
expect(adapter.processedTrack).toBeUndefined();
|
||||
|
||||
const live = { id: "live", readyState: "live" } as MediaStreamTrack;
|
||||
await adapter.restart({ kind: Track.Kind.Video, track: live });
|
||||
expect(processor.init).toHaveBeenCalledWith(live, undefined);
|
||||
expect(processor.restart).not.toHaveBeenCalled();
|
||||
expect(adapter.processedTrack).toEqual({ id: "processed" });
|
||||
});
|
||||
|
||||
// The processor may be the app's one pipeline, running on another track
|
||||
it("leaves the processor alone when let go before it was built", async () => {
|
||||
const processor = fakeProcessor();
|
||||
const adapter = toLivekitProcessor(processor);
|
||||
await adapter.init({ kind: Track.Kind.Video, track: ended });
|
||||
await adapter.destroy();
|
||||
expect(processor.destroy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
// LiveKit's own camera track, so that the order of its restart is what is
|
||||
// tested: a raw frame handed to the sender would reach the room
|
||||
describe("on LiveKit's camera track", () => {
|
||||
// jsdom has no MediaStream; LiveKit wraps every track in one
|
||||
beforeAll(() => {
|
||||
const original = globalThis.MediaStream;
|
||||
globalThis.MediaStream = class {
|
||||
public constructor(private readonly tracks: MediaStreamTrack[] = []) {}
|
||||
public getTracks(): MediaStreamTrack[] {
|
||||
return this.tracks;
|
||||
}
|
||||
public getVideoTracks(): MediaStreamTrack[] {
|
||||
return this.tracks.filter(({ kind }) => kind === "video");
|
||||
}
|
||||
public getAudioTracks(): MediaStreamTrack[] {
|
||||
return this.tracks.filter(({ kind }) => kind === "audio");
|
||||
}
|
||||
public addTrack(track: MediaStreamTrack): void {
|
||||
this.tracks.push(track);
|
||||
}
|
||||
public removeTrack(track: MediaStreamTrack): void {
|
||||
this.tracks.splice(this.tracks.indexOf(track), 1);
|
||||
}
|
||||
} as unknown as typeof MediaStream;
|
||||
// LiveKit plays its processor element and catches a refusal
|
||||
const play = vi
|
||||
.spyOn(HTMLMediaElement.prototype, "play")
|
||||
.mockResolvedValue(undefined);
|
||||
return (): void => {
|
||||
globalThis.MediaStream = original;
|
||||
play.mockRestore();
|
||||
};
|
||||
});
|
||||
|
||||
function deviceTrack(id: string): MediaStreamTrack {
|
||||
const track = {
|
||||
kind: "video",
|
||||
id,
|
||||
label: "",
|
||||
enabled: true,
|
||||
readyState: "live",
|
||||
getSettings: () => ({}),
|
||||
getConstraints: () => ({}),
|
||||
getCapabilities: () => ({}),
|
||||
applyConstraints: vi.fn().mockResolvedValue(undefined),
|
||||
addEventListener: vi.fn(),
|
||||
removeEventListener: vi.fn(),
|
||||
stop: vi.fn(() => {
|
||||
Object.assign(track, { readyState: "ended" });
|
||||
}),
|
||||
};
|
||||
return track as unknown as MediaStreamTrack;
|
||||
}
|
||||
|
||||
it("hands the sender only processed frames when the camera turns on", async () => {
|
||||
const reacquired = deviceTrack("reacquired");
|
||||
Object.defineProperty(navigator, "mediaDevices", {
|
||||
configurable: true,
|
||||
value: {
|
||||
getUserMedia: vi.fn().mockResolvedValue({
|
||||
getTracks: () => [reacquired],
|
||||
}),
|
||||
},
|
||||
});
|
||||
const processedTrack = { id: "processed" } as MediaStreamTrack;
|
||||
const processor = {
|
||||
...fakeProcessor(),
|
||||
init: vi.fn().mockResolvedValue(processedTrack),
|
||||
};
|
||||
const camera = new LocalVideoTrack(deviceTrack("first"), {}, false);
|
||||
camera.source = Track.Source.Camera;
|
||||
const replaceTrack = vi.fn().mockResolvedValue(undefined);
|
||||
Object.assign(camera, {
|
||||
sender: {
|
||||
replaceTrack,
|
||||
getParameters: () => ({ encodings: [] }),
|
||||
setParameters: vi.fn().mockResolvedValue(undefined),
|
||||
},
|
||||
});
|
||||
|
||||
await camera.mute();
|
||||
expect(camera.mediaStreamTrack.readyState).toBe("ended");
|
||||
// What the camera sent before it was turned off is not in question
|
||||
replaceTrack.mockClear();
|
||||
await camera.setProcessor(toLivekitProcessor(processor));
|
||||
await camera.unmute();
|
||||
|
||||
expect(processor.init).toHaveBeenCalledWith(
|
||||
reacquired,
|
||||
expect.anything(),
|
||||
);
|
||||
expect(replaceTrack.mock.calls).toEqual([[processedTrack]]);
|
||||
// LiveKit plays attached elements again on a timer
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("fromLivekitProcessor", () => {
|
||||
const captured = { id: "captured" } as MediaStreamTrack;
|
||||
const processed = { id: "processed" } as MediaStreamTrack;
|
||||
|
||||
function fakeLivekitProcessor(
|
||||
processedTrack: MediaStreamTrack | undefined,
|
||||
): TrackProcessor<Track.Kind.Video> {
|
||||
return {
|
||||
name: "background-blur",
|
||||
processedTrack,
|
||||
init: vi.fn().mockResolvedValue(undefined),
|
||||
restart: vi.fn().mockResolvedValue(undefined),
|
||||
destroy: vi.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
}
|
||||
|
||||
it("resolves with the track the LiveKit processor produced", async () => {
|
||||
const livekitProcessor = fakeLivekitProcessor(processed);
|
||||
const processor = fromLivekitProcessor(livekitProcessor);
|
||||
expect(processor.name).toBe("background-blur");
|
||||
await expect(processor.init(captured)).resolves.toBe(processed);
|
||||
expect(livekitProcessor.init).toHaveBeenCalledWith({
|
||||
kind: Track.Kind.Video,
|
||||
track: captured,
|
||||
});
|
||||
await expect(processor.restart(captured)).resolves.toBe(processed);
|
||||
await processor.destroy();
|
||||
expect(livekitProcessor.destroy).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects where the LiveKit processor produced no track", async () => {
|
||||
await expect(
|
||||
fromLivekitProcessor(fakeLivekitProcessor(undefined)).init(captured),
|
||||
).rejects.toThrow("produced no track");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { type Track, type TrackProcessor } from "livekit-client";
|
||||
import { Track, type TrackProcessor } from "livekit-client";
|
||||
|
||||
import { type VideoProcessor } from "../../media-api";
|
||||
import { inTurn } from "../../utils/processorTurns";
|
||||
@@ -16,34 +16,73 @@ export interface LivekitProcessor extends TrackProcessor<Track.Kind.Video> {
|
||||
}
|
||||
|
||||
/**
|
||||
* A `VideoProcessor` as livekit-client wants it. LiveKit reads the processed
|
||||
* track off the processor after `init` and `restart` rather than taking it as
|
||||
* their result, so the adapter keeps it there. A processor a lobby already
|
||||
* runs on the track is adopted with its output, `running`, so that the join
|
||||
* does not initialise it again. Every step takes its turn on the processor,
|
||||
* which a stopped track tears down without waiting for the next one's init.
|
||||
* A `VideoProcessor` as a livekit-client processor. LiveKit reads the
|
||||
* processed track off the processor after `init` and `restart` rather than
|
||||
* taking it as their result, so the adapter keeps it there. A processor a
|
||||
* lobby already runs on the track is adopted with its output, `running`, so
|
||||
* that the join does not initialise it again. Every step takes its turn on
|
||||
* the processor, which a stopped track tears down without waiting for the
|
||||
* next one's init.
|
||||
*
|
||||
* Set on a camera that is off, whose track LiveKit has stopped, the build
|
||||
* waits for the restart that turns it back on. LiveKit restarts a processor
|
||||
* before it hands the new track to the sender, so the camera's first frame
|
||||
* out is already processed.
|
||||
*/
|
||||
export function convertToLivekitProcessor(
|
||||
export function toLivekitProcessor(
|
||||
processor: VideoProcessor,
|
||||
running?: MediaStreamTrack,
|
||||
): LivekitProcessor {
|
||||
let built = false;
|
||||
const adapter: LivekitProcessor = {
|
||||
name: processor.name,
|
||||
source: processor,
|
||||
init: async (opts) =>
|
||||
inTurn(processor, async () => {
|
||||
if (running === undefined && opts.track.readyState === "ended") return;
|
||||
adapter.processedTrack =
|
||||
running ?? (await processor.init(opts.track, opts.element));
|
||||
running = undefined;
|
||||
built = true;
|
||||
}),
|
||||
restart: async (opts) =>
|
||||
inTurn(processor, async () => {
|
||||
adapter.processedTrack = await processor.restart(
|
||||
opts.track,
|
||||
opts.element,
|
||||
);
|
||||
adapter.processedTrack = built
|
||||
? await processor.restart(opts.track, opts.element)
|
||||
: await processor.init(opts.track, opts.element);
|
||||
built = true;
|
||||
}),
|
||||
destroy: async () =>
|
||||
inTurn(processor, async () => {
|
||||
if (built) await processor.destroy();
|
||||
}),
|
||||
destroy: async () => inTurn(processor, async () => processor.destroy()),
|
||||
};
|
||||
return adapter;
|
||||
}
|
||||
|
||||
/**
|
||||
* A livekit-client processor, such as one from `@livekit/track-processors`,
|
||||
* as a `VideoProcessor`: `init` and `restart` resolve with the track it
|
||||
* leaves on itself.
|
||||
*/
|
||||
export function fromLivekitProcessor(
|
||||
processor: TrackProcessor<Track.Kind.Video>,
|
||||
): VideoProcessor {
|
||||
const processedTrack = (): MediaStreamTrack => {
|
||||
if (processor.processedTrack === undefined)
|
||||
throw new Error(`Processor ${processor.name} produced no track`);
|
||||
return processor.processedTrack;
|
||||
};
|
||||
return {
|
||||
name: processor.name,
|
||||
init: async (track, element) => {
|
||||
await processor.init({ kind: Track.Kind.Video, track, element });
|
||||
return processedTrack();
|
||||
},
|
||||
restart: async (track, element) => {
|
||||
await processor.restart({ kind: Track.Kind.Video, track, element });
|
||||
return processedTrack();
|
||||
},
|
||||
destroy: async () => processor.destroy(),
|
||||
};
|
||||
}
|
||||
|
||||
+3
-3
@@ -35,7 +35,7 @@ import { ErrorPage, LoadingPage } from "./FullScreenView";
|
||||
import { Initializer } from "./initializer";
|
||||
import { type WidgetHelpers } from "./widget";
|
||||
import { useTheme } from "./useTheme";
|
||||
import { ProcessorProvider } from "./tracks/TrackProcessorContext";
|
||||
import { BackgroundEffectsProvider } from "./tracks/BackgroundEffectsContext";
|
||||
import { type AppViewModel } from "./state/AppViewModel";
|
||||
import { MediaDevicesContext } from "./MediaDevicesContext";
|
||||
import {
|
||||
@@ -148,7 +148,7 @@ export const App: FC<Props> = ({ vm, widget }) => {
|
||||
loaded && clientReady ? (
|
||||
<ClientProvider client={widgetClient}>
|
||||
<MediaDevicesContext value={vm.mediaDevices}>
|
||||
<ProcessorProvider>
|
||||
<BackgroundEffectsProvider effects={vm.backgroundEffects}>
|
||||
<Sentry.ErrorBoundary
|
||||
fallback={(error) => <ErrorPage error={error} />}
|
||||
>
|
||||
@@ -159,7 +159,7 @@ export const App: FC<Props> = ({ vm, widget }) => {
|
||||
<SentryRoute path="*" element={<RoomPage />} />
|
||||
</Routes>
|
||||
</Sentry.ErrorBoundary>
|
||||
</ProcessorProvider>
|
||||
</BackgroundEffectsProvider>
|
||||
</MediaDevicesContext>
|
||||
</ClientProvider>
|
||||
) : (
|
||||
|
||||
@@ -13,7 +13,10 @@ import type { CallViewModel } from "../state/CallViewModel/CallViewModel";
|
||||
import type { Layout } from "../state/layout-types";
|
||||
import type { SpotlightTileViewModel } from "../state/TileViewModel";
|
||||
import type { DeviceLabel } from "../state/MediaDevices";
|
||||
import { createCallFooterViewModel } from "./CallFooterViewModel";
|
||||
import {
|
||||
createCallFooterViewModel,
|
||||
createLobbyFooterViewModel,
|
||||
} from "./CallFooterViewModel";
|
||||
import { HeaderStyle } from "../UrlParams";
|
||||
|
||||
const platformMock = vi.hoisted(() => vi.fn(() => "desktop"));
|
||||
@@ -23,10 +26,10 @@ vi.mock("../Platform", () => ({
|
||||
},
|
||||
}));
|
||||
|
||||
// Prevent supportsBackgroundProcessors from throwing in jsdom – it is not
|
||||
// exercised by these tests (only used in `videoToggles`, not `videoOptions`).
|
||||
// The SDK's own check needs WebGL and canvas APIs jsdom does not have.
|
||||
const sdkSupportMock = vi.hoisted(() => vi.fn(() => false));
|
||||
vi.mock("@livekit/track-processors", () => ({
|
||||
supportsBackgroundProcessors: (): boolean => false,
|
||||
supportsBackgroundProcessors: (): boolean => sdkSupportMock(),
|
||||
}));
|
||||
|
||||
const outputSelectionMock = vi.hoisted(() => vi.fn(() => true));
|
||||
@@ -217,4 +220,55 @@ describe("createCallFooterViewModel", () => {
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("background effects", () => {
|
||||
// The lobby, because in a call the whole device switcher is already
|
||||
// withheld on a phone, whatever the verdict says.
|
||||
function lobbyOn(
|
||||
platform: string,
|
||||
): ReturnType<typeof createLobbyFooterViewModel> {
|
||||
platformMock.mockReturnValue(platform);
|
||||
return createLobbyFooterViewModel(
|
||||
testScope(),
|
||||
mockMuteStates(),
|
||||
twoMicsAndOneCamMediaDevices,
|
||||
/* openSettings */ undefined,
|
||||
/* hangup */ undefined,
|
||||
/* showLogo */ false,
|
||||
);
|
||||
}
|
||||
|
||||
it("offers nothing on a phone, where the pipeline would refuse them", () => {
|
||||
sdkSupportMock.mockReturnValue(true);
|
||||
expect(lobbyOn("ios").toggleBlur$.value).toBeUndefined();
|
||||
});
|
||||
|
||||
it("offers them where the pipeline will honour them", () => {
|
||||
sdkSupportMock.mockReturnValue(true);
|
||||
expect(lobbyOn("desktop").toggleBlur$.value).toBeDefined();
|
||||
});
|
||||
|
||||
it("offers nothing where the browser itself cannot run them", () => {
|
||||
sdkSupportMock.mockReturnValue(false);
|
||||
expect(lobbyOn("desktop").toggleBlur$.value).toBeUndefined();
|
||||
});
|
||||
|
||||
it("availability is the same before and during a call", () => {
|
||||
for (const supported of [true, false]) {
|
||||
sdkSupportMock.mockReturnValue(supported);
|
||||
const lobby = lobbyOn("desktop");
|
||||
const inCall = createCallFooterViewModel(
|
||||
testScope(),
|
||||
buildMinimalCallViewModel(gridLayout),
|
||||
mockMuteStates(),
|
||||
twoMicsAndOneCamMediaDevices,
|
||||
/* reactionIdentifier */ undefined,
|
||||
{ showControls: true, header: HeaderStyle.Standard },
|
||||
);
|
||||
const offeredInLobby = lobby.toggleBlur$.value !== undefined;
|
||||
expect(inCall.toggleBlur$.value !== undefined).toBe(offeredInLobby);
|
||||
expect(offeredInLobby).toBe(supported);
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -6,9 +6,10 @@ Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { combineLatest, map, type Observable, switchMap } from "rxjs";
|
||||
import { supportsBackgroundProcessors } from "@livekit/track-processors";
|
||||
import { supportsAudioOutputSelection } from "livekit-client";
|
||||
|
||||
import { supportsBackgroundProcessors } from "../tracks/backgroundProcessing";
|
||||
|
||||
import { type CallViewModel } from "../state/CallViewModel/CallViewModel";
|
||||
import { type MenuOptions } from "./MediaMuteAndSwitchButton";
|
||||
import { type MediaDevices } from "../state/MediaDevices";
|
||||
|
||||
@@ -50,6 +50,7 @@ import {
|
||||
mockMediaDevices,
|
||||
mockRtcMembership,
|
||||
MockRTCSession,
|
||||
testScope,
|
||||
} from "../utils/test";
|
||||
import { CallView } from "./CallView";
|
||||
import { GroupCallErrorBoundary } from "./GroupCallErrorBoundary";
|
||||
@@ -61,7 +62,10 @@ import {
|
||||
} from "../HostBridge";
|
||||
import { type JoinCallData } from "../widget";
|
||||
import { MatrixRTCTransportMissingError } from "../utils/errors";
|
||||
import { ProcessorProvider } from "../tracks/TrackProcessorContext";
|
||||
import {
|
||||
BackgroundEffectsProvider,
|
||||
createAppBackgroundEffects,
|
||||
} from "../tracks/BackgroundEffectsContext";
|
||||
import { MediaDevicesContext } from "../MediaDevicesContext";
|
||||
import { constant } from "../state/Behavior";
|
||||
|
||||
@@ -193,7 +197,9 @@ function createCallView(
|
||||
<HostBridgeProvider value={hostBridge}>
|
||||
<TooltipProvider>
|
||||
<MediaDevicesContext value={mockMediaDevices({})}>
|
||||
<ProcessorProvider>
|
||||
<BackgroundEffectsProvider
|
||||
effects={createAppBackgroundEffects(testScope())}
|
||||
>
|
||||
{options.withErrorBoundary ? (
|
||||
<GroupCallErrorBoundary recoveryActionHandler={vi.fn()}>
|
||||
{callView}
|
||||
@@ -201,7 +207,7 @@ function createCallView(
|
||||
) : (
|
||||
callView
|
||||
)}
|
||||
</ProcessorProvider>
|
||||
</BackgroundEffectsProvider>
|
||||
</MediaDevicesContext>
|
||||
</TooltipProvider>
|
||||
</HostBridgeProvider>
|
||||
|
||||
@@ -28,6 +28,7 @@ import {
|
||||
type MockRTCSession,
|
||||
mockRTCSlot,
|
||||
mockRTCMember,
|
||||
testScope,
|
||||
} from "../utils/test";
|
||||
import { E2eeType } from "../e2ee/e2eeType";
|
||||
import {
|
||||
@@ -48,7 +49,10 @@ import { MediaDevicesContext } from "../MediaDevicesContext";
|
||||
import { type MediaDevices as ECMediaDevices } from "../state/MediaDevices";
|
||||
import { AppBar } from "../AppBar";
|
||||
import { type MatrixInfo } from "./VideoPreview";
|
||||
import { ProcessorProvider } from "../tracks/TrackProcessorContext";
|
||||
import {
|
||||
BackgroundEffectsProvider,
|
||||
createAppBackgroundEffects,
|
||||
} from "../tracks/BackgroundEffectsContext";
|
||||
import { initializeWidget } from "../widget";
|
||||
import { RootElementProvider } from "../RootElementContext";
|
||||
|
||||
@@ -246,7 +250,9 @@ describe("ActiveCall", () => {
|
||||
const { findByTestId } = render(
|
||||
<BrowserRouter>
|
||||
<MediaDevicesContext value={mediaDevices}>
|
||||
<ProcessorProvider>
|
||||
<BackgroundEffectsProvider
|
||||
effects={createAppBackgroundEffects(testScope())}
|
||||
>
|
||||
<TooltipProvider>
|
||||
<ActiveCall
|
||||
client={matrixRoom.client}
|
||||
@@ -260,7 +266,7 @@ describe("ActiveCall", () => {
|
||||
onLeft={(): void => {}}
|
||||
/>
|
||||
</TooltipProvider>
|
||||
</ProcessorProvider>
|
||||
</BackgroundEffectsProvider>
|
||||
</MediaDevicesContext>
|
||||
</BrowserRouter>,
|
||||
);
|
||||
@@ -295,7 +301,9 @@ describe("ActiveCall", () => {
|
||||
<BrowserRouter>
|
||||
<RootElementProvider value={root}>
|
||||
<MediaDevicesContext value={mediaDevices}>
|
||||
<ProcessorProvider>
|
||||
<BackgroundEffectsProvider
|
||||
effects={createAppBackgroundEffects(testScope())}
|
||||
>
|
||||
<TooltipProvider>
|
||||
<ActiveCall
|
||||
client={matrixRoom.client}
|
||||
@@ -309,7 +317,7 @@ describe("ActiveCall", () => {
|
||||
onLeft={(): void => {}}
|
||||
/>
|
||||
</TooltipProvider>
|
||||
</ProcessorProvider>
|
||||
</BackgroundEffectsProvider>
|
||||
</MediaDevicesContext>
|
||||
</RootElementProvider>
|
||||
</BrowserRouter>,
|
||||
|
||||
+11
-6
@@ -110,7 +110,7 @@ import { useBehavior } from "../useBehavior.ts";
|
||||
import { useValueBehavior } from "../useValueBehavior.ts";
|
||||
import { Toast } from "../Toast.tsx";
|
||||
import overlayStyles from "../Overlay.module.css";
|
||||
import { useTrackProcessorState$ } from "../tracks/TrackProcessorContext.tsx";
|
||||
import { useBackgroundEffectsState$ } from "../tracks/BackgroundEffectsContext";
|
||||
import { type Layout } from "../state/layout-types.ts";
|
||||
import { ObservableScope } from "../state/ObservableScope.ts";
|
||||
import { CallFooter, type FooterSnapshot } from "../components/CallFooter.tsx";
|
||||
@@ -153,7 +153,7 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
|
||||
const urlParams = useUrlParams();
|
||||
const hostBridge = useHostBridge();
|
||||
const mediaDevices = useMediaDevices();
|
||||
const trackProcessorState$ = useTrackProcessorState$();
|
||||
const backgroundEffectsState$ = useBackgroundEffectsState$();
|
||||
// The element we have to draw the call in: the page, or the container a host
|
||||
// gave us. Its size, not the window's, decides how the call is laid out.
|
||||
const rootElement = useRootElement();
|
||||
@@ -177,7 +177,12 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
|
||||
publish:
|
||||
props.media.requests.length > 0
|
||||
? props.media
|
||||
: initialPublishRequests(props.muteStates, mediaDevices, capture),
|
||||
: initialPublishRequests(
|
||||
props.muteStates,
|
||||
mediaDevices,
|
||||
capture,
|
||||
backgroundEffectsState$.value.processor,
|
||||
),
|
||||
});
|
||||
|
||||
const vm = createCallViewModel$(
|
||||
@@ -191,7 +196,7 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
|
||||
encryptionSystem: props.e2eeSystem,
|
||||
capture,
|
||||
videoProcessor$: scope.behavior(
|
||||
trackProcessorState$.pipe(map((state) => state.processor)),
|
||||
backgroundEffectsState$.pipe(map((state) => state.processor)),
|
||||
),
|
||||
hostBridge,
|
||||
autoLeaveWhenOthersLeft,
|
||||
@@ -221,7 +226,7 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
|
||||
urlParams,
|
||||
hostBridge,
|
||||
mediaDevices,
|
||||
trackProcessorState$,
|
||||
backgroundEffectsState$,
|
||||
props.client,
|
||||
props.media,
|
||||
rootElement,
|
||||
@@ -253,7 +258,7 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
|
||||
props.onLeft,
|
||||
urlParams,
|
||||
mediaDevices,
|
||||
trackProcessorState$,
|
||||
backgroundEffectsState$,
|
||||
props.client,
|
||||
vm,
|
||||
]);
|
||||
|
||||
@@ -19,7 +19,7 @@ import {
|
||||
import { KnockLobbyView } from "./KnockLobbyView";
|
||||
import { LeaveToHomeProvider } from "../LeaveToHomeContext";
|
||||
import { MediaDevicesContext } from "../MediaDevicesContext";
|
||||
import { type ProcessorState } from "../tracks/TrackProcessorContext";
|
||||
import { type BackgroundEffectsState } from "../state/BackgroundEffects";
|
||||
import { mockMediaDevices } from "../utils/test";
|
||||
|
||||
// One instance each: the lobby keys effects on them
|
||||
@@ -27,12 +27,12 @@ const stable = vi.hoisted(() => ({
|
||||
processorState$: undefined as unknown,
|
||||
media: undefined as unknown,
|
||||
}));
|
||||
vi.mock("../tracks/TrackProcessorContext", () => ({
|
||||
useTrackProcessorState$: (): Behavior<ProcessorState> =>
|
||||
vi.mock("../tracks/BackgroundEffectsContext", () => ({
|
||||
useBackgroundEffectsState$: (): Behavior<BackgroundEffectsState> =>
|
||||
(stable.processorState$ ??= constant({
|
||||
supported: false,
|
||||
processor: undefined,
|
||||
})) as Behavior<ProcessorState>,
|
||||
})) as Behavior<BackgroundEffectsState>,
|
||||
}));
|
||||
// Media that opens no device: the view is what is under test
|
||||
vi.mock("../useLocalMedia", () => ({
|
||||
|
||||
@@ -33,7 +33,7 @@ import {
|
||||
} from "../utils/test";
|
||||
import { type MediaDevices } from "../state/MediaDevices";
|
||||
import { MediaDevicesContext } from "../MediaDevicesContext";
|
||||
import { type ProcessorState } from "../tracks/TrackProcessorContext";
|
||||
import { type BackgroundEffectsState } from "../state/BackgroundEffects";
|
||||
import { type EncryptionSystem } from "../e2ee/sharedKeyManagement";
|
||||
import lobbyStyles from "./LobbyView.module.css";
|
||||
import headerStyles from "../Header.module.css";
|
||||
@@ -44,12 +44,12 @@ const leaveToHome = vi.fn();
|
||||
|
||||
// One instance: the view keys an effect on it
|
||||
const processorState$ = vi.hoisted(() => ({ current: undefined as unknown }));
|
||||
vi.mock("../tracks/TrackProcessorContext", () => ({
|
||||
useTrackProcessorState$: (): Behavior<ProcessorState> =>
|
||||
vi.mock("../tracks/BackgroundEffectsContext", () => ({
|
||||
useBackgroundEffectsState$: (): Behavior<BackgroundEffectsState> =>
|
||||
(processorState$.current ??= constant({
|
||||
supported: false,
|
||||
processor: undefined,
|
||||
})) as Behavior<ProcessorState>,
|
||||
})) as Behavior<BackgroundEffectsState>,
|
||||
}));
|
||||
|
||||
// Media that opens no device: the view is what is under test
|
||||
|
||||
@@ -31,7 +31,7 @@ import { LeaveToHomeLink } from "../button/LeaveToHomeLink";
|
||||
import { useLeaveToHome } from "../LeaveToHomeContext";
|
||||
import { useMediaDevices } from "../MediaDevicesContext";
|
||||
import { ObservableScope } from "../state/ObservableScope";
|
||||
import { useTrackProcessorState$ } from "../tracks/TrackProcessorContext";
|
||||
import { useBackgroundEffectsState$ } from "../tracks/BackgroundEffectsContext";
|
||||
import { useBehavior } from "../useBehavior";
|
||||
import { CallFooter, type FooterSnapshot } from "../components/CallFooter";
|
||||
import { useCallViewKeyboardShortcuts } from "../useCallViewKeyboardShortcuts";
|
||||
@@ -122,7 +122,7 @@ export const LobbyView: FC<Props> = ({
|
||||
|
||||
const devices = useMediaDevices();
|
||||
const videoInputId = useBehavior(devices.videoInput.selected$)?.id;
|
||||
const trackProcessorState$ = useTrackProcessorState$();
|
||||
const backgroundEffectsState$ = useBackgroundEffectsState$();
|
||||
|
||||
const [mediaVm, setMediaVm] = useState<LobbyMediaViewModel | null>(null);
|
||||
useEffect(() => {
|
||||
@@ -135,14 +135,14 @@ export const LobbyView: FC<Props> = ({
|
||||
devices,
|
||||
captureSettings(),
|
||||
mediaScope.behavior(
|
||||
trackProcessorState$.pipe(map((state) => state.processor)),
|
||||
backgroundEffectsState$.pipe(map((state) => state.processor)),
|
||||
),
|
||||
),
|
||||
);
|
||||
return (): void => {
|
||||
mediaScope.end();
|
||||
};
|
||||
}, [media, muteStates, devices, trackProcessorState$]);
|
||||
}, [media, muteStates, devices, backgroundEffectsState$]);
|
||||
const videoTrack = useBehavior(mediaVm?.videoTrack$ ?? noVideoTrack);
|
||||
|
||||
useEffect(() => {
|
||||
|
||||
@@ -27,7 +27,7 @@ import {
|
||||
import { PreferencesSettingsTab } from "./PreferencesSettingsTab";
|
||||
import { Slider } from "../Slider";
|
||||
import { DeviceSelection } from "./DeviceSelection";
|
||||
import { useTrackProcessor } from "../tracks/TrackProcessorContext";
|
||||
import { useBackgroundEffects } from "../tracks/BackgroundEffectsContext";
|
||||
import {
|
||||
DeveloperSettingsTab,
|
||||
type DeveloperSettingsSnapshot,
|
||||
@@ -78,7 +78,7 @@ export const SettingsModal: FC<Props> = ({
|
||||
|
||||
// Generate a `Checkbox` input to turn blur on or off.
|
||||
const BlurCheckbox: React.FC = (): ReactNode => {
|
||||
const { supported } = useTrackProcessor();
|
||||
const { supported } = useBackgroundEffects();
|
||||
|
||||
const [blurActive, setBlurActive] = useSetting(backgroundBlurSetting);
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ Please see LICENSE in the repository root for full details.
|
||||
|
||||
import { type AudioOutputOptions, MediaDevices } from "./MediaDevices";
|
||||
import { type ObservableScope } from "./ObservableScope";
|
||||
import { createAppBackgroundEffects } from "../tracks/BackgroundEffectsContext";
|
||||
|
||||
/**
|
||||
* The top-level state holder for the application.
|
||||
@@ -16,6 +17,7 @@ export class AppViewModel {
|
||||
this.scope,
|
||||
this.audioOutputOptions,
|
||||
);
|
||||
public readonly backgroundEffects = createAppBackgroundEffects(this.scope);
|
||||
|
||||
// TODO: Move more application logic here. The CallViewModel, at the very
|
||||
// least, ought to be accessible from this object.
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { BehaviorSubject, distinctUntilChanged, map } from "rxjs";
|
||||
import { type BackgroundProcessorWrapper } from "@livekit/track-processors";
|
||||
|
||||
import {
|
||||
type BackgroundEffects,
|
||||
createBackgroundEffects,
|
||||
type BackgroundEffectsOptions,
|
||||
type BackgroundEffectsState,
|
||||
} from "./BackgroundEffects";
|
||||
import { flushPromises, testScope, withTestScheduler } from "../utils/test";
|
||||
|
||||
/** A pipeline that records what it is switched to. */
|
||||
function fakePipeline(): {
|
||||
pipeline: BackgroundProcessorWrapper;
|
||||
switches: unknown[];
|
||||
/** Holds the next switch until it resolves, as a picture loading does. */
|
||||
holdNext: () => () => void;
|
||||
} {
|
||||
const switches: unknown[] = [];
|
||||
let slow: Promise<void> | undefined;
|
||||
const pipeline = {
|
||||
switchTo: vi.fn(async (options: unknown) => {
|
||||
switches.push(options);
|
||||
const held = slow;
|
||||
slow = undefined;
|
||||
await held;
|
||||
}),
|
||||
} as unknown as BackgroundProcessorWrapper;
|
||||
return {
|
||||
pipeline,
|
||||
switches,
|
||||
holdNext: () => {
|
||||
let release!: () => void;
|
||||
slow = new Promise((resolve) => (release = resolve));
|
||||
return release;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** One letter per state: idle, attached. */
|
||||
function letter(state: BackgroundEffectsState): string {
|
||||
return state.processor === undefined ? "i" : "a";
|
||||
}
|
||||
|
||||
describe("the pipeline's state", () => {
|
||||
function testState({
|
||||
effect,
|
||||
expected,
|
||||
}: {
|
||||
effect: string;
|
||||
expected: string;
|
||||
}): void {
|
||||
withTestScheduler(({ behavior, expectObservable }) => {
|
||||
const effects = createBackgroundEffects(testScope(), {
|
||||
supported: true,
|
||||
blur$: behavior(effect, { n: false, b: true }),
|
||||
pipeline: fakePipeline().pipeline,
|
||||
});
|
||||
expectObservable(
|
||||
effects.state$.pipe(map(letter), distinctUntilChanged()),
|
||||
).toBe(expected);
|
||||
});
|
||||
}
|
||||
|
||||
it("attaches on first use and stays attached", () =>
|
||||
testState({ effect: "nbn", expected: "ia" }));
|
||||
});
|
||||
|
||||
describe("background effects", () => {
|
||||
let blur$: BehaviorSubject<boolean>;
|
||||
let fake: ReturnType<typeof fakePipeline>;
|
||||
|
||||
function build(
|
||||
options: Partial<BackgroundEffectsOptions> = {},
|
||||
): BackgroundEffects {
|
||||
return createBackgroundEffects(testScope(), {
|
||||
supported: true,
|
||||
blur$,
|
||||
pipeline: fake.pipeline,
|
||||
...options,
|
||||
});
|
||||
}
|
||||
const blur = async (on: boolean): Promise<void> => {
|
||||
blur$.next(on);
|
||||
await flushPromises();
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
blur$ = new BehaviorSubject(false);
|
||||
fake = fakePipeline();
|
||||
});
|
||||
|
||||
it("switches in place rather than reattaching", async () => {
|
||||
const effects = build();
|
||||
await blur(true);
|
||||
const pipeline = effects.state$.value.processor;
|
||||
await blur(false);
|
||||
await blur(true);
|
||||
|
||||
expect(effects.state$.value.processor).toBe(pipeline);
|
||||
expect(fake.switches).toEqual([
|
||||
{ mode: "background-blur", blurRadius: 15 },
|
||||
{ mode: "disabled" },
|
||||
{ mode: "background-blur", blurRadius: 15 },
|
||||
]);
|
||||
});
|
||||
|
||||
it("keeps switching after a switch fails", async () => {
|
||||
vi.mocked(fake.pipeline.switchTo).mockRejectedValueOnce(new Error("lost"));
|
||||
build();
|
||||
await blur(true);
|
||||
await blur(false);
|
||||
expect(fake.switches).toEqual([{ mode: "disabled" }]);
|
||||
});
|
||||
|
||||
it("switches one at a time, skipping those overtaken", async () => {
|
||||
const finished = fake.holdNext();
|
||||
build();
|
||||
await blur(true);
|
||||
await blur(false);
|
||||
await blur(true);
|
||||
await blur(false);
|
||||
expect(fake.switches).toHaveLength(1);
|
||||
|
||||
finished();
|
||||
await flushPromises();
|
||||
expect(fake.switches).toEqual([
|
||||
{ mode: "background-blur", blurRadius: 15 },
|
||||
{ mode: "disabled" },
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,91 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { combineLatest, distinctUntilChanged, map, scan } from "rxjs";
|
||||
import {
|
||||
type BackgroundProcessorWrapper,
|
||||
type SwitchBackgroundProcessorOptions,
|
||||
} from "@livekit/track-processors";
|
||||
import { logger } from "matrix-js-sdk/lib/logger";
|
||||
import { deepCompare } from "matrix-js-sdk/lib/utils";
|
||||
import {
|
||||
fromLivekitProcessor,
|
||||
type VideoProcessor,
|
||||
} from "@element-hq/matrixrtc-sdk";
|
||||
|
||||
import { type Behavior } from "./Behavior";
|
||||
import { type ObservableScope } from "./ObservableScope";
|
||||
|
||||
const blurRadius = 15;
|
||||
|
||||
export interface BackgroundEffectsState {
|
||||
supported: boolean | undefined;
|
||||
processor: undefined | VideoProcessor;
|
||||
}
|
||||
|
||||
export interface BackgroundEffectsOptions {
|
||||
/** Whether this browser can run a pipeline at all. */
|
||||
supported: boolean;
|
||||
/** Whether blur is chosen. */
|
||||
blur$: Behavior<boolean>;
|
||||
/** The background processor pipeline to be switched between effects. */
|
||||
pipeline: BackgroundProcessorWrapper;
|
||||
}
|
||||
|
||||
/** The background effect pipeline, as the camera tracks and the menus see it. */
|
||||
export interface BackgroundEffects {
|
||||
readonly state$: Behavior<BackgroundEffectsState>;
|
||||
}
|
||||
|
||||
/** Switches the pipeline as the choice changes, for as long as the scope lasts. */
|
||||
export function createBackgroundEffects(
|
||||
scope: ObservableScope,
|
||||
{ supported, blur$, pipeline }: BackgroundEffectsOptions,
|
||||
): BackgroundEffects {
|
||||
// Wrapped once, so the call and the preview see one processor throughout.
|
||||
const processor = fromLivekitProcessor(pipeline);
|
||||
const state$ = scope.behavior(
|
||||
blur$.pipe(
|
||||
scan<boolean, BackgroundEffectsState>(
|
||||
(previous, wanted) => {
|
||||
// Attached the first time an effect is wanted and never detached
|
||||
// after, so someone who never turns one on pays for none of it.
|
||||
const enable =
|
||||
previous.processor !== undefined || (supported && wanted);
|
||||
return { supported, processor: enable ? processor : undefined };
|
||||
},
|
||||
{ supported, processor: undefined },
|
||||
),
|
||||
),
|
||||
);
|
||||
|
||||
const switchOptions$ = scope.behavior<
|
||||
SwitchBackgroundProcessorOptions | undefined
|
||||
>(
|
||||
combineLatest([state$, blur$]).pipe(
|
||||
map(([{ processor }, blur]) =>
|
||||
processor === undefined
|
||||
? undefined
|
||||
: blur
|
||||
? { mode: "background-blur", blurRadius }
|
||||
: { mode: "disabled" },
|
||||
),
|
||||
distinctUntilChanged(deepCompare),
|
||||
),
|
||||
);
|
||||
// In turn: a picture's switch ends once it loads, so an earlier one could land last.
|
||||
scope.reconcile(switchOptions$, async (options) => {
|
||||
if (options === undefined) return;
|
||||
try {
|
||||
await pipeline.switchTo(options);
|
||||
} catch (e) {
|
||||
logger.warn("Failed to switch background effect", e);
|
||||
}
|
||||
});
|
||||
|
||||
return { state$ };
|
||||
}
|
||||
@@ -28,6 +28,8 @@ import { type Layout } from "../layout-types.ts";
|
||||
import {
|
||||
mockMatrixRoomMember,
|
||||
mockMediaDevices,
|
||||
mockMuteStates,
|
||||
flushPromises,
|
||||
mockRemoteParticipant,
|
||||
withTestScheduler,
|
||||
mockRtcMembership,
|
||||
@@ -62,7 +64,15 @@ import {
|
||||
import { MatrixRTCMode } from "../../config/ConfigOptions.ts";
|
||||
import { initializeWidget } from "../../widget.ts";
|
||||
import { computeUrlParams } from "../../UrlParams.ts";
|
||||
import { callViewModelOptionsFromParams } from "./CallViewModel.ts";
|
||||
import {
|
||||
callViewModelOptionsFromParams,
|
||||
initialPublishRequests,
|
||||
} from "./CallViewModel.ts";
|
||||
import { type Handler, MuteState } from "../MuteStates.ts";
|
||||
import {
|
||||
type RTCParticipation,
|
||||
type VideoProcessor,
|
||||
} from "@element-hq/matrixrtc-sdk";
|
||||
|
||||
initializeWidget();
|
||||
|
||||
@@ -1664,3 +1674,50 @@ describe("callViewModelOptionsFromParams", () => {
|
||||
expect(callViewModelOptionsFromParams(params).hideScreensharing).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("a camera with a video processor", () => {
|
||||
const withCallViewModel = withCallViewModelInMode(MatrixRTCMode.Matrix_2_0);
|
||||
const processor = { name: "effect" } as VideoProcessor;
|
||||
|
||||
/** The handler the view model gives the camera's mute switch. */
|
||||
const cameraHandler = (setHandler: {
|
||||
mock: { calls: [Handler][] };
|
||||
}): Handler => setHandler.mock.calls[1][0];
|
||||
|
||||
it("starts each camera track with the processor", () => {
|
||||
const muteStates = mockMuteStates();
|
||||
vi.spyOn(muteStates.video.enabled$, "value", "get").mockReturnValue(true);
|
||||
expect(
|
||||
initialPublishRequests(
|
||||
muteStates,
|
||||
mockMediaDevices({}),
|
||||
{},
|
||||
processor,
|
||||
).find((request) => request.source === "camera"),
|
||||
).toMatchObject({ processor });
|
||||
});
|
||||
|
||||
// The SDK builds the processor on the camera's restart, before the room
|
||||
// sees a frame, so the track is unmuted rather than published anew
|
||||
it("turns on a camera turned off before the processor was chosen", async () => {
|
||||
const setHandler = vi.spyOn(MuteState.prototype, "setHandler");
|
||||
onTestFinished(() => setHandler.mockRestore());
|
||||
const videoProcessor$ = new BehaviorSubject<VideoProcessor | undefined>(
|
||||
undefined,
|
||||
);
|
||||
let participation: RTCParticipation | undefined;
|
||||
withCallViewModel(
|
||||
{},
|
||||
(_vm, _session, { rtcParticipation }) => {
|
||||
participation = rtcParticipation;
|
||||
},
|
||||
{ videoProcessor$ },
|
||||
);
|
||||
videoProcessor$.next(processor);
|
||||
await cameraHandler(setHandler)(true);
|
||||
await flushPromises();
|
||||
|
||||
expect(participation!.unpublish).not.toHaveBeenCalled();
|
||||
expect(participation!.publish).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -140,14 +140,16 @@ export interface CaptureSettings {
|
||||
}
|
||||
|
||||
/**
|
||||
* What to publish for a microphone or camera: the device the user picked and
|
||||
* the host's capture settings. The video processor is not in here; the call
|
||||
* attaches it to the camera track once that exists.
|
||||
* What to publish for a microphone or camera: the device the user picked, the
|
||||
* host's capture settings and, for the camera, the video processor, so that
|
||||
* the track starts with it rather than letting the room through while it is
|
||||
* built.
|
||||
*/
|
||||
export function publishRequest(
|
||||
source: "microphone" | "camera",
|
||||
mediaDevices: MediaDevices,
|
||||
capture: CaptureSettings,
|
||||
processor?: VideoProcessor,
|
||||
): PublishRequest {
|
||||
return source === "microphone"
|
||||
? {
|
||||
@@ -159,6 +161,7 @@ export function publishRequest(
|
||||
source,
|
||||
deviceId: mediaDevices.videoInput.selected$.value?.id,
|
||||
capture: capture.camera,
|
||||
...(processor && { processor }),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -167,12 +170,13 @@ export function initialPublishRequests(
|
||||
muteStates: MuteStates,
|
||||
mediaDevices: MediaDevices,
|
||||
capture: CaptureSettings,
|
||||
processor?: VideoProcessor,
|
||||
): PublishRequest[] {
|
||||
const requests: PublishRequest[] = [];
|
||||
if (muteStates.audio.enabled$.value)
|
||||
requests.push(publishRequest("microphone", mediaDevices, capture));
|
||||
if (muteStates.video.enabled$.value)
|
||||
requests.push(publishRequest("camera", mediaDevices, capture));
|
||||
requests.push(publishRequest("camera", mediaDevices, capture, processor));
|
||||
return requests;
|
||||
}
|
||||
|
||||
@@ -510,7 +514,14 @@ export function createCallViewModel$(
|
||||
|
||||
const supply = (source: "microphone" | "camera"): TrackSupply => ({
|
||||
publish: async () =>
|
||||
rtcParticipation.publish(publishRequest(source, mediaDevices, capture)),
|
||||
rtcParticipation.publish(
|
||||
publishRequest(
|
||||
source,
|
||||
mediaDevices,
|
||||
capture,
|
||||
source === "camera" ? videoProcessor$.value : undefined,
|
||||
),
|
||||
),
|
||||
});
|
||||
followMuteState(
|
||||
scope,
|
||||
|
||||
@@ -29,7 +29,8 @@ describe("LobbyMediaViewModel", () => {
|
||||
|
||||
expect(media.capture).toHaveBeenCalledExactlyOnceWith([
|
||||
expect.objectContaining({ source: "microphone", deviceId: "mic" }),
|
||||
expect.objectContaining({ source: "camera", deviceId: "cam" }),
|
||||
// With the processor, so that the camera starts with it
|
||||
expect.objectContaining({ source: "camera", deviceId: "cam", processor }),
|
||||
]);
|
||||
const camera = vm.videoTrack$.value!;
|
||||
expect(camera.source).toBe("camera");
|
||||
|
||||
@@ -71,7 +71,14 @@ export function createLobbyMediaViewModel(
|
||||
if (audio && microphone === undefined && !opening.has("microphone"))
|
||||
open.push(publishRequest("microphone", mediaDevices, capture));
|
||||
if (video && camera === undefined && !opening.has("camera"))
|
||||
open.push(publishRequest("camera", mediaDevices, capture));
|
||||
open.push(
|
||||
publishRequest(
|
||||
"camera",
|
||||
mediaDevices,
|
||||
capture,
|
||||
videoProcessor$.value,
|
||||
),
|
||||
);
|
||||
if (open.length > 0) {
|
||||
for (const { source } of open) opening.add(source);
|
||||
media
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { BackgroundEffectTransformer } from "./BackgroundEffectTransformer";
|
||||
|
||||
describe("BackgroundEffectTransformer", () => {
|
||||
it("passes frames on untouched once disabled, even after blurring", async () => {
|
||||
const transformer = new BackgroundEffectTransformer({});
|
||||
await transformer.update({ blurRadius: 15, backgroundDisabled: false });
|
||||
await transformer.update({
|
||||
imagePath: undefined,
|
||||
backgroundDisabled: true,
|
||||
});
|
||||
// With neither, the library passes a frame on without segmenting it.
|
||||
expect(transformer.options.blurRadius).toBeUndefined();
|
||||
expect(transformer.options.imagePath).toBeUndefined();
|
||||
});
|
||||
});
|
||||
+19
-1
@@ -1,5 +1,6 @@
|
||||
/*
|
||||
Copyright 2024-2025 New Vector Ltd.
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
@@ -9,6 +10,7 @@ import {
|
||||
BackgroundTransformer,
|
||||
VideoTransformer,
|
||||
type VideoTransformerInitOptions,
|
||||
type BackgroundOptions,
|
||||
} from "@livekit/track-processors";
|
||||
import { ImageSegmenter } from "@mediapipe/tasks-vision";
|
||||
|
||||
@@ -49,7 +51,21 @@ const wasmFileset: WasmFileset = {
|
||||
* loads the segmentation models from our own bundle rather than as an external
|
||||
* resource fetched from the public internet.
|
||||
*/
|
||||
export class BlurBackgroundTransformer extends BackgroundTransformer {
|
||||
export class BackgroundEffectTransformer extends BackgroundTransformer {
|
||||
/**
|
||||
* As the library's, except that disabling also drops the blur radius. The
|
||||
* library skips a frame on `isDisabled ?? backgroundDisabled`, and
|
||||
* `isDisabled` is never undefined, so a disabled effect that keeps its
|
||||
* radius has every frame segmented and drawn for nothing. With neither a
|
||||
* radius nor a picture, frames are passed on untouched.
|
||||
* TODO: drop once fixed upstream in livekit/track-processors-js (still in 0.8.2).
|
||||
*/
|
||||
public override async update(opts: BackgroundOptions): Promise<void> {
|
||||
await super.update(
|
||||
opts.backgroundDisabled ? { ...opts, blurRadius: undefined } : opts,
|
||||
);
|
||||
}
|
||||
|
||||
public async init({
|
||||
outputCanvas,
|
||||
inputElement: inputVideo,
|
||||
@@ -73,8 +89,10 @@ export class BlurBackgroundTransformer extends BackgroundTransformer {
|
||||
outputConfidenceMasks: false,
|
||||
});
|
||||
|
||||
// BackgroundTransformer's own init applies these, and this one replaces it.
|
||||
if (this.options.blurRadius) {
|
||||
this.gl?.setBlurRadius(this.options.blurRadius);
|
||||
}
|
||||
this.gl?.setBackgroundDisabled(this.options.backgroundDisabled ?? false);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { act, createElement, type FC } from "react";
|
||||
import { render } from "@testing-library/react";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
import {
|
||||
BackgroundEffectsProvider,
|
||||
useBackgroundEffects,
|
||||
createAppBackgroundEffects,
|
||||
} from "./BackgroundEffectsContext";
|
||||
import {
|
||||
type BackgroundEffects,
|
||||
type BackgroundEffectsState,
|
||||
} from "../state/BackgroundEffects";
|
||||
import { backgroundBlur } from "../settings/settings";
|
||||
import { flushPromises, testScope } from "../utils/test";
|
||||
|
||||
const pipelines = vi.hoisted(() => ({
|
||||
built: 0,
|
||||
destroyed: 0,
|
||||
switches: [] as unknown[],
|
||||
}));
|
||||
|
||||
// The pipeline itself needs WebGL and MediaPipe; what these tests are about is
|
||||
// what the provider asks of it.
|
||||
vi.mock("@livekit/track-processors", () => ({
|
||||
BackgroundProcessorWrapper: vi.fn(function (this: Record<string, unknown>) {
|
||||
pipelines.built += 1;
|
||||
this.switchTo = vi.fn(async (options: unknown) => {
|
||||
pipelines.switches.push(options);
|
||||
return Promise.resolve();
|
||||
});
|
||||
this.destroy = vi.fn(async () => {
|
||||
pipelines.destroyed += 1;
|
||||
return Promise.resolve();
|
||||
});
|
||||
}),
|
||||
supportsBackgroundProcessors: (): boolean => true,
|
||||
}));
|
||||
const platformMock = vi.hoisted(() => ({ platform: "desktop" }));
|
||||
vi.mock("../Platform", () => platformMock);
|
||||
vi.mock("./BackgroundEffectTransformer", () => ({
|
||||
BackgroundEffectTransformer: vi.fn(),
|
||||
}));
|
||||
|
||||
describe("BackgroundEffectsProvider", () => {
|
||||
let seen: BackgroundEffectsState[];
|
||||
const Surface: FC = () => {
|
||||
seen.push(useBackgroundEffects());
|
||||
return null;
|
||||
};
|
||||
// Made on first render, once a test has set up the platform.
|
||||
let effects: BackgroundEffects | undefined;
|
||||
const surface = (): ReturnType<typeof createElement> =>
|
||||
createElement(BackgroundEffectsProvider, {
|
||||
effects: (effects ??= createAppBackgroundEffects(testScope())),
|
||||
children: createElement(Surface),
|
||||
});
|
||||
const blur = async (on: boolean): Promise<void> => {
|
||||
await act(async () => {
|
||||
backgroundBlur.setValue(on);
|
||||
await flushPromises();
|
||||
});
|
||||
};
|
||||
const latest = (): BackgroundEffectsState => seen[seen.length - 1];
|
||||
|
||||
beforeEach(() => {
|
||||
seen = [];
|
||||
effects = undefined;
|
||||
pipelines.built = 0;
|
||||
pipelines.destroyed = 0;
|
||||
pipelines.switches = [];
|
||||
platformMock.platform = "desktop";
|
||||
backgroundBlur.setValue(false);
|
||||
});
|
||||
afterEach(() => backgroundBlur.setValue(false));
|
||||
|
||||
// A pipeline primes itself when it is built and when it is destroyed, and a
|
||||
// primed pipeline lets one frame through untouched: so the frame is spent
|
||||
// once, as long as it is built once and never taken off the camera.
|
||||
it("builds one pipeline on first use and keeps it through every change", async () => {
|
||||
render(surface());
|
||||
for (const on of [true, false, true, false, true]) await blur(on);
|
||||
|
||||
expect(pipelines.built).toBe(1);
|
||||
expect(pipelines.destroyed).toBe(0);
|
||||
expect(latest().processor).toBeDefined();
|
||||
});
|
||||
|
||||
it("does not run on a phone, even where its browser could", async () => {
|
||||
platformMock.platform = "ios";
|
||||
render(surface());
|
||||
await blur(true);
|
||||
|
||||
expect(latest().supported).toBe(false);
|
||||
expect(latest().processor).toBeUndefined();
|
||||
});
|
||||
|
||||
it("switches to an effect chosen while the camera is off", async () => {
|
||||
render(surface());
|
||||
await blur(true);
|
||||
|
||||
// Already switched to blur, with nothing yet to attach it to
|
||||
expect(pipelines.switches).toEqual([
|
||||
{ mode: "background-blur", blurRadius: 15 },
|
||||
]);
|
||||
expect(pipelines.built).toBe(1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,67 @@
|
||||
/*
|
||||
Copyright 2024-2026 New Vector Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { BackgroundProcessorWrapper } from "@livekit/track-processors";
|
||||
import { createContext, type FC, type JSX, use } from "react";
|
||||
|
||||
import { backgroundBlur as backgroundBlurSettings } from "../settings/settings";
|
||||
import { BackgroundEffectTransformer } from "./BackgroundEffectTransformer";
|
||||
import { supportsBackgroundProcessors } from "./backgroundProcessing";
|
||||
import { type Behavior } from "../state/Behavior";
|
||||
import { type ObservableScope } from "../state/ObservableScope";
|
||||
import {
|
||||
type BackgroundEffects,
|
||||
type BackgroundEffectsState,
|
||||
createBackgroundEffects,
|
||||
} from "../state/BackgroundEffects";
|
||||
import { useBehavior } from "../useBehavior";
|
||||
|
||||
const BackgroundEffectsContext = createContext<BackgroundEffects | undefined>(
|
||||
undefined,
|
||||
);
|
||||
|
||||
export function useBackgroundEffects(): BackgroundEffectsState {
|
||||
return useBehavior(useBackgroundEffectsState$());
|
||||
}
|
||||
|
||||
export function useBackgroundEffectsState$(): Behavior<BackgroundEffectsState> {
|
||||
const effects = use(BackgroundEffectsContext);
|
||||
if (effects === undefined)
|
||||
throw new Error(
|
||||
"useBackgroundEffects must be used within a BackgroundEffectsProvider",
|
||||
);
|
||||
return effects.state$;
|
||||
}
|
||||
|
||||
/**
|
||||
* The app's one pipeline, shared by every camera track it opens. Its builds
|
||||
* and teardowns take their turns through the SDK, which runs one operation
|
||||
* at a time on a processor.
|
||||
*/
|
||||
export function createAppBackgroundEffects(
|
||||
scope: ObservableScope,
|
||||
): BackgroundEffects {
|
||||
return createBackgroundEffects(scope, {
|
||||
supported: supportsBackgroundProcessors(),
|
||||
blur$: backgroundBlurSettings.value$,
|
||||
pipeline: new BackgroundProcessorWrapper(
|
||||
new BackgroundEffectTransformer({ backgroundDisabled: true }),
|
||||
"background-effect",
|
||||
),
|
||||
});
|
||||
}
|
||||
|
||||
interface Props {
|
||||
effects: BackgroundEffects;
|
||||
children: JSX.Element;
|
||||
}
|
||||
|
||||
export const BackgroundEffectsProvider: FC<Props> = ({ effects, children }) => (
|
||||
<BackgroundEffectsContext value={effects}>
|
||||
{children}
|
||||
</BackgroundEffectsContext>
|
||||
);
|
||||
@@ -1,51 +0,0 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
type BackgroundOptions,
|
||||
type ProcessorWrapper,
|
||||
} from "@livekit/track-processors";
|
||||
|
||||
import { fromLivekitProcessor } from "./TrackProcessorContext";
|
||||
|
||||
describe("fromLivekitProcessor", () => {
|
||||
const captured = { id: "captured" } as MediaStreamTrack;
|
||||
const processed = { id: "processed" } as MediaStreamTrack;
|
||||
|
||||
function mockWrapper(
|
||||
processedTrack: MediaStreamTrack | undefined,
|
||||
): ProcessorWrapper<BackgroundOptions> {
|
||||
return {
|
||||
name: "background-blur",
|
||||
processedTrack,
|
||||
init: vi.fn().mockResolvedValue(undefined),
|
||||
restart: vi.fn().mockResolvedValue(undefined),
|
||||
destroy: vi.fn().mockResolvedValue(undefined),
|
||||
} as unknown as ProcessorWrapper<BackgroundOptions>;
|
||||
}
|
||||
|
||||
it("resolves with the track the wrapper produced", async () => {
|
||||
const wrapper = mockWrapper(processed);
|
||||
const sdkProcessor = fromLivekitProcessor(wrapper);
|
||||
expect(sdkProcessor.name).toBe("background-blur");
|
||||
await expect(sdkProcessor.init(captured)).resolves.toBe(processed);
|
||||
expect(wrapper.init).toHaveBeenCalledWith({
|
||||
kind: "video",
|
||||
track: captured,
|
||||
});
|
||||
await expect(sdkProcessor.restart(captured)).resolves.toBe(processed);
|
||||
await sdkProcessor.destroy();
|
||||
expect(wrapper.destroy).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects where the wrapper produced no track", async () => {
|
||||
await expect(
|
||||
fromLivekitProcessor(mockWrapper(undefined)).init(captured),
|
||||
).rejects.toThrow("produced no track");
|
||||
});
|
||||
});
|
||||
@@ -1,110 +0,0 @@
|
||||
/*
|
||||
Copyright 2024-2025 New Vector Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import {
|
||||
ProcessorWrapper,
|
||||
supportsBackgroundProcessors as supportsBackgroundProcessorsLivekitSdk,
|
||||
type BackgroundOptions,
|
||||
} from "@livekit/track-processors";
|
||||
import { createContext, type FC, type JSX, use, useMemo } from "react";
|
||||
import { Track } from "livekit-client";
|
||||
import { type VideoProcessor } from "@element-hq/matrixrtc-sdk";
|
||||
|
||||
import {
|
||||
backgroundBlur as backgroundBlurSettings,
|
||||
useSetting,
|
||||
} from "../settings/settings";
|
||||
import { BlurBackgroundTransformer } from "./BlurBackgroundTransformer";
|
||||
import { type Behavior } from "../state/Behavior";
|
||||
import { platform } from "../Platform";
|
||||
import { useValueBehavior } from "../useValueBehavior";
|
||||
|
||||
//TODO-MULTI-SFU: This is not yet fully there.
|
||||
// it is a combination of exposing observable and react hooks.
|
||||
// preferably we should not make this a context anymore and instead just a vm?
|
||||
|
||||
export type ProcessorState = {
|
||||
supported: boolean | undefined;
|
||||
processor: undefined | VideoProcessor;
|
||||
};
|
||||
|
||||
/**
|
||||
* This allows to use a LiveKit processor for the matrixRTCSDK (VideoProcessor).
|
||||
* They have slightly different apis so we need to convert them with this
|
||||
* function before we can use them in the matrixRTCSDK.
|
||||
*/
|
||||
export function fromLivekitProcessor(
|
||||
wrapper: ProcessorWrapper<BackgroundOptions>,
|
||||
): VideoProcessor {
|
||||
const processedTrack = (): MediaStreamTrack => {
|
||||
if (wrapper.processedTrack === undefined)
|
||||
throw new Error(`Processor ${wrapper.name} produced no track`);
|
||||
return wrapper.processedTrack;
|
||||
};
|
||||
return {
|
||||
name: wrapper.name,
|
||||
init: async (track: MediaStreamTrack, element?: HTMLMediaElement) => {
|
||||
await wrapper.init({ kind: Track.Kind.Video, track, element });
|
||||
return processedTrack();
|
||||
},
|
||||
restart: async (track: MediaStreamTrack, element?: HTMLMediaElement) => {
|
||||
await wrapper.restart({ kind: Track.Kind.Video, track, element });
|
||||
return processedTrack();
|
||||
},
|
||||
destroy: async () => wrapper.destroy(),
|
||||
};
|
||||
}
|
||||
|
||||
const ProcessorContext = createContext<ProcessorState | undefined>(undefined);
|
||||
|
||||
export function useTrackProcessor(): ProcessorState {
|
||||
const state = use(ProcessorContext);
|
||||
if (state === undefined)
|
||||
throw new Error(
|
||||
"useTrackProcessor must be used within a ProcessorProvider",
|
||||
);
|
||||
return state;
|
||||
}
|
||||
|
||||
export function useTrackProcessorState$(): Behavior<ProcessorState> {
|
||||
return useValueBehavior(useTrackProcessor());
|
||||
}
|
||||
|
||||
interface Props {
|
||||
children: JSX.Element;
|
||||
}
|
||||
|
||||
function supportsBackgroundProcessors(): boolean {
|
||||
return supportsBackgroundProcessorsLivekitSdk() && platform === "desktop";
|
||||
}
|
||||
|
||||
export const ProcessorProvider: FC<Props> = ({ children }) => {
|
||||
// The setting the user wants to have
|
||||
const [blurActivated] = useSetting(backgroundBlurSettings);
|
||||
const supported = useMemo(() => supportsBackgroundProcessors(), []);
|
||||
const blur = useMemo(
|
||||
() =>
|
||||
fromLivekitProcessor(
|
||||
new ProcessorWrapper(
|
||||
new BlurBackgroundTransformer({ blurRadius: 15 }),
|
||||
"background-blur",
|
||||
),
|
||||
),
|
||||
[],
|
||||
);
|
||||
|
||||
// This is the actual state exposed through the context
|
||||
const processorState = useMemo(
|
||||
() => ({
|
||||
supported,
|
||||
processor: supported && blurActivated ? blur : undefined,
|
||||
}),
|
||||
[supported, blurActivated, blur],
|
||||
);
|
||||
|
||||
return <ProcessorContext value={processorState}>{children}</ProcessorContext>;
|
||||
};
|
||||
@@ -0,0 +1,15 @@
|
||||
/*
|
||||
Copyright 2026 Element Creations Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { supportsBackgroundProcessors as supportsBackgroundProcessorsLivekitSdk } from "@livekit/track-processors";
|
||||
|
||||
import { platform } from "../Platform";
|
||||
|
||||
/** One answer for the controls and the pipeline, so neither offers what the other refuses. */
|
||||
export function supportsBackgroundProcessors(): boolean {
|
||||
return supportsBackgroundProcessorsLivekitSdk() && platform === "desktop";
|
||||
}
|
||||
Reference in New Issue
Block a user