diff --git a/sdk/src/LocalMedia.test.ts b/sdk/src/LocalMedia.test.ts index 25f8b9b70..f61be6716 100644 --- a/sdk/src/LocalMedia.test.ts +++ b/sdk/src/LocalMedia.test.ts @@ -162,6 +162,7 @@ describe("createLocalMedia", () => { media.release("camera"); expect(trackBehind(camera).stop).toHaveBeenCalled(); expect(media.tracks$.value).toEqual([microphone]); + expect(media.requests.map((r) => r.source)).toEqual(["microphone"]); media.stop(); expect(trackBehind(microphone).stop).toHaveBeenCalled(); diff --git a/sdk/src/LocalMedia.ts b/sdk/src/LocalMedia.ts index 1bb934c3d..7825566ad 100644 --- a/sdk/src/LocalMedia.ts +++ b/sdk/src/LocalMedia.ts @@ -107,6 +107,7 @@ export function createLocalMedia( if (source !== "microphone" && source !== "camera") return; capturedTracks.get(source)?.stop(); capturedTracks.delete(source); + requests.delete(source); publishCapturedTracks(); }; diff --git a/src/tracks/BackgroundEffectTransformer.test.ts b/src/tracks/BackgroundEffectTransformer.test.ts index 09de8ef5f..cceead032 100644 --- a/src/tracks/BackgroundEffectTransformer.test.ts +++ b/src/tracks/BackgroundEffectTransformer.test.ts @@ -20,6 +20,7 @@ import { const frame = {} as VideoFrame; const controller = {} as TransformStreamDefaultController; +const stats = { processingTimeMs: 1, segmentationTimeMs: 1, filterTimeMs: 0 }; describe("BackgroundEffectTransformer", () => { afterEach(() => { @@ -48,9 +49,10 @@ describe("BackgroundEffectTransformer", () => { }); it("reports the first frame carrying an effect, once", async () => { + // The library reports every frame it draws. vi.spyOn(BackgroundTransformer.prototype, "transform").mockImplementation( async function (this: BackgroundTransformer) { - this.isFirstFrame = false; + this.options.onFrameProcessed?.(stats); return Promise.resolve(); }, ); @@ -73,6 +75,37 @@ describe("BackgroundEffectTransformer", () => { expect(transformer.onFirstFrame).not.toHaveBeenCalled(); }); + it("keeps reporting drawn frames to whoever asked", async () => { + vi.spyOn(BackgroundTransformer.prototype, "transform").mockImplementation( + async function (this: BackgroundTransformer) { + this.options.onFrameProcessed?.(stats); + return Promise.resolve(); + }, + ); + const onFrameProcessed = vi.fn(); + const transformer = new BackgroundEffectTransformer({ onFrameProcessed }); + + await transformer.transform(frame, controller); + expect(onFrameProcessed).toHaveBeenCalledWith(stats); + }); + + it("never sends the first frame on untouched", async () => { + // The library copies its first frame through untouched, ahead of the + // drawn one, unless told the first frame is past + const seen: boolean[] = []; + vi.spyOn(BackgroundTransformer.prototype, "transform").mockImplementation( + async function (this: BackgroundTransformer) { + seen.push(this.isFirstFrame); + return Promise.resolve(); + }, + ); + const transformer = new BackgroundEffectTransformer({}); + expect(transformer.isFirstFrame).toBe(true); + + await transformer.transform(frame, controller); + expect(seen).toEqual([false]); + }); + it("passes frames on untouched once disabled, even after blurring", async () => { const transformer = new BackgroundEffectTransformer({}); await transformer.update({ blurRadius: 15, backgroundDisabled: false }); diff --git a/src/tracks/BackgroundEffectTransformer.ts b/src/tracks/BackgroundEffectTransformer.ts index 774251994..9c43b0648 100644 --- a/src/tracks/BackgroundEffectTransformer.ts +++ b/src/tracks/BackgroundEffectTransformer.ts @@ -56,16 +56,30 @@ const wasmFileset: WasmFileset = { export class BackgroundEffectTransformer extends BackgroundTransformer { /** Called once, when the first frame carrying an effect has been drawn. */ public onFirstFrame: (() => void) | undefined; + private drewAFrame = false; + + public constructor(opts: BackgroundOptions) { + super(opts); + // Called by the library for a frame it drew, never for one it passed on + // untouched. + this.options.onFrameProcessed = (stats): void => { + opts.onFrameProcessed?.(stats); + if (this.drewAFrame) return; + this.drewAFrame = true; + this.onFirstFrame?.(); + }; + } public override async transform( frame: VideoFrame, controller: TransformStreamDefaultController, ): Promise { - const priming = this.isFirstFrame; + // For the first frame, the library enqueues the raw frame ahead of the + // processed one, so that the output shows something while the first + // segmentation runs. The raw frame is the room without the effect, + // so no frame is allowed to count as the first. + this.isFirstFrame = false; await super.transform(frame, controller); - // Cleared only once a frame has been through the segmenter; a frame - // passed through untouched leaves it set. - if (priming && !this.isFirstFrame) this.onFirstFrame?.(); } /**