diff --git a/playwright/sdk/media.spec.ts b/playwright/sdk/media.spec.ts index b98a0a24d..418245044 100644 --- a/playwright/sdk/media.spec.ts +++ b/playwright/sdk/media.spec.ts @@ -90,6 +90,37 @@ test("muting the microphone and the camera is seen by the peer", async () => { }); }); +test("the peer's microphone is active while it sends sound, and not while muted", async ({ + browserName, +}) => { + // The SFU's speaker detection picks up Firefox's fake microphone, a steady + // tone. Chromium's fake microphone never registers as a speaker there, so + // the test has nothing to measure on it. + test.skip( + browserName === "chromium", + "Chromium's fake microphone is silent to the SFU", + ); + const [page] = pages; + const audio = remoteTile(1, 0).locator("audio"); + // The SFU only announces a change of speakers, so a tone that has been + // playing since before the peer subscribed is not announced to it. Muting + // and unmuting is the change that is. + await page.getByRole("button", { name: "Microphone" }).click(); + await expect(audio).toHaveAttribute("data-muted", "true", { + timeout: 20_000, + }); + await expect(audio).toHaveAttribute("data-active", "false"); + await page.getByRole("button", { name: "Microphone" }).click(); + await expect(audio).toHaveAttribute("data-active", "true", { + timeout: 30_000, + }); + await page.getByRole("button", { name: "Microphone" }).click(); + await expect(audio).toHaveAttribute("data-active", "false", { + timeout: 20_000, + }); + await page.getByRole("button", { name: "Microphone" }).click(); +}); + test("a larger tile receives a higher resolution", async () => { const peer = remoteTile(1, 0); const video = peer.locator("video"); diff --git a/sdk/MemberMediaPlan.md b/sdk/MemberMediaPlan.md index 87c35ed56..d006bdec9 100644 --- a/sdk/MemberMediaPlan.md +++ b/sdk/MemberMediaPlan.md @@ -380,3 +380,76 @@ No behaviour change; one PR of renames and type narrowing. `trackName`) is a one-line addition when the first non-call application asks for it; adding it now would be a guess at that application's needs. - `sendNotificationType` stays, as `SdkArchitecture.md` already decided. + +## Changed behaviour + +What a host sees differently, per slice. The first block is implemented; the +rest follows the slices above and is marked as planned. + +### `tracks$` and `isActive$` (this plan, implemented) + +| Before | After | +| ---------------------------------------------------------------------- | -------------------------------------------------------------------- | +| `media.camera$`, `.microphone$`, `.screenShare$`, `.screenShareAudio$` | `trackBySource$(scope, media.tracks$, "camera")`, and so on | +| `media.screenShareEnabled$` | `media.tracks$` mapped to `some((t) => t.source === "screenShare")` | +| `media.speaking$` | `isActive$` on the `"microphone"` track; `false` while there is none | +| `createLivekitMediaTrack(…, source)` | the source is read from the publication (internal) | + +Differences in what the values mean: + +- **Tracks published without a source now appear**, with `source: "unknown"`. + Before, they were invisible. A host that renders every track must be ready + for a track it has no name for. +- **`isActive$` is `false` while the track is muted**, even if LiveKit still + reports the member as speaking. `speaking$` was not gated by mute. +- **`isActive$` is the same on every audio track of a member**, because LiveKit + detects speakers per participant. A member sharing a screen with audio lights + both tracks while either carries sound. +- **`setVolume` applies to the track, not the source.** It is remembered and + reapplied when the remote track arrives on subscription, so a volume set + before the track exists is no longer lost. It still does nothing on our own + tracks. +- **One `MediaTrack` object per publication**, stable across unrelated media + events. Before, a slot's object also changed only with its publication, so + nothing changes for the four known sources. +- **The list is in publication order** (the participant's map order). Nothing + before had an order. +- The harness labels audio elements with `data-active`. Test ids are unchanged. + +### Slice 1, local track controls (planned) + +- `switchCamera()` on the local media becomes `switchFacingMode()` on the local + camera track; it rejects, instead of resolving `undefined`, when there is no + camera track. +- Input device and processor changes are calls on the track. The SDK no longer + reacts to `audioInputDeviceId$`, `videoInputDeviceId$` or `videoProcessor$`; + a host that changed a setting and expected the SDK to follow now calls + `setDevice` or `setProcessor` itself. +- `setEnabled(false)` on a local track mutes it and keeps the publication, as + `setMicrophoneEnabled(false)` does today. + +### Slice 2, `publish` and `unpublish` (planned) + +- Screen sharing is `publish({ source: "screenShare" })` and `unpublish(id)`. + A capture failure rejects the promise; there is no `screenShareError$` to + dismiss. +- "Am I sharing" is derived from `tracks$` by the host. The SDK no longer + exposes `sharingScreen$`. +- `setMicrophoneEnabled` / `setCameraEnabled` are gone. Before the transport + is connected, `publish` is remembered and applied once it is, which is what + the two methods did. +- The initial microphone and camera are `LocalMediaInputs.publish`, a list; + an empty list joins without publishing, which needed `microphoneEnabled: +false, cameraEnabled: false` before. +- Capture settings travel with each request. A host that set + `capture.screenShare` once sets it on each screen share publish instead. + +### Slice 3, names (planned) + +- `membership$` is typed as `RTCMembership`. A host that reached for a + `CallMembership` member outside the narrowed set must import the js-sdk type + itself; the object is unchanged. +- `UnknownCallError` is renamed; `index.ts` stops exporting it as + `UnknownRTCError`. Error codes do not change. +- `applicationData` is forwarded whole once the js-sdk takes it. Until then the + behaviour is as today: only `m.call.intent` reaches the membership. diff --git a/sdk/SdkArchitecture.md b/sdk/SdkArchitecture.md index aace36b8a..683a0c4fd 100644 --- a/sdk/SdkArchitecture.md +++ b/sdk/SdkArchitecture.md @@ -30,8 +30,8 @@ therefore `Element Call → sdk`, never the reverse. The migration from today's │ │ │ per member RTCMember { id userId deviceId membership$ displayName$ │ │ avatarUrl$ transport$ media$ } │ -│ per media MemberMedia { speaking$ microphone$ camera$ screenShare$ … } │ -│ per track MediaTrack { muted$ encrypted$ stats$ attach() detach() } │ +│ per media MemberMedia { tracks$ encryptionError$ } │ +│ per track MediaTrack { source kind muted$ encrypted$ stats$ attach() }│ │ │ │ ┌──────────────────────────┐ memberships ┌───────────────────────────┐ │ │ │ MatrixRTC core │ ───────────────▶ │ transports │ │ @@ -447,7 +447,12 @@ is wrapped in a `MemberMedia`, so that nothing outside the SDK imports ```ts export type MediaSource = - "microphone" | "camera" | "screenShare" | "screenShareAudio"; + | "microphone" + | "camera" + | "screenShare" + | "screenShareAudio" + /** Published without a source; the application knows what it is. */ + | "unknown"; export type MediaStreamStats = RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats | undefined; @@ -480,6 +485,13 @@ export interface MediaTrack { export interface AudioMediaTrack extends MediaTrack { kind: "audio"; + /** + * Whether the track carries sound right now, as the backend measures it. + * False while muted. A call reads this on the microphone track and calls it + * "speaking". LiveKit measures it per member, so every audio track of a + * member reports the same value. + */ + isActive$: Behavior; /** Route playback through Web Audio, for earpiece pan and gain. Undefined resets. */ setAudioContext(ctx: AudioContext | undefined, plugins?: AudioNode[]): void; /** No-op for our own audio, which is never played back. */ @@ -501,18 +513,23 @@ export type EncryptionError = "MissingKey" | "InvalidKey"; */ export interface MemberMedia { local: boolean; - speaking$: Behavior; - screenShareEnabled$: Behavior; - - microphone$: Behavior; - camera$: Behavior; - screenShare$: Behavior; - screenShareAudio$: Behavior; - + /** + * One entry per published track, in publication order. An entry stays the + * same object for as long as the same publication is behind it. Which + * track is which is in its `source`; `trackBySource$` picks one out. + */ + tracks$: Behavior<(AudioMediaTrack | VideoMediaTrack)[]>; /** Emits when the SFU reports a key problem for this member. */ encryptionError$: Observable; } +/** The member's first track of a source; undefined while there is none. */ +export function trackBySource$( + scope: ObservableScope, + tracks$: Behavior, + source: S, +): Behavior | undefined>; + export interface LocalMemberMedia extends MemberMedia { local: true; /** @@ -600,12 +617,14 @@ What it does: log in with a password, or with an access token the URL hands it ( tests register users through the admin API and skip `/login`, which the dev homeserver rate-limits); join the room and wait for the sync to hold it; create the client with microphone and camera enabled; show one tile per member with the display -name, a `