Merge branch 'livekit' into fkwp/change_video_codec

.github/workflows/build-element-call.yaml

@@ -9,6 +9,11 @@ on:
         type: string # This would ideally be a `choice` type, but that isn't supported yet
         description: The package type to be built. Must be one of 'full' or 'embedded'
         required: true
+      build_mode:
+        type: string # This would ideally be a `choice` type, but that isn't supported yet
+        description: The build mode for vite. Must be either 'development' or 'production'
+        required: false
+        default: production
     secrets:
       SENTRY_ORG:
         required: true
@@ -37,20 +42,8 @@ jobs:
           node-version-file: ".node-version"
       - name: Install dependencies
         run: "yarn install --immutable"
-      - name: Build full version
-        if: ${{ inputs.package == 'full' }}
-        run: "yarn run build:full"
-        env:
-          SENTRY_ORG: ${{ secrets.SENTRY_ORG }}
-          SENTRY_PROJECT: ${{ secrets.SENTRY_PROJECT }}
-          SENTRY_URL: ${{ secrets.SENTRY_URL }}
-          SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
-          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
-          VITE_APP_VERSION: ${{ inputs.vite_app_version }}
-          NODE_OPTIONS: "--max-old-space-size=4096"
-      - name: Build embedded
-        if: ${{ inputs.package == 'embedded' }}
-        run: "yarn run build:embedded"
+      - name: Build Element Call
+        run: ${{ format('yarn run build:{0}:{1}', inputs.package, inputs.build_mode) }}
         env:
           SENTRY_ORG: ${{ secrets.SENTRY_ORG }}
           SENTRY_PROJECT: ${{ secrets.SENTRY_PROJECT }}

.github/workflows/build.yaml

@@ -14,6 +14,7 @@ jobs:
     with:
       package: full
       vite_app_version: ${{ github.event.release.tag_name || github.sha }}
+      build_mode: ${{ github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'development build') && 'development' || 'production' }}
     secrets:
       SENTRY_ORG: ${{ secrets.SENTRY_ORG }}
       SENTRY_PROJECT: ${{ secrets.SENTRY_PROJECT }}
@@ -61,6 +62,7 @@ jobs:
     with:
       package: embedded
       vite_app_version: ${{ github.event.release.tag_name || github.sha }}
+      build_mode: ${{ github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'development build') && 'development' || 'production' }}
     secrets:
       SENTRY_ORG: ${{ secrets.SENTRY_ORG }}
       SENTRY_PROJECT: ${{ secrets.SENTRY_PROJECT }}

config/config.devenv.json

@@ -8,5 +8,12 @@
   "features": {
     "feature_use_device_session_member_events": true
   },
-  "ssla": "https://static.element.io/legal/element-software-and-services-license-agreement-uk-1.pdf"
+  "ssla": "https://static.element.io/legal/element-software-and-services-license-agreement-uk-1.pdf",
+  "matrix_rtc_session": {
+    "wait_for_key_rotation_ms": 3000,
+    "membership_event_expiry_ms": 180000000,
+    "delayed_leave_event_delay_ms": 18000,
+    "delayed_leave_event_restart_ms": 4000,
+    "network_error_retry_ms": 4000
+  }
 }

config/config.sample.json

@@ -11,5 +11,12 @@
   "features": {
     "feature_use_device_session_member_events": true
   },
-  "ssla": "https://static.element.io/legal/element-software-and-services-license-agreement-uk-1.pdf"
+  "ssla": "https://static.element.io/legal/element-software-and-services-license-agreement-uk-1.pdf",
+  "matrix_rtc_session": {
+    "wait_for_key_rotation_ms": 3000,
+    "membership_event_expiry_ms": 180000000,
+    "delayed_leave_event_delay_ms": 18000,
+    "delayed_leave_event_restart_ms": 4000,
+    "network_error_retry_ms": 4000
+  }
 }

config/config_netlify_preview.json

@@ -5,6 +5,14 @@
       "server_name": "call-unstable.ems.host"
     }
   },
+  "ssla": "https://static.element.io/legal/element-software-and-services-license-agreement-uk-1.pdf",
+  "matrix_rtc_session": {
+    "wait_for_key_rotation_ms": 3000,
+    "membership_event_expiry_ms": 180000000,
+    "delayed_leave_event_delay_ms": 18000,
+    "delayed_leave_event_restart_ms": 4000,
+    "network_error_retry_ms": 4000
+  },
   "posthog": {
     "api_key": "phc_rXGHx9vDmyEvyRxPziYtdVIv0ahEv8A9uLWFcCi1WcU",
     "api_host": "https://posthog-element-call.element.io"

docs/controls.md

@@ -19,3 +19,7 @@ On mobile platforms (iOS, Android), web views do not reliably support selecting
 - `controls.setAudioEnabled(enabled: boolean)` Enables/disables all audio output from the application. Output is enabled by default.
 - `showNativeAudioDevicePicker: (() => void) | undefined`. Callback called whenever the user presses the output button in the settings menu.
   This button is only shown on iOS. (`userAgent.includes("iPhone")`)
+- `controls.onAudioPlaybackStarted: ((id: string) => void) | undefined`: This will be called the first time we start
+  playing audio in the webview. It can be helpful to do device setup on the native app when the webviews audio is ready.
+  In particular android is using it to setup the output channel so that the call volume can
+  be controlled by the hardware volume rocker.

docs/self-hosting.md

@@ -255,13 +255,17 @@ self-hosters and developers working with Element Call.
 
 - [How to resolve stuck MatrixRTC calls](https://sspaeth.de/2025/02/how-to-resolve-stuck-matrixrtc-calls/)
 
-## 🛠️ How-Tos & Tutorials
+## 📝 How-Tos & Tutorials
 
 - [MatrixRTC aka Element-call setup (Geek warning)](https://sspaeth.de/2024/11/sfu/)
 - [MatrixRTC with Synology Container Manager (Docker)](https://ztfr.de/matrixrtc-with-synology-container-manager-docker/)
 - [Encrypted & Scalable Video Calls: How to deploy an Element Call backend with Synapse Using Docker-Compose](https://willlewis.co.uk/blog/posts/deploy-element-call-backend-with-synapse-and-docker-compose/)
 - [Element Call einrichten: Verschlüsselte Videoanrufe mit Element X und Matrix Synapse](https://www.cleveradmin.de/blog/2025/04/matrixrtc-element-call-backend-einrichten/)
 
+## 🛠️ Tools
+
+- [A Matrix server sanity tester including tests for proper MatrixRTC setup](https://codeberg.org/spaetz/testmatrix)
+
 ## 🤝 Want to Contribute?
 
 Have a guide or blog post you'd like to share? Open a

package.json

@@ -8,7 +8,11 @@
     "dev:embedded": "vite --config vite-embedded.config.js",
     "build": "yarn build:full",
     "build:full": "NODE_OPTIONS=--max-old-space-size=16384 vite build",
+    "build:full:production": "yarn build:full",
+    "build:full:development": "yarn build:full --mode development",
     "build:embedded": "yarn build:full --config vite-embedded.config.js",
+    "build:embedded:production": "yarn build:embedded",
+    "build:embedded:development": "yarn build:embedded --mode development",
     "serve": "vite preview",
     "prettier:check": "prettier -c .",
     "prettier:format": "prettier -w .",

src/config/ConfigOptions.ts

@@ -14,6 +14,7 @@ export interface ConfigOptions {
     api_key: string;
     api_host: string;
   };
+
   /**
    * The Sentry endpoint to which crash data will be sent.
    * This is only used in the full package of Element Call.
@@ -22,6 +23,7 @@ export interface ConfigOptions {
     DSN: string;
     environment: string;
   };
+
   /**
    * The rageshake server to which feedback and debug logs will be sent.
    * This is only used in the full package of Element Call.
@@ -66,6 +68,7 @@ export interface ConfigOptions {
      * Allow to join group calls without audio and video.
      */
     feature_group_calls_without_video_and_audio?: boolean;
+
     /**
      * Send device-specific call session membership state events instead of
      * legacy user-specific call membership state events.
@@ -86,6 +89,7 @@ export interface ConfigOptions {
      * Defines whether participants should start with audio enabled by default.
      */
     enable_audio?: boolean;
+
     /**
      * Defines whether participants should start with video enabled by default.
      */
@@ -109,19 +113,38 @@ export interface ConfigOptions {
      * How long (in milliseconds) to wait before rotating end-to-end media encryption keys
      * when someone leaves a call.
      */
+    wait_for_key_rotation_ms?: number;
+    /** @deprecated use wait_for_key_rotation_ms instead */
     key_rotation_on_leave_delay?: number;
 
     /**
-     * How often (in milliseconds) keep-alive messages should be sent to the server for
-     * the MatrixRTC membership event.
+     * The duration (in milliseconds) after the most recent keep-alive (delayed leave event restart)
+     * that the server waits before sending the leave MatrixRTC membership event.
      */
+    delayed_leave_event_delay_ms?: number;
+    /** @deprecated use delayed_leave_event_delay_ms instead */
+    membership_server_side_expiry_timeout?: number;
+
+    /**
+     * The time interval (in milliseconds) at which the client sends membership keep-alive
+     * messages to the server by restarting the timer for the delayed leave event.
+     */
+    delayed_leave_event_restart_ms?: number;
+    /** @deprecated use delayed_leave_event_restart_ms instead */
     membership_keep_alive_period?: number;
 
     /**
-     * How long (in milliseconds) after the last keep-alive the server should expire the
-     * MatrixRTC membership event.
+     * How long we wait before retrying after a network error on any of the requests.
      */
-    membership_server_side_expiry_timeout?: number;
+    network_error_retry_ms?: number;
+
+    /**
+     * The timeout (in milliseconds) after we joined the call, that our membership should expire
+     * unless we have explicitly updated it.
+     *
+     * This is what goes into the m.rtc.member event expiry field and is typically set to a number of hours.
+     */
+    membership_event_expiry_ms?: number;
   };
 }
 

src/controls.ts

@@ -20,6 +20,7 @@ export interface Controls {
   /** @deprecated use  onAudioDeviceSelect instead*/
   onOutputDeviceSelect?: (id: string) => void;
   onAudioDeviceSelect?: (id: string) => void;
+  onAudioPlaybackStarted?: () => void;
   /** @deprecated use  setAudioEnabled instead*/
   setOutputEnabled(enabled: boolean): void;
   setAudioEnabled(enabled: boolean): void;
@@ -54,7 +55,13 @@ export const outputDevice$ = new BehaviorSubject<string | undefined>(undefined);
  * This should also be used to display a darkened overlay screen letting the user know that audio is muted.
  */
 export const setAudioEnabled$ = new Subject<boolean>();
-
+let playbackStartedEmitted = false;
+export const setPlaybackStarted = (): void => {
+  if (!playbackStartedEmitted) {
+    playbackStartedEmitted = true;
+    window.controls.onAudioPlaybackStarted?.();
+  }
+};
 window.controls = {
   canEnterPip(): boolean {
     return setPipEnabled$.observed;

src/livekit/MatrixAudioRenderer.test.tsx

@@ -58,11 +58,13 @@ it("should render for member", () => {
   expect(container).toBeTruthy();
   expect(queryAllByTestId("audio")).toHaveLength(1);
 });
+
 it("should not render without member", () => {
+  const memberships = [
+    { sender: "othermember", deviceId: "123" },
+  ] as CallMembership[];
   const { container, queryAllByTestId } = render(
-    <MatrixAudioRenderer
-      members={[{ sender: "othermember", deviceId: "123" }] as CallMembership[]}
-    />,
+    <MatrixAudioRenderer members={memberships} />,
   );
   expect(container).toBeTruthy();
   expect(queryAllByTestId("audio")).toHaveLength(0);
@@ -84,6 +86,7 @@ it("should not setup audioContext gain and pan if there is no need to.", () => {
   expect(testAudioContext.gain.gain.value).toEqual(1);
   expect(testAudioContext.pan.pan.value).toEqual(0);
 });
+
 it("should setup audioContext gain and pan", () => {
   vi.spyOn(MediaDevicesContext, "useEarpieceAudioConfig").mockReturnValue({
     pan: 1,

src/livekit/MatrixAudioRenderer.tsx

@@ -14,11 +14,13 @@ import {
   type AudioTrackProps,
 } from "@livekit/components-react";
 import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc";
-import { logger } from "matrix-js-sdk/lib/logger";
+import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
 
 import { useEarpieceAudioConfig } from "./MediaDevicesContext";
 import { useReactiveState } from "../useReactiveState";
+import * as controls from "../controls";
 
+const logger = rootLogger.getChild("[MatrixAudioRenderer]");
 export interface MatrixAudioRendererProps {
   /**
    * The list of participants to render audio for.
@@ -59,6 +61,7 @@ export function MatrixAudioRenderer({
   );
 
   const loggedInvalidIdentities = useRef(new Set<string>());
+
   /**
    * Log an invalid livekit track identity.
    * A invalid identity is one that does not match any of the matrix rtc members.
@@ -96,6 +99,14 @@ export function MatrixAudioRenderer({
       isValid
     );
   });
+  useEffect(() => {
+    if (!tracks.some((t) => !validIdentities.has(t.participant.identity))) {
+      logger.debug(
+        `All audio tracks have a matching matrix call member identity.`,
+      );
+      loggedInvalidIdentities.current.clear();
+    }
+  }, [tracks, validIdentities]);
 
   // This component is also (in addition to the "only play audio for connected members" logic above)
   // responsible for mimicking earpiece audio on iPhones.
@@ -204,6 +215,7 @@ function AudioTrackWithAudioNodes({
       useContext ? [audioNodes.gain!, audioNodes.pan!] : [],
     );
     setTrackReady(true);
+    controls.setPlaybackStarted();
   }, [audioContext, audioNodes, setTrackReady, trackReady, trackRef]);
 
   return (

src/rtcSessionHelpers.ts

@@ -121,10 +121,18 @@ export async function enterRTCSession(
       ...(useDeviceSessionMemberEvents !== undefined && {
         useLegacyMemberEvents: !useDeviceSessionMemberEvents,
       }),
+      delayedLeaveEventRestartMs:
+        matrixRtcSessionConfig?.delayed_leave_event_restart_ms ??
+        matrixRtcSessionConfig?.membership_keep_alive_period,
       delayedLeaveEventDelayMs:
+        matrixRtcSessionConfig?.delayed_leave_event_delay_ms ??
         matrixRtcSessionConfig?.membership_server_side_expiry_timeout,
-      networkErrorRetryMs: matrixRtcSessionConfig?.membership_keep_alive_period,
-      makeKeyDelay: matrixRtcSessionConfig?.key_rotation_on_leave_delay,
+      networkErrorRetryMs: matrixRtcSessionConfig?.network_error_retry_ms,
+      makeKeyDelay:
+        matrixRtcSessionConfig?.wait_for_key_rotation_ms ??
+        matrixRtcSessionConfig?.key_rotation_on_leave_delay,
+      membershipEventExpiryMs:
+        matrixRtcSessionConfig?.membership_event_expiry_ms,
       useExperimentalToDeviceTransport,
     },
   );

src/useAudioContext.tsx

@@ -18,6 +18,7 @@ import {
 } from "./livekit/MediaDevicesContext";
 import { type PrefetchedSounds } from "./soundUtils";
 import { useUrlParams } from "./UrlParams";
+import * as controls from "./controls";
 
 /**
  * Play a sound though a given AudioContext. Will take
@@ -42,6 +43,7 @@ async function playSound(
   src.buffer = buffer;
   src.connect(gain).connect(pan).connect(ctx.destination);
   const p = new Promise<void>((r) => src.addEventListener("ended", () => r()));
+  controls.setPlaybackStarted();
   src.start();
   return p;
 }

vite-embedded.config.js

@@ -18,9 +18,9 @@ export default defineConfig((env) =>
             output: "./config.json",
             data: {
               matrix_rtc_session: {
-                key_rotation_on_leave_delay: 15000,
-                membership_keep_alive_period: 5000,
-                membership_server_side_expiry_timeout: 15000,
+                wait_for_key_rotation_ms: 5000,
+                delayed_leave_event_restart_ms: 4000,
+                delayed_leave_event_delay_ms: 18000,
               },
             },
           },

vite.config.js

@@ -89,6 +89,7 @@ export default defineConfig(({ mode, packageType }) => {
       },
     },
     build: {
+      minify: mode === "production" ? true : false,
       sourcemap: true,
       rollupOptions: {
         output: {

yarn.lock

@@ -2743,12 +2743,12 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@livekit/protocol@npm:1.38.0":
-  version: 1.38.0
-  resolution: "@livekit/protocol@npm:1.38.0"
+"@livekit/protocol@npm:1.39.2":
+  version: 1.39.2
+  resolution: "@livekit/protocol@npm:1.39.2"
   dependencies:
     "@bufbuild/protobuf": "npm:^1.10.0"
-  checksum: 10c0/ca64d4f984853054ff60574730b08a761afcd3bdc084e5218663e54b0e7f395aa2022d9d15d982fa094bbc0179cb19ef6a96ec74b1aa3265d118a85d1a4fde33
+  checksum: 10c0/ce5f3ee3ab10ea1578fb40c1d16224879e48143a1bbbfb5e36a080fa5468892eaccfe21aad1166b5443f67e2aa54236facafd4b3235e41b331e91ca405379368
   languageName: node
   linkType: hard
 
@@ -2780,13 +2780,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@matrix-org/olm@npm:3.2.15":
-  version: 3.2.15
-  resolution: "@matrix-org/olm@npm:3.2.15"
-  checksum: 10c0/82a40d6e4e632a90670d4f15e8962272e302f4b9deed4fc78455c5ca78422c13bde6b53ebfc406630336926c9574386c9d9069c9c023db1c3d143117985c1e50
-  languageName: node
-  linkType: hard
-
 "@mediapipe/tasks-vision@npm:^0.10.18":
   version: 0.10.21
   resolution: "@mediapipe/tasks-vision@npm:0.10.21"
@@ -5603,8 +5596,8 @@ __metadata:
   linkType: hard
 
 "@vector-im/compound-design-tokens@npm:^4.0.0":
-  version: 4.0.3
-  resolution: "@vector-im/compound-design-tokens@npm:4.0.3"
+  version: 4.0.4
+  resolution: "@vector-im/compound-design-tokens@npm:4.0.4"
   peerDependencies:
     "@types/react": "*"
     react: ^17 || ^18 || ^19.0.0
@@ -5613,7 +5606,7 @@ __metadata:
       optional: true
     react:
       optional: true
-  checksum: 10c0/4e32e46b4f0afef463ab7827c7bd1e0369bf83bab2cf86f182a5052b409c8b64e6fea84f818eee4d3f10e51be996e550d2b2dc664d7444f6685502f517d9a754
+  checksum: 10c0/e6ff6a956082f4a288237e7c7e60044319d7195cad0d5175dad7115270119f80c43252520db8f1a514b762f92dd5b7059c1217d7ccbe81daf71c426cbfeaf3dd
   languageName: node
   linkType: hard
 
@@ -9914,11 +9907,11 @@ __metadata:
   linkType: hard
 
 "livekit-client@npm:^2.13.0":
-  version: 2.13.3
-  resolution: "livekit-client@npm:2.13.3"
+  version: 2.13.6
+  resolution: "livekit-client@npm:2.13.6"
   dependencies:
     "@livekit/mutex": "npm:1.1.1"
-    "@livekit/protocol": "npm:1.38.0"
+    "@livekit/protocol": "npm:1.39.2"
     events: "npm:^3.3.0"
     loglevel: "npm:^1.9.2"
     sdp-transform: "npm:^2.15.0"
@@ -9928,7 +9921,7 @@ __metadata:
     webrtc-adapter: "npm:^9.0.1"
   peerDependencies:
     "@types/dom-mediacapture-record": ^1
-  checksum: 10c0/9eadb9835514551c8e834cebb607999e632bc4b3b3e5d5e206db90b66e2e5467b8a1f76a03c648e00cc96c87256b4eece58dbb2b92dfcafa84f4798bbb305060
+  checksum: 10c0/3023e65c15e6b757885f1ca7607833fdcca4c4f49d6c109ae45629f318d08b953b2f5c4b32d1381f057cf2f51f4def5f4c312e0ff4f3c67ee4584274cd62f4c8
   languageName: node
   linkType: hard
 
@@ -10140,12 +10133,11 @@ __metadata:
   linkType: hard
 
 "matrix-js-sdk@github:matrix-org/matrix-js-sdk#head=develop":
-  version: 37.6.0
-  resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=bf6dc16ad32d47f2c6e167236f4c853ceef01d4f"
+  version: 37.8.0
+  resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=adaf92162377414df620e004d8c58c67118490ca"
   dependencies:
     "@babel/runtime": "npm:^7.12.5"
     "@matrix-org/matrix-sdk-crypto-wasm": "npm:^14.2.0"
-    "@matrix-org/olm": "npm:3.2.15"
     another-json: "npm:^0.2.0"
     bs58: "npm:^6.0.0"
     content-type: "npm:^1.0.4"
@@ -10158,7 +10150,7 @@ __metadata:
     sdp-transform: "npm:^2.14.1"
     unhomoglyph: "npm:^1.0.6"
     uuid: "npm:11"
-  checksum: 10c0/2877e7c5b2779200b48f3152bb7b510e58899b1e779e7e6d2bc1a236ed178fa8858f0547b644a2029f48f55dc7cc3954f48e2598c8963d45293c8280ccc23039
+  checksum: 10c0/8e842056ff926c615d53a4a333c9e54d0fbe7f88b97b55629464e67850cc512606accde6a8724dd48ac82714483f685515913b5556e9824c4d63c443317be1ef
   languageName: node
   linkType: hard