use optional audio context and effect to initiate it + review

fix left right to match chromium + safari
(firefox is swapped)

earpice as setting

Simpler code and documentation
The doc explains, what this class actually does and why it is so complicated.

Signed-off-by: Timo K <toger5@hotmail.de>

use only one audioContext, remove (non working) standby fallback

README.md

@@ -192,11 +192,6 @@ To use it, create a local config by, e.g.,
 The `config.devenv.json` config should work with the backend development
 environment as outlined in the next section out of box.
 
-> [!NOTE]
-> Be aware, that this `config.devenv.json` is exposing a deprecated fallback
-> LiveKit config key. If the homeserver advertises SFU backend via
-> `.well-known/matrix/client` this has precedence.
-
 You're now ready to launch the development server:
 
 ```sh
@@ -212,12 +207,20 @@ See also:
 A docker compose file `dev-backend-docker-compose.yml` is provided to start the
 whole stack of components which is required for a local development environment:
 
-- Minimum Synapse Setup (servername: `synapse.localhost`)
-- LiveKit JWT Service (Note requires Federation API and hence a TLS reverse proxy)
-- Minimum TLS reverse proxy (servername: `synapse.localhost`) Note certificates
-  are valid for at least 10 years from now
+- Minimum Synapse Setup (servername: `synapse.m.localhost`)
+- LiveKit Authorization Service (Note requires Federation API and hence a TLS reverse proxy)
 - Minimum LiveKit SFU Setup using dev defaults for config
 - Redis db for completeness
+- Minimum `localhost` Certificate Authority (CA) for Transport Layer Security (TLS)
+  - Hostnames: `m.localhost`, `*.m.localhost`
+  - Add [./backend/dev_tls_local-ca.crt](./backend/dev_tls_local-ca.crt) to your web browsers trusted
+    certificates
+- Minimum TLS reverse proxy for
+  - Synapse homeserver: `synapse.m.localhost`
+  - MatrixRTC backend: `matrix-rtc.m.localhost`
+  - Local Element Call development `call.m.localhost` via `yarn dev --host `
+  - Element Web `app.m.localhost`
+  - Note certificates will expire on Thu, 03 May 2035 10:32:02 GMT
 
 These use a test 'secret' published in this repository, so this must be used
 only for local development and **_never be exposed to the public Internet._**
@@ -230,6 +233,16 @@ yarn backend
 # podman-compose -f dev-backend-docker-compose.yml up
 ```
 
+> [!NOTE]
+> To ensure your local development frontend functions properly, you’ll need to
+> add certificate exceptions in your browser for `https://localhost:3000`,
+> `https://matrix-rtc.m.localhost/livekit/jwt/healthz` and
+> `https://synapse.m.localhost/.well-known/matrix/client`. This can be either
+> done by adding the minimum localhost CA
+> ([./backend/dev_tls_local-ca.crt](./backend/dev_tls_local-ca.crt)) to your web
+> browsers trusted certificates or by simply copying and pasting each URL into
+> your browser’s address bar and follow the prompts to add the exception.
+
 ### Playwright tests
 
 Our Playwright tests run automatically as part of our CI along with our other

backend/dev_homeserver.yaml

@@ -1,5 +1,5 @@
-server_name: "synapse.localhost"
-public_baseurl: http://synapse.localhost:8008/
+server_name: "synapse.m.localhost"
+public_baseurl: https://synapse.m.localhost/
 
 pid_file: /data/homeserver.pid
 

backend/dev_nginx.conf

@@ -0,0 +1,155 @@
+# Synapse reverse proxy including .well-known/matrix/client
+server {
+    listen                  80;
+    listen                  [::]:80;
+    listen                  443 ssl;
+    listen                  8448 ssl;
+    listen                  [::]:443 ssl;
+    listen                  [::]:8448 ssl;
+    server_name             synapse.m.localhost;
+    ssl_certificate         /root/ssl/cert.pem;
+    ssl_certificate_key     /root/ssl/key.pem;
+
+    # well-known config adding rtc_foci backend
+    # Note well-known is currently not effective due to:
+    # https://spec.matrix.org/v1.12/client-server-api/#well-known-uri the spec
+    # says it must be at https://$server_name/... (implied port 443) Hence, we
+    # currently rely for local development environment on deprecated config.json
+    # setting for livekit_service_url
+    location /.well-known/matrix/client {
+        add_header Access-Control-Allow-Origin *;
+        return 200 '{"m.homeserver": {"base_url": "https://synapse.m.localhost"}, "org.matrix.msc4143.rtc_foci": [{"type": "livekit", "livekit_service_url": "https://matrix-rtc.m.localhost/livekit/jwt"}]}';
+        default_type application/json;
+    }
+
+    # Reverse proxy for Matrix Synapse Homeserver
+    # This is also required for development environment.
+    # Reason: the lk-jwt-service uses the federation API for the openid token
+    #         verification, which requires TLS
+    location / {
+        proxy_pass "http://homeserver:8008";
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection "upgrade";
+        proxy_set_header Host $host;
+        proxy_set_header X-Forwarded-For $remote_addr;
+        proxy_set_header X-Forwarded-Proto $scheme;
+    }
+
+    error_page   500 502 503 504  /50x.html;
+
+}
+
+# MatrixRTC reverse proxy
+# - MatrixRTC Authorization Service
+# - LiveKit SFU websocket signaling connection
+server {
+    listen                  80;
+    listen                  [::]:80;
+    listen                  443 ssl;
+    listen                  [::]:443 ssl;
+    listen                  8448 ssl;
+    listen                  [::]:8448 ssl;
+    server_name             matrix-rtc.m.localhost;
+    ssl_certificate         /root/ssl/cert.pem;
+    ssl_certificate_key     /root/ssl/key.pem;
+
+
+    location ^~ /livekit/jwt/ {
+
+      proxy_set_header Host $host;
+      proxy_set_header X-Real-IP $remote_addr;
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_set_header X-Forwarded-Proto $scheme;
+
+      # JWT Service running at port 8080
+      proxy_pass http://auth-server:8080/;
+    }
+
+    location ^~ /livekit/sfu/ {
+      proxy_set_header Host $host;
+      proxy_set_header X-Real-IP $remote_addr;
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_set_header X-Forwarded-Proto $scheme;
+
+      proxy_send_timeout 120;
+      proxy_read_timeout 120;
+      proxy_buffering off;
+
+      proxy_set_header Accept-Encoding gzip;
+      proxy_set_header Upgrade $http_upgrade;
+      proxy_set_header Connection "upgrade";
+
+      # LiveKit SFU websocket connection running at port 7880
+      proxy_pass http://livekit-sfu:7880/;
+    }
+    
+    error_page   500 502 503 504  /50x.html;
+
+}
+
+# Convenience reverse proxy for the call.m.localhost domain to yarn dev --host
+server {
+    listen                  80;
+    listen                  [::]:80;
+    server_name             call.m.localhost;
+
+    return 301 https://$host$request_uri;
+}
+
+server {
+    listen                  443 ssl;
+    listen                  [::]:443 ssl;
+    server_name             call.m.localhost;
+    ssl_certificate         /root/ssl/cert.pem;
+    ssl_certificate_key     /root/ssl/key.pem;
+
+
+    location ^~ / {
+
+      proxy_set_header Host $host;
+      proxy_set_header X-Real-IP $remote_addr;
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_set_header X-Forwarded-Proto $scheme;
+
+      proxy_pass https://host.docker.internal:3000;
+      proxy_ssl_verify              off;
+
+    }
+    
+    error_page   500 502 503 504  /50x.html;
+
+}
+
+# Convenience reverse proxy app.m.localhost for element web
+server {
+    listen                  80;
+    listen                  [::]:80;
+    server_name             app.m.localhost;
+
+    return 301 https://$host$request_uri;
+}
+
+server {
+    listen                  443 ssl;
+    listen                  [::]:443 ssl;
+    server_name             app.m.localhost;
+    ssl_certificate         /root/ssl/cert.pem;
+    ssl_certificate_key     /root/ssl/key.pem;
+
+
+    location ^~ / {
+
+      proxy_set_header Host $host;
+      proxy_set_header X-Real-IP $remote_addr;
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_set_header X-Forwarded-Proto $scheme;
+
+      proxy_pass http://element-web:81;
+      proxy_ssl_verify              off;
+
+    }
+    
+    error_page   500 502 503 504  /50x.html;
+
+}

backend/dev_tls_local-ca.crt

@@ -0,0 +1,19 @@
+-----BEGIN CERTIFICATE-----
+MIIDGjCCAgKgAwIBAgIUGdiFHhH4KL2pqBjMQHQ+PVIkSV8wDQYJKoZIhvcNAQEL
+BQAwHjEcMBoGA1UEAwwTRWxlbWVudCBDYWxsIERldiBDQTAeFw0yNTA1MDUxMDMy
+MDJaFw0zNTA1MDMxMDMyMDJaMB4xHDAaBgNVBAMME0VsZW1lbnQgQ2FsbCBEZXYg
+Q0EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDA2y0hjmNn1vRsVSdy
+8IOfo8N1q9UgkhQWpGKXzPh+D5d1fnuJEmHIVwtDEtS/PwQ43LTmegChPtKH9jdT
+tG0IihW9Ja5YNG+9xAwaoA/sB3CGCBYsz+2/XjVUpXoBJXIPoFBWsn+K0oeFw9fw
+eRO1z9abM4cl+LjKzMNM8CCyu9uI1MaGjYez2YIWvG854VucLxX7HSlMJxZNWnie
+Ui7fMakuJhB2+aiIQjdKxy4E5RHNhzYG/LXhvP+wBYBDPNRsP3rtzEaE9HAveL9K
+FGqd3R4cBia6r1WIXmpAzyu5RGP5Eou0TZlGkal96/bF0I7q/pKlL23Jt1BLPiQU
+KGKrAgMBAAGjUDBOMB0GA1UdDgQWBBQJqBjMu61c1p24txw/y+kv3D+V6DAfBgNV
+HSMEGDAWgBQJqBjMu61c1p24txw/y+kv3D+V6DAMBgNVHRMEBTADAQH/MA0GCSqG
+SIb3DQEBCwUAA4IBAQB8m2YfFGLugNt5vAAOvNxVqDA8c72yCVYr3CBCpmTIEY5Z
+d3qVGhG9//ux6+J8ntkSwd9nV5GJyYXHukCG1VavnAWolWdNF/WAllf0jhLuz7kD
+/cJnuI1By4tBsBmSz851i6HJ4t5k99Be+6GQVzi0e7zzfxTHZE4xP2J6Ox8QbPsP
+n0m76nIp/WbWaJqzvIIjJhmUUPPv+4wN+eOArgjiGLzptM2qTtGZtd0c9nS5gvep
++mEbSUN9zkhAroZf80wf+hEvy+fJ94VbZ9QjTzTg7odZLrsXGIe8DaG63EYRQ25b
+W5iYBAreln5fGSt7qHsGfqwZibTEk/Lx3dydO1Kg
+-----END CERTIFICATE-----

backend/dev_tls_local-ca.key

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDA2y0hjmNn1vRs
+VSdy8IOfo8N1q9UgkhQWpGKXzPh+D5d1fnuJEmHIVwtDEtS/PwQ43LTmegChPtKH
+9jdTtG0IihW9Ja5YNG+9xAwaoA/sB3CGCBYsz+2/XjVUpXoBJXIPoFBWsn+K0oeF
+w9fweRO1z9abM4cl+LjKzMNM8CCyu9uI1MaGjYez2YIWvG854VucLxX7HSlMJxZN
+WnieUi7fMakuJhB2+aiIQjdKxy4E5RHNhzYG/LXhvP+wBYBDPNRsP3rtzEaE9HAv
+eL9KFGqd3R4cBia6r1WIXmpAzyu5RGP5Eou0TZlGkal96/bF0I7q/pKlL23Jt1BL
+PiQUKGKrAgMBAAECggEAAPX2kxi5AQ7ul82SzT1KgpSXyDHLdYaUyAoYnaX9RO+B
+8ylmpyeqygs4+KQS4EMJm9jpo85Oy37bIKdG3kljU6wQcKlL5Y+ZUOo1nzpV6fid
+hGVs6ts8VXw8KshKQ9AyccZ8L/pirUfgOffgTwfjY7/90zceAL/s98GuZWc62nkX
+55joQv/OikqYfAGP/U6Bp2Zyf23DwJB09Z3B6NnZj/ZyAbDrDEHuA15LhCOcCczp
+IU/mFEywBPHT9Tg4w4Beq78PeAETvku2UalYRLhP3RLlXr2oEbwUtINRVt2QjZ85
+Esps4uCqL/mgQluIebtudD9HL/YMlNPXue1mDXFxJQKBgQDgZZY4yJBcf488T1V6
+HNm06b/LvVGj253pKgw14hpY1xQu3Ymgzv1GEqzhSYdzxhpmj0tMUNHxAp+YdGQu
+SZ0wcPKhw0aYVkIjDRYDC3Wn5GJhyIEYHGYMo/n4l49UzHRBPOTDzp49DkHTKBgh
+XgIIazYT3CkjTIMRrkUv+qfIPQKBgQDcBGu/mqbjxs4sN3zqPS4aB21o6t6W0sXs
+ZP9w6RlTPQi5U2oRbftjZtYc0bbEgkMUImB1HwYPQT5pJ+MyC414xDvSc2exBr5d
+To6yyPIy78Tf5PHM12fpKV92nSvoz/pSjYcGxxDtKfPqu+t8mOJfjCV1lLLA+xuB
+DDaE4p8dBwKBgQCdAne6A5v/HMH8UQZeCxHJpESvKiiVnnU/UEx651nID7XvlNNX
+0X0mKqsMd4ZvW43ddSYan/JF0LAa3FW8jYWO/3jF9vzOWoysOdvNBZetgf/Uq5ao
+aDZ/YbzmVCXWD7jIbPMkjs3pqrAkL0mzDzQc7+dGviWKrV6IYIfIqnn7gQKBgDCz
+vdIk/qpO+JZrFfiX4Fucp0hhLTJ/p5ZDaRPqVVPKn+K+Jy2ChfIj8mNgvK9VEloj
+nexvGJ1J2PHYBX+vdPp1nbRhHWPfVUY8PHQw7QP/dToGaMvqJrNDGEGeWvjnCMc7
+UtdaO1H0Rm0AegkTopB56lTTvJnhO95eALd7nrMDAoGAEPdzJtWoKafp49svhSj0
+hiXQv2SPBwVUN4LZ4SOWiXUcmYYm80aNpYKLkBxYjrfqFWhE7NUHLGp8YorQWKY2
+acD9AReHk/xku0ABy6jeYmSCmCxASxst5liKD+l12sk0gB0rk5MBxB4Uu1MIbQZ2
+aCASX3AVD2/XyC2MKkzc8Eg=
+-----END PRIVATE KEY-----

backend/dev_tls_m.localhost.crt

@@ -0,0 +1,21 @@
+-----BEGIN CERTIFICATE-----
+MIIDZzCCAk+gAwIBAgIUXizLjwkdqepX0bh0K3abeJxj68IwDQYJKoZIhvcNAQEL
+BQAwHjEcMBoGA1UEAwwTRWxlbWVudCBDYWxsIERldiBDQTAeFw0yNTA1MDUxMzU5
+MTFaFw0zNTA1MDMxMzU5MTFaMBgxFjAUBgNVBAMMDSoubS5sb2NhbGhvc3QwggEi
+MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCrzGSScSgaQuZdELGFYiLiYRwr
+LKyUdNr0rsPcOo0bvbeZ3zQMeUMRNlA69zGFdarumiDRXUoAmZI39WmH95aX3d+A
+U7EFnWev7xpWSVhSYj8T0d4rke8HjGk3LpaffJ93tbJuagBIH1ouuN6AOdzWs8hp
+RYIomWleEeeuVnnfaMwaXOdc+ihJJ6wzm2hwQSfdpjZPWBDd/DFft1ZXxIZOCjDs
+rEIiI7uU8iZPLB3QEM/tgxSSAOxrcKvQvxZokk+FD7aMJFP71IfieLCEzMTP1VXa
+tP7UTAKAqB2NyDJ8m3IHbOINiqcdFvFR3R1D9bXOYE4oRynNvYZrQUGnL2RtAgMB
+AAGjgaIwgZ8wHwYDVR0jBBgwFoAUCagYzLutXNaduLccP8vpL9w/legwCQYDVR0T
+BAIwADALBgNVHQ8EBAMCBaAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwMAYDVR0RBCkw
+J4IJbG9jYWxob3N0ggttLmxvY2FsaG9zdIINKi5tLmxvY2FsaG9zdDAdBgNVHQ4E
+FgQUfdh1p52ZgWyZcBgBXGwKi4EnUE0wDQYJKoZIhvcNAQELBQADggEBAKrHEuB6
+33j8+EwSHw3zrvt/DRXK2BDHI1Ir9JcztSunaKAjZXVvf/dvZp0Xs1dEdJIdnv6G
+iZYhBbOqDqpQZbf2h/h0kuu5yZSBUdnQXnYNxlhp2UaC/UEgw5iZT/p1rm7RjVie
+y4Dp2WytV5iZOLmLj6xDvd3DXazgJPWIRX8p8qJZbKTkwCjTr7nDIj8jjG1sVFf7
+1RJBO5/6WSnImrpDmlLUrvjiKvbxcdseDJyBOhTwdRdSk4S2M+s5tR5j2I1gXLOq
+J5ioN76+SCrTY0K0WKRy9oOXWO1/X3+VYcekp+0F3SGkd5w17jylCv1XIGHAdEsQ
+v2z2/aMI/7sAD2Q=
+-----END CERTIFICATE-----

backend/dev_tls_m.localhost.key

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCrzGSScSgaQuZd
+ELGFYiLiYRwrLKyUdNr0rsPcOo0bvbeZ3zQMeUMRNlA69zGFdarumiDRXUoAmZI3
+9WmH95aX3d+AU7EFnWev7xpWSVhSYj8T0d4rke8HjGk3LpaffJ93tbJuagBIH1ou
+uN6AOdzWs8hpRYIomWleEeeuVnnfaMwaXOdc+ihJJ6wzm2hwQSfdpjZPWBDd/DFf
+t1ZXxIZOCjDsrEIiI7uU8iZPLB3QEM/tgxSSAOxrcKvQvxZokk+FD7aMJFP71Ifi
+eLCEzMTP1VXatP7UTAKAqB2NyDJ8m3IHbOINiqcdFvFR3R1D9bXOYE4oRynNvYZr
+QUGnL2RtAgMBAAECggEAJaFQii8U/KOYt9vXNoMnZvSkaeSQLLhn2V6Kciu1CtWE
+aMTWLsFE6nk+G5xXkYcTmM3T0GghtH3u5CjyI6EcsEkeEorCZJt0wbmayDmqiekR
+LfMzOdHuTHX5+edPgMGYYG1BFyRKyYFsjH1b5zRFZhXdGQnrl5760GsVlz9D1KZQ
+iHcT+q1S2tmZeoUukQnADENKXUMCyTGM5FCddgNtsWnGDsTDayh7hUdvDkB+mW4G
+lSp+BZuc3PCwpbD6qkXvfugWs6CUAAtXoV3ceWgxQ+TEnNlwxaG1AyugfgNUBolk
+8xgeZt4r5QId03jsHDf7hpBAofcaCd5EMIIQYFvWoQKBgQDlbAvAzEFPTZZn2nRV
+Xagw4xjqVc1LLEKLCWq0N5rEkwn0h90Dz5N7/3NuonP/sIDsDHCbyiOYBI1Ck6Xi
+0WuB+OyKDh+xeF2mekN9G9ywPahdK5lT/TVsxXFyZlwtVv1x/6KBO4yv5URizxqU
+gyAPDDxfD/KcNjkOBaodWEwQGQKBgQC/s2gPDBtQkjLwkHXchBomLww5eLlVrac1
+WK4UX6uSdOgrjJ375OOgMTxe8NVZdOuAKytGXRWDwgH3nVWvuZhe7dGlX3JMuSer
+e9VwDpBESrvqcR4ruL6wm8wej6BXyjH0wD3FHb0S5HfuBDxTn+4bDwrbRzOUMNgy
+lSppuflxdQKBgQDiZcIfazFT8evn5nMAvuC4BZNTxIJHmZC9JfjPiUPIkpWzYtOe
+7BvNtKOT3Op9uw8uYYRKqKqBXJSNy6ha8XCXHS9HeXKbLn20SFkLQBCDNwVLlDfF
+40zyXtF6JDr4XyzSb4NM5pgKCER5AYloXxGm59s3sEQpFXUuOjbKqJS/GQKBgAoI
+c7vF4HAZFr1sch62cz/oWnVvkhOf4Q5zs7ixQSOLJtOQqnwSgK9TpFs7s47ZBbJR
+kBRAru2Ua9Hv1Bo8VnMxczV6h1roneDlvEf/GyHX33nnrbKQGrrXjJlU3wl5NaAf
+p5v3cHvapUQ5yIZ/6lBUOzc6xMJOxCHxmKSr7Rg5AoGAbEE4lt6Xh2dnBPJ81eNI
+IDrw/3ITY53qAY4Bx88CByIFuu8CEUdUZprh98jSl6ic1tMinZfUhRMwABLrUD51
+DGst8iGLPD9u83iMcUHI/L+p7AbxrKLvWXZrF5UZm440c9mSWqfhPaTBosPtNDsG
+LfETwH1flKXMTXd2xA9RTE4=
+-----END PRIVATE KEY-----

backend/dev_tls_setup

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+# Step 1: Create a Root CA key and cert
+openssl genrsa -out dev_tls_local-ca.key 2048
+openssl req -x509 -new -nodes \
+  -days 3650 \
+  -subj "/CN=Element Call Dev CA" \
+  -key dev_tls_local-ca.key \
+  -out dev_tls_local-ca.crt \
+  -sha256 -addext "basicConstraints=CA:TRUE"
+
+# Step 2: Create a private key and CSR for *.m.localhost
+openssl req -new -nodes -newkey rsa:2048 \
+  -keyout dev_tls_m.localhost.key \
+  -out dev_tls_m.localhost.csr \
+  -subj "/CN=*.m.localhost"
+
+# Step 3: Sign the CSR with your CA
+openssl x509 \
+  -req -in dev_tls_m.localhost.csr \
+  -CA dev_tls_local-ca.crt -CAkey dev_tls_local-ca.key \
+  -CAcreateserial \
+  -out dev_tls_m.localhost.crt \
+  -days 3650 \
+  -sha256 \
+  -extfile <( cat <<EOF
+authorityKeyIdentifier=keyid,issuer
+basicConstraints=CA:FALSE
+keyUsage = digitalSignature, keyEncipherment
+extendedKeyUsage = serverAuth
+subjectAltName = @alt_names
+
+[alt_names]
+DNS.1 = localhost
+DNS.2 = m.localhost
+DNS.3 = *.m.localhost
+EOF
+)

backend/ew.test.config.json

@@ -1,8 +1,8 @@
 {
   "default_server_config": {
     "m.homeserver": {
-      "base_url": "http://synapse.localhost:8008",
-      "server_name": "synapse.localhost"
+      "base_url": "https://synapse.m.localhost",
+      "server_name": "synapse.m.localhost"
     }
   },
   "disable_custom_urls": false,

backend/playwright_homeserver.yaml

@@ -1,5 +1,5 @@
-server_name: "synapse.localhost"
-public_baseurl: http://synapse.localhost:8008/
+server_name: "synapse.m.localhost"
+public_baseurl: https://synapse.m.localhost/
 
 pid_file: /data/homeserver.pid
 

backend/tls_localhost_cert.pem

@@ -1,22 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIDtzCCAp+gAwIBAgIUCmJjl3HAeLmrPwRg+/OzikW6peQwDQYJKoZIhvcNAQEL
-BQAwazELMAkGA1UEBhMCR0IxDzANBgNVBAgMBkxvbmRvbjEPMA0GA1UEBwwGTG9u
-ZG9uMQ4wDAYDVQQKDAVBbHJvczEWMBQGA1UECwwNSVQgRGVwYXJ0bWVudDESMBAG
-A1UEAwwJbG9jYWxob3N0MB4XDTI0MTEwNDIxNDcwMFoXDTM0MTEwMjIxNDcwMFow
-azELMAkGA1UEBhMCR0IxDzANBgNVBAgMBkxvbmRvbjEPMA0GA1UEBwwGTG9uZG9u
-MQ4wDAYDVQQKDAVBbHJvczEWMBQGA1UECwwNSVQgRGVwYXJ0bWVudDESMBAGA1UE
-AwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAs368
-ExLSudP8luNoY5UfaPqBSVJUPYBi+JGyd36tyN75p5OI7xSfHTttQxuD4KrExBFP
-C8mAhE1eoZPBVBOZJ4FYWBJfMaQnCjeqU+laP36td65kSJYbUYlKYH1WpxEpCdgx
-wWOKkP/kPX5YXbYqODx9aBJXgoT3yAJW7AniIoL+eLFnS9Xo86TPqCDBTJU9ocwK
-gPIDLhDv60724rhZT1kbGp7ECqRovndoDTQjuws2D3yNMfQ+4rrQGPXHGmP5PcaR
-0R7uueB+6APyC7MJbuhbxxg/+DFHrRi3lJsgwxuh2hi/+vWw8zgKlgYIwHFA9X0l
-cX0UlQdENMH3bgcGIwIDAQABo1MwUTAdBgNVHQ4EFgQUUFGxw7zoiHXGwRqtagjZ
-RPYc85cwHwYDVR0jBBgwFoAUUFGxw7zoiHXGwRqtagjZRPYc85cwDwYDVR0TAQH/
-BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEALokb1z2lu3qW141b2wm14ilZQKCZ
-reNNuUR95Uom96FXPH4QVEH+mYTXXJ5UrfNhQYKQFpdE+5S4HL/UqEOxtWvbAHpK
-nsLQ62J8m+0+uwiJGqeQpWr03KJgXDAVE9X3XwMlp/+buxSLhc+GIHWuXW56itV2
-jiZJYjhO5SnhhgTWNoVZk93qXuuWEN0yacw7c3Fr1IvFYYYWufbXTk70dbZihPDK
-VD141o8tpp6FerSKHNYDqkVFDyTz3DVOhQQJ59zfMre7bFr+PpTTl4vIuGzXEY+E
-HPjUSlOzwkCoh5fu7Fs3qG55rJt8akhTEoKpiBTaLucgAjVWNHeci1+Yxg==
------END CERTIFICATE-----

backend/tls_localhost_key.pem

@@ -1,28 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCzfrwTEtK50/yW
-42hjlR9o+oFJUlQ9gGL4kbJ3fq3I3vmnk4jvFJ8dO21DG4PgqsTEEU8LyYCETV6h
-k8FUE5kngVhYEl8xpCcKN6pT6Vo/fq13rmRIlhtRiUpgfVanESkJ2DHBY4qQ/+Q9
-flhdtio4PH1oEleChPfIAlbsCeIigv54sWdL1ejzpM+oIMFMlT2hzAqA8gMuEO/r
-TvbiuFlPWRsansQKpGi+d2gNNCO7CzYPfI0x9D7iutAY9ccaY/k9xpHRHu654H7o
-A/ILswlu6FvHGD/4MUetGLeUmyDDG6HaGL/69bDzOAqWBgjAcUD1fSVxfRSVB0Q0
-wfduBwYjAgMBAAECggEACTqdSExxzJ+LX5ARFaWyOBSWly2GKqSyR14+aInOklhx
-9QgkmfOxJrCf3TvJ8RWhXloW0Aqr8qGDxG0Ixgjn7rG7gskXCey1xn8MNppLS0kj
-ztaG+NB3AR89ABm8XdoHsSY45geh3/Ni9I0i1VardGQafUJhgNLTZqjwIodzkBtJ
-S/bi4uFk1lGNfuvWQvWqzGXUvd1l1YupV6iA4GfhXlUvrSBZwftLBD6xEvQaSqsA
-pHvBxTfMXG4RMAkNPDIElkuQ8++CGi1gIRkJfmrv4OgbbitteMnxqqqGYV0zSNCg
-R/5FG6umIV7lDLBHZCSCk7wmfmq2UUvzhHThHy4yMQKBgQDu4TwFJCIcVIj7Wj4r
-DUBFvz6Lgbltqb+YAMUBtpiDcAQxDJWmedh6dK04ts5CFAFRlRjjuz2uFn7qlVBm
-uye9R7tL+tOv5viqDXU78a4snFywoXub6yzpbxrW8B4W1pdIUvQmhwCcDwvO1V24
-7Vj2vxcM5I9dsk1aCQSi3VY5yQKBgQDAW/VoTRwhU6OUc6sji5Z5dnkMjkP6NZK9
-CSrTWLAMGaLPY+g6fFS7JMNSvfWm/okypD6rcN7p0cxMK3mfFKmMiyPRde0wdrci
-sGFjGxM/2d2D7KTMC9iMYwA0K17UIna+UiYPfhR/muIg/dCyjlkKDFs9Z4jk//r1
-91bmznt2iwKBgFdiYXhn/Wprqih4nKFXGZnqGdEixVhObl4GegrkZuo+AeqHdf8O
-N5ikMfG7PbyCYPEdH5u/FRMn+4mI0X6jHChroyJqQSHp1jEu9yHUiSicknOyvusM
-nsNN932FHRyxp2m3nsSxQhHUlzc0ajKJ8K9iu+XlfmSCIzW6cs25Nh+xAoGBAJro
-M0wIdPPdsCj3sUVRvx8XqknTM6kGhaIYBNXoYPWNm5BaC4U15OJEq8sxUOdnqcMP
-g6x6m/k+S8C3bh0O/a9Bydl/l0BlCfw0gGjYP/s2ju4Tn272xy/e9iYNGzPIgUmp
-TB9D0GwmpZ4d6HgyrD+sTbm4bATGpCp6QhBjDggbAoGBAJVMMtZ4pF8D6mLMRZGR
-pQjNPy+MH13XYmDRc/BSF8KJ4yKk3tohr9LSXzxR0SEB43NoL1bHkucZrNjGyL8x
-jktnwkoIs96kO2mPrl1TqWkXs5RjGkkSTbAJovIcvkRU31SWap/WzN2kHpmRVcQc
-KEFKXT5fUYZCLLWxhgZFlGPp
------END PRIVATE KEY-----

backend/tls_localhost_nginx.conf

@@ -1,40 +0,0 @@
-server {
-    listen                  80;
-    listen                  [::]:80;
-    listen                  443 ssl;
-    listen                  8448 ssl;
-    listen                  [::]:443 ssl;
-    listen                  [::]:8448 ssl;
-    server_name             synapse.localhost;
-    ssl_certificate         /root/ssl/cert.pem;
-    ssl_certificate_key     /root/ssl/key.pem;
-
-    # well-known config adding rtc_foci backend
-    # Note well-known is currently not effective due to:
-    # https://spec.matrix.org/v1.12/client-server-api/#well-known-uri the spec
-    # says it must be at https://$server_name/... (implied port 443) Hence, we
-    # currently rely for local development environment on deprecated config.json
-    # setting for livekit_service_url
-    location /.well-known/matrix/client {
-        return 200 '{"m.homeserver": {"base_url": "http://synapse.localhost:8008"}, "org.matrix.msc4143.rtc_foci": [{"type": "livekit", "livekit_service_url": "http://localhost:8080"}]}';
-        default_type application/json;
-        add_header Access-Control-Allow-Origin *;
-    }
-
-    # Reverse proxy for Matrix Synapse Homeserver
-    # This is also required for development environment.
-    # Reason: the lk-jwt-service uses the federation API for the openid token
-    #         verification, which requires TLS
-    location / {
-        proxy_pass "http://homeserver:8008";
-        proxy_http_version 1.1;
-        proxy_set_header Upgrade $http_upgrade;
-        proxy_set_header Connection "upgrade";
-        proxy_set_header Host $host;
-        proxy_set_header X-Forwarded-For $remote_addr;
-        proxy_set_header X-Forwarded-Proto $scheme;
-    }
-
-    error_page   500 502 503 504  /50x.html;
-
-}

config/config.devenv.json

@@ -1,13 +1,10 @@
 {
   "default_server_config": {
     "m.homeserver": {
-      "base_url": "http://synapse.localhost:8008",
-      "server_name": "synapse.localhost"
+      "base_url": "https://synapse.m.localhost",
+      "server_name": "synapse.m.localhost"
     }
   },
-  "livekit": {
-    "livekit_service_url": "http://localhost:8009"
-  },
   "features": {
     "feature_use_device_session_member_events": true
   },

dev-backend-docker-compose.yml

@@ -7,7 +7,7 @@ services:
     hostname: auth-server
     environment:
       - LK_JWT_PORT=8080
-      - LIVEKIT_URL=ws://localhost:7880
+      - LIVEKIT_URL=wss://matrix-rtc.m.localhost/livekit/sfu
       - LIVEKIT_KEY=devkey
       - LIVEKIT_SECRET=secret
       # If the configured homeserver runs on localhost, it'll probably be using
@@ -18,12 +18,13 @@ services:
         condition: on-failure
     ports:
       # HOST_PORT:CONTAINER_PORT
-      - 8009:8080
+      - 8080:8080
     networks:
       - ecbackend
 
   livekit:
     image: livekit/livekit-server:latest
+    hostname: livekit-sfu
     command: --dev --config /etc/livekit.yaml
     restart: unless-stopped
     # The SFU seems to work far more reliably when we let it share the host
@@ -81,17 +82,22 @@ services:
       - ecbackend
 
   nginx:
-    #  openssl req -x509 -nodes -days 3650 -newkey rsa:2048 -keyout tls_localhost_key.pem -out tls_localhost_cert.pem -subj "/C=GB/ST=London/L=London/O=Alros/OU=IT Department/CN=localhost"
-    hostname: synapse.localhost
+    #  see backend/dev_tls_setup for how to generate the tls certs
+    hostname: synapse.m.localhost
     image: nginx:latest
     volumes:
-      - ./backend/tls_localhost_nginx.conf:/etc/nginx/conf.d/default.conf:Z
-      - ./backend/tls_localhost_key.pem:/root/ssl/key.pem:Z
-      - ./backend/tls_localhost_cert.pem:/root/ssl/cert.pem:Z
+      - ./backend/dev_nginx.conf:/etc/nginx/conf.d/default.conf:Z
+      - ./backend/dev_tls_m.localhost.key:/root/ssl/key.pem:Z
+      - ./backend/dev_tls_m.localhost.crt:/root/ssl/cert.pem:Z
     ports:
       # HOST_PORT:CONTAINER_PORT
+      - "80:80"
+      - "443:443"
       - "8008:80"
       - "4443:443"
+      - "8448:8448"
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
     depends_on:
       - synapse
     networks:

docs/self-hosting.md

@@ -93,7 +93,7 @@ the example above, this results in:
 
 Using Nginx, you can achieve this by:
 
-```jsonc
+```nginx configuration file
 server {
     ...
     location ^~ /livekit/jwt/ {
@@ -203,7 +203,7 @@ Because Element Call uses client-side routing, your server must be able to route
 any requests to non-existing paths back to `/index.html`. For example, in Nginx
 you can achieve this with the `try_files` directive:
 
-```jsonc
+```nginx configuration file
 server {
     ...
     location / {

embedded/android/gradle/libs.versions.toml

@@ -2,11 +2,11 @@
 # https://docs.gradle.org/current/userguide/platforms.html#sub::toml-dependencies-format
 
 [versions]
-android_gradle_plugin = "8.8.0"
+android_gradle_plugin = "8.10.0"
 
 [libraries]
 android_gradle_plugin = { module = "com.android.tools.build:gradle", version.ref = "android_gradle_plugin" }
 
 [plugins]
 android_library = { id = "com.android.library", version.ref = "android_gradle_plugin" }
-maven_publish = { id = "com.vanniktech.maven.publish", version = "0.30.0" }
+maven_publish = { id = "com.vanniktech.maven.publish", version = "0.31.0" }

embedded/android/gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.10.2-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.14-all.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME

embedded/android/gradlew

@@ -86,8 +86,7 @@ done
 # shellcheck disable=SC2034
 APP_BASE_NAME=${0##*/}
 # Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036)
-APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s
-' "$PWD" ) || exit
+APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s\n' "$PWD" ) || exit
 
 # Use the maximum available, or set MAX_FD != -1 to use that value.
 MAX_FD=maximum
@@ -115,7 +114,7 @@ case "$( uname )" in                #(
   NONSTOP* )        nonstop=true ;;
 esac
 
-CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
+CLASSPATH="\\\"\\\""
 
 
 # Determine the Java command to use to start the JVM.
@@ -206,7 +205,7 @@ fi
 DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
 
 # Collect all arguments for the java command:
-#   * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments,
+#   * DEFAULT_JVM_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments,
 #     and any embedded shellness will be escaped.
 #   * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be
 #     treated as '${Hostname}' itself on the command line.
@@ -214,7 +213,7 @@ DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
 set -- \
         "-Dorg.gradle.appname=$APP_BASE_NAME" \
         -classpath "$CLASSPATH" \
-        org.gradle.wrapper.GradleWrapperMain \
+        -jar "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" \
         "$@"
 
 # Stop when "xargs" is not available.

embedded/android/gradlew.bat

@@ -70,11 +70,11 @@ goto fail
 :execute
 @rem Setup the command line
 
-set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
+set CLASSPATH=
 
 
 @rem Execute Gradle
-"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %*
+"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" -jar "%APP_HOME%\gradle\wrapper\gradle-wrapper.jar" %*
 
 :end
 @rem End local scope for the variables with windows NT shell

knip.ts

@@ -27,6 +27,10 @@ export default {
     // then Knip will flag it as a false positive
     // https://github.com/webpro-nl/knip/issues/766
     "@vector-im/compound-web",
+    // We need this so that TypeScript is happy with @livekit/track-processors.
+    // This might be a bug in the LiveKit repo but for now we fix it on the
+    // Element Call side.
+    "@types/dom-mediacapture-transform",
     "matrix-widget-api",
   ],
   ignoreExportsUsedInFile: true,

locales/en/app.json

@@ -61,6 +61,7 @@
     "video": "Video"
   },
   "developer_mode": {
+    "always_show_iphone_earpiece": "Show iPhone earpiece option on all platforms",
     "crypto_version": "Crypto version: {{version}}",
     "debug_tile_layout_label": "Debug tile layout",
     "device_id": "Device ID: {{id}}",
@@ -70,6 +71,7 @@
     "livekit_server_info": "LiveKit Server Info",
     "livekit_sfu": "LiveKit SFU: {{url}}",
     "matrix_id": "Matrix ID: {{id}}",
+    "mute_all_audio": "Mute all audio (participants, reactions, join sounds)",
     "show_connection_stats": "Show connection statistics",
     "show_non_member_tiles": "Show tiles for non-member media",
     "url_params": "URL parameters",
@@ -164,12 +166,16 @@
       "effect_volume_description": "Adjust the volume at which reactions and hand raised effects play.",
       "effect_volume_label": "Sound effect volume"
     },
+    "background_blur_header": "Background",
+    "background_blur_label": "Blur the background of the video",
+    "blur_not_supported_by_browser": "(Background blur is not supported by this device.)",
     "developer_tab_title": "Developer",
     "devices": {
       "camera": "Camera",
       "camera_numbered": "Camera {{n}}",
       "default": "Default",
       "default_named": "Default <2>({{name}})</2>",
+      "earpiece": "Earpiece",
       "microphone": "Microphone",
       "microphone_numbered": "Microphone {{n}}",
       "speaker": "Speaker",

package.json

@@ -39,7 +39,9 @@
     "@formatjs/intl-segmenter": "^11.7.3",
     "@livekit/components-core": "^0.12.0",
     "@livekit/components-react": "^2.0.0",
-    "@livekit/protocol": "^1.33.0",
+    "@livekit/protocol": "^1.38.0",
+    "@livekit/track-processors": "^0.5.5",
+    "@mediapipe/tasks-vision": "^0.10.18",
     "@opentelemetry/api": "^1.4.0",
     "@opentelemetry/core": "^1.25.1",
     "@opentelemetry/exporter-trace-otlp-http": "^0.57.0",
@@ -47,7 +49,7 @@
     "@opentelemetry/sdk-trace-base": "^1.25.1",
     "@opentelemetry/sdk-trace-web": "^1.9.1",
     "@opentelemetry/semantic-conventions": "^1.25.1",
-    "@playwright/test": "^1.51.0",
+    "@playwright/test": "^1.52.0",
     "@radix-ui/react-dialog": "^1.0.4",
     "@radix-ui/react-slider": "^1.1.2",
     "@radix-ui/react-visually-hidden": "^1.0.3",
@@ -60,6 +62,7 @@
     "@testing-library/react": "^16.0.0",
     "@testing-library/user-event": "^14.5.1",
     "@types/content-type": "^1.1.5",
+    "@types/dom-mediacapture-transform": "^0.1.11",
     "@types/grecaptcha": "^3.0.9",
     "@types/jsdom": "^21.1.7",
     "@types/lodash-es": "^4.17.12",
@@ -70,12 +73,11 @@
     "@types/react-dom": "^18.3.0",
     "@types/sdp-transform": "^2.4.5",
     "@types/uuid": "10",
-    "@typescript-eslint/eslint-plugin": "^8.0.0",
-    "@typescript-eslint/parser": "^8.0.0",
+    "@typescript-eslint/eslint-plugin": "^8.31.0",
+    "@typescript-eslint/parser": "^8.31.0",
     "@use-gesture/react": "^10.2.11",
     "@vector-im/compound-design-tokens": "^3.0.0",
     "@vector-im/compound-web": "^7.2.0",
-    "@vitejs/plugin-basic-ssl": "^1.0.1",
     "@vitejs/plugin-react": "^4.0.1",
     "@vitest/coverage-v8": "^3.0.0",
     "babel-plugin-transform-vite-meta-env": "^1.0.3",
@@ -118,7 +120,7 @@
     "react-use-measure": "^2.1.1",
     "rxjs": "^7.8.1",
     "sass": "^1.42.1",
-    "typescript": "^5.1.6",
+    "typescript": "^5.8.3",
     "typescript-eslint-language-service": "^5.0.5",
     "unique-names-generator": "^4.6.0",
     "vaul": "^1.0.0",
@@ -131,6 +133,7 @@
   },
   "resolutions": {
     "@livekit/components-core/rxjs": "^7.8.1",
+    "@livekit/track-processors/@mediapipe/tasks-vision": "^0.10.18",
     "matrix-widget-api": "1.11.0"
   },
   "packageManager": "yarn@4.7.0"

playwright-backend-docker-compose.override.yml

@@ -0,0 +1,4 @@
+services:
+  synapse:
+    volumes:
+      - ./backend/playwright_homeserver.yaml:/data/cfg/homeserver.yaml:Z

playwright-backend-docker-compose.yml

@@ -1,97 +1,2 @@
-networks:
-  ecbackend:
-
-services:
-  auth-service:
-    image: ghcr.io/element-hq/lk-jwt-service:latest-ci
-    hostname: auth-server
-    environment:
-      - LK_JWT_PORT=8080
-      - LIVEKIT_URL=ws://localhost:7880
-      - LIVEKIT_KEY=devkey
-      - LIVEKIT_SECRET=secret
-      # If the configured homeserver runs on localhost, it'll probably be using
-      # a self-signed certificate
-      - LIVEKIT_INSECURE_SKIP_VERIFY_TLS=YES_I_KNOW_WHAT_I_AM_DOING
-    deploy:
-      restart_policy:
-        condition: on-failure
-    ports:
-      # HOST_PORT:CONTAINER_PORT
-      - 8009:8080
-    networks:
-      - ecbackend
-
-  livekit:
-    image: livekit/livekit-server:latest
-    command: --dev --config /etc/livekit.yaml
-    restart: unless-stopped
-    # The SFU seems to work far more reliably when we let it share the host
-    # network rather than opening specific ports (but why?? we're not missing
-    # any…)
-    ports:
-      # HOST_PORT:CONTAINER_PORT
-      - 7880:7880/tcp
-      - 7881:7881/tcp
-      - 7882:7882/tcp
-      - 50100-50200:50100-50200/udp
-    volumes:
-      - ./backend/dev_livekit.yaml:/etc/livekit.yaml:Z
-    networks:
-      - ecbackend
-
-  redis:
-    image: redis:6-alpine
-    command: redis-server /etc/redis.conf
-    ports:
-      # HOST_PORT:CONTAINER_PORT
-      - 6379:6379
-    volumes:
-      - ./backend/redis.conf:/etc/redis.conf:Z
-    networks:
-      - ecbackend
-
-  element-web:
-    image: ghcr.io/element-hq/element-web:develop
-    volumes:
-      - ./backend/ew.test.config.json:/app/config.json
-    environment:
-      ELEMENT_WEB_PORT: 81
-    ports:
-      - "8081:81"
-    networks:
-      - ecbackend
-
-  synapse:
-    hostname: homeserver
-    image: docker.io/matrixdotorg/synapse:latest
-    environment:
-      - SYNAPSE_CONFIG_PATH=/data/cfg/homeserver.yaml
-      # Needed for rootless podman-compose such that the uid/gid mapping does
-      # fit local user uid. If the container runs as root (uid 0) it is fine as
-      # it actually maps to your non-root user on the host (e.g. 1000).
-      # Otherwise uid mapping will not match your non-root user.
-      - UID=0
-      - GID=0
-    volumes:
-      - ./backend/synapse_tmp:/data:Z
-      - ./backend/playwright_homeserver.yaml:/data/cfg/homeserver.yaml:Z
-    networks:
-      - ecbackend
-
-  nginx:
-    #  openssl req -x509 -nodes -days 3650 -newkey rsa:2048 -keyout tls_localhost_key.pem -out tls_localhost_cert.pem -subj "/C=GB/ST=London/L=London/O=Alros/OU=IT Department/CN=localhost"
-    hostname: synapse.localhost
-    image: nginx:latest
-    volumes:
-      - ./backend/tls_localhost_nginx.conf:/etc/nginx/conf.d/default.conf:Z
-      - ./backend/tls_localhost_key.pem:/root/ssl/key.pem:Z
-      - ./backend/tls_localhost_cert.pem:/root/ssl/cert.pem:Z
-    ports:
-      # HOST_PORT:CONTAINER_PORT
-      - "8008:80"
-      - "4443:443"
-    depends_on:
-      - synapse
-    networks:
-      - ecbackend
+include:
+  - dev-backend-docker-compose.yml

playwright/access.spec.ts

@@ -49,7 +49,7 @@ test("Sign up a new account, then login, then logout", async ({ browser }) => {
 
   // logout
   await returningUserPage.getByTestId("usermenu_open").click();
-  await returningUserPage.locator('[data-test-id="usermenu_logout"]').click();
+  await returningUserPage.locator('[data-testid="usermenu_logout"]').click();
 
   await expect(
     returningUserPage.getByRole("link", { name: "Log In" }),

playwright/fixtures/widget-user.ts

@@ -154,8 +154,13 @@ export const widgetTest = test.extend<MyFixtures>({
       ewPage1.getByRole("heading", { name: "Invite to Welcome Room" }),
     ).toBeVisible();
 
-    await ewPage1.getByRole("textbox").fill(whistlerMxId);
-    await ewPage1.getByRole("textbox").click();
+    // To get the invite textbox we need to specifically select within the
+    // dialog, since there is another textbox in the background (the message
+    // composer). In theory the composer shouldn't be visible to Playwright at
+    // all because the invite dialog has trapped focus, but the focus trap
+    // doesn't quite work right on Firefox.
+    await ewPage1.getByRole("dialog").getByRole("textbox").fill(whistlerMxId);
+    await ewPage1.getByRole("dialog").getByRole("textbox").click();
     await ewPage1.getByRole("button", { name: "Invite" }).click();
 
     // Accept the invite

src/App.tsx

@@ -22,6 +22,7 @@ import { Initializer } from "./initializer";
 import { MediaDevicesProvider } from "./livekit/MediaDevicesContext";
 import { widget } from "./widget";
 import { useTheme } from "./useTheme";
+import { ProcessorProvider } from "./livekit/TrackProcessorContext";
 
 const SentryRoute = Sentry.withSentryReactRouterV7Routing(Route);
 
@@ -72,22 +73,24 @@ export const App: FC = () => {
               <Suspense fallback={null}>
                 <ClientProvider>
                   <MediaDevicesProvider>
-                    <Sentry.ErrorBoundary
-                      fallback={(error) => (
-                        <ErrorPage error={error} widget={widget} />
-                      )}
-                    >
-                      <DisconnectedBanner />
-                      <Routes>
-                        <SentryRoute path="/" element={<HomePage />} />
-                        <SentryRoute path="/login" element={<LoginPage />} />
-                        <SentryRoute
-                          path="/register"
-                          element={<RegisterPage />}
-                        />
-                        <SentryRoute path="*" element={<RoomPage />} />
-                      </Routes>
-                    </Sentry.ErrorBoundary>
+                    <ProcessorProvider>
+                      <Sentry.ErrorBoundary
+                        fallback={(error) => (
+                          <ErrorPage error={error} widget={widget} />
+                        )}
+                      >
+                        <DisconnectedBanner />
+                        <Routes>
+                          <SentryRoute path="/" element={<HomePage />} />
+                          <SentryRoute path="/login" element={<LoginPage />} />
+                          <SentryRoute
+                            path="/register"
+                            element={<RegisterPage />}
+                          />
+                          <SentryRoute path="*" element={<RoomPage />} />
+                        </Routes>
+                      </Sentry.ErrorBoundary>
+                    </ProcessorProvider>
                   </MediaDevicesProvider>
                 </ClientProvider>
               </Suspense>

src/UserMenu.tsx

@@ -119,7 +119,7 @@ export const UserMenu: FC<Props> = ({
           key={key}
           Icon={Icon}
           label={label}
-          data-test-id={dataTestid}
+          data-testid={dataTestid}
           onSelect={() => onAction(key)}
         />
       ))}

src/initializer.tsx

@@ -136,6 +136,11 @@ export class Initializer {
       lookup: () => getUrlParams().lang ?? undefined,
     });
 
+    // Synchronise the HTML lang attribute with the i18next language
+    i18n.on("languageChanged", (lng) => {
+      document.documentElement.lang = lng;
+    });
+
     await i18n
       .use(Backend)
       .use(languageDetector)

src/livekit/BlurBackgroundTransformer.ts

@@ -0,0 +1,80 @@
+/*
+Copyright 2024-2025 New Vector Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only
+Please see LICENSE in the repository root for full details.
+*/
+
+import {
+  BackgroundTransformer,
+  VideoTransformer,
+  type VideoTransformerInitOptions,
+} from "@livekit/track-processors";
+import { ImageSegmenter } from "@mediapipe/tasks-vision";
+
+import modelAssetPath from "../mediapipe/imageSegmenter/selfie_segmenter.tflite?url";
+
+interface WasmFileset {
+  /** The path to the Wasm loader script. */
+  wasmLoaderPath: string;
+  /** The path to the Wasm binary. */
+  wasmBinaryPath: string;
+}
+
+// The MediaPipe package, by default, ships some alternative versions of the
+// WASM files which avoid SIMD for compatibility with older browsers. But SIMD
+// in WASM is actually fine by our support policy, so we include just the SIMD
+// versions.
+// It's really not ideal that we have to reference these internal files from
+// MediaPipe and depend on node_modules having this specific structure. It's
+// easy to see this breaking if our dependencies changed and MediaPipe were
+// no longer hoisted, or if we switched to another dependency loader such as
+// Yarn PnP.
+// https://github.com/google-ai-edge/mediapipe/issues/5961
+const wasmFileset: WasmFileset = {
+  wasmLoaderPath: new URL(
+    "../../node_modules/@mediapipe/tasks-vision/wasm/vision_wasm_internal.js",
+    import.meta.url,
+  ).href,
+  wasmBinaryPath: new URL(
+    "../../node_modules/@mediapipe/tasks-vision/wasm/vision_wasm_internal.wasm",
+    import.meta.url,
+  ).href,
+};
+
+/**
+ * Track processor that applies effects such as blurring to a user's background.
+ *
+ * This is just like LiveKit's prebuilt BackgroundTransformer except that it
+ * loads the segmentation models from our own bundle rather than as an external
+ * resource fetched from the public internet.
+ */
+export class BlurBackgroundTransformer extends BackgroundTransformer {
+  public async init({
+    outputCanvas,
+    inputElement: inputVideo,
+  }: VideoTransformerInitOptions): Promise<void> {
+    // Call super.super.init() since we're totally replacing the init method of
+    // BackgroundTransformer here, rather than extending it
+    await VideoTransformer.prototype.init.call(this, {
+      outputCanvas,
+      inputElement: inputVideo,
+    });
+
+    this.imageSegmenter = await ImageSegmenter.createFromOptions(wasmFileset, {
+      baseOptions: {
+        modelAssetPath,
+        delegate: "GPU",
+        ...this.options.segmenterOptions,
+      },
+      canvas: this.canvas,
+      runningMode: "VIDEO",
+      outputCategoryMask: true,
+      outputConfidenceMasks: false,
+    });
+
+    if (this.options.blurRadius) {
+      this.gl?.setBlurRadius(this.options.blurRadius);
+    }
+  }
+}

src/livekit/MatrixAudioRenderer.test.tsx

@@ -0,0 +1,104 @@
+/*
+Copyright 2023, 2024 New Vector Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+import { afterEach, beforeEach, expect, it, vi } from "vitest";
+import { render } from "@testing-library/react";
+import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc";
+import {
+  getTrackReferenceId,
+  type TrackReference,
+} from "@livekit/components-core";
+import { type RemoteAudioTrack } from "livekit-client";
+import { type ReactNode } from "react";
+import { useTracks } from "@livekit/components-react";
+
+import { testAudioContext } from "../useAudioContext.test";
+import * as MediaDevicesContext from "./MediaDevicesContext";
+import { MatrixAudioRenderer } from "./MatrixAudioRenderer";
+import { mockTrack } from "../utils/test";
+
+export const TestAudioContextConstructor = vi.fn(() => testAudioContext);
+
+beforeEach(() => {
+  vi.stubGlobal("AudioContext", TestAudioContextConstructor);
+});
+
+afterEach(() => {
+  vi.unstubAllGlobals();
+  vi.clearAllMocks();
+});
+
+vi.mock("@livekit/components-react", async (importOriginal) => {
+  return {
+    ...(await importOriginal()),
+    AudioTrack: (props: { trackRef: TrackReference }): ReactNode => {
+      return (
+        <audio data-testid={"audio"}>
+          {getTrackReferenceId(props.trackRef)}
+        </audio>
+      );
+    },
+    useTracks: vi.fn(),
+  };
+});
+
+const tracks = [mockTrack("test:123")];
+vi.mocked(useTracks).mockReturnValue(tracks);
+
+it("should render for member", () => {
+  const { container, queryAllByTestId } = render(
+    <MatrixAudioRenderer
+      members={[{ sender: "test", deviceId: "123" }] as CallMembership[]}
+    />,
+  );
+  expect(container).toBeTruthy();
+  expect(queryAllByTestId("audio")).toHaveLength(1);
+});
+it("should not render without member", () => {
+  const { container, queryAllByTestId } = render(
+    <MatrixAudioRenderer
+      members={[{ sender: "othermember", deviceId: "123" }] as CallMembership[]}
+    />,
+  );
+  expect(container).toBeTruthy();
+  expect(queryAllByTestId("audio")).toHaveLength(0);
+});
+
+it("should not setup audioContext gain and pan if there is no need to.", () => {
+  render(
+    <MatrixAudioRenderer
+      members={[{ sender: "test", deviceId: "123" }] as CallMembership[]}
+    />,
+  );
+  const audioTrack = tracks[0].publication.track! as RemoteAudioTrack;
+
+  expect(audioTrack.setAudioContext).toHaveBeenCalledTimes(1);
+  expect(audioTrack.setAudioContext).toHaveBeenCalledWith(undefined);
+  expect(audioTrack.setWebAudioPlugins).toHaveBeenCalledTimes(1);
+  expect(audioTrack.setWebAudioPlugins).toHaveBeenCalledWith([]);
+
+  expect(testAudioContext.gain.gain.value).toEqual(1);
+  expect(testAudioContext.pan.pan.value).toEqual(0);
+});
+it("should setup audioContext gain and pan", () => {
+  vi.spyOn(MediaDevicesContext, "useEarpieceAudioConfig").mockReturnValue({
+    pan: 1,
+    volume: 0.1,
+  });
+  render(
+    <MatrixAudioRenderer
+      members={[{ sender: "test", deviceId: "123" }] as CallMembership[]}
+    />,
+  );
+
+  const audioTrack = tracks[0].publication.track! as RemoteAudioTrack;
+  expect(audioTrack.setAudioContext).toHaveBeenCalled();
+  expect(audioTrack.setWebAudioPlugins).toHaveBeenCalled();
+
+  expect(testAudioContext.gain.gain.value).toEqual(0.1);
+  expect(testAudioContext.pan.pan.value).toEqual(1);
+});

src/livekit/MatrixAudioRenderer.tsx

@@ -0,0 +1,212 @@
+/*
+Copyright 2025 New Vector Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+import { getTrackReferenceId } from "@livekit/components-core";
+import { type RemoteAudioTrack, Track } from "livekit-client";
+import { useEffect, useMemo, useRef, useState, type ReactNode } from "react";
+import {
+  useTracks,
+  AudioTrack,
+  type AudioTrackProps,
+} from "@livekit/components-react";
+import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc";
+import { logger } from "matrix-js-sdk/lib/logger";
+
+import { useEarpieceAudioConfig } from "./MediaDevicesContext";
+import { useReactiveState } from "../useReactiveState";
+
+export interface MatrixAudioRendererProps {
+  /**
+   * The list of participants to render audio for.
+   * This list needs to be composed based on the matrixRTC members so that we do not play audio from users
+   * that are not expected to be in the rtc session.
+   */
+  members: CallMembership[];
+  /**
+   * If set to `true`, mutes all audio tracks rendered by the component.
+   * @remarks
+   * If set to `true`, the server will stop sending audio track data to the client.
+   */
+  muted?: boolean;
+}
+
+/**
+ * The `MatrixAudioRenderer` component is a drop-in solution for adding audio to your LiveKit app.
+ * It takes care of handling remote participants’ audio tracks and makes sure that microphones and screen share are audible.
+ *
+ * It also takes care of the earpiece audio configuration for iOS devices.
+ * This is done by using the WebAudio API to create a stereo pan effect that mimics the earpiece audio.
+ * @example
+ * ```tsx
+ * <LiveKitRoom>
+ *   <MatrixAudioRenderer />
+ * </LiveKitRoom>
+ * ```
+ * @public
+ */
+export function MatrixAudioRenderer({
+  members,
+  muted,
+}: MatrixAudioRendererProps): ReactNode {
+  const validIdentities = useMemo(
+    () =>
+      new Set(members?.map((member) => `${member.sender}:${member.deviceId}`)),
+    [members],
+  );
+
+  const loggedInvalidIdentities = useRef(new Set<string>());
+  /**
+   * Log an invalid livekit track identity.
+   * A invalid identity is one that does not match any of the matrix rtc members.
+   *
+   * @param identity The identity of the track that is invalid
+   * @param validIdentities The list of valid identities
+   */
+  const logInvalid = (identity: string, validIdentities: Set<string>): void => {
+    if (loggedInvalidIdentities.current.has(identity)) return;
+    logger.warn(
+      `Audio track ${identity} has no matching matrix call member`,
+      `current members: ${Array.from(validIdentities.values())}`,
+      `track will not get rendered`,
+    );
+    loggedInvalidIdentities.current.add(identity);
+  };
+
+  const tracks = useTracks(
+    [
+      Track.Source.Microphone,
+      Track.Source.ScreenShareAudio,
+      Track.Source.Unknown,
+    ],
+    {
+      updateOnlyOn: [],
+      onlySubscribed: true,
+    },
+  ).filter((ref) => {
+    const isValid = validIdentities?.has(ref.participant.identity);
+    if (!isValid && !ref.participant.isLocal)
+      logInvalid(ref.participant.identity, validIdentities);
+    return (
+      !ref.participant.isLocal &&
+      ref.publication.kind === Track.Kind.Audio &&
+      isValid
+    );
+  });
+
+  // This component is also (in addition to the "only play audio for connected members" logic above)
+  // responsible for mimicking earpiece audio on iPhones.
+  // The Safari audio devices enumeration does not expose an earpiece audio device.
+  // We alternatively use the audioContext pan node to only use one of the stereo channels.
+
+  // This component does get additionally complicated because of a Safari bug.
+  // (see: https://bugs.webkit.org/show_bug.cgi?id=251532
+  // and the related issues: https://bugs.webkit.org/show_bug.cgi?id=237878
+  // and https://bugs.webkit.org/show_bug.cgi?id=231105)
+  //
+  // AudioContext gets stopped if the webview gets moved into the background.
+  // Once the phone is in standby audio playback will stop.
+  // So we can only use the pan trick only works is the phone is not in standby.
+  // If earpiece mode is not used we do not use audioContext to allow standby playback.
+  // shouldUseAudioContext is set to false if stereoPan === 0 to allow standby bluetooth playback.
+
+  const { pan: stereoPan, volume: volumeFactor } = useEarpieceAudioConfig();
+  const shouldUseAudioContext = stereoPan !== 0;
+
+  // initialize the potentially used audio context.
+  const [audioContext, setAudioContext] = useState<AudioContext | undefined>(
+    undefined,
+  );
+  useEffect(() => {
+    const ctx = new AudioContext();
+    setAudioContext(ctx);
+    return (): void => {
+      void ctx.close();
+    };
+  }, []);
+  const audioNodes = useMemo(
+    () => ({
+      gain: audioContext?.createGain(),
+      pan: audioContext?.createStereoPanner(),
+    }),
+    [audioContext],
+  );
+
+  // Simple effects to update the gain and pan node based on the props
+  useEffect(() => {
+    if (audioNodes.pan) audioNodes.pan.pan.value = stereoPan;
+  }, [audioNodes.pan, stereoPan]);
+  useEffect(() => {
+    if (audioNodes.gain) audioNodes.gain.gain.value = volumeFactor;
+  }, [audioNodes.gain, volumeFactor]);
+
+  return (
+    // We add all audio elements into one <div> for the browser developer tool experience/tidyness.
+    <div style={{ display: "none" }}>
+      {tracks.map((trackRef) => (
+        <AudioTrackWithAudioNodes
+          key={getTrackReferenceId(trackRef)}
+          trackRef={trackRef}
+          muted={muted}
+          audioContext={shouldUseAudioContext ? audioContext : undefined}
+          audioNodes={audioNodes}
+        />
+      ))}
+    </div>
+  );
+}
+
+interface StereoPanAudioTrackProps {
+  muted?: boolean;
+  audioContext?: AudioContext;
+  audioNodes: {
+    gain?: GainNode;
+    pan?: StereoPannerNode;
+  };
+}
+
+/**
+ * This wraps `livekit.AudioTrack` to allow adding audio nodes to a track.
+ * It main purpose is to remount the AudioTrack component when switching from
+ * audiooContext to normal audio playback.
+ * As of now the AudioTrack component does not support adding audio nodes while being mounted.
+ * @param param0
+ * @returns
+ */
+function AudioTrackWithAudioNodes({
+  trackRef,
+  muted,
+  audioContext,
+  audioNodes,
+  ...props
+}: StereoPanAudioTrackProps &
+  AudioTrackProps &
+  React.RefAttributes<HTMLAudioElement>): ReactNode {
+  // This is used to unmount/remount the AudioTrack component.
+  // Mounting needs to happen after the audioContext is set.
+  // (adding the audio context when already mounted did not work outside strict mode)
+  const [trackReady, setTrackReady] = useReactiveState(
+    () => false,
+    // We only want the track to reset once both (audioNodes and audioContext) are set.
+    // for unsetting the audioContext its enough if one of the the is undefined.
+    [audioContext && audioNodes],
+  );
+
+  useEffect(() => {
+    if (!trackRef || trackReady) return;
+    const track = trackRef.publication.track as RemoteAudioTrack;
+    const useContext = audioContext && audioNodes.gain && audioNodes.pan;
+    track.setAudioContext(useContext ? audioContext : undefined);
+    track.setWebAudioPlugins(
+      useContext ? [audioNodes.gain!, audioNodes.pan!] : [],
+    );
+    setTrackReady(true);
+  }, [audioContext, audioNodes, setTrackReady, trackReady, trackRef]);
+
+  return (
+    trackReady && <AudioTrack trackRef={trackRef} muted={muted} {...props} />
+  );
+}

src/livekit/MediaDevicesContext.tsx

@@ -26,12 +26,16 @@ import {
   audioInput as audioInputSetting,
   audioOutput as audioOutputSetting,
   videoInput as videoInputSetting,
+  alwaysShowIphoneEarpiece as alwaysShowIphoneEarpieceSetting,
   type Setting,
 } from "../settings/settings";
 
+export const EARPIECE_CONFIG_ID = "earpiece-id";
+
 export type DeviceLabel =
   | { type: "name"; name: string }
   | { type: "number"; number: number }
+  | { type: "earpiece" }
   | { type: "default"; name: string | null };
 
 export interface MediaDevice {
@@ -40,6 +44,11 @@ export interface MediaDevice {
    */
   available: Map<string, DeviceLabel>;
   selectedId: string | undefined;
+  /**
+   * An additional device configuration that makes us use only one channel of the
+   * output device and a reduced volume.
+   */
+  useAsEarpiece: boolean | undefined;
   /**
    * The group ID of the selected device.
    */
@@ -65,6 +74,7 @@ function useMediaDevice(
 ): MediaDevice {
   // Make sure we don't needlessly reset to a device observer without names,
   // once permissions are already given
+  const [alwaysShowIphoneEarpice] = useSetting(alwaysShowIphoneEarpieceSetting);
   const hasRequestedPermissions = useRef(false);
   const requestPermissions = usingNames || hasRequestedPermissions.current;
   hasRequestedPermissions.current ||= usingNames;
@@ -102,27 +112,39 @@ function useMediaDevice(
             // Create a virtual default audio output for browsers that don't have one.
             // Its device ID must be the empty string because that's what setSinkId
             // recognizes.
+            // We also create this if we do not have any available devices, so that
+            // we can use the default or the earpiece.
+            const showEarpiece =
+              navigator.userAgent.match("iPhone") || alwaysShowIphoneEarpice;
             if (
               kind === "audiooutput" &&
-              available.size &&
               !available.has("") &&
-              !available.has("default")
+              !available.has("default") &&
+              (available.size || showEarpiece)
             )
               available = new Map([
                 ["", { type: "default", name: availableRaw[0]?.label || null }],
                 ...available,
               ]);
+            if (kind === "audiooutput" && showEarpiece)
+              // On IPhones we have to create a virtual earpiece device, because
+              // the earpiece is not available as a device ID.
+              available = new Map([
+                ...available,
+                [EARPIECE_CONFIG_ID, { type: "earpiece" }],
+              ]);
             // Note: creating virtual default input devices would be another problem
             // entirely, because requesting a media stream from deviceId "" won't
             // automatically track the default device.
             return available;
           }),
         ),
-      [kind, deviceObserver$],
+      [alwaysShowIphoneEarpice, deviceObserver$, kind],
     ),
   );
 
-  const [preferredId, select] = useSetting(setting);
+  const [preferredId, setPreferredId] = useSetting(setting);
+  const [asEarpice, setAsEarpiece] = useState(false);
   const selectedId = useMemo(() => {
     if (available.size) {
       // If the preferred device is available, use it. Or if every available
@@ -138,6 +160,7 @@ function useMediaDevice(
     }
     return undefined;
   }, [available, preferredId]);
+
   const selectedGroupId = useObservableEagerState(
     useMemo(
       () =>
@@ -151,14 +174,27 @@ function useMediaDevice(
     ),
   );
 
+  const select = useCallback(
+    (id: string) => {
+      if (id === EARPIECE_CONFIG_ID) {
+        setAsEarpiece(true);
+      } else {
+        setAsEarpiece(false);
+        setPreferredId(id);
+      }
+    },
+    [setPreferredId],
+  );
+
   return useMemo(
     () => ({
       available,
       selectedId,
+      useAsEarpiece: asEarpice,
       selectedGroupId,
       select,
     }),
-    [available, selectedId, selectedGroupId, select],
+    [available, selectedId, asEarpice, selectedGroupId, select],
   );
 }
 
@@ -167,6 +203,7 @@ export const deviceStub: MediaDevice = {
   selectedId: undefined,
   selectedGroupId: undefined,
   select: () => {},
+  useAsEarpiece: false,
 };
 export const devicesStub: MediaDevices = {
   audioInput: deviceStub,
@@ -255,3 +292,30 @@ export const useMediaDeviceNames = (
       return context.stopUsingDeviceNames;
     }
   }, [context, enabled]);
+
+/**
+ * A convenience hook to get the audio node configuration for the earpiece.
+ * It will check the `useAsEarpiece` of the `audioOutput` device and return
+ * the appropriate pan and volume values.
+ *
+ * @returns pan and volume values for the earpiece audio node configuration.
+ */
+export const useEarpieceAudioConfig = (): {
+  pan: number;
+  volume: number;
+} => {
+  const { audioOutput } = useMediaDevices();
+  // We use only the right speaker (pan = 1) for the earpiece.
+  // This mimics the behavior of the native earpiece speaker (only the top speaker on an iPhone)
+  const pan = useMemo(
+    () => (audioOutput.useAsEarpiece ? 1 : 0),
+    [audioOutput.useAsEarpiece],
+  );
+  // We also do lower the volume by a factor of 10 to optimize for the usecase where
+  // a user is holding the phone to their ear.
+  const volume = useMemo(
+    () => (audioOutput.useAsEarpiece ? 0.1 : 1),
+    [audioOutput.useAsEarpiece],
+  );
+  return { pan, volume };
+};

src/livekit/TrackProcessorContext.tsx

@@ -0,0 +1,84 @@
+/*
+Copyright 2024 New Vector Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only
+Please see LICENSE in the repository root for full details.
+*/
+
+import {
+  ProcessorWrapper,
+  supportsBackgroundProcessors,
+  type BackgroundOptions,
+} from "@livekit/track-processors";
+import { createContext, type FC, useContext, useEffect, useMemo } from "react";
+import { type LocalVideoTrack } from "livekit-client";
+
+import {
+  backgroundBlur as backgroundBlurSettings,
+  useSetting,
+} from "../settings/settings";
+import { BlurBackgroundTransformer } from "./BlurBackgroundTransformer";
+
+type ProcessorState = {
+  supported: boolean | undefined;
+  processor: undefined | ProcessorWrapper<BackgroundOptions>;
+};
+
+const ProcessorContext = createContext<ProcessorState | undefined>(undefined);
+
+export function useTrackProcessor(): ProcessorState {
+  const state = useContext(ProcessorContext);
+  if (state === undefined)
+    throw new Error(
+      "useTrackProcessor must be used within a ProcessorProvider",
+    );
+  return state;
+}
+
+export const useTrackProcessorSync = (
+  videoTrack: LocalVideoTrack | null,
+): void => {
+  const { processor } = useTrackProcessor();
+  useEffect(() => {
+    if (!videoTrack) return;
+    if (processor && !videoTrack.getProcessor()) {
+      void videoTrack.setProcessor(processor);
+    }
+    if (!processor && videoTrack.getProcessor()) {
+      void videoTrack.stopProcessor();
+    }
+  }, [processor, videoTrack]);
+};
+
+interface Props {
+  children: JSX.Element;
+}
+
+export const ProcessorProvider: FC<Props> = ({ children }) => {
+  // The setting the user wants to have
+  const [blurActivated] = useSetting(backgroundBlurSettings);
+  const supported = useMemo(() => supportsBackgroundProcessors(), []);
+  const blur = useMemo(
+    () =>
+      new ProcessorWrapper(
+        new BlurBackgroundTransformer({ blurRadius: 15 }),
+        "background-blur",
+      ),
+    [],
+  );
+
+  // This is the actual state exposed through the context
+  const processorState = useMemo(
+    () => ({
+      supported,
+      processor: supported && blurActivated ? blur : undefined,
+    }),
+    [supported, blurActivated, blur],
+  );
+
+  return (
+    <ProcessorContext.Provider value={processorState}>
+      {children}
+    </ProcessorContext.Provider>
+  );
+};

src/livekit/useLivekit.ts

@@ -9,6 +9,7 @@ import {
   ConnectionState,
   type E2EEManagerOptions,
   ExternalE2EEKeyProvider,
+  LocalVideoTrack,
   Room,
   type RoomOptions,
   Track,
@@ -17,6 +18,8 @@ import { useEffect, useMemo, useRef } from "react";
 import E2EEWorker from "livekit-client/e2ee-worker?worker";
 import { logger } from "matrix-js-sdk/lib/logger";
 import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc";
+import { useObservable, useObservableEagerState } from "observable-hooks";
+import { map } from "rxjs";
 
 import { defaultLiveKitOptions } from "./options";
 import { type SFUConfig } from "./openIDSFU";
@@ -33,13 +36,19 @@ import {
 import { MatrixKeyProvider } from "../e2ee/matrixKeyProvider";
 import { E2eeType } from "../e2ee/e2eeType";
 import { type EncryptionSystem } from "../e2ee/sharedKeyManagement";
+import {
+  useTrackProcessor,
+  useTrackProcessorSync,
+} from "./TrackProcessorContext";
+import { useInitial } from "../useInitial";
+import { observeTrackReference$ } from "../state/MediaViewModel";
 
 interface UseLivekitResult {
   livekitRoom?: Room;
   connState: ECConnectionState;
 }
 
-export function useLiveKit(
+export function useLivekit(
   rtcSession: MatrixRTCSession,
   muteStates: MuteStates,
   sfuConfig: SFUConfig | undefined,
@@ -82,12 +91,15 @@ export function useLiveKit(
   const devices = useMediaDevices();
   const initialDevices = useRef<MediaDevices>(devices);
 
+  const { processor } = useTrackProcessor();
+  const initialProcessor = useInitial(() => processor);
   const roomOptions = useMemo(
     (): RoomOptions => ({
       ...defaultLiveKitOptions,
       videoCaptureDefaults: {
         ...defaultLiveKitOptions.videoCaptureDefaults,
         deviceId: initialDevices.current.videoInput.selectedId,
+        processor: initialProcessor,
       },
       audioCaptureDefaults: {
         ...defaultLiveKitOptions.audioCaptureDefaults,
@@ -98,7 +110,7 @@ export function useLiveKit(
       },
       e2ee: e2eeOptions,
     }),
-    [e2eeOptions],
+    [e2eeOptions, initialProcessor],
   );
 
   // Store if audio/video are currently updating. If to prohibit unnecessary calls
@@ -123,6 +135,25 @@ export function useLiveKit(
     return r;
   }, [roomOptions, e2eeSystem]);
 
+  // Sync the requested track processors with LiveKit
+  useTrackProcessorSync(
+    useObservableEagerState(
+      useObservable(
+        (room$) =>
+          observeTrackReference$(
+            room$.pipe(map(([room]) => room.localParticipant)),
+            Track.Source.Camera,
+          ).pipe(
+            map((trackRef) => {
+              const track = trackRef?.publication?.track;
+              return track instanceof LocalVideoTrack ? track : null;
+            }),
+          ),
+        [room],
+      ),
+    ),
+  );
+
   const connectionState = useECConnectionState(
     {
       deviceId: initialDevices.current.audioInput.selectedId,
@@ -198,6 +229,7 @@ export function useLiveKit(
                 audioMuteUpdating.current = true;
                 trackPublication = await participant.setMicrophoneEnabled(
                   buttonEnabled.current.audio,
+                  room.options.audioCaptureDefaults,
                 );
                 audioMuteUpdating.current = false;
                 break;
@@ -205,6 +237,7 @@ export function useLiveKit(
                 videoMuteUpdating.current = true;
                 trackPublication = await participant.setCameraEnabled(
                   buttonEnabled.current.video,
+                  room.options.videoCaptureDefaults,
                 );
                 videoMuteUpdating.current = false;
                 break;

src/mediapipe/imageSegmenter/README.md

@@ -0,0 +1,5 @@
+# Google AI Edge MediaPipe Selfie Segmentation
+
+- See: https://ai.google.dev/edge/mediapipe/solutions/vision/image_segmenter
+- Latest: https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter/float16/latest/selfie_segmenter.tflite
+- License: Apache 2.0 as per https://storage.googleapis.com/mediapipe-assets/Model%20Card%20MediaPipe%20Selfie%20Segmentation.pdf

src/room/CallEventAudioRenderer.tsx

@@ -47,12 +47,15 @@ export const callEventAudioSounds = prefetchSounds({
 
 export function CallEventAudioRenderer({
   vm,
+  muted,
 }: {
   vm: CallViewModel;
+  muted?: boolean;
 }): ReactNode {
   const audioEngineCtx = useAudioContext({
     sounds: callEventAudioSounds,
     latencyHint: "interactive",
+    muted,
   });
   const audioEngineRef = useLatest(audioEngineCtx);
 

src/room/GroupCallView.test.tsx

@@ -38,6 +38,7 @@ import { GroupCallView } from "./GroupCallView";
 import { type WidgetHelpers } from "../widget";
 import { LazyEventEmitter } from "../LazyEventEmitter";
 import { MatrixRTCFocusMissingError } from "../utils/errors";
+import { ProcessorProvider } from "../livekit/TrackProcessorContext";
 
 vi.mock("../soundUtils");
 vi.mock("../useAudioContext");
@@ -46,6 +47,13 @@ vi.mock("react-use-measure", () => ({
   default: (): [() => void, object] => [(): void => {}, {}],
 }));
 
+vi.hoisted(
+  () =>
+    (global.ImageData = class MockImageData {
+      public data: number[] = [];
+    } as unknown as typeof ImageData),
+);
+
 const enterRTCSession = vi.hoisted(() => vi.fn(async () => Promise.resolve()));
 const leaveRTCSession = vi.hoisted(() =>
   vi.fn(
@@ -137,18 +145,20 @@ function createGroupCallView(
   const { getByText } = render(
     <BrowserRouter>
       <TooltipProvider>
-        <GroupCallView
-          client={client}
-          isPasswordlessUser={false}
-          confineToRoom={false}
-          preload={false}
-          skipLobby={false}
-          hideHeader={true}
-          rtcSession={rtcSession as unknown as MatrixRTCSession}
-          isJoined={joined}
-          muteStates={muteState}
-          widget={widget}
-        />
+        <ProcessorProvider>
+          <GroupCallView
+            client={client}
+            isPasswordlessUser={false}
+            confineToRoom={false}
+            preload={false}
+            skipLobby={false}
+            hideHeader={true}
+            rtcSession={rtcSession as unknown as MatrixRTCSession}
+            isJoined={joined}
+            muteStates={muteState}
+            widget={widget}
+          />
+        </ProcessorProvider>
       </TooltipProvider>
     </BrowserRouter>,
   );

src/room/GroupCallView.tsx

@@ -62,8 +62,9 @@ import {
 } from "../utils/errors.ts";
 import { GroupCallErrorBoundary } from "./GroupCallErrorBoundary.tsx";
 import {
-  useExperimentalToDeviceTransportSetting,
-  useNewMembershipManagerSetting as useNewMembershipManagerSetting,
+  useNewMembershipManager as useNewMembershipManagerSetting,
+  useExperimentalToDeviceTransport as useExperimentalToDeviceTransportSetting,
+  muteAllAudio as muteAllAudioSetting,
   useSetting,
 } from "../settings/settings";
 import { useTypedEventEmitter } from "../useEvents";
@@ -104,11 +105,13 @@ export const GroupCallView: FC<Props> = ({
     null,
   );
 
+  const [muteAllAudio] = useSetting(muteAllAudioSetting);
   const memberships = useMatrixRTCSessionMemberships(rtcSession);
   const leaveSoundContext = useLatest(
     useAudioContext({
       sounds: callEventAudioSounds,
       latencyHint: "interactive",
+      muted: muteAllAudio,
     }),
   );
   // This should use `useEffectEvent` (only available in experimental versions)

src/room/InCallView.test.tsx

@@ -0,0 +1,265 @@
+/*
+Copyright 2025 New Vector Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+import {
+  beforeEach,
+  describe,
+  expect,
+  it,
+  type MockedFunction,
+  vi,
+} from "vitest";
+import { act, render, type RenderResult } from "@testing-library/react";
+import { type MatrixClient, JoinRule, type RoomState } from "matrix-js-sdk";
+import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc";
+import { type RelationsContainer } from "matrix-js-sdk/lib/models/relations-container";
+import { ConnectionState, type LocalParticipant } from "livekit-client";
+import { of } from "rxjs";
+import { BrowserRouter } from "react-router-dom";
+import { TooltipProvider } from "@vector-im/compound-web";
+import { RoomContext, useLocalParticipant } from "@livekit/components-react";
+import { RoomAndToDeviceEvents } from "matrix-js-sdk/lib/matrixrtc/RoomAndToDeviceKeyTransport";
+
+import { type MuteStates } from "./MuteStates";
+import { InCallView } from "./InCallView";
+import {
+  mockLivekitRoom,
+  mockLocalParticipant,
+  mockMatrixRoom,
+  mockMatrixRoomMember,
+  mockRemoteParticipant,
+  mockRtcMembership,
+  type MockRTCSession,
+} from "../utils/test";
+import { E2eeType } from "../e2ee/e2eeType";
+import { getBasicCallViewModelEnvironment } from "../utils/test-viewmodel";
+import { alice, local } from "../utils/test-fixtures";
+import {
+  developerMode as developerModeSetting,
+  useExperimentalToDeviceTransport as useExperimentalToDeviceTransportSetting,
+} from "../settings/settings";
+import { ReactionsSenderProvider } from "../reactions/useReactionsSender";
+import { useRoomEncryptionSystem } from "../e2ee/sharedKeyManagement";
+import { MatrixAudioRenderer } from "../livekit/MatrixAudioRenderer";
+
+// vi.hoisted(() => {
+//   localStorage = {} as unknown as Storage;
+// });
+vi.hoisted(
+  () =>
+    (global.ImageData = class MockImageData {
+      public data: number[] = [];
+    } as unknown as typeof ImageData),
+);
+
+vi.mock("../soundUtils");
+vi.mock("../useAudioContext");
+vi.mock("../tile/GridTile");
+vi.mock("../tile/SpotlightTile");
+vi.mock("@livekit/components-react");
+vi.mock("../e2ee/sharedKeyManagement");
+vi.mock("../livekit/MatrixAudioRenderer");
+vi.mock("react-use-measure", () => ({
+  default: (): [() => void, object] => [(): void => {}, {}],
+}));
+
+const localRtcMember = mockRtcMembership("@carol:example.org", "CCCC");
+const localParticipant = mockLocalParticipant({
+  identity: "@local:example.org:AAAAAA",
+});
+const remoteParticipant = mockRemoteParticipant({
+  identity: "@alice:example.org:AAAAAA",
+});
+const carol = mockMatrixRoomMember(localRtcMember);
+const roomMembers = new Map([carol].map((p) => [p.userId, p]));
+
+const roomId = "!foo:bar";
+let useRoomEncryptionSystemMock: MockedFunction<typeof useRoomEncryptionSystem>;
+
+beforeEach(() => {
+  vi.clearAllMocks();
+
+  // MatrixAudioRenderer is tested separately.
+  (
+    MatrixAudioRenderer as MockedFunction<typeof MatrixAudioRenderer>
+  ).mockImplementation((_props) => {
+    return <div>mocked: MatrixAudioRenderer</div>;
+  });
+  (
+    useLocalParticipant as MockedFunction<typeof useLocalParticipant>
+  ).mockImplementation(
+    () =>
+      ({
+        isScreenShareEnabled: false,
+        localParticipant: localRtcMember as unknown as LocalParticipant,
+      }) as unknown as ReturnType<typeof useLocalParticipant>,
+  );
+  useRoomEncryptionSystemMock =
+    useRoomEncryptionSystem as typeof useRoomEncryptionSystemMock;
+  useRoomEncryptionSystemMock.mockReturnValue({ kind: E2eeType.NONE });
+});
+
+function createInCallView(): RenderResult & {
+  rtcSession: MockRTCSession;
+} {
+  const client = {
+    getUser: () => null,
+    getUserId: () => localRtcMember.sender,
+    getDeviceId: () => localRtcMember.deviceId,
+    getRoom: (rId) => (rId === roomId ? room : null),
+  } as Partial<MatrixClient> as MatrixClient;
+  const room = mockMatrixRoom({
+    relations: {
+      getChildEventsForEvent: () =>
+        vi.mocked({
+          getRelations: () => [],
+        }),
+    } as unknown as RelationsContainer,
+    client,
+    roomId,
+    getMember: (userId) => roomMembers.get(userId) ?? null,
+    getMxcAvatarUrl: () => null,
+    hasEncryptionStateEvent: vi.fn().mockReturnValue(true),
+    getCanonicalAlias: () => null,
+    currentState: {
+      getJoinRule: () => JoinRule.Invite,
+    } as Partial<RoomState> as RoomState,
+  });
+
+  const muteState = {
+    audio: { enabled: false },
+    video: { enabled: false },
+  } as MuteStates;
+  const livekitRoom = mockLivekitRoom(
+    {
+      localParticipant,
+    },
+    {
+      remoteParticipants$: of([remoteParticipant]),
+    },
+  );
+  const { vm, rtcSession } = getBasicCallViewModelEnvironment([local, alice]);
+
+  rtcSession.joined = true;
+  const renderResult = render(
+    <BrowserRouter>
+      <ReactionsSenderProvider
+        vm={vm}
+        rtcSession={rtcSession as unknown as MatrixRTCSession}
+      >
+        <TooltipProvider>
+          <RoomContext.Provider value={livekitRoom}>
+            <InCallView
+              client={client}
+              hideHeader={true}
+              rtcSession={rtcSession as unknown as MatrixRTCSession}
+              muteStates={muteState}
+              vm={vm}
+              matrixInfo={{
+                userId: "",
+                displayName: "",
+                avatarUrl: "",
+                roomId: "",
+                roomName: "",
+                roomAlias: null,
+                roomAvatar: null,
+                e2eeSystem: {
+                  kind: E2eeType.NONE,
+                },
+              }}
+              livekitRoom={livekitRoom}
+              participantCount={0}
+              onLeave={function (): void {
+                throw new Error("Function not implemented.");
+              }}
+              connState={ConnectionState.Connected}
+              onShareClick={null}
+            />
+          </RoomContext.Provider>
+        </TooltipProvider>
+      </ReactionsSenderProvider>
+    </BrowserRouter>,
+  );
+  return {
+    ...renderResult,
+    rtcSession,
+  };
+}
+
+describe("InCallView", () => {
+  describe("rendering", () => {
+    it("renders", () => {
+      const { container } = createInCallView();
+      expect(container).toMatchSnapshot();
+    });
+  });
+  describe("toDevice label", () => {
+    it("is shown if setting activated and room encrypted", () => {
+      useRoomEncryptionSystemMock.mockReturnValue({
+        kind: E2eeType.PER_PARTICIPANT,
+      });
+      useExperimentalToDeviceTransportSetting.setValue(true);
+      developerModeSetting.setValue(true);
+      const { getByText } = createInCallView();
+      expect(getByText("using to Device key transport")).toBeInTheDocument();
+    });
+
+    it("is not shown in unenecrypted room", () => {
+      useRoomEncryptionSystemMock.mockReturnValue({
+        kind: E2eeType.NONE,
+      });
+      useExperimentalToDeviceTransportSetting.setValue(true);
+      developerModeSetting.setValue(true);
+      const { queryByText } = createInCallView();
+      expect(
+        queryByText("using to Device key transport"),
+      ).not.toBeInTheDocument();
+    });
+
+    it("is hidden once fallback was triggered", async () => {
+      useRoomEncryptionSystemMock.mockReturnValue({
+        kind: E2eeType.PER_PARTICIPANT,
+      });
+      useExperimentalToDeviceTransportSetting.setValue(true);
+      developerModeSetting.setValue(true);
+      const { rtcSession, queryByText } = createInCallView();
+      expect(queryByText("using to Device key transport")).toBeInTheDocument();
+      expect(rtcSession).toBeDefined();
+      await act(() =>
+        rtcSession.emit(RoomAndToDeviceEvents.EnabledTransportsChanged, {
+          toDevice: true,
+          room: true,
+        }),
+      );
+      expect(
+        queryByText("using to Device key transport"),
+      ).not.toBeInTheDocument();
+    });
+    it("is not shown if setting is disabled", () => {
+      useExperimentalToDeviceTransportSetting.setValue(false);
+      developerModeSetting.setValue(true);
+      useRoomEncryptionSystemMock.mockReturnValue({
+        kind: E2eeType.PER_PARTICIPANT,
+      });
+      const { queryByText } = createInCallView();
+      expect(
+        queryByText("using to Device key transport"),
+      ).not.toBeInTheDocument();
+    });
+    it("is not shown if developer mode is disabled", () => {
+      useExperimentalToDeviceTransportSetting.setValue(true);
+      developerModeSetting.setValue(false);
+      useRoomEncryptionSystemMock.mockReturnValue({
+        kind: E2eeType.PER_PARTICIPANT,
+      });
+      const { queryByText } = createInCallView();
+      expect(
+        queryByText("using to Device key transport"),
+      ).not.toBeInTheDocument();
+    });
+  });
+});

src/room/InCallView.tsx

@@ -5,11 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
 Please see LICENSE in the repository root for full details.
 */
 
-import {
-  RoomAudioRenderer,
-  RoomContext,
-  useLocalParticipant,
-} from "@livekit/components-react";
+import { RoomContext, useLocalParticipant } from "@livekit/components-react";
 import { Text } from "@vector-im/compound-web";
 import { ConnectionState, type Room } from "livekit-client";
 import { type MatrixClient } from "matrix-js-sdk";
@@ -56,7 +52,7 @@ import { type OTelGroupCallMembership } from "../otel/OTelGroupCallMembership";
 import { SettingsModal, defaultSettingsTab } from "../settings/SettingsModal";
 import { useRageshakeRequestModal } from "../settings/submit-rageshake";
 import { RageshakeRequestModal } from "./RageshakeRequestModal";
-import { useLiveKit } from "../livekit/useLiveKit";
+import { useLivekit } from "../livekit/useLivekit.ts";
 import { useWakeLock } from "../useWakeLock";
 import { useMergedRefs } from "../useMergedRefs";
 import { type MuteStates } from "./MuteStates";
@@ -73,7 +69,10 @@ import {
 import { Grid, type TileProps } from "../grid/Grid";
 import { useInitial } from "../useInitial";
 import { SpotlightTile } from "../tile/SpotlightTile";
-import { type EncryptionSystem } from "../e2ee/sharedKeyManagement";
+import {
+  useRoomEncryptionSystem,
+  type EncryptionSystem,
+} from "../e2ee/sharedKeyManagement";
 import { E2eeType } from "../e2ee/e2eeType";
 import { makeGridLayout } from "../grid/GridLayout";
 import {
@@ -96,12 +95,15 @@ import { ReactionsOverlay } from "./ReactionsOverlay";
 import { CallEventAudioRenderer } from "./CallEventAudioRenderer";
 import {
   debugTileLayout as debugTileLayoutSetting,
-  useExperimentalToDeviceTransportSetting,
+  useExperimentalToDeviceTransport as useExperimentalToDeviceTransportSetting,
+  muteAllAudio as muteAllAudioSetting,
+  developerMode as developerModeSetting,
   useSetting,
 } from "../settings/settings";
 import { ReactionsReader } from "../reactions/ReactionsReader";
 import { ConnectionLostError } from "../utils/errors.ts";
 import { useTypedEventEmitter } from "../useEvents.ts";
+import { MatrixAudioRenderer } from "../livekit/MatrixAudioRenderer.tsx";
 
 const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {});
 
@@ -114,7 +116,7 @@ export interface ActiveCallProps
 
 export const ActiveCall: FC<ActiveCallProps> = (props) => {
   const sfuConfig = useOpenIDSFU(props.client, props.rtcSession);
-  const { livekitRoom, connState } = useLiveKit(
+  const { livekitRoom, connState } = useLivekit(
     props.rtcSession,
     props.muteStates,
     sfuConfig,
@@ -220,19 +222,34 @@ export const InCallView: FC<InCallViewProps> = ({
     room: livekitRoom,
   });
 
-  const [toDeviceEncryptionSetting] = useSetting(
-    useExperimentalToDeviceTransportSetting,
-  );
-  const [showToDeviceEncryption, setShowToDeviceEncryption] = useState(
-    () => toDeviceEncryptionSetting,
-  );
-  useEffect(() => {
-    setShowToDeviceEncryption(toDeviceEncryptionSetting);
-  }, [toDeviceEncryptionSetting]);
+  const [muteAllAudio] = useSetting(muteAllAudioSetting);
+
+  // This seems like it might be enough logic to use move it into the call view model?
+  const [didFallbackToRoomKey, setDidFallbackToRoomKey] = useState(false);
   useTypedEventEmitter(
     rtcSession,
     RoomAndToDeviceEvents.EnabledTransportsChanged,
-    (enabled) => setShowToDeviceEncryption(enabled.to_device),
+    (enabled) => setDidFallbackToRoomKey(enabled.room),
+  );
+
+  const [developerMode] = useSetting(developerModeSetting);
+  const [useExperimentalToDeviceTransport] = useSetting(
+    useExperimentalToDeviceTransportSetting,
+  );
+  const encryptionSystem = useRoomEncryptionSystem(rtcSession.room.roomId);
+
+  const showToDeviceEncryption = useMemo(
+    () =>
+      developerMode &&
+      useExperimentalToDeviceTransport &&
+      encryptionSystem.kind === E2eeType.PER_PARTICIPANT &&
+      !didFallbackToRoomKey,
+    [
+      developerMode,
+      useExperimentalToDeviceTransport,
+      encryptionSystem.kind,
+      didFallbackToRoomKey,
+    ],
   );
 
   const toggleMicrophone = useCallback(
@@ -693,10 +710,13 @@ export const InCallView: FC<InCallViewProps> = ({
           </Text>
         )
       }
-      <RoomAudioRenderer />
+      <MatrixAudioRenderer
+        members={rtcSession.memberships}
+        muted={muteAllAudio}
+      />
       {renderContent()}
-      <CallEventAudioRenderer vm={vm} />
-      <ReactionsAudioRenderer vm={vm} />
+      <CallEventAudioRenderer vm={vm} muted={muteAllAudio} />
+      <ReactionsAudioRenderer vm={vm} muted={muteAllAudio} />
       <ReactionsOverlay vm={vm} />
       {footer}
       {layout.type !== "pip" && (

src/room/LobbyView.tsx

@@ -12,7 +12,11 @@ import { Button } from "@vector-im/compound-web";
 import classNames from "classnames";
 import { logger } from "matrix-js-sdk/lib/logger";
 import { usePreviewTracks } from "@livekit/components-react";
-import { type LocalVideoTrack, Track } from "livekit-client";
+import {
+  type CreateLocalTracksOptions,
+  type LocalVideoTrack,
+  Track,
+} from "livekit-client";
 import { useObservable } from "observable-hooks";
 import { map } from "rxjs";
 import { useNavigate } from "react-router-dom";
@@ -36,7 +40,11 @@ import { E2eeType } from "../e2ee/e2eeType";
 import { Link } from "../button/Link";
 import { useMediaDevices } from "../livekit/MediaDevicesContext";
 import { useInitial } from "../useInitial";
-import { useSwitchCamera } from "./useSwitchCamera";
+import { useSwitchCamera as useShowSwitchCamera } from "./useSwitchCamera";
+import {
+  useTrackProcessor,
+  useTrackProcessorSync,
+} from "../livekit/TrackProcessorContext";
 import { usePageTitle } from "../usePageTitle";
 
 interface Props {
@@ -112,7 +120,10 @@ export const LobbyView: FC<Props> = ({
       muteStates.audio.enabled && { deviceId: devices.audioInput.selectedId },
   );
 
-  const localTrackOptions = useMemo(
+  const { processor } = useTrackProcessor();
+
+  const initialProcessor = useInitial(() => processor);
+  const localTrackOptions = useMemo<CreateLocalTracksOptions>(
     () => ({
       // The only reason we request audio here is to get the audio permission
       // request over with at the same time. But changing the audio settings
@@ -123,12 +134,14 @@ export const LobbyView: FC<Props> = ({
       audio: Object.assign({}, initialAudioOptions),
       video: muteStates.video.enabled && {
         deviceId: devices.videoInput.selectedId,
+        processor: initialProcessor,
       },
     }),
     [
       initialAudioOptions,
-      devices.videoInput.selectedId,
       muteStates.video.enabled,
+      devices.videoInput.selectedId,
+      initialProcessor,
     ],
   );
 
@@ -149,8 +162,8 @@ export const LobbyView: FC<Props> = ({
         null) as LocalVideoTrack | null,
     [tracks],
   );
-
-  const switchCamera = useSwitchCamera(
+  useTrackProcessorSync(videoTrack);
+  const showSwitchCamera = useShowSwitchCamera(
     useObservable(
       (inputs$) => inputs$.pipe(map(([video]) => video)),
       [videoTrack],
@@ -212,7 +225,9 @@ export const LobbyView: FC<Props> = ({
               onClick={onVideoPress}
               disabled={muteStates.video.setEnabled === null}
             />
-            {switchCamera && <SwitchCameraButton onClick={switchCamera} />}
+            {showSwitchCamera && (
+              <SwitchCameraButton onClick={showSwitchCamera} />
+            )}
             <SettingsButton onClick={openSettings} />
             {!confineToRoom && <EndCallButton onClick={onLeaveClick} />}
           </div>

src/room/MuteStates.test.tsx

@@ -79,6 +79,7 @@ function mockDevices(available: Map<string, DeviceLabel>): MediaDevice {
     selectedId: "",
     selectedGroupId: "",
     select: (): void => {},
+    useAsEarpiece: false,
   };
 }
 

src/room/ReactionAudioRenderer.test.tsx

@@ -21,8 +21,8 @@ import { act, type ReactNode } from "react";
 
 import { ReactionsAudioRenderer } from "./ReactionAudioRenderer";
 import {
-  playReactionsSound,
-  soundEffectVolumeSetting,
+  playReactionsSound as playReactionsSoundSetting,
+  soundEffectVolume as soundEffectVolumeSetting,
 } from "../settings/settings";
 import { useAudioContext } from "../useAudioContext";
 import { GenericReaction, ReactionSet } from "../reactions";
@@ -50,7 +50,7 @@ vitest.mock("../soundUtils");
 
 afterEach(() => {
   vitest.resetAllMocks();
-  playReactionsSound.setValue(playReactionsSound.defaultValue);
+  playReactionsSoundSetting.setValue(playReactionsSoundSetting.defaultValue);
   soundEffectVolumeSetting.setValue(soundEffectVolumeSetting.defaultValue);
 });
 
@@ -74,7 +74,7 @@ beforeEach(() => {
 
 test("preloads all audio elements", () => {
   const { vm } = getBasicCallViewModelEnvironment([local, alice]);
-  playReactionsSound.setValue(true);
+  playReactionsSoundSetting.setValue(true);
   render(<TestComponent vm={vm} />);
   expect(prefetchSounds).toHaveBeenCalledOnce();
 });
@@ -84,7 +84,7 @@ test("will play an audio sound when there is a reaction", () => {
     local,
     alice,
   ]);
-  playReactionsSound.setValue(true);
+  playReactionsSoundSetting.setValue(true);
   render(<TestComponent vm={vm} />);
 
   // Find the first reaction with a sound effect
@@ -110,7 +110,7 @@ test("will play the generic audio sound when there is soundless reaction", () =>
     local,
     alice,
   ]);
-  playReactionsSound.setValue(true);
+  playReactionsSoundSetting.setValue(true);
   render(<TestComponent vm={vm} />);
 
   // Find the first reaction with a sound effect
@@ -136,7 +136,7 @@ test("will play multiple audio sounds when there are multiple different reaction
     local,
     alice,
   ]);
-  playReactionsSound.setValue(true);
+  playReactionsSoundSetting.setValue(true);
   render(<TestComponent vm={vm} />);
 
   // Find the first reaction with a sound effect

src/room/ReactionAudioRenderer.tsx

@@ -24,8 +24,10 @@ const soundMap = Object.fromEntries([
 
 export function ReactionsAudioRenderer({
   vm,
+  muted,
 }: {
   vm: CallViewModel;
+  muted?: boolean;
 }): ReactNode {
   const [shouldPlay] = useSetting(playReactionsSound);
   const [soundCache, setSoundCache] = useState<ReturnType<
@@ -34,6 +36,7 @@ export function ReactionsAudioRenderer({
   const audioEngineCtx = useAudioContext({
     sounds: soundCache,
     latencyHint: "interactive",
+    muted,
   });
   const audioEngineRef = useLatest(audioEngineCtx);
 

src/room/__snapshots__/InCallView.test.tsx.snap

@@ -0,0 +1,181 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`InCallView > rendering > renders 1`] = `
+<div>
+  <div
+    class="inRoom"
+  >
+    <div
+      class="header filler"
+    />
+    <div>
+      mocked: MatrixAudioRenderer
+    </div>
+    <div
+      class="scrollingGrid grid"
+    >
+      <div
+        class="layer"
+      >
+        <div
+          class="container slot"
+          data-id="1"
+        >
+          <div
+            class="slot local slot"
+            data-block-alignment="start"
+            data-id="0"
+            data-inline-alignment="end"
+          />
+        </div>
+      </div>
+    </div>
+    <div
+      class="fixedGrid grid"
+    >
+      <div />
+    </div>
+    <div
+      class="container"
+    />
+    <div
+      class="footer"
+    >
+      <div
+        class="buttons"
+      >
+        <button
+          aria-disabled="false"
+          aria-labelledby=":r0:"
+          class="_button_i91xf_17 _has-icon_i91xf_66 _icon-only_i91xf_59"
+          data-kind="primary"
+          data-size="lg"
+          data-testid="incall_mute"
+          role="button"
+          tabindex="0"
+        >
+          <svg
+            aria-hidden="true"
+            fill="currentColor"
+            height="24"
+            viewBox="0 0 24 24"
+            width="24"
+            xmlns="http://www.w3.org/2000/svg"
+          >
+            <path
+              d="M8 8v-.006l6.831 6.832-.002.002 1.414 1.415.003-.003 1.414 1.414-.003.003L20.5 20.5a1 1 0 0 1-1.414 1.414l-3.022-3.022A7.949 7.949 0 0 1 13 19.938V21a1 1 0 0 1-2 0v-1.062A8.001 8.001 0 0 1 4 12a1 1 0 1 1 2 0 6 6 0 0 0 8.587 5.415l-1.55-1.55A4.005 4.005 0 0 1 8 12v-1.172L2.086 4.914A1 1 0 0 1 3.5 3.5L8 8Zm9.417 6.583 1.478 1.477A7.963 7.963 0 0 0 20 12a1 1 0 0 0-2 0c0 .925-.21 1.8-.583 2.583ZM8.073 5.238l7.793 7.793c.087-.329.134-.674.134-1.031V6a4 4 0 0 0-7.927-.762Z"
+            />
+          </svg>
+        </button>
+        <button
+          aria-disabled="false"
+          aria-labelledby=":r5:"
+          class="_button_i91xf_17 _has-icon_i91xf_66 _icon-only_i91xf_59"
+          data-kind="primary"
+          data-size="lg"
+          data-testid="incall_videomute"
+          role="button"
+          tabindex="0"
+        >
+          <svg
+            aria-hidden="true"
+            fill="currentColor"
+            height="24"
+            viewBox="0 0 24 24"
+            width="24"
+            xmlns="http://www.w3.org/2000/svg"
+          >
+            <path
+              d="M2.747 2.753 4.35 4.355l.007-.003L18 17.994v.012l3.247 3.247a1 1 0 0 1-1.414 1.414l-2.898-2.898A1.992 1.992 0 0 1 16 20H6a4 4 0 0 1-4-4V8c0-.892.292-1.715.785-2.38L1.333 4.166a1 1 0 0 1 1.414-1.414ZM18 15.166 6.834 4H16a2 2 0 0 1 2 2v4.286l3.35-2.871a1 1 0 0 1 1.65.76v7.65a1 1 0 0 1-1.65.76L18 13.715v1.45Z"
+            />
+          </svg>
+        </button>
+        <button
+          aria-labelledby=":ra:"
+          class="_button_i91xf_17 _has-icon_i91xf_66 _icon-only_i91xf_59"
+          data-kind="secondary"
+          data-size="lg"
+          role="button"
+          tabindex="0"
+        >
+          <svg
+            aria-hidden="true"
+            fill="currentColor"
+            height="24"
+            viewBox="0 0 24 24"
+            width="24"
+            xmlns="http://www.w3.org/2000/svg"
+          >
+            <path
+              d="M12.731 2C13.432 2 14 2.568 14 3.269c0 .578.396 1.074.935 1.286.085.034.17.07.253.106.531.23 1.162.16 1.572-.25a1.269 1.269 0 0 1 1.794 0l1.034 1.035a1.269 1.269 0 0 1 0 1.794c-.41.41-.48 1.04-.248 1.572.036.084.07.168.105.253.212.539.708.935 1.286.935.701 0 1.269.568 1.269 1.269v1.462c0 .701-.568 1.269-1.269 1.269-.578 0-1.074.396-1.287.935-.033.085-.068.17-.104.253-.232.531-.161 1.162.248 1.572a1.269 1.269 0 0 1 0 1.794l-1.034 1.034a1.269 1.269 0 0 1-1.794 0c-.41-.41-1.04-.48-1.572-.248a7.935 7.935 0 0 1-.253.105c-.539.212-.935.708-.935 1.286 0 .701-.568 1.269-1.269 1.269H11.27c-.702 0-1.27-.568-1.27-1.269 0-.578-.396-1.074-.935-1.287a7.975 7.975 0 0 1-.253-.104c-.531-.232-1.162-.161-1.572.248a1.269 1.269 0 0 1-1.794 0l-1.034-1.034a1.269 1.269 0 0 1 0-1.794c.41-.41.48-1.04.249-1.572a7.89 7.89 0 0 1-.106-.253C4.343 14.396 3.847 14 3.27 14 2.568 14 2 13.432 2 12.731V11.27c0-.702.568-1.27 1.269-1.27.578 0 1.074-.396 1.286-.935.034-.085.07-.17.106-.253.23-.531.16-1.162-.25-1.572a1.269 1.269 0 0 1 0-1.794l1.035-1.034a1.269 1.269 0 0 1 1.794 0c.41.41 1.04.48 1.572.249a7.93 7.93 0 0 1 .253-.106c.539-.212.935-.708.935-1.286C10 2.568 10.568 2 11.269 2h1.462ZM12 16a4 4 0 1 0 0-8 4 4 0 0 0 0 8Z"
+            />
+          </svg>
+        </button>
+        <button
+          aria-labelledby=":rf:"
+          class="_button_i91xf_17 endCall _has-icon_i91xf_66 _icon-only_i91xf_59 _destructive_i91xf_116"
+          data-kind="primary"
+          data-size="lg"
+          data-testid="incall_leave"
+          role="button"
+          tabindex="0"
+        >
+          <svg
+            aria-hidden="true"
+            fill="currentColor"
+            height="24"
+            viewBox="0 0 24 24"
+            width="24"
+            xmlns="http://www.w3.org/2000/svg"
+          >
+            <path
+              d="m2.765 16.02-2.47-2.416A1.018 1.018 0 0 1 0 12.852c0-.304.098-.555.295-.751a15.64 15.64 0 0 1 5.316-3.786A15.89 15.89 0 0 1 12 7c2.237 0 4.367.443 6.39 1.329a15.977 15.977 0 0 1 5.315 3.772c.197.196.295.447.295.751 0 .305-.098.555-.295.752l-2.47 2.416a1.047 1.047 0 0 1-1.396.108l-3.114-2.363a1.067 1.067 0 0 1-.322-.376 1.066 1.066 0 0 1-.108-.483v-2.27a13.593 13.593 0 0 0-2.12-.524C13.459 9.996 12 9.937 12 9.937s-1.459.059-2.175.175c-.715.116-1.422.29-2.12.523v2.271c0 .179-.036.34-.108.483a1.066 1.066 0 0 1-.322.376l-3.114 2.363a1.047 1.047 0 0 1-1.396-.107Z"
+            />
+          </svg>
+        </button>
+      </div>
+      <div
+        class="toggle layout"
+      >
+        <input
+          aria-labelledby=":rk:"
+          name="layout"
+          type="radio"
+          value="spotlight"
+        />
+        <svg
+          aria-hidden="true"
+          fill="currentColor"
+          height="24"
+          viewBox="0 0 24 24"
+          width="24"
+          xmlns="http://www.w3.org/2000/svg"
+        >
+          <path
+            d="M5 5h14v8h-5a1 1 0 0 0-1 1v5H5V5Zm10 14v-4h4v4h-4ZM5 21h14a2 2 0 0 0 2-2V5a2 2 0 0 0-2-2H5a2 2 0 0 0-2 2v14a2 2 0 0 0 2 2Z"
+          />
+        </svg>
+        <input
+          aria-labelledby=":rp:"
+          checked=""
+          name="layout"
+          type="radio"
+          value="grid"
+        />
+        <svg
+          aria-hidden="true"
+          fill="currentColor"
+          height="24"
+          viewBox="0 0 24 24"
+          width="24"
+          xmlns="http://www.w3.org/2000/svg"
+        >
+          <path
+            d="M4 11a.967.967 0 0 1-.712-.287A.968.968 0 0 1 3 10V4c0-.283.096-.52.288-.712A.968.968 0 0 1 4 3h6a.97.97 0 0 1 .713.288A.968.968 0 0 1 11 4v6c0 .283-.096.52-.287.713A.968.968 0 0 1 10 11H4Zm5-2V5H5v4h4Zm5 12a.968.968 0 0 1-.713-.288A.968.968 0 0 1 13 20v-6c0-.283.096-.52.287-.713A.968.968 0 0 1 14 13h6a.97.97 0 0 1 .712.287c.192.192.288.43.288.713v6c0 .283-.096.52-.288.712A.968.968 0 0 1 20 21h-6Zm5-2v-4h-4v4h4ZM4 21a.967.967 0 0 1-.712-.288A.968.968 0 0 1 3 20v-6a.97.97 0 0 1 .288-.713A.967.967 0 0 1 4 13h6c.283 0 .52.096.713.287.191.192.287.43.287.713v6a.97.97 0 0 1-.287.712A.968.968 0 0 1 10 21H4Zm5-2v-4H5v4h4Zm5-8a.968.968 0 0 1-.713-.287A.968.968 0 0 1 13 10V4a.97.97 0 0 1 .287-.712A.968.968 0 0 1 14 3h6c.283 0 .52.096.712.288A.965.965 0 0 1 21 4v6a.97.97 0 0 1-.288.713A.968.968 0 0 1 20 11h-6Zm5-2V5h-4v4h4Z"
+          />
+        </svg>
+      </div>
+    </div>
+  </div>
+</div>
+`;

src/settings/DeveloperSettingsTab.tsx

@@ -15,8 +15,10 @@ import {
   debugTileLayout as debugTileLayoutSetting,
   showNonMemberTiles as showNonMemberTilesSetting,
   showConnectionStats as showConnectionStatsSetting,
-  useNewMembershipManagerSetting,
-  useExperimentalToDeviceTransportSetting,
+  useNewMembershipManager as useNewMembershipManagerSetting,
+  useExperimentalToDeviceTransport as useExperimentalToDeviceTransportSetting,
+  muteAllAudio as muteAllAudioSetting,
+  alwaysShowIphoneEarpiece as alwaysShowIphoneEarpieceSetting,
 } from "./settings";
 import type { MatrixClient } from "matrix-js-sdk";
 import type { Room as LivekitRoom } from "livekit-client";
@@ -45,10 +47,16 @@ export const DeveloperSettingsTab: FC<Props> = ({ client, livekitRoom }) => {
     useNewMembershipManagerSetting,
   );
 
+  const [alwaysShowIphoneEarpiece, setAlwaysShowIphoneEarpiece] = useSetting(
+    alwaysShowIphoneEarpieceSetting,
+  );
   const [
     useExperimentalToDeviceTransport,
     setUseExperimentalToDeviceTransport,
   ] = useSetting(useExperimentalToDeviceTransportSetting);
+
+  const [muteAllAudio, setMuteAllAudio] = useSetting(muteAllAudioSetting);
+
   const urlParams = useUrlParams();
 
   const sfuUrl = useMemo((): URL | null => {
@@ -175,6 +183,34 @@ export const DeveloperSettingsTab: FC<Props> = ({ client, livekitRoom }) => {
           )}
         />
       </FieldRow>
+      <FieldRow>
+        <InputField
+          id="muteAllAudio"
+          type="checkbox"
+          label={t("developer_mode.mute_all_audio")}
+          checked={muteAllAudio}
+          onChange={useCallback(
+            (event: ChangeEvent<HTMLInputElement>): void => {
+              setMuteAllAudio(event.target.checked);
+            },
+            [setMuteAllAudio],
+          )}
+        />
+      </FieldRow>{" "}
+      <FieldRow>
+        <InputField
+          id="alwaysShowIphoneEarpiece"
+          type="checkbox"
+          label={t("developer_mode.always_show_iphone_earpiece")}
+          checked={alwaysShowIphoneEarpiece}
+          onChange={useCallback(
+            (event: ChangeEvent<HTMLInputElement>): void => {
+              setAlwaysShowIphoneEarpiece(event.target.checked);
+            },
+            [setAlwaysShowIphoneEarpiece],
+          )}
+        />{" "}
+      </FieldRow>
       {livekitRoom ? (
         <>
           <p>

src/settings/DeviceSelection.tsx

@@ -22,17 +22,20 @@ import {
 } from "@vector-im/compound-web";
 import { Trans, useTranslation } from "react-i18next";
 
-import { type MediaDevice } from "../livekit/MediaDevicesContext";
+import {
+  EARPIECE_CONFIG_ID,
+  type MediaDevice,
+} from "../livekit/MediaDevicesContext";
 import styles from "./DeviceSelection.module.css";
 
 interface Props {
-  devices: MediaDevice;
+  device: MediaDevice;
   title: string;
   numberedLabel: (number: number) => string;
 }
 
 export const DeviceSelection: FC<Props> = ({
-  devices,
+  device,
   title,
   numberedLabel,
 }) => {
@@ -40,12 +43,13 @@ export const DeviceSelection: FC<Props> = ({
   const groupId = useId();
   const onChange = useCallback(
     (e: ChangeEvent<HTMLInputElement>) => {
-      devices.select(e.target.value);
+      device.select(e.target.value);
     },
-    [devices],
+    [device],
   );
 
-  if (devices.available.size == 0) return null;
+  // There is no need to show the menu if there is no choice that can be made.
+  if (device.available.size <= 1) return null;
 
   return (
     <div className={styles.selection}>
@@ -60,7 +64,7 @@ export const DeviceSelection: FC<Props> = ({
       </Heading>
       <Separator className={styles.separator} />
       <div className={styles.options}>
-        {[...devices.available].map(([id, label]) => {
+        {[...device.available].map(([id, label]) => {
           let labelText: ReactNode;
           switch (label.type) {
             case "name":
@@ -85,6 +89,16 @@ export const DeviceSelection: FC<Props> = ({
                   </Trans>
                 );
               break;
+            case "earpiece":
+              labelText = t("settings.devices.earpiece");
+              break;
+          }
+
+          let isSelected = false;
+          if (device.useAsEarpiece) {
+            isSelected = id === EARPIECE_CONFIG_ID;
+          } else {
+            isSelected = id === device.selectedId;
           }
 
           return (
@@ -93,7 +107,7 @@ export const DeviceSelection: FC<Props> = ({
               name={groupId}
               control={
                 <RadioControl
-                  checked={id === devices.selectedId}
+                  checked={isSelected}
                   onChange={onChange}
                   value={id}
                 />

src/settings/SettingsModal.tsx

@@ -5,10 +5,10 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
 Please see LICENSE in the repository root for full details.
 */
 
-import { type FC, useState } from "react";
+import { type FC, type ReactNode, useState } from "react";
 import { useTranslation } from "react-i18next";
 import { type MatrixClient } from "matrix-js-sdk";
-import { Root as Form } from "@vector-im/compound-web";
+import { Root as Form, Separator } from "@vector-im/compound-web";
 import { type Room as LivekitRoom } from "livekit-client";
 
 import { Modal } from "../Modal";
@@ -23,13 +23,16 @@ import {
 import { widget } from "../widget";
 import {
   useSetting,
-  soundEffectVolumeSetting,
+  soundEffectVolume as soundEffectVolumeSetting,
+  backgroundBlur as backgroundBlurSetting,
   developerMode,
 } from "./settings";
 import { PreferencesSettingsTab } from "./PreferencesSettingsTab";
 import { Slider } from "../Slider";
 import { DeviceSelection } from "./DeviceSelection";
+import { useTrackProcessor } from "../livekit/TrackProcessorContext";
 import { DeveloperSettingsTab } from "./DeveloperSettingsTab";
+import { FieldRow, InputField } from "../input/Input";
 import { useSubmitRageshake } from "./submit-rageshake";
 
 type SettingsTab =
@@ -64,11 +67,37 @@ export const SettingsModal: FC<Props> = ({
 }) => {
   const { t } = useTranslation();
 
+  // Generate a `Checkbox` input to turn blur on or off.
+  const BlurCheckbox: React.FC = (): ReactNode => {
+    const { supported } = useTrackProcessor();
+
+    const [blurActive, setBlurActive] = useSetting(backgroundBlurSetting);
+
+    return (
+      <>
+        <h4>{t("settings.background_blur_header")}</h4>
+
+        <FieldRow>
+          <InputField
+            id="activateBackgroundBlur"
+            label={t("settings.background_blur_label")}
+            description={
+              supported ? "" : t("settings.blur_not_supported_by_browser")
+            }
+            type="checkbox"
+            checked={!!blurActive}
+            onChange={(b): void => setBlurActive(b.target.checked)}
+            disabled={!supported}
+          />
+        </FieldRow>
+      </>
+    );
+  };
+
   const devices = useMediaDevices();
   useMediaDeviceNames(devices, open);
   const [soundVolume, setSoundVolume] = useSetting(soundEffectVolumeSetting);
   const [soundVolumeRaw, setSoundVolumeRaw] = useState(soundVolume);
-
   const [showDeveloperSettingsTab] = useSetting(developerMode);
 
   const { available: isRageshakeAvailable } = useSubmitRageshake();
@@ -80,17 +109,18 @@ export const SettingsModal: FC<Props> = ({
       <>
         <Form>
           <DeviceSelection
-            devices={devices.audioInput}
+            device={devices.audioInput}
             title={t("settings.devices.microphone")}
             numberedLabel={(n) =>
               t("settings.devices.microphone_numbered", { n })
             }
           />
           <DeviceSelection
-            devices={devices.audioOutput}
+            device={devices.audioOutput}
             title={t("settings.devices.speaker")}
             numberedLabel={(n) => t("settings.devices.speaker_numbered", { n })}
           />
+
           <div className={styles.volumeSlider}>
             <label>{t("settings.audio_tab.effect_volume_label")}</label>
             <p>{t("settings.audio_tab.effect_volume_description")}</p>
@@ -113,13 +143,17 @@ export const SettingsModal: FC<Props> = ({
     key: "video",
     name: t("common.video"),
     content: (
-      <Form>
-        <DeviceSelection
-          devices={devices.videoInput}
-          title={t("settings.devices.camera")}
-          numberedLabel={(n) => t("settings.devices.camera_numbered", { n })}
-        />
-      </Form>
+      <>
+        <Form>
+          <DeviceSelection
+            device={devices.videoInput}
+            title={t("settings.devices.camera")}
+            numberedLabel={(n) => t("settings.devices.camera_numbered", { n })}
+          />
+        </Form>
+        <Separator />
+        <BlurCheckbox />
+      </>
     ),
   };
 

src/settings/settings.ts

@@ -44,6 +44,9 @@ export class Setting<T> {
     this._value$.next(value);
     localStorage.setItem(this.key, JSON.stringify(value));
   };
+  public readonly getValue = (): T => {
+    return this._value$.getValue();
+  };
 }
 
 /**
@@ -96,6 +99,8 @@ export const videoInput = new Setting<string | undefined>(
   undefined,
 );
 
+export const backgroundBlur = new Setting<boolean>("background-blur", false);
+
 export const showHandRaisedTimer = new Setting<boolean>(
   "hand-raised-show-timer",
   false,
@@ -108,19 +113,26 @@ export const playReactionsSound = new Setting<boolean>(
   true,
 );
 
-export const soundEffectVolumeSetting = new Setting<number>(
+export const soundEffectVolume = new Setting<number>(
   "sound-effect-volume",
   0.5,
 );
 
-export const useNewMembershipManagerSetting = new Setting<boolean>(
+export const useNewMembershipManager = new Setting<boolean>(
   "new-membership-manager",
   true,
 );
 
-export const useExperimentalToDeviceTransportSetting = new Setting<boolean>(
+export const useExperimentalToDeviceTransport = new Setting<boolean>(
   "experimental-to-device-transport",
   true,
 );
 
+export const muteAllAudio = new Setting<boolean>("mute-all-audio", false);
+
 export const alwaysShowSelf = new Setting<boolean>("always-show-self", true);
+
+export const alwaysShowIphoneEarpiece = new Setting<boolean>(
+  "always-show-iphone-earpice",
+  false,
+);

src/useAudioContext.test.tsx

@@ -5,14 +5,14 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
 Please see LICENSE in the repository root for full details.
 */
 
-import { expect, test, vitest, afterEach } from "vitest";
+import { expect, vi, afterEach, beforeEach, test } from "vitest";
 import { type FC } from "react";
 import { render } from "@testing-library/react";
-import userEvent from "@testing-library/user-event";
+import userEvent, { type UserEvent } from "@testing-library/user-event";
 
 import { deviceStub, MediaDevicesContext } from "./livekit/MediaDevicesContext";
 import { useAudioContext } from "./useAudioContext";
-import { soundEffectVolumeSetting } from "./settings/settings";
+import { soundEffectVolume as soundEffectVolumeSetting } from "./settings/settings";
 
 const staticSounds = Promise.resolve({
   aSound: new ArrayBuffer(0),
@@ -39,61 +39,73 @@ const TestComponent: FC = () => {
   );
 };
 
-class MockAudioContext {
-  public static testContext: MockAudioContext;
-
-  public constructor() {
-    MockAudioContext.testContext = this;
-  }
-
-  public gain = vitest.mocked(
-    {
-      connect: () => {},
-      gain: {
-        setValueAtTime: vitest.fn(),
-      },
+const gainNode = vi.mocked(
+  {
+    connect: (node: AudioNode) => node,
+    gain: {
+      setValueAtTime: vi.fn(),
+      value: 1,
     },
-    true,
-  );
-
-  public setSinkId = vitest.fn().mockResolvedValue(undefined);
-  public decodeAudioData = vitest.fn().mockReturnValue(1);
-  public createBufferSource = vitest.fn().mockReturnValue(
-    vitest.mocked({
+  },
+  true,
+);
+const panNode = vi.mocked(
+  {
+    connect: (node: AudioNode) => node,
+    pan: {
+      setValueAtTime: vi.fn(),
+      value: 0,
+    },
+  },
+  true,
+);
+/**
+ * A shared audio context test instance.
+ * It can also be used to mock the `AudioContext` constructor in tests:
+ * `vi.stubGlobal("AudioContext", () => testAudioContext);`
+ */
+export const testAudioContext = {
+  gain: gainNode,
+  pan: panNode,
+  setSinkId: vi.fn().mockResolvedValue(undefined),
+  decodeAudioData: vi.fn().mockReturnValue(1),
+  createBufferSource: vi.fn().mockReturnValue(
+    vi.mocked({
       connect: (v: unknown) => v,
       start: () => {},
       addEventListener: (_name: string, cb: () => void) => cb(),
     }),
-  );
-  public createGain = vitest.fn().mockReturnValue(this.gain);
-  public close = vitest.fn().mockResolvedValue(undefined);
-}
+  ),
+  createGain: vi.fn().mockReturnValue(gainNode),
+  createStereoPanner: vi.fn().mockReturnValue(panNode),
+  close: vi.fn().mockResolvedValue(undefined),
+};
+export const TestAudioContextConstructor = vi.fn(() => testAudioContext);
+
+let user: UserEvent;
+beforeEach(() => {
+  vi.stubGlobal("AudioContext", TestAudioContextConstructor);
+  user = userEvent.setup();
+});
 
 afterEach(() => {
-  vitest.unstubAllGlobals();
+  vi.unstubAllGlobals();
+  vi.clearAllMocks();
 });
 
 test("can play a single sound", async () => {
-  const user = userEvent.setup();
-  vitest.stubGlobal("AudioContext", MockAudioContext);
   const { findByText } = render(<TestComponent />);
   await user.click(await findByText("Valid sound"));
-  expect(
-    MockAudioContext.testContext.createBufferSource,
-  ).toHaveBeenCalledOnce();
+  expect(testAudioContext.createBufferSource).toHaveBeenCalledOnce();
 });
+
 test("will ignore sounds that are not registered", async () => {
-  const user = userEvent.setup();
-  vitest.stubGlobal("AudioContext", MockAudioContext);
   const { findByText } = render(<TestComponent />);
   await user.click(await findByText("Invalid sound"));
-  expect(
-    MockAudioContext.testContext.createBufferSource,
-  ).not.toHaveBeenCalled();
+  expect(testAudioContext.createBufferSource).not.toHaveBeenCalled();
 });
 
 test("will use the correct device", () => {
-  vitest.stubGlobal("AudioContext", MockAudioContext);
   render(
     <MediaDevicesContext.Provider
       value={{
@@ -103,6 +115,7 @@ test("will use the correct device", () => {
           selectedGroupId: "",
           available: new Map(),
           select: () => {},
+          useAsEarpiece: false,
         },
         videoInput: deviceStub,
         startUsingDeviceNames: () => {},
@@ -112,21 +125,46 @@ test("will use the correct device", () => {
       <TestComponent />
     </MediaDevicesContext.Provider>,
   );
-  expect(
-    MockAudioContext.testContext.createBufferSource,
-  ).not.toHaveBeenCalled();
-  expect(MockAudioContext.testContext.setSinkId).toHaveBeenCalledWith(
-    "chosen-device",
-  );
+  expect(testAudioContext.createBufferSource).not.toHaveBeenCalled();
+  expect(testAudioContext.setSinkId).toHaveBeenCalledWith("chosen-device");
 });
 
 test("will use the correct volume level", async () => {
-  const user = userEvent.setup();
-  vitest.stubGlobal("AudioContext", MockAudioContext);
   soundEffectVolumeSetting.setValue(0.33);
   const { findByText } = render(<TestComponent />);
   await user.click(await findByText("Valid sound"));
-  expect(
-    MockAudioContext.testContext.gain.gain.setValueAtTime,
-  ).toHaveBeenCalledWith(0.33, 0);
+  expect(testAudioContext.gain.gain.setValueAtTime).toHaveBeenCalledWith(
+    0.33,
+    0,
+  );
+  expect(testAudioContext.pan.pan.setValueAtTime).toHaveBeenCalledWith(0, 0);
+});
+
+test("will use the pan if earpice is selected", async () => {
+  const { findByText } = render(
+    <MediaDevicesContext.Provider
+      value={{
+        audioInput: deviceStub,
+        audioOutput: {
+          selectedId: "chosen-device",
+          selectedGroupId: "",
+          available: new Map(),
+          select: () => {},
+          useAsEarpiece: true,
+        },
+        videoInput: deviceStub,
+        startUsingDeviceNames: () => {},
+        stopUsingDeviceNames: () => {},
+      }}
+    >
+      <TestComponent />
+    </MediaDevicesContext.Provider>,
+  );
+  await user.click(await findByText("Valid sound"));
+  expect(testAudioContext.pan.pan.setValueAtTime).toHaveBeenCalledWith(1, 0);
+
+  expect(testAudioContext.gain.gain.setValueAtTime).toHaveBeenCalledWith(
+    soundEffectVolumeSetting.getValue() * 0.1,
+    0,
+  );
 });

src/useAudioContext.tsx

@@ -9,10 +9,13 @@ import { logger } from "matrix-js-sdk/lib/logger";
 import { useState, useEffect } from "react";
 
 import {
-  soundEffectVolumeSetting as effectSoundVolumeSetting,
+  soundEffectVolume as soundEffectVolumeSetting,
   useSetting,
 } from "./settings/settings";
-import { useMediaDevices } from "./livekit/MediaDevicesContext";
+import {
+  useEarpieceAudioConfig,
+  useMediaDevices,
+} from "./livekit/MediaDevicesContext";
 import { type PrefetchedSounds } from "./soundUtils";
 
 /**
@@ -28,12 +31,15 @@ async function playSound(
   ctx: AudioContext,
   buffer: AudioBuffer,
   volume: number,
+  stereoPan: number,
 ): Promise<void> {
   const gain = ctx.createGain();
   gain.gain.setValueAtTime(volume, 0);
+  const pan = ctx.createStereoPanner();
+  pan.pan.setValueAtTime(stereoPan, 0);
   const src = ctx.createBufferSource();
   src.buffer = buffer;
-  src.connect(gain).connect(ctx.destination);
+  src.connect(gain).connect(pan).connect(ctx.destination);
   const p = new Promise<void>((r) => src.addEventListener("ended", () => r()));
   src.start();
   return p;
@@ -47,6 +53,7 @@ interface Props<S extends string> {
    */
   sounds: PrefetchedSounds<S> | null;
   latencyHint: AudioContextLatencyCategory;
+  muted?: boolean;
 }
 
 interface UseAudioContext<S> {
@@ -62,8 +69,9 @@ interface UseAudioContext<S> {
 export function useAudioContext<S extends string>(
   props: Props<S>,
 ): UseAudioContext<S> | null {
-  const [effectSoundVolume] = useSetting(effectSoundVolumeSetting);
-  const devices = useMediaDevices();
+  const [soundEffectVolume] = useSetting(soundEffectVolumeSetting);
+  const { audioOutput } = useMediaDevices();
+
   const [audioContext, setAudioContext] = useState<AudioContext>();
   const [audioBuffers, setAudioBuffers] = useState<Record<S, AudioBuffer>>();
 
@@ -105,23 +113,30 @@ export function useAudioContext<S extends string>(
     if (audioContext && "setSinkId" in audioContext) {
       // https://developer.mozilla.org/en-US/docs/Web/API/AudioContext/setSinkId
       // @ts-expect-error - setSinkId doesn't exist yet in types, maybe because it's not supported everywhere.
-      audioContext.setSinkId(devices.audioOutput.selectedId).catch((ex) => {
+      audioContext.setSinkId(audioOutput.selectedId).catch((ex) => {
         logger.warn("Unable to change sink for audio context", ex);
       });
     }
-  }, [audioContext, devices]);
+  }, [audioContext, audioOutput.selectedId]);
+  const { pan: earpiecePan, volume: earpieceVolume } = useEarpieceAudioConfig();
 
   // Don't return a function until we're ready.
-  if (!audioContext || !audioBuffers) {
+  if (!audioContext || !audioBuffers || props.muted) {
     return null;
   }
+
   return {
     playSound: async (name): Promise<void> => {
       if (!audioBuffers[name]) {
         logger.debug(`Tried to play a sound that wasn't buffered (${name})`);
         return;
       }
-      return playSound(audioContext, audioBuffers[name], effectSoundVolume);
+      return playSound(
+        audioContext,
+        audioBuffers[name],
+        soundEffectVolume * earpieceVolume,
+        earpiecePan,
+      );
     },
   };
 }

src/utils/test.ts

@@ -27,8 +27,14 @@ import {
   type RemoteParticipant,
   type RemoteTrackPublication,
   type Room as LivekitRoom,
+  Track,
 } from "livekit-client";
 import { randomUUID } from "crypto";
+import {
+  type RoomAndToDeviceEvents,
+  type RoomAndToDeviceEventsHandlerMap,
+} from "matrix-js-sdk/lib/matrixrtc/RoomAndToDeviceKeyTransport";
+import { type TrackReference } from "@livekit/components-core";
 
 import {
   LocalUserMediaViewModel,
@@ -269,8 +275,8 @@ export function mockConfig(config: Partial<ResolvedConfigOptions> = {}): void {
 }
 
 export class MockRTCSession extends TypedEventEmitter<
-  MatrixRTCSessionEvent,
-  MatrixRTCSessionEventHandlerMap
+  MatrixRTCSessionEvent | RoomAndToDeviceEvents,
+  MatrixRTCSessionEventHandlerMap & RoomAndToDeviceEventsHandlerMap
 > {
   public readonly statistics = {
     counters: {},
@@ -305,3 +311,24 @@ export class MockRTCSession extends TypedEventEmitter<
     return this;
   }
 }
+
+export const mockTrack = (identity: string): TrackReference =>
+  ({
+    participant: {
+      identity,
+    },
+    publication: {
+      kind: Track.Kind.Audio,
+      source: "mic",
+      trackSid: "123",
+      track: {
+        attach: vi.fn(),
+        detach: vi.fn(),
+        setAudioContext: vi.fn(),
+        setWebAudioPlugins: vi.fn(),
+        setVolume: vi.fn(),
+      },
+    },
+    track: {},
+    source: {},
+  }) as unknown as TrackReference;

vite.config.js

@@ -11,8 +11,8 @@ import { createHtmlPlugin } from "vite-plugin-html";
 import { codecovVitePlugin } from "@codecov/vite-plugin";
 import { sentryVitePlugin } from "@sentry/vite-plugin";
 import react from "@vitejs/plugin-react";
-import basicSsl from "@vitejs/plugin-basic-ssl";
 import { realpathSync } from "fs";
+import * as fs from "node:fs";
 
 // https://vitejs.dev/config/
 export default defineConfig(({ mode, packageType }) => {
@@ -24,7 +24,6 @@ export default defineConfig(({ mode, packageType }) => {
   process.env.VITE_PACKAGE = packageType ?? "full";
   const plugins = [
     react(),
-    basicSsl(),
     svgrPlugin({
       svgrOptions: {
         // This enables ref forwarding on SVGR components, which is needed, for
@@ -84,6 +83,10 @@ export default defineConfig(({ mode, packageType }) => {
     server: {
       port: 3000,
       fs: { allow },
+      https: {
+        key: fs.readFileSync("./backend/dev_tls_m.localhost.key"),
+        cert: fs.readFileSync("./backend/dev_tls_m.localhost.crt"),
+      },
     },
     build: {
       sourcemap: true,

yarn.lock

@@ -2527,12 +2527,23 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@livekit/protocol@npm:1.36.1, @livekit/protocol@npm:^1.33.0":
-  version: 1.36.1
-  resolution: "@livekit/protocol@npm:1.36.1"
+"@livekit/protocol@npm:1.38.0, @livekit/protocol@npm:^1.38.0":
+  version: 1.38.0
+  resolution: "@livekit/protocol@npm:1.38.0"
   dependencies:
     "@bufbuild/protobuf": "npm:^1.10.0"
-  checksum: 10c0/bb2e56785c542446bef3e2f2fd20b33d01db43b786be87ccb834feee8a664fd32c8231e249b4e1915d7a8eda13af0d59eea479fa710327079a1a370daf05c42e
+  checksum: 10c0/ca64d4f984853054ff60574730b08a761afcd3bdc084e5218663e54b0e7f395aa2022d9d15d982fa094bbc0179cb19ef6a96ec74b1aa3265d118a85d1a4fde33
+  languageName: node
+  linkType: hard
+
+"@livekit/track-processors@npm:^0.5.5":
+  version: 0.5.5
+  resolution: "@livekit/track-processors@npm:0.5.5"
+  dependencies:
+    "@mediapipe/tasks-vision": "npm:0.10.14"
+  peerDependencies:
+    livekit-client: ^1.12.0 || ^2.1.0
+  checksum: 10c0/b86737c4ce6b93c714dfc814c997d07f1d40b0869b5e269b28e00034a3c06da811d9117c4914d8a2706ac760be31f2e9d0705108c07b29f4bbb4626dcd41ed64
   languageName: node
   linkType: hard
 
@@ -2550,6 +2561,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@mediapipe/tasks-vision@npm:^0.10.18":
+  version: 0.10.21
+  resolution: "@mediapipe/tasks-vision@npm:0.10.21"
+  checksum: 10c0/11b2bdf98b8cb6e044f2a954e7c8393169e62c86ff49b3d0b61c3b327d18e1ccd47a187999b023bad48380c9da41bfa66eb165301c80da07746390482cb18a19
+  languageName: node
+  linkType: hard
+
 "@nodelib/fs.scandir@npm:2.1.5":
   version: 2.1.5
   resolution: "@nodelib/fs.scandir@npm:2.1.5"
@@ -3045,14 +3063,14 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@playwright/test@npm:^1.51.0":
-  version: 1.51.0
-  resolution: "@playwright/test@npm:1.51.0"
+"@playwright/test@npm:^1.52.0":
+  version: 1.52.0
+  resolution: "@playwright/test@npm:1.52.0"
   dependencies:
-    playwright: "npm:1.51.0"
+    playwright: "npm:1.52.0"
   bin:
     playwright: cli.js
-  checksum: 10c0/ae83dd2c3a32133de58f44a9dbcd73a8059155ebd8acc736ba8bd0a7ca99b194afe2e8f5a500861d18b1c8f06b4e4ea8de4a2402297c59053d4becc404b47e0a
+  checksum: 10c0/1c428b421593eb4f79b7c99783a389c3ab3526c9051ec772749f4fca61414dfa9f2344eba846faac5f238084aa96c836364a91d81d3034ac54924f239a93e247
   languageName: node
   linkType: hard
 
@@ -4603,6 +4621,22 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@types/dom-mediacapture-transform@npm:^0.1.11":
+  version: 0.1.11
+  resolution: "@types/dom-mediacapture-transform@npm:0.1.11"
+  dependencies:
+    "@types/dom-webcodecs": "npm:*"
+  checksum: 10c0/19c76d54cf31aa2a925011fc5f973dff9a10bdecfdf2285e5e568e61850a0fa2b8c9f1807a1462cbefd57ec26d32eeaa9c359117aca9d9fe7f0d6f2fff33f51e
+  languageName: node
+  linkType: hard
+
+"@types/dom-webcodecs@npm:*":
+  version: 0.1.15
+  resolution: "@types/dom-webcodecs@npm:0.1.15"
+  checksum: 10c0/1407f0352156c99c9b4378fb4c0c799b061520d031903a7f359ad09a6f706cc1fd56bafb272bb1a3decffcb32e54a51d2f07442eb72622464a950cff7f9e8862
+  languageName: node
+  linkType: hard
+
 "@types/estree@npm:1.0.6, @types/estree@npm:^1.0.0":
   version: 1.0.6
   resolution: "@types/estree@npm:1.0.6"
@@ -4797,15 +4831,15 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@typescript-eslint/eslint-plugin@npm:^8.0.0":
-  version: 8.25.0
-  resolution: "@typescript-eslint/eslint-plugin@npm:8.25.0"
+"@typescript-eslint/eslint-plugin@npm:^8.31.0":
+  version: 8.31.0
+  resolution: "@typescript-eslint/eslint-plugin@npm:8.31.0"
   dependencies:
     "@eslint-community/regexpp": "npm:^4.10.0"
-    "@typescript-eslint/scope-manager": "npm:8.25.0"
-    "@typescript-eslint/type-utils": "npm:8.25.0"
-    "@typescript-eslint/utils": "npm:8.25.0"
-    "@typescript-eslint/visitor-keys": "npm:8.25.0"
+    "@typescript-eslint/scope-manager": "npm:8.31.0"
+    "@typescript-eslint/type-utils": "npm:8.31.0"
+    "@typescript-eslint/utils": "npm:8.31.0"
+    "@typescript-eslint/visitor-keys": "npm:8.31.0"
     graphemer: "npm:^1.4.0"
     ignore: "npm:^5.3.1"
     natural-compare: "npm:^1.4.0"
@@ -4813,8 +4847,8 @@ __metadata:
   peerDependencies:
     "@typescript-eslint/parser": ^8.0.0 || ^8.0.0-alpha.0
     eslint: ^8.57.0 || ^9.0.0
-    typescript: ">=4.8.4 <5.8.0"
-  checksum: 10c0/11d63850f5f03b29cd31166f8da111788dc74e46877c2e16a5c488d6c4aa4b6c68c0857b9a396ad920aa7f0f3e7166f4faecbb194c19cd2bb9d3f687c5d2b292
+    typescript: ">=4.8.4 <5.9.0"
+  checksum: 10c0/7d78e0cdcc967742752d49d2d38986ee38d0b7ca64af247e5fe0816cea9ae5f1bfa5c126154acc0846af515c4fb1c52c96926ee25c73b4c3f7e6fd73cb6d2b0e
   languageName: node
   linkType: hard
 
@@ -4829,19 +4863,19 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@typescript-eslint/parser@npm:^8.0.0":
-  version: 8.25.0
-  resolution: "@typescript-eslint/parser@npm:8.25.0"
+"@typescript-eslint/parser@npm:^8.31.0":
+  version: 8.31.0
+  resolution: "@typescript-eslint/parser@npm:8.31.0"
   dependencies:
-    "@typescript-eslint/scope-manager": "npm:8.25.0"
-    "@typescript-eslint/types": "npm:8.25.0"
-    "@typescript-eslint/typescript-estree": "npm:8.25.0"
-    "@typescript-eslint/visitor-keys": "npm:8.25.0"
+    "@typescript-eslint/scope-manager": "npm:8.31.0"
+    "@typescript-eslint/types": "npm:8.31.0"
+    "@typescript-eslint/typescript-estree": "npm:8.31.0"
+    "@typescript-eslint/visitor-keys": "npm:8.31.0"
     debug: "npm:^4.3.4"
   peerDependencies:
     eslint: ^8.57.0 || ^9.0.0
-    typescript: ">=4.8.4 <5.8.0"
-  checksum: 10c0/9a54539ba297791f23093ff42a885cc57d36b26205d7a390e114d1f01cc584ce91ac6ead01819daa46b48f873cac6c829fcf399a436610bdbfa98e5cd78148a2
+    typescript: ">=4.8.4 <5.9.0"
+  checksum: 10c0/9bd903b3ea4e24bfeb444d7a5c2ed82e591ef5cffc0874c609de854c05d34935cd85543e66678ecdb8e0e3eae2cda2df5c1ba66eb72010632cb9f8779031d56d
   languageName: node
   linkType: hard
 
@@ -4865,18 +4899,28 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@typescript-eslint/type-utils@npm:8.25.0":
-  version: 8.25.0
-  resolution: "@typescript-eslint/type-utils@npm:8.25.0"
+"@typescript-eslint/scope-manager@npm:8.31.0":
+  version: 8.31.0
+  resolution: "@typescript-eslint/scope-manager@npm:8.31.0"
   dependencies:
-    "@typescript-eslint/typescript-estree": "npm:8.25.0"
-    "@typescript-eslint/utils": "npm:8.25.0"
+    "@typescript-eslint/types": "npm:8.31.0"
+    "@typescript-eslint/visitor-keys": "npm:8.31.0"
+  checksum: 10c0/eae758a24cc578fa351b8bf0c30c50de384292c0b05a58762f9b632d65a009bd5d902d806eccb6b678cc0b09686289fb4f1fd67da7f12d59ad43ff033b35cc4f
+  languageName: node
+  linkType: hard
+
+"@typescript-eslint/type-utils@npm:8.31.0":
+  version: 8.31.0
+  resolution: "@typescript-eslint/type-utils@npm:8.31.0"
+  dependencies:
+    "@typescript-eslint/typescript-estree": "npm:8.31.0"
+    "@typescript-eslint/utils": "npm:8.31.0"
     debug: "npm:^4.3.4"
     ts-api-utils: "npm:^2.0.1"
   peerDependencies:
     eslint: ^8.57.0 || ^9.0.0
-    typescript: ">=4.8.4 <5.8.0"
-  checksum: 10c0/b7477a2d239cfd337f7d28641666763cf680a43a8d377a09dc42415f715670d35fbb4e772e103dfe8cd620c377e66bce740106bb3983ee65a739c28fab7325d1
+    typescript: ">=4.8.4 <5.9.0"
+  checksum: 10c0/f6938413a583430468b259f6823bb2ab1b5cd77cd6d4e21e1803df70e329046b9579aed5bdc9bdcf4046c8091615a911ac3990859db78d00210bb867915ba37f
   languageName: node
   linkType: hard
 
@@ -4894,6 +4938,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@typescript-eslint/types@npm:8.31.0":
+  version: 8.31.0
+  resolution: "@typescript-eslint/types@npm:8.31.0"
+  checksum: 10c0/04130a30aac477d36d6a155399b27773457aeb9b485ef8fb56fee05725b6e36768c9fac7e4d1f073fd16988de0eb7dffc743c3f834ae907cf918cabb075e5cd8
+  languageName: node
+  linkType: hard
+
 "@typescript-eslint/typescript-estree@npm:5.62.0":
   version: 5.62.0
   resolution: "@typescript-eslint/typescript-estree@npm:5.62.0"
@@ -4930,6 +4981,24 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@typescript-eslint/typescript-estree@npm:8.31.0":
+  version: 8.31.0
+  resolution: "@typescript-eslint/typescript-estree@npm:8.31.0"
+  dependencies:
+    "@typescript-eslint/types": "npm:8.31.0"
+    "@typescript-eslint/visitor-keys": "npm:8.31.0"
+    debug: "npm:^4.3.4"
+    fast-glob: "npm:^3.3.2"
+    is-glob: "npm:^4.0.3"
+    minimatch: "npm:^9.0.4"
+    semver: "npm:^7.6.0"
+    ts-api-utils: "npm:^2.0.1"
+  peerDependencies:
+    typescript: ">=4.8.4 <5.9.0"
+  checksum: 10c0/0ec074b2b9c49f80fafea716aa0cc4b05085e65730a3ef7c7d2d39db1657a40b38abe83f22bbe15ac4f6fdf576692f47d2d057347242e6cef5be81d070f55064
+  languageName: node
+  linkType: hard
+
 "@typescript-eslint/utils@npm:5.62.0":
   version: 5.62.0
   resolution: "@typescript-eslint/utils@npm:5.62.0"
@@ -4948,7 +5017,22 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@typescript-eslint/utils@npm:8.25.0, @typescript-eslint/utils@npm:^8.13.0":
+"@typescript-eslint/utils@npm:8.31.0":
+  version: 8.31.0
+  resolution: "@typescript-eslint/utils@npm:8.31.0"
+  dependencies:
+    "@eslint-community/eslint-utils": "npm:^4.4.0"
+    "@typescript-eslint/scope-manager": "npm:8.31.0"
+    "@typescript-eslint/types": "npm:8.31.0"
+    "@typescript-eslint/typescript-estree": "npm:8.31.0"
+  peerDependencies:
+    eslint: ^8.57.0 || ^9.0.0
+    typescript: ">=4.8.4 <5.9.0"
+  checksum: 10c0/1fd4f62e16a44a5be2de501f70ba4b2d64479e014370bde7bbc6de6897cf1699766a8b7be4deb9b0328e74c2b4171839336ede4e3c60fec6ac8378b623a75275
+  languageName: node
+  linkType: hard
+
+"@typescript-eslint/utils@npm:^8.13.0":
   version: 8.25.0
   resolution: "@typescript-eslint/utils@npm:8.25.0"
   dependencies:
@@ -4983,6 +5067,16 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@typescript-eslint/visitor-keys@npm:8.31.0":
+  version: 8.31.0
+  resolution: "@typescript-eslint/visitor-keys@npm:8.31.0"
+  dependencies:
+    "@typescript-eslint/types": "npm:8.31.0"
+    eslint-visitor-keys: "npm:^4.2.0"
+  checksum: 10c0/e41e2a9e287d11232cda6126377d1df4de69c6e9dc2a14058819cff15280ec654a3877886a6806728196f299766cfbb0b299eb021c2ce168eb15dff5eb07b51b
+  languageName: node
+  linkType: hard
+
 "@ungap/structured-clone@npm:^1.2.0":
   version: 1.2.0
   resolution: "@ungap/structured-clone@npm:1.2.0"
@@ -5049,15 +5143,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@vitejs/plugin-basic-ssl@npm:^1.0.1":
-  version: 1.2.0
-  resolution: "@vitejs/plugin-basic-ssl@npm:1.2.0"
-  peerDependencies:
-    vite: ^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0
-  checksum: 10c0/0d360fcca01f91ade6e451edbea09a107ff9e95cd3c3766c7a069d1a168709df92d96c0bd1eccc66e2739a153e07c75a45321ec487450c0da942606200d8441d
-  languageName: node
-  linkType: hard
-
 "@vitejs/plugin-react@npm:^4.0.1":
   version: 4.3.4
   resolution: "@vitejs/plugin-react@npm:4.3.4"
@@ -6852,7 +6937,9 @@ __metadata:
     "@formatjs/intl-segmenter": "npm:^11.7.3"
     "@livekit/components-core": "npm:^0.12.0"
     "@livekit/components-react": "npm:^2.0.0"
-    "@livekit/protocol": "npm:^1.33.0"
+    "@livekit/protocol": "npm:^1.38.0"
+    "@livekit/track-processors": "npm:^0.5.5"
+    "@mediapipe/tasks-vision": "npm:^0.10.18"
     "@opentelemetry/api": "npm:^1.4.0"
     "@opentelemetry/core": "npm:^1.25.1"
     "@opentelemetry/exporter-trace-otlp-http": "npm:^0.57.0"
@@ -6860,7 +6947,7 @@ __metadata:
     "@opentelemetry/sdk-trace-base": "npm:^1.25.1"
     "@opentelemetry/sdk-trace-web": "npm:^1.9.1"
     "@opentelemetry/semantic-conventions": "npm:^1.25.1"
-    "@playwright/test": "npm:^1.51.0"
+    "@playwright/test": "npm:^1.52.0"
     "@radix-ui/react-dialog": "npm:^1.0.4"
     "@radix-ui/react-slider": "npm:^1.1.2"
     "@radix-ui/react-visually-hidden": "npm:^1.0.3"
@@ -6873,6 +6960,7 @@ __metadata:
     "@testing-library/react": "npm:^16.0.0"
     "@testing-library/user-event": "npm:^14.5.1"
     "@types/content-type": "npm:^1.1.5"
+    "@types/dom-mediacapture-transform": "npm:^0.1.11"
     "@types/grecaptcha": "npm:^3.0.9"
     "@types/jsdom": "npm:^21.1.7"
     "@types/lodash-es": "npm:^4.17.12"
@@ -6883,12 +6971,11 @@ __metadata:
     "@types/react-dom": "npm:^18.3.0"
     "@types/sdp-transform": "npm:^2.4.5"
     "@types/uuid": "npm:10"
-    "@typescript-eslint/eslint-plugin": "npm:^8.0.0"
-    "@typescript-eslint/parser": "npm:^8.0.0"
+    "@typescript-eslint/eslint-plugin": "npm:^8.31.0"
+    "@typescript-eslint/parser": "npm:^8.31.0"
     "@use-gesture/react": "npm:^10.2.11"
     "@vector-im/compound-design-tokens": "npm:^3.0.0"
     "@vector-im/compound-web": "npm:^7.2.0"
-    "@vitejs/plugin-basic-ssl": "npm:^1.0.1"
     "@vitejs/plugin-react": "npm:^4.0.1"
     "@vitest/coverage-v8": "npm:^3.0.0"
     babel-plugin-transform-vite-meta-env: "npm:^1.0.3"
@@ -6931,7 +7018,7 @@ __metadata:
     react-use-measure: "npm:^2.1.1"
     rxjs: "npm:^7.8.1"
     sass: "npm:^1.42.1"
-    typescript: "npm:^5.1.6"
+    typescript: "npm:^5.8.3"
     typescript-eslint-language-service: "npm:^5.0.5"
     unique-names-generator: "npm:^4.6.0"
     vaul: "npm:^1.0.0"
@@ -9281,11 +9368,11 @@ __metadata:
   linkType: hard
 
 "livekit-client@npm:^2.11.3":
-  version: 2.11.3
-  resolution: "livekit-client@npm:2.11.3"
+  version: 2.12.0
+  resolution: "livekit-client@npm:2.12.0"
   dependencies:
     "@livekit/mutex": "npm:1.1.1"
-    "@livekit/protocol": "npm:1.36.1"
+    "@livekit/protocol": "npm:1.38.0"
     events: "npm:^3.3.0"
     loglevel: "npm:^1.9.2"
     sdp-transform: "npm:^2.15.0"
@@ -9293,7 +9380,7 @@ __metadata:
     tslib: "npm:2.8.1"
     typed-emitter: "npm:^2.1.0"
     webrtc-adapter: "npm:^9.0.1"
-  checksum: 10c0/d56444f31c107b46ccd5532038ac77bd21038042910619008267c17894f1d3f054262ae2354d89df6fe0ba325aba01909b0612ad4c290906487c40d91641f6e4
+  checksum: 10c0/8a4657aa6c0f0bc5d1fe77c2cd9603a3b07d4acefa634f1c5151190eed69711e7e599dd09c07915939a418dc8770d87e3529ecf1b029f2a9af7f2172d83acb1c
   languageName: node
   linkType: hard
 
@@ -10295,27 +10382,27 @@ __metadata:
   languageName: node
   linkType: hard
 
-"playwright-core@npm:1.51.0":
-  version: 1.51.0
-  resolution: "playwright-core@npm:1.51.0"
+"playwright-core@npm:1.52.0":
+  version: 1.52.0
+  resolution: "playwright-core@npm:1.52.0"
   bin:
     playwright-core: cli.js
-  checksum: 10c0/8f5de23088c5e97c00327f356b17e0223181e921baf99f4e38d9a3b18d0693db288f8b5389e96d0cb4a1b55f03870f140dd7346128a0c02ce36d11eb92153841
+  checksum: 10c0/640945507e6ca2144e9f596b2a6ecac042c2fd3683ff99e6271e9a7b38f3602d415f282609d569456f66680aab8b3c5bb1b257d8fb63a7fc0ed648261110421f
   languageName: node
   linkType: hard
 
-"playwright@npm:1.51.0":
-  version: 1.51.0
-  resolution: "playwright@npm:1.51.0"
+"playwright@npm:1.52.0":
+  version: 1.52.0
+  resolution: "playwright@npm:1.52.0"
   dependencies:
     fsevents: "npm:2.3.2"
-    playwright-core: "npm:1.51.0"
+    playwright-core: "npm:1.52.0"
   dependenciesMeta:
     fsevents:
       optional: true
   bin:
     playwright: cli.js
-  checksum: 10c0/e8509ea500e03e8051fd243f2347ac3196ff8dde4c20ae3aba4cf723e2b647a0158d209fba062995dab90590229a483d723562cf1ea8b2fc11698617027416fd
+  checksum: 10c0/2c6edf1e15e59bbaf77f3fa0fe0ac975793c17cff835d9c8b8bc6395a3b6f1c01898b3058ab37891b2e4d424bcc8f1b4844fe70d943e0143d239d7451408c579
   languageName: node
   linkType: hard
 
@@ -12672,13 +12759,13 @@ __metadata:
   languageName: node
   linkType: hard
 
-"typescript@npm:^5.1.6":
-  version: 5.7.3
-  resolution: "typescript@npm:5.7.3"
+"typescript@npm:^5.8.3":
+  version: 5.8.3
+  resolution: "typescript@npm:5.8.3"
   bin:
     tsc: bin/tsc
     tsserver: bin/tsserver
-  checksum: 10c0/b7580d716cf1824736cc6e628ab4cd8b51877408ba2be0869d2866da35ef8366dd6ae9eb9d0851470a39be17cbd61df1126f9e211d8799d764ea7431d5435afa
+  checksum: 10c0/5f8bb01196e542e64d44db3d16ee0e4063ce4f3e3966df6005f2588e86d91c03e1fb131c2581baf0fb65ee79669eea6e161cd448178986587e9f6844446dbb48
   languageName: node
   linkType: hard
 
@@ -12692,13 +12779,13 @@ __metadata:
   languageName: node
   linkType: hard
 
-"typescript@patch:typescript@npm%3A^5.1.6#optional!builtin<compat/typescript>":
-  version: 5.7.3
-  resolution: "typescript@patch:typescript@npm%3A5.7.3#optional!builtin<compat/typescript>::version=5.7.3&hash=5786d5"
+"typescript@patch:typescript@npm%3A^5.8.3#optional!builtin<compat/typescript>":
+  version: 5.8.3
+  resolution: "typescript@patch:typescript@npm%3A5.8.3#optional!builtin<compat/typescript>::version=5.8.3&hash=5786d5"
   bin:
     tsc: bin/tsc
     tsserver: bin/tsserver
-  checksum: 10c0/6fd7e0ed3bf23a81246878c613423730c40e8bdbfec4c6e4d7bf1b847cbb39076e56ad5f50aa9d7ebd89877999abaee216002d3f2818885e41c907caaa192cc4
+  checksum: 10c0/39117e346ff8ebd87ae1510b3a77d5d92dae5a89bde588c747d25da5c146603a99c8ee588c7ef80faaf123d89ed46f6dbd918d534d641083177d5fac38b8a1cb
   languageName: node
   linkType: hard