Remove patched version of WebCodecs types

The patch has now been upstreamed to @types/dom-webcodecs.

README.md

@@ -192,11 +192,6 @@ To use it, create a local config by, e.g.,
 The `config.devenv.json` config should work with the backend development
 environment as outlined in the next section out of box.
 
-> [!NOTE]
-> Be aware, that this `config.devenv.json` is exposing a deprecated fallback
-> LiveKit config key. If the homeserver advertises SFU backend via
-> `.well-known/matrix/client` this has precedence.
-
 You're now ready to launch the development server:
 
 ```sh
@@ -212,12 +207,20 @@ See also:
 A docker compose file `dev-backend-docker-compose.yml` is provided to start the
 whole stack of components which is required for a local development environment:
 
-- Minimum Synapse Setup (servername: `synapse.localhost`)
-- LiveKit JWT Service (Note requires Federation API and hence a TLS reverse proxy)
-- Minimum TLS reverse proxy (servername: `synapse.localhost`) Note certificates
-  are valid for at least 10 years from now
+- Minimum Synapse Setup (servername: `synapse.m.localhost`)
+- LiveKit Authorization Service (Note requires Federation API and hence a TLS reverse proxy)
 - Minimum LiveKit SFU Setup using dev defaults for config
 - Redis db for completeness
+- Minimum `localhost` Certificate Authority (CA) for Transport Layer Security (TLS)
+  - Hostnames: `m.localhost`, `*.m.localhost`
+  - Add [./backend/dev_tls_local-ca.crt](./backend/dev_tls_local-ca.crt) to your web browsers trusted
+    certificates
+- Minimum TLS reverse proxy for
+  - Synapse homeserver: `synapse.m.localhost`
+  - MatrixRTC backend: `matrix-rtc.m.localhost`
+  - Local Element Call development `call.m.localhost` via `yarn dev --host `
+  - Element Web `app.m.localhost`
+  - Note certificates will expire on Thu, 03 May 2035 10:32:02 GMT
 
 These use a test 'secret' published in this repository, so this must be used
 only for local development and **_never be exposed to the public Internet._**
@@ -230,6 +233,16 @@ yarn backend
 # podman-compose -f dev-backend-docker-compose.yml up
 ```
 
+> [!NOTE]
+> To ensure your local development frontend functions properly, you’ll need to
+> add certificate exceptions in your browser for `https://localhost:3000`,
+> `https://matrix-rtc.m.localhost/livekit/jwt/healthz` and
+> `https://synapse.m.localhost/.well-known/matrix/client`. This can be either
+> done by adding the minimum localhost CA
+> ([./backend/dev_tls_local-ca.crt](./backend/dev_tls_local-ca.crt)) to your web
+> browsers trusted certificates or by simply copying and pasting each URL into
+> your browser’s address bar and follow the prompts to add the exception.
+
 ### Playwright tests
 
 Our Playwright tests run automatically as part of our CI along with our other

backend/dev_homeserver.yaml

@@ -1,5 +1,5 @@
-server_name: "synapse.localhost"
-public_baseurl: http://synapse.localhost:8008/
+server_name: "synapse.m.localhost"
+public_baseurl: https://synapse.m.localhost/
 
 pid_file: /data/homeserver.pid
 

backend/dev_nginx.conf

@@ -0,0 +1,155 @@
+# Synapse reverse proxy including .well-known/matrix/client
+server {
+    listen                  80;
+    listen                  [::]:80;
+    listen                  443 ssl;
+    listen                  8448 ssl;
+    listen                  [::]:443 ssl;
+    listen                  [::]:8448 ssl;
+    server_name             synapse.m.localhost;
+    ssl_certificate         /root/ssl/cert.pem;
+    ssl_certificate_key     /root/ssl/key.pem;
+
+    # well-known config adding rtc_foci backend
+    # Note well-known is currently not effective due to:
+    # https://spec.matrix.org/v1.12/client-server-api/#well-known-uri the spec
+    # says it must be at https://$server_name/... (implied port 443) Hence, we
+    # currently rely for local development environment on deprecated config.json
+    # setting for livekit_service_url
+    location /.well-known/matrix/client {
+        add_header Access-Control-Allow-Origin *;
+        return 200 '{"m.homeserver": {"base_url": "https://synapse.m.localhost"}, "org.matrix.msc4143.rtc_foci": [{"type": "livekit", "livekit_service_url": "https://matrix-rtc.m.localhost/livekit/jwt"}]}';
+        default_type application/json;
+    }
+
+    # Reverse proxy for Matrix Synapse Homeserver
+    # This is also required for development environment.
+    # Reason: the lk-jwt-service uses the federation API for the openid token
+    #         verification, which requires TLS
+    location / {
+        proxy_pass "http://homeserver:8008";
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection "upgrade";
+        proxy_set_header Host $host;
+        proxy_set_header X-Forwarded-For $remote_addr;
+        proxy_set_header X-Forwarded-Proto $scheme;
+    }
+
+    error_page   500 502 503 504  /50x.html;
+
+}
+
+# MatrixRTC reverse proxy
+# - MatrixRTC Authorization Service
+# - LiveKit SFU websocket signaling connection
+server {
+    listen                  80;
+    listen                  [::]:80;
+    listen                  443 ssl;
+    listen                  [::]:443 ssl;
+    listen                  8448 ssl;
+    listen                  [::]:8448 ssl;
+    server_name             matrix-rtc.m.localhost;
+    ssl_certificate         /root/ssl/cert.pem;
+    ssl_certificate_key     /root/ssl/key.pem;
+
+
+    location ^~ /livekit/jwt/ {
+
+      proxy_set_header Host $host;
+      proxy_set_header X-Real-IP $remote_addr;
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_set_header X-Forwarded-Proto $scheme;
+
+      # JWT Service running at port 8080
+      proxy_pass http://auth-server:8080/;
+    }
+
+    location ^~ /livekit/sfu/ {
+      proxy_set_header Host $host;
+      proxy_set_header X-Real-IP $remote_addr;
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_set_header X-Forwarded-Proto $scheme;
+
+      proxy_send_timeout 120;
+      proxy_read_timeout 120;
+      proxy_buffering off;
+
+      proxy_set_header Accept-Encoding gzip;
+      proxy_set_header Upgrade $http_upgrade;
+      proxy_set_header Connection "upgrade";
+
+      # LiveKit SFU websocket connection running at port 7880
+      proxy_pass http://livekit-sfu:7880/;
+    }
+    
+    error_page   500 502 503 504  /50x.html;
+
+}
+
+# Convenience reverse proxy for the call.m.localhost domain to yarn dev --host
+server {
+    listen                  80;
+    listen                  [::]:80;
+    server_name             call.m.localhost;
+
+    return 301 https://$host$request_uri;
+}
+
+server {
+    listen                  443 ssl;
+    listen                  [::]:443 ssl;
+    server_name             call.m.localhost;
+    ssl_certificate         /root/ssl/cert.pem;
+    ssl_certificate_key     /root/ssl/key.pem;
+
+
+    location ^~ / {
+
+      proxy_set_header Host $host;
+      proxy_set_header X-Real-IP $remote_addr;
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_set_header X-Forwarded-Proto $scheme;
+
+      proxy_pass https://host.docker.internal:3000;
+      proxy_ssl_verify              off;
+
+    }
+    
+    error_page   500 502 503 504  /50x.html;
+
+}
+
+# Convenience reverse proxy app.m.localhost for element web
+server {
+    listen                  80;
+    listen                  [::]:80;
+    server_name             app.m.localhost;
+
+    return 301 https://$host$request_uri;
+}
+
+server {
+    listen                  443 ssl;
+    listen                  [::]:443 ssl;
+    server_name             app.m.localhost;
+    ssl_certificate         /root/ssl/cert.pem;
+    ssl_certificate_key     /root/ssl/key.pem;
+
+
+    location ^~ / {
+
+      proxy_set_header Host $host;
+      proxy_set_header X-Real-IP $remote_addr;
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_set_header X-Forwarded-Proto $scheme;
+
+      proxy_pass http://element-web:81;
+      proxy_ssl_verify              off;
+
+    }
+    
+    error_page   500 502 503 504  /50x.html;
+
+}

backend/dev_tls_local-ca.crt

@@ -0,0 +1,19 @@
+-----BEGIN CERTIFICATE-----
+MIIDGjCCAgKgAwIBAgIUGdiFHhH4KL2pqBjMQHQ+PVIkSV8wDQYJKoZIhvcNAQEL
+BQAwHjEcMBoGA1UEAwwTRWxlbWVudCBDYWxsIERldiBDQTAeFw0yNTA1MDUxMDMy
+MDJaFw0zNTA1MDMxMDMyMDJaMB4xHDAaBgNVBAMME0VsZW1lbnQgQ2FsbCBEZXYg
+Q0EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDA2y0hjmNn1vRsVSdy
+8IOfo8N1q9UgkhQWpGKXzPh+D5d1fnuJEmHIVwtDEtS/PwQ43LTmegChPtKH9jdT
+tG0IihW9Ja5YNG+9xAwaoA/sB3CGCBYsz+2/XjVUpXoBJXIPoFBWsn+K0oeFw9fw
+eRO1z9abM4cl+LjKzMNM8CCyu9uI1MaGjYez2YIWvG854VucLxX7HSlMJxZNWnie
+Ui7fMakuJhB2+aiIQjdKxy4E5RHNhzYG/LXhvP+wBYBDPNRsP3rtzEaE9HAveL9K
+FGqd3R4cBia6r1WIXmpAzyu5RGP5Eou0TZlGkal96/bF0I7q/pKlL23Jt1BLPiQU
+KGKrAgMBAAGjUDBOMB0GA1UdDgQWBBQJqBjMu61c1p24txw/y+kv3D+V6DAfBgNV
+HSMEGDAWgBQJqBjMu61c1p24txw/y+kv3D+V6DAMBgNVHRMEBTADAQH/MA0GCSqG
+SIb3DQEBCwUAA4IBAQB8m2YfFGLugNt5vAAOvNxVqDA8c72yCVYr3CBCpmTIEY5Z
+d3qVGhG9//ux6+J8ntkSwd9nV5GJyYXHukCG1VavnAWolWdNF/WAllf0jhLuz7kD
+/cJnuI1By4tBsBmSz851i6HJ4t5k99Be+6GQVzi0e7zzfxTHZE4xP2J6Ox8QbPsP
+n0m76nIp/WbWaJqzvIIjJhmUUPPv+4wN+eOArgjiGLzptM2qTtGZtd0c9nS5gvep
++mEbSUN9zkhAroZf80wf+hEvy+fJ94VbZ9QjTzTg7odZLrsXGIe8DaG63EYRQ25b
+W5iYBAreln5fGSt7qHsGfqwZibTEk/Lx3dydO1Kg
+-----END CERTIFICATE-----

backend/dev_tls_local-ca.key

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDA2y0hjmNn1vRs
+VSdy8IOfo8N1q9UgkhQWpGKXzPh+D5d1fnuJEmHIVwtDEtS/PwQ43LTmegChPtKH
+9jdTtG0IihW9Ja5YNG+9xAwaoA/sB3CGCBYsz+2/XjVUpXoBJXIPoFBWsn+K0oeF
+w9fweRO1z9abM4cl+LjKzMNM8CCyu9uI1MaGjYez2YIWvG854VucLxX7HSlMJxZN
+WnieUi7fMakuJhB2+aiIQjdKxy4E5RHNhzYG/LXhvP+wBYBDPNRsP3rtzEaE9HAv
+eL9KFGqd3R4cBia6r1WIXmpAzyu5RGP5Eou0TZlGkal96/bF0I7q/pKlL23Jt1BL
+PiQUKGKrAgMBAAECggEAAPX2kxi5AQ7ul82SzT1KgpSXyDHLdYaUyAoYnaX9RO+B
+8ylmpyeqygs4+KQS4EMJm9jpo85Oy37bIKdG3kljU6wQcKlL5Y+ZUOo1nzpV6fid
+hGVs6ts8VXw8KshKQ9AyccZ8L/pirUfgOffgTwfjY7/90zceAL/s98GuZWc62nkX
+55joQv/OikqYfAGP/U6Bp2Zyf23DwJB09Z3B6NnZj/ZyAbDrDEHuA15LhCOcCczp
+IU/mFEywBPHT9Tg4w4Beq78PeAETvku2UalYRLhP3RLlXr2oEbwUtINRVt2QjZ85
+Esps4uCqL/mgQluIebtudD9HL/YMlNPXue1mDXFxJQKBgQDgZZY4yJBcf488T1V6
+HNm06b/LvVGj253pKgw14hpY1xQu3Ymgzv1GEqzhSYdzxhpmj0tMUNHxAp+YdGQu
+SZ0wcPKhw0aYVkIjDRYDC3Wn5GJhyIEYHGYMo/n4l49UzHRBPOTDzp49DkHTKBgh
+XgIIazYT3CkjTIMRrkUv+qfIPQKBgQDcBGu/mqbjxs4sN3zqPS4aB21o6t6W0sXs
+ZP9w6RlTPQi5U2oRbftjZtYc0bbEgkMUImB1HwYPQT5pJ+MyC414xDvSc2exBr5d
+To6yyPIy78Tf5PHM12fpKV92nSvoz/pSjYcGxxDtKfPqu+t8mOJfjCV1lLLA+xuB
+DDaE4p8dBwKBgQCdAne6A5v/HMH8UQZeCxHJpESvKiiVnnU/UEx651nID7XvlNNX
+0X0mKqsMd4ZvW43ddSYan/JF0LAa3FW8jYWO/3jF9vzOWoysOdvNBZetgf/Uq5ao
+aDZ/YbzmVCXWD7jIbPMkjs3pqrAkL0mzDzQc7+dGviWKrV6IYIfIqnn7gQKBgDCz
+vdIk/qpO+JZrFfiX4Fucp0hhLTJ/p5ZDaRPqVVPKn+K+Jy2ChfIj8mNgvK9VEloj
+nexvGJ1J2PHYBX+vdPp1nbRhHWPfVUY8PHQw7QP/dToGaMvqJrNDGEGeWvjnCMc7
+UtdaO1H0Rm0AegkTopB56lTTvJnhO95eALd7nrMDAoGAEPdzJtWoKafp49svhSj0
+hiXQv2SPBwVUN4LZ4SOWiXUcmYYm80aNpYKLkBxYjrfqFWhE7NUHLGp8YorQWKY2
+acD9AReHk/xku0ABy6jeYmSCmCxASxst5liKD+l12sk0gB0rk5MBxB4Uu1MIbQZ2
+aCASX3AVD2/XyC2MKkzc8Eg=
+-----END PRIVATE KEY-----

backend/dev_tls_m.localhost.crt

@@ -0,0 +1,21 @@
+-----BEGIN CERTIFICATE-----
+MIIDZzCCAk+gAwIBAgIUXizLjwkdqepX0bh0K3abeJxj68IwDQYJKoZIhvcNAQEL
+BQAwHjEcMBoGA1UEAwwTRWxlbWVudCBDYWxsIERldiBDQTAeFw0yNTA1MDUxMzU5
+MTFaFw0zNTA1MDMxMzU5MTFaMBgxFjAUBgNVBAMMDSoubS5sb2NhbGhvc3QwggEi
+MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCrzGSScSgaQuZdELGFYiLiYRwr
+LKyUdNr0rsPcOo0bvbeZ3zQMeUMRNlA69zGFdarumiDRXUoAmZI39WmH95aX3d+A
+U7EFnWev7xpWSVhSYj8T0d4rke8HjGk3LpaffJ93tbJuagBIH1ouuN6AOdzWs8hp
+RYIomWleEeeuVnnfaMwaXOdc+ihJJ6wzm2hwQSfdpjZPWBDd/DFft1ZXxIZOCjDs
+rEIiI7uU8iZPLB3QEM/tgxSSAOxrcKvQvxZokk+FD7aMJFP71IfieLCEzMTP1VXa
+tP7UTAKAqB2NyDJ8m3IHbOINiqcdFvFR3R1D9bXOYE4oRynNvYZrQUGnL2RtAgMB
+AAGjgaIwgZ8wHwYDVR0jBBgwFoAUCagYzLutXNaduLccP8vpL9w/legwCQYDVR0T
+BAIwADALBgNVHQ8EBAMCBaAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwMAYDVR0RBCkw
+J4IJbG9jYWxob3N0ggttLmxvY2FsaG9zdIINKi5tLmxvY2FsaG9zdDAdBgNVHQ4E
+FgQUfdh1p52ZgWyZcBgBXGwKi4EnUE0wDQYJKoZIhvcNAQELBQADggEBAKrHEuB6
+33j8+EwSHw3zrvt/DRXK2BDHI1Ir9JcztSunaKAjZXVvf/dvZp0Xs1dEdJIdnv6G
+iZYhBbOqDqpQZbf2h/h0kuu5yZSBUdnQXnYNxlhp2UaC/UEgw5iZT/p1rm7RjVie
+y4Dp2WytV5iZOLmLj6xDvd3DXazgJPWIRX8p8qJZbKTkwCjTr7nDIj8jjG1sVFf7
+1RJBO5/6WSnImrpDmlLUrvjiKvbxcdseDJyBOhTwdRdSk4S2M+s5tR5j2I1gXLOq
+J5ioN76+SCrTY0K0WKRy9oOXWO1/X3+VYcekp+0F3SGkd5w17jylCv1XIGHAdEsQ
+v2z2/aMI/7sAD2Q=
+-----END CERTIFICATE-----

backend/dev_tls_m.localhost.key

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCrzGSScSgaQuZd
+ELGFYiLiYRwrLKyUdNr0rsPcOo0bvbeZ3zQMeUMRNlA69zGFdarumiDRXUoAmZI3
+9WmH95aX3d+AU7EFnWev7xpWSVhSYj8T0d4rke8HjGk3LpaffJ93tbJuagBIH1ou
+uN6AOdzWs8hpRYIomWleEeeuVnnfaMwaXOdc+ihJJ6wzm2hwQSfdpjZPWBDd/DFf
+t1ZXxIZOCjDsrEIiI7uU8iZPLB3QEM/tgxSSAOxrcKvQvxZokk+FD7aMJFP71Ifi
+eLCEzMTP1VXatP7UTAKAqB2NyDJ8m3IHbOINiqcdFvFR3R1D9bXOYE4oRynNvYZr
+QUGnL2RtAgMBAAECggEAJaFQii8U/KOYt9vXNoMnZvSkaeSQLLhn2V6Kciu1CtWE
+aMTWLsFE6nk+G5xXkYcTmM3T0GghtH3u5CjyI6EcsEkeEorCZJt0wbmayDmqiekR
+LfMzOdHuTHX5+edPgMGYYG1BFyRKyYFsjH1b5zRFZhXdGQnrl5760GsVlz9D1KZQ
+iHcT+q1S2tmZeoUukQnADENKXUMCyTGM5FCddgNtsWnGDsTDayh7hUdvDkB+mW4G
+lSp+BZuc3PCwpbD6qkXvfugWs6CUAAtXoV3ceWgxQ+TEnNlwxaG1AyugfgNUBolk
+8xgeZt4r5QId03jsHDf7hpBAofcaCd5EMIIQYFvWoQKBgQDlbAvAzEFPTZZn2nRV
+Xagw4xjqVc1LLEKLCWq0N5rEkwn0h90Dz5N7/3NuonP/sIDsDHCbyiOYBI1Ck6Xi
+0WuB+OyKDh+xeF2mekN9G9ywPahdK5lT/TVsxXFyZlwtVv1x/6KBO4yv5URizxqU
+gyAPDDxfD/KcNjkOBaodWEwQGQKBgQC/s2gPDBtQkjLwkHXchBomLww5eLlVrac1
+WK4UX6uSdOgrjJ375OOgMTxe8NVZdOuAKytGXRWDwgH3nVWvuZhe7dGlX3JMuSer
+e9VwDpBESrvqcR4ruL6wm8wej6BXyjH0wD3FHb0S5HfuBDxTn+4bDwrbRzOUMNgy
+lSppuflxdQKBgQDiZcIfazFT8evn5nMAvuC4BZNTxIJHmZC9JfjPiUPIkpWzYtOe
+7BvNtKOT3Op9uw8uYYRKqKqBXJSNy6ha8XCXHS9HeXKbLn20SFkLQBCDNwVLlDfF
+40zyXtF6JDr4XyzSb4NM5pgKCER5AYloXxGm59s3sEQpFXUuOjbKqJS/GQKBgAoI
+c7vF4HAZFr1sch62cz/oWnVvkhOf4Q5zs7ixQSOLJtOQqnwSgK9TpFs7s47ZBbJR
+kBRAru2Ua9Hv1Bo8VnMxczV6h1roneDlvEf/GyHX33nnrbKQGrrXjJlU3wl5NaAf
+p5v3cHvapUQ5yIZ/6lBUOzc6xMJOxCHxmKSr7Rg5AoGAbEE4lt6Xh2dnBPJ81eNI
+IDrw/3ITY53qAY4Bx88CByIFuu8CEUdUZprh98jSl6ic1tMinZfUhRMwABLrUD51
+DGst8iGLPD9u83iMcUHI/L+p7AbxrKLvWXZrF5UZm440c9mSWqfhPaTBosPtNDsG
+LfETwH1flKXMTXd2xA9RTE4=
+-----END PRIVATE KEY-----

backend/dev_tls_setup

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+# Step 1: Create a Root CA key and cert
+openssl genrsa -out dev_tls_local-ca.key 2048
+openssl req -x509 -new -nodes \
+  -days 3650 \
+  -subj "/CN=Element Call Dev CA" \
+  -key dev_tls_local-ca.key \
+  -out dev_tls_local-ca.crt \
+  -sha256 -addext "basicConstraints=CA:TRUE"
+
+# Step 2: Create a private key and CSR for *.m.localhost
+openssl req -new -nodes -newkey rsa:2048 \
+  -keyout dev_tls_m.localhost.key \
+  -out dev_tls_m.localhost.csr \
+  -subj "/CN=*.m.localhost"
+
+# Step 3: Sign the CSR with your CA
+openssl x509 \
+  -req -in dev_tls_m.localhost.csr \
+  -CA dev_tls_local-ca.crt -CAkey dev_tls_local-ca.key \
+  -CAcreateserial \
+  -out dev_tls_m.localhost.crt \
+  -days 3650 \
+  -sha256 \
+  -extfile <( cat <<EOF
+authorityKeyIdentifier=keyid,issuer
+basicConstraints=CA:FALSE
+keyUsage = digitalSignature, keyEncipherment
+extendedKeyUsage = serverAuth
+subjectAltName = @alt_names
+
+[alt_names]
+DNS.1 = localhost
+DNS.2 = m.localhost
+DNS.3 = *.m.localhost
+EOF
+)

backend/ew.test.config.json

@@ -1,8 +1,8 @@
 {
   "default_server_config": {
     "m.homeserver": {
-      "base_url": "http://synapse.localhost:8008",
-      "server_name": "synapse.localhost"
+      "base_url": "https://synapse.m.localhost",
+      "server_name": "synapse.m.localhost"
     }
   },
   "disable_custom_urls": false,

backend/playwright_homeserver.yaml

@@ -1,5 +1,5 @@
-server_name: "synapse.localhost"
-public_baseurl: http://synapse.localhost:8008/
+server_name: "synapse.m.localhost"
+public_baseurl: https://synapse.m.localhost/
 
 pid_file: /data/homeserver.pid
 

backend/tls_localhost_cert.pem

@@ -1,22 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIDtzCCAp+gAwIBAgIUCmJjl3HAeLmrPwRg+/OzikW6peQwDQYJKoZIhvcNAQEL
-BQAwazELMAkGA1UEBhMCR0IxDzANBgNVBAgMBkxvbmRvbjEPMA0GA1UEBwwGTG9u
-ZG9uMQ4wDAYDVQQKDAVBbHJvczEWMBQGA1UECwwNSVQgRGVwYXJ0bWVudDESMBAG
-A1UEAwwJbG9jYWxob3N0MB4XDTI0MTEwNDIxNDcwMFoXDTM0MTEwMjIxNDcwMFow
-azELMAkGA1UEBhMCR0IxDzANBgNVBAgMBkxvbmRvbjEPMA0GA1UEBwwGTG9uZG9u
-MQ4wDAYDVQQKDAVBbHJvczEWMBQGA1UECwwNSVQgRGVwYXJ0bWVudDESMBAGA1UE
-AwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAs368
-ExLSudP8luNoY5UfaPqBSVJUPYBi+JGyd36tyN75p5OI7xSfHTttQxuD4KrExBFP
-C8mAhE1eoZPBVBOZJ4FYWBJfMaQnCjeqU+laP36td65kSJYbUYlKYH1WpxEpCdgx
-wWOKkP/kPX5YXbYqODx9aBJXgoT3yAJW7AniIoL+eLFnS9Xo86TPqCDBTJU9ocwK
-gPIDLhDv60724rhZT1kbGp7ECqRovndoDTQjuws2D3yNMfQ+4rrQGPXHGmP5PcaR
-0R7uueB+6APyC7MJbuhbxxg/+DFHrRi3lJsgwxuh2hi/+vWw8zgKlgYIwHFA9X0l
-cX0UlQdENMH3bgcGIwIDAQABo1MwUTAdBgNVHQ4EFgQUUFGxw7zoiHXGwRqtagjZ
-RPYc85cwHwYDVR0jBBgwFoAUUFGxw7zoiHXGwRqtagjZRPYc85cwDwYDVR0TAQH/
-BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEALokb1z2lu3qW141b2wm14ilZQKCZ
-reNNuUR95Uom96FXPH4QVEH+mYTXXJ5UrfNhQYKQFpdE+5S4HL/UqEOxtWvbAHpK
-nsLQ62J8m+0+uwiJGqeQpWr03KJgXDAVE9X3XwMlp/+buxSLhc+GIHWuXW56itV2
-jiZJYjhO5SnhhgTWNoVZk93qXuuWEN0yacw7c3Fr1IvFYYYWufbXTk70dbZihPDK
-VD141o8tpp6FerSKHNYDqkVFDyTz3DVOhQQJ59zfMre7bFr+PpTTl4vIuGzXEY+E
-HPjUSlOzwkCoh5fu7Fs3qG55rJt8akhTEoKpiBTaLucgAjVWNHeci1+Yxg==
------END CERTIFICATE-----

backend/tls_localhost_key.pem

@@ -1,28 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCzfrwTEtK50/yW
-42hjlR9o+oFJUlQ9gGL4kbJ3fq3I3vmnk4jvFJ8dO21DG4PgqsTEEU8LyYCETV6h
-k8FUE5kngVhYEl8xpCcKN6pT6Vo/fq13rmRIlhtRiUpgfVanESkJ2DHBY4qQ/+Q9
-flhdtio4PH1oEleChPfIAlbsCeIigv54sWdL1ejzpM+oIMFMlT2hzAqA8gMuEO/r
-TvbiuFlPWRsansQKpGi+d2gNNCO7CzYPfI0x9D7iutAY9ccaY/k9xpHRHu654H7o
-A/ILswlu6FvHGD/4MUetGLeUmyDDG6HaGL/69bDzOAqWBgjAcUD1fSVxfRSVB0Q0
-wfduBwYjAgMBAAECggEACTqdSExxzJ+LX5ARFaWyOBSWly2GKqSyR14+aInOklhx
-9QgkmfOxJrCf3TvJ8RWhXloW0Aqr8qGDxG0Ixgjn7rG7gskXCey1xn8MNppLS0kj
-ztaG+NB3AR89ABm8XdoHsSY45geh3/Ni9I0i1VardGQafUJhgNLTZqjwIodzkBtJ
-S/bi4uFk1lGNfuvWQvWqzGXUvd1l1YupV6iA4GfhXlUvrSBZwftLBD6xEvQaSqsA
-pHvBxTfMXG4RMAkNPDIElkuQ8++CGi1gIRkJfmrv4OgbbitteMnxqqqGYV0zSNCg
-R/5FG6umIV7lDLBHZCSCk7wmfmq2UUvzhHThHy4yMQKBgQDu4TwFJCIcVIj7Wj4r
-DUBFvz6Lgbltqb+YAMUBtpiDcAQxDJWmedh6dK04ts5CFAFRlRjjuz2uFn7qlVBm
-uye9R7tL+tOv5viqDXU78a4snFywoXub6yzpbxrW8B4W1pdIUvQmhwCcDwvO1V24
-7Vj2vxcM5I9dsk1aCQSi3VY5yQKBgQDAW/VoTRwhU6OUc6sji5Z5dnkMjkP6NZK9
-CSrTWLAMGaLPY+g6fFS7JMNSvfWm/okypD6rcN7p0cxMK3mfFKmMiyPRde0wdrci
-sGFjGxM/2d2D7KTMC9iMYwA0K17UIna+UiYPfhR/muIg/dCyjlkKDFs9Z4jk//r1
-91bmznt2iwKBgFdiYXhn/Wprqih4nKFXGZnqGdEixVhObl4GegrkZuo+AeqHdf8O
-N5ikMfG7PbyCYPEdH5u/FRMn+4mI0X6jHChroyJqQSHp1jEu9yHUiSicknOyvusM
-nsNN932FHRyxp2m3nsSxQhHUlzc0ajKJ8K9iu+XlfmSCIzW6cs25Nh+xAoGBAJro
-M0wIdPPdsCj3sUVRvx8XqknTM6kGhaIYBNXoYPWNm5BaC4U15OJEq8sxUOdnqcMP
-g6x6m/k+S8C3bh0O/a9Bydl/l0BlCfw0gGjYP/s2ju4Tn272xy/e9iYNGzPIgUmp
-TB9D0GwmpZ4d6HgyrD+sTbm4bATGpCp6QhBjDggbAoGBAJVMMtZ4pF8D6mLMRZGR
-pQjNPy+MH13XYmDRc/BSF8KJ4yKk3tohr9LSXzxR0SEB43NoL1bHkucZrNjGyL8x
-jktnwkoIs96kO2mPrl1TqWkXs5RjGkkSTbAJovIcvkRU31SWap/WzN2kHpmRVcQc
-KEFKXT5fUYZCLLWxhgZFlGPp
------END PRIVATE KEY-----

backend/tls_localhost_nginx.conf

@@ -1,40 +0,0 @@
-server {
-    listen                  80;
-    listen                  [::]:80;
-    listen                  443 ssl;
-    listen                  8448 ssl;
-    listen                  [::]:443 ssl;
-    listen                  [::]:8448 ssl;
-    server_name             synapse.localhost;
-    ssl_certificate         /root/ssl/cert.pem;
-    ssl_certificate_key     /root/ssl/key.pem;
-
-    # well-known config adding rtc_foci backend
-    # Note well-known is currently not effective due to:
-    # https://spec.matrix.org/v1.12/client-server-api/#well-known-uri the spec
-    # says it must be at https://$server_name/... (implied port 443) Hence, we
-    # currently rely for local development environment on deprecated config.json
-    # setting for livekit_service_url
-    location /.well-known/matrix/client {
-        return 200 '{"m.homeserver": {"base_url": "http://synapse.localhost:8008"}, "org.matrix.msc4143.rtc_foci": [{"type": "livekit", "livekit_service_url": "http://localhost:8080"}]}';
-        default_type application/json;
-        add_header Access-Control-Allow-Origin *;
-    }
-
-    # Reverse proxy for Matrix Synapse Homeserver
-    # This is also required for development environment.
-    # Reason: the lk-jwt-service uses the federation API for the openid token
-    #         verification, which requires TLS
-    location / {
-        proxy_pass "http://homeserver:8008";
-        proxy_http_version 1.1;
-        proxy_set_header Upgrade $http_upgrade;
-        proxy_set_header Connection "upgrade";
-        proxy_set_header Host $host;
-        proxy_set_header X-Forwarded-For $remote_addr;
-        proxy_set_header X-Forwarded-Proto $scheme;
-    }
-
-    error_page   500 502 503 504  /50x.html;
-
-}

config/config.devenv.json

@@ -1,13 +1,10 @@
 {
   "default_server_config": {
     "m.homeserver": {
-      "base_url": "http://synapse.localhost:8008",
-      "server_name": "synapse.localhost"
+      "base_url": "https://synapse.m.localhost",
+      "server_name": "synapse.m.localhost"
     }
   },
-  "livekit": {
-    "livekit_service_url": "http://localhost:8009"
-  },
   "features": {
     "feature_use_device_session_member_events": true
   },

dev-backend-docker-compose.yml

@@ -7,7 +7,7 @@ services:
     hostname: auth-server
     environment:
       - LK_JWT_PORT=8080
-      - LIVEKIT_URL=ws://localhost:7880
+      - LIVEKIT_URL=wss://matrix-rtc.m.localhost/livekit/sfu
       - LIVEKIT_KEY=devkey
       - LIVEKIT_SECRET=secret
       # If the configured homeserver runs on localhost, it'll probably be using
@@ -18,12 +18,13 @@ services:
         condition: on-failure
     ports:
       # HOST_PORT:CONTAINER_PORT
-      - 8009:8080
+      - 8080:8080
     networks:
       - ecbackend
 
   livekit:
     image: livekit/livekit-server:latest
+    hostname: livekit-sfu
     command: --dev --config /etc/livekit.yaml
     restart: unless-stopped
     # The SFU seems to work far more reliably when we let it share the host
@@ -81,17 +82,22 @@ services:
       - ecbackend
 
   nginx:
-    #  openssl req -x509 -nodes -days 3650 -newkey rsa:2048 -keyout tls_localhost_key.pem -out tls_localhost_cert.pem -subj "/C=GB/ST=London/L=London/O=Alros/OU=IT Department/CN=localhost"
-    hostname: synapse.localhost
+    #  see backend/dev_tls_setup for how to generate the tls certs
+    hostname: synapse.m.localhost
     image: nginx:latest
     volumes:
-      - ./backend/tls_localhost_nginx.conf:/etc/nginx/conf.d/default.conf:Z
-      - ./backend/tls_localhost_key.pem:/root/ssl/key.pem:Z
-      - ./backend/tls_localhost_cert.pem:/root/ssl/cert.pem:Z
+      - ./backend/dev_nginx.conf:/etc/nginx/conf.d/default.conf:Z
+      - ./backend/dev_tls_m.localhost.key:/root/ssl/key.pem:Z
+      - ./backend/dev_tls_m.localhost.crt:/root/ssl/cert.pem:Z
     ports:
       # HOST_PORT:CONTAINER_PORT
+      - "80:80"
+      - "443:443"
       - "8008:80"
       - "4443:443"
+      - "8448:8448"
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
     depends_on:
       - synapse
     networks:

docs/self-hosting.md

@@ -93,7 +93,7 @@ the example above, this results in:
 
 Using Nginx, you can achieve this by:
 
-```jsonc
+```nginx configuration file
 server {
     ...
     location ^~ /livekit/jwt/ {
@@ -203,7 +203,7 @@ Because Element Call uses client-side routing, your server must be able to route
 any requests to non-existing paths back to `/index.html`. For example, in Nginx
 you can achieve this with the `try_files` directive:
 
-```jsonc
+```nginx configuration file
 server {
     ...
     location / {

knip.ts

@@ -27,6 +27,10 @@ export default {
     // then Knip will flag it as a false positive
     // https://github.com/webpro-nl/knip/issues/766
     "@vector-im/compound-web",
+    // We need this so that TypeScript is happy with @livekit/track-processors.
+    // This might be a bug in the LiveKit repo but for now we fix it on the
+    // Element Call side.
+    "@types/dom-mediacapture-transform",
     "matrix-widget-api",
   ],
   ignoreExportsUsedInFile: true,

locales/en/app.json

@@ -164,6 +164,9 @@
       "effect_volume_description": "Adjust the volume at which reactions and hand raised effects play.",
       "effect_volume_label": "Sound effect volume"
     },
+    "background_blur_header": "Background",
+    "background_blur_label": "Blur the background of the video",
+    "blur_not_supported_by_browser": "(Background blur is not supported by this device.)",
     "developer_tab_title": "Developer",
     "devices": {
       "camera": "Camera",

package.json

@@ -40,6 +40,8 @@
     "@livekit/components-core": "^0.12.0",
     "@livekit/components-react": "^2.0.0",
     "@livekit/protocol": "^1.33.0",
+    "@livekit/track-processors": "^0.5.5",
+    "@mediapipe/tasks-vision": "^0.10.18",
     "@opentelemetry/api": "^1.4.0",
     "@opentelemetry/core": "^1.25.1",
     "@opentelemetry/exporter-trace-otlp-http": "^0.57.0",
@@ -47,7 +49,7 @@
     "@opentelemetry/sdk-trace-base": "^1.25.1",
     "@opentelemetry/sdk-trace-web": "^1.9.1",
     "@opentelemetry/semantic-conventions": "^1.25.1",
-    "@playwright/test": "^1.51.0",
+    "@playwright/test": "^1.52.0",
     "@radix-ui/react-dialog": "^1.0.4",
     "@radix-ui/react-slider": "^1.1.2",
     "@radix-ui/react-visually-hidden": "^1.0.3",
@@ -60,6 +62,7 @@
     "@testing-library/react": "^16.0.0",
     "@testing-library/user-event": "^14.5.1",
     "@types/content-type": "^1.1.5",
+    "@types/dom-mediacapture-transform": "^0.1.11",
     "@types/grecaptcha": "^3.0.9",
     "@types/jsdom": "^21.1.7",
     "@types/lodash-es": "^4.17.12",
@@ -70,12 +73,11 @@
     "@types/react-dom": "^18.3.0",
     "@types/sdp-transform": "^2.4.5",
     "@types/uuid": "10",
-    "@typescript-eslint/eslint-plugin": "^8.0.0",
-    "@typescript-eslint/parser": "^8.0.0",
+    "@typescript-eslint/eslint-plugin": "^8.31.0",
+    "@typescript-eslint/parser": "^8.31.0",
     "@use-gesture/react": "^10.2.11",
     "@vector-im/compound-design-tokens": "^3.0.0",
     "@vector-im/compound-web": "^7.2.0",
-    "@vitejs/plugin-basic-ssl": "^1.0.1",
     "@vitejs/plugin-react": "^4.0.1",
     "@vitest/coverage-v8": "^3.0.0",
     "babel-plugin-transform-vite-meta-env": "^1.0.3",
@@ -118,7 +120,7 @@
     "react-use-measure": "^2.1.1",
     "rxjs": "^7.8.1",
     "sass": "^1.42.1",
-    "typescript": "^5.1.6",
+    "typescript": "^5.8.3",
     "typescript-eslint-language-service": "^5.0.5",
     "unique-names-generator": "^4.6.0",
     "vaul": "^1.0.0",
@@ -131,6 +133,7 @@
   },
   "resolutions": {
     "@livekit/components-core/rxjs": "^7.8.1",
+    "@livekit/track-processors/@mediapipe/tasks-vision": "^0.10.18",
     "matrix-widget-api": "1.11.0"
   },
   "packageManager": "yarn@4.7.0"

playwright-backend-docker-compose.override.yml

@@ -0,0 +1,4 @@
+services:
+  synapse:
+    volumes:
+      - ./backend/playwright_homeserver.yaml:/data/cfg/homeserver.yaml:Z

playwright-backend-docker-compose.yml

@@ -1,97 +1,2 @@
-networks:
-  ecbackend:
-
-services:
-  auth-service:
-    image: ghcr.io/element-hq/lk-jwt-service:latest-ci
-    hostname: auth-server
-    environment:
-      - LK_JWT_PORT=8080
-      - LIVEKIT_URL=ws://localhost:7880
-      - LIVEKIT_KEY=devkey
-      - LIVEKIT_SECRET=secret
-      # If the configured homeserver runs on localhost, it'll probably be using
-      # a self-signed certificate
-      - LIVEKIT_INSECURE_SKIP_VERIFY_TLS=YES_I_KNOW_WHAT_I_AM_DOING
-    deploy:
-      restart_policy:
-        condition: on-failure
-    ports:
-      # HOST_PORT:CONTAINER_PORT
-      - 8009:8080
-    networks:
-      - ecbackend
-
-  livekit:
-    image: livekit/livekit-server:latest
-    command: --dev --config /etc/livekit.yaml
-    restart: unless-stopped
-    # The SFU seems to work far more reliably when we let it share the host
-    # network rather than opening specific ports (but why?? we're not missing
-    # any…)
-    ports:
-      # HOST_PORT:CONTAINER_PORT
-      - 7880:7880/tcp
-      - 7881:7881/tcp
-      - 7882:7882/tcp
-      - 50100-50200:50100-50200/udp
-    volumes:
-      - ./backend/dev_livekit.yaml:/etc/livekit.yaml:Z
-    networks:
-      - ecbackend
-
-  redis:
-    image: redis:6-alpine
-    command: redis-server /etc/redis.conf
-    ports:
-      # HOST_PORT:CONTAINER_PORT
-      - 6379:6379
-    volumes:
-      - ./backend/redis.conf:/etc/redis.conf:Z
-    networks:
-      - ecbackend
-
-  element-web:
-    image: ghcr.io/element-hq/element-web:develop
-    volumes:
-      - ./backend/ew.test.config.json:/app/config.json
-    environment:
-      ELEMENT_WEB_PORT: 81
-    ports:
-      - "8081:81"
-    networks:
-      - ecbackend
-
-  synapse:
-    hostname: homeserver
-    image: docker.io/matrixdotorg/synapse:latest
-    environment:
-      - SYNAPSE_CONFIG_PATH=/data/cfg/homeserver.yaml
-      # Needed for rootless podman-compose such that the uid/gid mapping does
-      # fit local user uid. If the container runs as root (uid 0) it is fine as
-      # it actually maps to your non-root user on the host (e.g. 1000).
-      # Otherwise uid mapping will not match your non-root user.
-      - UID=0
-      - GID=0
-    volumes:
-      - ./backend/synapse_tmp:/data:Z
-      - ./backend/playwright_homeserver.yaml:/data/cfg/homeserver.yaml:Z
-    networks:
-      - ecbackend
-
-  nginx:
-    #  openssl req -x509 -nodes -days 3650 -newkey rsa:2048 -keyout tls_localhost_key.pem -out tls_localhost_cert.pem -subj "/C=GB/ST=London/L=London/O=Alros/OU=IT Department/CN=localhost"
-    hostname: synapse.localhost
-    image: nginx:latest
-    volumes:
-      - ./backend/tls_localhost_nginx.conf:/etc/nginx/conf.d/default.conf:Z
-      - ./backend/tls_localhost_key.pem:/root/ssl/key.pem:Z
-      - ./backend/tls_localhost_cert.pem:/root/ssl/cert.pem:Z
-    ports:
-      # HOST_PORT:CONTAINER_PORT
-      - "8008:80"
-      - "4443:443"
-    depends_on:
-      - synapse
-    networks:
-      - ecbackend
+include:
+  - dev-backend-docker-compose.yml

playwright/fixtures/widget-user.ts

@@ -154,8 +154,13 @@ export const widgetTest = test.extend<MyFixtures>({
       ewPage1.getByRole("heading", { name: "Invite to Welcome Room" }),
     ).toBeVisible();
 
-    await ewPage1.getByRole("textbox").fill(whistlerMxId);
-    await ewPage1.getByRole("textbox").click();
+    // To get the invite textbox we need to specifically select within the
+    // dialog, since there is another textbox in the background (the message
+    // composer). In theory the composer shouldn't be visible to Playwright at
+    // all because the invite dialog has trapped focus, but the focus trap
+    // doesn't quite work right on Firefox.
+    await ewPage1.getByRole("dialog").getByRole("textbox").fill(whistlerMxId);
+    await ewPage1.getByRole("dialog").getByRole("textbox").click();
     await ewPage1.getByRole("button", { name: "Invite" }).click();
 
     // Accept the invite

src/App.tsx

@@ -22,6 +22,7 @@ import { Initializer } from "./initializer";
 import { MediaDevicesProvider } from "./livekit/MediaDevicesContext";
 import { widget } from "./widget";
 import { useTheme } from "./useTheme";
+import { ProcessorProvider } from "./livekit/TrackProcessorContext";
 
 const SentryRoute = Sentry.withSentryReactRouterV7Routing(Route);
 
@@ -72,22 +73,24 @@ export const App: FC = () => {
               <Suspense fallback={null}>
                 <ClientProvider>
                   <MediaDevicesProvider>
-                    <Sentry.ErrorBoundary
-                      fallback={(error) => (
-                        <ErrorPage error={error} widget={widget} />
-                      )}
-                    >
-                      <DisconnectedBanner />
-                      <Routes>
-                        <SentryRoute path="/" element={<HomePage />} />
-                        <SentryRoute path="/login" element={<LoginPage />} />
-                        <SentryRoute
-                          path="/register"
-                          element={<RegisterPage />}
-                        />
-                        <SentryRoute path="*" element={<RoomPage />} />
-                      </Routes>
-                    </Sentry.ErrorBoundary>
+                    <ProcessorProvider>
+                      <Sentry.ErrorBoundary
+                        fallback={(error) => (
+                          <ErrorPage error={error} widget={widget} />
+                        )}
+                      >
+                        <DisconnectedBanner />
+                        <Routes>
+                          <SentryRoute path="/" element={<HomePage />} />
+                          <SentryRoute path="/login" element={<LoginPage />} />
+                          <SentryRoute
+                            path="/register"
+                            element={<RegisterPage />}
+                          />
+                          <SentryRoute path="*" element={<RoomPage />} />
+                        </Routes>
+                      </Sentry.ErrorBoundary>
+                    </ProcessorProvider>
                   </MediaDevicesProvider>
                 </ClientProvider>
               </Suspense>

src/initializer.tsx

@@ -136,6 +136,11 @@ export class Initializer {
       lookup: () => getUrlParams().lang ?? undefined,
     });
 
+    // Synchronise the HTML lang attribute with the i18next language
+    i18n.on("languageChanged", (lng) => {
+      document.documentElement.lang = lng;
+    });
+
     await i18n
       .use(Backend)
       .use(languageDetector)

src/livekit/BlurBackgroundTransformer.ts

@@ -0,0 +1,80 @@
+/*
+Copyright 2024-2025 New Vector Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only
+Please see LICENSE in the repository root for full details.
+*/
+
+import {
+  BackgroundTransformer,
+  VideoTransformer,
+  type VideoTransformerInitOptions,
+} from "@livekit/track-processors";
+import { ImageSegmenter } from "@mediapipe/tasks-vision";
+
+import modelAssetPath from "../mediapipe/imageSegmenter/selfie_segmenter.tflite?url";
+
+interface WasmFileset {
+  /** The path to the Wasm loader script. */
+  wasmLoaderPath: string;
+  /** The path to the Wasm binary. */
+  wasmBinaryPath: string;
+}
+
+// The MediaPipe package, by default, ships some alternative versions of the
+// WASM files which avoid SIMD for compatibility with older browsers. But SIMD
+// in WASM is actually fine by our support policy, so we include just the SIMD
+// versions.
+// It's really not ideal that we have to reference these internal files from
+// MediaPipe and depend on node_modules having this specific structure. It's
+// easy to see this breaking if our dependencies changed and MediaPipe were
+// no longer hoisted, or if we switched to another dependency loader such as
+// Yarn PnP.
+// https://github.com/google-ai-edge/mediapipe/issues/5961
+const wasmFileset: WasmFileset = {
+  wasmLoaderPath: new URL(
+    "../../node_modules/@mediapipe/tasks-vision/wasm/vision_wasm_internal.js",
+    import.meta.url,
+  ).href,
+  wasmBinaryPath: new URL(
+    "../../node_modules/@mediapipe/tasks-vision/wasm/vision_wasm_internal.wasm",
+    import.meta.url,
+  ).href,
+};
+
+/**
+ * Track processor that applies effects such as blurring to a user's background.
+ *
+ * This is just like LiveKit's prebuilt BackgroundTransformer except that it
+ * loads the segmentation models from our own bundle rather than as an external
+ * resource fetched from the public internet.
+ */
+export class BlurBackgroundTransformer extends BackgroundTransformer {
+  public async init({
+    outputCanvas,
+    inputElement: inputVideo,
+  }: VideoTransformerInitOptions): Promise<void> {
+    // Call super.super.init() since we're totally replacing the init method of
+    // BackgroundTransformer here, rather than extending it
+    await VideoTransformer.prototype.init.call(this, {
+      outputCanvas,
+      inputElement: inputVideo,
+    });
+
+    this.imageSegmenter = await ImageSegmenter.createFromOptions(wasmFileset, {
+      baseOptions: {
+        modelAssetPath,
+        delegate: "GPU",
+        ...this.options.segmenterOptions,
+      },
+      canvas: this.canvas,
+      runningMode: "VIDEO",
+      outputCategoryMask: true,
+      outputConfidenceMasks: false,
+    });
+
+    if (this.options.blurRadius) {
+      this.gl?.setBlurRadius(this.options.blurRadius);
+    }
+  }
+}

src/livekit/TrackProcessorContext.tsx

@@ -0,0 +1,84 @@
+/*
+Copyright 2024 New Vector Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only
+Please see LICENSE in the repository root for full details.
+*/
+
+import {
+  ProcessorWrapper,
+  supportsBackgroundProcessors,
+  type BackgroundOptions,
+} from "@livekit/track-processors";
+import { createContext, type FC, useContext, useEffect, useMemo } from "react";
+import { type LocalVideoTrack } from "livekit-client";
+
+import {
+  backgroundBlur as backgroundBlurSettings,
+  useSetting,
+} from "../settings/settings";
+import { BlurBackgroundTransformer } from "./BlurBackgroundTransformer";
+
+type ProcessorState = {
+  supported: boolean | undefined;
+  processor: undefined | ProcessorWrapper<BackgroundOptions>;
+};
+
+const ProcessorContext = createContext<ProcessorState | undefined>(undefined);
+
+export function useTrackProcessor(): ProcessorState {
+  const state = useContext(ProcessorContext);
+  if (state === undefined)
+    throw new Error(
+      "useTrackProcessor must be used within a ProcessorProvider",
+    );
+  return state;
+}
+
+export const useTrackProcessorSync = (
+  videoTrack: LocalVideoTrack | null,
+): void => {
+  const { processor } = useTrackProcessor();
+  useEffect(() => {
+    if (!videoTrack) return;
+    if (processor && !videoTrack.getProcessor()) {
+      void videoTrack.setProcessor(processor);
+    }
+    if (!processor && videoTrack.getProcessor()) {
+      void videoTrack.stopProcessor();
+    }
+  }, [processor, videoTrack]);
+};
+
+interface Props {
+  children: JSX.Element;
+}
+
+export const ProcessorProvider: FC<Props> = ({ children }) => {
+  // The setting the user wants to have
+  const [blurActivated] = useSetting(backgroundBlurSettings);
+  const supported = useMemo(() => supportsBackgroundProcessors(), []);
+  const blur = useMemo(
+    () =>
+      new ProcessorWrapper(
+        new BlurBackgroundTransformer({ blurRadius: 15 }),
+        "background-blur",
+      ),
+    [],
+  );
+
+  // This is the actual state exposed through the context
+  const processorState = useMemo(
+    () => ({
+      supported,
+      processor: supported && blurActivated ? blur : undefined,
+    }),
+    [supported, blurActivated, blur],
+  );
+
+  return (
+    <ProcessorContext.Provider value={processorState}>
+      {children}
+    </ProcessorContext.Provider>
+  );
+};

src/livekit/useLiveKit.ts

@@ -9,6 +9,7 @@ import {
   ConnectionState,
   type E2EEManagerOptions,
   ExternalE2EEKeyProvider,
+  LocalVideoTrack,
   Room,
   type RoomOptions,
   Track,
@@ -17,6 +18,8 @@ import { useEffect, useMemo, useRef } from "react";
 import E2EEWorker from "livekit-client/e2ee-worker?worker";
 import { logger } from "matrix-js-sdk/lib/logger";
 import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc";
+import { useObservable, useObservableEagerState } from "observable-hooks";
+import { map } from "rxjs";
 
 import { defaultLiveKitOptions } from "./options";
 import { type SFUConfig } from "./openIDSFU";
@@ -33,6 +36,12 @@ import {
 import { MatrixKeyProvider } from "../e2ee/matrixKeyProvider";
 import { E2eeType } from "../e2ee/e2eeType";
 import { type EncryptionSystem } from "../e2ee/sharedKeyManagement";
+import {
+  useTrackProcessor,
+  useTrackProcessorSync,
+} from "./TrackProcessorContext";
+import { useInitial } from "../useInitial";
+import { observeTrackReference$ } from "../state/MediaViewModel";
 
 interface UseLivekitResult {
   livekitRoom?: Room;
@@ -82,12 +91,15 @@ export function useLiveKit(
   const devices = useMediaDevices();
   const initialDevices = useRef<MediaDevices>(devices);
 
+  const { processor } = useTrackProcessor();
+  const initialProcessor = useInitial(() => processor);
   const roomOptions = useMemo(
     (): RoomOptions => ({
       ...defaultLiveKitOptions,
       videoCaptureDefaults: {
         ...defaultLiveKitOptions.videoCaptureDefaults,
         deviceId: initialDevices.current.videoInput.selectedId,
+        processor: initialProcessor,
       },
       audioCaptureDefaults: {
         ...defaultLiveKitOptions.audioCaptureDefaults,
@@ -98,7 +110,7 @@ export function useLiveKit(
       },
       e2ee: e2eeOptions,
     }),
-    [e2eeOptions],
+    [e2eeOptions, initialProcessor],
   );
 
   // Store if audio/video are currently updating. If to prohibit unnecessary calls
@@ -123,6 +135,25 @@ export function useLiveKit(
     return r;
   }, [roomOptions, e2eeSystem]);
 
+  // Sync the requested track processors with LiveKit
+  useTrackProcessorSync(
+    useObservableEagerState(
+      useObservable(
+        (room$) =>
+          observeTrackReference$(
+            room$.pipe(map(([room]) => room.localParticipant)),
+            Track.Source.Camera,
+          ).pipe(
+            map((trackRef) => {
+              const track = trackRef?.publication?.track;
+              return track instanceof LocalVideoTrack ? track : null;
+            }),
+          ),
+        [room],
+      ),
+    ),
+  );
+
   const connectionState = useECConnectionState(
     {
       deviceId: initialDevices.current.audioInput.selectedId,
@@ -198,6 +229,7 @@ export function useLiveKit(
                 audioMuteUpdating.current = true;
                 trackPublication = await participant.setMicrophoneEnabled(
                   buttonEnabled.current.audio,
+                  room.options.audioCaptureDefaults,
                 );
                 audioMuteUpdating.current = false;
                 break;
@@ -205,6 +237,7 @@ export function useLiveKit(
                 videoMuteUpdating.current = true;
                 trackPublication = await participant.setCameraEnabled(
                   buttonEnabled.current.video,
+                  room.options.videoCaptureDefaults,
                 );
                 videoMuteUpdating.current = false;
                 break;

src/mediapipe/imageSegmenter/README.md

@@ -0,0 +1,5 @@
+# Google AI Edge MediaPipe Selfie Segmentation
+
+- See: https://ai.google.dev/edge/mediapipe/solutions/vision/image_segmenter
+- Latest: https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter/float16/latest/selfie_segmenter.tflite
+- License: Apache 2.0 as per https://storage.googleapis.com/mediapipe-assets/Model%20Card%20MediaPipe%20Selfie%20Segmentation.pdf

src/room/GroupCallView.test.tsx

@@ -38,6 +38,7 @@ import { GroupCallView } from "./GroupCallView";
 import { type WidgetHelpers } from "../widget";
 import { LazyEventEmitter } from "../LazyEventEmitter";
 import { MatrixRTCFocusMissingError } from "../utils/errors";
+import { ProcessorProvider } from "../livekit/TrackProcessorContext";
 
 vi.mock("../soundUtils");
 vi.mock("../useAudioContext");
@@ -46,6 +47,13 @@ vi.mock("react-use-measure", () => ({
   default: (): [() => void, object] => [(): void => {}, {}],
 }));
 
+vi.hoisted(
+  () =>
+    (global.ImageData = class MockImageData {
+      public data: number[] = [];
+    } as unknown as typeof ImageData),
+);
+
 const enterRTCSession = vi.hoisted(() => vi.fn(async () => Promise.resolve()));
 const leaveRTCSession = vi.hoisted(() =>
   vi.fn(
@@ -137,18 +145,20 @@ function createGroupCallView(
   const { getByText } = render(
     <BrowserRouter>
       <TooltipProvider>
-        <GroupCallView
-          client={client}
-          isPasswordlessUser={false}
-          confineToRoom={false}
-          preload={false}
-          skipLobby={false}
-          hideHeader={true}
-          rtcSession={rtcSession as unknown as MatrixRTCSession}
-          isJoined={joined}
-          muteStates={muteState}
-          widget={widget}
-        />
+        <ProcessorProvider>
+          <GroupCallView
+            client={client}
+            isPasswordlessUser={false}
+            confineToRoom={false}
+            preload={false}
+            skipLobby={false}
+            hideHeader={true}
+            rtcSession={rtcSession as unknown as MatrixRTCSession}
+            isJoined={joined}
+            muteStates={muteState}
+            widget={widget}
+          />
+        </ProcessorProvider>
       </TooltipProvider>
     </BrowserRouter>,
   );

src/room/LobbyView.tsx

@@ -12,7 +12,11 @@ import { Button } from "@vector-im/compound-web";
 import classNames from "classnames";
 import { logger } from "matrix-js-sdk/lib/logger";
 import { usePreviewTracks } from "@livekit/components-react";
-import { type LocalVideoTrack, Track } from "livekit-client";
+import {
+  type CreateLocalTracksOptions,
+  type LocalVideoTrack,
+  Track,
+} from "livekit-client";
 import { useObservable } from "observable-hooks";
 import { map } from "rxjs";
 import { useNavigate } from "react-router-dom";
@@ -36,7 +40,11 @@ import { E2eeType } from "../e2ee/e2eeType";
 import { Link } from "../button/Link";
 import { useMediaDevices } from "../livekit/MediaDevicesContext";
 import { useInitial } from "../useInitial";
-import { useSwitchCamera } from "./useSwitchCamera";
+import { useSwitchCamera as useShowSwitchCamera } from "./useSwitchCamera";
+import {
+  useTrackProcessor,
+  useTrackProcessorSync,
+} from "../livekit/TrackProcessorContext";
 import { usePageTitle } from "../usePageTitle";
 
 interface Props {
@@ -112,7 +120,10 @@ export const LobbyView: FC<Props> = ({
       muteStates.audio.enabled && { deviceId: devices.audioInput.selectedId },
   );
 
-  const localTrackOptions = useMemo(
+  const { processor } = useTrackProcessor();
+
+  const initialProcessor = useInitial(() => processor);
+  const localTrackOptions = useMemo<CreateLocalTracksOptions>(
     () => ({
       // The only reason we request audio here is to get the audio permission
       // request over with at the same time. But changing the audio settings
@@ -123,12 +134,14 @@ export const LobbyView: FC<Props> = ({
       audio: Object.assign({}, initialAudioOptions),
       video: muteStates.video.enabled && {
         deviceId: devices.videoInput.selectedId,
+        processor: initialProcessor,
       },
     }),
     [
       initialAudioOptions,
-      devices.videoInput.selectedId,
       muteStates.video.enabled,
+      devices.videoInput.selectedId,
+      initialProcessor,
     ],
   );
 
@@ -149,8 +162,8 @@ export const LobbyView: FC<Props> = ({
         null) as LocalVideoTrack | null,
     [tracks],
   );
-
-  const switchCamera = useSwitchCamera(
+  useTrackProcessorSync(videoTrack);
+  const showSwitchCamera = useShowSwitchCamera(
     useObservable(
       (inputs$) => inputs$.pipe(map(([video]) => video)),
       [videoTrack],
@@ -212,7 +225,9 @@ export const LobbyView: FC<Props> = ({
               onClick={onVideoPress}
               disabled={muteStates.video.setEnabled === null}
             />
-            {switchCamera && <SwitchCameraButton onClick={switchCamera} />}
+            {showSwitchCamera && (
+              <SwitchCameraButton onClick={showSwitchCamera} />
+            )}
             <SettingsButton onClick={openSettings} />
             {!confineToRoom && <EndCallButton onClick={onLeaveClick} />}
           </div>

src/settings/SettingsModal.tsx

@@ -5,10 +5,10 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
 Please see LICENSE in the repository root for full details.
 */
 
-import { type FC, useState } from "react";
+import { type FC, type ReactNode, useState } from "react";
 import { useTranslation } from "react-i18next";
 import { type MatrixClient } from "matrix-js-sdk";
-import { Root as Form } from "@vector-im/compound-web";
+import { Root as Form, Separator } from "@vector-im/compound-web";
 import { type Room as LivekitRoom } from "livekit-client";
 
 import { Modal } from "../Modal";
@@ -24,12 +24,15 @@ import { widget } from "../widget";
 import {
   useSetting,
   soundEffectVolumeSetting,
+  backgroundBlur as backgroundBlurSetting,
   developerMode,
 } from "./settings";
 import { PreferencesSettingsTab } from "./PreferencesSettingsTab";
 import { Slider } from "../Slider";
 import { DeviceSelection } from "./DeviceSelection";
+import { useTrackProcessor } from "../livekit/TrackProcessorContext";
 import { DeveloperSettingsTab } from "./DeveloperSettingsTab";
+import { FieldRow, InputField } from "../input/Input";
 import { useSubmitRageshake } from "./submit-rageshake";
 
 type SettingsTab =
@@ -64,6 +67,33 @@ export const SettingsModal: FC<Props> = ({
 }) => {
   const { t } = useTranslation();
 
+  // Generate a `Checkbox` input to turn blur on or off.
+  const BlurCheckbox: React.FC = (): ReactNode => {
+    const { supported } = useTrackProcessor();
+
+    const [blurActive, setBlurActive] = useSetting(backgroundBlurSetting);
+
+    return (
+      <>
+        <h4>{t("settings.background_blur_header")}</h4>
+
+        <FieldRow>
+          <InputField
+            id="activateBackgroundBlur"
+            label={t("settings.background_blur_label")}
+            description={
+              supported ? "" : t("settings.blur_not_supported_by_browser")
+            }
+            type="checkbox"
+            checked={!!blurActive}
+            onChange={(b): void => setBlurActive(b.target.checked)}
+            disabled={!supported}
+          />
+        </FieldRow>
+      </>
+    );
+  };
+
   const devices = useMediaDevices();
   useMediaDeviceNames(devices, open);
   const [soundVolume, setSoundVolume] = useSetting(soundEffectVolumeSetting);
@@ -113,13 +143,17 @@ export const SettingsModal: FC<Props> = ({
     key: "video",
     name: t("common.video"),
     content: (
-      <Form>
-        <DeviceSelection
-          devices={devices.videoInput}
-          title={t("settings.devices.camera")}
-          numberedLabel={(n) => t("settings.devices.camera_numbered", { n })}
-        />
-      </Form>
+      <>
+        <Form>
+          <DeviceSelection
+            devices={devices.videoInput}
+            title={t("settings.devices.camera")}
+            numberedLabel={(n) => t("settings.devices.camera_numbered", { n })}
+          />
+        </Form>
+        <Separator />
+        <BlurCheckbox />
+      </>
     ),
   };
 

src/settings/settings.ts

@@ -96,6 +96,8 @@ export const videoInput = new Setting<string | undefined>(
   undefined,
 );
 
+export const backgroundBlur = new Setting<boolean>("background-blur", false);
+
 export const showHandRaisedTimer = new Setting<boolean>(
   "hand-raised-show-timer",
   false,

vite.config.js

@@ -11,8 +11,8 @@ import { createHtmlPlugin } from "vite-plugin-html";
 import { codecovVitePlugin } from "@codecov/vite-plugin";
 import { sentryVitePlugin } from "@sentry/vite-plugin";
 import react from "@vitejs/plugin-react";
-import basicSsl from "@vitejs/plugin-basic-ssl";
 import { realpathSync } from "fs";
+import * as fs from "node:fs";
 
 // https://vitejs.dev/config/
 export default defineConfig(({ mode, packageType }) => {
@@ -24,7 +24,6 @@ export default defineConfig(({ mode, packageType }) => {
   process.env.VITE_PACKAGE = packageType ?? "full";
   const plugins = [
     react(),
-    basicSsl(),
     svgrPlugin({
       svgrOptions: {
         // This enables ref forwarding on SVGR components, which is needed, for
@@ -84,6 +83,10 @@ export default defineConfig(({ mode, packageType }) => {
     server: {
       port: 3000,
       fs: { allow },
+      https: {
+        key: fs.readFileSync("./backend/dev_tls_m.localhost.key"),
+        cert: fs.readFileSync("./backend/dev_tls_m.localhost.crt"),
+      },
     },
     build: {
       sourcemap: true,

yarn.lock

@@ -2536,6 +2536,17 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@livekit/track-processors@npm:^0.5.5":
+  version: 0.5.5
+  resolution: "@livekit/track-processors@npm:0.5.5"
+  dependencies:
+    "@mediapipe/tasks-vision": "npm:0.10.14"
+  peerDependencies:
+    livekit-client: ^1.12.0 || ^2.1.0
+  checksum: 10c0/b86737c4ce6b93c714dfc814c997d07f1d40b0869b5e269b28e00034a3c06da811d9117c4914d8a2706ac760be31f2e9d0705108c07b29f4bbb4626dcd41ed64
+  languageName: node
+  linkType: hard
+
 "@matrix-org/matrix-sdk-crypto-wasm@npm:^14.0.1":
   version: 14.0.1
   resolution: "@matrix-org/matrix-sdk-crypto-wasm@npm:14.0.1"
@@ -2550,6 +2561,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@mediapipe/tasks-vision@npm:^0.10.18":
+  version: 0.10.21
+  resolution: "@mediapipe/tasks-vision@npm:0.10.21"
+  checksum: 10c0/11b2bdf98b8cb6e044f2a954e7c8393169e62c86ff49b3d0b61c3b327d18e1ccd47a187999b023bad48380c9da41bfa66eb165301c80da07746390482cb18a19
+  languageName: node
+  linkType: hard
+
 "@nodelib/fs.scandir@npm:2.1.5":
   version: 2.1.5
   resolution: "@nodelib/fs.scandir@npm:2.1.5"
@@ -3045,14 +3063,14 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@playwright/test@npm:^1.51.0":
-  version: 1.51.0
-  resolution: "@playwright/test@npm:1.51.0"
+"@playwright/test@npm:^1.52.0":
+  version: 1.52.0
+  resolution: "@playwright/test@npm:1.52.0"
   dependencies:
-    playwright: "npm:1.51.0"
+    playwright: "npm:1.52.0"
   bin:
     playwright: cli.js
-  checksum: 10c0/ae83dd2c3a32133de58f44a9dbcd73a8059155ebd8acc736ba8bd0a7ca99b194afe2e8f5a500861d18b1c8f06b4e4ea8de4a2402297c59053d4becc404b47e0a
+  checksum: 10c0/1c428b421593eb4f79b7c99783a389c3ab3526c9051ec772749f4fca61414dfa9f2344eba846faac5f238084aa96c836364a91d81d3034ac54924f239a93e247
   languageName: node
   linkType: hard
 
@@ -4603,6 +4621,22 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@types/dom-mediacapture-transform@npm:^0.1.11":
+  version: 0.1.11
+  resolution: "@types/dom-mediacapture-transform@npm:0.1.11"
+  dependencies:
+    "@types/dom-webcodecs": "npm:*"
+  checksum: 10c0/19c76d54cf31aa2a925011fc5f973dff9a10bdecfdf2285e5e568e61850a0fa2b8c9f1807a1462cbefd57ec26d32eeaa9c359117aca9d9fe7f0d6f2fff33f51e
+  languageName: node
+  linkType: hard
+
+"@types/dom-webcodecs@npm:*":
+  version: 0.1.15
+  resolution: "@types/dom-webcodecs@npm:0.1.15"
+  checksum: 10c0/1407f0352156c99c9b4378fb4c0c799b061520d031903a7f359ad09a6f706cc1fd56bafb272bb1a3decffcb32e54a51d2f07442eb72622464a950cff7f9e8862
+  languageName: node
+  linkType: hard
+
 "@types/estree@npm:1.0.6, @types/estree@npm:^1.0.0":
   version: 1.0.6
   resolution: "@types/estree@npm:1.0.6"
@@ -4797,15 +4831,15 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@typescript-eslint/eslint-plugin@npm:^8.0.0":
-  version: 8.25.0
-  resolution: "@typescript-eslint/eslint-plugin@npm:8.25.0"
+"@typescript-eslint/eslint-plugin@npm:^8.31.0":
+  version: 8.31.0
+  resolution: "@typescript-eslint/eslint-plugin@npm:8.31.0"
   dependencies:
     "@eslint-community/regexpp": "npm:^4.10.0"
-    "@typescript-eslint/scope-manager": "npm:8.25.0"
-    "@typescript-eslint/type-utils": "npm:8.25.0"
-    "@typescript-eslint/utils": "npm:8.25.0"
-    "@typescript-eslint/visitor-keys": "npm:8.25.0"
+    "@typescript-eslint/scope-manager": "npm:8.31.0"
+    "@typescript-eslint/type-utils": "npm:8.31.0"
+    "@typescript-eslint/utils": "npm:8.31.0"
+    "@typescript-eslint/visitor-keys": "npm:8.31.0"
     graphemer: "npm:^1.4.0"
     ignore: "npm:^5.3.1"
     natural-compare: "npm:^1.4.0"
@@ -4813,8 +4847,8 @@ __metadata:
   peerDependencies:
     "@typescript-eslint/parser": ^8.0.0 || ^8.0.0-alpha.0
     eslint: ^8.57.0 || ^9.0.0
-    typescript: ">=4.8.4 <5.8.0"
-  checksum: 10c0/11d63850f5f03b29cd31166f8da111788dc74e46877c2e16a5c488d6c4aa4b6c68c0857b9a396ad920aa7f0f3e7166f4faecbb194c19cd2bb9d3f687c5d2b292
+    typescript: ">=4.8.4 <5.9.0"
+  checksum: 10c0/7d78e0cdcc967742752d49d2d38986ee38d0b7ca64af247e5fe0816cea9ae5f1bfa5c126154acc0846af515c4fb1c52c96926ee25c73b4c3f7e6fd73cb6d2b0e
   languageName: node
   linkType: hard
 
@@ -4829,19 +4863,19 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@typescript-eslint/parser@npm:^8.0.0":
-  version: 8.25.0
-  resolution: "@typescript-eslint/parser@npm:8.25.0"
+"@typescript-eslint/parser@npm:^8.31.0":
+  version: 8.31.0
+  resolution: "@typescript-eslint/parser@npm:8.31.0"
   dependencies:
-    "@typescript-eslint/scope-manager": "npm:8.25.0"
-    "@typescript-eslint/types": "npm:8.25.0"
-    "@typescript-eslint/typescript-estree": "npm:8.25.0"
-    "@typescript-eslint/visitor-keys": "npm:8.25.0"
+    "@typescript-eslint/scope-manager": "npm:8.31.0"
+    "@typescript-eslint/types": "npm:8.31.0"
+    "@typescript-eslint/typescript-estree": "npm:8.31.0"
+    "@typescript-eslint/visitor-keys": "npm:8.31.0"
     debug: "npm:^4.3.4"
   peerDependencies:
     eslint: ^8.57.0 || ^9.0.0
-    typescript: ">=4.8.4 <5.8.0"
-  checksum: 10c0/9a54539ba297791f23093ff42a885cc57d36b26205d7a390e114d1f01cc584ce91ac6ead01819daa46b48f873cac6c829fcf399a436610bdbfa98e5cd78148a2
+    typescript: ">=4.8.4 <5.9.0"
+  checksum: 10c0/9bd903b3ea4e24bfeb444d7a5c2ed82e591ef5cffc0874c609de854c05d34935cd85543e66678ecdb8e0e3eae2cda2df5c1ba66eb72010632cb9f8779031d56d
   languageName: node
   linkType: hard
 
@@ -4865,18 +4899,28 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@typescript-eslint/type-utils@npm:8.25.0":
-  version: 8.25.0
-  resolution: "@typescript-eslint/type-utils@npm:8.25.0"
+"@typescript-eslint/scope-manager@npm:8.31.0":
+  version: 8.31.0
+  resolution: "@typescript-eslint/scope-manager@npm:8.31.0"
   dependencies:
-    "@typescript-eslint/typescript-estree": "npm:8.25.0"
-    "@typescript-eslint/utils": "npm:8.25.0"
+    "@typescript-eslint/types": "npm:8.31.0"
+    "@typescript-eslint/visitor-keys": "npm:8.31.0"
+  checksum: 10c0/eae758a24cc578fa351b8bf0c30c50de384292c0b05a58762f9b632d65a009bd5d902d806eccb6b678cc0b09686289fb4f1fd67da7f12d59ad43ff033b35cc4f
+  languageName: node
+  linkType: hard
+
+"@typescript-eslint/type-utils@npm:8.31.0":
+  version: 8.31.0
+  resolution: "@typescript-eslint/type-utils@npm:8.31.0"
+  dependencies:
+    "@typescript-eslint/typescript-estree": "npm:8.31.0"
+    "@typescript-eslint/utils": "npm:8.31.0"
     debug: "npm:^4.3.4"
     ts-api-utils: "npm:^2.0.1"
   peerDependencies:
     eslint: ^8.57.0 || ^9.0.0
-    typescript: ">=4.8.4 <5.8.0"
-  checksum: 10c0/b7477a2d239cfd337f7d28641666763cf680a43a8d377a09dc42415f715670d35fbb4e772e103dfe8cd620c377e66bce740106bb3983ee65a739c28fab7325d1
+    typescript: ">=4.8.4 <5.9.0"
+  checksum: 10c0/f6938413a583430468b259f6823bb2ab1b5cd77cd6d4e21e1803df70e329046b9579aed5bdc9bdcf4046c8091615a911ac3990859db78d00210bb867915ba37f
   languageName: node
   linkType: hard
 
@@ -4894,6 +4938,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@typescript-eslint/types@npm:8.31.0":
+  version: 8.31.0
+  resolution: "@typescript-eslint/types@npm:8.31.0"
+  checksum: 10c0/04130a30aac477d36d6a155399b27773457aeb9b485ef8fb56fee05725b6e36768c9fac7e4d1f073fd16988de0eb7dffc743c3f834ae907cf918cabb075e5cd8
+  languageName: node
+  linkType: hard
+
 "@typescript-eslint/typescript-estree@npm:5.62.0":
   version: 5.62.0
   resolution: "@typescript-eslint/typescript-estree@npm:5.62.0"
@@ -4930,6 +4981,24 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@typescript-eslint/typescript-estree@npm:8.31.0":
+  version: 8.31.0
+  resolution: "@typescript-eslint/typescript-estree@npm:8.31.0"
+  dependencies:
+    "@typescript-eslint/types": "npm:8.31.0"
+    "@typescript-eslint/visitor-keys": "npm:8.31.0"
+    debug: "npm:^4.3.4"
+    fast-glob: "npm:^3.3.2"
+    is-glob: "npm:^4.0.3"
+    minimatch: "npm:^9.0.4"
+    semver: "npm:^7.6.0"
+    ts-api-utils: "npm:^2.0.1"
+  peerDependencies:
+    typescript: ">=4.8.4 <5.9.0"
+  checksum: 10c0/0ec074b2b9c49f80fafea716aa0cc4b05085e65730a3ef7c7d2d39db1657a40b38abe83f22bbe15ac4f6fdf576692f47d2d057347242e6cef5be81d070f55064
+  languageName: node
+  linkType: hard
+
 "@typescript-eslint/utils@npm:5.62.0":
   version: 5.62.0
   resolution: "@typescript-eslint/utils@npm:5.62.0"
@@ -4948,7 +5017,22 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@typescript-eslint/utils@npm:8.25.0, @typescript-eslint/utils@npm:^8.13.0":
+"@typescript-eslint/utils@npm:8.31.0":
+  version: 8.31.0
+  resolution: "@typescript-eslint/utils@npm:8.31.0"
+  dependencies:
+    "@eslint-community/eslint-utils": "npm:^4.4.0"
+    "@typescript-eslint/scope-manager": "npm:8.31.0"
+    "@typescript-eslint/types": "npm:8.31.0"
+    "@typescript-eslint/typescript-estree": "npm:8.31.0"
+  peerDependencies:
+    eslint: ^8.57.0 || ^9.0.0
+    typescript: ">=4.8.4 <5.9.0"
+  checksum: 10c0/1fd4f62e16a44a5be2de501f70ba4b2d64479e014370bde7bbc6de6897cf1699766a8b7be4deb9b0328e74c2b4171839336ede4e3c60fec6ac8378b623a75275
+  languageName: node
+  linkType: hard
+
+"@typescript-eslint/utils@npm:^8.13.0":
   version: 8.25.0
   resolution: "@typescript-eslint/utils@npm:8.25.0"
   dependencies:
@@ -4983,6 +5067,16 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@typescript-eslint/visitor-keys@npm:8.31.0":
+  version: 8.31.0
+  resolution: "@typescript-eslint/visitor-keys@npm:8.31.0"
+  dependencies:
+    "@typescript-eslint/types": "npm:8.31.0"
+    eslint-visitor-keys: "npm:^4.2.0"
+  checksum: 10c0/e41e2a9e287d11232cda6126377d1df4de69c6e9dc2a14058819cff15280ec654a3877886a6806728196f299766cfbb0b299eb021c2ce168eb15dff5eb07b51b
+  languageName: node
+  linkType: hard
+
 "@ungap/structured-clone@npm:^1.2.0":
   version: 1.2.0
   resolution: "@ungap/structured-clone@npm:1.2.0"
@@ -5049,15 +5143,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@vitejs/plugin-basic-ssl@npm:^1.0.1":
-  version: 1.2.0
-  resolution: "@vitejs/plugin-basic-ssl@npm:1.2.0"
-  peerDependencies:
-    vite: ^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0
-  checksum: 10c0/0d360fcca01f91ade6e451edbea09a107ff9e95cd3c3766c7a069d1a168709df92d96c0bd1eccc66e2739a153e07c75a45321ec487450c0da942606200d8441d
-  languageName: node
-  linkType: hard
-
 "@vitejs/plugin-react@npm:^4.0.1":
   version: 4.3.4
   resolution: "@vitejs/plugin-react@npm:4.3.4"
@@ -6853,6 +6938,8 @@ __metadata:
     "@livekit/components-core": "npm:^0.12.0"
     "@livekit/components-react": "npm:^2.0.0"
     "@livekit/protocol": "npm:^1.33.0"
+    "@livekit/track-processors": "npm:^0.5.5"
+    "@mediapipe/tasks-vision": "npm:^0.10.18"
     "@opentelemetry/api": "npm:^1.4.0"
     "@opentelemetry/core": "npm:^1.25.1"
     "@opentelemetry/exporter-trace-otlp-http": "npm:^0.57.0"
@@ -6860,7 +6947,7 @@ __metadata:
     "@opentelemetry/sdk-trace-base": "npm:^1.25.1"
     "@opentelemetry/sdk-trace-web": "npm:^1.9.1"
     "@opentelemetry/semantic-conventions": "npm:^1.25.1"
-    "@playwright/test": "npm:^1.51.0"
+    "@playwright/test": "npm:^1.52.0"
     "@radix-ui/react-dialog": "npm:^1.0.4"
     "@radix-ui/react-slider": "npm:^1.1.2"
     "@radix-ui/react-visually-hidden": "npm:^1.0.3"
@@ -6873,6 +6960,7 @@ __metadata:
     "@testing-library/react": "npm:^16.0.0"
     "@testing-library/user-event": "npm:^14.5.1"
     "@types/content-type": "npm:^1.1.5"
+    "@types/dom-mediacapture-transform": "npm:^0.1.11"
     "@types/grecaptcha": "npm:^3.0.9"
     "@types/jsdom": "npm:^21.1.7"
     "@types/lodash-es": "npm:^4.17.12"
@@ -6883,12 +6971,11 @@ __metadata:
     "@types/react-dom": "npm:^18.3.0"
     "@types/sdp-transform": "npm:^2.4.5"
     "@types/uuid": "npm:10"
-    "@typescript-eslint/eslint-plugin": "npm:^8.0.0"
-    "@typescript-eslint/parser": "npm:^8.0.0"
+    "@typescript-eslint/eslint-plugin": "npm:^8.31.0"
+    "@typescript-eslint/parser": "npm:^8.31.0"
     "@use-gesture/react": "npm:^10.2.11"
     "@vector-im/compound-design-tokens": "npm:^3.0.0"
     "@vector-im/compound-web": "npm:^7.2.0"
-    "@vitejs/plugin-basic-ssl": "npm:^1.0.1"
     "@vitejs/plugin-react": "npm:^4.0.1"
     "@vitest/coverage-v8": "npm:^3.0.0"
     babel-plugin-transform-vite-meta-env: "npm:^1.0.3"
@@ -6931,7 +7018,7 @@ __metadata:
     react-use-measure: "npm:^2.1.1"
     rxjs: "npm:^7.8.1"
     sass: "npm:^1.42.1"
-    typescript: "npm:^5.1.6"
+    typescript: "npm:^5.8.3"
     typescript-eslint-language-service: "npm:^5.0.5"
     unique-names-generator: "npm:^4.6.0"
     vaul: "npm:^1.0.0"
@@ -10295,27 +10382,27 @@ __metadata:
   languageName: node
   linkType: hard
 
-"playwright-core@npm:1.51.0":
-  version: 1.51.0
-  resolution: "playwright-core@npm:1.51.0"
+"playwright-core@npm:1.52.0":
+  version: 1.52.0
+  resolution: "playwright-core@npm:1.52.0"
   bin:
     playwright-core: cli.js
-  checksum: 10c0/8f5de23088c5e97c00327f356b17e0223181e921baf99f4e38d9a3b18d0693db288f8b5389e96d0cb4a1b55f03870f140dd7346128a0c02ce36d11eb92153841
+  checksum: 10c0/640945507e6ca2144e9f596b2a6ecac042c2fd3683ff99e6271e9a7b38f3602d415f282609d569456f66680aab8b3c5bb1b257d8fb63a7fc0ed648261110421f
   languageName: node
   linkType: hard
 
-"playwright@npm:1.51.0":
-  version: 1.51.0
-  resolution: "playwright@npm:1.51.0"
+"playwright@npm:1.52.0":
+  version: 1.52.0
+  resolution: "playwright@npm:1.52.0"
   dependencies:
     fsevents: "npm:2.3.2"
-    playwright-core: "npm:1.51.0"
+    playwright-core: "npm:1.52.0"
   dependenciesMeta:
     fsevents:
       optional: true
   bin:
     playwright: cli.js
-  checksum: 10c0/e8509ea500e03e8051fd243f2347ac3196ff8dde4c20ae3aba4cf723e2b647a0158d209fba062995dab90590229a483d723562cf1ea8b2fc11698617027416fd
+  checksum: 10c0/2c6edf1e15e59bbaf77f3fa0fe0ac975793c17cff835d9c8b8bc6395a3b6f1c01898b3058ab37891b2e4d424bcc8f1b4844fe70d943e0143d239d7451408c579
   languageName: node
   linkType: hard
 
@@ -12672,13 +12759,13 @@ __metadata:
   languageName: node
   linkType: hard
 
-"typescript@npm:^5.1.6":
-  version: 5.7.3
-  resolution: "typescript@npm:5.7.3"
+"typescript@npm:^5.8.3":
+  version: 5.8.3
+  resolution: "typescript@npm:5.8.3"
   bin:
     tsc: bin/tsc
     tsserver: bin/tsserver
-  checksum: 10c0/b7580d716cf1824736cc6e628ab4cd8b51877408ba2be0869d2866da35ef8366dd6ae9eb9d0851470a39be17cbd61df1126f9e211d8799d764ea7431d5435afa
+  checksum: 10c0/5f8bb01196e542e64d44db3d16ee0e4063ce4f3e3966df6005f2588e86d91c03e1fb131c2581baf0fb65ee79669eea6e161cd448178986587e9f6844446dbb48
   languageName: node
   linkType: hard
 
@@ -12692,13 +12779,13 @@ __metadata:
   languageName: node
   linkType: hard
 
-"typescript@patch:typescript@npm%3A^5.1.6#optional!builtin<compat/typescript>":
-  version: 5.7.3
-  resolution: "typescript@patch:typescript@npm%3A5.7.3#optional!builtin<compat/typescript>::version=5.7.3&hash=5786d5"
+"typescript@patch:typescript@npm%3A^5.8.3#optional!builtin<compat/typescript>":
+  version: 5.8.3
+  resolution: "typescript@patch:typescript@npm%3A5.8.3#optional!builtin<compat/typescript>::version=5.8.3&hash=5786d5"
   bin:
     tsc: bin/tsc
     tsserver: bin/tsserver
-  checksum: 10c0/6fd7e0ed3bf23a81246878c613423730c40e8bdbfec4c6e4d7bf1b847cbb39076e56ad5f50aa9d7ebd89877999abaee216002d3f2818885e41c907caaa192cc4
+  checksum: 10c0/39117e346ff8ebd87ae1510b3a77d5d92dae5a89bde588c747d25da5c146603a99c8ee588c7ef80faaf123d89ed46f6dbd918d534d641083177d5fac38b8a1cb
   languageName: node
   linkType: hard