Compare commits

..

222 Commits

Author SHA1 Message Date
Valere
73245e875d fix: Netlify preview broken 2026-02-10 15:11:43 +01:00
Timo
a329ae0ad1 Merge pull request #3720 from element-hq/toger5/sdk-improvements-only
[MatrixRTC SDK] improvements - compatible with custom rtc application
2026-02-09 09:50:16 +01:00
Timo K
652ff0060c Back to develop branck (matrix-js-skd) 2026-02-09 09:07:54 +01:00
Robin
d3476c0446 Merge pull request #3726 from element-hq/robin/generate-items-logs
Add debug logs to generateItems
2026-02-06 13:45:13 +01:00
Robin
a0209eb433 Add debug logs to generateItems
It's always worth having logs for when state holders are created or destroyed (these are often the most interesting things happening in the application), so I thought it would be nice to have generateItems always log for you when it's doing that.
2026-02-06 13:26:02 +01:00
Timo K
e1ef28fdcc use js-sdk fixing room hack for any app 2026-02-05 09:04:03 +01:00
Timo K
2e26193c4a update leave 2026-02-05 06:33:51 +01:00
Timo K
42efc05bd0 Add more granular leave/join api 2026-02-05 06:06:15 +01:00
Valere
809186a2e7 test: test requesting send message permission in initialize widget 2026-02-04 21:16:50 +01:00
Timo K
927e8e195c add experimental label 2026-02-04 20:16:31 +01:00
Timo K
33f6271d13 sdk improvements - compatible with custom rtc application
- add local member
 - add optional make sticky
 - default to video not enabled
 - allow sending matrix events
2026-02-04 20:06:26 +01:00
Timo
2c1476f151 Merge pull request #3715 from element-hq/revert-3701-toger5/sdk-improvements
Revert "[WIP] SDK target improvements"
2026-02-04 18:41:54 +01:00
Timo
97ee4630b0 Revert "[WIP] SDK target improvements" 2026-02-04 08:51:01 +01:00
Timo
02b29f98b6 Merge pull request #3701 from element-hq/toger5/sdk-improvements
[WIP] SDK target improvements
2026-02-04 08:47:28 +01:00
Valere
fdc33a223f build against js-sdk rc 40.2.0-rc.0 2026-02-03 17:37:58 +01:00
Valere
7beafcac92 Merge branch 'livekit' into toger5/sdk-improvements 2026-02-03 16:32:36 +01:00
Valere Fedronic
5563f30bf2 Merge pull request #3705 from element-hq/toger5/fix-test-no-more-legacy-ring
[Matrix2.0 compatibility] stop advertising `livekit_alias`, use `"ROOM"` slot_id (backwards compat), stop sending deprecated notify
2026-02-03 16:26:52 +01:00
Valere
9d8ebf8ef3 fix import 2026-02-03 15:37:41 +01:00
Valere
a0ad238952 fix: DidSendCallNotification is not emitting array anymore 2026-02-03 15:29:19 +01:00
Timo K
18b771e620 Allow sending room events in rtc sdk 2026-02-03 11:10:57 +01:00
Timo K
461a1f1b70 add to sending room messages 2026-01-31 12:29:56 +01:00
Timo K
a066495c5f bump js-sdk fixup custom applications 2026-01-31 12:09:29 +01:00
Timo K
506b85770b Merge branch 'livekit' into toger5/sdk-improvements 2026-01-31 11:53:15 +01:00
Timo K
bd8c4188d0 bump js-sdk 2026-01-29 17:24:21 +01:00
Timo K
c78e8c54b8 js-sdk-bump 2026-01-29 17:22:56 +01:00
Valere Fedronic
c18cc964a8 Merge pull request #3706 from element-hq/valere/fix_remove_livekit_alias_signaling
The advertised livekit_alias in membership is deprecated
2026-01-29 16:13:53 +01:00
Valere
8ee098eef1 review: quick comment 2026-01-29 15:46:41 +01:00
Valere
a7bd2d3668 review: comment and better type 2026-01-29 14:58:54 +01:00
Valere
ca7c8a2f11 Send livekit_alias in multi sfu with state 2026-01-28 16:42:07 +01:00
Valere
146375e320 Only remove livekit_alias in sticky events 2026-01-28 16:33:52 +01:00
Valere
992fb9093c fix test and use anonymised alias 2026-01-28 15:25:19 +01:00
Valere
99ccfcfbd4 fix linter 2026-01-28 15:18:29 +01:00
Valere
420304fd9d fix test mocking 2026-01-28 15:16:02 +01:00
Valere
81cb89e599 fix test mocking 2026-01-28 15:11:10 +01:00
Valere
ee8b27837a fix lint issues 2026-01-28 14:38:23 +01:00
Valere
a645bbe0ba fix bad import 2026-01-28 14:29:16 +01:00
Valere
4c7db0147e The advertised livekit_alias in membership is deprecated 2026-01-28 14:22:21 +01:00
Timo K
c1b2346e22 Update tests based on new js-sdk changes
(no more legacy notify event)
2026-01-27 16:49:25 +01:00
Timo K
92a4150d6d more/better logging 2026-01-26 15:43:07 +01:00
Timo K
829619cca7 fix build:sdk:production 2026-01-26 12:44:25 +01:00
Timo K
408495b07a add back permissions to pr-deploy.yaml 2026-01-26 12:43:15 +01:00
Timo K
02f1a0da82 add local member and rtcBackendIdentity to sdk target 2026-01-26 12:43:01 +01:00
Timo K
6d8ade073a use sessionManager with custom slot instead of only session. 2026-01-25 19:27:07 +01:00
Timo K
1c946f7bb2 fix package full 2026-01-25 14:38:45 +01:00
Timo K
5724c3a09c add sdk:production 2026-01-25 14:37:37 +01:00
Timo K
d55464f3ab build sdk 2026-01-25 14:32:30 +01:00
Timo K
73a5a10c6d add workflow to publish sdk version to netlify 2026-01-25 14:25:15 +01:00
Timo K
112420cb5b fix widget api appliaction capabilities 2026-01-25 14:24:52 +01:00
Timo K
31823dbd53 improve widget tests
(this we might want to cherry pick in its own PR)
2026-01-25 14:24:31 +01:00
Timo
15c39372f4 Merge pull request #3697 from element-hq/renovate/npm-lodash-es-vulnerability
Update dependency lodash-es to v4.17.23 [SECURITY]
2026-01-23 10:51:41 +01:00
Timo
330ad3448d Merge pull request #3686 from element-hq/robin/sticky-widget
Make Element Call widgets compatible with Matrix 2.0 mode
2026-01-22 20:17:16 +01:00
Timo K
220fc51eb5 add initializeWidget to tests. 2026-01-22 19:38:39 +01:00
Timo K
b754106feb Add widget.ts tests 2026-01-22 19:23:08 +01:00
renovate[bot]
05107a3e5c Update dependency lodash-es to v4.17.23 [SECURITY] 2026-01-22 04:37:37 +00:00
Robin
d516f0f602 Make Element Call widgets compatible with Matrix 2.0 mode
This gives Element Call widgets the ability to send and receive sticky RTC membership events.
2026-01-21 19:59:34 +01:00
Timo
3eb88ec4a8 Merge pull request #3693 from element-hq/toger5/fix-sticky-rejoin-error
Fix rejoin EC crash
2026-01-20 16:30:09 +01:00
Timo K
745772f672 Fix rejoin EC crash
Due to a duplcaited key (the key not being specific enough)
2026-01-20 13:45:07 +01:00
Valere Fedronic
486c0b884b Merge pull request #3690 from element-hq/valere/playwright/improve_sticky_test
playwright: intercept send request to ensure sticky events are sent
2026-01-20 08:26:47 +01:00
Valere
9f323d21a8 review: quick comment 2026-01-20 08:10:50 +01:00
Timo
a72aae0b8d Merge pull request #3679 from element-hq/toger5/fix-unnecassary-error-logs
Fix some error log lines that lead to confusion
2026-01-19 17:57:34 +01:00
Timo
b1ceeda9ee Merge pull request #3687 from toger5/toger5/fix-reemitted-event-params
Fix delayId delegation. Use correct event reemitting types.
2026-01-19 16:00:16 +01:00
Valere
ed8cee9f58 review: fix/clean the route interception 2026-01-19 15:28:22 +01:00
Timo K
3a5b9dd519 remove overkill log 2026-01-19 14:58:45 +01:00
Valere
a985281b46 playwright: intercept send request to ensure sticky event are sent 2026-01-19 13:58:36 +01:00
Timo K
a7008d37c6 better log msg 2026-01-19 13:54:46 +01:00
Timo K
94bcb1bb27 remove leftover layout action code 2026-01-19 13:54:36 +01:00
Timo
36a1aa3c59 Merge pull request #3689 from toger5/toger5/remove-deprecated-layout-widget-actions
Remove deprecated widget layout actions
2026-01-19 13:20:04 +01:00
Valere Fedronic
b4ca55f7df Merge pull request #3688 from element-hq/valere/bug_deny_permissions
Do not end call with error in case of MediaDeviceError
2026-01-19 12:32:01 +01:00
Timo K
a102b3f4aa Fix some error log lines that lead to confusion.
In the past those log lines often were referenced for issues but they
are no real issues.
Either they are just deprecated code running or expected.
2026-01-19 12:20:37 +01:00
Timo K
d8802aa079 linter 2026-01-19 12:18:13 +01:00
Timo K
4833d4b867 Bump js-sdk to use version that properly emits delayId updates 2026-01-19 12:15:40 +01:00
Timo K
abb80e4e87 Fix event reemitting types
The callback args for reemitting will change to an array including the
originals emitter.
2026-01-19 12:15:15 +01:00
Timo K
2c3365a401 remove deprecated widget layout actions 2026-01-19 12:11:32 +01:00
Valere
2817362e15 Do not end call with error in case of MediaDeviceError 2026-01-19 11:46:17 +01:00
Timo
95b730e93f Merge pull request #3683 from element-hq/valere/playwright/spa_sticky_test
playwright: Test call in SPA mode with matrix 2.0
2026-01-19 09:34:49 +01:00
Valere
c2eea67b35 Merge branch 'livekit' into valere/playwright/spa_sticky_test 2026-01-16 17:41:39 +01:00
Valere
202e06a16d skip FF 2026-01-16 17:41:32 +01:00
Timo
e278752942 Merge pull request #3675 from element-hq/toger5/fixing-duble-publishing
Fix Publisher cleanup procedure
2026-01-16 17:37:46 +01:00
Valere
89c878cf3f playwright: Test call in SPA mode with matrix 2.0 2026-01-16 17:11:45 +01:00
Timo K
f41175444b revert changes unrelated to this PR 2026-01-16 16:23:05 +01:00
Timo K
102664482b Activate the hotswap test that was failing before. 2026-01-16 15:41:25 +01:00
Timo K
d5df9e65dd Merge branch 'livekit' into toger5/fixing-duble-publishing 2026-01-16 15:38:56 +01:00
Timo
f7255b3697 Merge pull request #3681 from element-hq/toger5/playwright-publisher-swap
Add playwright test for hotswatpping the transport (swap from `oldest_member` (singeSFU) to `multi_sfu`)
2026-01-16 15:34:15 +01:00
Timo K
3f8b3ba3f1 add test to check if publisher is properly destroyed 2026-01-16 14:41:11 +01:00
Timo K
22c1948a82 Add playwright test for hotswatpping transport 2026-01-16 14:27:52 +01:00
Timo K
a99a413b88 temp 2026-01-16 14:07:56 +01:00
Timo K
83df152608 review 2026-01-16 13:20:03 +01:00
Valere Fedronic
5c366320cb Merge pull request #3676 from element-hq/valere/playwright/compat_bug_reproduce
playwright: Federated call between legacy and compat
2026-01-16 13:09:02 +01:00
Valere
16b08994cc Try fixup mixed conten error loading widget 2026-01-16 12:53:10 +01:00
Timo K
f5c31626a6 fix unit tests 2026-01-16 12:43:13 +01:00
Timo K
5a48820927 remove widget layout widget action for good. 2026-01-16 10:22:23 +01:00
Timo K
971d16d243 revert jwt to stable version 2026-01-16 10:21:26 +01:00
Timo K
9b40d7420a refactor to use destroy method 2026-01-15 19:04:32 +01:00
Timo K
f71e483080 logging updates 2026-01-15 18:13:34 +01:00
Timo K
f7590a33d7 Fix publisher clean up! This is the acutal bug we chased 2026-01-15 18:12:00 +01:00
Timo K
c74accb906 remove "fetching" state again (we have to be able to deal with those
kind of switches!) Optimize to not always fetch the oldest member jwt
(only do so in legacy mode)
2026-01-15 18:11:08 +01:00
Valere
a00c6501c2 fix discarding pop ups 2026-01-15 15:22:04 +01:00
Valere
9e933b74ca cleanup notifications toast on start 2026-01-15 14:31:33 +01:00
Valere
d25d6d3196 Test all pairs of rtc mode in federated calls 2026-01-15 12:06:41 +01:00
Valere
243f369a24 re-use test helper in other files 2026-01-15 11:50:37 +01:00
Valere
0d1488af4c playwright: Federated call between legacy and compat 2026-01-14 18:07:12 +01:00
Timo K
b49411abfa dont set localTransport while still fetching oldest member transport 2026-01-14 17:35:35 +01:00
Timo
27e351270c Merge pull request #3669 from element-hq/toger5/livekit-alias-simple-roomid
Change the livekit alias to just be the room id (for backwards compatibility)
2026-01-14 12:43:08 +01:00
Timo K
41eb45b3c4 fix comments 2026-01-14 12:29:22 +01:00
Valere Fedronic
f5a6a05570 Merge pull request #3671 from element-hq/valere/playwright/huddle_test
playwright: End to end test for huddle calls in widget mode
2026-01-14 12:27:57 +01:00
Timo
31cc12eda4 Merge pull request #3670 from element-hq/toger5/livekitAlias-debugging-info
Add livekitAlias debugging info
2026-01-14 12:26:50 +01:00
Robin
39c24eb0f5 Merge pull request #3665 from element-hq/renovate/livekit-components
Update dependency @livekit/components-react to v2.9.17
2026-01-14 12:15:48 +01:00
Valere
7338a55e76 review: improve doc 2026-01-14 12:15:00 +01:00
Valere
68f04d46a9 playwright: End to end test for huddle calls in widget mode 2026-01-13 19:11:12 +01:00
Valere Fedronic
16ad89df47 Merge pull request #3667 from element-hq/valere/playwright/discard_call_widget_test
playwright: Test that declining a call in DM works as expected
2026-01-13 18:00:36 +01:00
Timo
3ce0afcacc Merge pull request #3664 from element-hq/toger5/userMedia-alternative
Small refactor: User media alternative
2026-01-13 17:51:36 +01:00
Timo K
4d35b77077 bump jwt docker 2026-01-13 17:41:46 +01:00
Timo K
20e63c8dc7 fix tests 2026-01-13 17:39:55 +01:00
Timo K
ff8d71380d Add livekitAlias debugging info 2026-01-13 15:14:34 +01:00
Timo K
d185b52091 more docstrings to docuemnt the migration away from livekit_alias 2026-01-13 12:44:17 +01:00
Timo K
72ca3821ab Change the livekit alias to just be the room id for backwards
compatibility.
2026-01-13 12:37:46 +01:00
Valere Fedronic
4b7dc8a290 Merge pull request #3659 from element-hq/valere/initial_mute_states
Fix: Ensure voice and audio intent are respected
2026-01-13 11:50:32 +01:00
Gordnho
f391c68055 Add Haproxy configuration for element call (#3661)
* Add Haproxy configuration for livekit SFU

---------

Co-authored-by: fkwp <fkwp@users.noreply.github.com>
2026-01-13 10:39:15 +00:00
Valere
e82a048088 playwright: Test that declining a call in DM works as expected 2026-01-13 09:46:44 +01:00
Valere
7a8c2d6ca9 Merge branch 'livekit' into valere/initial_mute_states 2026-01-12 19:13:40 +01:00
Valere
f3df8bc709 review: Use is_widget directly instead of using the packageType 2026-01-12 17:28:18 +01:00
Valere
534f934378 review: just inline intent and skipLobby from urlParam 2026-01-12 16:37:28 +01:00
Valere
56beb59346 review: Update copyright properly 2026-01-12 15:35:21 +01:00
Valere
fbdcacb344 review: remove dead comment 2026-01-12 15:23:02 +01:00
Timo K
89e5e5b10a fix uuid 2026-01-12 12:34:50 +01:00
Timo K
8d8eb09922 Merge branch 'livekit' into toger5/userMedia-alternative 2026-01-12 12:34:03 +01:00
renovate[bot]
fbdcde3058 Update dependency @livekit/components-react to v2.9.17 2026-01-12 01:33:44 +00:00
Timo
f444c9bc11 Merge pull request #3638 from element-hq/toger5/delayed-event-delegation
Pseudonomous identity and use the new jwt service endpoint (with delayed event delegation)
2026-01-09 19:58:30 +01:00
Timo K
47e389678b add uuid in our own package.json 2026-01-09 19:48:11 +01:00
Timo K
7a8f5cc859 use crypto.randomUUID() 2026-01-09 19:40:21 +01:00
Timo K
45bb233673 Merge branch 'toger5/delayed-event-delegation' into toger5/userMedia-alternative 2026-01-09 19:38:21 +01:00
Timo K
341277932c fix tests 2026-01-09 19:34:41 +01:00
Timo K
735c17d3e6 fix tile flicker on video mute/unmute
(only pass rtcBackendIdenitity instead of full rtcMember)
2026-01-09 19:31:40 +01:00
Timo K
1a9e4a1a5c bump js-sdk 2026-01-09 19:12:54 +01:00
Timo K
7387a00406 lint 2026-01-09 19:04:42 +01:00
Timo K
32694fddf6 improve rtcBackendIdentity debugging 2026-01-09 19:04:02 +01:00
Timo K
444c50d8c4 Change code style of user media 2026-01-09 18:27:56 +01:00
Timo K
9a88e3dab6 review valere (more readable force new endpoint logic) 2026-01-09 18:05:26 +01:00
Timo K
39432310e3 use actual uuid for member.id
review with some renaming and removal of unnecassary if
2026-01-09 17:56:30 +01:00
Timo
0addc7c09b Merge pull request #3662 from element-hq/toger5/simplify-sdk-example
Update sdk example
2026-01-09 17:34:57 +01:00
Timo
0b766472ed Update sdk/README.md
Co-authored-by: Robin <robin@robin.town>
2026-01-09 17:30:02 +01:00
Timo K
ed3d168ac3 fix tests 2026-01-09 17:29:48 +01:00
Timo K
70b7ebff80 prettier 2026-01-09 16:32:14 +01:00
Timo K
b7838e4353 Update README.md 2026-01-09 16:25:06 +01:00
Timo K
0eb0169ad9 update example 2026-01-09 16:01:03 +01:00
Timo K
c2e3774749 prettier 2026-01-09 15:40:39 +01:00
Valere
cf579638fd skip firefox 2026-01-09 15:39:53 +01:00
Timo K
c7c6dc18bf fix js-sdk dependency 2026-01-09 15:28:11 +01:00
Valere
8827ff2ea6 add option for firefox webRTC test to allow fake streams 2026-01-09 15:18:35 +01:00
Valere
8535494671 use config to unmute full package on localhost 2026-01-09 15:00:57 +01:00
Valere
ad66d10c69 run playwright in dev mode 2026-01-09 14:41:07 +01:00
Valere
e9ed9580db trust localhost for CI and devbuil 2026-01-09 14:18:14 +01:00
Timo K
7dbbd763b9 Refactor how we aquire the jwt token for the local user. (only fetch it
once)

The local jwt token needs to be aquired via the right endpoint. The
endpoint defines how our rtcBackendIdentity is computed. Based on us
using sticky events or state events we also need to use the right
endpoint. This cannot be done generically in the connection manager. The
jwt token now is computed in the localTransport and the resolved sfu
config is passed to the connection manager.

Add JWT endpoint version and SFU config support Pin matrix-js-sdk to a
specific commit and update dev auth image tag. Propagate SFU config and
JWT endpoint choice through local transport, ConnectionManager and
Connection; add JwtEndpointVersion enum and LocalTransportWithSFUConfig
type. Add NO_MATRIX_2 auth error and locale string, thread
rtcBackendIdentity through UI props, and include related test, CSS and
minor imports updates
2026-01-09 13:38:26 +01:00
Valere
0636678ad2 quick log for playwright problem 2026-01-09 13:25:16 +01:00
Robin
a567505ef5 Merge pull request #3478 from element-hq/renovate/compound
Update Compound
2026-01-09 13:12:48 +01:00
Robin
f89a7a8211 Fix tests 2026-01-09 12:58:06 +01:00
Valere Fedronic
29854a77b4 Merge pull request #3660 from element-hq/valere/add_codecov_shield
Add codecov shields.io badge
2026-01-09 12:57:56 +01:00
Valere
bf19b388ea Add codecov shields.io badge 2026-01-09 12:43:33 +01:00
Valere
d2907f51d3 prettier format 2026-01-09 12:35:25 +01:00
Valere
231a80d9de update snapshot, mute buttons have aria-label now 2026-01-09 12:32:36 +01:00
Valere
a9153f2781 fix: Regression on default mutestate for voicecall + end-2-end tests 2026-01-09 12:03:56 +01:00
Valere
f5f8bb549a delete outdated default mute state config 2026-01-09 12:03:45 +01:00
renovate[bot]
8fe49d681a Update Compound 2026-01-08 19:49:47 +00:00
Timo K
d4b06b0f9c fix connection recreation which breaks EC lk connection 2026-01-08 14:27:47 +01:00
Will Hunt
0439fdefef Remove duplicate IntentAndPlatformDerivedConfiguration interface (#3658) 2026-01-08 11:51:28 +00:00
Timo K
1909aef186 temp 2026-01-08 12:27:17 +01:00
Timo K
385f63e83e fix tests 2026-01-07 17:46:39 +01:00
Timo K
a5a4bb2b82 add retries inside the getLiveKitJWTWithDelayDelegation and
`getLiveKitJWT` functions.
2026-01-07 17:38:29 +01:00
Timo K
5556d363b9 add retries and be more specific when we fall back to legacy endpoint 2026-01-07 17:21:14 +01:00
Timo K
0eeed4e18e fix test snapshot 2026-01-07 17:05:00 +01:00
Timo K
d5ad2e38e2 fix up tests 2026-01-07 16:09:08 +01:00
Timo K
75fca3108a cleanup an rename compatibility mode 2026-01-07 16:08:20 +01:00
Timo K
d814f60f23 review (docstrings) and remove unused artifacts. 2026-01-07 15:47:44 +01:00
Timo K
dd562bdaf5 dont use throw for logic. 2026-01-07 15:36:32 +01:00
Timo K
d48042f522 fix lint 2026-01-07 15:18:37 +01:00
Timo K
6480df44e9 add tests for open id delay fallback 2026-01-07 15:10:32 +01:00
Timo K
83d04ac122 fix tests and remove duplicated mocks. 2026-01-07 13:26:37 +01:00
Timo K
556a87d141 fix js-doc 2026-01-05 22:20:19 +01:00
Timo K
69a4189517 self review 2026-01-05 22:00:02 +01:00
Timo
009c9e046c Merge branch 'livekit' into toger5/delayed-event-delegation 2026-01-05 21:26:15 +01:00
Timo K
1580087286 use latest js-sdk 2026-01-05 21:24:52 +01:00
Timo K
00fca6e3c7 simplify localTransport 2026-01-05 21:17:37 +01:00
Timo K
5d5d75ebdf fixup merge 2026-01-05 21:08:33 +01:00
Timo K
df2bd22005 Merge branch 'livekit' into toger5/delayed-event-delegation 2026-01-05 21:08:21 +01:00
Robin
de8fdcfa69 Merge pull request #3518 from element-hq/renovate/all-minor-patch
Update all non-major dependencies
2026-01-05 19:42:07 +01:00
Robin
6aaf2db626 Fix formatting 2026-01-05 19:35:09 +01:00
Robin
72b9b8231a Merge branch 'livekit' into renovate/all-minor-patch 2026-01-05 19:29:24 +01:00
Robin
7817b76a8b Downgrade react-i18next to fix type errors
Upgrade beyond 16.1.1 blocked by https://github.com/i18next/react-i18next/issues/1899
2026-01-05 19:28:07 +01:00
Robin
b585af41da Fix encryption keys type error
See commit 174439c2f0c09cf9926c28435ba4db1345df4aee in matrix-js-sdk for the other part of the fix.
2026-01-05 19:27:39 +01:00
Robin
1006d06b04 Fix WebCodecs type error 2026-01-05 17:46:04 +01:00
Robin
b9afaab556 Upgrade i18next to resolve a type error 2026-01-05 15:52:17 +01:00
Robin
a6cdb28e45 Roll back @livekit/track-processors to fix type error
While we await https://github.com/livekit/track-processors-js/pull/118 to be released
2026-01-05 15:36:43 +01:00
Timo
ce682232b8 Merge pull request #3586 from element-hq/toger5/otel-remove-experiment
Remove all `OpenTelementry` related code
2026-01-05 14:55:28 +01:00
Timo K
029c38be63 fix prettier 2026-01-05 14:30:55 +01:00
Timo K
5b1eb13d65 fix lints and add hint in the otel readme to the PR removing it. 2026-01-05 14:27:54 +01:00
Timo
d1aabb05ff Merge branch 'livekit' into toger5/otel-remove-experiment 2026-01-05 14:22:11 +01:00
Valere Fedronic
49cd980ddf Merge pull request #3644 from element-hq/valere/devx_check_doc
Add script to check that the tsdoc is correct and up-to-date
2026-01-05 14:08:08 +01:00
Robin
b6ca0c4cd6 Fix tests 2026-01-05 13:55:58 +01:00
Robin
5fa170c9c9 Fix builds 2026-01-05 13:54:08 +01:00
Will Hunt
f260a20728 Set latest tag when publishing a docker release. (#3650)
Fixes https://github.com/element-hq/element-call/issues/3647
2026-01-05 12:41:26 +00:00
Robin
749ba103a7 Merge pull request #3641 from element-hq/renovate/livekit-client
Update dependency livekit-client to v2.16.1
2026-01-05 13:28:17 +01:00
Timo
fef4db69d8 Merge pull request #3628 from element-hq/renovate/livekit-components
Update LiveKit components
2026-01-05 10:37:18 +01:00
renovate[bot]
057bc4e2ad Update all non-major dependencies 2026-01-03 18:06:04 +00:00
renovate[bot]
a6aa4526cf Update LiveKit components 2025-12-31 13:52:19 +00:00
renovate[bot]
88cfd32e51 Update dependency livekit-client to v2.16.1 2025-12-31 13:52:04 +00:00
Valere
5e715765d9 fix: un-needed tsdoc script 2025-12-30 18:17:07 +01:00
Valere
da55d84bde Add script to check that the tsdoc is correct and up-to-date 2025-12-30 17:02:44 +01:00
Timo K
0f5c5d8be5 cleanup based on new js-sdk impl 2025-12-29 17:38:54 +01:00
Timo K
7591e2bda1 Merge branch 'livekit' into toger5/delayed-event-delegation 2025-12-28 21:04:49 +01:00
Timo K
3ba2d13e27 use the js-sdk where the hashed rtcSessionIdeintity is already part of
the event (no need to compute it in the encryption manager)
2025-12-28 21:04:46 +01:00
Timo K
55d18f10fe temp refactored membership rtcidentity 2025-12-19 19:23:41 +01:00
Timo K
50f3bf00ae use correct js-sdk 2025-12-17 10:22:46 +01:00
Timo K
ab7e3486b3 Make use of the new jwt service endpoint (with delayed event delegation)
This also does all the compatibility work. When to use which endpoint to
authenticate agains a jwt service.
2025-12-17 09:53:49 +01:00
Timo K
9bd51fdfc4 Merge branch 'livekit' into toger5/pseudonomous-identities 2025-12-16 12:18:00 +01:00
fkwp
ff3d6bd088 enable sticky events 2025-12-16 11:22:25 +01:00
fkwp
6fe6daba31 switch synapse docker image to one with sticky event support 2025-12-16 11:20:38 +01:00
Timo
3d8d0ae9dd Merge branch 'livekit' into toger5/pseudonomous-identities 2025-12-15 21:20:55 +01:00
Timo K
5bc6ed5885 small refactor to make it testable. 2025-12-15 20:17:57 +01:00
Timo K
909d980dff still with broken tests... 2025-12-15 18:23:30 +01:00
Timo K
8bf6518371 remove span processor endpoints 2025-11-24 12:42:00 +01:00
Timo K
2f92867383 remove otel deps 2025-11-24 12:36:36 +01:00
Timo K
1fda5c7920 remove otel to see what impact it has on tests. 2025-11-24 12:33:02 +01:00
129 changed files with 5994 additions and 4316 deletions

View File

@@ -8,7 +8,7 @@ Please see LICENSE in the repository root for full details.
`;
module.exports = {
plugins: ["matrix-org", "rxjs"],
plugins: ["matrix-org", "rxjs", "jsdoc"],
extends: [
"plugin:matrix-org/react",
"plugin:matrix-org/a11y",
@@ -26,6 +26,13 @@ module.exports = {
node: true,
},
rules: {
"jsdoc/no-types": "error",
"jsdoc/empty-tags": "error",
"jsdoc/check-property-names": "error",
"jsdoc/check-values": "error",
"jsdoc/check-param-names": "warn",
// "jsdoc/require-param": "warn",
"jsdoc/require-param-description": "warn",
"matrix-org/require-copyright-header": ["error", COPYRIGHT_HEADER],
"jsx-a11y/media-has-caption": "off",
"react/display-name": "error",
@@ -75,6 +82,30 @@ module.exports = {
"no-console": ["error"],
},
},
{
files: [
"**/*.test.ts",
"**/*.test.tsx",
"**/test.ts",
"**/test.tsx",
"**/test-**",
],
rules: {
"jsdoc/no-types": "off",
"jsdoc/empty-tags": "off",
"jsdoc/check-property-names": "off",
"jsdoc/check-values": "off",
"jsdoc/check-param-names": "off",
"jsdoc/require-param-description": "off",
},
},
{
files: ["playwright/**"],
rules: {
// Playwright as a `use` function that has nothing to do with React hooks.
"react-hooks/rules-of-hooks": "off",
},
},
],
settings: {
react: {

View File

@@ -7,7 +7,7 @@ on:
type: string
package:
type: string # This would ideally be a `choice` type, but that isn't supported yet
description: The package type to be built. Must be one of 'full' or 'embedded'
description: The package type to be built. Must be one of 'full', 'embedded', or 'sdk'
required: true
build_mode:
type: string # This would ideally be a `choice` type, but that isn't supported yet

View File

@@ -69,3 +69,17 @@ jobs:
SENTRY_URL: ${{ secrets.SENTRY_URL }}
SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
build_sdk_element_call:
# Use the embedded package vite build
uses: ./.github/workflows/build-element-call.yaml
with:
package: sdk
vite_app_version: ${{ github.event.release.tag_name || github.sha }}
build_mode: ${{ github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'development build') && 'development' || 'production' }}
secrets:
SENTRY_ORG: ${{ secrets.SENTRY_ORG }}
SENTRY_PROJECT: ${{ secrets.SENTRY_PROJECT }}
SENTRY_URL: ${{ secrets.SENTRY_URL }}
SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

View File

@@ -14,6 +14,10 @@ on:
deployment_ref:
required: true
type: string
package:
required: true
type: string
description: Which package to deploy - 'full', 'embedded', or 'sdk'
artifact_run_id:
required: false
type: string
@@ -50,7 +54,7 @@ jobs:
with:
github-token: ${{ secrets.ELEMENT_BOT_TOKEN }}
run-id: ${{ inputs.artifact_run_id }}
name: build-output-full
name: build-output-${{ inputs.package }}
path: webapp
- name: Add redirects file
@@ -58,15 +62,19 @@ jobs:
run: curl -s https://raw.githubusercontent.com/element-hq/element-call/main/config/netlify_redirects > webapp/_redirects
- name: Add config file
run: curl -s "https://raw.githubusercontent.com/${{ inputs.pr_head_full_name }}/${{ inputs.pr_head_ref }}/config/config_netlify_preview.json" > webapp/config.json
run: |
if [ "${{ inputs.package }}" = "full" ]; then
curl -s "https://raw.githubusercontent.com/${{ inputs.pr_head_full_name }}/${{ inputs.pr_head_ref }}/config/config_netlify_preview.json" > webapp/config.json
else
curl -s "https://raw.githubusercontent.com/${{ inputs.pr_head_full_name }}/${{ inputs.pr_head_ref }}/config/config_netlify_preview_sdk.json" > webapp/config.json
fi
- name: ☁️ Deploy to Netlify
id: netlify
uses: nwtgck/actions-netlify@4cbaf4c08f1a7bfa537d6113472ef4424e4eb654 # v3.0
with:
publish-dir: webapp
deploy-message: "Deploy from GitHub Actions"
alias: pr${{ inputs.pr_number }}
alias: ${{ inputs.package == 'sdk' && format('pr{0}-sdk', inputs.pr_number) || format('pr{0}', inputs.pr_number) }}
env:
NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }}
NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }}

View File

@@ -20,7 +20,7 @@ jobs:
owner: ${{ github.event.workflow_run.head_repository.owner.login }}
branch: ${{ github.event.workflow_run.head_branch }}
netlify:
netlify-full:
needs: prdetails
permissions:
deployments: write
@@ -31,6 +31,24 @@ jobs:
pr_head_full_name: ${{ github.event.workflow_run.head_repository.full_name }}
pr_head_ref: ${{ needs.prdetails.outputs.pr_data_json && fromJSON(needs.prdetails.outputs.pr_data_json).head.ref }}
deployment_ref: ${{ needs.prdetails.outputs.pr_data_json && fromJSON(needs.prdetails.outputs.pr_data_json).head.sha || github.ref || github.head_ref }}
package: full
secrets:
ELEMENT_BOT_TOKEN: ${{ secrets.ELEMENT_BOT_TOKEN }}
NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }}
NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }}
netlify-sdk:
needs: prdetails
permissions:
deployments: write
uses: ./.github/workflows/deploy-to-netlify.yaml
with:
artifact_run_id: ${{ github.event.workflow_run.id || github.run_id }}
pr_number: ${{ needs.prdetails.outputs.pr_number }}
pr_head_full_name: ${{ github.event.workflow_run.head_repository.full_name }}
pr_head_ref: ${{ needs.prdetails.outputs.pr_data_json && fromJSON(needs.prdetails.outputs.pr_data_json).head.ref }}
deployment_ref: ${{ needs.prdetails.outputs.pr_data_json && fromJSON(needs.prdetails.outputs.pr_data_json).head.sha || github.ref || github.head_ref }}
package: sdk
secrets:
ELEMENT_BOT_TOKEN: ${{ secrets.ELEMENT_BOT_TOKEN }}
NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }}

View File

@@ -61,6 +61,7 @@ jobs:
docker_tags: |
type=sha,format=short,event=branch
type=raw,value=${{ github.event.release.tag_name }}
type=raw,value=latest
# Like before, using ${{ env.VERSION }} above doesn't work
add_docker_release_note:
needs: publish_docker

View File

@@ -3,6 +3,7 @@
[![Chat](https://img.shields.io/matrix/webrtc:matrix.org)](https://matrix.to/#/#webrtc:matrix.org)
[![Localazy](https://img.shields.io/endpoint?url=https%3A%2F%2Fconnect.localazy.com%2Fstatus%2Felement-call%2Fdata%3Fcontent%3Dall%26title%3Dlocalazy%26logo%3Dtrue)](https://localazy.com/p/element-call)
[![License](https://img.shields.io/github/license/element-hq/element-call)](LICENSE-AGPL-3.0)
[![Codecov](https://img.shields.io/codecov/c/github/element-hq/element-call)](https://app.codecov.io/gh/element-hq/element-call)
[🎬 Live Demo 🎬](https://call.element.io)

View File

@@ -38,6 +38,8 @@ experimental_features:
# MSC4222 needed for syncv2 state_after. This allow clients to
# correctly track the state of the room.
msc4222_enabled: true
# sticky events for MatrixRTC user state
msc4354_enabled: true
# The maximum allowed duration by which sent events can be delayed, as
# per MSC4140. Must be a positive value if set. Defaults to no

View File

@@ -38,7 +38,7 @@ experimental_features:
# MSC4222 needed for syncv2 state_after. This allow clients to
# correctly track the state of the room.
msc4222_enabled: true
# sticky events for matrixRTC user state
# sticky events for MatrixRTC user state
msc4354_enabled: true
# The maximum allowed duration by which sent events can be delayed, as

View File

@@ -181,7 +181,10 @@ server {
}
# Convenience reverse proxy for the call.m.localhost domain to yarn dev --host
# Convenience reverse proxy for the call.m.localhost domain to element call
# running on the host either via
# - yarn dev --host or
# - falling back to http (the element call docker container)
server {
listen 80;
listen [::]:80;
@@ -197,7 +200,7 @@ server {
ssl_certificate /root/ssl/cert.pem;
ssl_certificate_key /root/ssl/key.pem;
# 1. Attempt HTTPS first
location ^~ / {
proxy_set_header Host $host;
@@ -208,8 +211,23 @@ server {
proxy_pass https://host.docker.internal:3000;
proxy_ssl_verify off;
# 2. Redirect specific errors (e.g., 502 Bad Gateway or 504 Timeout)
# to the named fallback location
error_page 502 503 504 = @http_fallback;
}
# 3. Fallback location using HTTP
location @http_fallback {
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_pass http://host.docker.internal:8080;
}
error_page 500 502 503 504 /50x.html;
}

View File

@@ -38,6 +38,8 @@ experimental_features:
# MSC4222 needed for syncv2 state_after. This allow clients to
# correctly track the state of the room.
msc4222_enabled: true
# sticky events for MatrixRTC user state
msc4354_enabled: true
# The maximum allowed duration by which sent events can be delayed, as
# per MSC4140. Must be a positive value if set. Defaults to no

View File

@@ -38,6 +38,8 @@ experimental_features:
# MSC4222 needed for syncv2 state_after. This allow clients to
# correctly track the state of the room.
msc4222_enabled: true
# sticky events for MatrixRTC user state
msc4354_enabled: true
# The maximum allowed duration by which sent events can be delayed, as
# per MSC4140. Must be a positive value if set. Defaults to no

View File

@@ -0,0 +1,16 @@
{
"default_server_config": {
"m.homeserver": {
"base_url": "https://call-unstable.ems.host",
"server_name": "call-unstable.ems.host"
}
},
"ssla": "https://static.element.io/legal/element-software-and-services-license-agreement-uk-1.pdf",
"matrix_rtc_session": {
"wait_for_key_rotation_ms": 3000,
"membership_event_expiry_ms": 180000000,
"delayed_leave_event_delay_ms": 18000,
"delayed_leave_event_restart_ms": 4000,
"network_error_retry_ms": 100
}
}

View File

@@ -1,5 +1,12 @@
# OpenTelemetry Collector for development
## Edit:
Open telemetry has been removed in: https://github.com/element-hq/element-call/pull/3586
Check this PR to get back the implementation or to use it as reference to add it back.
---
This directory contains a docker compose file that starts a jaeger all-in-one instance
with an in-memory database, along with a standalone OpenTelemetry collector that forwards
traces into the jaeger. Jaeger has a built-in OpenTelemetry collector, but it can't be

View File

@@ -3,7 +3,7 @@ networks:
services:
auth-service:
image: ghcr.io/element-hq/lk-jwt-service:latest-ci
image: ghcr.io/element-hq/lk-jwt-service:sha-f8ddd00
pull_policy: always
hostname: auth-server
environment:
@@ -25,7 +25,7 @@ services:
- ecbackend
auth-service-1:
image: ghcr.io/element-hq/lk-jwt-service:latest-ci
image: ghcr.io/element-hq/lk-jwt-service:sha-f8ddd00
pull_policy: always
hostname: auth-server-1
environment:
@@ -88,7 +88,7 @@ services:
synapse:
hostname: homeserver
image: docker.io/matrixdotorg/synapse:latest
image: ghcr.io/element-hq/synapse:pr-18968-dcb7678281bc02d4551043a6338fe5b7e6aa47ce
pull_policy: always
environment:
- SYNAPSE_CONFIG_PATH=/data/cfg/homeserver.yaml
@@ -106,7 +106,7 @@ services:
synapse-1:
hostname: homeserver-1
image: docker.io/matrixdotorg/synapse:latest
image: ghcr.io/element-hq/synapse:pr-18968-dcb7678281bc02d4551043a6338fe5b7e6aa47ce
pull_policy: always
environment:
- SYNAPSE_CONFIG_PATH=/data/cfg/homeserver.yaml

View File

@@ -152,6 +152,40 @@ handle {
}
```
Using Haproxy, you can achieve this by:
```
# Frontend
# Match /livekit/sfu/ path
acl is_sfu path_beg -i /livekit/sfu/
use_backend sfu_backend if is_sfu matrixrtc_domain
acl is_mxrtc_auth path_beg -i /sfu/get
use_backend mxrtc_auth_backend if is_mxrtc_auth matrixrtc_domain
# Backend
## MatrixRTC backend
backend sfu_backend
server livekit 127.0.0.1:7880
http-request set-path %[path,regsub(^/livekit/sfu/,/)]
http-request set-header Host %[req.hdr(host)]
timeout server 120s
# WebSocket support
option forwardfor
option http-server-close
option http-buffer-request
backend mxrtc_auth_backend
server sfu 127.0.0.1:8070
http-request set-header Host %[req.hdr(host)]
timeout server 120s
# WebSocket support
option forwardfor
option http-server-close
option http-buffer-request
```
#### MatrixRTC backend announcement
> [!IMPORTANT]

View File

@@ -62,6 +62,7 @@ These parameters are relevant to both [widget](./embedded-standalone.md) and [st
| `homeserver` | | Not applicable | No | Homeserver for registering a new (guest) user, configures non-default guest user server when creating a spa link. |
| `lang` | [BCP 47](https://www.rfc-editor.org/info/bcp47) code | No | No | The language the app should use. |
| `password` | | No | No | E2EE password when using a shared secret. (For individual sender keys in embedded mode this is not required.) |
| `perParticipantE2EE` | `true` or `false` | No, defaults to `false` | No, defaults to `false` | Enables per participant encryption with Keys exchanged over encrypted matrix room messages. |
| `controlledAudioDevices` | `true` or `false` | No, defaults to `false` | No, defaults to `false` | Whether the [global JS controls for audio devices](./controls.md#audio-devices) should be enabled, allowing the list of audio devices to be controlled by the app hosting Element Call. |
| `roomId` | [Matrix Room ID](https://spec.matrix.org/v1.12/appendices/#room-ids) | Yes | No | Anything about what room we're pointed to should be from useRoomIdentifier which parses the path and resolves alias with respect to the default server name, however roomId is an exception as we need the room ID in embedded widget mode, and not the room alias (or even the via params because we are not trying to join it). This is also not validated, where it is in `useRoomIdentifier()`. |
| `showControls` | `true` or `false` | No, defaults to `true` | No, defaults to `true` | Displays controls like mute, screen-share, invite, and hangup buttons during a call. |

View File

@@ -34,10 +34,6 @@ export default {
// then Knip will flag it as a false positive
// https://github.com/webpro-nl/knip/issues/766
"@vector-im/compound-web",
// We need this so that TypeScript is happy with @livekit/track-processors.
// This might be a bug in the LiveKit repo but for now we fix it on the
// Element Call side.
"@types/dom-mediacapture-transform",
"matrix-widget-api",
],
ignoreExportsUsedInFile: true,

View File

@@ -116,6 +116,7 @@
"matrix_rtc_transport_missing": "The server is not configured to work with {{brand}}. Please contact your server admin (Domain: {{domain}}, Error Code: {{ errorCode }}).",
"membership_manager": "Membership Manager Error",
"membership_manager_description": "The Membership Manager had to shut down. This is caused by many consequtive failed network requests.",
"no_matrix_2_authorization_service": "The authorization service for your media server (SFU) is out of date.",
"open_elsewhere": "Opened in another tab",
"open_elsewhere_description": "{{brand}} has been opened in another tab. If that doesn't sound right, try reloading the page.",
"room_creation_restricted": "Failed to create call",

View File

@@ -13,8 +13,9 @@
"build:embedded": "yarn build:full --config vite-embedded.config.js",
"build:embedded:production": "yarn build:embedded",
"build:embedded:development": "yarn build:embedded --mode development",
"build:sdk": "yarn build:full --config vite-sdk.config.js",
"build:sdk:development": "yarn build:sdk --mode development",
"build:sdk": "yarn build:full --config vite-sdk.config.js",
"build:sdk:production": "yarn build:sdk",
"serve": "vite preview",
"prettier:check": "prettier -c .",
"prettier:format": "prettier -w .",
@@ -42,20 +43,13 @@
"@codecov/vite-plugin": "^1.3.0",
"@fontsource/inconsolata": "^5.1.0",
"@fontsource/inter": "^5.1.0",
"@formatjs/intl-durationformat": "^0.7.0",
"@formatjs/intl-durationformat": "^0.9.0",
"@formatjs/intl-segmenter": "^11.7.3",
"@livekit/components-core": "^0.12.0",
"@livekit/components-react": "^2.0.0",
"@livekit/protocol": "^1.42.2",
"@livekit/track-processors": "^0.5.5",
"@livekit/track-processors": "^0.6.0 || ^0.7.1",
"@mediapipe/tasks-vision": "^0.10.18",
"@opentelemetry/api": "^1.4.0",
"@opentelemetry/core": "^2.0.0",
"@opentelemetry/exporter-trace-otlp-http": "^0.203.0",
"@opentelemetry/resources": "^2.0.0",
"@opentelemetry/sdk-trace-base": "^2.0.0",
"@opentelemetry/sdk-trace-web": "^2.0.0",
"@opentelemetry/semantic-conventions": "^1.25.1",
"@playwright/test": "^1.57.0",
"@radix-ui/react-dialog": "^1.0.4",
"@radix-ui/react-slider": "^1.1.2",
@@ -69,7 +63,6 @@
"@testing-library/react": "^16.0.0",
"@testing-library/user-event": "^14.5.1",
"@types/content-type": "^1.1.5",
"@types/dom-mediacapture-transform": "^0.1.11",
"@types/grecaptcha": "^3.0.9",
"@types/jsdom": "^21.1.7",
"@types/lodash-es": "^4.17.12",
@@ -95,6 +88,7 @@
"eslint-config-prettier": "^10.0.0",
"eslint-plugin-deprecate": "^0.8.2",
"eslint-plugin-import": "^2.26.0",
"eslint-plugin-jsdoc": "^61.5.0",
"eslint-plugin-jsx-a11y": "^6.5.1",
"eslint-plugin-matrix-org": "2.1.0",
"eslint-plugin-react": "^7.29.4",
@@ -103,7 +97,7 @@
"eslint-plugin-unicorn": "^56.0.0",
"fetch-mock": "11.1.5",
"global-jsdom": "^26.0.0",
"i18next": "^24.0.0",
"i18next": "^25.0.0",
"i18next-browser-languagedetector": "^8.0.0",
"i18next-parser": "^9.1.0",
"jsdom": "^26.0.0",
@@ -111,8 +105,8 @@
"livekit-client": "^2.13.0",
"lodash-es": "^4.17.21",
"loglevel": "^1.9.1",
"matrix-js-sdk": "matrix-org/matrix-js-sdk#2218ec4e3102e841ba3e794e1c492c0a5aa6c1c3",
"matrix-widget-api": "^1.14.0",
"matrix-js-sdk": "matrix-org/matrix-js-sdk#6e3efef0c5f660df47cf00874927dec1c75cc3cf",
"matrix-widget-api": "^1.16.1",
"node-stdlib-browser": "^1.3.1",
"normalize.css": "^8.0.1",
"observable-hooks": "^4.2.3",
@@ -124,7 +118,7 @@
"qrcode": "^1.5.4",
"react": "19",
"react-dom": "19",
"react-i18next": "^15.0.0",
"react-i18next": "^16.0.0 <16.1.0",
"react-router-dom": "^7.0.0",
"react-use-measure": "^2.1.1",
"rxjs": "^7.8.1",
@@ -132,6 +126,7 @@
"typescript": "^5.8.3",
"typescript-eslint-language-service": "^5.0.5",
"unique-names-generator": "^4.6.0",
"uuid": "^13.0.0",
"vaul": "^1.0.0",
"vite": "^7.0.0",
"vite-plugin-generate-file": "^0.3.0",

View File

@@ -1,5 +1,6 @@
/*
Copyright 2025 New Vector Ltd.
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.

View File

@@ -7,6 +7,8 @@ Please see LICENSE in the repository root for full details.
import { expect, test } from "@playwright/test";
import { SpaHelpers } from "./spa-helpers.ts";
test("Sign up a new account, then login, then logout", async ({ browser }) => {
const userId = `test_user-id_${Date.now()}`;
@@ -69,12 +71,7 @@ test("As a guest, create a call, share link and other join", async ({
// ========
// ARRANGE: The first user creates a call as guest, join it, then click the invite button to copy the invite link
// ========
await creatorPage.getByTestId("home_callName").click();
await creatorPage.getByTestId("home_callName").fill("Welcome");
await creatorPage.getByTestId("home_displayName").click();
await creatorPage.getByTestId("home_displayName").fill("Inviter");
await creatorPage.getByTestId("home_go").click();
await expect(creatorPage.locator("video")).toBeVisible();
await SpaHelpers.createCall(creatorPage, "Inviter", "Welcome");
// join
await creatorPage.getByTestId("lobby_joinCall").click();
@@ -82,19 +79,7 @@ test("As a guest, create a call, share link and other join", async ({
await creatorPage.getByRole("radio", { name: "Spotlight" }).check();
// Get the invite link
await creatorPage.getByRole("button", { name: "Invite" }).click();
await expect(
creatorPage.getByRole("heading", { name: "Invite to this call" }),
).toBeVisible();
await expect(creatorPage.getByRole("img", { name: "QR Code" })).toBeVisible();
await expect(creatorPage.getByTestId("modal_inviteLink")).toBeVisible();
await expect(creatorPage.getByTestId("modal_inviteLink")).toBeVisible();
await creatorPage.getByTestId("modal_inviteLink").click();
const inviteLink = (await creatorPage.evaluate(
"navigator.clipboard.readText()",
)) as string;
expect(inviteLink).toContain("room/#/");
const inviteLink = await SpaHelpers.getCallInviteLink(creatorPage);
// ========
// ACT: The other user use the invite link to join the call as a guest
@@ -103,13 +88,7 @@ test("As a guest, create a call, share link and other join", async ({
reducedMotion: "reduce",
});
const guestPage = await guestInviteeContext.newPage();
await guestPage.goto(inviteLink);
await guestPage.getByTestId("joincall_displayName").fill("Invitee");
await expect(guestPage.getByTestId("joincall_joincall")).toBeVisible();
await guestPage.getByTestId("joincall_joincall").click();
await guestPage.getByTestId("lobby_joinCall").click();
await guestPage.getByRole("radio", { name: "Spotlight" }).check();
await SpaHelpers.joinCallFromInviteLink(guestPage, inviteLink);
// ========
// ASSERT: check that there are two members in the call

View File

@@ -1,22 +1,19 @@
/*
Copyright 2025 New Vector Ltd.
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
type Browser,
type Page,
test,
expect,
type JSHandle,
} from "@playwright/test";
import { type Page, test, expect, type JSHandle } from "@playwright/test";
import type { MatrixClient } from "matrix-js-sdk";
import { HOST1, TestHelpers } from "../widget/test-helpers.ts";
export type UserBaseFixture = {
mxId: string;
displayName: string;
page: Page;
clientHandle: JSHandle<MatrixClient>;
};
@@ -28,10 +25,10 @@ export type BaseWidgetSetup = {
export interface MyFixtures {
asWidget: BaseWidgetSetup;
callType: "room" | "dm";
addUser: (username: string, host: string) => Promise<UserBaseFixture>;
}
const PASSWORD = "foobarbaz1!";
// Minimal config.json for the local element-web instance
const CONFIG_JSON = {
default_server_config: {
@@ -65,144 +62,91 @@ const CONFIG_JSON = {
},
};
/**
* Set the Element Call URL in the dev tool settings using `window.mxSettingsStore` via `page.evaluate`.
* @param page
*/
const setDevToolElementCallDevUrl = process.env.USE_DOCKER
? async (page: Page): Promise<void> => {
await page.evaluate(() => {
window.mxSettingsStore.setValue(
"Developer.elementCallUrl",
null,
"device",
"http://localhost:8080/room",
);
});
}
: async (page: Page): Promise<void> => {
await page.evaluate(() => {
window.mxSettingsStore.setValue(
"Developer.elementCallUrl",
null,
"device",
"https://localhost:3000/room",
);
});
};
/**
* Registers a new user and returns page, clientHandle and mxId.
*/
async function registerUser(
browser: Browser,
username: string,
): Promise<{ page: Page; clientHandle: JSHandle<MatrixClient>; mxId: string }> {
const userContext = await browser.newContext({
reducedMotion: "reduce",
});
const page = await userContext.newPage();
await page.goto("http://localhost:8081/#/welcome");
await page.getByRole("link", { name: "Create Account" }).click();
await page.getByRole("textbox", { name: "Username" }).fill(username);
await page
.getByRole("textbox", { name: "Password", exact: true })
.fill(PASSWORD);
await page.getByRole("textbox", { name: "Confirm password" }).click();
await page.getByRole("textbox", { name: "Confirm password" }).fill(PASSWORD);
await page.getByRole("button", { name: "Register" }).click();
await expect(
page.getByRole("heading", { name: `Welcome ${username}` }),
).toBeVisible();
const browserUnsupportedToast = page
.getByText("Element does not support this browser")
.locator("..")
.locator("..");
// Dismiss incompatible browser toast
const dismissButton = browserUnsupportedToast.getByRole("button", {
name: "Dismiss",
});
try {
await expect(dismissButton).toBeVisible({ timeout: 700 });
await dismissButton.click();
} catch {
// dismissButton not visible, continue as normal
}
await setDevToolElementCallDevUrl(page);
const clientHandle = await page.evaluateHandle(() =>
window.mxMatrixClientPeg.get(),
);
const mxId = (await clientHandle.evaluate(
(cli: MatrixClient) => cli.getUserId(),
clientHandle,
))!;
return { page, clientHandle, mxId };
}
export const widgetTest = test.extend<MyFixtures>({
asWidget: async ({ browser, context }, pUse) => {
// allow per-test override: `widgetTest.use({ callType: "dm" })`
callType: ["room", { option: true }],
asWidget: async ({ browser, context, callType }, pUse) => {
await context.route(`http://localhost:8081/config.json*`, async (route) => {
await route.fulfill({ json: CONFIG_JSON });
});
const userA = `brooks_${Date.now()}`;
const userB = `whistler_${Date.now()}`;
const brooksDisplayName = `brooks_${Date.now()}`;
const whistlerDisplayName = `whistler_${Date.now()}`;
// Register users
const {
page: ewPage1,
clientHandle: brooksClientHandle,
mxId: brooksMxId,
} = await registerUser(browser, userA);
} = await TestHelpers.registerUser(browser, brooksDisplayName);
const {
page: ewPage2,
clientHandle: whistlerClientHandle,
mxId: whistlerMxId,
} = await registerUser(browser, userB);
} = await TestHelpers.registerUser(browser, whistlerDisplayName);
// Invite the second user
await ewPage1
.getByRole("navigation", { name: "Room list" })
.getByRole("button", { name: "New conversation" })
.click();
if (callType === "room") {
await TestHelpers.createRoom("Welcome Room", ewPage1);
await ewPage1.getByRole("menuitem", { name: "New Room" }).click();
await ewPage1.getByRole("textbox", { name: "Name" }).fill("Welcome Room");
await ewPage1.getByRole("button", { name: "Create room" }).click();
await expect(ewPage1.getByText("You created this room.")).toBeVisible();
await expect(ewPage1.getByText("Encryption enabled")).toBeVisible();
await ewPage1
.getByRole("button", { name: "Invite to this room", exact: true })
.click();
await expect(
ewPage1.getByRole("heading", { name: "Invite to Welcome Room" }),
).toBeVisible();
await ewPage1
.getByRole("button", { name: "Invite to this room", exact: true })
.click();
await expect(
ewPage1.getByRole("heading", { name: "Invite to Welcome Room" }),
).toBeVisible();
// To get the invite textbox we need to specifically select within the
// dialog, since there is another textbox in the background (the message
// composer). In theory the composer shouldn't be visible to Playwright at
// all because the invite dialog has trapped focus, but the focus trap
// doesn't quite work right on Firefox.
await ewPage1.getByRole("dialog").getByRole("textbox").fill(whistlerMxId);
await ewPage1.getByRole("dialog").getByRole("textbox").click();
await ewPage1.getByRole("button", { name: "Invite" }).click();
// To get the invite textbox we need to specifically select within the
// dialog, since there is another textbox in the background (the message
// composer). In theory the composer shouldn't be visible to Playwright at
// all because the invite dialog has trapped focus, but the focus trap
// doesn't quite work right on Firefox.
await ewPage1.getByRole("dialog").getByRole("textbox").fill(whistlerMxId);
await ewPage1.getByRole("dialog").getByRole("textbox").click();
await ewPage1.getByRole("button", { name: "Invite" }).click();
// Accept the invite
await expect(
ewPage2.getByRole("option", { name: "Welcome Room" }),
).toBeVisible();
await ewPage2.getByRole("option", { name: "Welcome Room" }).click();
await ewPage2.getByRole("button", { name: "Accept" }).click();
await expect(
ewPage2
.getByRole("main")
.getByRole("heading", { name: "Welcome Room" }),
).toBeVisible();
} else if (callType === "dm") {
await ewPage1
.getByRole("navigation", { name: "Room list" })
.getByRole("button", { name: "New conversation" })
.click();
// Accept the invite
await expect(
ewPage2.getByRole("option", { name: "Welcome Room" }),
).toBeVisible();
await ewPage2.getByRole("option", { name: "Welcome Room" }).click();
await ewPage2.getByRole("button", { name: "Accept" }).click();
await expect(
ewPage2.getByRole("main").getByRole("heading", { name: "Welcome Room" }),
).toBeVisible();
await ewPage1.getByRole("menuitem", { name: "Start chat" }).click();
await ewPage1.getByRole("textbox", { name: "Search" }).click();
await ewPage1.getByRole("textbox", { name: "Search" }).fill(whistlerMxId);
await ewPage1.getByRole("button", { name: "Go" }).click();
// Wait and send the first message to create the DM
await expect(
ewPage1.getByText(/Send your first message to invite/),
).toBeVisible();
await ewPage1.locator(".mx_BasicMessageComposer_input > div").click();
await ewPage1
.getByRole("textbox", { name: "Send a message…" })
.fill("Hello!");
await ewPage1.getByRole("button", { name: "Send message" }).click();
await expect(
ewPage1.getByText("This is the beginning of your"),
).toBeVisible();
// Accept the DM invite from brooks
// This how playwright record selects the DM invite in the room list
await ewPage2.getByRole("option", { name: "Open room" }).click();
await ewPage2.getByRole("button", { name: "Start chatting" }).click();
}
// Renamed use to pUse, as a workaround for eslint error that was thinking this use was a react use.
await pUse({
@@ -210,12 +154,40 @@ export const widgetTest = test.extend<MyFixtures>({
mxId: brooksMxId,
page: ewPage1,
clientHandle: brooksClientHandle,
displayName: brooksDisplayName,
},
whistler: {
mxId: whistlerMxId,
page: ewPage2,
clientHandle: whistlerClientHandle,
displayName: whistlerDisplayName,
},
});
},
/**
* Provide a way to add additional users within a test.
* The returned user will be registered on the default homeserver, the name will be made unique by appending a timestamp.
*/
addUser: async ({ browser }, use) => {
await use(
async (
username: string,
host: string = HOST1,
): Promise<UserBaseFixture> => {
const uniqueSuffix = Date.now();
const { page, clientHandle, mxId } = await TestHelpers.registerUser(
browser,
`${username.toLowerCase()}_${uniqueSuffix}`,
host,
);
return {
mxId,
displayName: username,
page,
clientHandle,
};
},
);
},
});

View File

@@ -0,0 +1,135 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
expect,
type Page,
test,
type Request,
type Browser,
} from "@playwright/test";
import { SpaHelpers } from "./spa-helpers";
async function setupTwoUserSpaCall(
browser: Browser,
page: Page,
browserName: string,
): Promise<{ guestPage: Page }> {
test.skip(
browserName === "firefox",
"The is test is not working on firefox CI environment. No mic/audio device inputs so cam/mic are disabled",
);
await page.goto("/");
let androlHasSentStickyEvent = false;
await interceptEventSend(
page,
// This room is not encrypted, so the event is sent in clear
"org.matrix.msc4143.rtc.member",
(req) => {
androlHasSentStickyEvent =
androlHasSentStickyEvent || isStickySend(req.url());
},
);
await SpaHelpers.createCall(page, "Androl", "HelloCall", true, "2_0");
const inviteLink = await SpaHelpers.getCallInviteLink(page);
// Other
const guestInviteeContext = await browser.newContext({
reducedMotion: "reduce",
});
const guestPage = await guestInviteeContext.newPage();
await guestPage.goto("/");
let pevaraHasSentStickyEvent = false;
await interceptEventSend(
guestPage,
// This room is not encrypted, so the event is sent in clear
"org.matrix.msc4143.rtc.member",
(req) => {
pevaraHasSentStickyEvent =
pevaraHasSentStickyEvent || isStickySend(req.url());
},
);
await SpaHelpers.joinCallFromInviteLink(
guestPage,
inviteLink,
"Pevara",
"2_0",
);
// Assert both sides have sent sticky membership events
expect(androlHasSentStickyEvent).toEqual(true);
expect(pevaraHasSentStickyEvent).toEqual(true);
return { guestPage };
}
test("One to One call using matrix rtc 2.0 aka sticky events", async ({
browser,
page,
browserName,
}) => {
const { guestPage } = await setupTwoUserSpaCall(browser, page, browserName);
await SpaHelpers.expectVideoTilesCount(page, 2);
await SpaHelpers.expectVideoTilesCount(guestPage, 2);
});
// This issue occurs when a member leave but does not clean up their sticky event.
// If they rejoin they will use a new stickye key (stickyKey = member.id = UUID())
// We end up with two memberships with the same user and device id. This previously
// was a impossible case since that would be the same state event. Now its possible.
// We need to ALWAYS key by userId, deviceId and member.id. This test checks that.
test("One to One rejoin after improper leave does not crash EC", async ({
browser,
page,
browserName,
}) => {
const { guestPage } = await setupTwoUserSpaCall(browser, page, browserName);
await SpaHelpers.expectVideoTilesCount(page, 2);
await SpaHelpers.expectVideoTilesCount(guestPage, 2);
await guestPage.reload();
await expect(guestPage.getByTestId("lobby_joinCall")).toBeVisible();
// Check if rejoining with the same browser context (device) breaks EC.
// This has happened on versions that do not consider the member.id as part of the key for a media tile.
await guestPage.getByTestId("lobby_joinCall").click();
// We cannot use the `expectVideoTilesCount` helper here since one of them is expected to show waiting for media
await expect(page.getByTestId("videoTile")).toHaveCount(3);
await expect(guestPage.getByTestId("videoTile")).toHaveCount(2);
});
function isStickySend(url: string): boolean {
return !!new URL(url).searchParams.get(
"org.matrix.msc4354.sticky_duration_ms",
);
}
async function interceptEventSend(
page: Page,
eventType: string,
callback: (request: Request) => void,
): Promise<void> {
await page.route(
`**/_matrix/client/v3/rooms/**/send/${eventType}/**`,
async (route, req) => {
callback(req);
return route.continue();
},
);
}

148
playwright/spa-helpers.ts Normal file
View File

@@ -0,0 +1,148 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { expect, type Page } from "@playwright/test";
import { type RtcMode } from "./widget/test-helpers.ts";
/**
* Create and join a call from the SPA home page.
*
* @param page - The Playwright page object
* @param userName - The display name to use for the call
* @param callName - The name of the call to create
* @param autoJoin - Whether to automatically join the call after creating it
* @param mode - The RTC mode to use for the call
*/
async function createCall(
page: Page,
userName: string,
callName: string,
autoJoin: boolean = false,
mode: RtcMode | undefined = undefined,
): Promise<void> {
await page.getByTestId("home_callName").click();
await page.getByTestId("home_callName").fill(callName);
await page.getByTestId("home_displayName").click();
await page.getByTestId("home_displayName").fill(userName);
await page.getByTestId("home_go").click();
await expect(page.locator("video")).toBeVisible();
await expect(page.getByTestId("lobby_joinCall")).toBeVisible();
if (mode) {
await setRtcModeFromSettings(page, mode);
}
if (autoJoin) {
// Join the call
await page.getByTestId("lobby_joinCall").click();
}
}
/**
* Get the invite link for the current call.
*/
async function getCallInviteLink(page: Page): Promise<string> {
await page.getByRole("button", { name: "Invite" }).click();
await expect(
page.getByRole("heading", { name: "Invite to this call" }),
).toBeVisible();
await expect(page.getByRole("img", { name: "QR Code" })).toBeVisible();
await expect(page.getByTestId("modal_inviteLink")).toBeVisible();
await expect(page.getByTestId("modal_inviteLink")).toBeVisible();
await page.getByTestId("modal_inviteLink").click();
const inviteLink = (await page.evaluate(
"navigator.clipboard.readText()",
)) as string;
expect(inviteLink).toContain("room/#/");
return inviteLink;
}
/**
* Join a call from an invitation link.
* @param page - The Playwright page object
* @param inviteLink - The invite link to join
* @param displayName - The display name to use when joining the call
* @param mode - The RTC mode to use for the call
*/
async function joinCallFromInviteLink(
page: Page,
inviteLink: string,
displayName: string = "Invitee",
mode: RtcMode | undefined = undefined,
): Promise<void> {
await page.goto(inviteLink);
await page.getByTestId("joincall_displayName").fill(displayName);
await expect(page.getByTestId("joincall_joincall")).toBeVisible();
await page.getByTestId("joincall_joincall").click();
if (mode) {
await setRtcModeFromSettings(page, mode);
}
await page.getByTestId("lobby_joinCall").click();
await page.getByRole("radio", { name: "Spotlight" }).check();
}
async function setRtcModeFromSettings(
page: Page,
mode: RtcMode,
): Promise<void> {
await page.getByRole("button", { name: "Settings" }).click();
await page.getByRole("tab", { name: "Preferences" }).click();
await page.getByText("Developer mode", { exact: true }).check(); // Idempotent: won't uncheck if already checked
// Move to Developer tab now
await page.getByRole("tab", { name: "Developer" }).click();
if (mode == "legacy") {
await page.getByText("Legacy: state events").click();
} else if (mode == "2_0") {
await page.getByText("Matrix 2.0").click();
} else {
// compat
await page.getByText("Compatibility: state events").click();
}
await page.getByTestId("modal_close").click();
}
/**
* Expect a certain number of video tiles to be present and visible.
*/
async function expectVideoTilesCount(page: Page, count: number): Promise<void> {
await expect(page.getByTestId("videoTile")).toHaveCount(2);
// There are no other options than to wait for all media to be ready?
// Or it is too flaky :/
await page.waitForTimeout(3000);
// No one should be waiting for media
await expect(page.getByText("Waiting for media...")).not.toBeVisible();
// There should be 5 video elements, visible and autoplaying
const videoElements = await page.locator("video").all();
expect(videoElements.length).toBe(count);
const blockDisplayCount = await page
.locator("video")
.evaluateAll(
(videos: Element[]) =>
videos.filter(
(v: Element) => window.getComputedStyle(v).display === "block",
).length,
);
expect(blockDisplayCount).toBe(count);
}
export const SpaHelpers = {
createCall,
getCallInviteLink,
joinCallFromInviteLink,
expectVideoTilesCount,
};

View File

@@ -0,0 +1,86 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { expect, test } from "@playwright/test";
import { widgetTest } from "../fixtures/widget-user";
import { HOST1, HOST2, type RtcMode, TestHelpers } from "./test-helpers";
const modePairs: [RtcMode, RtcMode][] = [
["compat", "compat"],
["legacy", "legacy"],
["legacy", "compat"],
["compat", "legacy"],
];
modePairs.forEach(([rtcMode1, rtcMode2]) => {
widgetTest(
`Test federated call with rtc modes ${rtcMode1} and ${rtcMode2}`,
async ({ addUser, browserName }) => {
test.skip(
browserName === "firefox",
"The is test is not working on firefox CI environment. No mic/audio device inputs so cam/mic are disabled",
);
const florian = await addUser("floriant", HOST1);
const timo = await addUser("timo", HOST2);
const roomName = "Call Room";
await TestHelpers.createRoom(roomName, florian.page, [timo.mxId]);
await TestHelpers.acceptRoomInvite(roomName, timo.page);
await florian.page.pause();
await TestHelpers.openWidgetSetEmbeddedElementCallRtcModeCloseWidget(
florian.page,
rtcMode1,
);
await TestHelpers.openWidgetSetEmbeddedElementCallRtcModeCloseWidget(
timo.page,
rtcMode2,
);
await TestHelpers.startCallInCurrentRoom(florian.page, false);
await TestHelpers.joinCallFromLobby(florian.page);
// timo joins
await TestHelpers.joinCallInCurrentRoom(timo.page);
// We should see 2 video tiles everywhere now
for (const user of [timo, florian]) {
const frame = user.page
.locator('iframe[title="Element Call"]')
.contentFrame();
await expect(frame.getByTestId("videoTile")).toHaveCount(2);
// There are no other options than to wait for all media to be ready?
// Or it is too flaky :/
await user.page.waitForTimeout(3000);
// No one should be waiting for media
await expect(frame.getByText("Waiting for media...")).not.toBeVisible();
// There should be 2 video elements, visible and autoplaying
const videoElements = await frame.locator("video").all();
expect(videoElements.length).toBe(2);
const blockDisplayCount = await frame
.locator("video")
.evaluateAll(
(videos: Element[]) =>
videos.filter(
(v: Element) => window.getComputedStyle(v).display === "block",
).length,
);
expect(blockDisplayCount).toBe(2);
}
// await florian.page.pause();
},
);
});

View File

@@ -0,0 +1,104 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { expect, test } from "@playwright/test";
import { widgetTest } from "../fixtures/widget-user";
import { HOST1, HOST2, TestHelpers } from "./test-helpers";
// ## Issue
// This test reproduces an issue with the publisher.
// When switching local focus, we need to recreate the publisher.
// This failed because of a dead lock in the old publishers destruction.
//
// There are numerus ways to enforece this situation:
// - oldest member swap (manually set the oldest member focus and leave with the prev oldest member)
// This almost never happens in the real worls since clients will set their preferredFoci list to what the oldest member is.
// - switch from oldest member to multi sfu as the NOT the first joiner + the first joiner is on a different sfu than your preferred sfu.
//
// This test uses the "switch from oldest member to multi sfu" approach.
//
// It is a copy of federated-call.test.ts in the `["legacy", "legacy"]` setup,
// which once connected will make the second user switch to multi sfu.
widgetTest(
`Test swapping publisher from ${HOST1} to ${HOST2}`,
async ({ addUser, browserName }) => {
test.skip(
browserName === "firefox",
"The is test is not working on firefox CI environment. No mic/audio device inputs so cam/mic are disabled",
);
const florian = await addUser("floriant", HOST1);
const timo = await addUser("timo", HOST2);
const roomName = "Call Room";
await TestHelpers.createRoom(roomName, florian.page, [timo.mxId]);
await TestHelpers.acceptRoomInvite(roomName, timo.page);
await florian.page.pause();
await TestHelpers.openWidgetSetEmbeddedElementCallRtcModeCloseWidget(
florian.page,
"legacy",
);
await TestHelpers.openWidgetSetEmbeddedElementCallRtcModeCloseWidget(
timo.page,
"legacy",
);
await TestHelpers.startCallInCurrentRoom(florian.page, false);
await TestHelpers.joinCallFromLobby(florian.page);
// timo joins
await TestHelpers.joinCallInCurrentRoom(timo.page);
// We should see 2 video tiles everywhere now
for (const user of [timo, florian]) {
const frame = user.page
.locator('iframe[title="Element Call"]')
.contentFrame();
await expect(frame.getByTestId("videoTile")).toHaveCount(2);
// There are no other options than to wait for all media to be ready?
// Or it is too flaky :/
await user.page.waitForTimeout(3000);
// No one should be waiting for media
await expect(frame.getByText("Waiting for media...")).not.toBeVisible();
// There should be 2 video elements, visible and autoplaying
const videoElements = await frame.locator("video").all();
expect(videoElements.length).toBe(2);
const blockDisplayCount = await frame
.locator("video")
.evaluateAll(
(videos: Element[]) =>
videos.filter(
(v: Element) => window.getComputedStyle(v).display === "block",
).length,
);
expect(blockDisplayCount).toBe(2);
}
// now we switch the mode for timo (second joiner on multi-sfu HOST2 but currently HOST1)
await TestHelpers.setEmbeddedElementCallRtcMode(timo.page, "compat");
await timo.page.waitForTimeout(3000);
const blockDisplayCount = await timo.page
.locator('iframe[title="Element Call"]')
.contentFrame()
.locator("video")
.evaluateAll(
(videos: Element[]) =>
videos.filter(
(v: Element) => window.getComputedStyle(v).display === "block",
).length,
);
expect(blockDisplayCount).toBe(2);
},
);

View File

@@ -0,0 +1,136 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { expect, test } from "@playwright/test";
import { widgetTest } from "../fixtures/widget-user.ts";
import { HOST1, TestHelpers } from "./test-helpers.ts";
widgetTest("Create and join a group call", async ({ addUser, browserName }) => {
test.skip(
browserName === "firefox",
"The is test is not working on firefox CI environment. No mic/audio device inputs so cam/mic are disabled",
);
test.slow(); // We are registering multiple users here, give it more time
const valere = await addUser("Valere", HOST1);
const timo = await addUser("Timo", HOST1);
const robin = await addUser("Robin", HOST1);
const halfshot = await addUser("Halfshot", HOST1);
const florian = await addUser("florian", HOST1);
const roomName = "Group Call Room";
await TestHelpers.createRoom(roomName, valere.page, [
timo.mxId,
robin.mxId,
halfshot.mxId,
florian.mxId,
]);
for (const user of [timo, robin, halfshot, florian]) {
// Accept the invite
// This isn't super stable to get this as this super generic locator,
// but it works for now.
await TestHelpers.acceptRoomInvite(roomName, user.page);
}
// Start the call as Valere
await TestHelpers.startCallInCurrentRoom(valere.page, false);
await expect(
valere.page.locator('iframe[title="Element Call"]'),
).toBeVisible();
await TestHelpers.joinCallFromLobby(valere.page);
for (const user of [timo, robin, halfshot, florian]) {
await TestHelpers.joinCallInCurrentRoom(user.page);
}
for (const user of [timo, robin, halfshot, florian]) {
const frame = user.page
.locator('iframe[title="Element Call"]')
.contentFrame();
// No lobby, should start with video on
// The only way to know if it is muted or not is to look at the data-kind attribute..
const videoButton = frame.getByTestId("incall_videomute");
await expect(videoButton).toBeVisible();
// video should be off by default in a voice call
await expect(videoButton).toHaveAttribute("aria-label", /^Stop video$/);
}
// We should see 5 video tiles everywhere now
for (const user of [valere, timo, robin, halfshot, florian]) {
const frame = user.page
.locator('iframe[title="Element Call"]')
.contentFrame();
await expect(frame.getByTestId("videoTile")).toHaveCount(5);
for (const participant of [valere, timo, robin, halfshot, florian]) {
// Check the names are correct
await expect(frame.getByText(participant.displayName)).toBeVisible();
}
// There is no other options than to wait for all media to be ready?
// Or it is too flaky :/
await user.page.waitForTimeout(5000);
// No one should be waiting for media
await expect(frame.getByText("Waiting for media...")).not.toBeVisible();
// There should be 5 video elements, visible and autoplaying
const videoElements = await frame.locator("video").all();
expect(videoElements.length).toBe(5);
await expect(frame.locator("video[autoplay]")).toHaveCount(5);
const blockDisplayCount = await frame
.locator("video")
.evaluateAll(
(videos: Element[]) =>
videos.filter(
(v: Element) => window.getComputedStyle(v).display === "block",
).length,
);
expect(blockDisplayCount).toBe(5);
}
// Quickly test muting one participant to see it reflects and that our asserts works
const florianFrame = florian.page
.locator('iframe[title="Element Call"]')
.contentFrame();
const florianMuteButton = florianFrame.getByTestId("incall_videomute");
await florianMuteButton.click();
// Now the button should indicate we can start video
await expect(florianMuteButton).toHaveAttribute(
"aria-label",
/^Start video$/,
);
// wait a bit for the state to propagate
await valere.page.waitForTimeout(3000);
{
const frame = valere.page
.locator('iframe[title="Element Call"]')
.contentFrame();
const videoElements = await frame.locator("video").all();
expect(videoElements.length).toBe(5);
const blockDisplayCount = await frame
.locator("video")
.evaluateAll(
(videos: Element[]) =>
videos.filter(
(v: Element) => window.getComputedStyle(v).display === "block",
).length,
);
// out of 5 ONLY 4 are visible (display:block) !!
// XXX we need to be better at our HTML markup and accessibility, it would make
// this kind of stuff way easier to test if we could look out for aria attributes.
expect(blockDisplayCount).toBe(4);
}
});

View File

@@ -0,0 +1,279 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
type Browser,
expect,
type JSHandle,
type Page,
} from "@playwright/test";
import { type MatrixClient } from "matrix-js-sdk";
const PASSWORD = "foobarbaz1!";
export const HOST1 = "https://app.m.localhost/#/welcome";
export const HOST2 = "https://app.othersite.m.localhost/#/welcome";
export type RtcMode = "legacy" | "compat" | "2_0";
export class TestHelpers {
public static async startCallInCurrentRoom(
page: Page,
voice: boolean = false,
): Promise<void> {
const buttonName = voice ? "Voice call" : "Video call";
await expect(page.getByRole("button", { name: buttonName })).toBeVisible();
await page.getByRole("button", { name: buttonName }).click();
await expect(
page.getByRole("menuitem", { name: "Element Call" }),
).toBeVisible();
await page.getByRole("menuitem", { name: "Element Call" }).click();
}
public static async joinCallFromLobby(page: Page): Promise<void> {
await expect(
page
.locator('iframe[title="Element Call"]')
.contentFrame()
.getByTestId("lobby_joinCall"),
).toBeVisible();
await page
.locator('iframe[title="Element Call"]')
.contentFrame()
.getByTestId("lobby_joinCall")
.click();
}
public static async joinCallInCurrentRoom(
page: Page,
audioOnly: boolean = false,
): Promise<void> {
// This is the header button that notifies about an ongoing call
const label = audioOnly ? "Voice call started" : "Video call started";
await expect(page.getByText(label)).toBeVisible();
await expect(page.getByRole("button", { name: "Join" })).toBeVisible();
await page.getByRole("button", { name: "Join" }).click();
}
/**
* Registers a new user and returns page, clientHandle and mxId.
*/
public static async registerUser(
browser: Browser,
username: string,
host: string = HOST1,
): Promise<{
page: Page;
clientHandle: JSHandle<MatrixClient>;
mxId: string;
}> {
const userContext = await browser.newContext({
reducedMotion: "reduce",
});
const page = await userContext.newPage();
await page.goto(host);
await page.getByRole("link", { name: "Create Account" }).click();
await page.getByRole("textbox", { name: "Username" }).fill(username);
await page.getByRole("textbox", { name: "Password", exact: true }).click();
await page
.getByRole("textbox", { name: "Password", exact: true })
.fill(PASSWORD);
await page.getByRole("textbox", { name: "Confirm password" }).click();
await page
.getByRole("textbox", { name: "Confirm password" })
.fill(PASSWORD);
await page.getByRole("button", { name: "Register" }).click();
await expect(
page.getByRole("heading", { name: `Welcome ${username}` }),
).toBeVisible({
// Increase timeout as registration can be slow :/
timeout: 15_000,
});
await this.maybeDismissBrowserNotSupportedToast(page);
await this.maybeDismissServiceWorkerWarningToast(page);
await TestHelpers.setDevToolElementCallDevUrl(page);
const clientHandle = await page.evaluateHandle(() =>
window.mxMatrixClientPeg.get(),
);
const mxId = (await clientHandle.evaluate(
(cli: MatrixClient) => cli.getUserId(),
clientHandle,
))!;
return { page, clientHandle, mxId };
}
private static async maybeDismissBrowserNotSupportedToast(
page: Page,
): Promise<void> {
const browserUnsupportedToast = page
.getByText("Element does not support this browser")
.locator("..")
.locator("..");
// Dismiss incompatible browser toast
const dismissButton = browserUnsupportedToast.getByRole("button", {
name: "Dismiss",
});
try {
await expect(dismissButton).toBeVisible({ timeout: 700 });
await dismissButton.click();
} catch {
// dismissButton not visible, continue as normal
}
}
private static async maybeDismissServiceWorkerWarningToast(
page: Page,
): Promise<void> {
const toast = page
.locator(".mx_Toast_toast")
.getByText("Failed to load service worker");
try {
await expect(toast).toBeVisible({ timeout: 700 });
await page
.locator(".mx_Toast_toast")
.getByRole("button", { name: "OK" })
.click();
} catch {
// toast not visible, continue as normal
}
}
public static async createRoom(
name: string,
page: Page,
andInvite: string[] = [],
): Promise<void> {
await page
.getByRole("navigation", { name: "Room list" })
.getByRole("button", { name: "New conversation" })
.click();
await page.getByRole("menuitem", { name: "New Room" }).click();
await page.getByRole("textbox", { name: "Name" }).fill(name);
await page.getByRole("button", { name: "Create room" }).click();
await expect(page.getByText("You created this room.")).toBeVisible();
await expect(page.getByText("Encryption enabled")).toBeVisible();
// Invite users if any
if (andInvite.length > 0) {
await page
.getByRole("button", { name: "Invite to this room", exact: true })
.click();
const inviteInput = page.getByRole("dialog").getByRole("textbox");
for (const mxId of andInvite) {
await inviteInput.focus();
await inviteInput.fill(mxId);
await inviteInput.press("Enter");
}
await page.getByRole("button", { name: "Invite" }).click();
}
}
/**
* Accepts a room invite using the room name.
* Locatest the invite in the room list.
*
*/
public static async acceptRoomInvite(
roomName: string,
page: Page,
): Promise<void> {
await expect(page.getByRole("option", { name: roomName })).toBeVisible();
await page.getByRole("option", { name: roomName }).click();
await page.getByRole("button", { name: "Accept" }).click();
await expect(
page.getByRole("main").getByRole("heading", { name: roomName }),
).toBeVisible();
}
/**
* Opens the widget and then goes to the settings to set the RTC mode.
* then closes the widget lobby.
*
* intended to be used before joining!
*
* WORKS IF A ROOM IS CURRENTLY OPENED IN THE PAGE
*/
public static async openWidgetSetEmbeddedElementCallRtcModeCloseWidget(
page: Page,
mode: RtcMode,
): Promise<void> {
await page.getByRole("button", { name: "Video call" }).click();
await page.getByRole("menuitem", { name: "Element Call" }).click();
await TestHelpers.setEmbeddedElementCallRtcMode(page, mode);
await page.getByRole("button", { name: "Close lobby" }).click();
}
/**
* Goes to the settings to set the RTC mode.
* then closes the settings modal.
*
* WORKS IF A ROOM IS CURRENTLY SHOWING THE EC WIDGET
*/
public static async setEmbeddedElementCallRtcMode(
page: Page,
mode: RtcMode,
): Promise<void> {
const iframe = page.locator('iframe[title="Element Call"]').contentFrame();
await iframe.getByRole("button", { name: "Settings" }).click();
await iframe.getByRole("tab", { name: "Preferences" }).click();
// await iframe.getByText("Developer mode", { exact: true }).click();
await iframe.getByText("Developer mode", { exact: true }).check(); // Idempotent: won't uncheck if already checked
// Move to Developer tab now
await iframe.getByRole("tab", { name: "Developer" }).click();
if (mode == "legacy") {
await iframe.getByText("Legacy: state events").click();
} else if (mode == "2_0") {
await iframe.getByText("Matrix 2.0").click();
} else {
// compat
await iframe.getByText("Compatibility: state events").click();
}
await iframe.getByTestId("modal_close").click();
}
/**
* Sets the current Element Web app to use the dev Element Call URL.
* @param page - The EW page
*/
public static async setDevToolElementCallDevUrl(page: Page): Promise<void> {
if (process.env.USE_DOCKER) {
await page.evaluate(() => {
window.mxSettingsStore.setValue(
"Developer.elementCallUrl",
null,
"device",
"https://call.m.localhost/room",
);
});
} else {
await page.evaluate(() => {
window.mxSettingsStore.setValue(
"Developer.elementCallUrl",
null,
"device",
"https://localhost:3000/room",
);
});
}
}
}

View File

@@ -0,0 +1,240 @@
/*
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { expect, test } from "@playwright/test";
import { widgetTest } from "../fixtures/widget-user.ts";
import { TestHelpers } from "./test-helpers.ts";
widgetTest.use({ callType: "dm" });
widgetTest(
"Start a new voice call in DM as widget",
async ({ asWidget, browserName }) => {
test.skip(
browserName === "firefox",
"The is test is not working on firefox CI environment. No mic/audio device inputs so cam/mic are disabled",
);
test.slow(); // Triples the timeout
const { brooks, whistler } = asWidget;
await TestHelpers.startCallInCurrentRoom(brooks.page, true);
await expect(
brooks.page.locator('iframe[title="Element Call"]'),
).toBeVisible();
const brooksFrame = brooks.page
.locator('iframe[title="Element Call"]')
.contentFrame();
// We should show a ringing overlay, let's check for that
await expect(
brooksFrame.getByText(`Waiting for ${whistler.displayName} to join…`),
).toBeVisible();
await expect(whistler.page.getByText("Incoming voice call")).toBeVisible();
await whistler.page.getByRole("button", { name: "Accept" }).click();
await expect(
whistler.page.locator('iframe[title="Element Call"]'),
).toBeVisible();
const whistlerFrame = whistler.page
.locator('iframe[title="Element Call"]')
.contentFrame();
// ASSERT the button states for whistler (the callee)
{
// The only way to know if it is muted or not is to look at the data-kind attribute..
const videoButton = whistlerFrame.getByTestId("incall_videomute");
// video should be off by default in a voice call
await expect(videoButton).toHaveAttribute("aria-label", /^Start video$/);
const audioButton = whistlerFrame.getByTestId("incall_mute");
// audio should be on for the voice call
await expect(audioButton).toHaveAttribute(
"aria-label",
/^Mute microphone$/,
);
}
// ASSERT the button states for brools (the caller)
{
// The only way to know if it is muted or not is to look at the data-kind attribute..
const videoButton = brooksFrame.getByTestId("incall_videomute");
// video should be off by default in a voice call
await expect(videoButton).toHaveAttribute("aria-label", /^Start video$/);
const audioButton = brooksFrame.getByTestId("incall_mute");
// audio should be on for the voice call
await expect(audioButton).toHaveAttribute(
"aria-label",
/^Mute microphone$/,
);
}
// In order to confirm that the call is disconnected we will check that the message composer is shown again.
// So first we need to confirm that it is hidden when in the call.
await expect(
whistler.page.locator(".mx_BasicMessageComposer"),
).not.toBeVisible();
await expect(
brooks.page.locator(".mx_BasicMessageComposer"),
).not.toBeVisible();
// ASSERT hanging up on one side ends the call for both
{
const hangupButton = brooksFrame.getByTestId("incall_leave");
await hangupButton.click();
}
// The widget should be closed on both sides and the timeline should be back on screen
await expect(
whistler.page.locator(".mx_BasicMessageComposer"),
).toBeVisible();
await expect(brooks.page.locator(".mx_BasicMessageComposer")).toBeVisible();
},
);
widgetTest(
"Start a new video call in DM as widget",
async ({ asWidget, browserName }) => {
test.skip(
browserName === "firefox",
"The is test is not working on firefox CI environment. No mic/audio device inputs so cam/mic are disabled",
);
test.slow(); // Triples the timeout
const { brooks, whistler } = asWidget;
await TestHelpers.startCallInCurrentRoom(brooks.page, false);
await expect(
brooks.page.locator('iframe[title="Element Call"]'),
).toBeVisible();
const brooksFrame = brooks.page
.locator('iframe[title="Element Call"]')
.contentFrame();
// We should show a ringing overlay, let's check for that
await expect(
brooksFrame.getByText(`Waiting for ${whistler.displayName} to join…`),
).toBeVisible();
await expect(whistler.page.getByText("Incoming video call")).toBeVisible();
await whistler.page.getByRole("button", { name: "Accept" }).click();
await expect(
whistler.page.locator('iframe[title="Element Call"]'),
).toBeVisible();
const whistlerFrame = whistler.page
.locator('iframe[title="Element Call"]')
.contentFrame();
// ASSERT the button states for whistler (the callee)
{
// The only way to know if it is muted or not is to look at the data-kind attribute..
const videoButton = whistlerFrame.getByTestId("incall_videomute");
// video should be on by default in a voice call
await expect(videoButton).toHaveAttribute("aria-label", /^Stop video$/);
const audioButton = whistlerFrame.getByTestId("incall_mute");
// audio should be on for the voice call
await expect(audioButton).toHaveAttribute(
"aria-label",
/^Mute microphone$/,
);
}
// ASSERT the button states for brools (the caller)
{
// The only way to know if it is muted or not is to look at the data-kind attribute..
const videoButton = brooksFrame.getByTestId("incall_videomute");
// video should be on by default in a voice call
await expect(videoButton).toHaveAttribute("aria-label", /^Stop video$/);
const audioButton = brooksFrame.getByTestId("incall_mute");
// audio should be on for the voice call
await expect(audioButton).toHaveAttribute(
"aria-label",
/^Mute microphone$/,
);
}
// In order to confirm that the call is disconnected we will check that the message composer is shown again.
// So first we need to confirm that it is hidden when in the call.
await expect(
whistler.page.locator(".mx_BasicMessageComposer"),
).not.toBeVisible();
await expect(
brooks.page.locator(".mx_BasicMessageComposer"),
).not.toBeVisible();
// ASSERT hanging up on one side ends the call for both
{
const hangupButton = brooksFrame.getByTestId("incall_leave");
await hangupButton.click();
}
// The widget should be closed on both sides and the timeline should be back on screen
await expect(
whistler.page.locator(".mx_BasicMessageComposer"),
).toBeVisible();
await expect(brooks.page.locator(".mx_BasicMessageComposer")).toBeVisible();
},
);
widgetTest(
"Decline a new video call in DM as widget",
async ({ asWidget, browserName }) => {
test.skip(
browserName === "firefox",
"The is test is not working on firefox CI environment. No mic/audio device inputs so cam/mic are disabled",
);
test.slow(); // Triples the timeout
const { brooks, whistler } = asWidget;
await TestHelpers.startCallInCurrentRoom(brooks.page, false);
await expect(
brooks.page.locator('iframe[title="Element Call"]'),
).toBeVisible();
const brooksFrame = brooks.page
.locator('iframe[title="Element Call"]')
.contentFrame();
// We should show a ringing overlay, let's check for that
await expect(
brooksFrame.getByText(`Waiting for ${whistler.displayName} to join…`),
).toBeVisible();
await expect(whistler.page.getByText("Incoming video call")).toBeVisible();
await whistler.page.getByRole("button", { name: "Decline" }).click();
await expect(
whistler.page.locator('iframe[title="Element Call"]'),
).not.toBeVisible();
// The widget should be closed and the timeline should be back on screen
await expect(
brooks.page.locator('iframe[title="Element Call"]'),
).not.toBeVisible();
await expect(
brooks.page.getByText("This is the beginning of your"),
).toBeVisible();
},
);

View File

@@ -1,4 +1,4 @@
# SDK mode
# SDK mode (EXPERIMENTAL)
EC can be build in sdk mode. This will result in a compiled js file that can be imported in very simple webapps.
@@ -6,18 +6,43 @@ It allows to use matrixRTC in combination with livekit without relying on elemen
This is done by instantiating the call view model and exposing some useful behaviors (observables) and methods.
This folder contains an example index.html file that showcases the sdk in use (hosted on localhost:8123 with a webserver ellowing cors (for example `npx serve -l 81234 --cors`)) as a godot engine HTML export template.
This folder contains an example index.html file that showcases the sdk in use (hosted on localhost:8123 with a webserver allowing cors (for example `npx serve -l 81234 --cors`)) as a godot engine HTML export template.
## Getting started
To get started run
```
yarn
yarn build:sdk
```
in the repository root.
It will create a `dist` folder containing the compiled js file.
This file needs to be hosted. Locally (via `npx serve -l 81234 --cors`) or on a remote server.
Now you just need to add the widget to element web via:
```
/addwidget http://localhost:3000?widgetId=$matrix_widget_id&perParticipantE2EE=true&userId=$matrix_user_id&deviceId=$org.matrix.msc3819.matrix_device_id&baseUrl=$org.matrix.msc4039.matrix_base_url&roomId=$matrix_room_id
```
## Widgets
The sdk mode is particularly interesting to be used in widgets where you do not need to pay attention to matrix login/cs api ...
To create a widget see the example index.html file in this folder. And add it to EW via:
The sdk mode is particularly interesting to be used in widgets. In widgets you do not need to pay attention to matrix login/cs api ...
To create a widget see the example `index.html` file in this folder. And add it to EW via:
`/addwidget <widgetUrl>` (see **url parameters** for more details on `<widgetUrl>`)
### url parameters
The url parameters are needed to pass initial data to the widget. They will automatically be used
by the matrixRTCSdk to start the postmessage widget api (communication between the client (e.g. Element Web) and the widget)
```
widgetId = $matrix_widget_id
perParticipantE2EE = true
userId = $matrix_user_id
deviceId = $org.matrix.msc3819.matrix_device_id
baseUrl = $org.matrix.msc4039.matrix_base_url
@@ -28,7 +53,7 @@ baseUrl = $org.matrix.msc4039.matrix_base_url
Full template use as `<widgetUrl>`:
```
http://localhost:3000?widgetId=$matrix_widget_id&userId=$matrix_user_id&deviceId=$org.matrix.msc3819.matrix_device_id&baseUrl=$org.matrix.msc4039.matrix_base_url&roomId=$matrix_room_id
http://localhost:3000?widgetId=$matrix_widget_id&perParticipantE2EE=true&userId=$matrix_user_id&deviceId=$org.matrix.msc3819.matrix_device_id&baseUrl=$org.matrix.msc4039.matrix_base_url&roomId=$matrix_room_id
```
the `$` prefixed variables will be replaced by EW on widget instantiation. (e.g. `$matrix_user_id` -> `@user:example.com` (url encoding will also be applied automatically by EW) -> `%40user%3Aexample.com`)

View File

@@ -12,15 +12,12 @@ Please see LICENSE in the repository root for full details.
import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
import { scan } from "rxjs";
import { widget as _widget } from "../src/widget";
import { type WidgetHelpers } from "../src/widget";
import { type LivekitRoomItem } from "../src/state/CallViewModel/CallViewModel";
export const logger = rootLogger.getChild("[MatrixRTCSdk]");
if (!_widget) throw Error("No widget. This webapp can only start as a widget");
export const widget = _widget;
export const tryMakeSticky = (): void => {
export const tryMakeSticky = (widget: WidgetHelpers): void => {
logger.info("try making sticky MatrixRTCSdk");
void widget.api
.setAlwaysOnScreen(true)

View File

@@ -1,7 +1,7 @@
<!doctype html>
<html>
<head>
<title>Godot MatrixRTC Widget</title>
<title>MatrixRTC Widget</title>
<meta charset="utf-8" />
<script type="module">
// TODO use the url where the matrixrtc-sdk.js file from dist is hosted
@@ -9,7 +9,7 @@
try {
window.matrixRTCSdk = await createMatrixRTCSdk(
"com.github.toger5.godot-game",
"com.github.toger5.rtc-application-type", // rtc application type
);
console.info("createMatrixRTCSdk was created!");
} catch (e) {
@@ -17,16 +17,6 @@
}
const sdk = window.matrixRTCSdk;
// This is the main bridging interface to godot
window.matrixRTCSdkGodot = {
dataObs: sdk.data$,
memberObs: sdk.members$,
// join: sdk.join, // lets stick with autojoin for now
sendData: sdk.sendData,
leave: sdk.leave,
connectedObs: sdk.connected$,
};
console.info("matrixRTCSdk join ", sdk);
const connectionState = sdk.join();
console.info("matrixRTCSdk joined");
@@ -38,15 +28,13 @@
const child = document.createElement("p");
child.innerHTML = JSON.stringify(data);
div.appendChild(child);
// TODO forward to godot
});
sdk.members$.subscribe((memberObjects) => {
// reset div
const div = document.getElementById("members");
div.innerHTML = "<h3>Members:</h3>";
// create member list
// Create member list
const members = memberObjects.map((member) => member.membership.sender);
console.info("members changed", members);
for (const m of members) {
@@ -62,22 +50,15 @@
const div = document.getElementById("connect_status");
div.innerHTML = connected ? "Connected" : "Disconnected";
});
let engine = new Engine($GODOT_CONFIG);
engine.startGame();
</script>
<!--// TODO use it as godot HTML template-->
<script src="$GODOT_URL"></script>
</head>
<body>
<canvas id="canvas"></canvas>
<div
id="overlay"
style="position: absolute; top: 0; right: 0; background-color: #ffffff10"
>
<div id="connect_status"></div>
<button onclick="window.matrixRTCSdk.leave();">Leave</button>
<button onclick="window.matrixRTCSdk.sendData({prop: 'Hello, world!'});">
<button onclick="window.matrixRTCSdk.leave()">Leave</button>
<button onclick="window.matrixRTCSdk.sendData({ prop: 'Hello, world!' })">
Send Text
</button>
<div id="members"></div>

View File

@@ -1,11 +1,13 @@
/*
Copyright 2025 Element Creations Ltd.
Copyright 2025-2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
/**
* EXPERIMENTAL
*
* This file is the entrypoint for the sdk build of element call: `yarn build:sdk`
* use in widgets.
* It exposes the `createMatrixRTCSdk` which creates the `MatrixRTCSdk` interface (see below) that
@@ -30,8 +32,8 @@ import {
} from "rxjs";
import {
type CallMembership,
MatrixRTCSession,
MatrixRTCSessionEvent,
MatrixRTCSessionManager,
} from "matrix-js-sdk/lib/matrixrtc";
import {
type Room as LivekitRoom,
@@ -50,14 +52,12 @@ import { getUrlParams } from "../src/UrlParams";
import { MuteStates } from "../src/state/MuteStates";
import { MediaDevices } from "../src/state/MediaDevices";
import { E2eeType } from "../src/e2ee/e2eeType";
import { currentAndPrev, logger, TEXT_LK_TOPIC, tryMakeSticky } from "./helper";
import {
currentAndPrev,
logger,
TEXT_LK_TOPIC,
tryMakeSticky,
widget,
} from "./helper";
import { ElementWidgetActions } from "../src/widget";
ElementWidgetActions,
widget as _widget,
initializeWidget,
} from "../src/widget";
import { type Connection } from "../src/state/CallViewModel/remoteMembers/Connection";
interface MatrixRTCSdk {
@@ -68,7 +68,13 @@ interface MatrixRTCSdk {
join: () => void;
/** @throws on leave errors */
leave: () => void;
data$: Observable<{ sender: string; data: string }>;
/**
* Ends the rtc sdk. This will unsubscribe any event listeners. And end the associated scope.
* No updates can be received from the rtc sdk. The sdk cannot be restarted after.
* A new sdk needs to be created via createMatrixRTCSdk.
*/
stop: () => void;
data$: Observable<{ rtcBackendIdentity: string; data: string }>;
/**
* flattened list of members
*/
@@ -79,34 +85,54 @@ interface MatrixRTCSdk {
participant: LocalParticipant | RemoteParticipant | null;
}[]
>;
/**
* flattened local members
*/
localMember$: Behavior<{
connection: Connection | null;
membership: CallMembership;
participant: LocalParticipant | null;
} | null>;
/** Use the LocalMemberConnectionState returned from `join` for a more detailed connection state */
connected$: Behavior<boolean>;
sendData?: (data: unknown) => Promise<void>;
sendRoomMessage?: (message: string) => Promise<void>;
}
export async function createMatrixRTCSdk(
application: string = "m.call",
id: string = "",
sticky: boolean = false,
): Promise<MatrixRTCSdk> {
logger.info("Hello");
const scope = new ObservableScope();
// widget client
initializeWidget(application, true);
const widget = _widget;
if (!widget) throw Error("No widget. This webapp can only start as a widget");
const client = await widget.client;
logger.info("client created");
const scope = new ObservableScope();
// url params
const { roomId } = getUrlParams();
if (roomId === null) throw Error("could not get roomId from url params");
const room = client.getRoom(roomId);
if (room === null) throw Error("could not get room from client");
const mediaDevices = new MediaDevices(scope);
const muteStates = new MuteStates(scope, mediaDevices, constant(true));
// rtc session
const slot = { application, id };
const rtcSession = new MatrixRTCSession(
client,
room,
MatrixRTCSession.sessionMembershipsForSlot(room, slot),
slot,
);
const rtcSessionManager = new MatrixRTCSessionManager(logger, client, slot);
rtcSessionManager.start();
const rtcSession = rtcSessionManager.getRoomSession(room);
// media devices
const mediaDevices = new MediaDevices(scope);
const muteStates = new MuteStates(scope, mediaDevices, {
audioEnabled: false,
videoEnabled: false,
});
// call view model
const callViewModel = createCallViewModel$(
scope,
rtcSession,
@@ -119,8 +145,9 @@ export async function createMatrixRTCSdk(
constant({ supported: false, processor: undefined }),
);
logger.info("CallViewModelCreated");
// create data listener
const data$ = new Subject<{ sender: string; data: string }>();
const data$ = new Subject<{ rtcBackendIdentity: string; data: string }>();
const lkTextStreamHandlerFunction = async (
reader: TextStreamReader,
@@ -142,7 +169,7 @@ export async function createMatrixRTCSdk(
if (participants && participants.includes(participantInfo.identity)) {
const text = await reader.readAll();
logger.info(`Received text: ${text}`);
data$.next({ sender: participantInfo.identity, data: text });
data$.next({ rtcBackendIdentity: participantInfo.identity, data: text });
} else {
logger.warn(
"Received text from unknown participant",
@@ -232,6 +259,16 @@ export async function createMatrixRTCSdk(
}
};
const sendRoomMessage = async (message: string): Promise<void> => {
const messageString = JSON.stringify(message);
logger.info("try sending to room: ", messageString);
try {
await client.sendTextMessage(room.roomId, message);
} catch (e) {
logger.error("failed sending to room: ", messageString, e);
}
};
// after hangup gets called
const leaveSubs = callViewModel.leave$.subscribe(() => {
const scheduleWidgetCloseOnLeave = async (): Promise<void> => {
@@ -259,9 +296,6 @@ export async function createMatrixRTCSdk(
// schedule close first and then leave (scope.end)
void scheduleWidgetCloseOnLeave();
// actual hangup (ending scope will send the leave event.. its kinda odd. since you might end up closing the widget too fast)
scope.end();
});
logger.info("createMatrixRTCSdk done");
@@ -269,15 +303,40 @@ export async function createMatrixRTCSdk(
return {
join: (): void => {
// first lets try making the widget sticky
tryMakeSticky();
if (sticky) tryMakeSticky(widget);
callViewModel.join();
},
leave: (): void => {
callViewModel.hangup();
callViewModel.leave();
},
stop: (): void => {
leaveSubs.unsubscribe();
livekitRoomItemsSub.unsubscribe();
scope.end();
},
data$,
localMember$: scope.behavior(
callViewModel.localMatrixLivekitMember$.pipe(
tap((member) =>
logger.info("localMatrixLivekitMember$ next: ", member),
),
switchMap((member) => {
if (member === null) return of(null);
return combineLatest([
member.connection$,
member.membership$,
member.participant.value$,
]).pipe(
map(([connection, membership, participant]) => ({
connection,
membership,
participant,
})),
);
}),
tap((member) => logger.info("localMember$ next: ", member)),
),
),
connected$: callViewModel.connected$,
members$: scope.behavior(
callViewModel.matrixLivekitMembers$.pipe(
@@ -304,5 +363,6 @@ export async function createMatrixRTCSdk(
[],
),
sendData,
sendRoomMessage,
};
}

View File

@@ -0,0 +1,147 @@
/* eslint-disable */
// The contents of this file below the line are copied from
// @types/dom-mediacapture-transform, which is inlined here into Element Call so
// that we can avoid the package's dependency on @types/dom-webcodecs, which is
// broken in TypeScript 5.9.
// (https://github.com/DefinitelyTyped/DefinitelyTyped/discussions/74294)
// If that issue is resolved, we can remove this file and return to depending on
// @types/dom-mediacapture-transform.
// -----------------------------------------------------------------------------
// This project is licensed under the MIT license.
// Copyrights are respective of each contributor listed at the beginning of each definition file.
// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
// The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
// In general, these types are only available behind a command line flag or an origin trial in
// Chrome 90+.
// This API depends on WebCodecs.
// Versioning:
// Until the above-mentioned spec is finalized, the major version number is 0. Although not
// necessary for version 0, consider incrementing the minor version number for breaking changes.
// The following modify existing DOM types to allow defining type-safe APIs on audio and video tracks.
/** Specialize MediaStreamTrack so that we can refer specifically to an audio track. */
interface MediaStreamAudioTrack extends MediaStreamTrack {
readonly kind: "audio";
clone(): MediaStreamAudioTrack;
}
/** Specialize MediaStreamTrack so that we can refer specifically to a video track. */
interface MediaStreamVideoTrack extends MediaStreamTrack {
readonly kind: "video";
clone(): MediaStreamVideoTrack;
}
/** Assert that getAudioTracks and getVideoTracks return the tracks with the appropriate kind. */
interface MediaStream {
getAudioTracks(): MediaStreamAudioTrack[];
getVideoTracks(): MediaStreamVideoTrack[];
}
// The following were originally generated from the spec using
// https://github.com/microsoft/TypeScript-DOM-lib-generator, then heavily modified.
/**
* A track sink that is capable of exposing the unencoded frames from the track to a
* ReadableStream, and exposes a control channel for signals going in the oppposite direction.
*/
interface MediaStreamTrackProcessor<T extends AudioData | VideoFrame> {
/**
* Allows reading the frames flowing through the MediaStreamTrack provided to the constructor.
*/
readonly readable: ReadableStream<T>;
/** Allows sending control signals to the MediaStreamTrack provided to the constructor. */
readonly writableControl: WritableStream<MediaStreamTrackSignal>;
}
declare var MediaStreamTrackProcessor: {
prototype: MediaStreamTrackProcessor<any>;
/** Constructor overrides based on the type of track. */
new (
init: MediaStreamTrackProcessorInit & { track: MediaStreamAudioTrack },
): MediaStreamTrackProcessor<AudioData>;
new (
init: MediaStreamTrackProcessorInit & { track: MediaStreamVideoTrack },
): MediaStreamTrackProcessor<VideoFrame>;
};
interface MediaStreamTrackProcessorInit {
track: MediaStreamTrack;
/**
* If media frames are not read from MediaStreamTrackProcessor.readable quickly enough, the
* MediaStreamTrackProcessor will internally buffer up to maxBufferSize of the frames produced
* by the track. If the internal buffer is full, each time the track produces a new frame, the
* oldest frame in the buffer will be dropped and the new frame will be added to the buffer.
*/
maxBufferSize?: number | undefined;
}
/**
* Takes video frames as input, and emits control signals that result from subsequent processing.
*/
interface MediaStreamTrackGenerator<
T extends AudioData | VideoFrame,
> extends MediaStreamTrack {
/**
* Allows writing media frames to the MediaStreamTrackGenerator, which is itself a
* MediaStreamTrack. When a frame is written to writable, the frames close() method is
* automatically invoked, so that its internal resources are no longer accessible from
* JavaScript.
*/
readonly writable: WritableStream<T>;
/**
* Allows reading control signals sent from any sinks connected to the
* MediaStreamTrackGenerator.
*/
readonly readableControl: ReadableStream<MediaStreamTrackSignal>;
}
type MediaStreamAudioTrackGenerator = MediaStreamTrackGenerator<AudioData> &
MediaStreamAudioTrack;
type MediaStreamVideoTrackGenerator = MediaStreamTrackGenerator<VideoFrame> &
MediaStreamVideoTrack;
declare var MediaStreamTrackGenerator: {
prototype: MediaStreamTrackGenerator<any>;
/** Constructor overrides based on the type of track. */
new (
init: MediaStreamTrackGeneratorInit & {
kind: "audio";
signalTarget?: MediaStreamAudioTrack | undefined;
},
): MediaStreamAudioTrackGenerator;
new (
init: MediaStreamTrackGeneratorInit & {
kind: "video";
signalTarget?: MediaStreamVideoTrack | undefined;
},
): MediaStreamVideoTrackGenerator;
};
interface MediaStreamTrackGeneratorInit {
kind: MediaStreamTrackGeneratorKind;
/**
* (Optional) track to which the MediaStreamTrackGenerator will automatically forward control
* signals. If signalTarget is provided and signalTarget.kind and kind do not match, the
* MediaStreamTrackGenerators constructor will raise an exception.
*/
signalTarget?: MediaStreamTrack | undefined;
}
type MediaStreamTrackGeneratorKind = "audio" | "video";
type MediaStreamTrackSignalType = "request-frame";
interface MediaStreamTrackSignal {
signalType: MediaStreamTrackSignalType;
}

View File

@@ -20,6 +20,7 @@ interface Props {
audio?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats;
video?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats;
focusUrl?: string;
rtcBackendIdentity?: string;
}
const extractDomain = (url: string): string => {
@@ -37,6 +38,7 @@ export const RTCConnectionStats: FC<Props> = ({
audio,
video,
focusUrl,
rtcBackendIdentity,
...rest
}) => {
const [showModal, setShowModal] = useState(false);
@@ -71,6 +73,9 @@ export const RTCConnectionStats: FC<Props> = ({
</pre>
</div>
</Modal>
<Text as="span" size="xs" title="rtcBackendIdentity">
rtcBackendIdentity:{rtcBackendIdentity}
</Text>
{focusUrl && (
<div>
<Text as="span" size="xs" title="focusURL">

View File

@@ -1,5 +1,6 @@
/*
Copyright 2023, 2024 New Vector Ltd.
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
@@ -223,6 +224,7 @@ describe("UrlParams", () => {
showControls: true,
hideScreensharing: false,
allowIceFallback: false,
perParticipantE2EE: false,
controlledAudioDevices: false,
skipLobby: false,
returnToLobby: false,
@@ -236,6 +238,7 @@ describe("UrlParams", () => {
showControls: true,
hideScreensharing: false,
allowIceFallback: true,
perParticipantE2EE: true,
controlledAudioDevices: platform === "desktop" ? false : true,
skipLobby: true,
returnToLobby: false,
@@ -249,12 +252,11 @@ describe("UrlParams", () => {
showControls: true,
hideScreensharing: false,
allowIceFallback: true,
perParticipantE2EE: true,
controlledAudioDevices: platform === "desktop" ? false : true,
skipLobby: false,
returnToLobby: false,
sendNotificationType: "notification",
defaultAudioEnabled: true,
defaultVideoEnabled: true,
});
it("use no-intent-defaults with unknown intent", () => {
expect(computeUrlParams()).toMatchObject(noIntentDefaults);
@@ -392,8 +394,6 @@ describe("UrlParams", () => {
expect.any(Object),
"configuration:",
expect.any(Object),
"intentAndPlatformDerivedConfiguration:",
{},
);
});
});

View File

@@ -1,5 +1,6 @@
/*
Copyright 2022-2024 New Vector Ltd.
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
@@ -190,6 +191,10 @@ export interface UrlConfiguration {
*/
allowIceFallback: boolean;
/**
* Whether the app should use per participant keys for E2EE.
*/
perParticipantE2EE: boolean;
/**
* Whether the global JS controls for audio output devices should be enabled,
* allowing the list of output devices to be controlled by the app hosting
@@ -242,24 +247,13 @@ export interface UrlConfiguration {
callIntent?: RTCCallIntent;
}
interface IntentAndPlatformDerivedConfiguration {
defaultAudioEnabled?: boolean;
defaultVideoEnabled?: boolean;
}
interface IntentAndPlatformDerivedConfiguration {
defaultAudioEnabled?: boolean;
defaultVideoEnabled?: boolean;
}
// If you need to add a new flag to this interface, prefer a name that describes
// a specific behavior (such as 'confineToRoom'), rather than one that describes
// the situations that call for this behavior ('isEmbedded'). This makes it
// clearer what each flag means, and helps us avoid coupling Element Call's
// behavior to the needs of specific consumers.
export interface UrlParams
extends UrlProperties,
UrlConfiguration,
IntentAndPlatformDerivedConfiguration {}
export interface UrlParams extends UrlProperties, UrlConfiguration {}
// This is here as a stopgap, but what would be far nicer is a function that
// takes a UrlParams and returns a query string. That would enable us to
@@ -402,6 +396,7 @@ export const computeUrlParams = (search = "", hash = ""): UrlParams => {
showControls: true,
hideScreensharing: false,
allowIceFallback: true,
perParticipantE2EE: true,
controlledAudioDevices: platform === "desktop" ? false : true,
skipLobby: true,
returnToLobby: false,
@@ -448,6 +443,7 @@ export const computeUrlParams = (search = "", hash = ""): UrlParams => {
showControls: true,
hideScreensharing: false,
allowIceFallback: false,
perParticipantE2EE: false,
controlledAudioDevices: false,
skipLobby: false,
returnToLobby: false,
@@ -457,29 +453,6 @@ export const computeUrlParams = (search = "", hash = ""): UrlParams => {
};
}
const intentAndPlatformDerivedConfiguration: IntentAndPlatformDerivedConfiguration =
{};
// Desktop also includes web. Its anything that is not mobile.
const desktopMobile = platform === "desktop" ? "desktop" : "mobile";
switch (desktopMobile) {
case "desktop":
case "mobile":
switch (intent) {
case UserIntent.StartNewCall:
case UserIntent.JoinExistingCall:
case UserIntent.StartNewCallDM:
case UserIntent.JoinExistingCallDM:
intentAndPlatformDerivedConfiguration.defaultAudioEnabled = true;
intentAndPlatformDerivedConfiguration.defaultVideoEnabled = true;
break;
case UserIntent.StartNewCallDMVoice:
case UserIntent.JoinExistingCallDMVoice:
intentAndPlatformDerivedConfiguration.defaultAudioEnabled = true;
intentAndPlatformDerivedConfiguration.defaultVideoEnabled = false;
break;
}
}
const properties: UrlProperties = {
widgetId,
parentUrl,
@@ -518,6 +491,7 @@ export const computeUrlParams = (search = "", hash = ""): UrlParams => {
showControls: parser.getFlag("showControls"),
hideScreensharing: parser.getFlag("hideScreensharing"),
allowIceFallback: parser.getFlag("allowIceFallback"),
perParticipantE2EE: parser.getFlag("perParticipantE2EE"),
controlledAudioDevices: parser.getFlag("controlledAudioDevices"),
skipLobby: isWidget ? parser.getFlag("skipLobby") : false,
// In SPA mode the user should always exit to the home screen when hanging
@@ -543,15 +517,12 @@ export const computeUrlParams = (search = "", hash = ""): UrlParams => {
properties,
"configuration:",
configuration,
"intentAndPlatformDerivedConfiguration:",
intentAndPlatformDerivedConfiguration,
);
return {
...properties,
...intentPreset,
...pickBy(configuration, (v?: unknown) => v !== undefined),
...intentAndPlatformDerivedConfiguration,
};
};
@@ -656,6 +627,9 @@ export function generateUrlSearchParams(
params.set("password", encodedPassword);
break;
}
case E2eeType.PER_PARTICIPANT:
params.set("perParticipantE2EE", "true");
break;
}
params.set("roomId", roomId);
viaServers?.forEach((s) => params.set("viaServers", s));

View File

@@ -13,7 +13,7 @@ exports[`AppBar > renders 1`] = `
class="nav leftNav"
>
<button
aria-labelledby="«r0»"
aria-labelledby="_r_0_"
class="_icon-button_1pz9o_8"
data-kind="primary"
role="button"
@@ -21,7 +21,7 @@ exports[`AppBar > renders 1`] = `
tabindex="0"
>
<div
class="_indicator-icon_zr2a0_17"
class="_indicator-icon_147l5_17"
style="--cpd-icon-button-size: 100%;"
>
<svg

View File

@@ -2,10 +2,10 @@
exports[`the content is rendered when the modal is open 1`] = `
<div
aria-labelledby="radix-«r4»"
aria-labelledby="radix-_r_4_"
class="overlay animate modal dialog _glass_sepwu_8"
data-state="open"
id="radix-«r3»"
id="radix-_r_3_"
role="dialog"
style="pointer-events: auto;"
tabindex="-1"
@@ -18,7 +18,7 @@ exports[`the content is rendered when the modal is open 1`] = `
>
<h2
class="_typography_6v6n8_153 _font-heading-md-semibold_6v6n8_112"
id="radix-«r4»"
id="radix-_r_4_"
>
My modal
</h2>
@@ -36,7 +36,7 @@ exports[`the content is rendered when the modal is open 1`] = `
exports[`the modal renders as a drawer in mobile viewports 1`] = `
<div
aria-labelledby="radix-«ra»"
aria-labelledby="radix-_r_a_"
class="overlay modal drawer"
data-state="open"
data-vaul-animate="true"
@@ -45,7 +45,7 @@ exports[`the modal renders as a drawer in mobile viewports 1`] = `
data-vaul-drawer=""
data-vaul-drawer-direction="bottom"
data-vaul-snap-points="false"
id="radix-«r9»"
id="radix-_r_9_"
role="dialog"
style="pointer-events: auto;"
tabindex="-1"
@@ -60,7 +60,7 @@ exports[`the modal renders as a drawer in mobile viewports 1`] = `
class="handle"
/>
<h2
id="radix-«ra»"
id="radix-_r_a_"
style="position: absolute; border: 0px; width: 1px; height: 1px; padding: 0px; margin: -1px; overflow: hidden; clip: rect(0px, 0px, 0px, 0px); white-space: nowrap; word-wrap: normal;"
>
My modal

View File

@@ -2,10 +2,10 @@
exports[`Toast > renders 1`] = `
<button
aria-labelledby="radix-«r4»"
aria-labelledby="radix-_r_4_"
class="overlay animate toast"
data-state="open"
id="radix-«r3»"
id="radix-_r_3_"
role="dialog"
style="pointer-events: auto;"
tabindex="-1"
@@ -13,7 +13,7 @@ exports[`Toast > renders 1`] = `
>
<h3
class="_typography_6v6n8_153 _font-body-sm-semibold_6v6n8_36"
id="radix-«r4»"
id="radix-_r_4_"
>
Hello world!
</h3>

View File

@@ -1,157 +0,0 @@
/*
Copyright 2023, 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
type SpanProcessor,
type ReadableSpan,
type Span,
} from "@opentelemetry/sdk-trace-base";
import { hrTimeToMilliseconds } from "@opentelemetry/core";
import { logger } from "matrix-js-sdk/lib/logger";
import { PosthogAnalytics } from "./PosthogAnalytics";
interface PrevCall {
callId: string;
hangupTs: number;
}
/**
* The maximum time between hanging up and joining the same call that we would
* consider a 'rejoin' on the user's part.
*/
const maxRejoinMs = 2 * 60 * 1000; // 2 minutes
/**
* Span processor that extracts certain metrics from spans to send to PostHog
*/
export class PosthogSpanProcessor implements SpanProcessor {
public async forceFlush(): Promise<void> {}
public onStart(span: Span): void {
// Hack: Yield to allow attributes to be set before processing
try {
switch (span.name) {
case "matrix.groupCallMembership":
this.onGroupCallMembershipStart(span);
return;
case "matrix.groupCallMembership.summaryReport":
this.onSummaryReportStart(span);
return;
}
} catch (e) {
// log to avoid tripping @typescript-eslint/no-unused-vars
logger.debug(e);
}
}
public onEnd(span: ReadableSpan): void {
switch (span.name) {
case "matrix.groupCallMembership":
this.onGroupCallMembershipEnd(span);
return;
}
}
private get prevCall(): PrevCall | null {
// This is stored in localStorage so we can remember the previous call
// across app restarts
const data = localStorage.getItem("matrix-prev-call");
if (data === null) return null;
try {
return JSON.parse(data);
} catch (e) {
logger.warn("Invalid prev call data", data, "error:", e);
return null;
}
}
private set prevCall(data: PrevCall | null) {
localStorage.setItem("matrix-prev-call", JSON.stringify(data));
}
private onGroupCallMembershipStart(span: ReadableSpan): void {
const prevCall = this.prevCall;
const newCallId = span.attributes["matrix.confId"] as string;
// If the user joined the same call within a short time frame, log this as a
// rejoin. This is interesting as a call quality metric, since rejoins may
// indicate that users had to intervene to make the product work.
if (prevCall !== null && newCallId === prevCall.callId) {
const duration = hrTimeToMilliseconds(span.startTime) - prevCall.hangupTs;
if (duration <= maxRejoinMs) {
PosthogAnalytics.instance.trackEvent({
eventName: "Rejoin",
callId: prevCall.callId,
rejoinDuration: duration,
});
}
}
}
private onGroupCallMembershipEnd(span: ReadableSpan): void {
this.prevCall = {
callId: span.attributes["matrix.confId"] as string,
hangupTs: hrTimeToMilliseconds(span.endTime),
};
}
private onSummaryReportStart(span: ReadableSpan): void {
// Searching for an event like this:
// matrix.stats.summary
// matrix.stats.summary.percentageReceivedAudioMedia: 0.75
// matrix.stats.summary.percentageReceivedMedia: 1
// matrix.stats.summary.percentageReceivedVideoMedia: 0.75
// matrix.stats.summary.maxJitter: 100
// matrix.stats.summary.maxPacketLoss: 20
const event = span.events.find((e) => e.name === "matrix.stats.summary");
if (event !== undefined) {
const attributes = event.attributes;
if (attributes) {
const mediaReceived = `${attributes["matrix.stats.summary.percentageReceivedMedia"]}`;
const videoReceived = `${attributes["matrix.stats.summary.percentageReceivedVideoMedia"]}`;
const audioReceived = `${attributes["matrix.stats.summary.percentageReceivedAudioMedia"]}`;
const maxJitter = `${attributes["matrix.stats.summary.maxJitter"]}`;
const maxPacketLoss = `${attributes["matrix.stats.summary.maxPacketLoss"]}`;
const peerConnections = `${attributes["matrix.stats.summary.peerConnections"]}`;
const percentageConcealedAudio = `${attributes["matrix.stats.summary.percentageConcealedAudio"]}`;
const opponentUsersInCall = `${attributes["matrix.stats.summary.opponentUsersInCall"]}`;
const opponentDevicesInCall = `${attributes["matrix.stats.summary.opponentDevicesInCall"]}`;
const diffDevicesToPeerConnections = `${attributes["matrix.stats.summary.diffDevicesToPeerConnections"]}`;
const ratioPeerConnectionToDevices = `${attributes["matrix.stats.summary.ratioPeerConnectionToDevices"]}`;
PosthogAnalytics.instance.trackEvent(
{
eventName: "MediaReceived",
callId: span.attributes["matrix.confId"] as string,
mediaReceived: mediaReceived,
audioReceived: audioReceived,
videoReceived: videoReceived,
maxJitter: maxJitter,
maxPacketLoss: maxPacketLoss,
peerConnections: peerConnections,
percentageConcealedAudio: percentageConcealedAudio,
opponentUsersInCall: opponentUsersInCall,
opponentDevicesInCall: opponentDevicesInCall,
diffDevicesToPeerConnections: diffDevicesToPeerConnections,
ratioPeerConnectionToDevices: ratioPeerConnectionToDevices,
},
// Send instantly because the window might be closing
{ send_instantly: true },
);
}
}
}
/**
* Shutdown the processor.
*/
public async shutdown(): Promise<void> {
return Promise.resolve();
}
}

View File

@@ -1,135 +0,0 @@
/*
Copyright 2023, 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type AttributeValue, type Attributes } from "@opentelemetry/api";
import { hrTimeToMicroseconds } from "@opentelemetry/core";
import {
type SpanProcessor,
type ReadableSpan,
type Span,
} from "@opentelemetry/sdk-trace-base";
const dumpAttributes = (
attr: Attributes,
): {
key: string;
type:
| "string"
| "number"
| "bigint"
| "boolean"
| "symbol"
| "undefined"
| "object"
| "function";
value: AttributeValue | undefined;
}[] =>
Object.entries(attr).map(([key, value]) => ({
key,
type: typeof value,
value,
}));
/**
* Exports spans on demand to the Jaeger JSON format, which can be attached to
* rageshakes and loaded into analysis tools like Jaeger and Stalk.
*/
export class RageshakeSpanProcessor implements SpanProcessor {
private readonly spans: ReadableSpan[] = [];
public async forceFlush(): Promise<void> {}
public onStart(span: Span): void {
this.spans.push(span);
}
public onEnd(): void {}
/**
* Dumps the spans collected so far as Jaeger-compatible JSON.
*/
public dump(): string {
const now = Date.now() * 1000; // Jaeger works in microseconds
const traces = new Map<string, ReadableSpan[]>();
// Organize spans by their trace IDs
for (const span of this.spans) {
const traceId = span.spanContext().traceId;
let trace = traces.get(traceId);
if (trace === undefined) {
trace = [];
traces.set(traceId, trace);
}
trace.push(span);
}
const processId = "p1";
const processes = {
[processId]: {
serviceName: "element-call",
tags: [],
},
warnings: null,
};
return JSON.stringify({
// Honestly not sure what some of these fields mean, I just know that
// they're present in Jaeger JSON exports
total: 0,
limit: 0,
offset: 0,
errors: null,
data: [...traces.entries()].map(([traceId, spans]) => ({
traceID: traceId,
warnings: null,
processes,
spans: spans.map((span) => {
const ctx = span.spanContext();
const startTime = hrTimeToMicroseconds(span.startTime);
// If the span has not yet ended, pretend that it ends now
const duration =
span.duration[0] === -1
? now - startTime
: hrTimeToMicroseconds(span.duration);
return {
traceID: traceId,
spanID: ctx.spanId,
operationName: span.name,
processID: processId,
warnings: null,
startTime,
duration,
references:
span.parentSpanContext?.spanId === undefined
? []
: [
{
refType: "CHILD_OF",
traceID: traceId,
spanID: span.parentSpanContext?.spanId,
},
],
tags: dumpAttributes(span.attributes),
logs: span.events.map((event) => ({
timestamp: hrTimeToMicroseconds(event.time),
// The name of the event is in the "event" field, aparently.
fields: [
...dumpAttributes(event.attributes ?? {}),
{ key: "event", type: "string", value: event.name },
],
})),
};
}),
})),
});
}
public async shutdown(): Promise<void> {}
}

View File

@@ -1,5 +1,6 @@
/*
Copyright 2022-2024 New Vector Ltd.
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
@@ -35,6 +36,7 @@ export const MicButton: FC<MicButtonProps> = ({ muted, ...props }) => {
<Tooltip label={label}>
<CpdButton
iconOnly
aria-label={label}
Icon={Icon}
kind={muted ? "primary" : "secondary"}
{...props}
@@ -58,6 +60,7 @@ export const VideoButton: FC<VideoButtonProps> = ({ muted, ...props }) => {
<Tooltip label={label}>
<CpdButton
iconOnly
aria-label={label}
Icon={Icon}
kind={muted ? "primary" : "secondary"}
{...props}
@@ -102,6 +105,7 @@ export const EndCallButton: FC<ComponentPropsWithoutRef<"button">> = ({
<CpdButton
className={classNames(className, styles.endCall)}
iconOnly
aria-label={t("hangup_button_label")}
Icon={EndCallIcon}
destructive
{...props}

View File

@@ -18,7 +18,8 @@ import { getBasicCallViewModelEnvironment } from "../utils/test-viewmodel";
import { alice, local, localRtcMember } from "../utils/test-fixtures";
import { type MockRTCSession } from "../utils/test";
import { ReactionsSenderProvider } from "../reactions/useReactionsSender";
import { initializeWidget } from "../widget";
initializeWidget();
vi.mock("livekit-client/e2ee-worker?worker");
const localIdent = `${localRtcMember.userId}:${localRtcMember.deviceId}`;

View File

@@ -9,8 +9,8 @@ exports[`Can close reaction dialog 1`] = `
aria-disabled="false"
aria-expanded="true"
aria-haspopup="true"
aria-labelledby="«rbb»"
class="_button_vczzf_8 _has-icon_vczzf_57 _icon-only_vczzf_50"
aria-labelledby="_r_bb_"
class="_button_13vu4_8 _has-icon_13vu4_60 _icon-only_13vu4_53"
data-kind="primary"
data-size="lg"
role="button"
@@ -43,8 +43,8 @@ exports[`Can fully expand emoji picker 1`] = `
aria-disabled="false"
aria-expanded="true"
aria-haspopup="true"
aria-labelledby="«r7m»"
class="_button_vczzf_8 _has-icon_vczzf_57 _icon-only_vczzf_50"
aria-labelledby="_r_7m_"
class="_button_13vu4_8 _has-icon_13vu4_60 _icon-only_13vu4_53"
data-kind="primary"
data-size="lg"
role="button"
@@ -74,8 +74,8 @@ exports[`Can lower hand 1`] = `
aria-disabled="false"
aria-expanded="false"
aria-haspopup="true"
aria-labelledby="«r36»"
class="_button_vczzf_8 _has-icon_vczzf_57 _icon-only_vczzf_50"
aria-labelledby="_r_36_"
class="_button_13vu4_8 _has-icon_13vu4_60 _icon-only_13vu4_53"
data-kind="secondary"
data-size="lg"
role="button"
@@ -108,8 +108,8 @@ exports[`Can open menu 1`] = `
aria-disabled="false"
aria-expanded="true"
aria-haspopup="true"
aria-labelledby="«r0»"
class="_button_vczzf_8 _has-icon_vczzf_57 _icon-only_vczzf_50"
aria-labelledby="_r_0_"
class="_button_13vu4_8 _has-icon_13vu4_60 _icon-only_13vu4_53"
data-kind="primary"
data-size="lg"
role="button"
@@ -139,8 +139,8 @@ exports[`Can raise hand 1`] = `
aria-disabled="false"
aria-expanded="false"
aria-haspopup="true"
aria-labelledby="«r1j»"
class="_button_vczzf_8 raisedButton _has-icon_vczzf_57 _icon-only_vczzf_50"
aria-labelledby="_r_1j_"
class="_button_13vu4_8 raisedButton _has-icon_13vu4_60 _icon-only_13vu4_53"
data-kind="primary"
data-size="lg"
role="button"

View File

@@ -1,5 +1,6 @@
/*
Copyright 2022-2024 New Vector Ltd.
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
@@ -163,10 +164,6 @@ export interface ResolvedConfigOptions extends ConfigOptions {
};
};
ssla: string;
media_devices: {
enable_audio: boolean;
enable_video: boolean;
};
app_prompt: boolean;
}
@@ -181,9 +178,5 @@ export const DEFAULT_CONFIG: ResolvedConfigOptions = {
feature_use_device_session_member_events: true,
},
ssla: "https://static.element.io/legal/element-software-and-services-license-agreement-uk-1.pdf",
media_devices: {
enable_audio: true,
enable_video: true,
},
app_prompt: true,
};

View File

@@ -6,11 +6,13 @@ Please see LICENSE in the repository root for full details.
*/
import { BaseKeyProvider } from "livekit-client";
import { logger } from "matrix-js-sdk/lib/logger";
import {
type MatrixRTCSession,
MatrixRTCSessionEvent,
} from "matrix-js-sdk/lib/matrixrtc";
import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
const logger = rootLogger.getChild("[MatrixKeyProvider]");
export class MatrixKeyProvider extends BaseKeyProvider {
private rtcSession?: MatrixRTCSession;
@@ -40,9 +42,10 @@ export class MatrixKeyProvider extends BaseKeyProvider {
}
private onEncryptionKeyChanged = (
encryptionKey: Uint8Array,
encryptionKey: Uint8Array<ArrayBuffer>,
encryptionKeyIndex: number,
participantId: string,
membershipParts: CallMembershipIdentityParts,
rtcBackendIdentity: string,
): void => {
crypto.subtle
.importKey("raw", encryptionKey, "HKDF", false, [
@@ -53,17 +56,17 @@ export class MatrixKeyProvider extends BaseKeyProvider {
(keyMaterial) => {
this.onSetEncryptionKey(
keyMaterial,
participantId,
rtcBackendIdentity,
encryptionKeyIndex,
);
logger.debug(
`Sent new key to livekit room=${this.rtcSession?.room.roomId} participantId=${participantId} encryptionKeyIndex=${encryptionKeyIndex}`,
`Sent new key to livekit room=${this.rtcSession?.room.roomId} participantId=${rtcBackendIdentity} (before hash: ${membershipParts.userId}:${membershipParts.deviceId}) encryptionKeyIndex=${encryptionKeyIndex}`,
);
},
(e) => {
logger.error(
`Failed to create key material from buffer for livekit room=${this.rtcSession?.room.roomId} participantId=${participantId} encryptionKeyIndex=${encryptionKeyIndex}`,
`Failed to create key material from buffer for livekit room=${this.rtcSession?.room.roomId} participantId before hash=${membershipParts.userId}:${membershipParts.deviceId} encryptionKeyIndex=${encryptionKeyIndex}`,
e,
);
},

View File

@@ -34,8 +34,8 @@ const getRoomSharedKeyLocalStorageKey = (roomId: string): string =>
`room-shared-key-${roomId}`;
/**
* An upto-date shared key for the room. Either from local storage or the value from `setInitialValue`.
* @param roomId
* An up-to-date shared key for the room. Either from local storage or the value from `setInitialValue`.
* @param roomId The room ID we want the shared key for.
* @param setInitialValue The value we get from the URL. The hook will overwrite the local storage value with this.
* @returns [roomSharedKey, setRoomSharedKey] like a react useState hook.
*/

View File

@@ -200,8 +200,11 @@ interface Drag {
export type DragCallback = (drag: Drag) => void;
interface LayoutMemoProps<LayoutModel, TileModel, R extends HTMLElement>
extends LayoutProps<LayoutModel, TileModel, R> {
interface LayoutMemoProps<
LayoutModel,
TileModel,
R extends HTMLElement,
> extends LayoutProps<LayoutModel, TileModel, R> {
Layout: ComponentType<LayoutProps<LayoutModel, TileModel, R>>;
}

View File

@@ -16,7 +16,7 @@ import LanguageDetector from "i18next-browser-languagedetector";
import * as Sentry from "@sentry/react";
import { logger } from "matrix-js-sdk/lib/logger";
import { shouldPolyfill as shouldPolyfillSegmenter } from "@formatjs/intl-segmenter/should-polyfill";
import { shouldPolyfill as shouldPolyfillDurationFormat } from "@formatjs/intl-durationformat/should-polyfill";
import { shouldPolyfill as shouldPolyfillDurationFormat } from "@formatjs/intl-durationformat/should-polyfill.js";
import {
useLocation,
useNavigationType,
@@ -26,9 +26,9 @@ import {
import { getUrlParams } from "./UrlParams";
import { Config } from "./config/Config";
import { ElementCallOpenTelemetry } from "./otel/otel";
import { platform } from "./Platform";
import { isFailure } from "./utils/fetch";
import { initializeWidget } from "./widget";
// This generates a map of locale names to their URL (based on import.meta.url), which looks like this:
// {
@@ -101,7 +101,6 @@ enum LoadState {
class DependencyLoadStates {
public config: LoadState = LoadState.None;
public sentry: LoadState = LoadState.None;
public openTelemetry: LoadState = LoadState.None;
public allDepsAreLoaded(): boolean {
return !Object.values(this).some((s) => s !== LoadState.Loaded);
@@ -117,13 +116,15 @@ export class Initializer {
}
public static async initBeforeReact(): Promise<void> {
initializeWidget();
const polyfills: Promise<unknown>[] = [];
if (shouldPolyfillSegmenter()) {
polyfills.push(import("@formatjs/intl-segmenter/polyfill-force"));
}
if (shouldPolyfillDurationFormat()) {
polyfills.push(import("@formatjs/intl-durationformat/polyfill-force"));
polyfills.push(import("@formatjs/intl-durationformat/polyfill-force.js"));
}
await Promise.all(polyfills);
@@ -266,15 +267,6 @@ export class Initializer {
this.loadStates.sentry = LoadState.Loaded;
}
// OpenTelemetry (also only after config loaded)
if (
this.loadStates.openTelemetry === LoadState.None &&
this.loadStates.config === LoadState.Loaded
) {
ElementCallOpenTelemetry.globalInit();
this.loadStates.openTelemetry = LoadState.Loaded;
}
if (this.loadStates.allDepsAreLoaded()) {
// resolve if there is no dependency that is not loaded
resolve();

View File

@@ -28,7 +28,8 @@ import {
mockRemoteParticipant,
mockTrack,
} from "../utils/test";
import { initializeWidget } from "../widget";
initializeWidget();
export const TestAudioContextConstructor = vi.fn(() => testAudioContext);
const MediaDevicesProvider = MediaDevicesContext.MediaDevicesContext.Provider;

View File

@@ -15,7 +15,6 @@ import {
type AudioTrackProps,
} from "@livekit/components-react";
import { logger } from "matrix-js-sdk/lib/logger";
import { type ParticipantId } from "matrix-js-sdk/lib/matrixrtc";
import { useEarpieceAudioConfig } from "../MediaDevicesContext";
import { useReactiveState } from "../useReactiveState";
@@ -32,7 +31,7 @@ export interface MatrixAudioRendererProps {
* This list needs to be composed based on the matrixRTC members so that we do not play audio from users
* that are not expected to be in the rtc session (local user is excluded).
*/
validIdentities: ParticipantId[];
validIdentities: string[];
/**
* If set to `true`, mutes all audio tracks rendered by the component.
* @remarks
@@ -79,6 +78,7 @@ export function LivekitRoomAudioRenderer({
.filter((ref) => {
const isValid = validIdentities.includes(ref.participant.identity);
if (!isValid) {
// TODO make sure to also skip the warn logging for the local identity
// Log that there is an invalid identity, that means that someone is publishing audio that is not expected to be in the call.
prefixedLogger.warn(
`Audio track ${ref.participant.identity} from ${url} has no matching matrix call member`,
@@ -166,7 +166,11 @@ interface StereoPanAudioTrackProps {
* It main purpose is to remount the AudioTrack component when switching from
* audioContext to normal audio playback.
* As of now the AudioTrack component does not support adding audio nodes while being mounted.
* @param param0
* @param props The component props
* @param props.trackRef The track reference
* @param props.muted If the track should be muted
* @param props.audioContext The audio context to use
* @param props.audioNodes The audio nodes to use
* @returns
*/
function AudioTrackWithAudioNodes({

View File

@@ -18,6 +18,7 @@ import fetchMock from "fetch-mock";
import { getSFUConfigWithOpenID, type OpenIDClientParts } from "./openIDSFU";
import { testJWTToken } from "../utils/test-fixtures";
import { ownMemberMock } from "../utils/test";
const sfuUrl = "https://sfu.example.org";
@@ -33,6 +34,7 @@ describe("getSFUConfigWithOpenID", () => {
vitest.clearAllMocks();
fetchMock.reset();
});
it("should handle fetching a token", async () => {
fetchMock.post("https://sfu.example.org/sfu/get", () => {
return {
@@ -42,6 +44,7 @@ describe("getSFUConfigWithOpenID", () => {
});
const config = await getSFUConfigWithOpenID(
matrixClient,
ownMemberMock,
"https://sfu.example.org",
"!example_room_id",
);
@@ -53,6 +56,7 @@ describe("getSFUConfigWithOpenID", () => {
});
void (await fetchMock.flush());
});
it("should fail if the SFU errors", async () => {
fetchMock.post("https://sfu.example.org/sfu/get", () => {
return {
@@ -63,11 +67,12 @@ describe("getSFUConfigWithOpenID", () => {
try {
await getSFUConfigWithOpenID(
matrixClient,
ownMemberMock,
"https://sfu.example.org",
"!example_room_id",
);
} catch (ex) {
expect(((ex as Error).cause as Error).message).toEqual(
expect((ex as Error).message).toEqual(
"SFU Config fetch failed with status code 500",
);
void (await fetchMock.flush());
@@ -76,6 +81,104 @@ describe("getSFUConfigWithOpenID", () => {
expect.fail("Expected test to throw;");
});
it("should try legacy and then new endpoint with delay delegation", async () => {
fetchMock.post("https://sfu.example.org/get_token", () => {
return {
status: 500,
body: { error: "Test failure" },
};
});
fetchMock.post("https://sfu.example.org/sfu/get", () => {
return {
status: 500,
body: { error: "Test failure" },
};
});
try {
await getSFUConfigWithOpenID(
matrixClient,
ownMemberMock,
"https://sfu.example.org",
"!example_room_id",
{
delayEndpointBaseUrl: "https://matrix.homeserverserver.org",
delayId: "mock_delay_id",
},
);
} catch (ex) {
expect((ex as Error).message).toEqual(
"SFU Config fetch failed with status code 500",
);
void (await fetchMock.flush());
}
const calls = fetchMock.calls();
expect(calls.length).toBe(2);
expect(calls[0][0]).toStrictEqual("https://sfu.example.org/get_token");
expect(calls[0][1]).toStrictEqual({
// check if it uses correct delayID!
body: '{"room_id":"!example_room_id","slot_id":"m.call#ROOM","member":{"id":"@alice:example.org:DEVICE","claimed_user_id":"@alice:example.org","claimed_device_id":"DEVICE"},"delay_id":"mock_delay_id","delay_timeout":1000,"delay_cs_api_url":"https://matrix.homeserverserver.org"}',
method: "POST",
headers: {
"Content-Type": "application/json",
},
});
expect(calls[1][0]).toStrictEqual("https://sfu.example.org/sfu/get");
expect(calls[1][1]).toStrictEqual({
body: '{"room":"!example_room_id","device_id":"DEVICE"}',
headers: {
"Content-Type": "application/json",
},
method: "POST",
});
});
it("dont try legacy if endpoint with delay delegation is sucessful", async () => {
fetchMock.post("https://sfu.example.org/get_token", () => {
return {
status: 200,
body: { url: sfuUrl, jwt: testJWTToken },
};
});
fetchMock.post("https://sfu.example.org/sfu/get", () => {
return {
status: 500,
body: { error: "Test failure" },
};
});
try {
await getSFUConfigWithOpenID(
matrixClient,
ownMemberMock,
"https://sfu.example.org",
"!example_room_id",
{
delayEndpointBaseUrl: "https://matrix.homeserverserver.org",
delayId: "mock_delay_id",
},
);
} catch (ex) {
expect((ex as Error).message).toEqual(
"SFU Config fetch failed with status code 500",
);
void (await fetchMock.flush());
}
const calls = fetchMock.calls();
expect(calls.length).toBe(1);
expect(calls[0][0]).toStrictEqual("https://sfu.example.org/get_token");
expect(calls[0][1]).toStrictEqual({
// check if it uses correct delayID!
body: '{"room_id":"!example_room_id","slot_id":"m.call#ROOM","member":{"id":"@alice:example.org:DEVICE","claimed_user_id":"@alice:example.org","claimed_device_id":"DEVICE"},"delay_id":"mock_delay_id","delay_timeout":1000,"delay_cs_api_url":"https://matrix.homeserverserver.org"}',
method: "POST",
headers: {
"Content-Type": "application/json",
},
});
});
it("should retry fetching the openid token", async () => {
let count = 0;
matrixClient.getOpenIdToken.mockImplementation(async () => {
@@ -98,6 +201,7 @@ describe("getSFUConfigWithOpenID", () => {
});
const config = await getSFUConfigWithOpenID(
matrixClient,
ownMemberMock,
"https://sfu.example.org",
"!example_room_id",
);

View File

@@ -5,11 +5,21 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type IOpenIDToken, type MatrixClient } from "matrix-js-sdk";
import { logger } from "matrix-js-sdk/lib/logger";
import {
retryNetworkOperation,
type IOpenIDToken,
type MatrixClient,
} from "matrix-js-sdk";
import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
import { type Logger } from "matrix-js-sdk/lib/logger";
import { FailToGetOpenIdToken } from "../utils/errors";
import {
FailToGetOpenIdToken,
NoMatrix2AuthorizationService,
} from "../utils/errors";
import { doNetworkOperationWithRetry } from "../utils/matrix";
import { Config } from "../config/Config";
import { JwtEndpointVersion } from "../state/CallViewModel/localMember/LocalTransport";
/**
* Configuration and access tokens provided by the SFU on successful authentication.
@@ -18,6 +28,7 @@ export interface SFUConfig {
url: string;
jwt: string;
livekitAlias: string;
// NOTE: Currently unused.
livekitIdentity: string;
}
@@ -63,16 +74,33 @@ export type OpenIDClientParts = Pick<
* Gets a bearer token from the homeserver and then use it to authenticate
* to the matrix RTC backend in order to get acces to the SFU.
* It has built-in retry for calls to the homeserver with a backoff policy.
* @param client
* @param serviceUrl
* @param matrixRoomId
* @param client The Matrix client
* @param membership Our own membership identity parts used to send to jwt service.
* @param serviceUrl The URL of the livekit SFU service
* @param roomId The room id used in the jwt request. This is NOT the livekit_alias. The jwt service will provide the alias. It maps matrix room ids <-> Livekit aliases.
* @param opts Additional options to modify which endpoint with which data will be used to acquire the jwt token.
* @param opts.forceJwtEndpoint This will use the old jwt endpoint which will create the rtc backend identity based on string concatenation
* instead of a hash.
* This function by default uses whatever is possible with the current jwt service installed next to the SFU.
* For remote connections this does not matter, since we will not publish there we can rely on the newest option.
* For our own connection we can only use the hashed version if we also send the new matrix2.0 sticky events.
* @param opts.delayEndpointBaseUrl The URL of the matrix homeserver.
* @param opts.delayId The delay id used for the jwt service to manage.
* @param logger optional logger.
* @returns Object containing the token information
* @throws FailToGetOpenIdToken
*/
export async function getSFUConfigWithOpenID(
client: OpenIDClientParts,
membership: CallMembershipIdentityParts,
serviceUrl: string,
matrixRoomId: string,
roomId: string,
opts?: {
forceJwtEndpoint?: JwtEndpointVersion;
delayEndpointBaseUrl?: string;
delayId?: string;
},
logger?: Logger,
): Promise<SFUConfig> {
let openIdToken: IOpenIDToken;
try {
@@ -84,16 +112,67 @@ export async function getSFUConfigWithOpenID(
error instanceof Error ? error : new Error("Unknown error"),
);
}
logger.debug("Got openID token", openIdToken);
logger?.debug("Got openID token", openIdToken);
logger.info(`Trying to get JWT for focus ${serviceUrl}...`);
const sfuConfig = await getLiveKitJWT(
client,
serviceUrl,
matrixRoomId,
openIdToken,
);
logger.info(`Got JWT from call's active focus URL.`);
logger?.info(`Trying to get JWT for focus ${serviceUrl}...`);
let sfuConfig: { url: string; jwt: string } | undefined;
const tryBothJwtEndpoints = opts?.forceJwtEndpoint === undefined; // This is for SFUs where we do not publish.
const forceMatrix2Jwt =
opts?.forceJwtEndpoint === JwtEndpointVersion.Matrix_2_0;
// We want to start using the new endpoint (with optional delay delegation)
// if we can use both or if we are forced to use the new one.
if (tryBothJwtEndpoints || forceMatrix2Jwt) {
try {
sfuConfig = await getLiveKitJWTWithDelayDelegation(
membership,
serviceUrl,
roomId,
openIdToken,
opts?.delayEndpointBaseUrl,
opts?.delayId,
);
logger?.info(`Got JWT from call's active focus URL.`);
} catch (e) {
if (e instanceof NotSupportedError) {
logger?.warn(
`Failed fetching jwt with matrix 2.0 endpoint (retry with legacy) Not supported`,
e,
);
sfuConfig = undefined;
} else {
logger?.warn(
`Failed fetching jwt with matrix 2.0 endpoint other issues ->`,
`(not going to try with legacy endpoint: forceOldJwtEndpoint is set to false, we did not get a not supported error from the sfu)`,
e,
);
// Make this throw a hard error in case we force the matrix2.0 endpoint.
if (forceMatrix2Jwt)
throw new NoMatrix2AuthorizationService(e as Error);
// NEVER get bejond this point if we forceMatrix2 and it failed!
}
}
}
// DEPRECATED
// here we either have a sfuConfig or we alredy exited because of `if (forceMatrix2) throw ...`
// The only case we can get into this condition is, if `forceMatrix2` is `false`
if (sfuConfig === undefined) {
sfuConfig = await getLiveKitJWT(
membership.deviceId,
serviceUrl,
roomId,
openIdToken,
);
logger?.info(`Got JWT from call's active focus URL.`);
}
if (!sfuConfig) {
throw new Error("No `sfuConfig` after trying with old and new endpoints");
}
// Pull the details from the JWT
const [, payloadStr] = sfuConfig.jwt.split(".");
@@ -104,33 +183,108 @@ export async function getSFUConfigWithOpenID(
url: sfuConfig.url,
livekitAlias: payload.video.room,
// NOTE: Currently unused.
// Probably also not helpful since we now compute the backendIdentity on joining the call so we can use it for the encryption manager.
// The only reason for us to know it locally is to connect the right users with the lk world. (and to set our own keys)
livekitIdentity: payload.sub,
};
}
const RETRIES = 4;
async function getLiveKitJWT(
client: OpenIDClientParts,
deviceId: string,
livekitServiceURL: string,
roomName: string,
matrixRoomId: string,
openIDToken: IOpenIDToken,
): Promise<{ url: string; jwt: string }> {
try {
const res = await fetch(livekitServiceURL + "/sfu/get", {
let res: Response | undefined;
await retryNetworkOperation(RETRIES, async () => {
res = await fetch(livekitServiceURL + "/sfu/get", {
method: "POST",
headers: {
"Content-Type": "application/json",
},
body: JSON.stringify({
room: roomName,
// This is the actual livekit room alias. For the legacy jwt endpoint simply the room id was used.
room: matrixRoomId,
openid_token: openIDToken,
device_id: client.getDeviceId(),
device_id: deviceId,
}),
});
if (!res.ok) {
throw new Error("SFU Config fetch failed with status code " + res.status);
}
return await res.json();
} catch (e) {
throw new Error("SFU Config fetch failed with exception", { cause: e });
});
if (!res) {
throw new Error(
`Network error while connecting to jwt service after ${RETRIES} retries`,
);
}
if (!res.ok) {
throw new Error("SFU Config fetch failed with status code " + res.status);
}
return await res.json();
}
class NotSupportedError extends Error {
public constructor(message: string) {
super(message);
this.name = "NotSupported";
}
}
export async function getLiveKitJWTWithDelayDelegation(
membership: CallMembershipIdentityParts,
livekitServiceURL: string,
matrixRoomId: string,
openIDToken: IOpenIDToken,
delayEndpointBaseUrl?: string,
delayId?: string,
): Promise<{ url: string; jwt: string }> {
const { userId, deviceId, memberId } = membership;
const body = {
room_id: matrixRoomId,
slot_id: "m.call#ROOM",
openid_token: openIDToken,
member: {
id: memberId,
claimed_user_id: userId,
claimed_device_id: deviceId,
},
};
let bodyDalayParts = {};
// Also check for empty string
if (delayId && delayEndpointBaseUrl) {
const delayTimeoutMs =
Config.get().matrix_rtc_session?.delayed_leave_event_delay_ms ?? 1000;
bodyDalayParts = {
delay_id: delayId,
delay_timeout: delayTimeoutMs,
delay_cs_api_url: delayEndpointBaseUrl,
};
}
let res: Response | undefined;
await retryNetworkOperation(RETRIES, async () => {
res = await fetch(livekitServiceURL + "/get_token", {
method: "POST",
headers: {
"Content-Type": "application/json",
},
body: JSON.stringify({ ...body, ...bodyDalayParts }),
});
});
if (!res) {
throw new Error(
`Network error while connecting to jwt service after ${RETRIES} retries`,
);
}
if (!res.ok) {
const msg = "SFU Config fetch failed with status code " + res.status;
if (res.status === 404) {
throw new NotSupportedError(msg);
} else {
throw new Error(msg);
}
}
return await res.json();
}

View File

@@ -1,188 +0,0 @@
/*
Copyright 2023, 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type Span } from "@opentelemetry/api";
import { type MatrixCall } from "matrix-js-sdk";
import { CallEvent } from "matrix-js-sdk/lib/webrtc/call";
import {
type TransceiverStats,
type CallFeedStats,
} from "matrix-js-sdk/lib/webrtc/stats/statsReport";
import { ObjectFlattener } from "./ObjectFlattener";
import { ElementCallOpenTelemetry } from "./otel";
import { type OTelCallAbstractMediaStreamSpan } from "./OTelCallAbstractMediaStreamSpan";
import { OTelCallTransceiverMediaStreamSpan } from "./OTelCallTransceiverMediaStreamSpan";
import { OTelCallFeedMediaStreamSpan } from "./OTelCallFeedMediaStreamSpan";
type StreamId = string;
type MID = string;
/**
* Tracks an individual call within a group call, either to a full-mesh peer or a focus
*/
export class OTelCall {
private readonly trackFeedSpan = new Map<
StreamId,
OTelCallAbstractMediaStreamSpan
>();
private readonly trackTransceiverSpan = new Map<
MID,
OTelCallAbstractMediaStreamSpan
>();
public constructor(
public userId: string,
public deviceId: string,
public call: MatrixCall,
public span: Span,
) {
if (call.peerConn) {
this.addCallPeerConnListeners();
} else {
this.call.once(
CallEvent.PeerConnectionCreated,
this.addCallPeerConnListeners,
);
}
}
public dispose(): void {
this.call.peerConn?.removeEventListener(
"connectionstatechange",
this.onCallConnectionStateChanged,
);
this.call.peerConn?.removeEventListener(
"signalingstatechange",
this.onCallSignalingStateChanged,
);
this.call.peerConn?.removeEventListener(
"iceconnectionstatechange",
this.onIceConnectionStateChanged,
);
this.call.peerConn?.removeEventListener(
"icegatheringstatechange",
this.onIceGatheringStateChanged,
);
this.call.peerConn?.removeEventListener(
"icecandidateerror",
this.onIceCandidateError,
);
}
private addCallPeerConnListeners = (): void => {
this.call.peerConn?.addEventListener(
"connectionstatechange",
this.onCallConnectionStateChanged,
);
this.call.peerConn?.addEventListener(
"signalingstatechange",
this.onCallSignalingStateChanged,
);
this.call.peerConn?.addEventListener(
"iceconnectionstatechange",
this.onIceConnectionStateChanged,
);
this.call.peerConn?.addEventListener(
"icegatheringstatechange",
this.onIceGatheringStateChanged,
);
this.call.peerConn?.addEventListener(
"icecandidateerror",
this.onIceCandidateError,
);
};
public onCallConnectionStateChanged = (): void => {
this.span.addEvent("matrix.call.callConnectionStateChange", {
callConnectionState: this.call.peerConn?.connectionState,
});
};
public onCallSignalingStateChanged = (): void => {
this.span.addEvent("matrix.call.callSignalingStateChange", {
callSignalingState: this.call.peerConn?.signalingState,
});
};
public onIceConnectionStateChanged = (): void => {
this.span.addEvent("matrix.call.iceConnectionStateChange", {
iceConnectionState: this.call.peerConn?.iceConnectionState,
});
};
public onIceGatheringStateChanged = (): void => {
this.span.addEvent("matrix.call.iceGatheringStateChange", {
iceGatheringState: this.call.peerConn?.iceGatheringState,
});
};
public onIceCandidateError = (ev: Event): void => {
const flatObject = {};
ObjectFlattener.flattenObjectRecursive(ev, flatObject, "error.", 0);
this.span.addEvent("matrix.call.iceCandidateError", flatObject);
};
public onCallFeedStats(callFeeds: CallFeedStats[]): void {
let prvFeeds: StreamId[] = [...this.trackFeedSpan.keys()];
callFeeds.forEach((feed) => {
if (!this.trackFeedSpan.has(feed.stream)) {
this.trackFeedSpan.set(
feed.stream,
new OTelCallFeedMediaStreamSpan(
ElementCallOpenTelemetry.instance,
this.span,
feed,
),
);
}
this.trackFeedSpan.get(feed.stream)?.update(feed);
prvFeeds = prvFeeds.filter((prvStreamId) => prvStreamId !== feed.stream);
});
prvFeeds.forEach((prvStreamId) => {
this.trackFeedSpan.get(prvStreamId)?.end();
this.trackFeedSpan.delete(prvStreamId);
});
}
public onTransceiverStats(transceiverStats: TransceiverStats[]): void {
let prvTransSpan: MID[] = [...this.trackTransceiverSpan.keys()];
transceiverStats.forEach((transStats) => {
if (!this.trackTransceiverSpan.has(transStats.mid)) {
this.trackTransceiverSpan.set(
transStats.mid,
new OTelCallTransceiverMediaStreamSpan(
ElementCallOpenTelemetry.instance,
this.span,
transStats,
),
);
}
this.trackTransceiverSpan.get(transStats.mid)?.update(transStats);
prvTransSpan = prvTransSpan.filter(
(prvStreamId) => prvStreamId !== transStats.mid,
);
});
prvTransSpan.forEach((prvMID) => {
this.trackTransceiverSpan.get(prvMID)?.end();
this.trackTransceiverSpan.delete(prvMID);
});
}
public end(): void {
this.trackFeedSpan.forEach((feedSpan) => feedSpan.end());
this.trackTransceiverSpan.forEach((transceiverSpan) =>
transceiverSpan.end(),
);
this.span.end();
}
}

View File

@@ -1,69 +0,0 @@
/*
Copyright 2023, 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import opentelemetry, { type Span } from "@opentelemetry/api";
import { type TrackStats } from "matrix-js-sdk/lib/webrtc/stats/statsReport";
import { type ElementCallOpenTelemetry } from "./otel";
import { OTelCallMediaStreamTrackSpan } from "./OTelCallMediaStreamTrackSpan";
type TrackId = string;
export abstract class OTelCallAbstractMediaStreamSpan {
protected readonly trackSpans = new Map<
TrackId,
OTelCallMediaStreamTrackSpan
>();
public readonly span;
public constructor(
protected readonly oTel: ElementCallOpenTelemetry,
protected readonly callSpan: Span,
protected readonly type: string,
) {
const ctx = opentelemetry.trace.setSpan(
opentelemetry.context.active(),
callSpan,
);
const options = {
links: [
{
context: callSpan.spanContext(),
},
],
};
this.span = oTel.tracer.startSpan(this.type, options, ctx);
}
protected upsertTrackSpans(tracks: TrackStats[]): void {
let prvTracks: TrackId[] = [...this.trackSpans.keys()];
tracks.forEach((t) => {
if (!this.trackSpans.has(t.id)) {
this.trackSpans.set(
t.id,
new OTelCallMediaStreamTrackSpan(this.oTel, this.span, t),
);
}
this.trackSpans.get(t.id)?.update(t);
prvTracks = prvTracks.filter((prvTrackId) => prvTrackId !== t.id);
});
prvTracks.forEach((prvTrackId) => {
this.trackSpans.get(prvTrackId)?.end();
this.trackSpans.delete(prvTrackId);
});
}
public abstract update(data: object): void;
public end(): void {
this.trackSpans.forEach((tSpan) => {
tSpan.end();
});
this.span.end();
}
}

View File

@@ -1,64 +0,0 @@
/*
Copyright 2023, 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type Span } from "@opentelemetry/api";
import {
type CallFeedStats,
type TrackStats,
} from "matrix-js-sdk/lib/webrtc/stats/statsReport";
import { type ElementCallOpenTelemetry } from "./otel";
import { OTelCallAbstractMediaStreamSpan } from "./OTelCallAbstractMediaStreamSpan";
export class OTelCallFeedMediaStreamSpan extends OTelCallAbstractMediaStreamSpan {
private readonly prev: { isAudioMuted: boolean; isVideoMuted: boolean };
public constructor(
protected readonly oTel: ElementCallOpenTelemetry,
protected readonly callSpan: Span,
callFeed: CallFeedStats,
) {
const postFix =
callFeed.type === "local" && callFeed.prefix === "from-call-feed"
? "(clone)"
: "";
super(oTel, callSpan, `matrix.call.feed.${callFeed.type}${postFix}`);
this.span.setAttribute("feed.streamId", callFeed.stream);
this.span.setAttribute("feed.type", callFeed.type);
this.span.setAttribute("feed.readFrom", callFeed.prefix);
this.span.setAttribute("feed.purpose", callFeed.purpose);
this.prev = {
isAudioMuted: callFeed.isAudioMuted,
isVideoMuted: callFeed.isVideoMuted,
};
this.span.addEvent("matrix.call.feed.initState", this.prev);
}
public update(callFeed: CallFeedStats): void {
if (this.prev.isAudioMuted !== callFeed.isAudioMuted) {
this.span.addEvent("matrix.call.feed.audioMuted", {
isAudioMuted: callFeed.isAudioMuted,
});
this.prev.isAudioMuted = callFeed.isAudioMuted;
}
if (this.prev.isVideoMuted !== callFeed.isVideoMuted) {
this.span.addEvent("matrix.call.feed.isVideoMuted", {
isVideoMuted: callFeed.isVideoMuted,
});
this.prev.isVideoMuted = callFeed.isVideoMuted;
}
const trackStats: TrackStats[] = [];
if (callFeed.video) {
trackStats.push(callFeed.video);
}
if (callFeed.audio) {
trackStats.push(callFeed.audio);
}
this.upsertTrackSpans(trackStats);
}
}

View File

@@ -1,69 +0,0 @@
/*
Copyright 2023, 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type TrackStats } from "matrix-js-sdk/lib/webrtc/stats/statsReport";
import opentelemetry, { type Span } from "@opentelemetry/api";
import { type ElementCallOpenTelemetry } from "./otel";
export class OTelCallMediaStreamTrackSpan {
private readonly span: Span;
private prev: TrackStats;
public constructor(
protected readonly oTel: ElementCallOpenTelemetry,
protected readonly streamSpan: Span,
data: TrackStats,
) {
const ctx = opentelemetry.trace.setSpan(
opentelemetry.context.active(),
streamSpan,
);
const options = {
links: [
{
context: streamSpan.spanContext(),
},
],
};
const type = `matrix.call.track.${data.label}.${data.kind}`;
this.span = oTel.tracer.startSpan(type, options, ctx);
this.span.setAttribute("track.trackId", data.id);
this.span.setAttribute("track.kind", data.kind);
this.span.setAttribute("track.constrainDeviceId", data.constrainDeviceId);
this.span.setAttribute("track.settingDeviceId", data.settingDeviceId);
this.span.setAttribute("track.label", data.label);
this.span.addEvent("matrix.call.track.initState", {
readyState: data.readyState,
muted: data.muted,
enabled: data.enabled,
});
this.prev = data;
}
public update(data: TrackStats): void {
if (this.prev.muted !== data.muted) {
this.span.addEvent("matrix.call.track.muted", { muted: data.muted });
}
if (this.prev.enabled !== data.enabled) {
this.span.addEvent("matrix.call.track.enabled", {
enabled: data.enabled,
});
}
if (this.prev.readyState !== data.readyState) {
this.span.addEvent("matrix.call.track.readyState", {
readyState: data.readyState,
});
}
this.prev = data;
}
public end(): void {
this.span.end();
}
}

View File

@@ -1,61 +0,0 @@
/*
Copyright 2023, 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type Span } from "@opentelemetry/api";
import {
type TrackStats,
type TransceiverStats,
} from "matrix-js-sdk/lib/webrtc/stats/statsReport";
import { type ElementCallOpenTelemetry } from "./otel";
import { OTelCallAbstractMediaStreamSpan } from "./OTelCallAbstractMediaStreamSpan";
export class OTelCallTransceiverMediaStreamSpan extends OTelCallAbstractMediaStreamSpan {
private readonly prev: {
direction: string;
currentDirection: string;
};
public constructor(
protected readonly oTel: ElementCallOpenTelemetry,
protected readonly callSpan: Span,
stats: TransceiverStats,
) {
super(oTel, callSpan, `matrix.call.transceiver.${stats.mid}`);
this.span.setAttribute("transceiver.mid", stats.mid);
this.prev = {
direction: stats.direction,
currentDirection: stats.currentDirection,
};
this.span.addEvent("matrix.call.transceiver.initState", this.prev);
}
public update(stats: TransceiverStats): void {
if (this.prev.currentDirection !== stats.currentDirection) {
this.span.addEvent("matrix.call.transceiver.currentDirection", {
currentDirection: stats.currentDirection,
});
this.prev.currentDirection = stats.currentDirection;
}
if (this.prev.direction !== stats.direction) {
this.span.addEvent("matrix.call.transceiver.direction", {
direction: stats.direction,
});
this.prev.direction = stats.direction;
}
const trackStats: TrackStats[] = [];
if (stats.sender) {
trackStats.push(stats.sender);
}
if (stats.receiver) {
trackStats.push(stats.receiver);
}
this.upsertTrackSpans(trackStats);
}
}

View File

@@ -1,477 +0,0 @@
/*
Copyright 2023, 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import opentelemetry, {
type Span,
type Attributes,
type Context,
} from "@opentelemetry/api";
import {
type GroupCall,
type MatrixClient,
type MatrixEvent,
type RoomMember,
} from "matrix-js-sdk";
import { logger } from "matrix-js-sdk/lib/logger";
import {
type CallError,
type CallState,
type MatrixCall,
type VoipEvent,
} from "matrix-js-sdk/lib/webrtc/call";
import {
type CallsByUserAndDevice,
type GroupCallError,
GroupCallEvent,
type GroupCallStatsReport,
} from "matrix-js-sdk/lib/webrtc/groupCall";
import {
type ConnectionStatsReport,
type ByteSentStatsReport,
type SummaryStatsReport,
type CallFeedReport,
} from "matrix-js-sdk/lib/webrtc/stats/statsReport";
import { ElementCallOpenTelemetry } from "./otel";
import { ObjectFlattener } from "./ObjectFlattener";
import { OTelCall } from "./OTelCall";
/**
* Represent the span of time which we intend to be joined to a group call
*/
export class OTelGroupCallMembership {
private callMembershipSpan?: Span;
private groupCallContext?: Context;
private myUserId = "unknown";
private myDeviceId: string;
private myMember?: RoomMember;
private callsByCallId = new Map<string, OTelCall>();
private statsReportSpan: {
span: Span | undefined;
stats: OTelStatsReportEvent[];
};
private readonly speakingSpans = new Map<RoomMember, Map<string, Span>>();
public constructor(
private groupCall: GroupCall,
client: MatrixClient,
) {
const clientId = client.getUserId();
if (clientId) {
this.myUserId = clientId;
const myMember = groupCall.room.getMember(clientId);
if (myMember) {
this.myMember = myMember;
}
}
this.myDeviceId = client.getDeviceId() || "unknown";
this.statsReportSpan = { span: undefined, stats: [] };
this.groupCall.on(GroupCallEvent.CallsChanged, this.onCallsChanged);
}
public dispose(): void {
this.groupCall.removeListener(
GroupCallEvent.CallsChanged,
this.onCallsChanged,
);
}
public onJoinCall(): void {
if (!ElementCallOpenTelemetry.instance) return;
if (this.callMembershipSpan !== undefined) {
logger.warn("Call membership span is already started");
return;
}
// Create the main span that tracks the time we intend to be in the call
this.callMembershipSpan =
ElementCallOpenTelemetry.instance.tracer.startSpan(
"matrix.groupCallMembership",
);
this.callMembershipSpan.setAttribute(
"matrix.confId",
this.groupCall.groupCallId,
);
this.callMembershipSpan.setAttribute("matrix.userId", this.myUserId);
this.callMembershipSpan.setAttribute("matrix.deviceId", this.myDeviceId);
this.callMembershipSpan.setAttribute(
"matrix.displayName",
this.myMember ? this.myMember.name : "unknown-name",
);
this.groupCallContext = opentelemetry.trace.setSpan(
opentelemetry.context.active(),
this.callMembershipSpan,
);
this.callMembershipSpan?.addEvent("matrix.joinCall");
}
public onLeaveCall(): void {
if (this.callMembershipSpan === undefined) {
logger.warn("Call membership span is already ended");
return;
}
this.callMembershipSpan.addEvent("matrix.leaveCall");
// and end the span to indicate we've left
this.callMembershipSpan.end();
this.callMembershipSpan = undefined;
this.groupCallContext = undefined;
}
public onUpdateRoomState(event: MatrixEvent): void {
if (
!event ||
(!event.getType().startsWith("m.call") &&
!event.getType().startsWith("org.matrix.msc3401.call"))
) {
return;
}
this.callMembershipSpan?.addEvent(
`matrix.roomStateEvent_${event.getType()}`,
ObjectFlattener.flattenVoipEvent(event.getContent()),
);
}
public onCallsChanged(calls: CallsByUserAndDevice): void {
for (const [userId, userCalls] of calls.entries()) {
for (const [deviceId, call] of userCalls.entries()) {
if (!this.callsByCallId.has(call.callId)) {
if (ElementCallOpenTelemetry.instance) {
const span = ElementCallOpenTelemetry.instance.tracer.startSpan(
`matrix.call`,
undefined,
this.groupCallContext,
);
// XXX: anonymity
span.setAttribute("matrix.call.target.userId", userId);
span.setAttribute("matrix.call.target.deviceId", deviceId);
const displayName =
this.groupCall.room.getMember(userId)?.name ?? "unknown";
span.setAttribute("matrix.call.target.displayName", displayName);
this.callsByCallId.set(
call.callId,
new OTelCall(userId, deviceId, call, span),
);
}
}
}
}
for (const callTrackingInfo of this.callsByCallId.values()) {
const userCalls = calls.get(callTrackingInfo.userId);
if (
!userCalls ||
!userCalls.has(callTrackingInfo.deviceId) ||
userCalls.get(callTrackingInfo.deviceId)?.callId !==
callTrackingInfo.call.callId
) {
callTrackingInfo.end();
this.callsByCallId.delete(callTrackingInfo.call.callId);
}
}
}
public onCallStateChange(call: MatrixCall, newState: CallState): void {
const callTrackingInfo = this.callsByCallId.get(call.callId);
if (!callTrackingInfo) {
logger.error(`Got call state change for unknown call ID ${call.callId}`);
return;
}
callTrackingInfo.span.addEvent("matrix.call.stateChange", {
state: newState,
});
}
public onSendEvent(call: MatrixCall, event: VoipEvent): void {
const eventType = event.eventType as string;
if (
!eventType.startsWith("m.call") &&
!eventType.startsWith("org.matrix.call")
)
return;
const callTrackingInfo = this.callsByCallId.get(call.callId);
if (!callTrackingInfo) {
logger.error(`Got call send event for unknown call ID ${call.callId}`);
return;
}
if (event.type === "toDevice") {
callTrackingInfo.span.addEvent(
`matrix.sendToDeviceEvent_${event.eventType}`,
ObjectFlattener.flattenVoipEvent(event),
);
} else if (event.type === "sendEvent") {
callTrackingInfo.span.addEvent(
`matrix.sendToRoomEvent_${event.eventType}`,
ObjectFlattener.flattenVoipEvent(event),
);
}
}
public onReceivedVoipEvent(event: MatrixEvent): void {
// These come straight from CallEventHandler so don't have
// a call already associated (in principle we could receive
// events for calls we don't know about).
const callId = event.getContent().call_id;
if (!callId) {
this.callMembershipSpan?.addEvent("matrix.receive_voip_event_no_callid", {
"sender.userId": event.getSender(),
});
logger.error("Received call event with no call ID!");
return;
}
const call = this.callsByCallId.get(callId);
if (!call) {
this.callMembershipSpan?.addEvent(
"matrix.receive_voip_event_unknown_callid",
{
"sender.userId": event.getSender(),
},
);
logger.error("Received call event for unknown call ID " + callId);
return;
}
call.span.addEvent("matrix.receive_voip_event", {
"sender.userId": event.getSender(),
...ObjectFlattener.flattenVoipEvent(event.getContent()),
});
}
public onToggleMicrophoneMuted(newValue: boolean): void {
this.callMembershipSpan?.addEvent("matrix.toggleMicMuted", {
"matrix.microphone.muted": newValue,
});
}
public onSetMicrophoneMuted(setMuted: boolean): void {
this.callMembershipSpan?.addEvent("matrix.setMicMuted", {
"matrix.microphone.muted": setMuted,
});
}
public onToggleLocalVideoMuted(newValue: boolean): void {
this.callMembershipSpan?.addEvent("matrix.toggleVidMuted", {
"matrix.video.muted": newValue,
});
}
public onSetLocalVideoMuted(setMuted: boolean): void {
this.callMembershipSpan?.addEvent("matrix.setVidMuted", {
"matrix.video.muted": setMuted,
});
}
public onToggleScreensharing(newValue: boolean): void {
this.callMembershipSpan?.addEvent("matrix.setVidMuted", {
"matrix.screensharing.enabled": newValue,
});
}
public onSpeaking(
member: RoomMember,
deviceId: string,
speaking: boolean,
): void {
if (speaking) {
// Ensure that there's an audio activity span for this speaker
let deviceMap = this.speakingSpans.get(member);
if (deviceMap === undefined) {
deviceMap = new Map();
this.speakingSpans.set(member, deviceMap);
}
if (!deviceMap.has(deviceId)) {
const span = ElementCallOpenTelemetry.instance.tracer.startSpan(
"matrix.audioActivity",
undefined,
this.groupCallContext,
);
span.setAttribute("matrix.userId", member.userId);
span.setAttribute("matrix.displayName", member.rawDisplayName);
deviceMap.set(deviceId, span);
}
} else {
// End the audio activity span for this speaker, if any
const deviceMap = this.speakingSpans.get(member);
deviceMap?.get(deviceId)?.end();
deviceMap?.delete(deviceId);
if (deviceMap?.size === 0) this.speakingSpans.delete(member);
}
}
public onCallError(error: CallError, call: MatrixCall): void {
const callTrackingInfo = this.callsByCallId.get(call.callId);
if (!callTrackingInfo) {
logger.error(`Got error for unknown call ID ${call.callId}`);
return;
}
callTrackingInfo.span.recordException(error);
}
public onGroupCallError(error: GroupCallError): void {
this.callMembershipSpan?.recordException(error);
}
public onUndecryptableToDevice(event: MatrixEvent): void {
this.callMembershipSpan?.addEvent("matrix.toDevice.undecryptable", {
"sender.userId": event.getSender(),
});
}
public onCallFeedStatsReport(
report: GroupCallStatsReport<CallFeedReport>,
): void {
if (!ElementCallOpenTelemetry.instance) return;
let call: OTelCall | undefined;
const callId = report.report?.callId;
if (callId) {
call = this.callsByCallId.get(callId);
}
if (!call) {
this.callMembershipSpan?.addEvent(
OTelStatsReportType.CallFeedReport + "_unknown_callId",
{
"call.callId": callId,
"call.opponentMemberId": report.report?.opponentMemberId
? report.report?.opponentMemberId
: "unknown",
},
);
logger.error(
`Received ${OTelStatsReportType.CallFeedReport} with unknown call ID: ${callId}`,
);
return;
} else {
call.onCallFeedStats(report.report.callFeeds);
call.onTransceiverStats(report.report.transceiver);
}
}
public onConnectionStatsReport(
statsReport: GroupCallStatsReport<ConnectionStatsReport>,
): void {
this.buildCallStatsSpan(
OTelStatsReportType.ConnectionReport,
statsReport.report,
);
}
public onByteSentStatsReport(
statsReport: GroupCallStatsReport<ByteSentStatsReport>,
): void {
this.buildCallStatsSpan(
OTelStatsReportType.ByteSentReport,
statsReport.report,
);
}
public buildCallStatsSpan(
type: OTelStatsReportType,
report: ByteSentStatsReport | ConnectionStatsReport,
): void {
if (!ElementCallOpenTelemetry.instance) return;
let call: OTelCall | undefined;
const callId = report?.callId;
if (callId) {
call = this.callsByCallId.get(callId);
}
if (!call) {
this.callMembershipSpan?.addEvent(type + "_unknown_callid", {
"call.callId": callId,
"call.opponentMemberId": report.opponentMemberId
? report.opponentMemberId
: "unknown",
});
logger.error(`Received ${type} with unknown call ID: ${callId}`);
return;
}
const data = ObjectFlattener.flattenReportObject(type, report);
const ctx = opentelemetry.trace.setSpan(
opentelemetry.context.active(),
call.span,
);
const options = {
links: [
{
context: call.span.spanContext(),
},
],
};
const span = ElementCallOpenTelemetry.instance.tracer.startSpan(
type,
options,
ctx,
);
span.setAttribute("matrix.callId", callId ?? "unknown");
span.setAttribute(
"matrix.opponentMemberId",
report.opponentMemberId ? report.opponentMemberId : "unknown",
);
span.addEvent("matrix.call.connection_stats_event", data);
span.end();
}
public onSummaryStatsReport(
statsReport: GroupCallStatsReport<SummaryStatsReport>,
): void {
if (!ElementCallOpenTelemetry.instance) return;
const type = OTelStatsReportType.SummaryReport;
const data = ObjectFlattener.flattenSummaryStatsReportObject(statsReport);
if (this.statsReportSpan.span === undefined && this.callMembershipSpan) {
const ctx = opentelemetry.trace.setSpan(
opentelemetry.context.active(),
this.callMembershipSpan,
);
const span = ElementCallOpenTelemetry.instance?.tracer.startSpan(
"matrix.groupCallMembership.summaryReport",
undefined,
ctx,
);
if (span === undefined) {
return;
}
span.setAttribute("matrix.confId", this.groupCall.groupCallId);
span.setAttribute("matrix.userId", this.myUserId);
span.setAttribute(
"matrix.displayName",
this.myMember ? this.myMember.name : "unknown-name",
);
span.addEvent(type, data);
span.end();
}
}
}
interface OTelStatsReportEvent {
type: OTelStatsReportType;
data: Attributes;
}
enum OTelStatsReportType {
ConnectionReport = "matrix.call.stats.connection",
ByteSentReport = "matrix.call.stats.byteSent",
SummaryReport = "matrix.stats.summary",
CallFeedReport = "matrix.stats.call_feed",
}

View File

@@ -1,266 +0,0 @@
/*
Copyright 2023, 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type GroupCallStatsReport } from "matrix-js-sdk/lib/webrtc/groupCall";
import {
type AudioConcealment,
type ByteSentStatsReport,
type ConnectionStatsReport,
} from "matrix-js-sdk/lib/webrtc/stats/statsReport";
import { describe, expect, it } from "vitest";
import { ObjectFlattener } from "../../src/otel/ObjectFlattener";
describe("ObjectFlattener", () => {
const noConcealment: AudioConcealment = {
concealedAudio: 0,
totalAudioDuration: 0,
};
const statsReport: GroupCallStatsReport<ConnectionStatsReport> = {
report: {
callId: "callId",
opponentMemberId: "opponentMemberId",
bandwidth: { upload: 426, download: 0 },
bitrate: {
upload: 426,
download: 0,
audio: {
upload: 124,
download: 0,
},
video: {
upload: 302,
download: 0,
},
},
packetLoss: {
total: 0,
download: 0,
upload: 0,
},
framerate: {
local: new Map([
["LOCAL_AUDIO_TRACK_ID", 0],
["LOCAL_VIDEO_TRACK_ID", 30],
]),
remote: new Map([
["REMOTE_AUDIO_TRACK_ID", 0],
["REMOTE_VIDEO_TRACK_ID", 60],
]),
},
resolution: {
local: new Map([
["LOCAL_AUDIO_TRACK_ID", { height: -1, width: -1 }],
["LOCAL_VIDEO_TRACK_ID", { height: 460, width: 780 }],
]),
remote: new Map([
["REMOTE_AUDIO_TRACK_ID", { height: -1, width: -1 }],
["REMOTE_VIDEO_TRACK_ID", { height: 960, width: 1080 }],
]),
},
jitter: new Map([
["REMOTE_AUDIO_TRACK_ID", 2],
["REMOTE_VIDEO_TRACK_ID", 50],
]),
codec: {
local: new Map([
["LOCAL_AUDIO_TRACK_ID", "opus"],
["LOCAL_VIDEO_TRACK_ID", "v8"],
]),
remote: new Map([
["REMOTE_AUDIO_TRACK_ID", "opus"],
["REMOTE_VIDEO_TRACK_ID", "v9"],
]),
},
transport: [
{
ip: "ff11::5fa:abcd:999c:c5c5:50000",
type: "udp",
localIp: "2aaa:9999:2aaa:999:8888:2aaa:2aaa:7777:50000",
isFocus: true,
localCandidateType: "host",
remoteCandidateType: "host",
networkType: "ethernet",
rtt: NaN,
},
{
ip: "10.10.10.2:22222",
type: "tcp",
localIp: "10.10.10.100:33333",
isFocus: true,
localCandidateType: "srfx",
remoteCandidateType: "srfx",
networkType: "ethernet",
rtt: 0,
},
],
audioConcealment: new Map([
["REMOTE_AUDIO_TRACK_ID", noConcealment],
["REMOTE_VIDEO_TRACK_ID", noConcealment],
]),
totalAudioConcealment: noConcealment,
},
};
describe("on flattenObjectRecursive", () => {
it("should flatter an Map object", () => {
const flatObject = {};
ObjectFlattener.flattenObjectRecursive(
statsReport.report.resolution,
flatObject,
"matrix.call.stats.connection.resolution.",
0,
);
expect(flatObject).toEqual({
"matrix.call.stats.connection.resolution.local.LOCAL_AUDIO_TRACK_ID.height":
-1,
"matrix.call.stats.connection.resolution.local.LOCAL_AUDIO_TRACK_ID.width":
-1,
"matrix.call.stats.connection.resolution.local.LOCAL_VIDEO_TRACK_ID.height": 460,
"matrix.call.stats.connection.resolution.local.LOCAL_VIDEO_TRACK_ID.width": 780,
"matrix.call.stats.connection.resolution.remote.REMOTE_AUDIO_TRACK_ID.height":
-1,
"matrix.call.stats.connection.resolution.remote.REMOTE_AUDIO_TRACK_ID.width":
-1,
"matrix.call.stats.connection.resolution.remote.REMOTE_VIDEO_TRACK_ID.height": 960,
"matrix.call.stats.connection.resolution.remote.REMOTE_VIDEO_TRACK_ID.width": 1080,
});
});
it("should flatter an Array object", () => {
const flatObject = {};
ObjectFlattener.flattenObjectRecursive(
statsReport.report.transport,
flatObject,
"matrix.call.stats.connection.transport.",
0,
);
expect(flatObject).toEqual({
"matrix.call.stats.connection.transport.0.ip":
"ff11::5fa:abcd:999c:c5c5:50000",
"matrix.call.stats.connection.transport.0.type": "udp",
"matrix.call.stats.connection.transport.0.localIp":
"2aaa:9999:2aaa:999:8888:2aaa:2aaa:7777:50000",
"matrix.call.stats.connection.transport.0.isFocus": true,
"matrix.call.stats.connection.transport.0.localCandidateType": "host",
"matrix.call.stats.connection.transport.0.remoteCandidateType": "host",
"matrix.call.stats.connection.transport.0.networkType": "ethernet",
"matrix.call.stats.connection.transport.0.rtt": "NaN",
"matrix.call.stats.connection.transport.1.ip": "10.10.10.2:22222",
"matrix.call.stats.connection.transport.1.type": "tcp",
"matrix.call.stats.connection.transport.1.localIp":
"10.10.10.100:33333",
"matrix.call.stats.connection.transport.1.isFocus": true,
"matrix.call.stats.connection.transport.1.localCandidateType": "srfx",
"matrix.call.stats.connection.transport.1.remoteCandidateType": "srfx",
"matrix.call.stats.connection.transport.1.networkType": "ethernet",
"matrix.call.stats.connection.transport.1.rtt": 0,
});
});
});
describe("on flattenReportObject Connection Stats", () => {
it("should flatten a Report to otel Attributes Object", () => {
expect(
ObjectFlattener.flattenReportObject(
"matrix.call.stats.connection",
statsReport.report,
),
).toEqual({
"matrix.call.stats.connection.callId": "callId",
"matrix.call.stats.connection.opponentMemberId": "opponentMemberId",
"matrix.call.stats.connection.bandwidth.download": 0,
"matrix.call.stats.connection.bandwidth.upload": 426,
"matrix.call.stats.connection.bitrate.audio.download": 0,
"matrix.call.stats.connection.bitrate.audio.upload": 124,
"matrix.call.stats.connection.bitrate.download": 0,
"matrix.call.stats.connection.bitrate.upload": 426,
"matrix.call.stats.connection.bitrate.video.download": 0,
"matrix.call.stats.connection.bitrate.video.upload": 302,
"matrix.call.stats.connection.codec.local.LOCAL_AUDIO_TRACK_ID": "opus",
"matrix.call.stats.connection.codec.local.LOCAL_VIDEO_TRACK_ID": "v8",
"matrix.call.stats.connection.codec.remote.REMOTE_AUDIO_TRACK_ID":
"opus",
"matrix.call.stats.connection.codec.remote.REMOTE_VIDEO_TRACK_ID": "v9",
"matrix.call.stats.connection.framerate.local.LOCAL_AUDIO_TRACK_ID": 0,
"matrix.call.stats.connection.framerate.local.LOCAL_VIDEO_TRACK_ID": 30,
"matrix.call.stats.connection.framerate.remote.REMOTE_AUDIO_TRACK_ID": 0,
"matrix.call.stats.connection.framerate.remote.REMOTE_VIDEO_TRACK_ID": 60,
"matrix.call.stats.connection.jitter.REMOTE_AUDIO_TRACK_ID": 2,
"matrix.call.stats.connection.jitter.REMOTE_VIDEO_TRACK_ID": 50,
"matrix.call.stats.connection.packetLoss.download": 0,
"matrix.call.stats.connection.packetLoss.total": 0,
"matrix.call.stats.connection.packetLoss.upload": 0,
"matrix.call.stats.connection.resolution.local.LOCAL_AUDIO_TRACK_ID.height":
-1,
"matrix.call.stats.connection.resolution.local.LOCAL_AUDIO_TRACK_ID.width":
-1,
"matrix.call.stats.connection.resolution.local.LOCAL_VIDEO_TRACK_ID.height": 460,
"matrix.call.stats.connection.resolution.local.LOCAL_VIDEO_TRACK_ID.width": 780,
"matrix.call.stats.connection.resolution.remote.REMOTE_AUDIO_TRACK_ID.height":
-1,
"matrix.call.stats.connection.resolution.remote.REMOTE_AUDIO_TRACK_ID.width":
-1,
"matrix.call.stats.connection.resolution.remote.REMOTE_VIDEO_TRACK_ID.height": 960,
"matrix.call.stats.connection.resolution.remote.REMOTE_VIDEO_TRACK_ID.width": 1080,
"matrix.call.stats.connection.transport.0.ip":
"ff11::5fa:abcd:999c:c5c5:50000",
"matrix.call.stats.connection.transport.0.type": "udp",
"matrix.call.stats.connection.transport.0.localIp":
"2aaa:9999:2aaa:999:8888:2aaa:2aaa:7777:50000",
"matrix.call.stats.connection.transport.0.isFocus": true,
"matrix.call.stats.connection.transport.0.localCandidateType": "host",
"matrix.call.stats.connection.transport.0.remoteCandidateType": "host",
"matrix.call.stats.connection.transport.0.networkType": "ethernet",
"matrix.call.stats.connection.transport.0.rtt": "NaN",
"matrix.call.stats.connection.transport.1.ip": "10.10.10.2:22222",
"matrix.call.stats.connection.transport.1.type": "tcp",
"matrix.call.stats.connection.transport.1.localIp":
"10.10.10.100:33333",
"matrix.call.stats.connection.transport.1.isFocus": true,
"matrix.call.stats.connection.transport.1.localCandidateType": "srfx",
"matrix.call.stats.connection.transport.1.remoteCandidateType": "srfx",
"matrix.call.stats.connection.transport.1.networkType": "ethernet",
"matrix.call.stats.connection.transport.1.rtt": 0,
"matrix.call.stats.connection.audioConcealment.REMOTE_AUDIO_TRACK_ID.concealedAudio": 0,
"matrix.call.stats.connection.audioConcealment.REMOTE_AUDIO_TRACK_ID.totalAudioDuration": 0,
"matrix.call.stats.connection.audioConcealment.REMOTE_VIDEO_TRACK_ID.concealedAudio": 0,
"matrix.call.stats.connection.audioConcealment.REMOTE_VIDEO_TRACK_ID.totalAudioDuration": 0,
"matrix.call.stats.connection.totalAudioConcealment.concealedAudio": 0,
"matrix.call.stats.connection.totalAudioConcealment.totalAudioDuration": 0,
});
});
});
describe("on flattenByteSendStatsReportObject", () => {
const byteSentStatsReport = new Map<
string,
number
>() as ByteSentStatsReport;
byteSentStatsReport.callId = "callId";
byteSentStatsReport.opponentMemberId = "opponentMemberId";
byteSentStatsReport.set("4aa92608-04c6-428e-8312-93e17602a959", 132093);
byteSentStatsReport.set("a08e4237-ee30-4015-a932-b676aec894b1", 913448);
it("should flatten a Report to otel Attributes Object", () => {
expect(
ObjectFlattener.flattenReportObject(
"matrix.call.stats.bytesSend",
byteSentStatsReport,
),
).toEqual({
"matrix.call.stats.bytesSend.4aa92608-04c6-428e-8312-93e17602a959": 132093,
"matrix.call.stats.bytesSend.a08e4237-ee30-4015-a932-b676aec894b1": 913448,
});
expect(byteSentStatsReport.callId).toEqual("callId");
expect(byteSentStatsReport.opponentMemberId).toEqual("opponentMemberId");
});
});
});

View File

@@ -1,100 +0,0 @@
/*
Copyright 2023, 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type Attributes } from "@opentelemetry/api";
import { type VoipEvent } from "matrix-js-sdk/lib/webrtc/call";
import { type GroupCallStatsReport } from "matrix-js-sdk/lib/webrtc/groupCall";
import {
type ByteSentStatsReport,
type ConnectionStatsReport,
type SummaryStatsReport,
} from "matrix-js-sdk/lib/webrtc/stats/statsReport";
export class ObjectFlattener {
public static flattenReportObject(
prefix: string,
report: ConnectionStatsReport | ByteSentStatsReport,
): Attributes {
const flatObject = {};
ObjectFlattener.flattenObjectRecursive(report, flatObject, `${prefix}.`, 0);
return flatObject;
}
public static flattenByteSentStatsReportObject(
statsReport: GroupCallStatsReport<ByteSentStatsReport>,
): Attributes {
const flatObject = {};
ObjectFlattener.flattenObjectRecursive(
statsReport.report,
flatObject,
"matrix.stats.bytesSent.",
0,
);
return flatObject;
}
public static flattenSummaryStatsReportObject(
statsReport: GroupCallStatsReport<SummaryStatsReport>,
): Attributes {
const flatObject = {};
ObjectFlattener.flattenObjectRecursive(
statsReport.report,
flatObject,
"matrix.stats.summary.",
0,
);
return flatObject;
}
/* Flattens out an object into a single layer with components
* of the key separated by dots
*/
public static flattenVoipEvent(event: VoipEvent): Attributes {
const flatObject = {};
ObjectFlattener.flattenObjectRecursive(
event as unknown as Record<string, unknown>, // XXX Types
flatObject,
"matrix.event.",
0,
);
return flatObject;
}
public static flattenObjectRecursive(
obj: object,
flatObject: Attributes,
prefix: string,
depth: number,
): void {
if (depth > 10)
throw new Error(
"Depth limit exceeded: aborting VoipEvent recursion. Prefix is " +
prefix,
);
let entries;
if (obj instanceof Map) {
entries = obj.entries();
} else {
entries = Object.entries(obj);
}
for (const [k, v] of entries) {
if (["string", "number", "boolean"].includes(typeof v) || v === null) {
let value;
value = v === null ? "null" : v;
value = typeof v === "number" && Number.isNaN(v) ? "NaN" : value;
flatObject[prefix + k] = value;
} else if (typeof v === "object") {
ObjectFlattener.flattenObjectRecursive(
v,
flatObject,
prefix + k + ".",
depth + 1,
);
}
}
}
}

View File

@@ -1,79 +0,0 @@
/*
Copyright 2025 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
expect,
describe,
it,
vi,
beforeEach,
beforeAll,
afterAll,
} from "vitest";
import { ElementCallOpenTelemetry } from "./otel";
import { mockConfig } from "../utils/test";
describe("ElementCallOpenTelemetry", () => {
describe("embedded package", () => {
beforeAll(() => {
vi.stubEnv("VITE_PACKAGE", "embedded");
});
beforeEach(() => {
mockConfig({});
});
afterAll(() => {
vi.unstubAllEnvs();
});
it("does not create instance without config value", () => {
ElementCallOpenTelemetry.globalInit();
expect(ElementCallOpenTelemetry.instance?.isOtlpEnabled).toBe(false);
});
it("ignores config value and does not create instance", () => {
mockConfig({
opentelemetry: {
collector_url: "https://collector.example.com.localhost",
},
});
ElementCallOpenTelemetry.globalInit();
expect(ElementCallOpenTelemetry.instance?.isOtlpEnabled).toBe(false);
});
});
describe("full package", () => {
beforeAll(() => {
vi.stubEnv("VITE_PACKAGE", "full");
});
beforeEach(() => {
mockConfig({});
});
afterAll(() => {
vi.unstubAllEnvs();
});
it("does not create instance without config value", () => {
ElementCallOpenTelemetry.globalInit();
expect(ElementCallOpenTelemetry.instance?.isOtlpEnabled).toBe(false);
});
it("creates instance with config value", () => {
mockConfig({
opentelemetry: {
collector_url: "https://collector.example.com.localhost",
},
});
ElementCallOpenTelemetry.globalInit();
expect(ElementCallOpenTelemetry.instance?.isOtlpEnabled).toBe(true);
});
});
});

View File

@@ -1,117 +0,0 @@
/*
Copyright 2023, 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
SimpleSpanProcessor,
type SpanProcessor,
} from "@opentelemetry/sdk-trace-base";
import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
import { WebTracerProvider } from "@opentelemetry/sdk-trace-web";
import opentelemetry, { type Tracer } from "@opentelemetry/api";
import { resourceFromAttributes } from "@opentelemetry/resources";
import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions";
import { logger } from "matrix-js-sdk/lib/logger";
import { PosthogSpanProcessor } from "../analytics/PosthogSpanProcessor";
import { Config } from "../config/Config";
import { RageshakeSpanProcessor } from "../analytics/RageshakeSpanProcessor";
import { getRageshakeSubmitUrl } from "../settings/submit-rageshake";
const SERVICE_NAME = "element-call";
let sharedInstance: ElementCallOpenTelemetry;
export class ElementCallOpenTelemetry {
private _provider: WebTracerProvider;
private _tracer: Tracer;
private otlpExporter?: OTLPTraceExporter;
public readonly rageshakeProcessor?: RageshakeSpanProcessor;
public static globalInit(): void {
// this is only supported in the full package as the is currently no support for passing in the collector URL from the widget host
const collectorUrl =
import.meta.env.VITE_PACKAGE === "full"
? Config.get().opentelemetry?.collector_url
: undefined;
// we always enable opentelemetry in general. We only enable the OTLP
// collector if a URL is defined (and in future if another setting is defined)
// Posthog reporting is enabled or disabled
// within the posthog code.
const shouldEnableOtlp = Boolean(collectorUrl);
if (!sharedInstance || sharedInstance.isOtlpEnabled !== shouldEnableOtlp) {
logger.info("(Re)starting OpenTelemetry debug reporting");
sharedInstance?.dispose();
sharedInstance = new ElementCallOpenTelemetry(
collectorUrl,
getRageshakeSubmitUrl(),
);
}
}
public static get instance(): ElementCallOpenTelemetry {
return sharedInstance;
}
private constructor(
collectorUrl: string | undefined,
rageshakeUrl: string | undefined,
) {
const spanProcessors: SpanProcessor[] = [];
if (collectorUrl) {
logger.info("Enabling OTLP collector with URL " + collectorUrl);
this.otlpExporter = new OTLPTraceExporter({
url: collectorUrl,
});
spanProcessors.push(new SimpleSpanProcessor(this.otlpExporter));
} else {
logger.info("OTLP collector disabled");
}
if (rageshakeUrl) {
this.rageshakeProcessor = new RageshakeSpanProcessor();
spanProcessors.push(this.rageshakeProcessor);
}
spanProcessors.push(new PosthogSpanProcessor());
this._provider = new WebTracerProvider({
resource: resourceFromAttributes({
// This is how we can make Jaeger show a reasonable service in the dropdown on the left.
[ATTR_SERVICE_NAME]: SERVICE_NAME,
}),
spanProcessors,
});
opentelemetry.trace.setGlobalTracerProvider(this._provider);
this._tracer = opentelemetry.trace.getTracer(
// This is not the serviceName shown in jaeger
"my-element-call-otl-tracer",
);
}
public dispose(): void {
opentelemetry.trace.disable();
this._provider?.shutdown().catch((e) => {
logger.error("Failed to shutdown OpenTelemetry", e);
});
}
public get isOtlpEnabled(): boolean {
return Boolean(this.otlpExporter);
}
public get tracer(): Tracer {
return this._tracer;
}
public get provider(): WebTracerProvider {
return this._provider;
}
}

View File

@@ -135,10 +135,10 @@ export class ReactionsReader {
}
/**
* Fetchest any hand wave reactions by the given sender on the given
* Fetches any hand wave reactions by the given sender on the given
* membership event.
* @param membershipEventId
* @param expectedSender
* @param membershipEventId - The user membership event id.
* @param expectedSender - The expected sender of the reaction.
* @returns A MatrixEvent if one was found.
*/
private getLastReactionEvent(

View File

@@ -39,7 +39,8 @@ import {
localRtcMember,
} from "../utils/test-fixtures";
import { MAX_PARTICIPANT_COUNT_FOR_SOUND } from "../state/CallViewModel/CallViewModel";
import { initializeWidget } from "../widget";
initializeWidget();
vitest.mock("livekit-client/e2ee-worker?worker");
vitest.mock("../useAudioContext");
vitest.mock("../soundUtils");

View File

@@ -178,7 +178,11 @@ export const GroupCallView: FC<Props> = ({
const { displayName, avatarUrl } = useProfile(client);
const roomName = useRoomName(room);
const roomAvatar = useRoomAvatar(room);
const { returnToLobby, password: passwordFromUrl } = useUrlParams();
const {
perParticipantE2EE,
returnToLobby,
password: passwordFromUrl,
} = useUrlParams();
const e2eeSystem = useRoomEncryptionSystem(room.roomId);
// Save the password once we start the groupCallView
@@ -299,6 +303,7 @@ export const GroupCallView: FC<Props> = ({
rtcSession,
preload,
skipLobby,
perParticipantE2EE,
mediaDevices,
latestMuteStates,
setJoined,

View File

@@ -1,5 +1,6 @@
/*
Copyright 2025 New Vector Ltd.
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
@@ -42,7 +43,8 @@ import { useRoomEncryptionSystem } from "../e2ee/sharedKeyManagement";
import { LivekitRoomAudioRenderer } from "../livekit/MatrixAudioRenderer";
import { MediaDevicesContext } from "../MediaDevicesContext";
import { HeaderStyle } from "../UrlParams";
import { initializeWidget } from "../widget";
initializeWidget();
vi.hoisted(
() =>
(global.ImageData = class MockImageData {

View File

@@ -33,7 +33,6 @@ import { useTranslation } from "react-i18next";
import LogoMark from "../icons/LogoMark.svg?react";
import LogoType from "../icons/LogoType.svg?react";
import type { IWidgetApiRequest } from "matrix-widget-api";
import {
EndCallButton,
MicButton,
@@ -45,10 +44,9 @@ import {
import { Header, LeftNav, RightNav, RoomHeaderInfo } from "../Header";
import { type HeaderStyle, useUrlParams } from "../UrlParams";
import { useCallViewKeyboardShortcuts } from "../useCallViewKeyboardShortcuts";
import { ElementWidgetActions, widget } from "../widget";
import { widget } from "../widget";
import styles from "./InCallView.module.css";
import { GridTile } from "../tile/GridTile";
import { type OTelGroupCallMembership } from "../otel/OTelGroupCallMembership";
import { SettingsModal, defaultSettingsTab } from "../settings/SettingsModal";
import { useRageshakeRequestModal } from "../settings/submit-rageshake";
import { RageshakeRequestModal } from "./RageshakeRequestModal";
@@ -114,8 +112,10 @@ const logger = rootLogger.getChild("[InCallView]");
const maxTapDurationMs = 400;
export interface ActiveCallProps
extends Omit<InCallViewProps, "vm" | "livekitRoom" | "connState"> {
export interface ActiveCallProps extends Omit<
InCallViewProps,
"vm" | "livekitRoom" | "connState"
> {
e2eeSystem: EncryptionSystem;
// TODO refactor those reasons into an enum
onLeft: (
@@ -189,7 +189,6 @@ export interface InCallViewProps {
matrixRoom: MatrixRoom;
muteStates: MuteStates;
header: HeaderStyle;
otelGroupCallMembership?: OTelGroupCallMembership;
onShareClick: (() => void) | null;
}
@@ -435,46 +434,6 @@ export const InCallView: FC<InCallViewProps> = ({
[vm],
);
useEffect(() => {
widget?.api.transport
.send(
gridMode === "grid"
? ElementWidgetActions.TileLayout
: ElementWidgetActions.SpotlightLayout,
{},
)
.catch((e) => {
logger.error("Failed to send layout change to widget API", e);
});
}, [gridMode]);
useEffect(() => {
if (widget) {
const onTileLayout = (ev: CustomEvent<IWidgetApiRequest>): void => {
setGridMode("grid");
widget!.api.transport.reply(ev.detail, {});
};
const onSpotlightLayout = (ev: CustomEvent<IWidgetApiRequest>): void => {
setGridMode("spotlight");
widget!.api.transport.reply(ev.detail, {});
};
widget.lazyActions.on(ElementWidgetActions.TileLayout, onTileLayout);
widget.lazyActions.on(
ElementWidgetActions.SpotlightLayout,
onSpotlightLayout,
);
return (): void => {
widget!.lazyActions.off(ElementWidgetActions.TileLayout, onTileLayout);
widget!.lazyActions.off(
ElementWidgetActions.SpotlightLayout,
onSpotlightLayout,
);
};
}
}, [setGridMode]);
useAppBarSecondaryButton(
useMemo(() => {
if (audioOutputSwitcher === null) return null;
@@ -798,6 +757,8 @@ export const InCallView: FC<InCallViewProps> = ({
</div>
);
const allConnections = useBehavior(vm.allConnections$);
return (
<div
className={styles.inRoom}
@@ -836,8 +797,15 @@ export const InCallView: FC<InCallViewProps> = ({
onDismiss={closeSettings}
tab={settingsTab}
onTabChange={setSettingsTab}
// TODO expose correct data to setttings modal
livekitRooms={[]}
livekitRooms={allConnections
.getConnections()
.map((connectionItem) => ({
room: connectionItem.livekitRoom,
livekitAlias: connectionItem.livekitAlias,
// TODO compute is local or tag it in the livekit room items already
isLocal: undefined,
url: connectionItem.transport.livekit_service_url,
}))}
/>
</>
)}

View File

@@ -37,6 +37,8 @@ import {
local,
localRtcMember,
} from "../utils/test-fixtures";
import { initializeWidget } from "../widget";
initializeWidget();
function TestComponent({ vm }: { vm: CallViewModel }): ReactNode {
return (

View File

@@ -19,6 +19,8 @@ import {
bobRtcMember,
} from "../utils/test-fixtures";
import { getBasicCallViewModelEnvironment } from "../utils/test-viewmodel";
import { initializeWidget } from "../widget";
initializeWidget();
vi.mock("livekit-client/e2ee-worker?worker");

View File

@@ -1,5 +1,6 @@
/*
Copyright 2021-2024 New Vector Ltd.
Copyright 2026 Element Creations Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
@@ -20,8 +21,6 @@ import {
CheckIcon,
UnknownSolidIcon,
} from "@vector-im/compound-design-tokens/assets/web/icons";
import { useObservable } from "observable-hooks";
import { map } from "rxjs";
import { useClientLegacy } from "../ClientContext";
import { ErrorPage, FullScreenView, LoadingPage } from "../FullScreenView";
@@ -44,10 +43,12 @@ import { ErrorView } from "../ErrorView";
import { useMediaDevices } from "../MediaDevicesContext";
import { MuteStates } from "../state/MuteStates";
import { ObservableScope } from "../state/ObservableScope";
import { calculateInitialMuteState } from "../state/initialMuteState.ts";
export const RoomPage: FC = () => {
const urlParams = useUrlParams();
const { confineToRoom, appPrompt, preload, header, displayName, skipLobby } =
useUrlParams();
urlParams;
const { t } = useTranslation();
const { roomAlias, roomId, viaServers } = useRoomIdentifier();
@@ -68,15 +69,22 @@ export const RoomPage: FC = () => {
const devices = useMediaDevices();
const [muteStates, setMuteStates] = useState<MuteStates | null>(null);
const joined$ = useObservable(
(inputs$) => inputs$.pipe(map(([joined]) => joined)),
[joined],
);
useEffect(() => {
const scope = new ObservableScope();
setMuteStates(new MuteStates(scope, devices, joined$));
setMuteStates(
new MuteStates(
scope,
devices,
calculateInitialMuteState(
urlParams.skipLobby,
urlParams.callIntent,
widget !== null,
),
),
);
return (): void => scope.end();
}, [devices, joined$]);
}, [devices, urlParams]);
useEffect(() => {
// If we've finished loading, are not already authed and we've been given a display name as

View File

@@ -108,7 +108,7 @@ exports[`ConnectionLostError: Action handling should reset error state 1`] = `
class="error"
>
<div
class="_content_o77nw_8 icon"
class="_content_1r8kr_8 icon"
data-size="large"
>
<svg
@@ -133,7 +133,7 @@ exports[`ConnectionLostError: Action handling should reset error state 1`] = `
You were disconnected from the call.
</p>
<button
class="_button_vczzf_8"
class="_button_13vu4_8"
data-kind="secondary"
data-size="lg"
role="button"
@@ -142,7 +142,7 @@ exports[`ConnectionLostError: Action handling should reset error state 1`] = `
Reconnect
</button>
<button
class="_button_vczzf_8 homeLink"
class="_button_13vu4_8 homeLink"
data-kind="tertiary"
data-size="lg"
role="button"
@@ -265,7 +265,7 @@ exports[`should have a close button in widget mode 1`] = `
class="error"
>
<div
class="_content_o77nw_8 icon"
class="_content_1r8kr_8 icon"
data-size="large"
>
<svg
@@ -295,7 +295,7 @@ exports[`should have a close button in widget mode 1`] = `
The server is not configured to work with Element Call. Please contact your server admin (Domain: example.com, Error Code: MISSING_MATRIX_RTC_TRANSPORT).
</p>
<button
class="_button_vczzf_8"
class="_button_13vu4_8"
data-kind="primary"
data-size="lg"
role="button"
@@ -418,7 +418,7 @@ exports[`should render the error page with link back to home 1`] = `
class="error"
>
<div
class="_content_o77nw_8 icon"
class="_content_1r8kr_8 icon"
data-size="large"
>
<svg
@@ -448,7 +448,7 @@ exports[`should render the error page with link back to home 1`] = `
The server is not configured to work with Element Call. Please contact your server admin (Domain: example.com, Error Code: MISSING_MATRIX_RTC_TRANSPORT).
</p>
<button
class="_button_vczzf_8 homeLink"
class="_button_13vu4_8 homeLink"
data-kind="tertiary"
data-size="lg"
role="button"
@@ -571,7 +571,7 @@ exports[`should report correct error for 'Call is not supported' 1`] = `
class="error"
>
<div
class="_content_o77nw_8 icon"
class="_content_1r8kr_8 icon"
data-size="large"
>
<svg
@@ -601,7 +601,7 @@ exports[`should report correct error for 'Call is not supported' 1`] = `
The server is not configured to work with Element Call. Please contact your server admin (Domain: example.com, Error Code: MISSING_MATRIX_RTC_TRANSPORT).
</p>
<button
class="_button_vczzf_8 homeLink"
class="_button_13vu4_8 homeLink"
data-kind="tertiary"
data-size="lg"
role="button"
@@ -724,7 +724,7 @@ exports[`should report correct error for 'Connection lost' 1`] = `
class="error"
>
<div
class="_content_o77nw_8 icon"
class="_content_1r8kr_8 icon"
data-size="large"
>
<svg
@@ -749,7 +749,7 @@ exports[`should report correct error for 'Connection lost' 1`] = `
You were disconnected from the call.
</p>
<button
class="_button_vczzf_8"
class="_button_13vu4_8"
data-kind="secondary"
data-size="lg"
role="button"
@@ -758,7 +758,7 @@ exports[`should report correct error for 'Connection lost' 1`] = `
Reconnect
</button>
<button
class="_button_vczzf_8 homeLink"
class="_button_13vu4_8 homeLink"
data-kind="tertiary"
data-size="lg"
role="button"
@@ -881,7 +881,7 @@ exports[`should report correct error for 'Incompatible browser' 1`] = `
class="error"
>
<div
class="_content_o77nw_8 icon"
class="_content_1r8kr_8 icon"
data-size="large"
>
<svg
@@ -906,7 +906,7 @@ exports[`should report correct error for 'Incompatible browser' 1`] = `
Your web browser does not support encrypted calls. Supported browsers include Chrome, Safari, and Firefox 117+.
</p>
<button
class="_button_vczzf_8 homeLink"
class="_button_13vu4_8 homeLink"
data-kind="tertiary"
data-size="lg"
role="button"
@@ -1029,7 +1029,7 @@ exports[`should report correct error for 'Insufficient capacity' 1`] = `
class="error"
>
<div
class="_content_o77nw_8 icon"
class="_content_1r8kr_8 icon"
data-size="large"
>
<svg
@@ -1054,7 +1054,7 @@ exports[`should report correct error for 'Insufficient capacity' 1`] = `
The server has reached its maximum capacity and you cannot join the call at this time. Try again later, or contact your server admin if the problem persists.
</p>
<button
class="_button_vczzf_8 homeLink"
class="_button_13vu4_8 homeLink"
data-kind="tertiary"
data-size="lg"
role="button"

View File

@@ -17,7 +17,7 @@ exports[`InCallView > rendering > renders 1`] = `
>
<span
aria-label=""
class="_avatar_1qbcf_8 roomAvatar _avatar-imageless_1qbcf_52"
class="_avatar_zysgz_8 roomAvatar _avatar-imageless_zysgz_55"
data-color="1"
data-type="round"
role="img"
@@ -34,7 +34,7 @@ exports[`InCallView > rendering > renders 1`] = `
tabindex="0"
>
<svg
aria-labelledby="«r0»"
aria-labelledby="_r_0_"
class="lock"
data-encrypted="false"
fill="currentColor"
@@ -117,7 +117,7 @@ exports[`InCallView > rendering > renders 1`] = `
data-show="false"
>
<div
class="_content_o77nw_8 icon"
class="_content_1r8kr_8 icon"
data-size="large"
>
<svg
@@ -159,7 +159,7 @@ exports[`InCallView > rendering > renders 1`] = `
Only works while using app
</p>
<button
class="_button_vczzf_8"
class="_button_13vu4_8"
data-kind="primary"
data-size="sm"
role="button"
@@ -286,8 +286,9 @@ exports[`InCallView > rendering > renders 1`] = `
>
<button
aria-disabled="true"
aria-labelledby="«r8»"
class="_button_vczzf_8 _has-icon_vczzf_57 _icon-only_vczzf_50"
aria-label="Unmute microphone"
aria-labelledby="_r_8_"
class="_button_13vu4_8 _has-icon_13vu4_60 _icon-only_13vu4_53"
data-kind="primary"
data-size="lg"
data-testid="incall_mute"
@@ -309,8 +310,9 @@ exports[`InCallView > rendering > renders 1`] = `
</button>
<button
aria-disabled="true"
aria-labelledby="«rd»"
class="_button_vczzf_8 _has-icon_vczzf_57 _icon-only_vczzf_50"
aria-label="Start video"
aria-labelledby="_r_d_"
class="_button_13vu4_8 _has-icon_13vu4_60 _icon-only_13vu4_53"
data-kind="primary"
data-size="lg"
data-testid="incall_videomute"
@@ -331,8 +333,8 @@ exports[`InCallView > rendering > renders 1`] = `
</svg>
</button>
<button
aria-labelledby="«ri»"
class="_button_vczzf_8 _has-icon_vczzf_57 _icon-only_vczzf_50"
aria-labelledby="_r_i_"
class="_button_13vu4_8 _has-icon_13vu4_60 _icon-only_13vu4_53"
data-kind="secondary"
data-size="lg"
role="button"
@@ -352,8 +354,9 @@ exports[`InCallView > rendering > renders 1`] = `
</svg>
</button>
<button
aria-labelledby="«rn»"
class="_button_vczzf_8 endCall _has-icon_vczzf_57 _icon-only_vczzf_50 _destructive_vczzf_107"
aria-label="End call"
aria-labelledby="_r_n_"
class="_button_13vu4_8 endCall _has-icon_13vu4_60 _icon-only_13vu4_53 _destructive_13vu4_110"
data-kind="primary"
data-size="lg"
data-testid="incall_leave"
@@ -378,7 +381,7 @@ exports[`InCallView > rendering > renders 1`] = `
class="toggle layout"
>
<input
aria-labelledby="«rs»"
aria-labelledby="_r_s_"
name="layout"
type="radio"
value="spotlight"
@@ -396,7 +399,7 @@ exports[`InCallView > rendering > renders 1`] = `
/>
</svg>
<input
aria-labelledby="«r11»"
aria-labelledby="_r_11_"
checked=""
name="layout"
type="radio"

View File

@@ -106,22 +106,18 @@ async function joinRoomAfterInvite(
export class CallTerminatedMessage extends Error {
/**
* Creates a new CallTerminatedMessage.
*
* @param icon The icon to display with the message
* @param messageTitle The title of the call ended screen message (translated)
* @param messageBody The message explaining the kind of termination
* (kick, ban, knock reject, etc.) (translated)
* @param reason The user-provided reason for the termination (kick/ban)
*/
public constructor(
/**
* The icon to display with the message.
*/
public readonly icon: ComponentType<SVGAttributes<SVGElement>>,
messageTitle: string,
/**
* The message explaining the kind of termination (kick, ban, knock reject,
* etc.) (translated)
*/
public readonly messageBody: string,
/**
* The user-provided reason for the termination (kick/ban)
*/
public readonly reason?: string,
) {
super(messageTitle);
@@ -232,7 +228,7 @@ export const useLoadGroupCall = (
});
};
const fetchRoom = async (): Promise<Room> => {
const fetchOrCreateRoom = async (): Promise<Room> => {
let room: Room | null = null;
if (roomIdOrAlias[0] === "#") {
const alias = roomIdOrAlias;
@@ -330,8 +326,8 @@ export const useLoadGroupCall = (
return room;
};
const fetchGroupCall = async (): Promise<MatrixRTCSession> => {
const room = await fetchRoom();
const fetchOrCreateGroupCall = async (): Promise<MatrixRTCSession> => {
const room = await fetchOrCreateRoom();
activeRoom.current = room;
logger.debug(`Fetched / joined room ${roomIdOrAlias}`);
@@ -369,7 +365,7 @@ export const useLoadGroupCall = (
if (state.kind === "loading") {
logger.log("Start loading group call");
waitForClientSyncing()
.then(fetchGroupCall)
.then(fetchOrCreateGroupCall)
.then((rtcSession) => setState({ kind: "loaded", rtcSession }))
.then(observeMyMembership)
.catch((error) => setState({ kind: "failed", error }));

View File

@@ -8,3 +8,14 @@ Please see LICENSE in the repository root for full details.
pre {
font-size: var(--font-size-micro);
}
.livekit_room_box {
border: 3px solid var(--cpd-color-bg-subtle-secondary);
border-radius: var(--cpd-space-8x);
padding: var(--cpd-space-4x);
margin-bottom: var(--cpd-space-4x);
margin-top: var(--cpd-space-4x);
li {
font-size: var(--font-size-micro);
}
}

View File

@@ -7,9 +7,9 @@ Please see LICENSE in the repository root for full details.
import { describe, expect, it, vi } from "vitest";
import { render, waitFor } from "@testing-library/react";
import { type Room as LivekitRoom } from "livekit-client";
import type { MatrixClient } from "matrix-js-sdk";
import type { Room as LivekitRoom } from "livekit-client";
import { DeveloperSettingsTab } from "./DeveloperSettingsTab";
// Mock url params hook to avoid environment-dependent snapshot churn.
@@ -25,17 +25,20 @@ function createMockLivekitRoom(
wsUrl: string,
serverInfo: object,
metadata: string,
): { isLocal: boolean; url: string; room: LivekitRoom } {
): { isLocal: boolean; url: string; room: LivekitRoom; livekitAlias: string } {
const mockRoom = {
serverInfo,
metadata,
engine: { client: { ws: { url: wsUrl } } },
localParticipant: { identity: "localParticipantIdentity" },
remoteParticipants: new Map(),
} as unknown as LivekitRoom;
return {
isLocal: true,
url: wsUrl,
room: mockRoom,
livekitAlias: "TestAlias",
};
}
@@ -59,6 +62,7 @@ describe("DeveloperSettingsTab", () => {
room: LivekitRoom;
url: string;
isLocal?: boolean;
livekitAlias: string;
}[] = [
createMockLivekitRoom(
"wss://local-sfu.example.org",
@@ -67,8 +71,11 @@ describe("DeveloperSettingsTab", () => {
),
{
isLocal: false,
livekitAlias: "TestAlias2",
url: "wss://remote-sfu.example.org",
room: {
localParticipant: { identity: "localParticipantIdentity" },
remoteParticipants: new Map(),
serverInfo: { region: "remote", version: "4.5.6" },
metadata: "remote-metadata",
engine: { client: { ws: { url: "wss://remote-sfu.example.org" } } },

View File

@@ -29,6 +29,7 @@ import {
Label,
RadioControl,
} from "@vector-im/compound-web";
import { type Room as LivekitRoom } from "livekit-client";
import { FieldRow, InputField } from "../input/Input";
import {
@@ -42,13 +43,17 @@ import {
customLivekitUrl as customLivekitUrlSetting,
MatrixRTCMode,
} from "./settings";
import type { Room as LivekitRoom } from "livekit-client";
import styles from "./DeveloperSettingsTab.module.css";
import { useUrlParams } from "../UrlParams";
interface Props {
client: MatrixClient;
livekitRooms?: { room: LivekitRoom; url: string; isLocal?: boolean }[];
livekitRooms?: {
room: LivekitRoom;
url: string;
isLocal?: boolean;
livekitAlias?: string;
}[];
env: ImportMetaEnv;
}
@@ -275,8 +280,8 @@ export const DeveloperSettingsTab: FC<Props> = ({
name={matrixRTCModeRadioGroup}
control={
<RadioControl
checked={matrixRTCMode === MatrixRTCMode.Compatibil}
value={MatrixRTCMode.Compatibil}
checked={matrixRTCMode === MatrixRTCMode.Compatibility}
value={MatrixRTCMode.Compatibility}
onChange={onMatrixRTCModeChange}
/>
}
@@ -304,12 +309,14 @@ export const DeveloperSettingsTab: FC<Props> = ({
</InlineField>
</Form>
{livekitRooms?.map((livekitRoom) => (
<>
<h3>
<div className={styles.livekit_room_box}>
<h4>
{t("developer_mode.livekit_sfu", {
url: livekitRoom.url || "unknown",
})}
</h3>
</h4>
<p>LivekitAlias: {livekitRoom.livekitAlias}</p>
<p>connectionState (wont hot reload): {livekitRoom.room.state}</p>
{livekitRoom.isLocal && <p>ws-url: {localSfuUrl?.href}</p>}
<p>
{t("developer_mode.livekit_server_info")}(
@@ -321,7 +328,19 @@ export const DeveloperSettingsTab: FC<Props> = ({
: "undefined"}
{livekitRoom.room.metadata}
</pre>
</>
<p>Local Participant</p>
<pre className={styles.pre}>
{livekitRoom.room.localParticipant.identity}
</pre>
<p>Remote Participants</p>
<ul>
{Array.from(livekitRoom.room.remoteParticipants.keys()).map(
(id) => (
<li key={id}>{id}</li>
),
)}
</ul>
</div>
))}
<p>{t("developer_mode.environment_variables")}</p>
<pre>{JSON.stringify(env, null, 2)}</pre>

View File

@@ -24,7 +24,7 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
class="field inputField"
>
<input
aria-describedby="«r1»"
aria-describedby="_r_1_"
id="duplicateTiles"
min="0"
type="number"
@@ -44,7 +44,7 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
class="field checkboxField"
>
<input
aria-describedby="«r2»"
aria-describedby="_r_2_"
id="debugTileLayout"
type="checkbox"
/>
@@ -81,7 +81,7 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
class="field checkboxField"
>
<input
aria-describedby="«r3»"
aria-describedby="_r_3_"
id="showConnectionStats"
type="checkbox"
/>
@@ -118,7 +118,7 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
class="field checkboxField"
>
<input
aria-describedby="«r4»"
aria-describedby="_r_4_"
id="muteAllAudio"
type="checkbox"
/>
@@ -156,7 +156,7 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
class="field checkboxField"
>
<input
aria-describedby="«r5»"
aria-describedby="_r_5_"
id="alwaysShowIphoneEarpiece"
type="checkbox"
/>
@@ -195,7 +195,7 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
>
<label
class="_label_19upo_59"
for="radix-«r6»"
for="radix-_r_6_"
>
Custom Livekit-url
</label>
@@ -203,9 +203,9 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
class="_controls_17lij_8"
>
<input
aria-describedby="radix-«r7»"
aria-describedby="radix-_r_7_"
class="_control_sqdq4_10"
id="radix-«r6»"
id="radix-_r_6_"
name="input"
title=""
value=""
@@ -213,7 +213,7 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
</div>
<span
class="_message_19upo_85 _help-message_19upo_91"
id="radix-«r7»"
id="radix-_r_7_"
>
Currently, no overwrite is set. Url from well-known or config is used.
</span>
@@ -234,20 +234,20 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
class="_inline-field-control_19upo_44"
>
<div
class="_container_1e0uz_10"
class="_container_1qhtc_10"
>
<input
aria-describedby="radix-«r9» radix-«rb» radix-«rd»"
aria-describedby="radix-_r_9_ radix-_r_b_ radix-_r_d_"
checked=""
class="_input_1e0uz_18"
id="radix-«r8»"
name="«r0»"
class="_input_1qhtc_18"
id="radix-_r_8_"
name="_r_0_"
title=""
type="radio"
value="legacy"
/>
<div
class="_ui_1e0uz_19"
class="_ui_1qhtc_19"
/>
</div>
</div>
@@ -256,13 +256,13 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
>
<label
class="_label_19upo_59"
for="radix-«r8»"
for="radix-_r_8_"
>
Legacy: state events & oldest membership SFU
</label>
<span
class="_message_19upo_85 _help-message_19upo_91"
id="radix-«r9»"
id="radix-_r_9_"
>
Compatible with old versions of EC that do not support multi SFU
</span>
@@ -275,19 +275,19 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
class="_inline-field-control_19upo_44"
>
<div
class="_container_1e0uz_10"
class="_container_1qhtc_10"
>
<input
aria-describedby="radix-«r9» radix-«rb» radix-«rd»"
class="_input_1e0uz_18"
id="radix-«ra»"
name="«r0»"
aria-describedby="radix-_r_9_ radix-_r_b_ radix-_r_d_"
class="_input_1qhtc_18"
id="radix-_r_a_"
name="_r_0_"
title=""
type="radio"
value="compatibil"
value="compatibility"
/>
<div
class="_ui_1e0uz_19"
class="_ui_1qhtc_19"
/>
</div>
</div>
@@ -296,13 +296,13 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
>
<label
class="_label_19upo_59"
for="radix-«ra»"
for="radix-_r_a_"
>
Compatibility: state events & multi SFU
</label>
<span
class="_message_19upo_85 _help-message_19upo_91"
id="radix-«rb»"
id="radix-_r_b_"
>
Compatible with homeservers that do not support sticky events (but all other EC clients are v0.17.0 or later)
</span>
@@ -315,19 +315,19 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
class="_inline-field-control_19upo_44"
>
<div
class="_container_1e0uz_10"
class="_container_1qhtc_10"
>
<input
aria-describedby="radix-«r9» radix-«rb» radix-«rd»"
class="_input_1e0uz_18"
id="radix-«rc»"
name="«r0»"
aria-describedby="radix-_r_9_ radix-_r_b_ radix-_r_d_"
class="_input_1qhtc_18"
id="radix-_r_c_"
name="_r_0_"
title=""
type="radio"
value="matrix_2_0"
/>
<div
class="_ui_1e0uz_19"
class="_ui_1qhtc_19"
/>
</div>
</div>
@@ -336,59 +336,105 @@ exports[`DeveloperSettingsTab > renders and matches snapshot 1`] = `
>
<label
class="_label_19upo_59"
for="radix-«rc»"
for="radix-_r_c_"
>
Matrix 2.0: sticky events & multi SFU
</label>
<span
class="_message_19upo_85 _help-message_19upo_91"
id="radix-«rd»"
id="radix-_r_d_"
>
Compatible only with homservers supporting sticky events and all EC clients v0.17.0 or later
</span>
</div>
</div>
</form>
<h3>
LiveKit SFU: wss://local-sfu.example.org
</h3>
<p>
ws-url:
wss://local-sfu.example.org/
</p>
<p>
LiveKit Server Info
(
local
)
</p>
<pre
class="pre"
<div
class="livekit_room_box"
>
{
<h4>
LiveKit SFU: wss://local-sfu.example.org
</h4>
<p>
LivekitAlias:
TestAlias
</p>
<p>
connectionState (wont hot reload):
</p>
<p>
ws-url:
wss://local-sfu.example.org/
</p>
<p>
LiveKit Server Info
(
local
)
</p>
<pre
class="pre"
>
{
"region": "local",
"version": "1.2.3"
}
local-metadata
</pre>
<h3>
LiveKit SFU: wss://remote-sfu.example.org
</h3>
<p>
LiveKit Server Info
(
remote
)
</p>
<pre
class="pre"
local-metadata
</pre>
<p>
Local Participant
</p>
<pre
class="pre"
>
localParticipantIdentity
</pre>
<p>
Remote Participants
</p>
<ul />
</div>
<div
class="livekit_room_box"
>
{
<h4>
LiveKit SFU: wss://remote-sfu.example.org
</h4>
<p>
LivekitAlias:
TestAlias2
</p>
<p>
connectionState (wont hot reload):
</p>
<p>
LiveKit Server Info
(
remote
)
</p>
<pre
class="pre"
>
{
"region": "remote",
"version": "4.5.6"
}
remote-metadata
</pre>
remote-metadata
</pre>
<p>
Local Participant
</p>
<pre
class="pre"
>
localParticipantIdentity
</pre>
<p>
Remote Participants
</p>
<ul />
</div>
<p>
Environment variables
</p>

View File

@@ -99,7 +99,7 @@ class ConsoleLogger extends EventEmitter {
/**
* Returns the log lines to flush to disk and empties the internal log buffer
* @return {string} \n delimited log lines
* @return \n delimited log lines
*/
public popLogs(): string {
const logsToFlush = this.logs;
@@ -109,7 +109,7 @@ class ConsoleLogger extends EventEmitter {
/**
* Returns lines currently in the log buffer without removing them
* @return {string} \n delimited log lines
* @return \n delimited log lines
*/
public peekLogs(): string {
return this.logs;
@@ -139,7 +139,7 @@ class IndexedDBLogStore {
}
/**
* @return {Promise} Resolves when the store is ready.
* @return Resolves when the store is ready.
*/
public async connect(): Promise<void> {
const req = this.indexedDB.open("logs");
@@ -219,7 +219,7 @@ class IndexedDBLogStore {
* This guarantees that we will always eventually do a flush when flush() is
* called.
*
* @return {Promise} Resolved when the logs have been flushed.
* @return Resolved when the logs have been flushed.
*/
public flush = async (): Promise<void> => {
// check if a flush() operation is ongoing
@@ -270,7 +270,7 @@ class IndexedDBLogStore {
* returned are deleted at the same time, so this can be called at startup
* to do house-keeping to keep the logs from growing too large.
*
* @return {Promise<Object[]>} Resolves to an array of objects. The array is
* @return Resolves to an array of objects. The array is
* sorted in time (oldest first) based on when the log file was created (the
* log ID). The objects have said log ID in an "id" field and "lines" which
* is a big string with all the new-line delimited logs.
@@ -421,12 +421,12 @@ class IndexedDBLogStore {
/**
* Helper method to collect results from a Cursor and promiseify it.
* @param {ObjectStore|Index} store The store to perform openCursor on.
* @param {IDBKeyRange=} keyRange Optional key range to apply on the cursor.
* @param {Function} resultMapper A function which is repeatedly called with a
* @param store - The store to perform openCursor on.
* @param keyRange - Optional key range to apply on the cursor.
* @param resultMapper - A function which is repeatedly called with a
* Cursor.
* Return the data you want to keep.
* @return {Promise<T[]>} Resolves to an array of whatever you returned from
* @return Resolves to an array of whatever you returned from
* resultMapper.
*/
async function selectQuery<T>(
@@ -464,9 +464,7 @@ declare global {
/**
* Configure rage shaking support for sending bug reports.
* Modifies globals.
* @param {boolean} setUpPersistence When true (default), the persistence will
* be set up immediately for the logs.
* @return {Promise} Resolves when set up.
* @return Resolves when set up.
*/
export async function init(): Promise<void> {
global.mx_rage_logger = new ConsoleLogger();
@@ -503,7 +501,7 @@ export async function init(): Promise<void> {
/**
* Try to start up the rageshake storage for logs. If not possible (client unsupported)
* then this no-ops.
* @return {Promise} Resolves when complete.
* @return Resolves when complete.
*/
async function tryInitStorage(): Promise<void> {
if (global.mx_rage_initStoragePromise) {
@@ -536,7 +534,7 @@ async function tryInitStorage(): Promise<void> {
/**
* Get a recent snapshot of the logs, ready for attaching to a bug report
*
* @return {LogEntry[]} list of log data
* @return list of log data
*/
export async function getLogsForReport(): Promise<LogEntry[]> {
if (!global.mx_rage_logger) {

View File

@@ -126,7 +126,13 @@ export const alwaysShowIphoneEarpiece = new Setting<boolean>(
export enum MatrixRTCMode {
Legacy = "legacy",
Compatibil = "compatibil",
Compatibility = "compatibility",
/** This implies using
* - sticky events
* - hashed RTC backend identity
* - the new endpoint for the jwt token on the local membership (remote memberships will always try the new jwt endpoint first -> then the legacy one)
* - use the hashed identity for the local membership
*/
Matrix_2_0 = "matrix_2_0",
}

View File

@@ -17,7 +17,6 @@ import { type CryptoApi } from "matrix-js-sdk/lib/crypto-api";
import { getLogsForReport } from "./rageshake";
import { useClient } from "../ClientContext";
import { Config } from "../config/Config";
import { ElementCallOpenTelemetry } from "../otel/otel";
import { type RageshakeRequestModal } from "../room/RageshakeRequestModal";
import { getUrlParams } from "../UrlParams";
@@ -274,14 +273,6 @@ export function useSubmitRageshake(
for (const entry of logs) {
body.append("compressed-log", await gzip(entry.lines), entry.id);
}
body.append(
"file",
await gzip(
ElementCallOpenTelemetry.instance.rageshakeProcessor!.dump(),
),
"traces.json.gz",
);
}
if (opts.rageshakeRequestId) {

View File

@@ -5,10 +5,6 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import {
type ICallNotifyContent,
type IRTCNotificationContent,
} from "matrix-js-sdk/lib/matrixrtc";
import { describe, it } from "vitest";
import {
EventType,
@@ -25,23 +21,23 @@ import {
localRtcMember,
} from "../../utils/test-fixtures";
import {
type CallNotificationWrapper,
createCallNotificationLifecycle$,
type Props as CallNotificationLifecycleProps,
} from "./CallNotificationLifecycle";
import { trackEpoch } from "../ObservableScope";
const mockLegacyRingEvent = {} as { event_id: string } & ICallNotifyContent;
function mockRingEvent(
eventId: string,
lifetimeMs: number | undefined,
sender = local.userId,
): { event_id: string } & IRTCNotificationContent {
): CallNotificationWrapper {
return {
event_id: eventId,
...(lifetimeMs === undefined ? {} : { lifetime: lifetimeMs }),
notification_type: "ring",
sender,
} as unknown as { event_id: string } & IRTCNotificationContent;
} as unknown as CallNotificationWrapper;
}
describe("waitForCallPickup$", () => {
@@ -54,7 +50,7 @@ describe("waitForCallPickup$", () => {
behavior("a", { a: [] }).pipe(trackEpoch()),
),
sentCallNotification$: hot("10ms a", {
a: [mockRingEvent("$notif1", 30), mockLegacyRingEvent],
a: mockRingEvent("$notif1", 30),
}),
receivedDecline$: hot(""),
options: {
@@ -86,7 +82,7 @@ describe("waitForCallPickup$", () => {
}).pipe(trackEpoch()),
),
sentCallNotification$: hot("5ms a", {
a: [mockRingEvent("$notif2", 100), mockLegacyRingEvent],
a: mockRingEvent("$notif2", 100),
}),
receivedDecline$: hot(""),
options: {
@@ -115,7 +111,7 @@ describe("waitForCallPickup$", () => {
}).pipe(trackEpoch()),
),
sentCallNotification$: hot("20ms a", {
a: [mockRingEvent("$notif2", 50), mockLegacyRingEvent],
a: mockRingEvent("$notif2", 50),
}),
receivedDecline$: hot(""),
options: {
@@ -142,7 +138,7 @@ describe("waitForCallPickup$", () => {
}).pipe(trackEpoch()),
),
sentCallNotification$: hot("10ms a", {
a: [mockRingEvent("$notif2", undefined), mockLegacyRingEvent],
a: mockRingEvent("$notif2", undefined),
}),
receivedDecline$: hot(""),
options: {
@@ -171,7 +167,7 @@ describe("waitForCallPickup$", () => {
}).pipe(trackEpoch()),
),
sentCallNotification$: hot("10ms a", {
a: [mockRingEvent("$notif5", 30), mockLegacyRingEvent],
a: mockRingEvent("$notif5", 30),
}),
receivedDecline$: hot(""),
options: {
@@ -210,7 +206,7 @@ describe("waitForCallPickup$", () => {
}).pipe(trackEpoch()),
),
sentCallNotification$: hot("10ms a", {
a: [mockRingEvent("$decl1", 50), mockLegacyRingEvent],
a: mockRingEvent("$decl1", 50),
}),
receivedDecline$: hot("40ms d", {
d: [
@@ -254,7 +250,7 @@ describe("waitForCallPickup$", () => {
}).pipe(trackEpoch()),
),
sentCallNotification$: hot("10ms a", {
a: [mockRingEvent("$decl", 20), mockLegacyRingEvent],
a: mockRingEvent("$decl", 20),
}),
receivedDecline$: hot("40ms d", {
d: [
@@ -305,7 +301,7 @@ describe("waitForCallPickup$", () => {
}).pipe(trackEpoch()),
),
sentCallNotification$: hot("10ms a", {
a: [mockRingEvent("$right", 50), mockLegacyRingEvent],
a: mockRingEvent("$right", 50),
}),
receivedDecline$: hot("20ms d", {
d: [

View File

@@ -7,9 +7,9 @@ Please see LICENSE in the repository root for full details.
import {
type CallMembership,
type IRTCNotificationContent,
type MatrixRTCSession,
MatrixRTCSessionEvent,
type MatrixRTCSessionEventHandlerMap,
} from "matrix-js-sdk/lib/matrixrtc";
import {
combineLatest,
@@ -38,6 +38,7 @@ import {
import { type Behavior } from "../Behavior";
import { type Epoch, mapEpoch, type ObservableScope } from "../ObservableScope";
export type AutoLeaveReason = "allOthersLeft" | "timeout" | "decline";
export type CallPickupState =
| "unknown"
@@ -46,9 +47,11 @@ export type CallPickupState =
| "decline"
| "success"
| null;
export type CallNotificationWrapper = Parameters<
MatrixRTCSessionEventHandlerMap[MatrixRTCSessionEvent.DidSendCallNotification]
>;
export type CallNotificationWrapper = {
event_id: string;
} & IRTCNotificationContent;
export function createSentCallNotification$(
scope: ObservableScope,
matrixRTCSession: MatrixRTCSession,
@@ -80,8 +83,9 @@ export interface Props {
options: { waitForCallPickup?: boolean; autoLeaveWhenOthersLeft?: boolean };
localUser: { deviceId: string; userId: string };
}
/**
* @returns {callPickupState$, autoLeave$}
* @returns two observables:
* `callPickupState$` The current call pickup state of the call.
* - "unknown": The client has not yet sent the notification event. We don't know if it will because it first needs to send its own membership.
* Then we can conclude if we were the first one to join or not.
@@ -140,12 +144,12 @@ export function createCallNotificationLifecycle$({
scope.behavior(
sentCallNotification$.pipe(
filter(
(newAndLegacyEvents) =>
(notificationEventArgs: CallNotificationWrapper | null) =>
// only care about new events (legacy do not have decline pattern)
newAndLegacyEvents?.[0].notification_type === "ring",
notificationEventArgs?.notification_type === "ring",
),
map((e) => e as CallNotificationWrapper),
switchMap(([notificationEvent]) => {
switchMap((notificationEvent) => {
const lifetimeMs = notificationEvent?.lifetime ?? 0;
return concat(
lifetimeMs === 0

View File

@@ -29,7 +29,6 @@ import {
Status,
type CallMembership,
type IRTCNotificationContent,
type ICallNotifyContent,
MatrixRTCSessionEvent,
type LivekitTransport,
} from "matrix-js-sdk/lib/matrixrtc";
@@ -62,6 +61,9 @@ import { getValue } from "../../utils/observable.ts";
import { type Behavior, constant } from "../Behavior.ts";
import { withCallViewModel as withCallViewModelInMode } from "./CallViewModelTestUtils.ts";
import { MatrixRTCMode } from "../../settings/settings.ts";
import { initializeWidget } from "../../widget.ts";
initializeWidget();
vi.mock("rxjs", async (importOriginal) => ({
...(await importOriginal()),
@@ -78,11 +80,14 @@ vi.mock("../e2ee/matrixKeyProvider");
const getUrlParams = vi.hoisted(() => vi.fn(() => ({})));
vi.mock("../UrlParams", () => ({ getUrlParams }));
vi.mock("../rtcSessionHelpers", async (importOriginal) => ({
...(await importOriginal()),
makeTransport: async (): Promise<LivekitTransport> =>
Promise.resolve(exampleTransport),
}));
vi.mock(
"../state/CallViewModel/localMember/localTransport",
async (importOriginal) => ({
...(await importOriginal()),
makeTransport: async (): Promise<LivekitTransport> =>
Promise.resolve(exampleTransport),
}),
);
const yesNo = {
y: true,
@@ -226,13 +231,9 @@ function mockRingEvent(
} as unknown as { event_id: string } & IRTCNotificationContent;
}
// The app doesn't really care about the content of these legacy events, we just
// need a value to fill in for them when emitting notifications
const mockLegacyRingEvent = {} as { event_id: string } & ICallNotifyContent;
describe.each([
[MatrixRTCMode.Legacy],
[MatrixRTCMode.Compatibil],
[MatrixRTCMode.Compatibility],
[MatrixRTCMode.Matrix_2_0],
])("CallViewModel (%s mode)", (mode) => {
const withCallViewModel = withCallViewModelInMode(mode);
@@ -1103,7 +1104,6 @@ describe.each([
rtcSession.emit(
MatrixRTCSessionEvent.DidSendCallNotification,
mockRingEvent("$notif1", 30),
mockLegacyRingEvent,
);
},
});
@@ -1145,7 +1145,6 @@ describe.each([
rtcSession.emit(
MatrixRTCSessionEvent.DidSendCallNotification,
mockRingEvent("$notif2", 100),
mockLegacyRingEvent,
);
},
d: () => {
@@ -1255,11 +1254,6 @@ describe.each([
y: () => {
rtcSession.membershipStatus = Status.Connected;
},
n: () => {
// NOTE: This was removed in https://github.com/matrix-org/matrix-js-sdk/pull/5103 accidentally.
// eslint-disable-next-line @typescript-eslint/no-explicit-any
rtcSession.membershipStatus = "Reconnecting" as any;
},
});
schedule(probablyLeftMarbles, {
y: () => {

View File

@@ -41,10 +41,14 @@ import {
} from "rxjs";
import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
import {
type LivekitTransport,
MembershipManagerEvent,
type LivekitTransportConfig,
type MatrixRTCSession,
} from "matrix-js-sdk/lib/matrixrtc";
import { type IWidgetApiRequest } from "matrix-widget-api";
import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
import { v4 as uuidv4 } from "uuid";
import { type IMembershipManager } from "matrix-js-sdk/lib/matrixrtc/IMembershipManager";
import {
LocalUserMediaViewModel,
@@ -98,26 +102,32 @@ import {
type SpotlightLandscapeLayoutMedia,
type SpotlightPortraitLayoutMedia,
} from "../layout-types.ts";
import { ElementCallError } from "../../utils/errors.ts";
import { type ObservableScope } from "../ObservableScope.ts";
import { ElementCallError, UnknownCallError } from "../../utils/errors.ts";
import { type Epoch, type ObservableScope } from "../ObservableScope.ts";
import { createHomeserverConnected$ } from "./localMember/HomeserverConnected.ts";
import {
createLocalMembership$,
enterRTCSession,
TransportState,
} from "./localMember/LocalMember.ts";
import { createLocalTransport$ } from "./localMember/LocalTransport.ts";
import {
createLocalTransport$,
JwtEndpointVersion,
} from "./localMember/LocalTransport.ts";
import {
createMemberships$,
membershipsAndTransports$,
} from "../SessionBehaviors.ts";
import { ECConnectionFactory } from "./remoteMembers/ConnectionFactory.ts";
import { createConnectionManager$ } from "./remoteMembers/ConnectionManager.ts";
import {
type ConnectionManagerData,
createConnectionManager$,
} from "./remoteMembers/ConnectionManager.ts";
import {
createMatrixLivekitMembers$,
type TaggedParticipant,
type LocalMatrixLivekitMember,
type RemoteMatrixLivekitMember,
type MatrixLivekitMember,
} from "./remoteMembers/MatrixLivekitMembers.ts";
import {
type AutoLeaveReason,
@@ -207,15 +217,23 @@ export interface CallViewModel {
"unknown" | "ringing" | "timeout" | "decline" | "success" | null
>;
/** Observable that emits when the user should leave the call (hangup pressed, widget action, error).
* THIS DOES NOT LEAVE THE CALL YET. The only way to leave the call (send the hangup event) is by ending the scope.
* THIS DOES NOT LEAVE THE CALL YET. The only way to leave the call (send the hangup event) is
* - by ending the scope
* - or calling requestDisconnect
*
* TODO: it seems more reasonable to add a leave() method (that calls requestDisconnect) that will then update leave$ and remove the hangup pattern
*/
leave$: Observable<"user" | AutoLeaveReason>;
/** Call to initiate hangup. Use in conbination with reconnectino state track the async hangup process. */
/** Call to initiate hangup. Use in conbination with reconnection state track the async hangup process. */
hangup: () => void;
// joining
join: () => void;
/**
* calls requestDisconnect. The async leave state can than be observed via connected$
*/
leave: () => void;
// screen sharing
/**
* Callback to toggle screen sharing. If null, screen sharing is not possible.
@@ -261,6 +279,7 @@ export interface CallViewModel {
* multiple devices.
*/
participantCount$: Behavior<number>;
allConnections$: Behavior<ConnectionManagerData>;
/** Participants sorted by livekit room so they can be used in the audio rendering */
livekitRoomItems$: Behavior<LivekitRoomItem[]>;
userMedia$: Behavior<UserMedia[]>;
@@ -381,8 +400,11 @@ export function createCallViewModel$(
trackProcessorState$: Behavior<ProcessorState>,
): CallViewModel {
const client = matrixRoom.client;
const userId = client.getUserId()!;
const deviceId = client.getDeviceId()!;
const userId = client.getUserId();
const deviceId = client.getDeviceId();
if (!(userId && deviceId))
throw new UnknownCallError(new Error("userId and deviceId are required"));
const livekitKeyProvider = getE2eeKeyProvider(
options.encryptionSystem,
matrixRTCSession,
@@ -415,11 +437,38 @@ export function createCallViewModel$(
memberships$,
);
const ownMembershipIdentity: CallMembershipIdentityParts = {
userId,
deviceId,
// This will only be consumed by the sticky membership manager. So it has no impact on legacy calls.
memberId: uuidv4(),
};
const localTransport$ = createLocalTransport$({
scope: scope,
memberships$: memberships$,
ownMembershipIdentity,
client,
delayId$: scope.behavior(
(
fromEvent(
matrixRTCSession,
MembershipManagerEvent.DelayIdChanged,
// The type of reemitted event includes the original emitted as the second arg.
) as Observable<[string | undefined, IMembershipManager]>
).pipe(map(([delayId]) => delayId ?? null)),
matrixRTCSession.delayId ?? null,
),
roomId: matrixRoom.roomId,
forceJwtEndpoint$: scope.behavior(
matrixRTCMode$.pipe(
map((v) =>
v === MatrixRTCMode.Matrix_2_0
? JwtEndpointVersion.Matrix_2_0
: JwtEndpointVersion.Legacy,
),
),
),
useOldestMember$: scope.behavior(
matrixRTCMode$.pipe(map((v) => v === MatrixRTCMode.Legacy)),
),
@@ -427,6 +476,7 @@ export function createCallViewModel$(
const connectionFactory = new ECConnectionFactory(
client,
matrixRoom.roomId,
mediaDevices,
trackProcessorState$,
livekitKeyProvider,
@@ -439,38 +489,29 @@ export function createCallViewModel$(
const connectionManager = createConnectionManager$({
scope: scope,
connectionFactory: connectionFactory,
inputTransports$: scope.behavior(
combineLatest(
[
localTransport$.pipe(
catchError((e: unknown) => {
logger.info(
"dont pass local transport to createConnectionManager$. localTransport$ threw an error",
e,
);
return of(null);
}),
),
membershipsAndTransports.transports$,
],
(localTransport, transports) => {
const localTransportAsArray = localTransport ? [localTransport] : [];
return transports.mapInner((transports) => [
...localTransportAsArray,
...transports,
]);
},
localTransport$: scope.behavior(
localTransport$.pipe(
catchError((e: unknown) => {
logger.info(
"could not pass local transport to createConnectionManager$. localTransport$ threw an error",
e,
);
return of(null);
}),
),
),
logger,
remoteTransports$: membershipsAndTransports.transports$,
logger: logger,
ownMembershipIdentity,
});
const matrixLivekitMembers$ = createMatrixLivekitMembers$({
scope: scope,
membershipsWithTransport$:
membershipsAndTransports.membershipsWithTransport$,
connectionManager: connectionManager,
});
const matrixLivekitMembers$: Behavior<Epoch<RemoteMatrixLivekitMember[]>> =
createMatrixLivekitMembers$({
scope: scope,
membershipsWithTransport$:
membershipsAndTransports.membershipsWithTransport$,
connectionManager: connectionManager,
});
const connectOptions$ = scope.behavior(
matrixRTCMode$.pipe(
@@ -490,16 +531,16 @@ export function createCallViewModel$(
matrixRTCSession,
),
muteStates: muteStates,
joinMatrixRTC: (transport: LivekitTransport) => {
joinMatrixRTC: (transport: LivekitTransportConfig) => {
return enterRTCSession(
matrixRTCSession,
ownMembershipIdentity,
transport,
connectOptions$.value,
);
},
createPublisherFactory: (connection: Connection) => {
return new Publisher(
scope,
connection,
mediaDevices,
muteStates,
@@ -604,15 +645,14 @@ export function createCallViewModel$(
),
);
const allConnections$ = scope.behavior(
connectionManager.connectionManagerData$.pipe(map((d) => d.value)),
);
const livekitRoomItems$ = scope.behavior(
matrixLivekitMembers$.pipe(
tap((val) => {
logger.debug("matrixLivekitMembers$ updated", val.value);
}),
switchMap((membersWithEpoch) => {
const members = membersWithEpoch.value;
switchMap((members) => {
const a$ = combineLatest(
members.map((member) =>
members.value.map((member) =>
combineLatest([member.connection$, member.participant.value$]).pipe(
map(([connection, participant]) => {
// do not render audio for local participant
@@ -683,60 +723,44 @@ export function createCallViewModel$(
// Generate a collection of MediaItems from the list of expected (whether
// present or missing) LiveKit participants.
generateItems(
"CallViewModel userMedia$",
function* ([
localMatrixLivekitMember,
{ value: matrixLivekitMembers },
matrixLivekitMembers,
duplicateTiles,
]) {
let localParticipantId: string | undefined = undefined;
// add local member if available
if (localMatrixLivekitMember) {
const computeMediaId = (m: MatrixLivekitMember): string =>
`${m.userId}:${m.membership$.value.deviceId}`;
const localUserMediaId = localMatrixLivekitMember
? computeMediaId(localMatrixLivekitMember)
: undefined;
const localAsArray = localMatrixLivekitMember
? [localMatrixLivekitMember]
: [];
const remoteWithoutLocal = matrixLivekitMembers.value.filter(
(m) => computeMediaId(m) !== localUserMediaId,
);
const allMatrixLivekitMembers = [
...localAsArray,
...remoteWithoutLocal,
];
for (const matrixLivekitMember of allMatrixLivekitMembers) {
const { userId, participant, connection$, membership$ } =
localMatrixLivekitMember;
localParticipantId = `${userId}:${membership$.value.deviceId}`; // should be membership$.value.membershipID which is not optional
// const participantId = membership$.value.membershipID;
if (localParticipantId) {
for (let dup = 0; dup < 1 + duplicateTiles; dup++) {
yield {
keys: [
dup,
localParticipantId,
userId,
participant satisfies TaggedParticipant as TaggedParticipant, // Widen the type safely
connection$,
],
data: undefined,
};
}
}
}
// add remote members that are available
for (const {
userId,
participant,
connection$,
membership$,
} of matrixLivekitMembers) {
const participantId = `${userId}:${membership$.value.deviceId}`;
if (participantId === localParticipantId) continue;
// const participantId = membership$.value?.identity;
matrixLivekitMember;
const rtcId = membership$.value.rtcBackendIdentity; // rtcBackendIdentity
const mediaId = computeMediaId(matrixLivekitMember);
for (let dup = 0; dup < 1 + duplicateTiles; dup++) {
yield {
keys: [dup, participantId, userId, participant, connection$],
keys: [dup, mediaId, userId, participant, connection$, rtcId],
data: undefined,
};
}
}
},
(
scope,
_data$,
dup,
participantId,
userId,
participant,
connection$,
) => {
(scope, _, dup, mediaId, userId, participant, connection$, rtcId) => {
const livekitRoom$ = scope.behavior(
connection$.pipe(map((c) => c?.livekitRoom)),
);
@@ -751,8 +775,9 @@ export function createCallViewModel$(
return new UserMedia(
scope,
`${participantId}:${dup}`,
`${mediaId}:${dup}`,
userId,
rtcId,
participant,
options.encryptionSystem,
livekitRoom$,
@@ -761,8 +786,8 @@ export function createCallViewModel$(
localMembership.reconnecting$,
displayName$,
matrixMemberMetadataStore.createAvatarUrlBehavior$(userId),
handsRaised$.pipe(map((v) => v[participantId]?.time ?? null)),
reactions$.pipe(map((v) => v[participantId] ?? undefined)),
handsRaised$.pipe(map((v) => v[mediaId]?.time ?? null)),
reactions$.pipe(map((v) => v[mediaId] ?? undefined)),
);
},
),
@@ -1480,6 +1505,7 @@ export function createCallViewModel$(
leave$: leave$,
hangup: (): void => userHangup$.next(),
join: localMembership.requestJoinAndPublish,
leave: localMembership.requestDisconnect,
toggleScreenSharing: toggleScreenSharing,
sharingScreen$: sharingScreen$,
@@ -1503,6 +1529,7 @@ export function createCallViewModel$(
),
null,
),
allConnections$,
participantCount$: participantCount$,
handsRaised$: handsRaised$,
reactions$: reactions$,

View File

@@ -8,7 +8,7 @@ Please see LICENSE in the repository root for full details.
import {
Status as RTCMemberStatus,
type LivekitTransport,
type LivekitTransportConfig,
type MatrixRTCSession,
} from "matrix-js-sdk/lib/matrixrtc";
import { describe, expect, it, vi } from "vitest";
@@ -24,6 +24,7 @@ import {
mockLivekitRoom,
mockMuteStates,
withTestScheduler,
ownMemberMock,
} from "../../../utils/test";
import {
TransportState,
@@ -38,6 +39,10 @@ import { constant } from "../../Behavior";
import { ConnectionManagerData } from "../remoteMembers/ConnectionManager";
import { ConnectionState, type Connection } from "../remoteMembers/Connection";
import { type Publisher } from "./Publisher";
import { type LocalTransportWithSFUConfig } from "./LocalTransport";
import { initializeWidget } from "../../../widget";
initializeWidget();
const MATRIX_RTC_MODE = MatrixRTCMode.Legacy;
const getUrlParams = vi.hoisted(() => vi.fn(() => ({})));
@@ -103,11 +108,12 @@ describe("LocalMembership", () => {
getOldestMembership: vi.fn().mockReturnValue({
getPreferredFoci: vi.fn().mockReturnValue([focusFromOlderMembership]),
}),
joinRoomSession: vi.fn(),
joinRTCSession: vi.fn(),
}) as unknown as MatrixRTCSession;
enterRTCSession(
mockedSession,
ownMemberMock,
{
livekit_alias: "roomId",
livekit_service_url: "http://my-well-known-service-url.com",
@@ -119,7 +125,12 @@ describe("LocalMembership", () => {
},
);
expect(mockedSession.joinRoomSession).toHaveBeenLastCalledWith(
expect(mockedSession.joinRTCSession).toHaveBeenLastCalledWith(
{
deviceId: "DEVICE",
memberId: "@alice:example.org:DEVICE",
userId: "@alice:example.org",
},
[
{
livekit_alias: "roomId",
@@ -161,11 +172,12 @@ describe("LocalMembership", () => {
},
memberships: [],
getFocusInUse: vi.fn(),
joinRoomSession: vi.fn(),
joinRTCSession: vi.fn(),
}) as unknown as MatrixRTCSession;
enterRTCSession(
mockedSession,
ownMemberMock,
{
livekit_alias: "roomId",
livekit_service_url: "http://my-well-known-service-url.com",
@@ -185,8 +197,8 @@ describe("LocalMembership", () => {
matrixRTCMode: MatrixRTCMode.Matrix_2_0,
}),
matrixRTCSession: {
updateCallIntent: () => {},
leaveRoomSession: () => {},
updateCallIntent: vi.fn().mockReturnValue(Promise.resolve()),
leaveRoomSession: vi.fn(),
} as unknown as MatrixRTCSession,
muteStates: mockMuteStates(),
trackProcessorState$: constant({
@@ -204,10 +216,11 @@ describe("LocalMembership", () => {
it("throws error on missing RTC config error", () => {
withTestScheduler(({ scope, hot, expectObservable }) => {
const localTransport$ = scope.behavior<null | LivekitTransport>(
hot("1ms #", {}, new MatrixRTCTransportMissingError("domain.com")),
null,
);
const localTransport$ =
scope.behavior<null | LocalTransportWithSFUConfig>(
hot("1ms #", {}, new MatrixRTCTransportMissingError("domain.com")),
null,
);
// we do not need any connection data since we want to fail before reaching that.
const mockConnectionManager = {
@@ -234,12 +247,55 @@ describe("LocalMembership", () => {
});
});
it("logs if callIntent cannot be updated", async () => {
const scope = new ObservableScope();
const localTransport$ = new BehaviorSubject(aTransport);
const mockConnectionManager = {
transports$: constant(new Epoch([])),
connectionManagerData$: constant(new Epoch(new ConnectionManagerData())),
};
async function reject(): Promise<void> {
return Promise.reject(new Error("Not connected yet"));
}
const localMembership = createLocalMembership$({
scope,
...defaultCreateLocalMemberValues,
matrixRTCSession: {
updateCallIntent: vi.fn().mockImplementation(reject),
leaveRoomSession: vi.fn(),
},
connectionManager: mockConnectionManager,
localTransport$,
});
const expextedLog =
"'not connected yet' while updating the call intent (this is expected on startup)";
const internalLogger = vi.spyOn(localMembership.internalLoggerRef, "debug");
await flushPromises();
defaultCreateLocalMemberValues.muteStates.video.setEnabled$.value?.(true);
expect(internalLogger).toHaveBeenCalledWith(expextedLog);
scope.end();
});
const aTransport = {
livekit_service_url: "a",
} as LivekitTransport;
transport: {
livekit_service_url: "a",
} as LivekitTransportConfig,
sfuConfig: {
url: "sfu-url",
jwt: "sfu-token",
},
} as LocalTransportWithSFUConfig;
const bTransport = {
livekit_service_url: "b",
} as LivekitTransport;
transport: {
livekit_service_url: "b",
} as LivekitTransportConfig,
sfuConfig: {
url: "sfu-url",
jwt: "sfu-token",
},
} as LocalTransportWithSFUConfig;
const connectionTransportAConnected = {
livekitRoom: mockLivekitRoom({
@@ -249,7 +305,7 @@ describe("LocalMembership", () => {
} as unknown as LocalParticipant,
}),
state$: constant(ConnectionState.LivekitConnected),
transport: aTransport,
transport: aTransport.transport,
} as unknown as Connection;
const connectionTransportAConnecting = {
...connectionTransportAConnected,
@@ -258,11 +314,11 @@ describe("LocalMembership", () => {
} as unknown as Connection;
const connectionTransportBConnected = {
state$: constant(ConnectionState.LivekitConnected),
transport: bTransport,
transport: bTransport.transport,
livekitRoom: mockLivekitRoom({}),
} as unknown as Connection;
it("recreates publisher if new connection is used and ENDS always unpublish and end tracks", async () => {
it("recreates publisher if new connection is used, always unpublish and end tracks", async () => {
const scope = new ObservableScope();
const localTransport$ = new BehaviorSubject(aTransport);
@@ -275,6 +331,9 @@ describe("LocalMembership", () => {
seed += 1;
logger.info(`creating [${a}]`);
const p = {
// It is enought to check if destroy is called. Destroy itself is tested in the publisher to make sure it does
// all the cleanup we need.
destroy: vi.fn(),
stopPublishing: vi.fn().mockImplementation(() => {
logger.info(`stopPublishing [${a}]`);
}),
@@ -303,19 +362,22 @@ describe("LocalMembership", () => {
await flushPromises();
localTransport$.next(bTransport);
await flushPromises();
expect(publisherFactory).toHaveBeenCalledTimes(2);
expect(publishers.length).toBe(2);
// stop the first Publisher and let the second one life.
expect(publishers[0].stopTracks).toHaveBeenCalled();
expect(publishers[1].stopTracks).not.toHaveBeenCalled();
expect(publishers[0].stopPublishing).toHaveBeenCalled();
expect(publishers[1].stopPublishing).not.toHaveBeenCalled();
expect(publisherFactory.mock.calls[0][0].transport).toBe(aTransport);
expect(publisherFactory.mock.calls[1][0].transport).toBe(bTransport);
expect(publishers[0].destroy).toHaveBeenCalled();
expect(publishers[1].destroy).not.toHaveBeenCalled();
expect(publisherFactory.mock.calls[0][0].transport).toBe(
aTransport.transport,
);
expect(publisherFactory.mock.calls[1][0].transport).toBe(
bTransport.transport,
);
scope.end();
await flushPromises();
// stop all tracks after ending scopes
expect(publishers[1].stopPublishing).toHaveBeenCalled();
expect(publishers[1].destroy).toHaveBeenCalled();
// expect(publishers[1].stopTracks).toHaveBeenCalled();
defaultCreateLocalMemberValues.createPublisherFactory.mockReset();
@@ -333,8 +395,9 @@ describe("LocalMembership", () => {
defaultCreateLocalMemberValues.createPublisherFactory.mockImplementation(
() => {
const p = {
stopPublishing: vi.fn(),
stopTracks: vi.fn(),
// It is enought to check if destroy is called. Destroy itself is tested in the publisher to make sure it does
// all the cleanup we need.
destroy: vi.fn(),
createAndSetupTracks: vi.fn().mockImplementation(async () => {
tracks$.next([{}, {}] as LocalTrack[]);
return Promise.resolve();
@@ -369,11 +432,11 @@ describe("LocalMembership", () => {
localMembership.startTracks();
await flushPromises();
expect(publishers[0].createAndSetupTracks).toHaveBeenCalled();
// expect(localMembership.tracks$.value.length).toBe(2);
scope.end();
await flushPromises();
// stop all tracks after ending scopes
expect(publishers[0].stopPublishing).toHaveBeenCalled();
expect(publishers[0].destroy).toHaveBeenCalled();
// expect(publishers[0].stopTracks).toHaveBeenCalled();
publisherFactory.mockClear();
});
@@ -383,33 +446,29 @@ describe("LocalMembership", () => {
const scope = new ObservableScope();
const connectionManagerData = new ConnectionManagerData();
const localTransport$ = new BehaviorSubject<null | LivekitTransport>(null);
const localTransport$ =
new BehaviorSubject<null | LocalTransportWithSFUConfig>(null);
const connectionManagerData$ = new BehaviorSubject(
new Epoch(connectionManagerData),
);
const publishers: Publisher[] = [];
const tracks$ = new BehaviorSubject<LocalTrack[]>([]);
const publishing$ = new BehaviorSubject<boolean>(false);
const createTrackResolver = Promise.withResolvers<void>();
const publishResolver = Promise.withResolvers<void>();
defaultCreateLocalMemberValues.createPublisherFactory.mockImplementation(
() => {
const p = {
stopPublishing: vi.fn(),
stopTracks: vi.fn().mockImplementation(() => {
logger.info("stopTracks");
tracks$.next([]);
}),
// It is enought to check if destroy is called. Destroy itself is tested in the publisher to make sure it does
// all the cleanup we need.
destroy: vi.fn(),
createAndSetupTracks: vi.fn().mockImplementation(async () => {
await createTrackResolver.promise;
tracks$.next([{}, {}] as LocalTrack[]);
}),
startPublishing: vi.fn().mockImplementation(async () => {
await publishResolver.promise;
publishing$.next(true);
}),
tracks$,
publishing$,
};
publishers.push(p as unknown as Publisher);
@@ -460,7 +519,7 @@ describe("LocalMembership", () => {
});
(
connectionManagerData2.getConnectionForTransport(aTransport)!
connectionManagerData2.getConnectionForTransport(aTransport.transport)!
.state$ as BehaviorSubject<ConnectionState>
).next(ConnectionState.LivekitConnected);
expect(localMembership.localMemberState$.value).toStrictEqual({
@@ -509,7 +568,7 @@ describe("LocalMembership", () => {
(localMembership.localMemberState$.value as any).media,
).toStrictEqual(PublishState.Publishing);
expect(publishers[0].stopPublishing).not.toHaveBeenCalled();
expect(publishers[0].destroy).not.toHaveBeenCalled();
expect(localMembership.localMemberState$.isStopped).toBe(false);
scope.end();
@@ -520,7 +579,7 @@ describe("LocalMembership", () => {
(localMembership.localMemberState$.value as any).media,
).toStrictEqual(PublishState.Publishing);
// stop all tracks after ending scopes
expect(publishers[0].stopPublishing).toHaveBeenCalled();
expect(publishers[0].destroy).toHaveBeenCalled();
// expect(publishers[0].stopTracks).toHaveBeenCalled();
});
// TODO add tests for matrix local matrix participation.

View File

@@ -17,6 +17,7 @@ import { observeParticipantEvents } from "@livekit/components-core";
import {
Status as RTCSessionStatus,
type LivekitTransport,
type LivekitTransportConfig,
type MatrixRTCSession,
} from "matrix-js-sdk/lib/matrixrtc";
import {
@@ -36,6 +37,7 @@ import {
} from "rxjs";
import { type Logger } from "matrix-js-sdk/lib/logger";
import { deepCompare } from "matrix-js-sdk/lib/utils";
import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
import { type Behavior } from "../../Behavior.ts";
import { type IConnectionManager } from "../remoteMembers/ConnectionManager.ts";
@@ -60,6 +62,7 @@ import {
} from "../remoteMembers/Connection.ts";
import { type HomeserverConnected } from "./HomeserverConnected.ts";
import { and$ } from "../../../utils/observable.ts";
import { type LocalTransportWithSFUConfig } from "./LocalTransport.ts";
export enum TransportState {
/** Not even a transport is available to the LocalMembership */
@@ -123,9 +126,9 @@ interface Props {
muteStates: MuteStates;
connectionManager: IConnectionManager;
createPublisherFactory: (connection: Connection) => Publisher;
joinMatrixRTC: (transport: LivekitTransport) => void;
joinMatrixRTC: (transport: LivekitTransportConfig) => void;
homeserverConnected: HomeserverConnected;
localTransport$: Behavior<LivekitTransport | null>;
localTransport$: Behavior<LocalTransportWithSFUConfig | null>;
matrixRTCSession: Pick<
MatrixRTCSession,
"updateCallIntent" | "leaveRoomSession"
@@ -138,7 +141,16 @@ interface Props {
* We want
* - a publisher
* -
* @param param0
* @param props The properties required to create the local membership.
* @param props.scope The observable scope to use.
* @param props.connectionManager The connection manager to get connections from.
* @param props.createPublisherFactory Factory to create a publisher once we have a connection.
* @param props.joinMatrixRTC Callback to join the matrix RTC session once we have a transport.
* @param props.homeserverConnected The homeserver connected state.
* @param props.localTransport$ The local transport to use for publishing.
* @param props.logger The logger to use.
* @param props.muteStates The mute states for video and audio.
* @param props.matrixRTCSession The matrix RTC session to join.
* @returns
* - publisher: The handle to create tracks and publish them to the room.
* - connected$: the current connection state. Including matrix server and livekit server connection. (only considering the livekit server we are using for our own media publication)
@@ -189,6 +201,7 @@ export const createLocalMembership$ = ({
* Fully connected
*/
connected$: Behavior<boolean>;
internalLoggerRef: Logger;
} => {
const logger = parentLogger.getChild("[LocalMembership]");
logger.debug(`Creating local membership..`);
@@ -224,7 +237,9 @@ export const createLocalMembership$ = ({
return null;
}
return connectionData.getConnectionForTransport(localTransport);
return connectionData.getConnectionForTransport(
localTransport.transport,
);
}),
tap((connection) => {
logger.info(
@@ -253,11 +268,11 @@ export const createLocalMembership$ = ({
mediaErrors$.pipe(scope.bind()).subscribe((error) => {
if (error) {
// This is a MediaDevice error, can be PermissionDenied, NotFound, DeviceInUse, Other.
// Will also occurs if you cancel screen sharing browser prompt.
// This is not necessarily fatal, since the user might be able to join without media.
// XXX We might want to give some user feedback here to let them know their media is not working.
logger.error(`Failed to create local tracks:`, error);
setMatrixError(
// TODO is it fatal? Do we need to create a new Specialized Error?
new UnknownCallError(new Error(`Media device error: ${error}`)),
);
}
});
// MATRIX RELATED
@@ -297,13 +312,17 @@ export const createLocalMembership$ = ({
// - destruct all current streams
// - overwrite current publisher
scope.reconcile(localConnection$, async (connection) => {
logger.info(
"reconcile based on new localConnection:",
connection?.transport.livekit_service_url,
);
if (connection !== null) {
const publisher = createPublisherFactory(connection);
publisher$.next(publisher);
// Clean-up callback
return Promise.resolve(async (): Promise<void> => {
await publisher.stopPublishing();
await publisher.stopTracks();
await publisher.destroy();
});
}
});
@@ -507,12 +526,19 @@ export const createLocalMembership$ = ({
}
});
combineLatest([muteStates.video.enabled$, homeserverConnected.combined$])
.pipe(scope.bind())
.subscribe(([videoEnabled, connected]) => {
if (!connected) return;
void matrixRTCSession.updateCallIntent(videoEnabled ? "video" : "audio");
});
muteStates.video.enabled$.pipe(scope.bind()).subscribe((videoEnabled) => {
void matrixRTCSession
.updateCallIntent(videoEnabled ? "video" : "audio")
.catch((e) => {
if (e instanceof Error && e.message === "Not connected yet") {
logger.debug(
"'not connected yet' while updating the call intent (this is expected on startup)",
);
} else {
throw e;
}
});
});
// Keep matrix rtc session in sync with localTransport$, connectRequested$
scope.reconcile(
@@ -523,7 +549,7 @@ export const createLocalMembership$ = ({
if (!shouldConnect) return;
try {
joinMatrixRTC(transport);
joinMatrixRTC(transport.transport);
} catch (error) {
logger.error("Error entering RTC session", error);
if (error instanceof Error)
@@ -542,7 +568,12 @@ export const createLocalMembership$ = ({
);
const participant$ = scope.behavior(
localConnection$.pipe(map((c) => c?.livekitRoom?.localParticipant ?? null)),
localConnection$.pipe(
map((c) => c?.livekitRoom?.localParticipant ?? null),
tap((p) => {
logger.debug("participant$ updated:", p?.identity);
}),
),
);
// Pause upstream of all local media tracks when we're disconnected from
@@ -651,6 +682,7 @@ export const createLocalMembership$ = ({
sharingScreen$,
toggleScreenSharing,
connection$: localConnection$,
internalLoggerRef: logger,
};
};
@@ -676,17 +708,20 @@ interface EnterRTCSessionOptions {
* - Delay events management
* - Handles retries (fails only after several attempts)
*
* @param rtcSession
* @param transport
* @param options
* @param rtcSession - The MatrixRTCSession to join.
* @param ownMembershipIdentity - Options for entering the RTC session.
* @param transport - The LivekitTransport to use for this session.
* @param options - `encryptMedia`: Whether to encrypt media `matrixRTCMode`: The Matrix RTC mode to use.
* @throws If the widget could not send ElementWidgetActions.JoinCall action.
*/
// Exported for unit testing
export function enterRTCSession(
rtcSession: MatrixRTCSession,
transport: LivekitTransport,
{ encryptMedia, matrixRTCMode }: EnterRTCSessionOptions,
ownMembershipIdentity: CallMembershipIdentityParts,
transport: LivekitTransportConfig,
options: EnterRTCSessionOptions,
): void {
const { encryptMedia, matrixRTCMode } = options;
PosthogAnalytics.instance.eventCallEnded.cacheStartCall(new Date());
PosthogAnalytics.instance.eventCallStarted.track(rtcSession.room.roomId);
@@ -698,12 +733,29 @@ export function enterRTCSession(
const useDeviceSessionMemberEvents =
features?.feature_use_device_session_member_events;
const { sendNotificationType: notificationType, callIntent } = getUrlParams();
const multiSFU = matrixRTCMode !== MatrixRTCMode.Legacy;
const multiSFU =
matrixRTCMode === MatrixRTCMode.Compatibility ||
matrixRTCMode === MatrixRTCMode.Matrix_2_0;
// For backwards compatibility with Element Call versions that do not do Matrix 2.0,
// we add the livekit alias to the transport.
let backwardCompatibleTransport: LivekitTransport | LivekitTransportConfig;
if (matrixRTCMode === MatrixRTCMode.Matrix_2_0) {
backwardCompatibleTransport = transport;
} else {
backwardCompatibleTransport = {
livekit_alias: rtcSession.room.roomId,
...transport,
};
}
// Multi-sfu does not need a preferred foci list. just the focus that is actually used.
// TODO where/how do we track errors originating from the ongoing rtcSession?
rtcSession.joinRoomSession(
multiSFU ? [] : [transport],
multiSFU ? transport : undefined,
rtcSession.joinRTCSession(
ownMembershipIdentity,
multiSFU ? [] : [backwardCompatibleTransport],
multiSFU ? backwardCompatibleTransport : undefined,
{
notificationType,
callIntent,

View File

@@ -18,8 +18,8 @@ import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc";
import { BehaviorSubject, lastValueFrom } from "rxjs";
import fetchMock from "fetch-mock";
import { mockConfig, flushPromises } from "../../../utils/test";
import { createLocalTransport$ } from "./LocalTransport";
import { mockConfig, flushPromises, ownMemberMock } from "../../../utils/test";
import { createLocalTransport$, JwtEndpointVersion } from "./LocalTransport";
import { constant } from "../../Behavior";
import { Epoch, ObservableScope } from "../../ObservableScope";
import {
@@ -34,16 +34,40 @@ describe("LocalTransport", () => {
const openIdResponse: openIDSFU.SFUConfig = {
url: "https://lk.example.org",
jwt: testJWTToken,
livekitAlias: "!example_room_id",
livekitAlias: "Akph4alDMhen",
livekitIdentity: "@lk_user:ABCDEF",
};
let scope: ObservableScope;
beforeEach(() => {
scope = new ObservableScope();
});
beforeEach(() => (scope = new ObservableScope()));
afterEach(() => scope.end());
it("throws if config is missing", async () => {
const localTransport$ = createLocalTransport$({
scope,
roomId: "!room:example.org",
useOldestMember$: constant(false),
memberships$: constant(new Epoch<CallMembership[]>([])),
client: {
// eslint-disable-next-line @typescript-eslint/naming-convention
_unstable_getRTCTransports: async () => Promise.resolve([]),
getDomain: () => "",
baseUrl: "example.org",
// These won't be called in this error path but satisfy the type
getOpenIdToken: vi.fn(),
getDeviceId: vi.fn(),
},
ownMembershipIdentity: ownMemberMock,
forceJwtEndpoint$: constant(JwtEndpointVersion.Legacy),
delayId$: constant("delay_id_mock"),
});
await flushPromises();
expect(() => localTransport$.value).toThrow(
new MatrixRTCTransportMissingError(""),
);
});
it("throws FailToGetOpenIdToken when OpenID fetch fails", async () => {
// Provide a valid config so makeTransportInternal resolves a transport
const scope = new ObservableScope();
@@ -65,6 +89,7 @@ describe("LocalTransport", () => {
useOldestMember$: constant(false),
memberships$: constant(new Epoch<CallMembership[]>([])),
client: {
baseUrl: "https://lk.example.org",
// Use empty domain to skip .well-known and use config directly
getDomain: () => "",
// eslint-disable-next-line @typescript-eslint/naming-convention
@@ -72,6 +97,9 @@ describe("LocalTransport", () => {
getOpenIdToken: vi.fn(),
getDeviceId: vi.fn(),
},
ownMembershipIdentity: ownMemberMock,
forceJwtEndpoint$: constant(JwtEndpointVersion.Legacy),
delayId$: constant("delay_id_mock"),
});
localTransport$.subscribe(
(o) => observations.push(o),
@@ -86,6 +114,59 @@ describe("LocalTransport", () => {
expect(() => localTransport$.value).toThrow(expectedError);
});
it("emits preferred transport after OpenID resolves", async () => {
// Use config so transport discovery succeeds, but delay OpenID JWT fetch
mockConfig({
livekit: { livekit_service_url: "https://lk.example.org" },
});
const openIdResolver = Promise.withResolvers<openIDSFU.SFUConfig>();
vi.spyOn(openIDSFU, "getSFUConfigWithOpenID").mockReturnValue(
openIdResolver.promise,
);
const localTransport$ = createLocalTransport$({
scope,
roomId: "!room:example.org",
useOldestMember$: constant(false),
memberships$: constant(new Epoch<CallMembership[]>([])),
client: {
// eslint-disable-next-line @typescript-eslint/naming-convention
_unstable_getRTCTransports: async () => Promise.resolve([]),
getDomain: () => "",
getOpenIdToken: vi.fn(),
getDeviceId: vi.fn(),
baseUrl: "https://lk.example.org",
},
ownMembershipIdentity: ownMemberMock,
forceJwtEndpoint$: constant(JwtEndpointVersion.Legacy),
delayId$: constant("delay_id_mock"),
});
openIdResolver.resolve?.({
url: "https://lk.example.org",
jwt: "jwt",
livekitAlias: "Akph4alDMhen",
livekitIdentity: ownMemberMock.userId + ":" + ownMemberMock.deviceId,
});
expect(localTransport$.value).toBe(null);
await flushPromises();
// final
expect(localTransport$.value).toStrictEqual({
transport: {
livekit_service_url: "https://lk.example.org",
type: "livekit",
},
sfuConfig: {
jwt: "jwt",
livekitAlias: "Akph4alDMhen",
livekitIdentity: "@alice:example.org:DEVICE",
url: "https://lk.example.org",
},
});
});
it("updates local transport when oldest member changes", async () => {
// Use config so transport discovery succeeds, but delay OpenID JWT fetch
mockConfig({
@@ -109,7 +190,11 @@ describe("LocalTransport", () => {
_unstable_getRTCTransports: async () => Promise.resolve([]),
getOpenIdToken: vi.fn(),
getDeviceId: vi.fn(),
baseUrl: "https://lk.example.org",
},
ownMembershipIdentity: ownMemberMock,
forceJwtEndpoint$: constant(JwtEndpointVersion.Legacy),
delayId$: constant("delay_id_mock"),
});
openIdResolver.resolve?.(openIdResponse);
@@ -117,9 +202,16 @@ describe("LocalTransport", () => {
await flushPromises();
// final
expect(localTransport$.value).toStrictEqual({
livekit_alias: "!example_room_id",
livekit_service_url: "https://lk.example.org",
type: "livekit",
transport: {
livekit_service_url: "https://lk.example.org",
type: "livekit",
},
sfuConfig: {
jwt: "e30=.eyJzdWIiOiJAbWU6ZXhhbXBsZS5vcmc6QUJDREVGIiwidmlkZW8iOnsicm9vbSI6IiFleGFtcGxlX3Jvb21faWQifX0=.e30=",
livekitAlias: "Akph4alDMhen",
livekitIdentity: "@lk_user:ABCDEF",
url: "https://lk.example.org",
},
});
});
@@ -134,11 +226,15 @@ describe("LocalTransport", () => {
mockConfig({});
customLivekitUrl.setValue(customLivekitUrl.defaultValue);
localTransportOpts = {
ownMembershipIdentity: ownMemberMock,
scope,
roomId: "!example_room_id",
useOldestMember$: constant(false),
forceJwtEndpoint$: constant(JwtEndpointVersion.Legacy),
delayId$: constant(null),
memberships$: constant(new Epoch<CallMembership[]>([])),
client: {
baseUrl: "https://example.org",
getDomain: vi.fn().mockReturnValue(""),
// eslint-disable-next-line @typescript-eslint/naming-convention
_unstable_getRTCTransports: vi.fn().mockResolvedValue([]),
@@ -165,9 +261,16 @@ describe("LocalTransport", () => {
expect(localTransport$.value).toBe(null);
await flushPromises();
expect(localTransport$.value).toStrictEqual({
livekit_alias: "!example_room_id",
livekit_service_url: "https://lk.example.org",
type: "livekit",
transport: {
livekit_service_url: "https://lk.example.org",
type: "livekit",
},
sfuConfig: {
jwt: "e30=.eyJzdWIiOiJAbWU6ZXhhbXBsZS5vcmc6QUJDREVGIiwidmlkZW8iOnsicm9vbSI6IiFleGFtcGxlX3Jvb21faWQifX0=.e30=",
livekitAlias: "Akph4alDMhen",
livekitIdentity: "@lk_user:ABCDEF",
url: "https://lk.example.org",
},
});
});
it("supports getting transport via user settings", async () => {
@@ -177,9 +280,16 @@ describe("LocalTransport", () => {
expect(localTransport$.value).toBe(null);
await flushPromises();
expect(localTransport$.value).toStrictEqual({
livekit_alias: "!example_room_id",
livekit_service_url: "https://lk.example.org",
type: "livekit",
transport: {
livekit_service_url: "https://lk.example.org",
type: "livekit",
},
sfuConfig: {
jwt: "e30=.eyJzdWIiOiJAbWU6ZXhhbXBsZS5vcmc6QUJDREVGIiwidmlkZW8iOnsicm9vbSI6IiFleGFtcGxlX3Jvb21faWQifX0=.e30=",
livekitAlias: "Akph4alDMhen",
livekitIdentity: "@lk_user:ABCDEF",
url: "https://lk.example.org",
},
});
});
it("supports getting transport via backend", async () => {
@@ -191,9 +301,16 @@ describe("LocalTransport", () => {
expect(localTransport$.value).toBe(null);
await flushPromises();
expect(localTransport$.value).toStrictEqual({
livekit_alias: "!example_room_id",
livekit_service_url: "https://lk.example.org",
type: "livekit",
transport: {
livekit_service_url: "https://lk.example.org",
type: "livekit",
},
sfuConfig: {
jwt: "e30=.eyJzdWIiOiJAbWU6ZXhhbXBsZS5vcmc6QUJDREVGIiwidmlkZW8iOnsicm9vbSI6IiFleGFtcGxlX3Jvb21faWQifX0=.e30=",
livekitAlias: "Akph4alDMhen",
livekitIdentity: "@lk_user:ABCDEF",
url: "https://lk.example.org",
},
});
});
it("fails fast if the openID request fails for backend config", async () => {
@@ -222,9 +339,16 @@ describe("LocalTransport", () => {
expect(localTransport$.value).toBe(null);
await flushPromises();
expect(localTransport$.value).toStrictEqual({
livekit_alias: "!example_room_id",
livekit_service_url: "https://lk.example.org",
type: "livekit",
transport: {
livekit_service_url: "https://lk.example.org",
type: "livekit",
},
sfuConfig: {
jwt: "e30=.eyJzdWIiOiJAbWU6ZXhhbXBsZS5vcmc6QUJDREVGIiwidmlkZW8iOnsicm9vbSI6IiFleGFtcGxlX3Jvb21faWQifX0=.e30=",
livekitAlias: "Akph4alDMhen",
livekitIdentity: "@lk_user:ABCDEF",
url: "https://lk.example.org",
},
});
expect(fetchMock.done()).toEqual(true);
});
@@ -248,11 +372,15 @@ describe("LocalTransport", () => {
it("throws if no options are available", async () => {
const localTransport$ = createLocalTransport$({
scope,
ownMembershipIdentity: ownMemberMock,
roomId: "!example_room_id",
useOldestMember$: constant(false),
forceJwtEndpoint$: constant(JwtEndpointVersion.Legacy),
delayId$: constant(null),
memberships$: constant(new Epoch<CallMembership[]>([])),
client: {
getDomain: () => "",
baseUrl: "https://example.org",
// eslint-disable-next-line @typescript-eslint/naming-convention
_unstable_getRTCTransports: async () => Promise.resolve([]),
// These won't be called in this error path but satisfy the type

View File

@@ -7,22 +7,22 @@ Please see LICENSE in the repository root for full details.
import {
type CallMembership,
isLivekitTransport,
type LivekitTransport,
isLivekitTransportConfig,
type Transport,
type LivekitTransportConfig,
} from "matrix-js-sdk/lib/matrixrtc";
import { MatrixError, type MatrixClient } from "matrix-js-sdk";
import {
combineLatest,
distinctUntilChanged,
first,
from,
map,
of,
switchMap,
} from "rxjs";
import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
import { AutoDiscovery } from "matrix-js-sdk/lib/autodiscovery";
import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
import { type Behavior } from "../../Behavior.ts";
import { type Epoch, type ObservableScope } from "../../ObservableScope.ts";
@@ -30,9 +30,11 @@ import { Config } from "../../../config/Config.ts";
import {
FailToGetOpenIdToken,
MatrixRTCTransportMissingError,
NoMatrix2AuthorizationService,
} from "../../../utils/errors.ts";
import {
getSFUConfigWithOpenID,
type SFUConfig,
type OpenIDClientParts,
} from "../../../livekit/openIDSFU.ts";
import { areLivekitTransportsEqual } from "../remoteMembers/MatrixLivekitMembers.ts";
@@ -47,11 +49,54 @@ const logger = rootLogger.getChild("[LocalTransport]");
*/
interface Props {
scope: ObservableScope;
ownMembershipIdentity: CallMembershipIdentityParts;
memberships$: Behavior<Epoch<CallMembership[]>>;
client: Pick<MatrixClient, "getDomain" | "_unstable_getRTCTransports"> &
client: Pick<
MatrixClient,
"getDomain" | "baseUrl" | "_unstable_getRTCTransports"
> &
OpenIDClientParts;
// Used by the jwt service to create the livekit room and compute the livekit alias.
roomId: string;
useOldestMember$: Behavior<boolean>;
forceJwtEndpoint$: Behavior<JwtEndpointVersion>;
delayId$: Behavior<string | null>;
}
export enum JwtEndpointVersion {
Legacy = "legacy",
Matrix_2_0 = "matrix_2_0",
}
// TODO livekit_alias-cleanup
// 1. We need to move away from transports map to connections!!!
//
// 2. We need to stop sending livekit_alias all together
//
//
// 1.
// Transports are just the jwt service adress but do not contain the information which room on this transport to use.
// That requires slot and roomId.
//
// We need one connection per room on the transport.
//
// We need an object that contains:
// transport
// roomId
// slotId
//
// To map to the connections. Prosposal: `ConnectionIdentifier`
//
// 2.
// We need to make sure we do not sent livekit_alias in sticky events and that we drop all code for sending state events!
export interface LocalTransportWithSFUConfig {
transport: LivekitTransportConfig;
sfuConfig: SFUConfig;
}
export function isLocalTransportWithSFUConfig(
obj: LivekitTransportConfig | LocalTransportWithSFUConfig,
): obj is LocalTransportWithSFUConfig {
return "transport" in obj && "sfuConfig" in obj;
}
/**
@@ -61,29 +106,60 @@ interface Props {
* @prop useOldestMember Whether to use the same transport as the oldest member.
* This will only update once the first oldest member appears. Will not recompute if the oldest member leaves.
*
* @prop useOldJwtEndpoint$ Whether to set forceOldJwtEndpoint on the returned transport and to use the old JWT endpoint.
* This is used when the connection manager needs to know if it has to use the legacy endpoint which implies a string concatenated rtcBackendIdentity.
* (which is expected for non sticky event based rtc member events)
* @returns The local transport. It will be created using the correct sfu endpoint based on the useOldJwtEndpoint$ value.
* @throws MatrixRTCTransportMissingError | FailToGetOpenIdToken
*/
export const createLocalTransport$ = ({
scope,
memberships$,
ownMembershipIdentity,
client,
roomId,
useOldestMember$,
}: Props): Behavior<LivekitTransport | null> => {
forceJwtEndpoint$,
delayId$,
}: Props): Behavior<LocalTransportWithSFUConfig | null> => {
/**
* The transport over which we should be actively publishing our media.
* undefined when not joined.
*/
const oldestMemberTransport$ = scope.behavior(
memberships$.pipe(
map(
(memberships) =>
memberships.value[0]?.getTransport(memberships.value[0]) ?? null,
const oldestMemberTransport$ =
scope.behavior<LocalTransportWithSFUConfig | null>(
combineLatest([memberships$, useOldestMember$]).pipe(
map(([memberships, useOldestMember]) => {
if (!useOldestMember) return null; // No need to do any prefetching if not using oldest member
const oldestMember = memberships.value[0];
const transport = oldestMember?.getTransport(oldestMember);
if (!transport) return null;
return transport;
}),
switchMap((transport) => {
if (transport !== null && isLivekitTransportConfig(transport)) {
// Get the open jwt token to connect to the sfu
const computeLocalTransportWithSFUConfig =
async (): Promise<LocalTransportWithSFUConfig> => {
return {
transport,
sfuConfig: await getSFUConfigWithOpenID(
client,
ownMembershipIdentity,
transport.livekit_service_url,
roomId,
{ forceJwtEndpoint: JwtEndpointVersion.Legacy },
logger,
),
};
};
return from(computeLocalTransportWithSFUConfig());
}
return of(null);
}),
),
first((t) => t != null && isLivekitTransport(t)),
),
null,
);
null,
);
/**
* The transport that we would personally prefer to publish on (if not for the
@@ -91,9 +167,33 @@ export const createLocalTransport$ = ({
*
* @throws MatrixRTCTransportMissingError | FailToGetOpenIdToken
*/
const preferredTransport$: Behavior<LivekitTransport | null> = scope.behavior(
customLivekitUrl.value$.pipe(
switchMap((customUrl) => from(makeTransport(client, roomId, customUrl))),
const preferredTransport$ = scope.behavior(
// preferredTransport$ (used for multi sfu) needs to know if we are using the old or new
// jwt endpoint (`get_token` vs `sfu/get`) based on that the jwt endpoint will compute the rtcBackendIdentity
// differently. (sha(`${userId}|${deviceId}|${memberId}`) vs `${userId}|${deviceId}|${memberId}`)
// When using sticky events (we need to use the new endpoint).
combineLatest([customLivekitUrl.value$, delayId$, forceJwtEndpoint$]).pipe(
switchMap(([customUrl, delayId, forceEndpoint]) => {
logger.info(
"Creating preferred transport based on: ",
"customUrl: ",
customUrl,
"delayId: ",
delayId,
"forceEndpoint: ",
forceEndpoint,
);
return from(
makeTransport(
client,
ownMembershipIdentity,
roomId,
customUrl,
forceEndpoint,
delayId ?? undefined,
),
);
}),
),
null,
);
@@ -107,12 +207,23 @@ export const createLocalTransport$ = ({
oldestMemberTransport$,
preferredTransport$,
]).pipe(
map(([useOldestMember, oldestMemberTransport, preferredTransport]) =>
useOldestMember
map(([useOldestMember, oldestMemberTransport, preferredTransport]) => {
return useOldestMember
? (oldestMemberTransport ?? preferredTransport)
: preferredTransport,
),
distinctUntilChanged(areLivekitTransportsEqual),
: preferredTransport;
}),
distinctUntilChanged((t1, t2) => {
logger.info(
"Local Transport Update from:",
t1?.transport.livekit_service_url,
" to ",
t2?.transport.livekit_service_url,
);
return areLivekitTransportsEqual(
t1?.transport ?? null,
t2?.transport ?? null,
);
}),
),
);
};
@@ -124,25 +235,62 @@ const FOCI_WK_KEY = "org.matrix.msc4143.rtc_foci";
* validating auth against the service to ensure it's correct.
* Prefers in order:
*
* 1. The `urlFromDevSettings` value. If this cannot be validated, the function will throw.
* 2. The transports returned via the homeserver.
* 3. The transports returned via .well-known.
* 4. The transport configured in Element Call's config.
*
* @param client The authenticated Matrix client for the current user
* @param membership The membership identity of the user.
* @param roomId The ID of the room to be connected to.
* @param urlFromDevSettings Override URL provided by the user's local config.
* @param forceJwtEndpoint Whether to force a specific JWT endpoint
* - `Legacy` / `Matrix_2_0`
* - `get_token` / `sfu/get`
* - not hashing / hashing the backendIdentity
* @param delayId the delay id passed to the jwt service.
*
* @returns A fully validated transport config.
* @throws MatrixRTCTransportMissingError | FailToGetOpenIdToken
*/
async function makeTransport(
client: Pick<MatrixClient, "getDomain" | "_unstable_getRTCTransports"> &
client: Pick<
MatrixClient,
"getDomain" | "baseUrl" | "_unstable_getRTCTransports"
> &
OpenIDClientParts,
membership: CallMembershipIdentityParts,
roomId: string,
urlFromDevSettings: string | null,
): Promise<LivekitTransport> {
forceJwtEndpoint: JwtEndpointVersion,
delayId?: string,
): Promise<LocalTransportWithSFUConfig> {
logger.trace("Searching for a preferred transport");
async function doOpenIdAndJWTFromUrl(
url: string,
): Promise<LocalTransportWithSFUConfig> {
const sfuConfig = await getSFUConfigWithOpenID(
client,
membership,
url,
roomId,
{
forceJwtEndpoint: forceJwtEndpoint,
delayEndpointBaseUrl: client.baseUrl,
delayId,
},
logger,
);
return {
transport: {
type: "livekit",
livekit_service_url: url,
},
sfuConfig,
};
}
// We will call `getSFUConfigWithOpenID` once per transport here as it's our
// only mechanism of valiation. This means we will also ask the
// homeserver for a OpenID token a few times. Since OpenID tokens are single
@@ -153,39 +301,29 @@ async function makeTransport(
// DEVTOOL: Highest priority: Load from devtool setting
if (urlFromDevSettings !== null) {
logger.info("Using LiveKit transport from dev tools: ", urlFromDevSettings);
// Validate that the SFU is up. Otherwise, we want to fail on this
// as we don't permit other SFUs.
const config = await getSFUConfigWithOpenID(
client,
urlFromDevSettings,
roomId,
);
return {
type: "livekit",
livekit_service_url: urlFromDevSettings,
livekit_alias: config.livekitAlias,
};
// This will call the jwt/sfu/get endpoint to pre create the livekit room.
logger.info("Using LiveKit transport from dev tools: ", urlFromDevSettings);
return await doOpenIdAndJWTFromUrl(urlFromDevSettings);
}
async function getFirstUsableTransport(
transports: Transport[],
): Promise<LivekitTransport | null> {
): Promise<LocalTransportWithSFUConfig | null> {
for (const potentialTransport of transports) {
if (isLivekitTransportConfig(potentialTransport)) {
try {
const { livekitAlias } = await getSFUConfigWithOpenID(
client,
// This will call the jwt/sfu/get endpoint to pre create the livekit room.
return await doOpenIdAndJWTFromUrl(
potentialTransport.livekit_service_url,
roomId,
);
return {
...potentialTransport,
livekit_alias: livekitAlias,
};
} catch (ex) {
// Explictly throw these
if (ex instanceof FailToGetOpenIdToken) {
// Explictly throw these
throw ex;
}
if (ex instanceof NoMatrix2AuthorizationService) {
throw ex;
}
logger.debug(
@@ -201,17 +339,22 @@ async function makeTransport(
// MSC4143: Attempt to fetch transports from backend.
if ("_unstable_getRTCTransports" in client) {
try {
const selectedTransport = await getFirstUsableTransport(
await client._unstable_getRTCTransports(),
);
const transportList = await client._unstable_getRTCTransports();
const selectedTransport = await getFirstUsableTransport(transportList);
if (selectedTransport) {
logger.info("Using backend-configured SFU", selectedTransport);
logger.info(
"Using backend-configured (client.getRTCTransports) SFU",
selectedTransport,
);
return selectedTransport;
}
} catch (ex) {
if (ex instanceof MatrixError && ex.httpStatus === 404) {
// Expected, this is an unstable endpoint and it's not required.
logger.debug("Backend does not provide any RTC transports", ex);
// There will be expected 404 errors in the console. When we check if synapse supports the endpoint.
logger.debug(
"Matrix homeserver does not provide any RTC transports via `/rtc/transports` (will retry with well-known.)",
);
} else if (ex instanceof FailToGetOpenIdToken) {
throw ex;
} else {
@@ -245,18 +388,9 @@ async function makeTransport(
const urlFromConf = Config.get().livekit?.livekit_service_url;
if (urlFromConf) {
try {
const { livekitAlias } = await getSFUConfigWithOpenID(
client,
urlFromConf,
roomId,
);
const selectedTransport: LivekitTransport = {
type: "livekit",
livekit_service_url: urlFromConf,
livekit_alias: livekitAlias,
};
logger.info("Using config SFU", selectedTransport);
return selectedTransport;
// This will call the jwt/sfu/get endpoint to pre create the livekit room.
logger.info("Using config SFU", urlFromConf);
return await doOpenIdAndJWTFromUrl(urlFromConf);
} catch (ex) {
if (ex instanceof FailToGetOpenIdToken) {
throw ex;
@@ -265,5 +399,6 @@ async function makeTransport(
}
}
// If we do not have returned a transport by now we throw an error
throw new MatrixRTCTransportMissingError(domain ?? "");
}

View File

@@ -183,7 +183,6 @@ describe("Publisher", () => {
beforeEach(() => {
publisher = new Publisher(
scope,
connection,
mockMediaDevices({}),
muteStates,
@@ -192,7 +191,9 @@ describe("Publisher", () => {
);
});
afterEach(() => {});
afterEach(async () => {
await publisher.destroy();
});
it("Should not create tracks if started muted to avoid unneeded permission requests", async () => {
const createTracksSpy = vi.spyOn(
@@ -207,6 +208,38 @@ describe("Publisher", () => {
expect(createTracksSpy).not.toHaveBeenCalled();
});
it("should unsetHandler and stop tracks on destroy", async () => {
// setup all spies
const unsetVideoSpy = vi.spyOn(
(
publisher as unknown as {
muteStates: { video: { unsetHandler: () => void } };
}
).muteStates.video,
"unsetHandler",
);
const unsetAudioSpy = vi.spyOn(
(
publisher as unknown as {
muteStates: { audio: { unsetHandler: () => void } };
}
).muteStates.audio,
"unsetHandler",
);
const scopeEndSpy = vi.spyOn(
(publisher as unknown as { scope: { end: () => void } }).scope,
"end",
);
const stopTracksSpy = vi.spyOn(publisher, "stopTracks");
// destroy publisher
await publisher.destroy();
expect(stopTracksSpy).toHaveBeenCalledOnce();
expect(unsetVideoSpy).toHaveBeenCalledOnce();
expect(unsetAudioSpy).toHaveBeenCalledOnce();
expect(scopeEndSpy).toHaveBeenCalled();
});
it("Should minimize permission request by querying create at once", async () => {
const enableCameraAndMicrophoneSpy = vi.spyOn(
localParticipant,
@@ -267,7 +300,6 @@ describe("Publisher", () => {
let publisher: Publisher;
beforeEach(() => {
publisher = new Publisher(
scope,
connection,
mockMediaDevices({}),
muteStates,
@@ -275,6 +307,9 @@ describe("Publisher", () => {
logger,
);
});
afterEach(async () => {
await publisher.destroy();
});
test.each([
{ mutes: { audioEnabled: true, videoEnabled: false } },
@@ -320,7 +355,6 @@ describe("Bug fix", () => {
it("wrongly publish tracks while muted", async () => {
// setLogLevel(`debug`);
const publisher = new Publisher(
scope,
connection,
mockMediaDevices({}),
muteStates,
@@ -356,5 +390,6 @@ describe("Bug fix", () => {
expect(track!.mute).toHaveBeenCalled();
expect(track!.isMuted).toBe(true);
}
await publisher.destroy();
});
});

View File

@@ -32,7 +32,7 @@ import {
import { getUrlParams } from "../../../UrlParams.ts";
import { observeTrackReference$ } from "../../MediaViewModel.ts";
import { type Connection } from "../remoteMembers/Connection.ts";
import { type ObservableScope } from "../../ObservableScope.ts";
import { ObservableScope } from "../../ObservableScope.ts";
/**
* A wrapper for a Connection object.
@@ -47,9 +47,10 @@ export class Publisher {
*/
public shouldPublish = false;
private readonly scope = new ObservableScope();
/**
* Creates a new Publisher.
* @param scope - The observable scope to use for managing the publisher.
* @param connection - The connection to use for publishing.
* @param devices - The media devices to use for audio and video input.
* @param muteStates - The mute states for audio and video.
@@ -57,7 +58,6 @@ export class Publisher {
* @param logger - The logger to use for logging :D.
*/
public constructor(
private scope: ObservableScope,
private connection: Pick<Connection, "livekitRoom" | "state$">, //setE2EEEnabled,
devices: MediaDevices,
private readonly muteStates: MuteStates,
@@ -65,7 +65,6 @@ export class Publisher {
private logger: Logger,
) {
const { controlledAudioDevices } = getUrlParams();
const room = connection.livekitRoom;
room.setE2EEEnabled(room.options.e2ee !== undefined)?.catch((e: Error) => {
@@ -73,17 +72,11 @@ export class Publisher {
});
// Setup track processor syncing (blur)
this.observeTrackProcessors(scope, room, trackerProcessorState$);
this.observeTrackProcessors(this.scope, room, trackerProcessorState$);
// Observe media device changes and update LiveKit active devices accordingly
this.observeMediaDevices(scope, devices, controlledAudioDevices);
this.observeMediaDevices(this.scope, devices, controlledAudioDevices);
this.workaroundRestartAudioInputTrackChrome(devices, scope);
this.scope.onEnd(() => {
this.logger.info("Scope ended -> stop publishing all tracks");
void this.stopPublishing();
muteStates.audio.unsetHandler();
muteStates.video.unsetHandler();
});
this.workaroundRestartAudioInputTrackChrome(devices, this.scope);
this.connection.livekitRoom.localParticipant.on(
ParticipantEvent.LocalTrackPublished,
@@ -91,6 +84,21 @@ export class Publisher {
);
}
public async destroy(): Promise<void> {
this.scope.end();
this.logger.info("Scope ended -> unset handler");
this.muteStates.audio.unsetHandler();
this.muteStates.video.unsetHandler();
this.logger.info(`Start to stop tracks`);
try {
await this.stopTracks();
this.logger.info(`Done to stop tracks`);
} catch (e) {
this.logger.error(`Failed to stop tracks: ${e}`);
}
}
// LiveKit will publish the tracks as soon as they are created
// but we want to control when tracks are published.
// We cannot just mute the tracks, even if this will effectively stop the publishing,
@@ -143,7 +151,7 @@ export class Publisher {
this.logger.debug("createAndSetupTracks called");
const lkRoom = this.connection.livekitRoom;
// Observe mute state changes and update LiveKit microphone/camera states accordingly
this.observeMuteStates(this.scope);
this.observeMuteStates();
// Check if audio and/or video is enabled. We only create tracks if enabled,
// because it could prompt for permission, and we don't want to do that unnecessarily.
@@ -356,10 +364,9 @@ export class Publisher {
/**
* Observe changes in the mute states and update the LiveKit room accordingly.
* @param scope
* @private
*/
private observeMuteStates(scope: ObservableScope): void {
private observeMuteStates(): void {
const lkRoom = this.connection.livekitRoom;
this.muteStates.audio.setHandler(async (enable) => {
try {

View File

@@ -26,8 +26,8 @@ import fetchMock from "fetch-mock";
import EventEmitter from "events";
import { type IOpenIDToken } from "matrix-js-sdk";
import { logger } from "matrix-js-sdk/lib/logger";
import { type LivekitTransportConfig } from "matrix-js-sdk/lib/matrixrtc";
import type { LivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
import {
Connection,
ConnectionState,
@@ -40,7 +40,7 @@ import {
FailToGetOpenIdToken,
} from "../../../utils/errors.ts";
import { testJWTToken } from "../../../utils/test-fixtures.ts";
import { mockRemoteParticipant } from "../../../utils/test.ts";
import { mockRemoteParticipant, ownMemberMock } from "../../../utils/test.ts";
let testScope: ObservableScope;
@@ -51,8 +51,9 @@ let fakeLivekitRoom: MockedObject<LivekitRoom>;
let localParticipantEventEmiter: EventEmitter;
let fakeLocalParticipant: MockedObject<LocalParticipant>;
const livekitFocus: LivekitTransport = {
livekit_alias: "!roomID:example.org",
const ROOM_ID = "!roomID:example.org";
const livekitFocus: LivekitTransportConfig = {
livekit_service_url: "https://matrix-rtc.example.org/livekit/jwt",
type: "livekit",
};
@@ -112,8 +113,10 @@ function setupTest(): void {
function setupRemoteConnection(): Connection {
const opts: ConnectionOpts = {
client: client,
roomId: ROOM_ID,
transport: livekitFocus,
scope: testScope,
ownMembershipIdentity: ownMemberMock,
livekitRoomFactory: () => fakeLivekitRoom,
};
@@ -153,8 +156,10 @@ describe("Start connection states", () => {
const opts: ConnectionOpts = {
client: client,
roomId: ROOM_ID,
transport: livekitFocus,
scope: testScope,
ownMembershipIdentity: ownMemberMock,
livekitRoomFactory: () => fakeLivekitRoom,
};
const connection = new Connection(opts, logger);
@@ -168,8 +173,10 @@ describe("Start connection states", () => {
const opts: ConnectionOpts = {
client: client,
roomId: ROOM_ID,
transport: livekitFocus,
scope: testScope,
ownMembershipIdentity: ownMemberMock,
livekitRoomFactory: () => fakeLivekitRoom,
};
@@ -218,8 +225,10 @@ describe("Start connection states", () => {
const opts: ConnectionOpts = {
client: client,
roomId: ROOM_ID,
transport: livekitFocus,
scope: testScope,
ownMembershipIdentity: ownMemberMock,
livekitRoomFactory: () => fakeLivekitRoom,
};
@@ -259,7 +268,7 @@ describe("Start connection states", () => {
capturedState.cause instanceof Error
) {
expect(capturedState.cause.message).toContain(
"SFU Config fetch failed with exception",
"SFU Config fetch failed with status code 500",
);
expect(connection.transport.livekit_alias).toEqual(
livekitFocus.livekit_alias,
@@ -275,8 +284,10 @@ describe("Start connection states", () => {
const opts: ConnectionOpts = {
client: client,
roomId: ROOM_ID,
transport: livekitFocus,
scope: testScope,
ownMembershipIdentity: ownMemberMock,
livekitRoomFactory: () => fakeLivekitRoom,
};

View File

@@ -15,9 +15,10 @@ import {
type Room as LivekitRoom,
type RemoteParticipant,
} from "livekit-client";
import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
import { type LivekitTransportConfig } from "matrix-js-sdk/lib/matrixrtc";
import { BehaviorSubject, map } from "rxjs";
import { type Logger } from "matrix-js-sdk/lib/logger";
import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
import {
getSFUConfigWithOpenID,
@@ -32,12 +33,27 @@ import {
SFURoomCreationRestrictedError,
UnknownCallError,
} from "../../../utils/errors.ts";
import { type JwtEndpointVersion } from "../localMember/LocalTransport.ts";
export interface ConnectionOpts {
/**
* For the local transport we already do know the jwt token and url. We can reuse it.
* On top the local transport will send additional data to the jwt server to use delayed event delegation.
*/
existingSFUConfig?: SFUConfig;
/**
* For local connections that use the oldest member pattern. here we have not prefetched the sfuConfig
* and hence we need to let the connection do the jwt token fetching.
*/
forceJwtEndpoint?: JwtEndpointVersion;
/** The identity parts to use on this connection */
ownMembershipIdentity: CallMembershipIdentityParts;
/** The media transport to connect to. */
transport: LivekitTransport;
transport: LivekitTransportConfig;
/** The Matrix client to use for OpenID and SFU config requests. */
client: OpenIDClientParts;
/** The room ID this connection is associated with. */
roomId: string;
/** The observable scope to use for this connection. */
scope: ObservableScope;
@@ -88,7 +104,7 @@ export class Connection {
/**
* The media transport to connect to.
*/
public readonly transport: LivekitTransport;
public readonly transport: LivekitTransportConfig;
public readonly livekitRoom: LivekitRoom;
@@ -103,12 +119,61 @@ export class Connection {
*/
public readonly remoteParticipants$: Behavior<RemoteParticipant[]>;
/**
* The alias of the LiveKit room.
*/
public get livekitAlias(): string | undefined {
return this._livekitAlias;
}
private _livekitAlias?: string;
/**
* Whether the connection has been stopped.
* @see Connection.stop
* */
protected stopped = false;
// TODO: can we just keep the ConnectionOpts object instead of spreading?
private readonly client: OpenIDClientParts;
private readonly roomId: string;
private readonly logger: Logger;
private readonly ownMembershipIdentity: CallMembershipIdentityParts;
private readonly existingSFUConfig?: SFUConfig;
/**
* Creates a new connection to a matrix RTC LiveKit backend.
*
* @param opts - Connection options {@link ConnectionOpts}.
*
* @param logger - The logger to use.
*/
public constructor(opts: ConnectionOpts, logger: Logger) {
this.ownMembershipIdentity = opts.ownMembershipIdentity;
this.existingSFUConfig = opts.existingSFUConfig;
this.roomId = opts.roomId;
this.logger = logger.getChild(
"[Connection " + opts.transport.livekit_service_url + "]",
);
this.logger.info(
`constructor: ${opts.transport.livekit_service_url} roomId: ${this.roomId} withSfuConfig?: ${opts.existingSFUConfig ? JSON.stringify(opts.existingSFUConfig) : "undefined"}`,
);
const { transport, client, scope } = opts;
this.scope = scope;
this.livekitRoom = opts.livekitRoomFactory();
this.transport = transport;
this.client = client;
this.remoteParticipants$ = scope.behavior(
// Only tracks remote participants
connectedParticipantsObserver(this.livekitRoom),
);
scope.onEnd(() => {
this.logger.info(`Connection scope ended, stopping connection`);
void this.stop();
});
}
/**
* Starts the connection.
*
@@ -129,8 +194,21 @@ export class Connection {
try {
this._state$.next(ConnectionState.FetchingConfig);
// We should already have this information after creating the localTransport.
// It would probably be better to forward this here.
const { url, jwt } = await this.getSFUConfigWithOpenID();
// only call getSFUConfigWithOpenID for connections where we do not have a token yet. (existingJwtTokenData === undefined)
const { url, jwt, livekitAlias } =
this.existingSFUConfig ??
(await this.getSFUConfigForRemoteConnection());
this.logger.debug(
"Starting Connection to: ",
this.transport.livekit_service_url,
"jwt: ",
jwt,
"wss: ",
url,
"livekitAlias: ",
livekitAlias,
);
this._livekitAlias = livekitAlias;
// If we were stopped while fetching the config, don't proceed to connect
if (this.stopped) return;
@@ -146,8 +224,11 @@ export class Connection {
});
try {
this.logger.info(`livekitRoom.connect ${url}`);
await this.livekitRoom.connect(url, jwt);
this.logger.info(`livekitRoom.connect SUCCESS ${url}`);
} catch (e) {
this.logger.info(`livekitRoom.connect FAILED ${url}`, e);
// LiveKit uses 503 to indicate that the server has hit its track limits.
// https://github.com/livekit/livekit/blob/fcb05e97c5a31812ecf0ca6f7efa57c485cea9fb/pkg/service/rtcservice.go#L171
// It also errors with a status code of 200 (yes, really) for room
@@ -186,11 +267,17 @@ export class Connection {
}
}
protected async getSFUConfigWithOpenID(): Promise<SFUConfig> {
protected async getSFUConfigForRemoteConnection(): Promise<SFUConfig> {
// This will only be called for sfu's where we do not publish ourselves.
// For the local connection we will use the existingJwtTokenData
return await getSFUConfigWithOpenID(
this.client,
this.ownMembershipIdentity,
this.transport.livekit_service_url,
this.transport.livekit_alias,
this.roomId,
// dont pass any custom opts for the subscribe only connections
{},
this.logger,
);
}
@@ -202,44 +289,14 @@ export class Connection {
*/
public async stop(): Promise<void> {
this.logger.debug(
`Stopping connection to ${this.transport.livekit_service_url}`,
`stop: disconnecing from lk room ${this.transport.livekit_service_url}`,
);
if (this.stopped) return;
await this.livekitRoom.disconnect();
this._state$.next(ConnectionState.Stopped);
this.stopped = true;
}
private readonly client: OpenIDClientParts;
private readonly logger: Logger;
/**
* Creates a new connection to a matrix RTC LiveKit backend.
*
* @param opts - Connection options {@link ConnectionOpts}.
*
* @param logger
*/
public constructor(opts: ConnectionOpts, logger: Logger) {
this.logger = logger.getChild("[Connection]");
this.logger.info(
`Creating new connection to ${opts.transport.livekit_service_url} ${opts.transport.livekit_alias}`,
this.logger.debug(
`stop: DONE disconnecing from lk room ${this.transport.livekit_service_url}`,
);
const { transport, client, scope } = opts;
this.scope = scope;
this.livekitRoom = opts.livekitRoomFactory();
this.transport = transport;
this.client = client;
this.remoteParticipants$ = scope.behavior(
// Only tracks remote participants
connectedParticipantsObserver(this.livekitRoom),
);
scope.onEnd(() => {
this.logger.info(`Connection scope ended, stopping connection`);
void this.stop();
});
}
}

View File

@@ -5,7 +5,6 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
import {
Room as LivekitRoom,
type RoomOptions,
@@ -16,10 +15,15 @@ import {
import { type Logger } from "matrix-js-sdk/lib/logger";
// imported as inline to support worker when loaded from a cdn (cross domain)
import E2EEWorker from "livekit-client/e2ee-worker?worker&inline";
import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
import { type LivekitTransportConfig } from "matrix-js-sdk/lib/matrixrtc";
import { type ObservableScope } from "../../ObservableScope.ts";
import { Connection } from "./Connection.ts";
import type { OpenIDClientParts } from "../../../livekit/openIDSFU.ts";
import type {
OpenIDClientParts,
SFUConfig,
} from "../../../livekit/openIDSFU.ts";
import type { MediaDevices } from "../../MediaDevices.ts";
import type { Behavior } from "../../Behavior.ts";
import type { ProcessorState } from "../../../livekit/TrackProcessorContext.tsx";
@@ -28,9 +32,11 @@ import { defaultLiveKitOptions } from "../../../livekit/options.ts";
// TODO evaluate if this should be done like the Publisher Factory
export interface ConnectionFactory {
createConnection(
transport: LivekitTransport,
scope: ObservableScope,
transport: LivekitTransportConfig,
ownMembershipIdentity: CallMembershipIdentityParts,
logger: Logger,
sfuConfig?: SFUConfig,
): Connection;
}
@@ -41,16 +47,18 @@ export class ECConnectionFactory implements ConnectionFactory {
* Creates a ConnectionFactory for LiveKit connections.
*
* @param client - The OpenID client parts for authentication, needed to get openID and JWT tokens.
* @param roomId - The current room ID.
* @param devices - Used for video/audio out/in capture options.
* @param processorState$ - Effects like background blur (only for publishing connection?)
* @param livekitKeyProvider
* @param livekitKeyProvider - Optional key provider for end-to-end encryption.
* @param controlledAudioDevices - Option to indicate whether audio output device is controlled externally (native mobile app).
* @param livekitRoomFactory - Optional factory function (for testing) to create LivekitRoom instances. If not provided, a default factory is used.
* @param echoCancellation - Whether to enable echo cancellation for audio capture.
* @param noiseSuppression - Whether to enable noise suppression for audio capture.
* @param livekitRoomFactory - Optional factory function (for testing) to create LivekitRoom instances. If not provided, a default factory is used.
*/
public constructor(
private client: OpenIDClientParts,
private readonly roomId: string,
private devices: MediaDevices,
private processorState$: Behavior<ProcessorState>,
livekitKeyProvider: BaseKeyProvider | undefined,
@@ -78,17 +86,31 @@ export class ECConnectionFactory implements ConnectionFactory {
this.livekitRoomFactory = livekitRoomFactory ?? defaultFactory;
}
/**
*
* @param scope The observable scope (used for clean-up)
* @param transport The transport to use for this connection.
* @param ownMembershipIdentity required to connect (using the jwt service) with the SFU.
* @param logger The logger instance to use for this connection.
* @param sfuConfig optional config in case we already have a token for this connection.
* @returns
*/
public createConnection(
transport: LivekitTransport,
scope: ObservableScope,
transport: LivekitTransportConfig,
ownMembershipIdentity: CallMembershipIdentityParts,
logger: Logger,
sfuConfig?: SFUConfig,
): Connection {
return new Connection(
{
existingSFUConfig: sfuConfig,
roomId: this.roomId,
transport,
client: this.client,
scope: scope,
livekitRoomFactory: this.livekitRoomFactory,
ownMembershipIdentity,
},
logger,
);

View File

@@ -7,7 +7,7 @@ Please see LICENSE in the repository root for full details.
import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";
import { BehaviorSubject } from "rxjs";
import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
import { type LivekitTransportConfig } from "matrix-js-sdk/lib/matrixrtc";
import { type RemoteParticipant } from "livekit-client";
import { logger } from "matrix-js-sdk/lib/logger";
@@ -18,22 +18,20 @@ import {
} from "./ConnectionManager.ts";
import { type ConnectionFactory } from "./ConnectionFactory.ts";
import { type Connection } from "./Connection.ts";
import { withTestScheduler } from "../../../utils/test.ts";
import { ownMemberMock, withTestScheduler } from "../../../utils/test.ts";
import { areLivekitTransportsEqual } from "./MatrixLivekitMembers.ts";
import { type Behavior } from "../../Behavior.ts";
import { constant, type Behavior } from "../../Behavior.ts";
// Some test constants
const TRANSPORT_1: LivekitTransport = {
const TRANSPORT_1: LivekitTransportConfig = {
type: "livekit",
livekit_service_url: "https://lk.example.org",
livekit_alias: "!alias:example.org",
};
const TRANSPORT_2: LivekitTransport = {
const TRANSPORT_2: LivekitTransportConfig = {
type: "livekit",
livekit_service_url: "https://lk.sample.com",
livekit_alias: "!alias:sample.com",
};
let fakeConnectionFactory: ConnectionFactory;
@@ -49,7 +47,7 @@ beforeEach(() => {
vi.mocked(fakeConnectionFactory).createConnection = vi
.fn()
.mockImplementation(
(transport: LivekitTransport, scope: ObservableScope) => {
(scope: ObservableScope, transport: LivekitTransportConfig) => {
const mockConnection = {
transport,
remoteParticipants$: new BehaviorSubject([]),
@@ -76,10 +74,12 @@ describe("connections$ stream", () => {
const { connectionManagerData$ } = createConnectionManager$({
scope: testScope,
connectionFactory: fakeConnectionFactory,
inputTransports$: behavior("a", {
localTransport$: constant(null),
remoteTransports$: behavior("a", {
a: new Epoch([TRANSPORT_1, TRANSPORT_2], 0),
}),
logger: logger,
ownMembershipIdentity: ownMemberMock,
});
expectObservable(
@@ -115,7 +115,8 @@ describe("connections$ stream", () => {
const { connectionManagerData$ } = createConnectionManager$({
scope: testScope,
connectionFactory: fakeConnectionFactory,
inputTransports$: behavior("abcdef", {
localTransport$: constant(null),
remoteTransports$: behavior("abcdef", {
a: new Epoch([TRANSPORT_1], 0),
b: new Epoch([TRANSPORT_1], 1),
c: new Epoch([TRANSPORT_1], 2),
@@ -124,6 +125,7 @@ describe("connections$ stream", () => {
f: new Epoch([TRANSPORT_1, TRANSPORT_2], 5),
}),
logger: logger,
ownMembershipIdentity: ownMemberMock,
});
expectObservable(
@@ -160,12 +162,14 @@ describe("connections$ stream", () => {
const { connectionManagerData$ } = createConnectionManager$({
scope: testScope,
connectionFactory: fakeConnectionFactory,
inputTransports$: behavior("abc", {
localTransport$: constant(null),
remoteTransports$: behavior("abc", {
a: new Epoch([TRANSPORT_1], 0),
b: new Epoch([TRANSPORT_1, TRANSPORT_2], 1),
c: new Epoch([TRANSPORT_1], 2),
}),
logger: logger,
ownMembershipIdentity: ownMemberMock,
});
expectObservable(
@@ -203,15 +207,15 @@ describe("connectionManagerData$ stream", () => {
// Used in test to control fake connections' remoteParticipants$ streams
let fakeRemoteParticipantsStreams: Map<string, Behavior<RemoteParticipant[]>>;
function keyForTransport(transport: LivekitTransport): string {
return `${transport.livekit_service_url}|${transport.livekit_alias}`;
function keyForTransport(transport: LivekitTransportConfig): string {
return `${transport.livekit_service_url}`;
}
beforeEach(() => {
fakeRemoteParticipantsStreams = new Map();
function getRemoteParticipantsFor(
transport: LivekitTransport,
transport: LivekitTransportConfig,
): Behavior<RemoteParticipant[]> {
return (
fakeRemoteParticipantsStreams.get(keyForTransport(transport)) ??
@@ -223,7 +227,7 @@ describe("connectionManagerData$ stream", () => {
vi.mocked(fakeConnectionFactory).createConnection = vi
.fn()
.mockImplementation(
(transport: LivekitTransport, scope: ObservableScope) => {
(scope: ObservableScope, transport: LivekitTransportConfig) => {
const fakeRemoteParticipants$ = new BehaviorSubject<
RemoteParticipant[]
>([]);
@@ -275,10 +279,12 @@ describe("connectionManagerData$ stream", () => {
const { connectionManagerData$ } = createConnectionManager$({
scope: testScope,
connectionFactory: fakeConnectionFactory,
inputTransports$: behavior("a", {
localTransport$: constant(null),
remoteTransports$: behavior("a", {
a: new Epoch([TRANSPORT_1, TRANSPORT_2], 0),
}),
logger,
ownMembershipIdentity: ownMemberMock,
});
expectObservable(connectionManagerData$).toBe("abcd", {

View File

@@ -6,10 +6,11 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
import { combineLatest, map, of, switchMap, tap } from "rxjs";
import { type LivekitTransportConfig } from "matrix-js-sdk/lib/matrixrtc";
import { combineLatest, map, of, switchMap } from "rxjs";
import { type Logger } from "matrix-js-sdk/lib/logger";
import { type RemoteParticipant } from "livekit-client";
import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
import { type Behavior } from "../../Behavior.ts";
import { type Connection } from "./Connection.ts";
@@ -17,6 +18,11 @@ import { Epoch, type ObservableScope } from "../../ObservableScope.ts";
import { generateItemsWithEpoch } from "../../../utils/observable.ts";
import { areLivekitTransportsEqual } from "./MatrixLivekitMembers.ts";
import { type ConnectionFactory } from "./ConnectionFactory.ts";
import {
isLocalTransportWithSFUConfig,
type LocalTransportWithSFUConfig,
} from "../localMember/LocalTransport.ts";
import { type SFUConfig } from "../../../livekit/openIDSFU.ts";
export class ConnectionManagerData {
private readonly store: Map<
@@ -36,8 +42,10 @@ export class ConnectionManagerData {
}
}
private getKey(transport: LivekitTransport): string {
return transport.livekit_service_url + "|" + transport.livekit_alias;
private getKey(transport: LivekitTransportConfig): string {
// This is enough as a key because the ConnectionManager is already scoped by room.
// We also do not need to consider the slotId at this point since each `MatrixRTCSession` is already scoped by `slotDescription: {id, application}`.
return transport.livekit_service_url;
}
public getConnections(): Connection[] {
@@ -45,15 +53,15 @@ export class ConnectionManagerData {
}
public getConnectionForTransport(
transport: LivekitTransport,
transport: LivekitTransportConfig,
): Connection | null {
return this.store.get(this.getKey(transport))?.connection ?? null;
}
public getParticipantsForTransport(
transport: LivekitTransport,
transport: LivekitTransportConfig,
): RemoteParticipant[] {
const key = transport.livekit_service_url + "|" + transport.livekit_alias;
const key = this.getKey(transport);
const existing = this.store.get(key);
if (existing) {
return existing.participants;
@@ -65,8 +73,11 @@ export class ConnectionManagerData {
interface Props {
scope: ObservableScope;
connectionFactory: ConnectionFactory;
inputTransports$: Behavior<Epoch<LivekitTransport[]>>;
localTransport$: Behavior<LocalTransportWithSFUConfig | null>;
remoteTransports$: Behavior<Epoch<LivekitTransportConfig[]>>;
logger: Logger;
ownMembershipIdentity: CallMembershipIdentityParts;
}
// TODO - write test for scopes (do we really need to bind scope)
@@ -76,9 +87,15 @@ export interface IConnectionManager {
/**
* Crete a `ConnectionManager`
* @param scope the observable scope used by this object.
* @param connectionFactory used to create new connections.
* @param _transportsSubscriptions$ A list of Behaviors each containing a LIST of LivekitTransport.
* @param props - Configuration object
* @param props.scope - The observable scope used by this object
* @param props.connectionFactory - Used to create new connections
* @param props.localTransport$ - The local transport to use. (deduplicated with remoteTransports$)
* @param props.remoteTransports$ - All other transports. The connection manager will create connections for each transport. (deduplicated with localTransport$)
* @param props.ownMembershipIdentity - The own membership identity to use.
* @param props.logger - The logger to use.
*
* Each of these behaviors can be interpreted as subscribed list of transports.
*
* Using `registerTransports` independent external modules can control what connections
@@ -91,8 +108,10 @@ export interface IConnectionManager {
export function createConnectionManager$({
scope,
connectionFactory,
inputTransports$,
localTransport$,
remoteTransports$,
logger: parentLogger,
ownMembershipIdentity,
}: Props): IConnectionManager {
const logger = parentLogger.getChild("[ConnectionManager]");
// TODO logger: only construct one logger from the client and make it compatible via a EC specific sing
@@ -105,12 +124,33 @@ export function createConnectionManager$({
* It is build based on the list of subscribed transports (`transportsSubscriptions$`).
* externally this is modified via `registerTransports()`.
*/
const transports$ = scope.behavior(
inputTransports$.pipe(
map((transports) => transports.mapInner(removeDuplicateTransports)),
tap(({ value: transports }) => {
logger.trace(
`Managing transports: ${transports.map((t) => t.livekit_service_url).join(", ")}`,
const localAndRemoteTransports$: Behavior<
Epoch<(LivekitTransportConfig | LocalTransportWithSFUConfig)[]>
> = scope.behavior(
combineLatest([remoteTransports$, localTransport$]).pipe(
// Combine local and remote transports into one transport array
// and set the forceOldJwtEndpoint property on the local transport
map(([remoteTransports, localTransport]) => {
let localTransportAsArray: LocalTransportWithSFUConfig[] = [];
if (localTransport) {
localTransportAsArray = [localTransport];
}
const dedupedRemote = removeDuplicateTransports(remoteTransports.value);
const remoteWithoutLocal = dedupedRemote.filter(
(transport) =>
!localTransportAsArray.find((l) =>
areLivekitTransportsEqual(l.transport, transport),
),
);
logger.debug(
"remoteWithoutLocal",
remoteWithoutLocal,
"localTransportAsArray",
localTransportAsArray,
);
return new Epoch(
[...localTransportAsArray, ...remoteWithoutLocal],
remoteTransports.epoch,
);
}),
),
@@ -120,25 +160,41 @@ export function createConnectionManager$({
* Connections for each transport in use by one or more session members.
*/
const connections$ = scope.behavior(
transports$.pipe(
localAndRemoteTransports$.pipe(
generateItemsWithEpoch(
"ConnectionManager connections$",
function* (transports) {
for (const transport of transports)
yield {
keys: [transport.livekit_service_url, transport.livekit_alias],
data: undefined,
};
for (const transportWithOrWithoutSfuConfig of transports) {
if (
isLocalTransportWithSFUConfig(transportWithOrWithoutSfuConfig)
) {
// This is the local transport only the `LocalTransportWithSFUConfig` has a `sfuConfig` field
const { transport, sfuConfig } = transportWithOrWithoutSfuConfig;
yield {
keys: [transport.livekit_service_url, sfuConfig],
data: undefined,
};
} else {
yield {
keys: [
transportWithOrWithoutSfuConfig.livekit_service_url,
undefined as undefined | SFUConfig,
],
data: undefined,
};
}
}
},
(scope, _data$, serviceUrl, alias) => {
logger.debug(`Creating connection to ${serviceUrl} (${alias})`);
(scope, _data$, serviceUrl, sfuConfig) => {
const connection = connectionFactory.createConnection(
scope,
{
type: "livekit",
livekit_service_url: serviceUrl,
livekit_alias: alias,
},
scope,
ownMembershipIdentity,
logger,
sfuConfig,
);
// Start the connection immediately
// Use connection state to track connection progress
@@ -188,18 +244,18 @@ export function createConnectionManager$({
);
}),
),
new Epoch(new ConnectionManagerData()),
new Epoch(new ConnectionManagerData(), -1),
);
return { connectionManagerData$ };
}
function removeDuplicateTransports(
transports: LivekitTransport[],
): LivekitTransport[] {
function removeDuplicateTransports<T extends LivekitTransportConfig>(
transports: T[],
): T[] {
return transports.reduce((acc, transport) => {
if (!acc.some((t) => areLivekitTransportsEqual(t, transport)))
acc.push(transport);
return acc;
}, [] as LivekitTransport[]);
}, [] as T[]);
}

View File

@@ -15,7 +15,11 @@ import EventEmitter from "events";
import { ObservableScope } from "../../ObservableScope.ts";
import { ECConnectionFactory } from "./ConnectionFactory.ts";
import type { OpenIDClientParts } from "../../../livekit/openIDSFU.ts";
import { exampleTransport, mockMediaDevices } from "../../../utils/test.ts";
import {
exampleTransport,
mockMediaDevices,
ownMemberMock,
} from "../../../utils/test.ts";
import type { ProcessorState } from "../../../livekit/TrackProcessorContext.tsx";
import { constant } from "../../Behavior";
@@ -61,6 +65,7 @@ describe("ECConnectionFactory - Audio inputs options", () => {
const ecConnectionFactory = new ECConnectionFactory(
mockClient,
"!roomid:example.org",
mockMediaDevices({}),
new BehaviorSubject<ProcessorState>({
supported: true,
@@ -72,7 +77,12 @@ describe("ECConnectionFactory - Audio inputs options", () => {
echo,
noise,
);
ecConnectionFactory.createConnection(exampleTransport, testScope, logger);
ecConnectionFactory.createConnection(
testScope,
exampleTransport,
ownMemberMock,
logger,
);
// Check if Room was constructed with expected options
expect(RoomConstructor).toHaveBeenCalledWith(
@@ -96,6 +106,7 @@ describe("ECConnectionFactory - ControlledAudioDevice", () => {
const ecConnectionFactory = new ECConnectionFactory(
mockClient,
"!roomid:example.org",
mockMediaDevices({
audioOutput: {
available$: constant(new Map<never, never>()),
@@ -113,7 +124,12 @@ describe("ECConnectionFactory - ControlledAudioDevice", () => {
false,
false,
);
ecConnectionFactory.createConnection(exampleTransport, testScope, logger);
ecConnectionFactory.createConnection(
testScope,
exampleTransport,
ownMemberMock,
logger,
);
// Check if Room was constructed with expected options
expect(RoomConstructor).toHaveBeenCalledWith(

View File

@@ -10,8 +10,7 @@ import {
type CallMembership,
type LivekitTransport,
} from "matrix-js-sdk/lib/matrixrtc";
import { getParticipantId } from "matrix-js-sdk/lib/matrixrtc/utils";
import { combineLatest, map, type Observable } from "rxjs";
import { BehaviorSubject, combineLatest, map, type Observable } from "rxjs";
import { type IConnectionManager } from "./ConnectionManager.ts";
import {
@@ -26,14 +25,18 @@ import {
} from "../../ObservableScope.ts";
import { ConnectionManagerData } from "./ConnectionManager.ts";
import {
mockCallMembership,
flushPromises,
mockRtcMembership,
mockRemoteParticipant,
withTestScheduler,
} from "../../../utils/test.ts";
import { type Connection } from "./Connection.ts";
import { constant } from "../../Behavior.ts";
let testScope: ObservableScope;
const fallbackMemberId = (userId: string, deviceId: string): string =>
`${userId}:${deviceId}`;
const transportA: LivekitTransport = {
type: "livekit",
livekit_service_url: "https://lk.example.org",
@@ -46,16 +49,12 @@ const transportB: LivekitTransport = {
livekit_alias: "!alias:sample.com",
};
const bobMembership = mockCallMembership(
"@bob:example.org",
"DEV000",
transportA,
);
const carlMembership = mockCallMembership(
"@carl:sample.com",
"DEV111",
transportB,
);
const bobMembership = mockRtcMembership("@bob:example.org", "DEV000", {
fociPreferred: [transportA],
});
const carlMembership = mockRtcMembership("@carl:sample.com", "DEV111", {
fociPreferred: [transportB],
});
beforeEach(() => {
testScope = new ObservableScope();
@@ -76,52 +75,41 @@ function epochMeWith$<T, U>(
);
}
test("should signal participant not yet connected to livekit", () => {
withTestScheduler(({ behavior, expectObservable }) => {
const { memberships$, membershipsWithTransport$ } = fromMemberships$(
behavior("a", {
a: [bobMembership],
}),
);
test("should signal participant not yet connected to livekit", async () => {
const mockedMemberships$ = new BehaviorSubject([bobMembership]);
const mockConnectionManagerData$ = new BehaviorSubject(
new ConnectionManagerData(),
);
const { memberships$, membershipsWithTransport$ } =
createEpochedMemberships$(mockedMemberships$);
const connectionManagerData$ = epochMeWith$(
memberships$,
behavior("a", {
a: new ConnectionManagerData(),
}),
);
const connectionManagerData$ = epochMeWith$(
memberships$,
mockConnectionManagerData$,
);
const matrixLivekitMembers$ = createMatrixLivekitMembers$({
scope: testScope,
membershipsWithTransport$: testScope.behavior(membershipsWithTransport$),
connectionManager: {
connectionManagerData$: connectionManagerData$,
} as unknown as IConnectionManager,
});
expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe(
"a",
{
a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => {
expect(data.length).toEqual(1);
expectObservable(data[0].membership$).toBe("a", {
a: bobMembership,
});
expectObservable(data[0].participant.value$).toBe("a", {
a: null,
});
expectObservable(data[0].connection$).toBe("a", {
a: null,
});
return true;
}),
},
);
const matrixLivekitMember$ = createMatrixLivekitMembers$({
scope: testScope,
membershipsWithTransport$: testScope.behavior(membershipsWithTransport$),
connectionManager: {
connectionManagerData$: connectionManagerData$,
} as unknown as IConnectionManager,
});
await flushPromises();
expect(matrixLivekitMember$.value.value).toSatisfy(
(data: RemoteMatrixLivekitMember[]) => {
expect(data.length).toEqual(1);
expect(data[0].membership$.value).toBe(bobMembership);
expect(data[0].participant.value$.value).toBe(null);
expect(data[0].connection$.value).toBe(null);
return true;
},
);
});
// Helper to create epoch'ed memberships$ and membershipsWithTransport$ from memberships observable.
function fromMemberships$(m$: Observable<CallMembership[]>): {
function createEpochedMemberships$(m$: Observable<CallMembership[]>): {
memberships$: Observable<Epoch<CallMembership[]>>;
membershipsWithTransport$: Observable<
Epoch<{ membership: CallMembership; transport?: LivekitTransport }[]>
@@ -146,32 +134,115 @@ function fromMemberships$(m$: Observable<CallMembership[]>): {
};
}
test("should signal participant on a connection that is publishing", () => {
withTestScheduler(({ behavior, expectObservable }) => {
const bobParticipantId = getParticipantId(
test("should signal participant on a connection that is publishing", async () => {
const bobParticipantId = fallbackMemberId(
bobMembership.userId,
bobMembership.deviceId,
);
const { memberships$, membershipsWithTransport$ } = createEpochedMemberships$(
constant([bobMembership]),
);
const connection = {
transport: bobMembership.getTransport(bobMembership),
} as unknown as Connection;
const dataWithPublisher = new ConnectionManagerData();
dataWithPublisher.add(connection, [
mockRemoteParticipant({ identity: bobParticipantId }),
]);
const connectionManagerData$ = epochMeWith$(
memberships$,
constant(dataWithPublisher),
);
const matrixLivekitMember$ = createMatrixLivekitMembers$({
scope: testScope,
membershipsWithTransport$: testScope.behavior(membershipsWithTransport$),
connectionManager: {
connectionManagerData$: connectionManagerData$,
} as unknown as IConnectionManager,
});
await flushPromises();
expect(matrixLivekitMember$.value.value).toSatisfy(
(data: RemoteMatrixLivekitMember[]) => {
expect(data.length).toEqual(1);
expect(data[0].membership$.value).toBe(bobMembership);
expect(data[0].participant.value$.value).toSatisfy((participant) => {
expect(participant).toBeDefined();
expect(participant!.identity).toEqual(bobParticipantId);
return true;
});
expect(data[0].connection$.value).toBe(connection);
return true;
},
);
});
test("should signal participant on a connection that is not publishing", async () => {
const { memberships$, membershipsWithTransport$ } = createEpochedMemberships$(
constant([bobMembership]),
);
const connection = {
transport: bobMembership.getTransport(bobMembership),
} as unknown as Connection;
const dataWithPublisher = new ConnectionManagerData();
dataWithPublisher.add(connection, []);
const connectionManagerData$ = epochMeWith$(
memberships$,
constant(dataWithPublisher),
);
const matrixLivekitMember$ = createMatrixLivekitMembers$({
scope: testScope,
membershipsWithTransport$: testScope.behavior(membershipsWithTransport$),
connectionManager: {
connectionManagerData$: connectionManagerData$,
} as unknown as IConnectionManager,
});
await flushPromises();
expect(matrixLivekitMember$.value.value).toSatisfy(
(data: RemoteMatrixLivekitMember[]) => {
expect(data.length).toEqual(1);
expect(data[0].membership$.value).toBe(bobMembership);
expect(data[0].participant.value$.value).toBe(null);
expect(data[0].connection$.value).toBe(connection);
return true;
},
);
});
describe("Publication edge case", () => {
test("bob is publishing in several connections", async () => {
const { memberships$, membershipsWithTransport$ } =
createEpochedMemberships$(constant([bobMembership, carlMembership]));
const connectionWithPublisher = new ConnectionManagerData();
const bobParticipantId = fallbackMemberId(
bobMembership.userId,
bobMembership.deviceId,
);
const { memberships$, membershipsWithTransport$ } = fromMemberships$(
behavior("a", {
a: [bobMembership],
}),
);
const connection = {
transport: bobMembership.getTransport(bobMembership),
const connectionA = {
transport: transportA,
} as unknown as Connection;
const dataWithPublisher = new ConnectionManagerData();
dataWithPublisher.add(connection, [
const connectionB = {
transport: transportB,
} as unknown as Connection;
connectionWithPublisher.add(connectionA, [
mockRemoteParticipant({ identity: bobParticipantId }),
]);
connectionWithPublisher.add(connectionB, [
mockRemoteParticipant({ identity: bobParticipantId }),
]);
const connectionManagerData$ = epochMeWith$(
memberships$,
behavior("a", {
a: dataWithPublisher,
}),
constant(connectionWithPublisher),
);
const matrixLivekitMembers$ = createMatrixLivekitMembers$({
@@ -181,213 +252,73 @@ test("should signal participant on a connection that is publishing", () => {
connectionManagerData$: connectionManagerData$,
} as unknown as IConnectionManager,
});
expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe(
"a",
{
a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => {
expect(data.length).toEqual(1);
expectObservable(data[0].membership$).toBe("a", {
a: bobMembership,
});
expectObservable(data[0].participant.value$).toBe("a", {
a: expect.toSatisfy((participant) => {
expect(participant).toBeDefined();
expect(participant!.identity).toEqual(bobParticipantId);
return true;
}),
});
expectObservable(data[0].connection$).toBe("a", {
a: connection,
});
await flushPromises();
expect(matrixLivekitMembers$.value.value).toSatisfy(
(data: RemoteMatrixLivekitMember[]) => {
expect(data.length).toEqual(2);
expect(data[0].membership$.value).toBe(bobMembership);
expect(data[0].connection$.value).toBe(connectionA);
expect(data[0].participant.value$.value).toSatisfy((participant) => {
expect(participant).toBeDefined();
expect(participant!.identity).toEqual(bobParticipantId);
return true;
}),
});
return true;
},
);
});
});
test("should signal participant on a connection that is not publishing", () => {
withTestScheduler(({ behavior, expectObservable }) => {
const { memberships$, membershipsWithTransport$ } = fromMemberships$(
behavior("a", {
a: [bobMembership],
}),
);
test("bob is publishing in the wrong connection", async () => {
const mockedMemberships$ = new BehaviorSubject([
bobMembership,
carlMembership,
]);
const connection = {
transport: bobMembership.getTransport(bobMembership),
} as unknown as Connection;
const dataWithPublisher = new ConnectionManagerData();
dataWithPublisher.add(connection, []);
const { memberships$, membershipsWithTransport$ } =
createEpochedMemberships$(mockedMemberships$);
const connectionManagerData$ = epochMeWith$(
memberships$,
behavior("a", {
a: dataWithPublisher,
}),
);
const connectionWithPublisher = new ConnectionManagerData();
const matrixLivekitMembers$ = createMatrixLivekitMembers$({
scope: testScope,
membershipsWithTransport$: testScope.behavior(membershipsWithTransport$),
connectionManager: {
connectionManagerData$: connectionManagerData$,
} as unknown as IConnectionManager,
});
const bobParticipantId = fallbackMemberId(
bobMembership.userId,
bobMembership.deviceId,
);
const connectionA = { transport: transportA } as unknown as Connection;
const connectionB = { transport: transportB } as unknown as Connection;
expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe(
"a",
{
a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => {
expect(data.length).toEqual(1);
expectObservable(data[0].membership$).toBe("a", {
a: bobMembership,
});
expectObservable(data[0].participant.value$).toBe("a", {
a: null,
});
expectObservable(data[0].connection$).toBe("a", {
a: connection,
});
return true;
}),
},
);
});
});
describe("Publication edge case", () => {
test("bob is publishing in several connections", () => {
withTestScheduler(({ behavior, expectObservable }) => {
const { memberships$, membershipsWithTransport$ } = fromMemberships$(
behavior("a", {
a: [bobMembership, carlMembership],
}),
);
const connectionWithPublisher = new ConnectionManagerData();
const bobParticipantId = getParticipantId(
bobMembership.userId,
bobMembership.deviceId,
);
const connectionA = {
transport: transportA,
} as unknown as Connection;
const connectionB = {
transport: transportB,
} as unknown as Connection;
connectionWithPublisher.add(connectionA, [
mockRemoteParticipant({ identity: bobParticipantId }),
]);
connectionWithPublisher.add(connectionB, [
mockRemoteParticipant({ identity: bobParticipantId }),
]);
const connectionManagerData$ = epochMeWith$(
memberships$,
behavior("a", {
a: connectionWithPublisher,
}),
);
const matrixLivekitMembers$ = createMatrixLivekitMembers$({
scope: testScope,
membershipsWithTransport$: testScope.behavior(
membershipsWithTransport$,
),
connectionManager: {
connectionManagerData$: connectionManagerData$,
} as unknown as IConnectionManager,
});
expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe(
"a",
{
a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => {
expect(data.length).toEqual(2);
expectObservable(data[0].membership$).toBe("a", {
a: bobMembership,
});
expectObservable(data[0].connection$).toBe("a", {
// The real connection should be from transportA as per the membership
a: connectionA,
});
expectObservable(data[0].participant.value$).toBe("a", {
a: expect.toSatisfy((participant) => {
expect(participant).toBeDefined();
expect(participant!.identity).toEqual(bobParticipantId);
return true;
}),
});
return true;
}),
},
);
});
});
test("bob is publishing in the wrong connection", () => {
withTestScheduler(({ behavior, expectObservable }) => {
const { memberships$, membershipsWithTransport$ } = fromMemberships$(
behavior("a", {
a: [bobMembership, carlMembership],
}),
);
const connectionWithPublisher = new ConnectionManagerData();
const bobParticipantId = getParticipantId(
bobMembership.userId,
bobMembership.deviceId,
);
const connectionA = { transport: transportA } as unknown as Connection;
const connectionB = { transport: transportB } as unknown as Connection;
// Bob is not publishing on A
connectionWithPublisher.add(connectionA, []);
// Bob is publishing on B but his membership says A
connectionWithPublisher.add(connectionB, [
mockRemoteParticipant({ identity: bobParticipantId }),
]);
const connectionManagerData$ = epochMeWith$(
memberships$,
behavior("a", {
a: connectionWithPublisher,
}),
);
const matrixLivekitMembers$ = createMatrixLivekitMembers$({
scope: testScope,
membershipsWithTransport$: testScope.behavior(
membershipsWithTransport$,
),
connectionManager: {
connectionManagerData$: connectionManagerData$,
} as unknown as IConnectionManager,
});
expectObservable(matrixLivekitMembers$.pipe(map((e) => e.value))).toBe(
"a",
{
a: expect.toSatisfy((data: RemoteMatrixLivekitMember[]) => {
expect(data.length).toEqual(2);
expectObservable(data[0].membership$).toBe("a", {
a: bobMembership,
});
expectObservable(data[0].connection$).toBe("a", {
// The real connection should be from transportA as per the membership
a: connectionA,
});
expectObservable(data[0].participant.value$).toBe("a", {
// No participant as Bob is not publishing on his membership transport
a: null,
});
return true;
}),
},
);
});
// Bob is not publishing on A
connectionWithPublisher.add(connectionA, []);
// Bob is publishing on B but his membership says A
connectionWithPublisher.add(connectionB, [
mockRemoteParticipant({ identity: bobParticipantId }),
]);
const connectionsWithPublisher$ = new BehaviorSubject(
connectionWithPublisher,
);
const connectionManagerData$ = epochMeWith$(
memberships$,
connectionsWithPublisher$,
);
const matrixLivekitMember$ = createMatrixLivekitMembers$({
scope: testScope,
membershipsWithTransport$: testScope.behavior(membershipsWithTransport$),
connectionManager: {
connectionManagerData$: connectionManagerData$,
} as unknown as IConnectionManager,
});
await flushPromises();
expect(matrixLivekitMember$.value.value).toSatisfy(
(data: RemoteMatrixLivekitMember[]) => {
expect(data.length).toEqual(2);
expect(data[0].membership$.value).toBe(bobMembership);
expect(data[0].connection$.value).toBe(connectionA);
expect(data[0].participant.value$.value).toBe(null);
return true;
},
);
});

Some files were not shown because too many files have changed in this diff Show More