diff --git a/.github/workflows/build-element-call.yaml b/.github/workflows/build-element-call.yaml index 01553fec..4ca5ccad 100644 --- a/.github/workflows/build-element-call.yaml +++ b/.github/workflows/build-element-call.yaml @@ -7,7 +7,7 @@ on: type: string package: type: string # This would ideally be a `choice` type, but that isn't supported yet - description: The package type to be built. Must be one of 'full' or 'embedded' + description: The package type to be built. Must be one of 'full', 'embedded', or 'sdk' required: true build_mode: type: string # This would ideally be a `choice` type, but that isn't supported yet diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 6aa5fae6..9b86215e 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -69,3 +69,17 @@ jobs: SENTRY_URL: ${{ secrets.SENTRY_URL }} SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + + build_sdk_element_call: + # Use the embedded package vite build + uses: ./.github/workflows/build-element-call.yaml + with: + package: sdk + vite_app_version: ${{ github.event.release.tag_name || github.sha }} + build_mode: ${{ github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'development build') && 'development' || 'production' }} + secrets: + SENTRY_ORG: ${{ secrets.SENTRY_ORG }} + SENTRY_PROJECT: ${{ secrets.SENTRY_PROJECT }} + SENTRY_URL: ${{ secrets.SENTRY_URL }} + SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/deploy-to-netlify.yaml b/.github/workflows/deploy-to-netlify.yaml index 388192e4..4b7ba22f 100644 --- a/.github/workflows/deploy-to-netlify.yaml +++ b/.github/workflows/deploy-to-netlify.yaml @@ -14,6 +14,10 @@ on: deployment_ref: required: true type: string + package: + required: true + type: string + description: Which package to deploy - 'full', 'embedded', or 'sdk' artifact_run_id: required: false type: string @@ -50,7 +54,7 @@ jobs: with: github-token: ${{ secrets.ELEMENT_BOT_TOKEN }} run-id: ${{ inputs.artifact_run_id }} - name: build-output-full + name: build-output-${{ inputs.package }} path: webapp - name: Add redirects file @@ -58,15 +62,17 @@ jobs: run: curl -s https://raw.githubusercontent.com/element-hq/element-call/main/config/netlify_redirects > webapp/_redirects - name: Add config file - run: curl -s "https://raw.githubusercontent.com/${{ inputs.pr_head_full_name }}/${{ inputs.pr_head_ref }}/config/config_netlify_preview.json" > webapp/config.json - + run: | + if [ "${{ inputs.package }}" = "full" ]; then + curl -s "https://raw.githubusercontent.com/${{ inputs.pr_head_full_name }}/${{ inputs.pr_head_ref }}/config/config_netlify_preview_sdk.json" > webapp/config.json + fi - name: ☁️ Deploy to Netlify id: netlify uses: nwtgck/actions-netlify@4cbaf4c08f1a7bfa537d6113472ef4424e4eb654 # v3.0 with: publish-dir: webapp deploy-message: "Deploy from GitHub Actions" - alias: pr${{ inputs.pr_number }} + alias: ${{ inputs.package == 'sdk' && format('pr{0}-sdk', inputs.pr_number) || format('pr{0}', inputs.pr_number) }} env: NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }} NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }} diff --git a/.github/workflows/pr-deploy.yaml b/.github/workflows/pr-deploy.yaml index 7b128352..fe934162 100644 --- a/.github/workflows/pr-deploy.yaml +++ b/.github/workflows/pr-deploy.yaml @@ -20,7 +20,7 @@ jobs: owner: ${{ github.event.workflow_run.head_repository.owner.login }} branch: ${{ github.event.workflow_run.head_branch }} - netlify: + netlify-full: needs: prdetails permissions: deployments: write @@ -31,6 +31,24 @@ jobs: pr_head_full_name: ${{ github.event.workflow_run.head_repository.full_name }} pr_head_ref: ${{ needs.prdetails.outputs.pr_data_json && fromJSON(needs.prdetails.outputs.pr_data_json).head.ref }} deployment_ref: ${{ needs.prdetails.outputs.pr_data_json && fromJSON(needs.prdetails.outputs.pr_data_json).head.sha || github.ref || github.head_ref }} + package: full + secrets: + ELEMENT_BOT_TOKEN: ${{ secrets.ELEMENT_BOT_TOKEN }} + NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }} + NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }} + + netlify-sdk: + needs: prdetails + permissions: + deployments: write + uses: ./.github/workflows/deploy-to-netlify.yaml + with: + artifact_run_id: ${{ github.event.workflow_run.id || github.run_id }} + pr_number: ${{ needs.prdetails.outputs.pr_number }} + pr_head_full_name: ${{ github.event.workflow_run.head_repository.full_name }} + pr_head_ref: ${{ needs.prdetails.outputs.pr_data_json && fromJSON(needs.prdetails.outputs.pr_data_json).head.ref }} + deployment_ref: ${{ needs.prdetails.outputs.pr_data_json && fromJSON(needs.prdetails.outputs.pr_data_json).head.sha || github.ref || github.head_ref }} + package: sdk secrets: ELEMENT_BOT_TOKEN: ${{ secrets.ELEMENT_BOT_TOKEN }} NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }} diff --git a/package.json b/package.json index 14193013..b835a128 100644 --- a/package.json +++ b/package.json @@ -13,8 +13,9 @@ "build:embedded": "yarn build:full --config vite-embedded.config.js", "build:embedded:production": "yarn build:embedded", "build:embedded:development": "yarn build:embedded --mode development", - "build:sdk": "yarn build:full --config vite-sdk.config.js", "build:sdk:development": "yarn build:sdk --mode development", + "build:sdk": "yarn build:full --config vite-sdk.config.js", + "build:sdk:production": "yarn build:sdk", "serve": "vite preview", "prettier:check": "prettier -c .", "prettier:format": "prettier -w .", @@ -104,7 +105,7 @@ "livekit-client": "^2.13.0", "lodash-es": "^4.17.21", "loglevel": "^1.9.1", - "matrix-js-sdk": "matrix-org/matrix-js-sdk#develop", + "matrix-js-sdk": "40.2.0-rc.0", "matrix-widget-api": "^1.16.1", "node-stdlib-browser": "^1.3.1", "normalize.css": "^8.0.1", diff --git a/sdk/README.md b/sdk/README.md index 91337f10..25c0de28 100644 --- a/sdk/README.md +++ b/sdk/README.md @@ -21,7 +21,7 @@ in the repository root. It will create a `dist` folder containing the compiled js file. -This file needs to be hosted. Locally (via `npx serve -l 81234 --cors`) or on a remote server. +This file needs to be hosted. Locally (via `npx serve -l 1234 --cors`) or on a remote server. Now you just need to add the widget to element web via: diff --git a/sdk/helper.ts b/sdk/helper.ts index a3d597be..47de4a93 100644 --- a/sdk/helper.ts +++ b/sdk/helper.ts @@ -12,15 +12,12 @@ Please see LICENSE in the repository root for full details. import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; import { scan } from "rxjs"; -import { widget as _widget } from "../src/widget"; +import { type WidgetHelpers } from "../src/widget"; import { type LivekitRoomItem } from "../src/state/CallViewModel/CallViewModel"; export const logger = rootLogger.getChild("[MatrixRTCSdk]"); -if (!_widget) throw Error("No widget. This webapp can only start as a widget"); -export const widget = _widget; - -export const tryMakeSticky = (): void => { +export const tryMakeSticky = (widget: WidgetHelpers): void => { logger.info("try making sticky MatrixRTCSdk"); void widget.api .setAlwaysOnScreen(true) diff --git a/sdk/main.ts b/sdk/main.ts index a273ed8a..fddba53c 100644 --- a/sdk/main.ts +++ b/sdk/main.ts @@ -30,8 +30,8 @@ import { } from "rxjs"; import { type CallMembership, - MatrixRTCSession, MatrixRTCSessionEvent, + MatrixRTCSessionManager, } from "matrix-js-sdk/lib/matrixrtc"; import { type Room as LivekitRoom, @@ -50,14 +50,12 @@ import { getUrlParams } from "../src/UrlParams"; import { MuteStates } from "../src/state/MuteStates"; import { MediaDevices } from "../src/state/MediaDevices"; import { E2eeType } from "../src/e2ee/e2eeType"; +import { currentAndPrev, logger, TEXT_LK_TOPIC, tryMakeSticky } from "./helper"; import { - currentAndPrev, - logger, - TEXT_LK_TOPIC, - tryMakeSticky, - widget, -} from "./helper"; -import { ElementWidgetActions, initializeWidget } from "../src/widget"; + ElementWidgetActions, + widget as _widget, + initializeWidget, +} from "../src/widget"; import { type Connection } from "../src/state/CallViewModel/remoteMembers/Connection"; interface MatrixRTCSdk { @@ -68,7 +66,7 @@ interface MatrixRTCSdk { join: () => void; /** @throws on leave errors */ leave: () => void; - data$: Observable<{ sender: string; data: string }>; + data$: Observable<{ rtcBackendIdentity: string; data: string }>; /** * flattened list of members */ @@ -79,32 +77,54 @@ interface MatrixRTCSdk { participant: LocalParticipant | RemoteParticipant | null; }[] >; + /** + * flattened local members + */ + localMember$: Behavior<{ + connection: Connection | null; + membership: CallMembership; + participant: LocalParticipant | null; + } | null>; /** Use the LocalMemberConnectionState returned from `join` for a more detailed connection state */ connected$: Behavior; sendData?: (data: unknown) => Promise; + sendRoomMessage?: (message: string) => Promise; } export async function createMatrixRTCSdk( application: string = "m.call", id: string = "", + sticky: boolean = false, ): Promise { - initializeWidget(); + const scope = new ObservableScope(); + + // widget client + initializeWidget(application, true); + const widget = _widget; + if (!widget) throw Error("No widget. This webapp can only start as a widget"); const client = await widget.client; logger.info("client created"); - const scope = new ObservableScope(); + + // url params const { roomId } = getUrlParams(); if (roomId === null) throw Error("could not get roomId from url params"); - const room = client.getRoom(roomId); if (room === null) throw Error("could not get room from client"); + // rtc session + const slot = { application, id }; + const rtcSessionManager = new MatrixRTCSessionManager(logger, client, slot); + rtcSessionManager.start(); + const rtcSession = rtcSessionManager.getRoomSession(room); + + // media devices const mediaDevices = new MediaDevices(scope); const muteStates = new MuteStates(scope, mediaDevices, { - audioEnabled: true, - videoEnabled: true, + audioEnabled: false, + videoEnabled: false, }); - const slot = { application, id }; - const rtcSession = new MatrixRTCSession(client, room, slot); + + // call view model const callViewModel = createCallViewModel$( scope, rtcSession, @@ -117,8 +137,9 @@ export async function createMatrixRTCSdk( constant({ supported: false, processor: undefined }), ); logger.info("CallViewModelCreated"); + // create data listener - const data$ = new Subject<{ sender: string; data: string }>(); + const data$ = new Subject<{ rtcBackendIdentity: string; data: string }>(); const lkTextStreamHandlerFunction = async ( reader: TextStreamReader, @@ -140,7 +161,7 @@ export async function createMatrixRTCSdk( if (participants && participants.includes(participantInfo.identity)) { const text = await reader.readAll(); logger.info(`Received text: ${text}`); - data$.next({ sender: participantInfo.identity, data: text }); + data$.next({ rtcBackendIdentity: participantInfo.identity, data: text }); } else { logger.warn( "Received text from unknown participant", @@ -230,6 +251,16 @@ export async function createMatrixRTCSdk( } }; + const sendRoomMessage = async (message: string): Promise => { + const messageString = JSON.stringify(message); + logger.info("try sending to room: ", messageString); + try { + await client.sendTextMessage(room.roomId, message); + } catch (e) { + logger.error("failed sending to room: ", messageString, e); + } + }; + // after hangup gets called const leaveSubs = callViewModel.leave$.subscribe(() => { const scheduleWidgetCloseOnLeave = async (): Promise => { @@ -267,7 +298,7 @@ export async function createMatrixRTCSdk( return { join: (): void => { // first lets try making the widget sticky - tryMakeSticky(); + if (sticky) tryMakeSticky(widget); callViewModel.join(); }, leave: (): void => { @@ -276,6 +307,28 @@ export async function createMatrixRTCSdk( livekitRoomItemsSub.unsubscribe(); }, data$, + localMember$: scope.behavior( + callViewModel.localMatrixLivekitMember$.pipe( + tap((member) => + logger.info("localMatrixLivekitMember$ next: ", member), + ), + switchMap((member) => { + if (member === null) return of(null); + return combineLatest([ + member.connection$, + member.membership$, + member.participant.value$, + ]).pipe( + map(([connection, membership, participant]) => ({ + connection, + membership, + participant, + })), + ); + }), + tap((member) => logger.info("localMember$ next: ", member)), + ), + ), connected$: callViewModel.connected$, members$: scope.behavior( callViewModel.matrixLivekitMembers$.pipe( @@ -302,5 +355,6 @@ export async function createMatrixRTCSdk( [], ), sendData, + sendRoomMessage, }; } diff --git a/src/livekit/openIDSFU.ts b/src/livekit/openIDSFU.ts index 8360cdc7..e1aa72be 100644 --- a/src/livekit/openIDSFU.ts +++ b/src/livekit/openIDSFU.ts @@ -146,7 +146,7 @@ export async function getSFUConfigWithOpenID( } else { logger?.warn( `Failed fetching jwt with matrix 2.0 endpoint other issues ->`, - `(not going to try with legacy endpoint: forceOldJwtEndpoint is set to false, we did not get a not supported error from the sfu)`, + `(not going to try with legacy endpoint if forceMatrix2Jwt is set to false (it is ${forceMatrix2Jwt}), we did not get a not supported error from the sfu)`, e, ); // Make this throw a hard error in case we force the matrix2.0 endpoint. diff --git a/src/widget.test.ts b/src/widget.test.ts index f85c56bc..8852f799 100644 --- a/src/widget.test.ts +++ b/src/widget.test.ts @@ -6,12 +6,11 @@ Please see LICENSE in the repository root for full details. */ import { beforeAll, describe, expect, vi, it } from "vitest"; -import { createRoomWidgetClient, EventType } from "matrix-js-sdk"; +import { createRoomWidgetClient } from "matrix-js-sdk"; import { getUrlParams } from "./UrlParams"; import { initializeWidget, widget } from "./widget"; import { Config } from "./config/Config"; -import { ElementCallReactionEventType } from "./reactions"; vi.mock("matrix-js-sdk", { spy: true }); const createRoomWidgetClientSpy = vi.mocked(createRoomWidgetClient); @@ -35,7 +34,7 @@ vi.mock("./UrlParams", () => ({ })), })); -initializeWidget(); +initializeWidget("ANYRTCAPP"); describe("widget", () => { beforeAll(() => {}); @@ -52,48 +51,39 @@ describe("widget", () => { expect(widget).toBeDefined(); expect(configInitSpy).toHaveBeenCalled(); const sendEvent = [ - EventType.CallNotify, // Sent as a deprecated fallback - EventType.RTCNotification, + "org.matrix.msc4075.call.notify", // Sent as a deprecated fallback + "org.matrix.msc4075.rtc.notification", ]; const sendRecvEvent = [ "org.matrix.rageshake_request", - EventType.CallEncryptionKeysPrefix, - EventType.Reaction, - EventType.RoomRedaction, - ElementCallReactionEventType, - EventType.RTCDecline, - EventType.RTCMembership, + "io.element.call.encryption_keys", + "m.reaction", + "m.room.redaction", + "io.element.call.reaction", + "org.matrix.msc4310.rtc.decline", + "org.matrix.msc4143.rtc.member", ]; const sendState = [ - "myYser", // Legacy call membership events - `_myYser_AAAAA_m.call`, // Session membership events - `myYser_AAAAA_m.call`, // The above with no leading underscore, for room versions whose auth rules allow it - ].map((stateKey) => ({ - eventType: EventType.GroupCallMemberPrefix, - stateKey, - })); + { eventType: "org.matrix.msc3401.call.member", stateKey: "myYser" }, // Legacy call membership events + { + eventType: "org.matrix.msc3401.call.member", + stateKey: `_myYser_AAAAA_ANYRTCAPP`, + }, // Session membership events + { + eventType: "org.matrix.msc3401.call.member", + stateKey: `myYser_AAAAA_ANYRTCAPP`, + }, // The above with no leading underscore, for room versions whose auth rules allow it + ]; const receiveState = [ - { eventType: EventType.RoomCreate }, - { eventType: EventType.RoomName }, - { eventType: EventType.RoomMember }, - { eventType: EventType.RoomEncryption }, - { eventType: EventType.GroupCallMemberPrefix }, + { eventType: "m.room.create" }, + { eventType: "m.room.name" }, + { eventType: "m.room.member" }, + { eventType: "m.room.encryption" }, + { eventType: "org.matrix.msc3401.call.member" }, ]; - const sendRecvToDevice = [ - EventType.CallInvite, - EventType.CallCandidates, - EventType.CallAnswer, - EventType.CallHangup, - EventType.CallReject, - EventType.CallSelectAnswer, - EventType.CallNegotiate, - EventType.CallSDPStreamMetadataChanged, - EventType.CallSDPStreamMetadataChangedPrefix, - EventType.CallReplaces, - EventType.CallEncryptionKeysPrefix, - ]; + const sendRecvToDevice = ["io.element.call.encryption_keys"]; expect(createRoomWidgetClientSpy.mock.calls[0][1]).toStrictEqual({ sendEvent: [...sendEvent, ...sendRecvEvent], diff --git a/src/widget.ts b/src/widget.ts index 16dbf514..e04fd794 100644 --- a/src/widget.ts +++ b/src/widget.ts @@ -68,7 +68,10 @@ export let widget: WidgetHelpers | null; */ // this needs to be a seperate call and cannot be done on import to allow us to spy on methods in here before // execution. -export const initializeWidget = (): void => { +export const initializeWidget = ( + rtcApplication: string = "m.call", + sendRoomEvents = false, +): void => { try { const { widgetId, @@ -116,6 +119,9 @@ export const initializeWidget = (): void => { EventType.CallNotify, // Sent as a deprecated fallback EventType.RTCNotification, ]; + if (sendRoomEvents) { + sendEvent.push(EventType.RoomMessage); + } const sendRecvEvent = [ "org.matrix.rageshake_request", EventType.CallEncryptionKeysPrefix, @@ -128,8 +134,8 @@ export const initializeWidget = (): void => { const sendState = [ userId, // Legacy call membership events - `_${userId}_${deviceId}_m.call`, // Session membership events - `${userId}_${deviceId}_m.call`, // The above with no leading underscore, for room versions whose auth rules allow it + `_${userId}_${deviceId}_${rtcApplication}`, // Session membership events + `${userId}_${deviceId}_${rtcApplication}`, // The above with no leading underscore, for room versions whose auth rules allow it ].map((stateKey) => ({ eventType: EventType.GroupCallMemberPrefix, stateKey, @@ -142,19 +148,7 @@ export const initializeWidget = (): void => { { eventType: EventType.GroupCallMemberPrefix }, ]; - const sendRecvToDevice = [ - EventType.CallInvite, - EventType.CallCandidates, - EventType.CallAnswer, - EventType.CallHangup, - EventType.CallReject, - EventType.CallSelectAnswer, - EventType.CallNegotiate, - EventType.CallSDPStreamMetadataChanged, - EventType.CallSDPStreamMetadataChangedPrefix, - EventType.CallReplaces, - EventType.CallEncryptionKeysPrefix, - ]; + const sendRecvToDevice = [EventType.CallEncryptionKeysPrefix]; const client = createRoomWidgetClient( api, diff --git a/yarn.lock b/yarn.lock index e486bf6b..43ec9545 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8364,7 +8364,7 @@ __metadata: livekit-client: "npm:^2.13.0" lodash-es: "npm:^4.17.21" loglevel: "npm:^1.9.1" - matrix-js-sdk: "matrix-org/matrix-js-sdk#develop" + matrix-js-sdk: "npm:40.2.0-rc.0" matrix-widget-api: "npm:^1.16.1" node-stdlib-browser: "npm:^1.3.1" normalize.css: "npm:^8.0.1" @@ -11452,9 +11452,9 @@ __metadata: languageName: node linkType: hard -"matrix-js-sdk@matrix-org/matrix-js-sdk#develop": - version: 40.1.0 - resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=f2157f28bbadf2898fe21991f69ccb2af40df326" +"matrix-js-sdk@npm:40.2.0-rc.0": + version: 40.2.0-rc.0 + resolution: "matrix-js-sdk@npm:40.2.0-rc.0" dependencies: "@babel/runtime": "npm:^7.12.5" "@matrix-org/matrix-sdk-crypto-wasm": "npm:^17.0.0" @@ -11470,7 +11470,7 @@ __metadata: sdp-transform: "npm:^3.0.0" unhomoglyph: "npm:^1.0.6" uuid: "npm:13" - checksum: 10c0/d646b9214abbf0b9126760105edd9c57be7ffe8b53ae4acd5fefe841a51ad7d78fa57130922b3eac65ff2266b43f31ea60b4bdda9481e6bf8f1808d96726ed8a + checksum: 10c0/82311a60bc0fd2c8f5dff5219d05744d45577c2ea3145d17bef71e6ea194f4bb16f4557a5e74839dbc1b17fe95e08f0f510b7fd0da10f82dda8cb55ce28cd5f5 languageName: node linkType: hard