From 63fe888544f1765851edb520bbf5d504c683bfe1 Mon Sep 17 00:00:00 2001 From: Tiago Jacobs Date: Sun, 27 Mar 2022 22:18:34 -0300 Subject: [PATCH] Add ICE negotiation --- .../Classes/BBBSampleHandler.swift | 2 +- .../Classes/ScreenBroadcaster.swift | 93 ++++++++++++++++++- ios-common/Classes/BBBSharedData.swift | 6 ++ ios-common/Classes/IceCandidate.swift | 26 ++++++ ios-common/Classes/UserDefaults.swift | 10 ++ ios-common/Classes/WebRTCClient.swift | 31 ++++--- ios/NativeOnly/BigBlueButtonSDK.swift | 28 ++++++ .../ReactNativeEventEmitter.swift | 2 + src/events/onScreenShareLocalIceCandidate.tsx | 17 ++++ .../onScreenShareSignalingStateChange.tsx | 14 +++ src/index.tsx | 9 +- 11 files changed, 224 insertions(+), 14 deletions(-) create mode 100644 ios-common/Classes/IceCandidate.swift create mode 100644 src/events/onScreenShareLocalIceCandidate.tsx create mode 100644 src/events/onScreenShareSignalingStateChange.tsx diff --git a/ios-broadcast-upload-extension/Classes/BBBSampleHandler.swift b/ios-broadcast-upload-extension/Classes/BBBSampleHandler.swift index 33706ef..9a5d97d 100644 --- a/ios-broadcast-upload-extension/Classes/BBBSampleHandler.swift +++ b/ios-broadcast-upload-extension/Classes/BBBSampleHandler.swift @@ -33,7 +33,7 @@ open class BBBSampleHandler : RPBroadcastSampleHandler { logger.info("ReplayKit2 event - broadcastStarted - persisting information on UserDefaults") userDefaults.set(BBBSharedData.generatePayload(), forKey: BBBSharedData.SharedData.broadcastStarted) - self.screenBroadcaster = ScreenBroadcaster() + self.screenBroadcaster = ScreenBroadcaster(appGroupName: appGroupName) // Listen for createOffer requests from the UI APP logger.info("Configuring observer for createOffer") diff --git a/ios-broadcast-upload-extension/Classes/ScreenBroadcaster.swift b/ios-broadcast-upload-extension/Classes/ScreenBroadcaster.swift index 09be567..3816f49 100644 --- a/ios-broadcast-upload-extension/Classes/ScreenBroadcaster.swift +++ b/ios-broadcast-upload-extension/Classes/ScreenBroadcaster.swift @@ -5,18 +5,24 @@ // import os import bigbluebutton_mobile_sdk_common +import WebRTC open class ScreenBroadcaster { // Logger (these messages are displayed in the console application) private var logger = os.Logger(subsystem: "BigBlueButtonMobileSDK", category: "ScreenBroadcaster") private var webRTCClient:WebRTCClient + private var appGroupName:String + private let encoder = JSONEncoder() - init() { + init(appGroupName: String) { + self.appGroupName = appGroupName + webRTCClient = WebRTCClient(iceServers: ["stun:stun.l.google.com:19302", "stun:stun1.l.google.com:19302", "stun:stun2.l.google.com:19302", "stun:stun3.l.google.com:19302", "stun:stun4.l.google.com:19302"]) + webRTCClient.delegate = self } public func createOffer() async -> String? { @@ -38,6 +44,91 @@ open class ScreenBroadcaster { return false } } +} + +extension ScreenBroadcaster: WebRTCClientDelegate { + + public func webRTCClient(_ client: WebRTCClient, didDiscoverLocalCandidate rtcIceCandidate: RTCIceCandidate) { + do { + let iceCandidate = IceCandidate(from: rtcIceCandidate) + let iceCandidateAsJsonData = try self.encoder.encode(iceCandidate) + let iceCandidateAsJsonString = String(decoding: iceCandidateAsJsonData, as: UTF8.self) + + + BBBSharedData + .getUserDefaults(appGroupName: self.appGroupName) + .set(BBBSharedData.generatePayload(properties: [ + "iceJson": iceCandidateAsJsonString + ]), forKey: BBBSharedData.SharedData.onScreenShareLocalIceCandidate) + } catch { + self.logger.error("Error handling ICE candidate") + } + } + + public func webRTCClient(_ client: WebRTCClient, didChangeIceConnectionState state: RTCIceConnectionState) { + switch state { + case .connected: + self.logger.info("didChangeConnectionState -> connected") + case .completed: + self.logger.info("didChangeConnectionState -> completed") + case .disconnected: + self.logger.info("didChangeConnectionState -> disconnected") + case .failed: + self.logger.info("didChangeConnectionState -> failed") + case .closed: + self.logger.info("didChangeConnectionState -> closed") + case .new, .checking, .count: + break + @unknown default: + print("Unknown connection state.") + } + } + + public func webRTCClient(_ client: WebRTCClient, didChangeIceGatheringState state: RTCIceGatheringState) { + switch state { + case .new: + self.logger.info("didChangeGatheringState -> new") + case .gathering: + self.logger.info("didChangeGatheringState -> gathering") + case .complete: + self.logger.info("didChangeGatheringState -> complete") + @unknown default: + self.logger.error("Unknown gathering state: \(state.rawValue)") + } + } + + public func webRTCClient(_ client: WebRTCClient, didChangeSignalingState state: RTCSignalingState) { + var stateString = "" + switch(state) { + case .haveLocalOffer: + self.logger.info("peerConnection new signaling state -> haveLocalOffer") + stateString = "have-local-offer" + case .haveLocalPrAnswer: + self.logger.info("peerConnection new signaling state -> haveLocalPrAnswer") + stateString = "have-local-pranswer" + case .haveRemoteOffer: + self.logger.info("peerConnection new signaling state -> haveRemoteOffer") + stateString = "have-remote-offer" + case .haveRemotePrAnswer: + self.logger.info("peerConnection new signaling state -> haveRemotePrAnswer") + stateString = "have-remote-pranswer" + case .stable: + self.logger.info("peerConnection new signaling state -> stable") + stateString = "stable" + case .closed: + self.logger.info("peerConnection new signaling state -> closed") + stateString = "closed" + default: + self.logger.error("peerConnection new signaling state -> UNKNOWN") + } + + BBBSharedData + .getUserDefaults(appGroupName: self.appGroupName) + .set(BBBSharedData.generatePayload(properties: [ + "newState": stateString + ]), forKey: BBBSharedData.SharedData.onScreenShareSignalingStateChange) + } } + diff --git a/ios-common/Classes/BBBSharedData.swift b/ios-common/Classes/BBBSharedData.swift index 251968d..29aa34d 100644 --- a/ios-common/Classes/BBBSharedData.swift +++ b/ios-common/Classes/BBBSharedData.swift @@ -18,10 +18,16 @@ open class BBBSharedData { public static let broadcastPaused = "broadcastPaused" // Broadcaster -> UI APP public static let broadcastResumed = "broadcastResumed" // Broadcaster -> UI APP public static let broadcastFinished = "broadcastFinished" // Broadcaster -> UI APP + public static let createScreenShareOffer = "createScreenShareOffer" // UI APP -> Broadcaster public static let screenShareOfferCreated = "screenShareOfferCreated" // Broadcaster -> UI APP + public static let setScreenShareRemoteSDP = "setScreenShareRemoteSDP" // UI APP -> Broadcaster public static let setScreenShareRemoteSDPCompleted = "setScreenShareRemoteSDPCompleted" // Broadcaster -> UI APP + + public static let onScreenShareLocalIceCandidate = "onScreenShareLocalIceCandidate" // Broadcaster -> UI APP + + public static let onScreenShareSignalingStateChange = "onScreenShareSignalingStateChange" // Broadcaster -> UI APP } // Get reference to userDefaults object (that's actually the object used to share information among UI APP and the BroadcastUploadExtension APP) diff --git a/ios-common/Classes/IceCandidate.swift b/ios-common/Classes/IceCandidate.swift new file mode 100644 index 0000000..0cb7e46 --- /dev/null +++ b/ios-common/Classes/IceCandidate.swift @@ -0,0 +1,26 @@ +// +// IceCandidate.swift +// WebRTC-Demo +// +// Created by Stasel on 20/02/2019. +// Copyright © 2019 Stasel. All rights reserved. +// +import Foundation +import WebRTC + +/// This struct is a swift wrapper over `RTCIceCandidate` for easy encode and decode +public struct IceCandidate: Codable { + let candidate: String + let sdpMLineIndex: Int32 + let sdpMid: String? + + public init(from iceCandidate: RTCIceCandidate) { + self.sdpMLineIndex = iceCandidate.sdpMLineIndex + self.sdpMid = iceCandidate.sdpMid + self.candidate = iceCandidate.sdp + } + + var rtcIceCandidate: RTCIceCandidate { + return RTCIceCandidate(sdp: self.candidate, sdpMLineIndex: self.sdpMLineIndex, sdpMid: self.sdpMid) + } +} diff --git a/ios-common/Classes/UserDefaults.swift b/ios-common/Classes/UserDefaults.swift index d9464d6..a1c8d03 100644 --- a/ios-common/Classes/UserDefaults.swift +++ b/ios-common/Classes/UserDefaults.swift @@ -44,5 +44,15 @@ extension UserDefaults { @objc open dynamic var setScreenShareRemoteSDPCompleted: String { return string(forKey: BBBSharedData.SharedData.setScreenShareRemoteSDPCompleted) ?? "" } + + // Broadcaster -> UI APP + @objc open dynamic var onScreenShareLocalIceCandidate: String { + return string(forKey: BBBSharedData.SharedData.onScreenShareLocalIceCandidate) ?? "" + } + + // Broadcaster -> UI APP + @objc open dynamic var onScreenShareSignalingStateChange: String { + return string(forKey: BBBSharedData.SharedData.onScreenShareSignalingStateChange) ?? "" + } } diff --git a/ios-common/Classes/WebRTCClient.swift b/ios-common/Classes/WebRTCClient.swift index b31b6ed..2f69052 100644 --- a/ios-common/Classes/WebRTCClient.swift +++ b/ios-common/Classes/WebRTCClient.swift @@ -9,9 +9,11 @@ import Foundation import WebRTC import os -protocol WebRTCClientDelegate: AnyObject { +public protocol WebRTCClientDelegate: AnyObject { func webRTCClient(_ client: WebRTCClient, didDiscoverLocalCandidate candidate: RTCIceCandidate) - func webRTCClient(_ client: WebRTCClient, didChangeConnectionState state: RTCIceConnectionState) + func webRTCClient(_ client: WebRTCClient, didChangeIceConnectionState state: RTCIceConnectionState) + func webRTCClient(_ client: WebRTCClient, didChangeIceGatheringState state: RTCIceGatheringState) + func webRTCClient(_ client: WebRTCClient, didChangeSignalingState state: RTCSignalingState) } open class WebRTCClient: NSObject { @@ -27,7 +29,7 @@ open class WebRTCClient: NSObject { return RTCPeerConnectionFactory(encoderFactory: videoEncoderFactory, decoderFactory: videoDecoderFactory) }() - weak var delegate: WebRTCClientDelegate? + public weak var delegate: WebRTCClientDelegate? private let peerConnection: RTCPeerConnection private let rtcAudioSession = RTCAudioSession.sharedInstance() private let audioQueue = DispatchQueue(label: "audio") @@ -78,13 +80,9 @@ open class WebRTCClient: NSObject { return sdp } - public func setRemoteSDP(remoteSDP: String) async { - do { - let rtcSessionDescription = RTCSessionDescription(type: RTCSdpType.answer, sdp: remoteSDP) - try await self.peerConnection.setRemoteDescription(rtcSessionDescription) - } catch { - self.logger.error("Error setting remote SDP") - } + public func setRemoteSDP(remoteSDP: String) async throws { + let rtcSessionDescription = RTCSessionDescription(type: RTCSdpType.answer, sdp: remoteSDP) + try await self.peerConnection.setRemoteDescription(rtcSessionDescription) } @@ -147,6 +145,8 @@ extension WebRTCClient: RTCPeerConnectionDelegate { public func peerConnection(_ peerConnection: RTCPeerConnection, didChange stateChanged: RTCSignalingState) { self.logger.info("peerConnection new signaling state: \(stateChanged.rawValue)") + + self.delegate?.webRTCClient(self, didChangeSignalingState: stateChanged) } public func peerConnection(_ peerConnection: RTCPeerConnection, didAdd stream: RTCMediaStream) { @@ -163,11 +163,20 @@ extension WebRTCClient: RTCPeerConnectionDelegate { public func peerConnection(_ peerConnection: RTCPeerConnection, didChange newState: RTCIceConnectionState) { self.logger.info("peerConnection new connection state: \(newState.rawValue)") - self.delegate?.webRTCClient(self, didChangeConnectionState: newState) + self.delegate?.webRTCClient(self, didChangeIceConnectionState: newState) } public func peerConnection(_ peerConnection: RTCPeerConnection, didChange newState: RTCIceGatheringState) { self.logger.info("peerConnection new gathering state: \(newState.rawValue)") + self.delegate?.webRTCClient(self, didChangeIceGatheringState: newState) + + if(newState == .complete) { + self.logger.info("peerConnection new gathering state is COMPLETE") + } else if(newState == .gathering) { + self.logger.info("peerConnection new gathering state is GATHERING") + } else if(newState == .new) { + self.logger.info("peerConnection new gathering state is NEW") + } } public func peerConnection(_ peerConnection: RTCPeerConnection, didGenerate candidate: RTCIceCandidate) { diff --git a/ios/NativeOnly/BigBlueButtonSDK.swift b/ios/NativeOnly/BigBlueButtonSDK.swift index bbd9d71..7555da6 100644 --- a/ios/NativeOnly/BigBlueButtonSDK.swift +++ b/ios/NativeOnly/BigBlueButtonSDK.swift @@ -18,6 +18,8 @@ open class BigBlueButtonSDK: NSObject { private static var observer1: NSKeyValueObservation? private static var observer2: NSKeyValueObservation? private static var observer3: NSKeyValueObservation? + private static var observer4: NSKeyValueObservation? + private static var observer5: NSKeyValueObservation? public static func initialize(broadcastExtensionBundleId:String, appGroupName:String) { self.broadcastExtensionBundleId = broadcastExtensionBundleId @@ -50,6 +52,32 @@ open class BigBlueButtonSDK: NSObject { logger.info("Detected a change in userDefaults for key setScreenShareRemoteSDPCompleted") ReactNativeEventEmitter.emitter.sendEvent(withName: ReactNativeEventEmitter.EVENT.onSetScreenShareRemoteSDPCompleted.rawValue, body: nil) } + + //setScreenShareRemoteSDPCompleted + observer4 = userDefaults?.observe(\.onScreenShareLocalIceCandidate, options: [.new]) { (defaults, change) in + let payload:String = (change.newValue!); + logger.info("Detected a change in userDefaults for key onScreenShareLocalIceCandidate \(payload)") + let payloadData = payload.data(using: .utf8)! + + let decodedPayload = (try? JSONDecoder().decode([String: String].self, from: payloadData))! + let iceJson = decodedPayload["iceJson"] + + logger.info("") + ReactNativeEventEmitter.emitter.sendEvent(withName: ReactNativeEventEmitter.EVENT.onScreenShareLocalIceCandidate.rawValue, body: iceJson) + } + + + //onScreenShareSignalingStateChange + observer5 = userDefaults?.observe(\.onScreenShareSignalingStateChange, options: [.new]) { (defaults, change) in + let payload:String = (change.newValue!); + logger.info("Detected a change in userDefaults for key onScreenShareSignalingStateChange \(payload)") + let payloadData = payload.data(using: .utf8)! + + let decodedPayload = (try? JSONDecoder().decode([String: String].self, from: payloadData))! + let newState = decodedPayload["newState"] + + ReactNativeEventEmitter.emitter.sendEvent(withName: ReactNativeEventEmitter.EVENT.onScreenShareSignalingStateChange.rawValue, body: newState) + } } public static func getBroadcastExtensionBundleId() -> String { diff --git a/ios/ReactExported/ReactNativeEventEmitter.swift b/ios/ReactExported/ReactNativeEventEmitter.swift index faff596..17942cf 100644 --- a/ios/ReactExported/ReactNativeEventEmitter.swift +++ b/ios/ReactExported/ReactNativeEventEmitter.swift @@ -20,6 +20,8 @@ open class ReactNativeEventEmitter: RCTEventEmitter { case onBroadcastFinished = "onBroadcastFinished" case onScreenShareOfferCreated = "onScreenShareOfferCreated" case onSetScreenShareRemoteSDPCompleted = "onSetScreenShareRemoteSDPCompleted" + case onScreenShareLocalIceCandidate = "onScreenShareLocalIceCandidate" + case onScreenShareSignalingStateChange = "onScreenShareSignalingStateChange" } override init() { diff --git a/src/events/onScreenShareLocalIceCandidate.tsx b/src/events/onScreenShareLocalIceCandidate.tsx new file mode 100644 index 0000000..e4d64c5 --- /dev/null +++ b/src/events/onScreenShareLocalIceCandidate.tsx @@ -0,0 +1,17 @@ +import type { MutableRefObject } from 'react'; +import nativeEmitter from '../native-messaging/emitter'; + +export function setupListener(_webViewRef: MutableRefObject) { + // Resolve promise when SDP offer is available + nativeEmitter.addListener( + 'onScreenShareLocalIceCandidate', + (jsonEncodedIceCandidate) => { + const iceCandidate = JSON.parse(jsonEncodedIceCandidate); + _webViewRef.current.injectJavaScript( + `window.bbbMobileScreenShareIceCandidateCallback(${JSON.stringify( + iceCandidate + )});` + ); + } + ); +} diff --git a/src/events/onScreenShareSignalingStateChange.tsx b/src/events/onScreenShareSignalingStateChange.tsx new file mode 100644 index 0000000..b69e8ce --- /dev/null +++ b/src/events/onScreenShareSignalingStateChange.tsx @@ -0,0 +1,14 @@ +import type { MutableRefObject } from 'react'; +import nativeEmitter from '../native-messaging/emitter'; + +export function setupListener(_webViewRef: MutableRefObject) { + // Resolve promise when SDP offer is available + nativeEmitter.addListener('onScreenShareSignalingStateChange', (newState) => { + console.log(`Temos um novo state: ${newState}`); + _webViewRef.current.injectJavaScript( + `window.bbbMobileScreenShareSignalingStateChangeCallback(${JSON.stringify( + newState + )});` + ); + }); +} diff --git a/src/index.tsx b/src/index.tsx index 4a2ae7b..ffc445c 100644 --- a/src/index.tsx +++ b/src/index.tsx @@ -1,8 +1,10 @@ import { Platform, ViewStyle } from 'react-native'; -import React, { useRef } from 'react'; +import React, { useEffect, useRef } from 'react'; import BBBN_SystemBroadcastPicker from './native-components/BBBN_SystemBroadcastPicker'; import { WebView } from 'react-native-webview'; import { handleWebviewMessage } from './webview/message-handler'; +import * as onScreenShareLocalIceCandidate from './events/onScreenShareLocalIceCandidate'; +import * as onScreenShareSignalingStateChange from './events/onScreenShareSignalingStateChange'; type BigbluebuttonMobileSdkProps = { url: string; @@ -21,6 +23,11 @@ export const BigbluebuttonMobile = ({ }: BigbluebuttonMobileSdkProps) => { const webViewRef = useRef(null); + useEffect(() => { + onScreenShareLocalIceCandidate.setupListener(webViewRef); + onScreenShareSignalingStateChange.setupListener(webViewRef); + }, [webViewRef]); + return ( <> {renderPlatformSpecificComponents()}