From 4e9c02165bab3f0256c90e2286f70ce3ea920ed8 Mon Sep 17 00:00:00 2001 From: Denis Koltovich Date: Mon, 13 Jan 2025 17:59:02 +0300 Subject: [PATCH 1/4] feat: add voice activity handler --- .../WebRTCModule/PeerConnectionObserver.java | 4 + .../com/oney/WebRTCModule/WebRTCModule.java | 156 ++++++++++++++++++ .../WebRTCModule+RTCPeerConnection.m | 53 +++++- ios/RCTWebRTC/WebRTCModule.h | 3 + ios/RCTWebRTC/WebRTCModule.m | 1 + ios/RCTWebRTC/WebRTCVideoCaptureHandler.swift | 95 +++++++++++ package.json | 2 +- src/EventEmitter.ts | 1 + src/RTCPeerConnection.ts | 19 +++ 9 files changed, 325 insertions(+), 9 deletions(-) diff --git a/android/src/main/java/com/oney/WebRTCModule/PeerConnectionObserver.java b/android/src/main/java/com/oney/WebRTCModule/PeerConnectionObserver.java index de7ccf518..8652b1ccc 100644 --- a/android/src/main/java/com/oney/WebRTCModule/PeerConnectionObserver.java +++ b/android/src/main/java/com/oney/WebRTCModule/PeerConnectionObserver.java @@ -58,6 +58,10 @@ PeerConnection getPeerConnection() { return peerConnection; } + int getId() { + return id; + } + void setPeerConnection(PeerConnection peerConnection) { this.peerConnection = peerConnection; } diff --git a/android/src/main/java/com/oney/WebRTCModule/WebRTCModule.java b/android/src/main/java/com/oney/WebRTCModule/WebRTCModule.java index 41a23ba6e..7cda29350 100644 --- a/android/src/main/java/com/oney/WebRTCModule/WebRTCModule.java +++ b/android/src/main/java/com/oney/WebRTCModule/WebRTCModule.java @@ -33,9 +33,38 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Timer; +import java.util.TimerTask; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; +interface OnValueChangeListener { + void onValueChanged(PeerConnectionObserver peerConnection, boolean isSpeak, double audioLevel); +} + +class AudioLevelValueHolder { + private double audioLevel; + private boolean isSpeak; + private PeerConnectionObserver peerConnection; + private OnValueChangeListener listener; + + void setOnValueChangeListener(OnValueChangeListener listener) { + this.listener = listener; + } + + void setValue(PeerConnectionObserver peerConnection, boolean isSpeak, double audioLevel) { + if (this.isSpeak != isSpeak) { + this.isSpeak = isSpeak; + this.audioLevel = audioLevel; + this.peerConnection = peerConnection; + + if (listener != null) { + listener.onValueChanged(peerConnection, isSpeak, audioLevel); + } + } + } +} + @ReactModule(name = "WebRTCModule") public class WebRTCModule extends ReactContextBaseJavaModule { static final String TAG = WebRTCModule.class.getCanonicalName(); @@ -49,6 +78,12 @@ public class WebRTCModule extends ReactContextBaseJavaModule { private final SparseArray mPeerConnectionObservers; final Map localStreams; + Timer voiceTimer = new Timer(); + boolean isVoiceTimerRunning = false; + + AudioLevelValueHolder incomingAudioLevelHolder; + AudioLevelValueHolder outgoingAudioLevelHolder; + private final GetUserMediaImpl getUserMediaImpl; public WebRTCModule(ReactApplicationContext reactContext) { @@ -57,6 +92,39 @@ public WebRTCModule(ReactApplicationContext reactContext) { mPeerConnectionObservers = new SparseArray<>(); localStreams = new HashMap<>(); + incomingAudioLevelHolder = new AudioLevelValueHolder(); + outgoingAudioLevelHolder = new AudioLevelValueHolder(); + + incomingAudioLevelHolder.setOnValueChangeListener(new OnValueChangeListener() { + @Override + public void onValueChanged(PeerConnectionObserver peerConnection, boolean isSpeak, double audioLevel) { + ThreadUtils.runOnExecutor(() -> { + WritableMap params = Arguments.createMap(); + WritableMap childParams = Arguments.createMap(); + childParams.putBoolean("isSpeak", isSpeak); + childParams.putDouble("audioLevel", audioLevel); + + params.putInt("pcId", peerConnection.getId()); + params.putMap("incoming", childParams); + sendEvent("peerVoiceStateChanged", params); + }); + } + }); + + outgoingAudioLevelHolder.setOnValueChangeListener(new OnValueChangeListener() { + @Override + public void onValueChanged(PeerConnectionObserver peerConnection, boolean isSpeak, double audioLevel) { + WritableMap params = Arguments.createMap(); + WritableMap childParams = Arguments.createMap(); + childParams.putBoolean("isSpeak", isSpeak); + childParams.putDouble("audioLevel", audioLevel); + + params.putInt("pcId", peerConnection.getId()); + params.putMap("outgoing", childParams); + sendEvent("peerVoiceStateChanged", params); + } + }); + WebRTCModuleOptions options = WebRTCModuleOptions.getInstance(); AudioDeviceModule adm = options.audioDeviceModule; @@ -386,6 +454,8 @@ public boolean peerConnectionInit(ReadableMap configuration, int id) { } observer.setPeerConnection(peerConnection); mPeerConnectionObservers.put(id, observer); + observeVoiceActivity(); + return true; }) .get(); @@ -395,6 +465,92 @@ public boolean peerConnectionInit(ReadableMap configuration, int id) { } } + void stopObserveVoiceActivity() { + if (isVoiceTimerRunning) { + try { + voiceTimer.cancel(); + } catch (RuntimeException e) { + Log.i("RuntimeException", e.getLocalizedMessage()); + } + + isVoiceTimerRunning = false; + } + } + void observeVoiceActivity() { + double checkInterval = 0.1; + double silenceThreshold = 0.3; + + final double[] silenceIncomingCount = {0}; + final double[] silenceOutgoingCount = {0}; + + stopObserveVoiceActivity(); + + if (mPeerConnectionObservers.size() == 0) { + return; + } + + voiceTimer.schedule(new TimerTask() { + @Override + public void run() { + for (int i = 0; i < mPeerConnectionObservers.size(); i++) { + int key = mPeerConnectionObservers.keyAt(i); + PeerConnectionObserver peer = mPeerConnectionObservers.get(key); + + if (peer.getPeerConnection().connectionState() == PeerConnection.PeerConnectionState.CONNECTED) { + peer.getPeerConnection().getStats(new RTCStatsCollectorCallback() { + @Override + public void onStatsDelivered(RTCStatsReport rtcStatsReport) { + + for (RTCStats stats : rtcStatsReport.getStatsMap().values()) { + if (stats.getType().equals("inbound-rtp")) { + Object audioLevelObject = stats.getMembers().get("audioLevel"); + + if (audioLevelObject instanceof Double) { + Double audioLevel = ((Double) audioLevelObject); + + if (audioLevel > 0.1) { + incomingAudioLevelHolder.setValue(peer, true, audioLevel.doubleValue()); + } else { + silenceIncomingCount[0] += 1; + + if (silenceIncomingCount[0] > silenceThreshold / checkInterval) { + incomingAudioLevelHolder.setValue(peer, false, 0); + } + } + } + } + + if (stats.getType().equals("media-source")) { + Object audioLevelObject = stats.getMembers().get("audioLevel"); + + if (audioLevelObject instanceof Double) { + Double audioLevel = ((Double) audioLevelObject); + + if (audioLevel > 0.1) { + outgoingAudioLevelHolder.setValue(peer, true, audioLevel.doubleValue()); + } else { + silenceOutgoingCount[0] += 1; + + if (silenceOutgoingCount[0] > silenceThreshold / checkInterval) { + outgoingAudioLevelHolder.setValue(peer, false, 0); + } + } + } + } + } + } + }); + } else { + incomingAudioLevelHolder.setValue(peer, false, 0); + outgoingAudioLevelHolder.setValue(peer, false, 0); + } + } + } + }, 0, 100); + + isVoiceTimerRunning = true; + } + MediaStream getStreamForReactTag(String streamReactTag) { // This function _only_ gets called from WebRTCView, in the UI thread. // Hence make sure we run this code in the executor or we run at the risk diff --git a/ios/RCTWebRTC/WebRTCModule+RTCPeerConnection.m b/ios/RCTWebRTC/WebRTCModule+RTCPeerConnection.m index 802c39330..7f79e97fe 100644 --- a/ios/RCTWebRTC/WebRTCModule+RTCPeerConnection.m +++ b/ios/RCTWebRTC/WebRTCModule+RTCPeerConnection.m @@ -22,8 +22,10 @@ #import "WebRTCModule.h" #import "WebRTCAudioSession.h" -@implementation RTCPeerConnection (React) +#import "react_native_webrtc-Swift.h" + +@implementation RTCPeerConnection (React) - (NSMutableDictionary *)dataChannels { return objc_getAssociatedObject(self, _cmd); } @@ -98,11 +100,44 @@ @implementation WebRTCModule (RTCPeerConnection) peerConnection.webRTCModule = self; self.peerConnections[objectID] = peerConnection; + + [self checkAudioLevel]; }); return @(ret); } +- (void)checkAudioLevel { + // Cancel prev observer + [(WebRTCVoiceHandler*)self.voiceHandler stopObserve]; + + if (self.peerConnections.count == 0) { + return; + } + + self.voiceHandler = [[WebRTCVoiceHandler new] + startObserveWithPeerConnections:self.peerConnections.allValues + voiceClosure:^(RTCPeerConnection* peerConnection, BOOL outgoing, BOOL isSpeak, double audioLevel) { + + dispatch_async(self.workerQueue, ^{ + [self sendEventWithName:kEventPeerVoiceStateChanged + body: + outgoing + ? + @{@"outgoing": @{ + @"isSpeak" : @(isSpeak), + @"audioLevel" : @(audioLevel) + }, @"pcId" : peerConnection.reactTag} + : + @{@"incoming": @{ + @"isSpeak" : @(isSpeak), + @"audioLevel" : @(audioLevel) + }, @"pcId" : peerConnection.reactTag} + ]; + }); + }]; +} + RCT_EXPORT_METHOD(peerConnectionSetConfiguration : (RTCConfiguration *)configuration objectID : (nonnull NSNumber *)objectID) { @@ -857,9 +892,9 @@ - (void)peerConnection:(RTC_OBJC_TYPE(RTCPeerConnection) *)peerConnection streams:(NSArray *)mediaStreams { dispatch_async(self.workerQueue, ^{ RCTLogWarn(@"PeerConnection %@ didAddReceiver %@", peerConnection.reactTag, rtpReceiver.receiverId); - + RTCRtpTransceiver *transceiver = nil; - + for (RTCRtpTransceiver *t in peerConnection.transceivers) { if ([rtpReceiver.receiverId isEqual:t.receiver.receiverId]) { transceiver = t; @@ -878,13 +913,13 @@ - (void)peerConnection:(RTC_OBJC_TYPE(RTCPeerConnection) *)peerConnection RTCVideoTrack *videoTrack = (RTCVideoTrack *)track; [peerConnection addVideoTrackAdapter:videoTrack]; } - + peerConnection.remoteTracks[track.trackId] = track; - + NSMutableArray *streams = [NSMutableArray new]; NSMutableDictionary *params = [NSMutableDictionary new]; - - + + // NSLog(@"TEST: Remote tracks: "); // for (NSString * key in [peerConnection.remoteTracks allKeys]) { // NSLog(@"TEST: Remote trackId: %@", key); @@ -896,7 +931,7 @@ - (void)peerConnection:(RTC_OBJC_TYPE(RTCPeerConnection) *)peerConnection // RTCMediaStream* remoteStream = [peerConnection.remoteStreams objectForKey:key]; // NSLog(@"TEST: stream: %@", remoteStream); // } - + for (RTCMediaStream * stream in mediaStreams) { // NSLog(@"TEST: Coming stream: %@", stream); NSString *streamReactTag = nil; @@ -957,4 +992,6 @@ - (void)peerConnection:(nonnull RTCPeerConnection *)peerConnection didRemoveStre RCTLogWarn(@"PeerConnection %@ didRemoveStream %@", peerConnection.reactTag, stream.streamId); } + + @end diff --git a/ios/RCTWebRTC/WebRTCModule.h b/ios/RCTWebRTC/WebRTCModule.h index 040dfde99..12fdc7952 100644 --- a/ios/RCTWebRTC/WebRTCModule.h +++ b/ios/RCTWebRTC/WebRTCModule.h @@ -8,6 +8,7 @@ #import #import "VideoCaptureController.h" +static NSString *const kEventPeerVoiceStateChanged = @"peerVoiceStateChanged"; static NSString *const kEventPeerConnectionSignalingStateChanged = @"peerConnectionSignalingStateChanged"; static NSString *const kEventPeerConnectionStateChanged = @"peerConnectionStateChanged"; static NSString *const kEventPeerConnectionOnRenegotiationNeeded = @"peerConnectionOnRenegotiationNeeded"; @@ -26,6 +27,7 @@ static NSString *const kEventPeerConnectionOnTrack = @"peerConnectionOnTrack"; @interface WebRTCModule : RCTEventEmitter @property(nonatomic, strong) dispatch_queue_t workerQueue; +@property(nonatomic, strong) dispatch_source_t timer; @property(nonatomic, strong) RTCPeerConnectionFactory *peerConnectionFactory; @property(nonatomic, strong) id decoderFactory; @@ -35,6 +37,7 @@ static NSString *const kEventPeerConnectionOnTrack = @"peerConnectionOnTrack"; @property(nonatomic, strong) NSMutableDictionary *localStreams; @property(nonatomic, strong) NSMutableDictionary *localTracks; @property (nonatomic, strong) id videoSourceInterceptor; +@property (nonatomic, strong) id voiceHandler; @property (nonatomic, strong) VideoCaptureController *videoCaptureController; - (RTCMediaStream *)streamForReactTag:(NSString *)reactTag; diff --git a/ios/RCTWebRTC/WebRTCModule.m b/ios/RCTWebRTC/WebRTCModule.m index 93ff75097..6c3eabd16 100644 --- a/ios/RCTWebRTC/WebRTCModule.m +++ b/ios/RCTWebRTC/WebRTCModule.m @@ -107,6 +107,7 @@ - (dispatch_queue_t)methodQueue { - (NSArray *)supportedEvents { return @[ + kEventPeerVoiceStateChanged, kEventPeerConnectionSignalingStateChanged, kEventPeerConnectionStateChanged, kEventPeerConnectionOnRenegotiationNeeded, diff --git a/ios/RCTWebRTC/WebRTCVideoCaptureHandler.swift b/ios/RCTWebRTC/WebRTCVideoCaptureHandler.swift index a7173d00a..46e5b6eee 100644 --- a/ios/RCTWebRTC/WebRTCVideoCaptureHandler.swift +++ b/ios/RCTWebRTC/WebRTCVideoCaptureHandler.swift @@ -13,6 +13,101 @@ import CoreImage import Foundation import CoreImage.CIFilterBuiltins +final public class WebRTCVoiceHandler: NSObject { + private var outgoingVoicePublisher: CurrentValueSubject<(RTCPeerConnection?, Bool, Double), Never> = .init((nil, false, 0)) + private var incomingVoicePublisher: CurrentValueSubject<(RTCPeerConnection?, Bool, Double), Never> = .init((nil, false, 0)) + + private var observeTask: Task? + private var disposeBag: Set = [] + + @objc + public func startObserve(peerConnections: [RTCPeerConnection], + voiceClosure: @escaping (RTCPeerConnection, Bool, Bool, Double) -> Void) -> Self { + let _ = stopObserve() + + observeTask = Task { + await observeVoiceActivity(peerConnections: peerConnections) + } + + outgoingVoicePublisher.removeDuplicates(by: { $0.1 == $1.1 }).sink { (peer, isSpeak, audioLevel) in + guard let peer else { return } + voiceClosure(peer, true, isSpeak, audioLevel) + }.store(in: &disposeBag) + + incomingVoicePublisher.removeDuplicates(by: { $0.1 == $1.1 }).sink { (peer, isSpeak, audioLevel) in + guard let peer else { return } + voiceClosure(peer, false, isSpeak, audioLevel) + }.store(in: &disposeBag) + + return self + } + + @objc + public func stopObserve() -> Self { + observeTask?.cancel() + disposeBag.forEach { $0.cancel() } + + return self + } + + private func observeVoiceActivity(peerConnections: [RTCPeerConnection]) async { + let checkInterval: Double = 0.1 + let silenceThreshold: Double = 0.3 + + var silenceIncomingCount: Double = 0 + var silenceOutgoingCount: Double = 0 + + while !Task.isCancelled { + do { + peerConnections.forEach { [weak self] peerConnection in + guard let self = self else { return } + + if peerConnection.connectionState == .connected { + peerConnection.statistics { reports in + for statistic in reports.statistics.values { + if statistic.type == "inbound-rtp" { + guard let audioLevel = statistic.values["audioLevel"] as? Double else { return } + + if audioLevel > 0.1 { + self.incomingVoicePublisher.send((peerConnection, true, audioLevel)) + } else { + silenceIncomingCount += 1 + + if silenceIncomingCount > silenceThreshold / checkInterval { + self.incomingVoicePublisher.send((peerConnection, false, audioLevel)) + } + } + } + + if statistic.type == "media-source" { + guard let audioLevel = statistic.values["audioLevel"] as? Double else { return } + + if audioLevel > 0.1 { + self.outgoingVoicePublisher.send((peerConnection, true, audioLevel)) + } else { + silenceOutgoingCount += 1 + + if silenceOutgoingCount > silenceThreshold / checkInterval { + self.outgoingVoicePublisher.send((peerConnection, false, audioLevel)) + } + } + } + } + } + } else { + self.incomingVoicePublisher.send((peerConnection, false, 0)) + self.outgoingVoicePublisher.send((peerConnection, false, 0)) + } + } + + try await Task.sleep(nanoseconds: UInt64(checkInterval * 1000000000)) + } catch { + print(error) + } + } + } +} + final public class WebRTCVideoCaptureHandler: NSObject, RTCVideoCapturerDelegate { var selectedFilter: VideoFilter? diff --git a/package.json b/package.json index 1d7185327..2fc693596 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "react-native-webrtc", - "version": "124.0.6", + "version": "124.0.7", "repository": { "type": "git", "url": "git+https://github.com/react-native-webrtc/react-native-webrtc.git" diff --git a/src/EventEmitter.ts b/src/EventEmitter.ts index 48adb14fb..59a3515a0 100644 --- a/src/EventEmitter.ts +++ b/src/EventEmitter.ts @@ -9,6 +9,7 @@ const { WebRTCModule } = NativeModules; const nativeEmitter = new NativeEventEmitter(WebRTCModule); const NATIVE_EVENTS = [ + 'peerVoiceStateChanged', 'peerConnectionSignalingStateChanged', 'peerConnectionStateChanged', 'peerConnectionOnRenegotiationNeeded', diff --git a/src/RTCPeerConnection.ts b/src/RTCPeerConnection.ts index cc88293d7..fd585052d 100644 --- a/src/RTCPeerConnection.ts +++ b/src/RTCPeerConnection.ts @@ -45,6 +45,11 @@ type RTCDataChannelInit = { id?: number }; +type RTCVoice = { + incoming: { isSpeak: boolean, audioLevel: number }, + outgoing: { isSpeak: boolean, audioLevel: number } +}; + type RTCIceServer = { credential?: string, url?: string, // Deprecated. @@ -68,6 +73,7 @@ type RTCPeerConnectionEventMap = { icegatheringstatechange: Event<'icegatheringstatechange'> negotiationneeded: Event<'negotiationneeded'> signalingstatechange: Event<'signalingstatechange'> + voicestatechange: Event<'voicestatechange'> datachannel: RTCDataChannelEvent<'datachannel'> track: RTCTrackEvent<'track'> error: Event<'error'> @@ -83,6 +89,8 @@ export default class RTCPeerConnection extends EventTarget; @@ -598,6 +606,16 @@ export default class RTCPeerConnection extends EventTarget { + if (ev.pcId !== this._pcId) { + return; + } + + this.voiceState = { ...this.voiceState, ...ev }; + + this.dispatchEvent(new Event('voicestatechange')); + }); addListener(this, 'peerConnectionSignalingStateChanged', (ev: any) => { if (ev.pcId !== this._pcId) { @@ -828,6 +846,7 @@ defineEventAttribute(proto, 'iceconnectionstatechange'); defineEventAttribute(proto, 'icegatheringstatechange'); defineEventAttribute(proto, 'negotiationneeded'); defineEventAttribute(proto, 'signalingstatechange'); +defineEventAttribute(proto, 'voicestatechange'); defineEventAttribute(proto, 'datachannel'); defineEventAttribute(proto, 'track'); defineEventAttribute(proto, 'error'); From 8e694dbd0f3e6ea0efff724a9fa131e10adb44a7 Mon Sep 17 00:00:00 2001 From: Denis Koltovich Date: Thu, 19 Jun 2025 11:06:42 +0300 Subject: [PATCH 2/4] fix: Rename --- .../com/oney/WebRTCModule/WebRTCModule.java | 20 +++++++++---------- .../WebRTCModule+RTCPeerConnection.m | 6 +++--- ios/RCTWebRTC/WebRTCVideoCaptureHandler.swift | 8 ++++---- src/RTCPeerConnection.ts | 6 +++--- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/android/src/main/java/com/oney/WebRTCModule/WebRTCModule.java b/android/src/main/java/com/oney/WebRTCModule/WebRTCModule.java index 7cda29350..ff242da3b 100644 --- a/android/src/main/java/com/oney/WebRTCModule/WebRTCModule.java +++ b/android/src/main/java/com/oney/WebRTCModule/WebRTCModule.java @@ -39,12 +39,12 @@ import java.util.concurrent.ExecutionException; interface OnValueChangeListener { - void onValueChanged(PeerConnectionObserver peerConnection, boolean isSpeak, double audioLevel); + void onValueChanged(PeerConnectionObserver peerConnection, boolean isSpeaking, double audioLevel); } class AudioLevelValueHolder { private double audioLevel; - private boolean isSpeak; + private boolean isSpeaking; private PeerConnectionObserver peerConnection; private OnValueChangeListener listener; @@ -52,14 +52,14 @@ void setOnValueChangeListener(OnValueChangeListener listener) { this.listener = listener; } - void setValue(PeerConnectionObserver peerConnection, boolean isSpeak, double audioLevel) { - if (this.isSpeak != isSpeak) { - this.isSpeak = isSpeak; + void setValue(PeerConnectionObserver peerConnection, boolean isSpeaking, double audioLevel) { + if (this.isSpeaking != isSpeaking) { + this.isSpeaking = isSpeaking; this.audioLevel = audioLevel; this.peerConnection = peerConnection; if (listener != null) { - listener.onValueChanged(peerConnection, isSpeak, audioLevel); + listener.onValueChanged(peerConnection, isSpeaking, audioLevel); } } } @@ -97,11 +97,11 @@ public WebRTCModule(ReactApplicationContext reactContext) { incomingAudioLevelHolder.setOnValueChangeListener(new OnValueChangeListener() { @Override - public void onValueChanged(PeerConnectionObserver peerConnection, boolean isSpeak, double audioLevel) { + public void onValueChanged(PeerConnectionObserver peerConnection, boolean isSpeaking, double audioLevel) { ThreadUtils.runOnExecutor(() -> { WritableMap params = Arguments.createMap(); WritableMap childParams = Arguments.createMap(); - childParams.putBoolean("isSpeak", isSpeak); + childParams.putBoolean("isSpeaking", isSpeaking); childParams.putDouble("audioLevel", audioLevel); params.putInt("pcId", peerConnection.getId()); @@ -113,10 +113,10 @@ public void onValueChanged(PeerConnectionObserver peerConnection, boolean isSpea outgoingAudioLevelHolder.setOnValueChangeListener(new OnValueChangeListener() { @Override - public void onValueChanged(PeerConnectionObserver peerConnection, boolean isSpeak, double audioLevel) { + public void onValueChanged(PeerConnectionObserver peerConnection, boolean isSpeaking, double audioLevel) { WritableMap params = Arguments.createMap(); WritableMap childParams = Arguments.createMap(); - childParams.putBoolean("isSpeak", isSpeak); + childParams.putBoolean("isSpeaking", isSpeaking); childParams.putDouble("audioLevel", audioLevel); params.putInt("pcId", peerConnection.getId()); diff --git a/ios/RCTWebRTC/WebRTCModule+RTCPeerConnection.m b/ios/RCTWebRTC/WebRTCModule+RTCPeerConnection.m index 7f79e97fe..3aa71e034 100644 --- a/ios/RCTWebRTC/WebRTCModule+RTCPeerConnection.m +++ b/ios/RCTWebRTC/WebRTCModule+RTCPeerConnection.m @@ -117,7 +117,7 @@ - (void)checkAudioLevel { self.voiceHandler = [[WebRTCVoiceHandler new] startObserveWithPeerConnections:self.peerConnections.allValues - voiceClosure:^(RTCPeerConnection* peerConnection, BOOL outgoing, BOOL isSpeak, double audioLevel) { + voiceClosure:^(RTCPeerConnection* peerConnection, BOOL outgoing, BOOL isSpeaking, double audioLevel) { dispatch_async(self.workerQueue, ^{ [self sendEventWithName:kEventPeerVoiceStateChanged @@ -125,12 +125,12 @@ - (void)checkAudioLevel { outgoing ? @{@"outgoing": @{ - @"isSpeak" : @(isSpeak), + @"isSpeaking" : @(isSpeaking), @"audioLevel" : @(audioLevel) }, @"pcId" : peerConnection.reactTag} : @{@"incoming": @{ - @"isSpeak" : @(isSpeak), + @"isSpeaking" : @(isSpeaking), @"audioLevel" : @(audioLevel) }, @"pcId" : peerConnection.reactTag} ]; diff --git a/ios/RCTWebRTC/WebRTCVideoCaptureHandler.swift b/ios/RCTWebRTC/WebRTCVideoCaptureHandler.swift index 46e5b6eee..9149d9dc5 100644 --- a/ios/RCTWebRTC/WebRTCVideoCaptureHandler.swift +++ b/ios/RCTWebRTC/WebRTCVideoCaptureHandler.swift @@ -29,14 +29,14 @@ final public class WebRTCVoiceHandler: NSObject { await observeVoiceActivity(peerConnections: peerConnections) } - outgoingVoicePublisher.removeDuplicates(by: { $0.1 == $1.1 }).sink { (peer, isSpeak, audioLevel) in + outgoingVoicePublisher.removeDuplicates(by: { $0.1 == $1.1 }).sink { (peer, isSpeaking, audioLevel) in guard let peer else { return } - voiceClosure(peer, true, isSpeak, audioLevel) + voiceClosure(peer, true, isSpeaking, audioLevel) }.store(in: &disposeBag) - incomingVoicePublisher.removeDuplicates(by: { $0.1 == $1.1 }).sink { (peer, isSpeak, audioLevel) in + incomingVoicePublisher.removeDuplicates(by: { $0.1 == $1.1 }).sink { (peer, isSpeaking, audioLevel) in guard let peer else { return } - voiceClosure(peer, false, isSpeak, audioLevel) + voiceClosure(peer, false, isSpeaking, audioLevel) }.store(in: &disposeBag) return self diff --git a/src/RTCPeerConnection.ts b/src/RTCPeerConnection.ts index fd585052d..7081261d7 100644 --- a/src/RTCPeerConnection.ts +++ b/src/RTCPeerConnection.ts @@ -46,8 +46,8 @@ type RTCDataChannelInit = { }; type RTCVoice = { - incoming: { isSpeak: boolean, audioLevel: number }, - outgoing: { isSpeak: boolean, audioLevel: number } + incoming: { isSpeaking: boolean, audioLevel: number }, + outgoing: { isSpeaking: boolean, audioLevel: number } }; type RTCIceServer = { @@ -89,7 +89,7 @@ export default class RTCPeerConnection extends EventTarget Date: Tue, 8 Jul 2025 18:39:08 +0300 Subject: [PATCH 3/4] fix: Android call stuck --- .../com/oney/WebRTCModule/WebRTCModule.java | 30 +++++++------------ 1 file changed, 11 insertions(+), 19 deletions(-) diff --git a/android/src/main/java/com/oney/WebRTCModule/WebRTCModule.java b/android/src/main/java/com/oney/WebRTCModule/WebRTCModule.java index ff242da3b..a6002435b 100644 --- a/android/src/main/java/com/oney/WebRTCModule/WebRTCModule.java +++ b/android/src/main/java/com/oney/WebRTCModule/WebRTCModule.java @@ -78,7 +78,7 @@ public class WebRTCModule extends ReactContextBaseJavaModule { private final SparseArray mPeerConnectionObservers; final Map localStreams; - Timer voiceTimer = new Timer(); + Timer voiceTimer; boolean isVoiceTimerRunning = false; AudioLevelValueHolder incomingAudioLevelHolder; @@ -444,6 +444,7 @@ private PeerConnection.RTCConfiguration parseRTCConfiguration(ReadableMap map) { public boolean peerConnectionInit(ReadableMap configuration, int id) { PeerConnection.RTCConfiguration rtcConfiguration = parseRTCConfiguration(configuration); + observeVoiceActivity(); try { return (boolean) ThreadUtils .submitToExecutor(() -> { @@ -454,7 +455,6 @@ public boolean peerConnectionInit(ReadableMap configuration, int id) { } observer.setPeerConnection(peerConnection); mPeerConnectionObservers.put(id, observer); - observeVoiceActivity(); return true; }) @@ -465,17 +465,6 @@ public boolean peerConnectionInit(ReadableMap configuration, int id) { } } - void stopObserveVoiceActivity() { - if (isVoiceTimerRunning) { - try { - voiceTimer.cancel(); - } catch (RuntimeException e) { - Log.i("RuntimeException", e.getLocalizedMessage()); - } - - isVoiceTimerRunning = false; - } - } void observeVoiceActivity() { double checkInterval = 0.1; double silenceThreshold = 0.3; @@ -483,12 +472,7 @@ void observeVoiceActivity() { final double[] silenceIncomingCount = {0}; final double[] silenceOutgoingCount = {0}; - stopObserveVoiceActivity(); - - if (mPeerConnectionObservers.size() == 0) { - return; - } - + voiceTimer = new Timer(); voiceTimer.schedule(new TimerTask() { @Override public void run() { @@ -1529,6 +1513,10 @@ public WritableMap createDataChannel(int peerConnectionId, String label, Readabl @ReactMethod public void dataChannelClose(int peerConnectionId, String reactTag) { + if (isVoiceTimerRunning) { + voiceTimer.cancel(); + } + ThreadUtils.runOnExecutor(() -> { // Forward to PeerConnectionObserver which deals with DataChannels // because DataChannel is owned by PeerConnection. @@ -1544,6 +1532,10 @@ public void dataChannelClose(int peerConnectionId, String reactTag) { @ReactMethod public void dataChannelDispose(int peerConnectionId, String reactTag) { + if (isVoiceTimerRunning) { + voiceTimer.cancel(); + } + ThreadUtils.runOnExecutor(() -> { PeerConnectionObserver pco = mPeerConnectionObservers.get(peerConnectionId); if (pco == null || pco.getPeerConnection() == null) { From b4c7864743521857456e1a1c9c9164073bf87b19 Mon Sep 17 00:00:00 2001 From: Denis Koltovich Date: Wed, 9 Jul 2025 14:11:58 +0300 Subject: [PATCH 4/4] fix: Threashold --- .../main/java/com/oney/WebRTCModule/WebRTCModule.java | 8 +++++--- ios/RCTWebRTC/WebRTCVideoCaptureHandler.swift | 9 +++++---- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/android/src/main/java/com/oney/WebRTCModule/WebRTCModule.java b/android/src/main/java/com/oney/WebRTCModule/WebRTCModule.java index a6002435b..ead2cc394 100644 --- a/android/src/main/java/com/oney/WebRTCModule/WebRTCModule.java +++ b/android/src/main/java/com/oney/WebRTCModule/WebRTCModule.java @@ -466,7 +466,7 @@ public boolean peerConnectionInit(ReadableMap configuration, int id) { } void observeVoiceActivity() { - double checkInterval = 0.1; + double checkInterval = 0.3; double silenceThreshold = 0.3; final double[] silenceIncomingCount = {0}; @@ -493,11 +493,12 @@ public void onStatsDelivered(RTCStatsReport rtcStatsReport) { Double audioLevel = ((Double) audioLevelObject); if (audioLevel > 0.1) { + silenceIncomingCount[0] = 0; incomingAudioLevelHolder.setValue(peer, true, audioLevel.doubleValue()); } else { silenceIncomingCount[0] += 1; - if (silenceIncomingCount[0] > silenceThreshold / checkInterval) { + if (silenceIncomingCount[0] > 5.0) { incomingAudioLevelHolder.setValue(peer, false, 0); } } @@ -511,11 +512,12 @@ public void onStatsDelivered(RTCStatsReport rtcStatsReport) { Double audioLevel = ((Double) audioLevelObject); if (audioLevel > 0.1) { + silenceOutgoingCount[0] = 0; outgoingAudioLevelHolder.setValue(peer, true, audioLevel.doubleValue()); } else { silenceOutgoingCount[0] += 1; - if (silenceOutgoingCount[0] > silenceThreshold / checkInterval) { + if (silenceOutgoingCount[0] > 5.0) { outgoingAudioLevelHolder.setValue(peer, false, 0); } } diff --git a/ios/RCTWebRTC/WebRTCVideoCaptureHandler.swift b/ios/RCTWebRTC/WebRTCVideoCaptureHandler.swift index 9149d9dc5..c810d4f1c 100644 --- a/ios/RCTWebRTC/WebRTCVideoCaptureHandler.swift +++ b/ios/RCTWebRTC/WebRTCVideoCaptureHandler.swift @@ -51,8 +51,7 @@ final public class WebRTCVoiceHandler: NSObject { } private func observeVoiceActivity(peerConnections: [RTCPeerConnection]) async { - let checkInterval: Double = 0.1 - let silenceThreshold: Double = 0.3 + let checkInterval: Double = 0.3 var silenceIncomingCount: Double = 0 var silenceOutgoingCount: Double = 0 @@ -69,11 +68,12 @@ final public class WebRTCVoiceHandler: NSObject { guard let audioLevel = statistic.values["audioLevel"] as? Double else { return } if audioLevel > 0.1 { + silenceIncomingCount = 0 self.incomingVoicePublisher.send((peerConnection, true, audioLevel)) } else { silenceIncomingCount += 1 - if silenceIncomingCount > silenceThreshold / checkInterval { + if silenceIncomingCount > 5 { self.incomingVoicePublisher.send((peerConnection, false, audioLevel)) } } @@ -83,11 +83,12 @@ final public class WebRTCVoiceHandler: NSObject { guard let audioLevel = statistic.values["audioLevel"] as? Double else { return } if audioLevel > 0.1 { + silenceOutgoingCount = 0 self.outgoingVoicePublisher.send((peerConnection, true, audioLevel)) } else { silenceOutgoingCount += 1 - if silenceOutgoingCount > silenceThreshold / checkInterval { + if silenceOutgoingCount > 5 { self.outgoingVoicePublisher.send((peerConnection, false, audioLevel)) } }