M-1920 Investigate & create the task: Green Frame to indicate when someone talks

android/src/main/java/com/oney/WebRTCModule/PeerConnectionObserver.java

@@ -58,6 +58,10 @@ PeerConnection getPeerConnection() {
         return peerConnection;
     }
 
+    int getId() {
+        return id;
+    }
+
     void setPeerConnection(PeerConnection peerConnection) {
         this.peerConnection = peerConnection;
     }

android/src/main/java/com/oney/WebRTCModule/WebRTCModule.java

@@ -33,9 +33,38 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Timer;
+import java.util.TimerTask;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutionException;
 
+interface OnValueChangeListener {
+    void onValueChanged(PeerConnectionObserver peerConnection, boolean isSpeaking, double audioLevel);
+}
+
+class AudioLevelValueHolder {
+    private double audioLevel;
+    private boolean isSpeaking;
+    private PeerConnectionObserver peerConnection;
+    private OnValueChangeListener listener;
+
+    void setOnValueChangeListener(OnValueChangeListener listener) {
+        this.listener = listener;
+    }
+
+    void setValue(PeerConnectionObserver peerConnection, boolean isSpeaking, double audioLevel) {
+        if (this.isSpeaking != isSpeaking) {
+            this.isSpeaking = isSpeaking;
+            this.audioLevel = audioLevel;
+            this.peerConnection = peerConnection;
+
+            if (listener != null) {
+                listener.onValueChanged(peerConnection, isSpeaking, audioLevel);
+            }
+        }
+    }
+}
+
 @ReactModule(name = "WebRTCModule")
 public class WebRTCModule extends ReactContextBaseJavaModule {
     static final String TAG = WebRTCModule.class.getCanonicalName();
@@ -49,6 +78,12 @@ public class WebRTCModule extends ReactContextBaseJavaModule {
     private final SparseArray<PeerConnectionObserver> mPeerConnectionObservers;
     final Map<String, MediaStream> localStreams;
 
+    Timer voiceTimer;
+    boolean isVoiceTimerRunning = false;
+
+    AudioLevelValueHolder incomingAudioLevelHolder;
+    AudioLevelValueHolder outgoingAudioLevelHolder;
+
     private final GetUserMediaImpl getUserMediaImpl;
 
     public WebRTCModule(ReactApplicationContext reactContext) {
@@ -57,6 +92,39 @@ public WebRTCModule(ReactApplicationContext reactContext) {
         mPeerConnectionObservers = new SparseArray<>();
         localStreams = new HashMap<>();
 
+        incomingAudioLevelHolder = new AudioLevelValueHolder();
+        outgoingAudioLevelHolder = new AudioLevelValueHolder();
+
+        incomingAudioLevelHolder.setOnValueChangeListener(new OnValueChangeListener() {
+            @Override
+            public void onValueChanged(PeerConnectionObserver peerConnection, boolean isSpeaking, double audioLevel) {
+                ThreadUtils.runOnExecutor(() -> {
+                    WritableMap params = Arguments.createMap();
+                    WritableMap childParams = Arguments.createMap();
+                    childParams.putBoolean("isSpeaking", isSpeaking);
+                    childParams.putDouble("audioLevel", audioLevel);
+
+                    params.putInt("pcId", peerConnection.getId());
+                    params.putMap("incoming", childParams);
+                    sendEvent("peerVoiceStateChanged", params);
+                });
+            }
+        });
+
+        outgoingAudioLevelHolder.setOnValueChangeListener(new OnValueChangeListener() {
+            @Override
+            public void onValueChanged(PeerConnectionObserver peerConnection, boolean isSpeaking, double audioLevel) {
+                WritableMap params = Arguments.createMap();
+                WritableMap childParams = Arguments.createMap();
+                childParams.putBoolean("isSpeaking", isSpeaking);
+                childParams.putDouble("audioLevel", audioLevel);
+
+                params.putInt("pcId", peerConnection.getId());
+                params.putMap("outgoing", childParams);
+                sendEvent("peerVoiceStateChanged", params);
+            }
+        });
+
         WebRTCModuleOptions options = WebRTCModuleOptions.getInstance();
 
         AudioDeviceModule adm = options.audioDeviceModule;
@@ -376,6 +444,7 @@ private PeerConnection.RTCConfiguration parseRTCConfiguration(ReadableMap map) {
     public boolean peerConnectionInit(ReadableMap configuration, int id) {
         PeerConnection.RTCConfiguration rtcConfiguration = parseRTCConfiguration(configuration);
 
+        observeVoiceActivity();
         try {
             return (boolean) ThreadUtils
                     .submitToExecutor(() -> {
@@ -386,6 +455,7 @@ public boolean peerConnectionInit(ReadableMap configuration, int id) {
                         }
                         observer.setPeerConnection(peerConnection);
                         mPeerConnectionObservers.put(id, observer);
+
                         return true;
                     })
                     .get();
@@ -395,6 +465,78 @@ public boolean peerConnectionInit(ReadableMap configuration, int id) {
         }
     }
 
+    void observeVoiceActivity() {
+        double checkInterval = 0.3;
+        double silenceThreshold = 0.3;
+
+        final double[] silenceIncomingCount = {0};
+        final double[] silenceOutgoingCount = {0};
+
+        voiceTimer = new Timer();
+        voiceTimer.schedule(new TimerTask() {
+            @Override
+            public void run() {
+                for (int i = 0; i < mPeerConnectionObservers.size(); i++) {
+                    int key = mPeerConnectionObservers.keyAt(i);
+                    PeerConnectionObserver peer = mPeerConnectionObservers.get(key);
+
+                    if (peer.getPeerConnection().connectionState() == PeerConnection.PeerConnectionState.CONNECTED) {
+                        peer.getPeerConnection().getStats(new RTCStatsCollectorCallback() {
+                            @Override
+                            public void onStatsDelivered(RTCStatsReport rtcStatsReport) {
+
+                                for (RTCStats stats : rtcStatsReport.getStatsMap().values()) {
+                                    if (stats.getType().equals("inbound-rtp")) {
+                                        Object audioLevelObject = stats.getMembers().get("audioLevel");
+
+                                        if (audioLevelObject instanceof Double) {
+                                            Double audioLevel = ((Double) audioLevelObject);
+
+                                            if (audioLevel > 0.1) {
+                                                silenceIncomingCount[0] = 0;
+                                                incomingAudioLevelHolder.setValue(peer, true, audioLevel.doubleValue());
+                                            } else {
+                                                silenceIncomingCount[0] += 1;
+
+                                                if (silenceIncomingCount[0] > 5.0) {
+                                                    incomingAudioLevelHolder.setValue(peer, false, 0);
+                                                }
+                                            }
+                                        }
+                                    }
+
+                                    if (stats.getType().equals("media-source")) {
+                                        Object audioLevelObject = stats.getMembers().get("audioLevel");
+
+                                        if (audioLevelObject instanceof Double) {
+                                            Double audioLevel = ((Double) audioLevelObject);
+
+                                            if (audioLevel > 0.1) {
+                                                silenceOutgoingCount[0] = 0;
+                                                outgoingAudioLevelHolder.setValue(peer, true, audioLevel.doubleValue());
+                                            } else {
+                                                silenceOutgoingCount[0] += 1;
+
+                                                if (silenceOutgoingCount[0] > 5.0) {
+                                                    outgoingAudioLevelHolder.setValue(peer, false, 0);
+                                                }
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                        });
+                    } else {
+                        incomingAudioLevelHolder.setValue(peer, false, 0);
+                        outgoingAudioLevelHolder.setValue(peer, false, 0);
+                    }
+                }
+            }
+        }, 0, 100);
+
+        isVoiceTimerRunning = true;
+    }
+
     MediaStream getStreamForReactTag(String streamReactTag) {
         // This function _only_ gets called from WebRTCView, in the UI thread.
         // Hence make sure we run this code in the executor or we run at the risk
@@ -1373,6 +1515,10 @@ public WritableMap createDataChannel(int peerConnectionId, String label, Readabl
 
     @ReactMethod
     public void dataChannelClose(int peerConnectionId, String reactTag) {
+        if (isVoiceTimerRunning) {
+            voiceTimer.cancel();
+        }
+
         ThreadUtils.runOnExecutor(() -> {
             // Forward to PeerConnectionObserver which deals with DataChannels
             // because DataChannel is owned by PeerConnection.
@@ -1388,6 +1534,10 @@ public void dataChannelClose(int peerConnectionId, String reactTag) {
 
     @ReactMethod
     public void dataChannelDispose(int peerConnectionId, String reactTag) {
+        if (isVoiceTimerRunning) {
+            voiceTimer.cancel();
+        }
+
         ThreadUtils.runOnExecutor(() -> {
             PeerConnectionObserver pco = mPeerConnectionObservers.get(peerConnectionId);
             if (pco == null || pco.getPeerConnection() == null) {

ios/RCTWebRTC/WebRTCModule+RTCPeerConnection.m

@@ -22,8 +22,10 @@
 #import "WebRTCModule.h"
 #import "WebRTCAudioSession.h"
 
-@implementation RTCPeerConnection (React)
+#import "react_native_webrtc-Swift.h"
+
 
+@implementation RTCPeerConnection (React)
 - (NSMutableDictionary<NSString *, DataChannelWrapper *> *)dataChannels {
     return objc_getAssociatedObject(self, _cmd);
 }
@@ -98,11 +100,44 @@ @implementation WebRTCModule (RTCPeerConnection)
         peerConnection.webRTCModule = self;
 
         self.peerConnections[objectID] = peerConnection;
+
+        [self checkAudioLevel];
     });
 
     return @(ret);
 }
 
+- (void)checkAudioLevel {
+  // Cancel prev observer
+  [(WebRTCVoiceHandler*)self.voiceHandler stopObserve];
+
+  if (self.peerConnections.count == 0) {
+    return;
+  }
+
+  self.voiceHandler = [[WebRTCVoiceHandler new]
+                       startObserveWithPeerConnections:self.peerConnections.allValues
+                       voiceClosure:^(RTCPeerConnection* peerConnection, BOOL outgoing, BOOL isSpeaking, double audioLevel) {
+
+    dispatch_async(self.workerQueue, ^{
+      [self sendEventWithName:kEventPeerVoiceStateChanged
+                         body:
+       outgoing
+       ?
+       @{@"outgoing": @{
+         @"isSpeaking" : @(isSpeaking),
+         @"audioLevel" : @(audioLevel)
+       }, @"pcId" : peerConnection.reactTag}
+       :
+       @{@"incoming": @{
+         @"isSpeaking" : @(isSpeaking),
+         @"audioLevel" : @(audioLevel)
+       }, @"pcId" : peerConnection.reactTag}
+      ];
+    });
+  }];
+}
+
 RCT_EXPORT_METHOD(peerConnectionSetConfiguration
                   : (RTCConfiguration *)configuration objectID
                   : (nonnull NSNumber *)objectID) {
@@ -857,9 +892,9 @@ - (void)peerConnection:(RTC_OBJC_TYPE(RTCPeerConnection) *)peerConnection
                streams:(NSArray<RTC_OBJC_TYPE(RTCMediaStream) *> *)mediaStreams {
     dispatch_async(self.workerQueue, ^{
         RCTLogWarn(@"PeerConnection %@ didAddReceiver %@", peerConnection.reactTag, rtpReceiver.receiverId);
-        
+
         RTCRtpTransceiver *transceiver = nil;
-        
+
         for (RTCRtpTransceiver *t in peerConnection.transceivers) {
             if ([rtpReceiver.receiverId isEqual:t.receiver.receiverId]) {
                 transceiver = t;
@@ -878,13 +913,13 @@ - (void)peerConnection:(RTC_OBJC_TYPE(RTCPeerConnection) *)peerConnection
             RTCVideoTrack *videoTrack = (RTCVideoTrack *)track;
             [peerConnection addVideoTrackAdapter:videoTrack];
         }
-        
+
         peerConnection.remoteTracks[track.trackId] = track;
-        
+
         NSMutableArray *streams = [NSMutableArray new];
         NSMutableDictionary *params = [NSMutableDictionary new];
-        
-        
+
+
 //        NSLog(@"TEST: Remote tracks: ");
 //        for (NSString * key in [peerConnection.remoteTracks allKeys]) {
 //            NSLog(@"TEST: Remote trackId: %@", key);
@@ -896,7 +931,7 @@ - (void)peerConnection:(RTC_OBJC_TYPE(RTCPeerConnection) *)peerConnection
 //            RTCMediaStream* remoteStream = [peerConnection.remoteStreams objectForKey:key];
 //            NSLog(@"TEST: stream: %@", remoteStream);
 //        }
-        
+
         for (RTCMediaStream * stream in mediaStreams) {
           //  NSLog(@"TEST: Coming stream: %@", stream);
             NSString *streamReactTag = nil;
@@ -957,4 +992,6 @@ - (void)peerConnection:(nonnull RTCPeerConnection *)peerConnection didRemoveStre
     RCTLogWarn(@"PeerConnection %@ didRemoveStream %@", peerConnection.reactTag, stream.streamId);
 }
 
+
+
 @end

ios/RCTWebRTC/WebRTCModule.h

@@ -8,6 +8,7 @@
 #import <WebRTC/WebRTC.h>
 #import "VideoCaptureController.h"
 
+static NSString *const kEventPeerVoiceStateChanged = @"peerVoiceStateChanged";
 static NSString *const kEventPeerConnectionSignalingStateChanged = @"peerConnectionSignalingStateChanged";
 static NSString *const kEventPeerConnectionStateChanged = @"peerConnectionStateChanged";
 static NSString *const kEventPeerConnectionOnRenegotiationNeeded = @"peerConnectionOnRenegotiationNeeded";
@@ -26,6 +27,7 @@ static NSString *const kEventPeerConnectionOnTrack = @"peerConnectionOnTrack";
 @interface WebRTCModule : RCTEventEmitter<RCTBridgeModule>
 
 @property(nonatomic, strong) dispatch_queue_t workerQueue;
+@property(nonatomic, strong) dispatch_source_t timer;
 
 @property(nonatomic, strong) RTCPeerConnectionFactory *peerConnectionFactory;
 @property(nonatomic, strong) id<RTCVideoDecoderFactory> decoderFactory;
@@ -35,6 +37,7 @@ static NSString *const kEventPeerConnectionOnTrack = @"peerConnectionOnTrack";
 @property(nonatomic, strong) NSMutableDictionary<NSString *, RTCMediaStream *> *localStreams;
 @property(nonatomic, strong) NSMutableDictionary<NSString *, RTCMediaStreamTrack *> *localTracks;
 @property (nonatomic, strong) id videoSourceInterceptor;
+@property (nonatomic, strong) id voiceHandler;
 @property (nonatomic, strong) VideoCaptureController *videoCaptureController;
 
 - (RTCMediaStream *)streamForReactTag:(NSString *)reactTag;

ios/RCTWebRTC/WebRTCModule.m

@@ -107,6 +107,7 @@ - (dispatch_queue_t)methodQueue {
 
 - (NSArray<NSString *> *)supportedEvents {
     return @[
+        kEventPeerVoiceStateChanged,
         kEventPeerConnectionSignalingStateChanged,
         kEventPeerConnectionStateChanged,
         kEventPeerConnectionOnRenegotiationNeeded,