DEV vs MASTER

android/build.gradle

@@ -32,4 +32,9 @@ dependencies {
     implementation 'com.facebook.react:react-native:+'
     api 'org.jitsi:webrtc:124.+'
     implementation "androidx.core:core:1.7.0"
+
+    implementation "com.google.android.gms:play-services-tasks:18.2.0"
+    implementation "com.google.android.gms:play-services-mlkit-face-detection:17.1.0"
+    implementation "com.google.mlkit:segmentation-selfie:16.0.0-beta6"
+    implementation "com.google.mlkit:camera:16.0.0-beta3"
 }

android/local.properties

@@ -0,0 +1,8 @@
+## This file must *NOT* be checked into Version Control Systems,
+# as it contains information specific to your local configuration.
+#
+# Location of the SDK. This is only used by Gradle.
+# For customization when using a Version Control System, please read the
+# header note.
+#Tue Jan 03 11:44:06 MSK 2023
+sdk.dir=/Users/vasil/Library/Android/sdk

android/src/main/AndroidManifest.xml

@@ -2,6 +2,9 @@
           package="com.oney.WebRTCModule"
           xmlns:tools="http://schemas.android.com/tools"
     >
+
+    <uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" />
+
     <application>
         <service
                 android:name=".MediaProjectionService"

android/src/main/java/com/oney/WebRTCModule/DataChannelWrapper.java

@@ -59,31 +59,33 @@ public void onBufferedAmountChange(long amount) {
 
     @Override
     public void onMessage(DataChannel.Buffer buffer) {
-        WritableMap params = Arguments.createMap();
-        params.putString("reactTag", reactTag);
-        params.putInt("peerConnectionId", peerConnectionId);
-
-        byte[] bytes;
-        if (buffer.data.hasArray()) {
-            bytes = buffer.data.array();
-        } else {
-            bytes = new byte[buffer.data.remaining()];
-            buffer.data.get(bytes);
+        final byte[] copiedBytes;
+        try {
+            if (buffer.data.hasArray()) {
+                int offset = buffer.data.arrayOffset() + buffer.data.position();
+                int length = buffer.data.remaining();
+                copiedBytes = new byte[length];
+                System.arraycopy(buffer.data.array(), offset, copiedBytes, 0, length);
+            } else {
+                copiedBytes = new byte[buffer.data.remaining()];
+                buffer.data.get(copiedBytes);
+            }
+        } finally {
+            buffer.data.rewind();
         }
 
-        String type;
-        String data;
-        if (buffer.binary) {
-            type = "binary";
-            data = Base64.encodeToString(bytes, Base64.NO_WRAP);
-        } else {
-            type = "text";
-            data = new String(bytes, StandardCharsets.UTF_8);
-        }
-        params.putString("type", type);
-        params.putString("data", data);
+        ThreadUtils.runOnExecutor(() -> {
+            WritableMap params = Arguments.createMap();
+            params.putString("reactTag", reactTag);
+            params.putInt("peerConnectionId", peerConnectionId);
+
+            String type = "text";
+            String data = new String(copiedBytes, StandardCharsets.UTF_8);
+            params.putString("type", type);
+            params.putString("data", data);
 
-        webRTCModule.sendEvent("dataChannelReceiveMessage", params);
+            webRTCModule.sendEvent("dataChannelReceiveMessage", params);
+        });
     }
 
     @Override

android/src/main/java/com/oney/WebRTCModule/GetUserMediaImpl.java

@@ -4,6 +4,8 @@
 import android.content.Context;
 import android.content.Intent;
 import android.media.projection.MediaProjectionManager;
+import android.os.Handler;
+import android.os.Looper;
 import android.util.DisplayMetrics;
 import android.util.Log;
 import androidx.core.util.Consumer;
@@ -195,7 +197,7 @@ void getUserMedia(final ReadableMap constraints, final Callback successCallback,
             CameraCaptureController cameraCaptureController =
                     new CameraCaptureController(reactContext.getCurrentActivity(), getCameraEnumerator(), videoConstraintsMap);
 
-            videoTrack = createVideoTrack(cameraCaptureController);
+            videoTrack = createVideoTrack(cameraCaptureController, videoConstraintsMap.hasKey("enableVirtualBackgroud"), videoConstraintsMap.hasKey("enableBlurBackgroud"), videoConstraintsMap.getString("backgroundImageBase64"));
         }
 
         if (audioTrack == null && videoTrack == null) {
@@ -362,12 +364,12 @@ private VideoTrack createScreenTrack() {
         DisplayMetrics displayMetrics = DisplayUtils.getDisplayMetrics(reactContext.getCurrentActivity());
         int width = displayMetrics.widthPixels;
         int height = displayMetrics.heightPixels;
-        ScreenCaptureController screenCaptureController = new ScreenCaptureController(
-                reactContext.getCurrentActivity(), width, height, mediaProjectionPermissionResultData);
-        return createVideoTrack(screenCaptureController);
+        ScreenCaptureController screenCaptureController
+            = new ScreenCaptureController(reactContext.getCurrentActivity(), width, height, mediaProjectionPermissionResultData);
+        return createVideoTrack(screenCaptureController, null, false, "");
     }
 
-    VideoTrack createVideoTrack(AbstractVideoCaptureController videoCaptureController) {
+    VideoTrack createVideoTrack(AbstractVideoCaptureController videoCaptureController, Boolean enableVirtualBackgroud, Boolean enableBlurBackgroud, String backgroundImageBase64) {
         videoCaptureController.initializeVideoCapturer();
 
         VideoCapturer videoCapturer = videoCaptureController.videoCapturer;
@@ -392,12 +394,19 @@ VideoTrack createVideoTrack(AbstractVideoCaptureController videoCaptureControlle
         VideoSource videoSource = pcFactory.createVideoSource(videoCapturer.isScreencast());
         videoCapturer.initialize(surfaceTextureHelper, reactContext, videoSource.getCapturerObserver());
 
-        VideoTrack track = pcFactory.createVideoTrack(id, videoSource);
+        if (enableVirtualBackgroud || enableBlurBackgroud) {
+            VideoProcessor p = new VirtualBackgroundVideoProcessor(reactContext, surfaceTextureHelper, enableVirtualBackgroud, enableBlurBackgroud, backgroundImageBase64);
+            videoSource.setVideoProcessor(p);
+        }
 
-        track.setEnabled(true);
-        tracks.put(id, new TrackPrivate(track, videoSource, videoCaptureController, surfaceTextureHelper));
+        String videoTrackId = UUID.randomUUID().toString();
+        VideoTrack track = pcFactory.createVideoTrack(videoTrackId, videoSource);
 
-        videoCaptureController.startCapture();
+        track.setEnabled(true);
+        tracks.put(videoTrackId, new TrackPrivate(track, videoSource, videoCaptureController, surfaceTextureHelper));
+
+        // Delay fix issue with app freezing on fold devices
+        new Handler(Looper.getMainLooper()).postDelayed(videoCaptureController::startCapture, 250);
 
         return track;
     }

android/src/main/java/com/oney/WebRTCModule/PeerConnectionObserver.java

@@ -58,19 +58,35 @@ PeerConnection getPeerConnection() {
         return peerConnection;
     }
 
+    int getId() {
+        return id;
+    }
+
     void setPeerConnection(PeerConnection peerConnection) {
         this.peerConnection = peerConnection;
     }
 
     void close() {
         Log.d(TAG, "PeerConnection.close() for " + id);
 
-        peerConnection.close();
+        PeerConnection pc = peerConnection;
+        if (pc == null) {
+            return;
+        }
+
+        pc.close();
     }
 
     void dispose() {
         Log.d(TAG, "PeerConnection.dispose() for " + id);
 
+        PeerConnection pc = peerConnection;
+        if (pc == null) {
+            return;
+        }
+        // Make future calls observe disposal immediately.
+        peerConnection = null;
+
         // Remove video track adapters
         for (MediaStreamTrack track : this.remoteTracks.values()) {
             if (track instanceof VideoTrack) {
@@ -87,7 +103,7 @@ void dispose() {
         // At this point there should be no local MediaStreams in the associated
         // PeerConnection. Call dispose() to free all remaining resources held
         // by the PeerConnection instance (RtpReceivers, RtpSenders, etc.)
-        peerConnection.dispose();
+        pc.dispose();
 
         remoteStreamIds.clear();
         remoteStreams.clear();
@@ -470,7 +486,9 @@ public void onAddTrack(final RtpReceiver receiver, final MediaStream[] mediaStre
      * semantics are specified. The transceiver will be disposed automatically.
      */
     @Override
-    public void onTrack(final RtpTransceiver transceiver) {}
+    public void onTrack(final RtpTransceiver transceiver) {
+        Log.w(TAG, "PeerConnection didStartReceivingOnTransceiver" + transceiver.getReceiver().id());
+    }
 
     /*
      * Triggered when a previously added remote track is removed by the remote