diff --git a/android/.gradle/7.4/checksums/checksums.lock b/android/.gradle/7.4/checksums/checksums.lock new file mode 100644 index 000000000..1a5adb6d1 Binary files /dev/null and b/android/.gradle/7.4/checksums/checksums.lock differ diff --git a/android/.gradle/7.4/dependencies-accessors/dependencies-accessors.lock b/android/.gradle/7.4/dependencies-accessors/dependencies-accessors.lock new file mode 100644 index 000000000..e77ba0b82 Binary files /dev/null and b/android/.gradle/7.4/dependencies-accessors/dependencies-accessors.lock differ diff --git a/android/.gradle/7.4/dependencies-accessors/gc.properties b/android/.gradle/7.4/dependencies-accessors/gc.properties new file mode 100644 index 000000000..e69de29bb diff --git a/android/.gradle/7.4/fileChanges/last-build.bin b/android/.gradle/7.4/fileChanges/last-build.bin new file mode 100644 index 000000000..f76dd238a Binary files /dev/null and b/android/.gradle/7.4/fileChanges/last-build.bin differ diff --git a/android/.gradle/7.4/fileHashes/fileHashes.lock b/android/.gradle/7.4/fileHashes/fileHashes.lock new file mode 100644 index 000000000..f6c5385c1 Binary files /dev/null and b/android/.gradle/7.4/fileHashes/fileHashes.lock differ diff --git a/android/.gradle/7.4/gc.properties b/android/.gradle/7.4/gc.properties new file mode 100644 index 000000000..e69de29bb diff --git a/android/.gradle/vcs-1/gc.properties b/android/.gradle/vcs-1/gc.properties new file mode 100644 index 000000000..e69de29bb diff --git a/android/build.gradle b/android/build.gradle index 050469457..2a5944269 100644 --- a/android/build.gradle +++ b/android/build.gradle @@ -32,4 +32,9 @@ dependencies { implementation 'com.facebook.react:react-native:+' api 'org.jitsi:webrtc:124.+' implementation "androidx.core:core:1.7.0" + + implementation "com.google.android.gms:play-services-tasks:18.2.0" + implementation "com.google.android.gms:play-services-mlkit-face-detection:17.1.0" + implementation "com.google.mlkit:segmentation-selfie:16.0.0-beta6" + implementation "com.google.mlkit:camera:16.0.0-beta3" } diff --git a/android/local.properties b/android/local.properties new file mode 100644 index 000000000..5ca093dcb --- /dev/null +++ b/android/local.properties @@ -0,0 +1,8 @@ +## This file must *NOT* be checked into Version Control Systems, +# as it contains information specific to your local configuration. +# +# Location of the SDK. This is only used by Gradle. +# For customization when using a Version Control System, please read the +# header note. +#Tue Jan 03 11:44:06 MSK 2023 +sdk.dir=/Users/vasil/Library/Android/sdk diff --git a/android/src/main/AndroidManifest.xml b/android/src/main/AndroidManifest.xml index 321315114..760554341 100644 --- a/android/src/main/AndroidManifest.xml +++ b/android/src/main/AndroidManifest.xml @@ -2,6 +2,9 @@ package="com.oney.WebRTCModule" xmlns:tools="http://schemas.android.com/tools" > + + + { + WritableMap params = Arguments.createMap(); + params.putString("reactTag", reactTag); + params.putInt("peerConnectionId", peerConnectionId); + + String type = "text"; + String data = new String(copiedBytes, StandardCharsets.UTF_8); + params.putString("type", type); + params.putString("data", data); - webRTCModule.sendEvent("dataChannelReceiveMessage", params); + webRTCModule.sendEvent("dataChannelReceiveMessage", params); + }); } @Override diff --git a/android/src/main/java/com/oney/WebRTCModule/GetUserMediaImpl.java b/android/src/main/java/com/oney/WebRTCModule/GetUserMediaImpl.java index 3e9234b9d..66a448a16 100644 --- a/android/src/main/java/com/oney/WebRTCModule/GetUserMediaImpl.java +++ b/android/src/main/java/com/oney/WebRTCModule/GetUserMediaImpl.java @@ -4,6 +4,8 @@ import android.content.Context; import android.content.Intent; import android.media.projection.MediaProjectionManager; +import android.os.Handler; +import android.os.Looper; import android.util.DisplayMetrics; import android.util.Log; import androidx.core.util.Consumer; @@ -195,7 +197,7 @@ void getUserMedia(final ReadableMap constraints, final Callback successCallback, CameraCaptureController cameraCaptureController = new CameraCaptureController(reactContext.getCurrentActivity(), getCameraEnumerator(), videoConstraintsMap); - videoTrack = createVideoTrack(cameraCaptureController); + videoTrack = createVideoTrack(cameraCaptureController, videoConstraintsMap.hasKey("enableVirtualBackgroud"), videoConstraintsMap.hasKey("enableBlurBackgroud"), videoConstraintsMap.getString("backgroundImageBase64")); } if (audioTrack == null && videoTrack == null) { @@ -362,12 +364,12 @@ private VideoTrack createScreenTrack() { DisplayMetrics displayMetrics = DisplayUtils.getDisplayMetrics(reactContext.getCurrentActivity()); int width = displayMetrics.widthPixels; int height = displayMetrics.heightPixels; - ScreenCaptureController screenCaptureController = new ScreenCaptureController( - reactContext.getCurrentActivity(), width, height, mediaProjectionPermissionResultData); - return createVideoTrack(screenCaptureController); + ScreenCaptureController screenCaptureController + = new ScreenCaptureController(reactContext.getCurrentActivity(), width, height, mediaProjectionPermissionResultData); + return createVideoTrack(screenCaptureController, null, false, ""); } - VideoTrack createVideoTrack(AbstractVideoCaptureController videoCaptureController) { + VideoTrack createVideoTrack(AbstractVideoCaptureController videoCaptureController, Boolean enableVirtualBackgroud, Boolean enableBlurBackgroud, String backgroundImageBase64) { videoCaptureController.initializeVideoCapturer(); VideoCapturer videoCapturer = videoCaptureController.videoCapturer; @@ -392,12 +394,19 @@ VideoTrack createVideoTrack(AbstractVideoCaptureController videoCaptureControlle VideoSource videoSource = pcFactory.createVideoSource(videoCapturer.isScreencast()); videoCapturer.initialize(surfaceTextureHelper, reactContext, videoSource.getCapturerObserver()); - VideoTrack track = pcFactory.createVideoTrack(id, videoSource); + if (enableVirtualBackgroud || enableBlurBackgroud) { + VideoProcessor p = new VirtualBackgroundVideoProcessor(reactContext, surfaceTextureHelper, enableVirtualBackgroud, enableBlurBackgroud, backgroundImageBase64); + videoSource.setVideoProcessor(p); + } - track.setEnabled(true); - tracks.put(id, new TrackPrivate(track, videoSource, videoCaptureController, surfaceTextureHelper)); + String videoTrackId = UUID.randomUUID().toString(); + VideoTrack track = pcFactory.createVideoTrack(videoTrackId, videoSource); - videoCaptureController.startCapture(); + track.setEnabled(true); + tracks.put(videoTrackId, new TrackPrivate(track, videoSource, videoCaptureController, surfaceTextureHelper)); + + // Delay fix issue with app freezing on fold devices + new Handler(Looper.getMainLooper()).postDelayed(videoCaptureController::startCapture, 250); return track; } diff --git a/android/src/main/java/com/oney/WebRTCModule/PeerConnectionObserver.java b/android/src/main/java/com/oney/WebRTCModule/PeerConnectionObserver.java index dfc2901a8..0f41ab1e0 100644 --- a/android/src/main/java/com/oney/WebRTCModule/PeerConnectionObserver.java +++ b/android/src/main/java/com/oney/WebRTCModule/PeerConnectionObserver.java @@ -58,6 +58,10 @@ PeerConnection getPeerConnection() { return peerConnection; } + int getId() { + return id; + } + void setPeerConnection(PeerConnection peerConnection) { this.peerConnection = peerConnection; } @@ -65,12 +69,24 @@ void setPeerConnection(PeerConnection peerConnection) { void close() { Log.d(TAG, "PeerConnection.close() for " + id); - peerConnection.close(); + PeerConnection pc = peerConnection; + if (pc == null) { + return; + } + + pc.close(); } void dispose() { Log.d(TAG, "PeerConnection.dispose() for " + id); + PeerConnection pc = peerConnection; + if (pc == null) { + return; + } + // Make future calls observe disposal immediately. + peerConnection = null; + // Remove video track adapters for (MediaStreamTrack track : this.remoteTracks.values()) { if (track instanceof VideoTrack) { @@ -87,7 +103,7 @@ void dispose() { // At this point there should be no local MediaStreams in the associated // PeerConnection. Call dispose() to free all remaining resources held // by the PeerConnection instance (RtpReceivers, RtpSenders, etc.) - peerConnection.dispose(); + pc.dispose(); remoteStreamIds.clear(); remoteStreams.clear(); @@ -470,7 +486,9 @@ public void onAddTrack(final RtpReceiver receiver, final MediaStream[] mediaStre * semantics are specified. The transceiver will be disposed automatically. */ @Override - public void onTrack(final RtpTransceiver transceiver) {} + public void onTrack(final RtpTransceiver transceiver) { + Log.w(TAG, "PeerConnection didStartReceivingOnTransceiver" + transceiver.getReceiver().id()); + } /* * Triggered when a previously added remote track is removed by the remote diff --git a/android/src/main/java/com/oney/WebRTCModule/VirtualBackgroundVideoProcessor.java b/android/src/main/java/com/oney/WebRTCModule/VirtualBackgroundVideoProcessor.java new file mode 100644 index 000000000..ca0ac66e8 --- /dev/null +++ b/android/src/main/java/com/oney/WebRTCModule/VirtualBackgroundVideoProcessor.java @@ -0,0 +1,216 @@ +package com.oney.WebRTCModule; + +import static android.graphics.Color.argb; +import static android.graphics.PorterDuff.Mode.DST_OVER; +import static android.graphics.PorterDuff.Mode.SRC_IN; + +import android.graphics.Bitmap; +import android.graphics.BitmapFactory; +import android.graphics.Canvas; +import android.graphics.Matrix; +import android.graphics.Paint; +import android.graphics.PorterDuffXfermode; +import android.opengl.GLES20; +import android.opengl.GLUtils; +import android.renderscript.Allocation; +import android.renderscript.Element; +import android.renderscript.RenderScript; +import android.renderscript.ScriptIntrinsicBlur; +import android.util.Base64; +import android.util.Log; +import android.view.Surface; + +import androidx.annotation.Nullable; + +import com.facebook.react.bridge.ReactApplicationContext; +import com.google.android.gms.tasks.OnSuccessListener; +import com.google.android.gms.tasks.Task; +import com.google.mlkit.vision.common.InputImage; +import com.google.mlkit.vision.segmentation.Segmentation; +import com.google.mlkit.vision.segmentation.SegmentationMask; +import com.google.mlkit.vision.segmentation.Segmenter; +import com.google.mlkit.vision.segmentation.selfie.SelfieSegmenterOptions; + +import org.webrtc.SurfaceTextureHelper; +import org.webrtc.TextureBufferImpl; +import org.webrtc.VideoFrame; +import org.webrtc.VideoProcessor; +import org.webrtc.VideoSink; +import org.webrtc.YuvConverter; + +public class VirtualBackgroundVideoProcessor implements VideoProcessor { + + private VideoSink target; + private final SurfaceTextureHelper surfaceTextureHelper; + final YuvConverter yuvConverter = new YuvConverter(); + + private YuvFrame yuvFrame; + private Bitmap inputFrameBitmap; + private ReactApplicationContext reactContext; + + public Boolean enableBlurBackground = false; + + final Bitmap backgroundImage; + final Bitmap scaled; + + final SelfieSegmenterOptions options = + new SelfieSegmenterOptions.Builder() + .setDetectorMode(SelfieSegmenterOptions.STREAM_MODE) + .build(); + final Segmenter segmenter = Segmentation.getClient(options); + + public VirtualBackgroundVideoProcessor(ReactApplicationContext context, SurfaceTextureHelper surfaceTextureHelper, Boolean enableVirtualBackgroud, Boolean enableBlur, String backgroundImageBase64) { + super(); + + this.surfaceTextureHelper = surfaceTextureHelper; + reactContext = context; + enableBlurBackground = enableBlur; + + if (enableVirtualBackgroud && !backgroundImageBase64.isEmpty()) { + byte[] decodedString = Base64.decode(backgroundImageBase64, Base64.DEFAULT); + backgroundImage = BitmapFactory.decodeByteArray(decodedString, 0, decodedString.length); + int sourceWidth = backgroundImage.getWidth(); + int sourceHeight = backgroundImage.getHeight(); + + if (sourceWidth > sourceHeight) { + scaled = Bitmap.createScaledBitmap(backgroundImage, sourceHeight, sourceWidth, false); + } else { + scaled = Bitmap.createScaledBitmap(backgroundImage, sourceWidth, sourceHeight, false); + } + } else { + scaled = null; + backgroundImage = null; + } + } + + @Override + public void setSink(@Nullable VideoSink videoSink) { + target = videoSink; + } + + @Override + public void onCapturerStarted(boolean b) { + + } + + @Override + public void onCapturerStopped() { + + } + + private Bitmap blurImage(Bitmap input) + { + try + { + RenderScript rsScript = RenderScript.create(reactContext); + Allocation alloc = Allocation.createFromBitmap(rsScript, input); + + ScriptIntrinsicBlur blur = ScriptIntrinsicBlur.create(rsScript, Element.U8_4(rsScript)); + blur.setRadius(25); + blur.setInput(alloc); + + Bitmap result = Bitmap.createBitmap(input.getWidth(), input.getHeight(), Bitmap.Config.ARGB_8888); + Allocation outAlloc = Allocation.createFromBitmap(rsScript, result); + + blur.forEach(outAlloc); + outAlloc.copyTo(result); + + rsScript.destroy(); + return result; + } + catch (Exception e) { + // TODO: handle exception + return input; + } + + } + + @Override + public void onFrameCaptured(VideoFrame videoFrame) { + yuvFrame = new YuvFrame(videoFrame); + inputFrameBitmap = yuvFrame.getBitmap(); + InputImage image = InputImage.fromBitmap(inputFrameBitmap, 0); + + Task result = + segmenter.process(image) + .addOnSuccessListener( + new OnSuccessListener() { + @Override + public void onSuccess(SegmentationMask mask) { + + mask.getBuffer().rewind(); + int[] arr = maskColorsFromByteBuffer(mask); + Bitmap segmentedBitmap = Bitmap.createBitmap( + arr, mask.getWidth(), mask.getHeight(), Bitmap.Config.ARGB_8888 + ); + arr = null; + + Bitmap segmentedBitmapMutable = segmentedBitmap.copy(Bitmap.Config.ARGB_8888, true); + segmentedBitmap.recycle(); + Canvas canvas = new Canvas(segmentedBitmapMutable); + + Paint paint = new Paint(); + paint.setXfermode(new PorterDuffXfermode(SRC_IN)); + canvas.drawBitmap(enableBlurBackground ? blurImage(inputFrameBitmap) : scaled, 0, 0, paint); + paint.setXfermode(new PorterDuffXfermode(DST_OVER)); + canvas.drawBitmap(inputFrameBitmap, 0, 0, paint); + + surfaceTextureHelper.getHandler().post(new Runnable() { + @Override + public void run() { + + GLES20.glActiveTexture(GLES20.GL_TEXTURE0); + TextureBufferImpl buffer = new TextureBufferImpl(segmentedBitmapMutable.getWidth(), + segmentedBitmapMutable.getHeight(), VideoFrame.TextureBuffer.Type.RGB, + GLES20.GL_TEXTURE0, new Matrix(), surfaceTextureHelper.getHandler(), yuvConverter, null); + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE0); + + GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_NEAREST); + GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_NEAREST); + GLUtils.texImage2D(GLES20.GL_TEXTURE_2D, 0, segmentedBitmapMutable, 0); + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, 0); + + VideoFrame.I420Buffer i420Buf = yuvConverter.convert(buffer); + + int orientationDegrees; + switch (videoFrame.getRotation()) { + case 90: + orientationDegrees = 0; + break; + default: + orientationDegrees = 180; + break; + } + + VideoFrame out = new VideoFrame(i420Buf, orientationDegrees, videoFrame.getTimestampNs()); + + buffer.release(); + yuvFrame.dispose(); + + target.onFrame(out); + out.release(); + } + }); + + } + }); + } + + private int[] maskColorsFromByteBuffer(SegmentationMask mask) { + int[] colors = new int[mask.getHeight() * mask.getWidth()]; + for (int i = 0; i < mask.getHeight() * mask.getWidth(); i++) { + float backgroundLikelihood = 1 - mask.getBuffer().getFloat(); + if (backgroundLikelihood > 0.9) { + colors[i] = argb(255, 255, 0, 255); + } else if (backgroundLikelihood > 0.2) { + // Linear interpolation to make sure when backgroundLikelihood is 0.2, the alpha is 0 and + // when backgroundLikelihood is 0.9, the alpha is 128. + // +0.5 to round the float value to the nearest int. + double d = 182.9 * backgroundLikelihood - 36.6 + 0.5; + int alpha = (int) d; + colors[i] = argb(alpha, 255, 0, 255); + } + } + return colors; + } +} \ No newline at end of file diff --git a/android/src/main/java/com/oney/WebRTCModule/WebRTCModule.java b/android/src/main/java/com/oney/WebRTCModule/WebRTCModule.java index 245ccf373..ef24aeb2b 100644 --- a/android/src/main/java/com/oney/WebRTCModule/WebRTCModule.java +++ b/android/src/main/java/com/oney/WebRTCModule/WebRTCModule.java @@ -1,8 +1,13 @@ package com.oney.WebRTCModule; +import android.net.ConnectivityManager; +import android.net.Network; +import android.net.NetworkRequest; import android.util.Log; import android.util.Pair; import android.util.SparseArray; +import android.util.SparseBooleanArray; +import android.util.SparseIntArray; import androidx.annotation.NonNull; import androidx.annotation.Nullable; @@ -11,6 +16,7 @@ import com.facebook.react.bridge.Callback; import com.facebook.react.bridge.Promise; import com.facebook.react.bridge.ReactApplicationContext; +import com.facebook.react.bridge.ReactContext; import com.facebook.react.bridge.ReactContextBaseJavaModule; import com.facebook.react.bridge.ReactMethod; import com.facebook.react.bridge.ReadableArray; @@ -33,8 +39,51 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Timer; +import java.util.TimerTask; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +interface OnValueChangeListener { + void onValueChanged(PeerConnectionObserver peerConnection, boolean isSpeaking, double audioLevel); +} + +class AudioLevelValueHolder { + private OnValueChangeListener listener; + SparseBooleanArray storedPeerConnections = new SparseBooleanArray(); + + void setOnValueChangeListener(OnValueChangeListener listener) { + this.listener = listener; + } + + void setValue(PeerConnectionObserver peerConnection, boolean isSpeaking, double audioLevel) { + int id = peerConnection.getId(); + if (this.storedPeerConnections.indexOfKey(id) >= 0) { + boolean isSpeak = this.storedPeerConnections.get(id); + + if (isSpeak != isSpeaking) { + this.storedPeerConnections.put(id, isSpeaking); + + if (listener != null) { + listener.onValueChanged(peerConnection, isSpeaking, audioLevel); + } + } + } else { + this.storedPeerConnections.put(id, isSpeaking); + + if (listener != null) { + listener.onValueChanged(peerConnection, isSpeaking, audioLevel); + } + } + } + + void cleanup() { + this.storedPeerConnections.clear(); + } +} @ReactModule(name = "WebRTCModule") public class WebRTCModule extends ReactContextBaseJavaModule { @@ -49,6 +98,16 @@ public class WebRTCModule extends ReactContextBaseJavaModule { private final SparseArray mPeerConnectionObservers; final Map localStreams; + static Timer voiceTimer; + boolean isVoiceTimerRunning = false; + + AudioLevelValueHolder incomingAudioLevelHolder; + AudioLevelValueHolder outgoingAudioLevelHolder; + + private final AtomicInteger voicePollGeneration = new AtomicInteger(0); + private final AtomicReference> voicePollTaskRef = new AtomicReference<>(null); + + private final GetUserMediaImpl getUserMediaImpl; public WebRTCModule(ReactApplicationContext reactContext) { @@ -57,6 +116,35 @@ public WebRTCModule(ReactApplicationContext reactContext) { mPeerConnectionObservers = new SparseArray<>(); localStreams = new HashMap<>(); + incomingAudioLevelHolder = new AudioLevelValueHolder(); + outgoingAudioLevelHolder = new AudioLevelValueHolder(); + + incomingAudioLevelHolder.setOnValueChangeListener(new OnValueChangeListener() { + @Override + public void onValueChanged(PeerConnectionObserver peerConnection, boolean isSpeaking, double audioLevel) { + ThreadUtils.runOnExecutor(() -> { + WritableMap params = Arguments.createMap(); + params.putBoolean("isSpeaking", isSpeaking); + params.putDouble("audioLevel", audioLevel); + + params.putInt("pcId", peerConnection.getId()); + sendEvent("peerVoiceIncomingStateChanged", params); + }); + } + }); + + outgoingAudioLevelHolder.setOnValueChangeListener(new OnValueChangeListener() { + @Override + public void onValueChanged(PeerConnectionObserver peerConnection, boolean isSpeaking, double audioLevel) { + WritableMap params = Arguments.createMap(); + params.putBoolean("isSpeaking", isSpeaking); + params.putDouble("audioLevel", audioLevel); + + params.putInt("pcId", peerConnection.getId()); + sendEvent("peerVoiceOutgoingStateChanged", params); + } + }); + WebRTCModuleOptions options = WebRTCModuleOptions.getInstance(); AudioDeviceModule adm = options.audioDeviceModule; @@ -111,6 +199,25 @@ public WebRTCModule(ReactApplicationContext reactContext) { mAudioDeviceModule = adm; getUserMediaImpl = new GetUserMediaImpl(this, reactContext); + + observeReconnection(reactContext); + } + + private void observeReconnection(ReactContext reactContext) { + ConnectivityManager connectivityManager = + (ConnectivityManager) reactContext.getSystemService(reactContext.CONNECTIVITY_SERVICE); + NetworkRequest networkRequest = new NetworkRequest.Builder().build(); + + connectivityManager.registerNetworkCallback(networkRequest, new ConnectivityManager.NetworkCallback() { + @Override + public void onAvailable(Network network) { + super.onAvailable(network); + for (int i = 0, size = mPeerConnectionObservers.size(); i < size; i++) { + PeerConnectionObserver pco = mPeerConnectionObservers.valueAt(i); + pco.getPeerConnection().restartIce(); + } + } + }); } @NonNull @@ -182,7 +289,8 @@ private PeerConnection.RTCConfiguration parseRTCConfiguration(ReadableMap map) { PeerConnection.RTCConfiguration conf = new PeerConnection.RTCConfiguration(iceServers); conf.sdpSemantics = PeerConnection.SdpSemantics.UNIFIED_PLAN; - + conf.continualGatheringPolicy = PeerConnection.ContinualGatheringPolicy.GATHER_CONTINUALLY; + conf.candidateNetworkPolicy = PeerConnection.CandidateNetworkPolicy.ALL; // Required for perfect negotiation. conf.enableImplicitRollback = true; @@ -376,6 +484,7 @@ private PeerConnection.RTCConfiguration parseRTCConfiguration(ReadableMap map) { public boolean peerConnectionInit(ReadableMap configuration, int id) { PeerConnection.RTCConfiguration rtcConfiguration = parseRTCConfiguration(configuration); + observeVoiceActivity(); try { return (boolean) ThreadUtils .submitToExecutor(() -> { @@ -386,6 +495,7 @@ public boolean peerConnectionInit(ReadableMap configuration, int id) { } observer.setPeerConnection(peerConnection); mPeerConnectionObservers.put(id, observer); + return true; }) .get(); @@ -395,6 +505,155 @@ public boolean peerConnectionInit(ReadableMap configuration, int id) { } } + void cancelVoiceActivity() { + // Invalidate all queued/running poll tasks from previous timer cycle. + voicePollGeneration.incrementAndGet(); + + Future pending = voicePollTaskRef.getAndSet(null); + if (pending != null) { + pending.cancel(false); + } + + if (WebRTCModule.voiceTimer != null) { + WebRTCModule.voiceTimer.cancel(); + WebRTCModule.voiceTimer.purge(); + WebRTCModule.voiceTimer = null; + } + + incomingAudioLevelHolder.cleanup(); + outgoingAudioLevelHolder.cleanup(); + } + + + void observeVoiceActivity() { + int checkInterval = 300; + + final SparseIntArray silenceIncomingCount = new SparseIntArray(); + final double[] silenceOutgoingCount = {0}; + + for (int i = 0; i < mPeerConnectionObservers.size(); i++) { + int key = mPeerConnectionObservers.keyAt(i); + PeerConnectionObserver peer = mPeerConnectionObservers.get(key); + if (peer != null) { + silenceIncomingCount.put(peer.getId(), 0); + } + } + + cancelVoiceActivity(); + final int generation = voicePollGeneration.incrementAndGet(); + + WebRTCModule.voiceTimer = new Timer(); + WebRTCModule.voiceTimer.schedule(new TimerTask() { + @Override + public void run() { + if (voicePollGeneration.get() != generation) { + return; + } + + Future current = voicePollTaskRef.get(); + if (current != null && !current.isDone()) { + return; // previous tick still queued/running + } + + Future next = ThreadUtils.submitToExecutor(() -> { + if (voicePollGeneration.get() != generation) { + return; + } + + for (int i = 0; i < mPeerConnectionObservers.size(); i++) { + int key = mPeerConnectionObservers.keyAt(i); + PeerConnectionObserver peer = mPeerConnectionObservers.get(key); + if (peer == null) { + continue; + } + + PeerConnection peerConnection = peer.getPeerConnection(); + if (peerConnection == null) { + continue; + } + + int id = peer.getId(); + + if (peerConnection.connectionState() == PeerConnection.PeerConnectionState.CONNECTED) { + peerConnection.getStats(new RTCStatsCollectorCallback() { + @Override + public void onStatsDelivered(RTCStatsReport rtcStatsReport) { + for (RTCStats stats : rtcStatsReport.getStatsMap().values()) { + if (stats.getType().equals("inbound-rtp")) { + Object audioLevelObject = stats.getMembers().get("audioLevel"); + + if (audioLevelObject instanceof Double) { + Double audioLevel = ((Double) audioLevelObject); + + if (audioLevel > 0.025) { + silenceIncomingCount.put(id, 0); + incomingAudioLevelHolder.setValue(peer, true, audioLevel.doubleValue()); + } else { + silenceIncomingCount.put(id, silenceIncomingCount.get(id) + 1); + + if (silenceIncomingCount.get(id) > 4.0) { + incomingAudioLevelHolder.setValue(peer, false, 0); + } + } + } + } + } + } + }); + } + } + + for (int i = 0; i < mPeerConnectionObservers.size(); i++) { + int key = mPeerConnectionObservers.keyAt(i); + PeerConnectionObserver peer = mPeerConnectionObservers.get(key); + if (peer == null) { + continue; + } + + PeerConnection peerConnection = peer.getPeerConnection(); + if (peerConnection == null) { + continue; + } + + if (peerConnection.connectionState() == PeerConnection.PeerConnectionState.CONNECTED) { + peerConnection.getStats(new RTCStatsCollectorCallback() { + @Override + public void onStatsDelivered(RTCStatsReport rtcStatsReport) { + for (RTCStats stats : rtcStatsReport.getStatsMap().values()) { + if (stats.getType().equals("media-source")) { + Object audioLevelObject = stats.getMembers().get("audioLevel"); + + if (audioLevelObject instanceof Double) { + Double audioLevel = ((Double) audioLevelObject); + + if (audioLevel > 0.01) { + silenceOutgoingCount[0] = 0; + outgoingAudioLevelHolder.setValue(peer, true, audioLevel.doubleValue()); + } else { + silenceOutgoingCount[0] += 1; + + if (silenceOutgoingCount[0] > 4.0) { + outgoingAudioLevelHolder.setValue(peer, false, 0); + } + } + } + } + } + } + }); + + return; + } + } + }); + + voicePollTaskRef.set(next); + } + }, 0, checkInterval); + + isVoiceTimerRunning = true; + } + MediaStream getStreamForReactTag(String streamReactTag) { // This function _only_ gets called from WebRTCView, in the UI thread. // Hence make sure we run this code in the executor or we run at the risk @@ -443,7 +702,7 @@ MediaStreamTrack getLocalTrack(String trackId) { } public VideoTrack createVideoTrack(AbstractVideoCaptureController videoCaptureController) { - return getUserMediaImpl.createVideoTrack(videoCaptureController); + return getUserMediaImpl.createVideoTrack(videoCaptureController, false, false, ""); } public void createStream( @@ -705,7 +964,7 @@ public void transceiverSetDirection(int id, String senderId, String direction, P }); } - @ReactMethod(isBlockingSynchronousMethod = true) + @ReactMethod public void transceiverSetCodecPreferences(int id, String senderId, ReadableArray codecPreferences) { ThreadUtils.runOnExecutor(() -> { WritableMap identifier = Arguments.createMap(); @@ -1281,7 +1540,7 @@ public void peerConnectionAddICECandidate(int pcId, ReadableMap candidateMap, Pr candidateMap.hasKey("sdpMid") && !candidateMap.isNull("sdpMid") ? candidateMap.getString("sdpMid") : "", candidateMap.hasKey("sdpMLineIndex") && !candidateMap.isNull("sdpMLineIndex") ? candidateMap.getInt("sdpMLineIndex") : 0, candidateMap.getString("candidate")); - + peerConnection.addIceCandidate(candidate, new AddIceObserver() { @Override public void onAddSuccess() { @@ -1334,8 +1593,16 @@ public void peerConnectionDispose(int id) { if (pco == null || pco.getPeerConnection() == null) { Log.d(TAG, "peerConnectionDispose() peerConnection is null"); } - pco.dispose(); + + if (pco != null) { + pco.dispose(); + } + mPeerConnectionObservers.remove(id); + + if (mPeerConnectionObservers.size() == 0) { + cancelVoiceActivity(); + } }); } diff --git a/android/src/main/java/com/oney/WebRTCModule/WebRTCView.java b/android/src/main/java/com/oney/WebRTCModule/WebRTCView.java index 95ede811e..ee6fb239d 100644 --- a/android/src/main/java/com/oney/WebRTCModule/WebRTCView.java +++ b/android/src/main/java/com/oney/WebRTCModule/WebRTCView.java @@ -175,7 +175,7 @@ private VideoTrack getVideoTrackForStreamURL(String streamURL) { List videoTracks = stream.videoTracks; if (!videoTracks.isEmpty()) { - videoTrack = videoTracks.get(0); + videoTrack = videoTracks.get(videoTracks.size() - 1); } } diff --git a/android/src/main/java/com/oney/WebRTCModule/YuvFrame.java b/android/src/main/java/com/oney/WebRTCModule/YuvFrame.java new file mode 100644 index 000000000..3b3da0a67 --- /dev/null +++ b/android/src/main/java/com/oney/WebRTCModule/YuvFrame.java @@ -0,0 +1,195 @@ +package com.oney.WebRTCModule; + +import android.graphics.Bitmap; +import android.graphics.Matrix; + +import org.webrtc.VideoFrame; + +import java.nio.ByteBuffer; + +public class YuvFrame { + public int width; + public int height; + public byte[] nv21Buffer; + public int rotationDegree; + public long timestamp; + + private final Object planeLock = new Object(); + + public YuvFrame(final VideoFrame videoFrame) { + fromVideoFrame(videoFrame, System.nanoTime()); + } + + public void fromVideoFrame(final VideoFrame videoFrame, final long timestamp) { + if (videoFrame == null) { + return; + } + + synchronized (planeLock) { + try { + // Save timestamp + this.timestamp = timestamp; + + // Copy rotation information + rotationDegree = videoFrame.getRotation(); // Just save rotation info for now, doing actual rotation can wait until per-pixel processing. + + // Copy the pixel data, processing as requested. + copyPlanes(videoFrame.getBuffer()); + } catch (Throwable t) { + dispose(); + } + } + } + + public void dispose() { + nv21Buffer = null; + } + + private void copyPlanes( final VideoFrame.Buffer videoFrameBuffer ) + { + VideoFrame.I420Buffer i420Buffer = null; + + if ( videoFrameBuffer != null ) + { + i420Buffer = videoFrameBuffer.toI420(); + } + + if ( i420Buffer == null ) + { + return; + } + + synchronized ( planeLock ) + { + // Set the width and height of the frame. + width = i420Buffer.getWidth(); + height = i420Buffer.getHeight(); + + // Calculate sizes needed to convert to NV21 buffer format + final int size = width * height; + final int chromaStride = width; + final int chromaWidth = ( width + 1 ) / 2; + final int chromaHeight = ( height + 1 ) / 2; + final int nv21Size = size + chromaStride * chromaHeight; + + if ( nv21Buffer == null || nv21Buffer.length != nv21Size ) + { + nv21Buffer = new byte[nv21Size]; + } + + final ByteBuffer yPlane = i420Buffer.getDataY(); + final ByteBuffer uPlane = i420Buffer.getDataU(); + final ByteBuffer vPlane = i420Buffer.getDataV(); + final int yStride = i420Buffer.getStrideY(); + final int uStride = i420Buffer.getStrideU(); + final int vStride = i420Buffer.getStrideV(); + + // Populate a buffer in NV21 format because that's what the converter wants + for ( int y = 0; y < height; y++ ) + { + for ( int x = 0; x < width; x++ ) + { + nv21Buffer[y * width + x] = yPlane.get( y * yStride + x ); + } + } + + for ( int y = 0; y < chromaHeight; y++ ) + { + for ( int x = 0; x < chromaWidth; x++ ) + { + // Swapping U and V values here because it makes the image the right color + + // Store V + nv21Buffer[size + y * chromaStride + 2 * x + 1] = uPlane.get( y * uStride + x ); + + // Store U + nv21Buffer[size + y * chromaStride + 2 * x] = vPlane.get( y * vStride + x ); + } + } + } + i420Buffer.release(); + } + + public Bitmap getBitmap() + { + if ( nv21Buffer == null ) + { + return null; + } + + // Calculate the size of the frame + final int size = width * height; + + // Allocate an array to hold the ARGB pixel data + int[] argb = new int[size]; + + // Use the converter (based on WebRTC source) to change to ARGB format + YUV_NV21_TO_RGB(argb, nv21Buffer, width, height); + + // Construct a Bitmap based on the new pixel data + Bitmap bitmap = Bitmap.createBitmap( argb, width, height, Bitmap.Config.ARGB_8888 ); + argb = null; + + // If necessary, generate a rotated version of the Bitmap + if ( rotationDegree == 90 || rotationDegree == -270 ) + { + final Matrix m = new Matrix(); + m.preScale(-1, 1); + m.postRotate( 90 ); + + return Bitmap.createBitmap( bitmap, 0, 0, bitmap.getWidth(), bitmap.getHeight(), m, true ); + } + else if ( rotationDegree == 180 || rotationDegree == -180 ) + { + final Matrix m = new Matrix(); + m.preScale(-1, 1); + m.postRotate( 180 ); + + return Bitmap.createBitmap( bitmap, 0, 0, bitmap.getWidth(), bitmap.getHeight(), m, true ); + } + else if ( rotationDegree == 270 || rotationDegree == -90 ) + { + final Matrix m = new Matrix(); + m.preScale(1, -1); + m.postRotate( 270 ); + + return Bitmap.createBitmap( bitmap, 0, 0, bitmap.getWidth(), bitmap.getHeight(), m, true ); + } + else + { + final Matrix m = new Matrix(); + m.preScale(-1, 1); + + return Bitmap.createBitmap( bitmap, 0, 0, bitmap.getWidth(), bitmap.getHeight(), m, true ); + } + } + + public static void YUV_NV21_TO_RGB(int[] argb, byte[] yuv, int width, int height) { + final int frameSize = width * height; + + final int ii = 0; + final int ij = 0; + final int di = +1; + final int dj = +1; + + int a = 0; + for (int i = 0, ci = ii; i < height; ++i, ci += di) { + for (int j = 0, cj = ij; j < width; ++j, cj += dj) { + int y = (0xff & ((int) yuv[ci * width + cj])); + int v = (0xff & ((int) yuv[frameSize + (ci >> 1) * width + (cj & ~1) + 0])); + int u = (0xff & ((int) yuv[frameSize + (ci >> 1) * width + (cj & ~1) + 1])); + y = y < 16 ? 16 : y; + + int r = (int) (1.164f * (y - 16) + 1.596f * (v - 128)); + int g = (int) (1.164f * (y - 16) - 0.813f * (v - 128) - 0.391f * (u - 128)); + int b = (int) (1.164f * (y - 16) + 2.018f * (u - 128)); + + r = r < 0 ? 0 : (r > 255 ? 255 : r); + g = g < 0 ? 0 : (g > 255 ? 255 : g); + b = b < 0 ? 0 : (b > 255 ? 255 : b); + + argb[a++] = 0xff000000 | (r << 16) | (g << 8) | b; + } + } + } +} diff --git a/examples/GumTestApp/.buckconfig b/examples/GumTestApp/.buckconfig deleted file mode 100644 index 934256cb2..000000000 --- a/examples/GumTestApp/.buckconfig +++ /dev/null @@ -1,6 +0,0 @@ - -[android] - target = Google Inc.:Google APIs:23 - -[maven_repositories] - central = https://repo1.maven.org/maven2 diff --git a/examples/GumTestApp/.eslintrc.js b/examples/GumTestApp/.eslintrc.js deleted file mode 100644 index 40c6dcd05..000000000 --- a/examples/GumTestApp/.eslintrc.js +++ /dev/null @@ -1,4 +0,0 @@ -module.exports = { - root: true, - extends: '@react-native-community', -}; diff --git a/examples/GumTestApp/.flowconfig b/examples/GumTestApp/.flowconfig deleted file mode 100644 index b274ad1d6..000000000 --- a/examples/GumTestApp/.flowconfig +++ /dev/null @@ -1,73 +0,0 @@ -[ignore] -; We fork some components by platform -.*/*[.]android.js - -; Ignore "BUCK" generated dirs -/\.buckd/ - -; Ignore polyfills -node_modules/react-native/Libraries/polyfills/.* - -; These should not be required directly -; require from fbjs/lib instead: require('fbjs/lib/warning') -node_modules/warning/.* - -; Flow doesn't support platforms -.*/Libraries/Utilities/LoadingView.js - -[untyped] -.*/node_modules/@react-native-community/cli/.*/.* - -[include] - -[libs] -node_modules/react-native/interface.js -node_modules/react-native/flow/ - -[options] -emoji=true - -esproposal.optional_chaining=enable -esproposal.nullish_coalescing=enable - -module.file_ext=.js -module.file_ext=.json -module.file_ext=.ios.js - -munge_underscores=true - -module.name_mapper='^react-native/\(.*\)$' -> '/node_modules/react-native/\1' -module.name_mapper='^@?[./a-zA-Z0-9$_-]+\.\(bmp\|gif\|jpg\|jpeg\|png\|psd\|svg\|webp\|m4v\|mov\|mp4\|mpeg\|mpg\|webm\|aac\|aiff\|caf\|m4a\|mp3\|wav\|html\|pdf\)$' -> '/node_modules/react-native/Libraries/Image/RelativeImageStub' - -suppress_type=$FlowIssue -suppress_type=$FlowFixMe -suppress_type=$FlowFixMeProps -suppress_type=$FlowFixMeState - -suppress_comment=\\(.\\|\n\\)*\\$FlowFixMe\\($\\|[^(]\\|(\\(\\)? *\\(site=[a-z,_]*react_native\\(_ios\\)?_\\(oss\\|fb\\)[a-z,_]*\\)?)\\) -suppress_comment=\\(.\\|\n\\)*\\$FlowIssue\\((\\(\\)? *\\(site=[a-z,_]*react_native\\(_ios\\)?_\\(oss\\|fb\\)[a-z,_]*\\)?)\\)?:? #[0-9]+ -suppress_comment=\\(.\\|\n\\)*\\$FlowExpectedError - -[lints] -sketchy-null-number=warn -sketchy-null-mixed=warn -sketchy-number=warn -untyped-type-import=warn -nonstrict-import=warn -deprecated-type=warn -unsafe-getters-setters=warn -unnecessary-invariant=warn -signature-verification-failure=warn -deprecated-utility=error - -[strict] -deprecated-type -nonstrict-import -sketchy-null -unclear-type -unsafe-getters-setters -untyped-import -untyped-type-import - -[version] -^0.122.0 diff --git a/examples/GumTestApp/.gitattributes b/examples/GumTestApp/.gitattributes deleted file mode 100644 index d42ff1835..000000000 --- a/examples/GumTestApp/.gitattributes +++ /dev/null @@ -1 +0,0 @@ -*.pbxproj -text diff --git a/examples/GumTestApp/.gitignore b/examples/GumTestApp/.gitignore deleted file mode 100644 index ad572e632..000000000 --- a/examples/GumTestApp/.gitignore +++ /dev/null @@ -1,59 +0,0 @@ -# OSX -# -.DS_Store - -# Xcode -# -build/ -*.pbxuser -!default.pbxuser -*.mode1v3 -!default.mode1v3 -*.mode2v3 -!default.mode2v3 -*.perspectivev3 -!default.perspectivev3 -xcuserdata -*.xccheckout -*.moved-aside -DerivedData -*.hmap -*.ipa -*.xcuserstate - -# Android/IntelliJ -# -build/ -.idea -.gradle -local.properties -*.iml - -# node.js -# -node_modules/ -npm-debug.log -yarn-error.log - -# BUCK -buck-out/ -\.buckd/ -*.keystore -!debug.keystore - -# fastlane -# -# It is recommended to not store the screenshots in the git repo. Instead, use fastlane to re-generate the -# screenshots whenever they are needed. -# For more information about the recommended setup visit: -# https://docs.fastlane.tools/best-practices/source-control/ - -*/fastlane/report.xml -*/fastlane/Preview.html -*/fastlane/screenshots - -# Bundle artifact -*.jsbundle - -# CocoaPods -/ios/Pods/ diff --git a/examples/GumTestApp/.npmrc b/examples/GumTestApp/.npmrc deleted file mode 100644 index b15cbc2c0..000000000 --- a/examples/GumTestApp/.npmrc +++ /dev/null @@ -1,2 +0,0 @@ -package-lock=false - diff --git a/examples/GumTestApp/.prettierrc.js b/examples/GumTestApp/.prettierrc.js deleted file mode 100644 index 5c4de1a4f..000000000 --- a/examples/GumTestApp/.prettierrc.js +++ /dev/null @@ -1,6 +0,0 @@ -module.exports = { - bracketSpacing: false, - jsxBracketSameLine: true, - singleQuote: true, - trailingComma: 'all', -}; diff --git a/examples/GumTestApp/.watchmanconfig b/examples/GumTestApp/.watchmanconfig deleted file mode 100644 index 9e26dfeeb..000000000 --- a/examples/GumTestApp/.watchmanconfig +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/examples/GumTestApp/App.js b/examples/GumTestApp/App.js deleted file mode 100644 index e9e9fe6c3..000000000 --- a/examples/GumTestApp/App.js +++ /dev/null @@ -1,82 +0,0 @@ -/** - * Sample React Native App - * https://github.com/facebook/react-native - * - * @format - * @flow strict-local - */ - -import React, {useState} from 'react'; -import { - Button, - SafeAreaView, - StyleSheet, - View, - StatusBar, -} from 'react-native'; -import { Colors } from 'react-native/Libraries/NewAppScreen'; -import { mediaDevices, RTCView } from 'react-native-webrtc'; - - -const App = () => { - const [stream, setStream] = useState(null); - const start = async () => { - console.log('start'); - if (!stream) { - try { - const s = await mediaDevices.getUserMedia({ video: true }); - setStream(s); - } catch(e) { - console.error(e); - } - } - }; - const stop = () => { - console.log('stop'); - if (stream) { - stream.release(); - setStream(null); - } - }; - return ( - <> - - - { - stream && - - } - -