diff --git a/app/src/main/java/org/thoughtcrime/securesms/components/webrtc/BroadcastVideoSink.java b/app/src/main/java/org/thoughtcrime/securesms/components/webrtc/BroadcastVideoSink.java index b7c45771b..757094c66 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/components/webrtc/BroadcastVideoSink.java +++ b/app/src/main/java/org/thoughtcrime/securesms/components/webrtc/BroadcastVideoSink.java @@ -22,7 +22,7 @@ public class BroadcastVideoSink implements VideoSink { public static final int DEVICE_ROTATION_IGNORE = -1; - private final EglBase eglBase; + private final EglBaseWrapper eglBase; private final WeakHashMap sinks; private final WeakHashMap requestingSizes; private boolean dirtySizes; @@ -32,7 +32,7 @@ public class BroadcastVideoSink implements VideoSink { private boolean rotateWithDevice; public BroadcastVideoSink() { - this(null, false, true, 0); + this(new EglBaseWrapper(null), false, true, 0); } /** @@ -41,7 +41,7 @@ public class BroadcastVideoSink implements VideoSink { * @param rotateWithDevice Rotate video frame to match device orientation * @param deviceOrientationDegrees Device orientation in degrees */ - public BroadcastVideoSink(@Nullable EglBase eglBase, boolean forceRotate, boolean rotateWithDevice, int deviceOrientationDegrees) { + public BroadcastVideoSink(@NonNull EglBaseWrapper eglBase, boolean forceRotate, boolean rotateWithDevice, int deviceOrientationDegrees) { this.eglBase = eglBase; this.sinks = new WeakHashMap<>(); this.requestingSizes = new WeakHashMap<>(); @@ -52,7 +52,7 @@ public class BroadcastVideoSink implements VideoSink { this.rotateWithDevice = rotateWithDevice; } - public @Nullable EglBase getEglBase() { + public @NonNull EglBaseWrapper getLockableEglBase() { return eglBase; } diff --git a/app/src/main/java/org/thoughtcrime/securesms/components/webrtc/CallParticipantView.java b/app/src/main/java/org/thoughtcrime/securesms/components/webrtc/CallParticipantView.java index 12695edcc..845ede228 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/components/webrtc/CallParticipantView.java +++ b/app/src/main/java/org/thoughtcrime/securesms/components/webrtc/CallParticipantView.java @@ -144,9 +144,9 @@ public class CallParticipantView extends ConstraintLayout { renderer.setVisibility(hasContentToRender ? View.VISIBLE : View.GONE); if (participant.isVideoEnabled()) { - if (participant.getVideoSink().getEglBase() != null) { - renderer.init(participant.getVideoSink().getEglBase()); - } + participant.getVideoSink().getLockableEglBase().performWithValidEglBase(eglBase -> { + renderer.init(eglBase); + }); renderer.attachBroadcastVideoSink(participant.getVideoSink()); } else { renderer.attachBroadcastVideoSink(null); diff --git a/app/src/main/java/org/thoughtcrime/securesms/components/webrtc/EglBaseWrapper.kt b/app/src/main/java/org/thoughtcrime/securesms/components/webrtc/EglBaseWrapper.kt new file mode 100644 index 000000000..6a78d27ca --- /dev/null +++ b/app/src/main/java/org/thoughtcrime/securesms/components/webrtc/EglBaseWrapper.kt @@ -0,0 +1,70 @@ +package org.thoughtcrime.securesms.components.webrtc + +import android.opengl.EGL14 +import org.signal.core.util.logging.Log +import org.webrtc.EglBase +import org.webrtc.EglBase10 +import org.webrtc.EglBase14 +import java.util.concurrent.locks.Lock +import java.util.concurrent.locks.ReentrantLock +import java.util.function.Consumer +import javax.microedition.khronos.egl.EGL10 +import kotlin.concurrent.withLock + +private val TAG = Log.tag(EglBaseWrapper::class.java) + +/** + * Wrapper which allows caller to perform synchronized actions on an EglBase object. + */ +class EglBaseWrapper(val eglBase: EglBase?) { + + private val lock: Lock = ReentrantLock() + + fun require(): EglBase = requireNotNull(eglBase) + + @Volatile + private var isReleased: Boolean = false + + fun performWithValidEglBase(consumer: Consumer) { + if (isReleased) { + Log.d(TAG, "Tried to use a released EglBase", Exception()) + return + } + + if (eglBase == null) { + return + } + + lock.withLock { + if (isReleased) { + Log.d(TAG, "Tried to use a released EglBase", Exception()) + return + } + + val hasSharedContext = when (val context: EglBase.Context = eglBase.eglBaseContext) { + is EglBase14.Context -> context.rawContext != EGL14.EGL_NO_CONTEXT + is EglBase10.Context -> context.rawContext != EGL10.EGL_NO_CONTEXT + else -> throw IllegalStateException("Unknown context") + } + + if (hasSharedContext) { + consumer.accept(eglBase) + } + } + } + + fun releaseEglBase() { + if (isReleased || eglBase == null) { + return + } + + lock.withLock { + if (isReleased) { + return + } + + isReleased = true + eglBase.release() + } + } +} diff --git a/app/src/main/java/org/thoughtcrime/securesms/components/webrtc/WebRtcCallView.java b/app/src/main/java/org/thoughtcrime/securesms/components/webrtc/WebRtcCallView.java index b18865ad5..cbcb706fa 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/components/webrtc/WebRtcCallView.java +++ b/app/src/main/java/org/thoughtcrime/securesms/components/webrtc/WebRtcCallView.java @@ -47,7 +47,6 @@ import org.thoughtcrime.securesms.recipients.Recipient; import org.thoughtcrime.securesms.recipients.RecipientId; import org.thoughtcrime.securesms.ringrtc.CameraState; import org.thoughtcrime.securesms.util.BlurTransformation; -import org.thoughtcrime.securesms.util.ServiceUtil; import org.thoughtcrime.securesms.util.SetUtil; import org.thoughtcrime.securesms.util.ViewUtil; import org.thoughtcrime.securesms.util.views.Stub; @@ -377,9 +376,10 @@ public class WebRtcCallView extends ConstraintLayout { smallLocalRender.setScalingType(RendererCommon.ScalingType.SCALE_ASPECT_FILL); largeLocalRender.setScalingType(RendererCommon.ScalingType.SCALE_ASPECT_FILL); - if (localCallParticipant.getVideoSink().getEglBase() != null) { - largeLocalRender.init(localCallParticipant.getVideoSink().getEglBase()); - } + localCallParticipant.getVideoSink().getLockableEglBase().performWithValidEglBase(eglBase -> { + largeLocalRender.init(eglBase); + }); + videoToggle.setChecked(localCallParticipant.isVideoEnabled(), false); smallLocalRender.setRenderInPip(true); diff --git a/app/src/main/java/org/thoughtcrime/securesms/ringrtc/Camera.java b/app/src/main/java/org/thoughtcrime/securesms/ringrtc/Camera.java index 0387aee92..30f67b0f3 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/ringrtc/Camera.java +++ b/app/src/main/java/org/thoughtcrime/securesms/ringrtc/Camera.java @@ -14,13 +14,13 @@ import com.annimon.stream.Stream; import org.signal.core.util.logging.Log; import org.signal.ringrtc.CameraControl; +import org.thoughtcrime.securesms.components.webrtc.EglBaseWrapper; import org.webrtc.Camera1Enumerator; import org.webrtc.Camera2Capturer; import org.webrtc.Camera2Enumerator; import org.webrtc.CameraEnumerator; import org.webrtc.CameraVideoCapturer; import org.webrtc.CapturerObserver; -import org.webrtc.EglBase; import org.webrtc.SurfaceTextureHelper; import org.webrtc.VideoFrame; @@ -39,19 +39,19 @@ public class Camera implements CameraControl, CameraVideoCapturer.CameraSwitchHa private static final String TAG = Log.tag(Camera.class); - @NonNull private final Context context; - @Nullable private final CameraVideoCapturer capturer; - @NonNull private final CameraEventListener cameraEventListener; - @NonNull private final EglBase eglBase; - private final int cameraCount; - @NonNull private CameraState.Direction activeDirection; - private boolean enabled; - private boolean isInitialized; - private int orientation; + @NonNull private final Context context; + @Nullable private final CameraVideoCapturer capturer; + @NonNull private final CameraEventListener cameraEventListener; + @NonNull private final EglBaseWrapper eglBase; + private final int cameraCount; + @NonNull private CameraState.Direction activeDirection; + private boolean enabled; + private boolean isInitialized; + private int orientation; public Camera(@NonNull Context context, @NonNull CameraEventListener cameraEventListener, - @NonNull EglBase eglBase, + @NonNull EglBaseWrapper eglBase, @NonNull CameraState.Direction desiredCameraDirection) { this.context = context; @@ -80,11 +80,13 @@ public class Camera implements CameraControl, CameraVideoCapturer.CameraSwitchHa @Override public void initCapturer(@NonNull CapturerObserver observer) { if (capturer != null) { - capturer.initialize(SurfaceTextureHelper.create("WebRTC-SurfaceTextureHelper", eglBase.getEglBaseContext()), - context, - new CameraCapturerWrapper(observer)); - capturer.setOrientation(orientation); - isInitialized = true; + eglBase.performWithValidEglBase(base -> { + capturer.initialize(SurfaceTextureHelper.create("WebRTC-SurfaceTextureHelper", base.getEglBaseContext()), + context, + new CameraCapturerWrapper(observer)); + capturer.setOrientation(orientation); + isInitialized = true; + }); } } diff --git a/app/src/main/java/org/thoughtcrime/securesms/service/webrtc/BeginCallActionProcessorDelegate.java b/app/src/main/java/org/thoughtcrime/securesms/service/webrtc/BeginCallActionProcessorDelegate.java index d87c2bc70..913765217 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/service/webrtc/BeginCallActionProcessorDelegate.java +++ b/app/src/main/java/org/thoughtcrime/securesms/service/webrtc/BeginCallActionProcessorDelegate.java @@ -45,7 +45,7 @@ public class BeginCallActionProcessorDelegate extends WebRtcActionProcessor { CallParticipant.createRemote(new CallParticipantId(remotePeer.getRecipient()), remotePeer.getRecipient(), null, - new BroadcastVideoSink(currentState.getVideoState().getEglBase(), + new BroadcastVideoSink(currentState.getVideoState().getLockableEglBase(), false, true, currentState.getLocalDeviceState().getOrientation().getDegrees()), @@ -92,7 +92,7 @@ public class BeginCallActionProcessorDelegate extends WebRtcActionProcessor { CallParticipant.createRemote(new CallParticipantId(remotePeer.getRecipient()), remotePeer.getRecipient(), null, - new BroadcastVideoSink(currentState.getVideoState().getEglBase(), + new BroadcastVideoSink(currentState.getVideoState().getLockableEglBase(), false, true, currentState.getLocalDeviceState().getOrientation().getDegrees()), diff --git a/app/src/main/java/org/thoughtcrime/securesms/service/webrtc/GroupActionProcessor.java b/app/src/main/java/org/thoughtcrime/securesms/service/webrtc/GroupActionProcessor.java index 4206076de..e15454101 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/service/webrtc/GroupActionProcessor.java +++ b/app/src/main/java/org/thoughtcrime/securesms/service/webrtc/GroupActionProcessor.java @@ -92,11 +92,11 @@ public class GroupActionProcessor extends DeviceAwareActionProcessor { BroadcastVideoSink videoSink; VideoTrack videoTrack = device.getVideoTrack(); if (videoTrack != null) { - videoSink = (callParticipant != null && callParticipant.getVideoSink().getEglBase() != null) ? callParticipant.getVideoSink() - : new BroadcastVideoSink(currentState.getVideoState().requireEglBase(), - true, - true, - currentState.getLocalDeviceState().getOrientation().getDegrees()); + videoSink = (callParticipant != null && callParticipant.getVideoSink().getLockableEglBase().getEglBase() != null) ? callParticipant.getVideoSink() + : new BroadcastVideoSink(currentState.getVideoState().getLockableEglBase(), + true, + true, + currentState.getLocalDeviceState().getOrientation().getDegrees()); videoTrack.addSink(videoSink); } else { videoSink = new BroadcastVideoSink(); @@ -267,7 +267,7 @@ public class GroupActionProcessor extends DeviceAwareActionProcessor { currentState = terminateGroupCall(currentState, false).builder() .actionProcessor(new GroupNetworkUnavailableActionProcessor(webRtcInteractor)) .changeVideoState() - .eglBase(videoState.getEglBase()) + .eglBase(videoState.getLockableEglBase()) .camera(videoState.getCamera()) .localSink(videoState.getLocalSink()) .commit() diff --git a/app/src/main/java/org/thoughtcrime/securesms/service/webrtc/GroupNetworkUnavailableActionProcessor.java b/app/src/main/java/org/thoughtcrime/securesms/service/webrtc/GroupNetworkUnavailableActionProcessor.java index f81824588..f63695f5d 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/service/webrtc/GroupNetworkUnavailableActionProcessor.java +++ b/app/src/main/java/org/thoughtcrime/securesms/service/webrtc/GroupNetworkUnavailableActionProcessor.java @@ -44,7 +44,7 @@ class GroupNetworkUnavailableActionProcessor extends WebRtcActionProcessor { byte[] groupId = currentState.getCallInfoState().getCallRecipient().requireGroupId().getDecodedId(); GroupCall groupCall = webRtcInteractor.getCallManager().createGroupCall(groupId, SignalStore.internalValues().groupCallingServer(), - currentState.getVideoState().requireEglBase(), + currentState.getVideoState().getLockableEglBase().require(), webRtcInteractor.getGroupCallObserver()); return currentState.builder() diff --git a/app/src/main/java/org/thoughtcrime/securesms/service/webrtc/GroupPreJoinActionProcessor.java b/app/src/main/java/org/thoughtcrime/securesms/service/webrtc/GroupPreJoinActionProcessor.java index 43a9b7af3..d1f848477 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/service/webrtc/GroupPreJoinActionProcessor.java +++ b/app/src/main/java/org/thoughtcrime/securesms/service/webrtc/GroupPreJoinActionProcessor.java @@ -45,7 +45,7 @@ public class GroupPreJoinActionProcessor extends GroupActionProcessor { byte[] groupId = currentState.getCallInfoState().getCallRecipient().requireGroupId().getDecodedId(); GroupCall groupCall = webRtcInteractor.getCallManager().createGroupCall(groupId, SignalStore.internalValues().groupCallingServer(), - currentState.getVideoState().requireEglBase(), + currentState.getVideoState().getLockableEglBase().require(), webRtcInteractor.getGroupCallObserver()); try { diff --git a/app/src/main/java/org/thoughtcrime/securesms/service/webrtc/IncomingCallActionProcessor.java b/app/src/main/java/org/thoughtcrime/securesms/service/webrtc/IncomingCallActionProcessor.java index 3f8c36347..d3deaf591 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/service/webrtc/IncomingCallActionProcessor.java +++ b/app/src/main/java/org/thoughtcrime/securesms/service/webrtc/IncomingCallActionProcessor.java @@ -86,7 +86,7 @@ public class IncomingCallActionProcessor extends DeviceAwareActionProcessor { try { webRtcInteractor.getCallManager().proceed(activePeer.getCallId(), context, - videoState.requireEglBase(), + videoState.getLockableEglBase().require(), videoState.requireLocalSink(), callParticipant.getVideoSink(), videoState.requireCamera(), diff --git a/app/src/main/java/org/thoughtcrime/securesms/service/webrtc/OutgoingCallActionProcessor.java b/app/src/main/java/org/thoughtcrime/securesms/service/webrtc/OutgoingCallActionProcessor.java index 8e6e75444..87bcb1210 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/service/webrtc/OutgoingCallActionProcessor.java +++ b/app/src/main/java/org/thoughtcrime/securesms/service/webrtc/OutgoingCallActionProcessor.java @@ -118,7 +118,7 @@ public class OutgoingCallActionProcessor extends DeviceAwareActionProcessor { webRtcInteractor.getCallManager().proceed(activePeer.getCallId(), context, - videoState.requireEglBase(), + videoState.getLockableEglBase().require(), videoState.requireLocalSink(), callParticipant.getVideoSink(), videoState.requireCamera(), diff --git a/app/src/main/java/org/thoughtcrime/securesms/service/webrtc/WebRtcVideoUtil.java b/app/src/main/java/org/thoughtcrime/securesms/service/webrtc/WebRtcVideoUtil.java index 12cd7a6ac..8b422d303 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/service/webrtc/WebRtcVideoUtil.java +++ b/app/src/main/java/org/thoughtcrime/securesms/service/webrtc/WebRtcVideoUtil.java @@ -6,6 +6,7 @@ import androidx.annotation.NonNull; import org.signal.core.util.ThreadUtil; import org.thoughtcrime.securesms.components.webrtc.BroadcastVideoSink; +import org.thoughtcrime.securesms.components.webrtc.EglBaseWrapper; import org.thoughtcrime.securesms.ringrtc.Camera; import org.thoughtcrime.securesms.ringrtc.CameraEventListener; import org.thoughtcrime.securesms.ringrtc.CameraState; @@ -31,7 +32,7 @@ public final class WebRtcVideoUtil { final WebRtcServiceStateBuilder builder = currentState.builder(); ThreadUtil.runOnMainSync(() -> { - EglBase eglBase = EglBase.create(); + EglBaseWrapper eglBase = new EglBaseWrapper(EglBase.create()); BroadcastVideoSink localSink = new BroadcastVideoSink(eglBase, true, false, @@ -66,7 +67,7 @@ public final class WebRtcVideoUtil { camera = new Camera(context, cameraEventListener, - currentState.getVideoState().requireEglBase(), + currentState.getVideoState().getLockableEglBase(), currentState.getLocalDeviceState().getCameraState().getActiveDirection()); camera.setOrientation(currentState.getLocalDeviceState().getOrientation().getDegrees()); @@ -88,10 +89,7 @@ public final class WebRtcVideoUtil { camera.dispose(); } - EglBase eglBase = currentState.getVideoState().getEglBase(); - if (eglBase != null) { - eglBase.release(); - } + currentState.getVideoState().getLockableEglBase().releaseEglBase(); return currentState.builder() .changeVideoState() diff --git a/app/src/main/java/org/thoughtcrime/securesms/service/webrtc/state/VideoState.java b/app/src/main/java/org/thoughtcrime/securesms/service/webrtc/state/VideoState.java index 4fd127876..f850b7ed8 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/service/webrtc/state/VideoState.java +++ b/app/src/main/java/org/thoughtcrime/securesms/service/webrtc/state/VideoState.java @@ -4,8 +4,8 @@ import androidx.annotation.NonNull; import androidx.annotation.Nullable; import org.thoughtcrime.securesms.components.webrtc.BroadcastVideoSink; +import org.thoughtcrime.securesms.components.webrtc.EglBaseWrapper; import org.thoughtcrime.securesms.ringrtc.Camera; -import org.webrtc.EglBase; import java.util.Objects; @@ -13,7 +13,7 @@ import java.util.Objects; * Local device video state and infrastructure. */ public final class VideoState { - EglBase eglBase; + EglBaseWrapper eglBase; BroadcastVideoSink localSink; Camera camera; @@ -25,20 +25,16 @@ public final class VideoState { this(toCopy.eglBase, toCopy.localSink, toCopy.camera); } - VideoState(@Nullable EglBase eglBase, @Nullable BroadcastVideoSink localSink, @Nullable Camera camera) { + VideoState(@NonNull EglBaseWrapper eglBase, @Nullable BroadcastVideoSink localSink, @Nullable Camera camera) { this.eglBase = eglBase; this.localSink = localSink; this.camera = camera; } - public @Nullable EglBase getEglBase() { + public @NonNull EglBaseWrapper getLockableEglBase() { return eglBase; } - public @NonNull EglBase requireEglBase() { - return Objects.requireNonNull(eglBase); - } - public @Nullable BroadcastVideoSink getLocalSink() { return localSink; } diff --git a/app/src/main/java/org/thoughtcrime/securesms/service/webrtc/state/WebRtcServiceStateBuilder.java b/app/src/main/java/org/thoughtcrime/securesms/service/webrtc/state/WebRtcServiceStateBuilder.java index 100c198ce..e78f2826b 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/service/webrtc/state/WebRtcServiceStateBuilder.java +++ b/app/src/main/java/org/thoughtcrime/securesms/service/webrtc/state/WebRtcServiceStateBuilder.java @@ -8,6 +8,7 @@ import com.annimon.stream.OptionalLong; import org.signal.ringrtc.GroupCall; import org.thoughtcrime.securesms.components.sensors.Orientation; import org.thoughtcrime.securesms.components.webrtc.BroadcastVideoSink; +import org.thoughtcrime.securesms.components.webrtc.EglBaseWrapper; import org.thoughtcrime.securesms.events.CallParticipant; import org.thoughtcrime.securesms.events.CallParticipantId; import org.thoughtcrime.securesms.events.WebRtcViewModel; @@ -17,7 +18,6 @@ import org.thoughtcrime.securesms.ringrtc.Camera; import org.thoughtcrime.securesms.ringrtc.CameraState; import org.thoughtcrime.securesms.ringrtc.RemotePeer; import org.thoughtcrime.securesms.service.webrtc.WebRtcActionProcessor; -import org.webrtc.EglBase; import java.util.Collection; @@ -177,7 +177,7 @@ public class WebRtcServiceStateBuilder { return WebRtcServiceStateBuilder.this.build(); } - public @NonNull VideoStateBuilder eglBase(@Nullable EglBase eglBase) { + public @NonNull VideoStateBuilder eglBase(@Nullable EglBaseWrapper eglBase) { toBuild.eglBase = eglBase; return this; }