Video management logic update

Rounded corners for floating inset
Proper handling of video scaling based on video proportions
Proper handling of mirroring logic for floating/fullscreen videos depending on whether they are the user or the remote video and whether the camera is front facing or not

app/src/main/java/org/thoughtcrime/securesms/calls/WebRtcCallActivity.kt

@@ -5,6 +5,7 @@ import android.content.BroadcastReceiver
 import android.content.Context
 import android.content.Intent
 import android.content.IntentFilter
+import android.graphics.Outline
 import android.hardware.Sensor
 import android.hardware.SensorEvent
 import android.hardware.SensorEventListener
@@ -14,7 +15,11 @@ import android.os.Build
 import android.os.Bundle
 import android.provider.Settings
 import android.view.MenuItem
+import android.view.View
+import android.view.ViewGroup
+import android.view.ViewOutlineProvider
 import android.view.WindowManager
+import android.widget.FrameLayout
 import androidx.activity.viewModels
 import androidx.core.content.ContextCompat
 import androidx.core.view.isVisible
@@ -49,6 +54,7 @@ import org.thoughtcrime.securesms.webrtc.CallViewModel.State.CALL_RINGING
 import org.thoughtcrime.securesms.webrtc.Orientation
 import org.thoughtcrime.securesms.webrtc.audio.SignalAudioManager.AudioDevice.EARPIECE
 import org.thoughtcrime.securesms.webrtc.audio.SignalAudioManager.AudioDevice.SPEAKER_PHONE
+import org.webrtc.RendererCommon
 import kotlin.math.asin
 
 @AndroidEntryPoint
@@ -137,9 +143,7 @@ class WebRtcCallActivity : PassphraseRequiredActionBarActivity(), SensorEventLis
         }
 
         binding.floatingRendererContainer.setOnClickListener {
-            val swapVideoViewIntent =
-                WebRtcCallService.swapVideoViews(this, viewModel.toggleVideoSwap())
-            startService(swapVideoViewIntent)
+            viewModel.swapVideos()
         }
 
         binding.microphoneButton.setOnClickListener {
@@ -180,7 +184,7 @@ class WebRtcCallActivity : PassphraseRequiredActionBarActivity(), SensorEventLis
             Permissions.with(this)
                 .request(Manifest.permission.CAMERA)
                 .onAllGranted {
-                    val intent = WebRtcCallService.cameraEnabled(this, !viewModel.videoEnabled)
+                    val intent = WebRtcCallService.cameraEnabled(this, !viewModel.videoState.value.userVideoEnabled)
                     startService(intent)
                 }
                 .execute()
@@ -197,6 +201,26 @@ class WebRtcCallActivity : PassphraseRequiredActionBarActivity(), SensorEventLis
             onBackPressed()
         }
 
+        clipFloatingInsets()
+    }
+
+    /**
+     * Makes sure the floating video inset has clipped rounded corners, included with the video stream itself
+     */
+    private fun clipFloatingInsets() {
+        // clip the video inset with rounded corners
+        val videoInsetProvider = object : ViewOutlineProvider() {
+            override fun getOutline(view: View, outline: Outline) {
+                // all corners
+                outline.setRoundRect(
+                    0, 0, view.width, view.height,
+                    resources.getDimensionPixelSize(R.dimen.video_inset_radius).toFloat()
+                )
+            }
+        }
+
+        binding.floatingRendererContainer.outlineProvider = videoInsetProvider
+        binding.floatingRendererContainer.clipToOutline = true
     }
 
     //Function to check if Android System Auto-rotate is on or off
@@ -216,7 +240,11 @@ class WebRtcCallActivity : PassphraseRequiredActionBarActivity(), SensorEventLis
 
     override fun onPause() {
         super.onPause()
-        sensorManager.unregisterListener(this)
+        try {
+            sensorManager.unregisterListener(this)
+        } catch (e: Exception) {
+            // the unregister can throw if the activity dies too quickly and the sensorManager is not initialised yet
+        }
     }
 
     override fun onSensorChanged(event: SensorEvent) {
@@ -394,33 +422,37 @@ class WebRtcCallActivity : PassphraseRequiredActionBarActivity(), SensorEventLis
                 }
             }
 
+            // handle video windows
             launch {
-                viewModel.localVideoEnabledState.collect { isEnabled ->
+                viewModel.videoState.collect { state ->
                     binding.floatingRenderer.removeAllViews()
-                    if (isEnabled) {
-                        viewModel.floatingRenderer?.let { surfaceView ->
-                            surfaceView.setZOrderOnTop(true)
-                            binding.floatingRenderer.addView(surfaceView)
-                        }
-                    }
-
-                    binding.floatingRenderer.isVisible = isEnabled
-                    binding.enableCameraButton.isSelected = isEnabled
-                    //binding.swapViewIcon.bringToFront()
-                }
-            }
-
-            launch {
-                viewModel.remoteVideoEnabledState.collect { isEnabled ->
                     binding.fullscreenRenderer.removeAllViews()
-                    if (isEnabled) {
+
+                    // handle fullscreen video window
+                    if(state.showFullscreenVideo()){
                         viewModel.fullscreenRenderer?.let { surfaceView ->
                             binding.fullscreenRenderer.addView(surfaceView)
+                            binding.fullscreenRenderer.isVisible = true
+                            binding.remoteRecipient.isVisible = false
                         }
+                    } else {
+                        binding.fullscreenRenderer.isVisible = false
+                        binding.remoteRecipient.isVisible = true
                     }
-                    binding.fullscreenRenderer.isVisible = isEnabled
-                    binding.remoteRecipient.isVisible = !isEnabled
-                    //binding.swapViewIcon.bringToFront()
+
+                    // handle floating video window
+                    if(state.showFloatingVideo()){
+                        viewModel.floatingRenderer?.let { surfaceView ->
+                            binding.floatingRenderer.addView(surfaceView)
+                            binding.floatingRenderer.isVisible = true
+                            binding.swapViewIcon.bringToFront()
+                        }
+                    } else {
+                        binding.floatingRenderer.isVisible = false
+                    }
+
+                    // handle buttons
+                    binding.enableCameraButton.isSelected = state.userVideoEnabled
                 }
             }
         }

app/src/main/java/org/thoughtcrime/securesms/service/WebRtcCallService.kt

@@ -62,7 +62,6 @@ class WebRtcCallService : LifecycleService(), CallManager.WebRtcListener {
         const val ACTION_LOCAL_HANGUP = "LOCAL_HANGUP"
         const val ACTION_SET_MUTE_AUDIO = "SET_MUTE_AUDIO"
         const val ACTION_SET_MUTE_VIDEO = "SET_MUTE_VIDEO"
-        const val ACTION_SWAP_VIDEO_VIEW = "SWAP_VIDEO_VIEW"
         const val ACTION_FLIP_CAMERA = "FLIP_CAMERA"
         const val ACTION_UPDATE_AUDIO = "UPDATE_AUDIO"
         const val ACTION_WIRED_HEADSET_CHANGE = "WIRED_HEADSET_CHANGE"
@@ -110,11 +109,6 @@ class WebRtcCallService : LifecycleService(), CallManager.WebRtcListener {
         fun acceptCallIntent(context: Context) = Intent(context, WebRtcCallService::class.java)
             .setAction(ACTION_ANSWER_CALL)
 
-        fun swapVideoViews(context: Context, swapped: Boolean) =
-             Intent(context, WebRtcCallService::class.java)
-                .setAction(ACTION_SWAP_VIDEO_VIEW)
-                .putExtra(EXTRA_SWAPPED, swapped)
-
         fun microphoneIntent(context: Context, enabled: Boolean) =
             Intent(context, WebRtcCallService::class.java)
                 .setAction(ACTION_SET_MUTE_AUDIO)
@@ -299,7 +293,6 @@ class WebRtcCallService : LifecycleService(), CallManager.WebRtcListener {
                 action == ACTION_DENY_CALL -> handleDenyCall(intent)
                 action == ACTION_LOCAL_HANGUP -> handleLocalHangup(intent)
                 action == ACTION_REMOTE_HANGUP -> handleRemoteHangup(intent)
-                action == ACTION_SWAP_VIDEO_VIEW ->handleSwapVideoView(intent)
                 action == ACTION_SET_MUTE_AUDIO -> handleSetMuteAudio(intent)
                 action == ACTION_SET_MUTE_VIDEO -> handleSetMuteVideo(intent)
                 action == ACTION_FLIP_CAMERA -> handleSetCameraFlip(intent)
@@ -602,11 +595,6 @@ class WebRtcCallService : LifecycleService(), CallManager.WebRtcListener {
         onHangup()
     }
 
-    private fun handleSwapVideoView(intent: Intent) {
-        val swapped = intent.getBooleanExtra(EXTRA_SWAPPED, false)
-        callManager.handleSwapVideoView(swapped)
-    }
-
     private fun handleSetMuteAudio(intent: Intent) {
         val muted = intent.getBooleanExtra(EXTRA_MUTE, false)
         callManager.handleSetMuteAudio(muted)

app/src/main/java/org/thoughtcrime/securesms/webrtc/CallManager.kt

@@ -6,6 +6,7 @@ import android.telephony.TelephonyManager
 import androidx.core.content.ContextCompat
 import kotlinx.coroutines.flow.MutableStateFlow
 import kotlinx.coroutines.flow.asSharedFlow
+import kotlinx.coroutines.flow.asStateFlow
 import kotlinx.serialization.json.Json
 import kotlinx.serialization.json.JsonObject
 import kotlinx.serialization.json.JsonPrimitive
@@ -51,6 +52,7 @@ import org.webrtc.MediaStream
 import org.webrtc.PeerConnection
 import org.webrtc.PeerConnection.IceConnectionState
 import org.webrtc.PeerConnectionFactory
+import org.webrtc.RendererCommon
 import org.webrtc.RtpReceiver
 import org.webrtc.SessionDescription
 import org.webrtc.SurfaceViewRenderer
@@ -67,7 +69,6 @@ class CallManager(
     SignalAudioManager.EventListener, CameraEventListener, DataChannel.Observer {
 
     sealed class StateEvent {
-        data class VideoSwapped(val isSwapped: Boolean): StateEvent()
         data class AudioEnabled(val isEnabled: Boolean): StateEvent()
         data class VideoEnabled(val isEnabled: Boolean): StateEvent()
         data class CallStateUpdate(val state: CallState): StateEvent()
@@ -106,10 +107,15 @@ class CallManager(
 
     private val _audioEvents = MutableStateFlow(AudioEnabled(false))
     val audioEvents = _audioEvents.asSharedFlow()
-    private val _videoEvents = MutableStateFlow(VideoEnabled(false))
-    val videoEvents = _videoEvents.asSharedFlow()
-    private val _remoteVideoEvents = MutableStateFlow(VideoEnabled(false))
-    val remoteVideoEvents = _remoteVideoEvents.asSharedFlow()
+
+    private val _videoState: MutableStateFlow<VideoState> = MutableStateFlow(
+        VideoState(
+            swapped = false,
+            userVideoEnabled = false,
+            remoteVideoEnabled = false
+        )
+    )
+    val videoState = _videoState
 
     private val stateProcessor = StateProcessor(CallState.Idle)
 
@@ -221,8 +227,10 @@ class CallManager(
             val base = EglBase.create()
             eglBase = base
             floatingRenderer = SurfaceViewRenderer(context)
+            floatingRenderer?.setScalingType(RendererCommon.ScalingType.SCALE_ASPECT_FIT)
 
             fullscreenRenderer = SurfaceViewRenderer(context)
+            fullscreenRenderer?.setScalingType(RendererCommon.ScalingType.SCALE_ASPECT_FIT)
 
             remoteRotationSink = RemoteRotationVideoProxySink()
 
@@ -363,7 +371,8 @@ class CallManager(
             val byteArray = ByteArray(buffer.data.remaining()) { buffer.data[it] }
             val json = Json.parseToJsonElement(byteArray.decodeToString()) as JsonObject
             if (json.containsKey("video")) {
-                _remoteVideoEvents.value = VideoEnabled((json["video"] as JsonPrimitive).boolean)
+                _videoState.value = _videoState.value.copy(remoteVideoEnabled = (json["video"] as JsonPrimitive).boolean)
+                handleMirroring()
             } else if (json.containsKey("hangup")) {
                 peerConnectionObservers.forEach(WebRtcListener::onHangup)
             }
@@ -399,8 +408,11 @@ class CallManager(
             pendingOffer = null
             callStartTime = -1
             _audioEvents.value = AudioEnabled(false)
-            _videoEvents.value = VideoEnabled(false)
-            _remoteVideoEvents.value = VideoEnabled(false)
+            _videoState.value = VideoState(
+                swapped = false,
+                userVideoEnabled = false,
+                remoteVideoEnabled = false
+            )
             pendingOutgoingIceUpdates.clear()
             pendingIncomingIceUpdates.clear()
         }
@@ -411,7 +423,7 @@ class CallManager(
 
         // If the camera we've switched to is the front one then mirror it to match what someone
         // would see when looking in the mirror rather than the left<-->right flipped version.
-       handleUserMirroring()
+       handleMirroring()
     }
 
     fun onPreOffer(callId: UUID, recipient: Recipient, onSuccess: () -> Unit) {
@@ -609,10 +621,19 @@ class CallManager(
         }
     }
 
-    fun handleSwapVideoView(swapped: Boolean) {
-        videoSwapped = swapped
+    fun swapVideos() {
+        videoSwapped = !videoSwapped
 
-        if (!swapped) {
+        // update the state
+        _videoState.value = _videoState.value.copy(swapped = videoSwapped)
+        handleMirroring()
+
+//todo TOM received rotated video shouldn't be full scale
+        //todo TOM make sure the swap icon is visible
+        //todo TOM Should we show the 'no video' inset straight away?
+        //todo TOM ios rotates the controls in landscape ( just the buttons though, not the whole ui??)
+
+        if (!videoSwapped) {
             peerConnection?.rotationVideoSink?.apply {
                 setSink(floatingRenderer)
             }
@@ -633,18 +654,31 @@ class CallManager(
      */
     private fun getUserRenderer() = if(videoSwapped) fullscreenRenderer else floatingRenderer
 
+    /**
+     * Returns the renderer currently showing the contact's video, not the user's
+     */
+    private fun getRemoteRenderer() = if(videoSwapped) floatingRenderer else fullscreenRenderer
+
     /**
      * Makes sure the user's renderer applies mirroring if necessary
      */
-    private fun handleUserMirroring() = getUserRenderer()?.setMirror(isCameraFrontFacing())
+    private fun handleMirroring() {
+        val videoState = _videoState.value
+
+        // if we have user video and the camera is front facing, make sure to mirror stream
+        if(videoState.userVideoEnabled) {
+            getUserRenderer()?.setMirror(isCameraFrontFacing())
+        }
+
+        // the remote video is never mirrored
+        if(videoState.remoteVideoEnabled){
+            getRemoteRenderer()?.setMirror(false)
+        }
+    }
 
     fun handleSetMuteVideo(muted: Boolean, lockManager: LockManager) {
-        _videoEvents.value = VideoEnabled(!muted)
-
-        // if we have video and the camera is not front facing, make sure to mirror stream
-        if(!muted){
-            handleUserMirroring()
-        }
+        _videoState.value = _videoState.value.copy(userVideoEnabled = !muted)
+        handleMirroring()
 
         val connection = peerConnection ?: return
         connection.setVideoEnabled(!muted)
@@ -761,7 +795,7 @@ class CallManager(
         connection.setCommunicationMode()
         setAudioEnabled(true)
         dataChannel?.let { channel ->
-            val toSend = if (!_videoEvents.value.isEnabled) VIDEO_DISABLED_JSON else VIDEO_ENABLED_JSON
+            val toSend = if (!_videoState.value.userVideoEnabled) VIDEO_DISABLED_JSON else VIDEO_ENABLED_JSON
             val buffer = DataChannel.Buffer(ByteBuffer.wrap(toSend.toString().encodeToByteArray()), false)
             channel.send(buffer)
         }
@@ -790,7 +824,7 @@ class CallManager(
 
     fun isInitiator(): Boolean = peerConnection?.isInitiator() == true
 
-    fun isCameraFrontFacing() = localCameraState.activeDirection == CameraState.Direction.FRONT
+    fun isCameraFrontFacing() = localCameraState.activeDirection != CameraState.Direction.BACK
 
     interface WebRtcListener: PeerConnection.Observer {
         fun onHangup()

app/src/main/java/org/thoughtcrime/securesms/webrtc/CallViewModel.kt

@@ -2,6 +2,8 @@ package org.thoughtcrime.securesms.webrtc
 
 import androidx.lifecycle.ViewModel
 import dagger.hilt.android.lifecycle.HiltViewModel
+import kotlinx.coroutines.flow.SharedFlow
+import kotlinx.coroutines.flow.StateFlow
 import kotlinx.coroutines.flow.map
 import kotlinx.coroutines.flow.onEach
 import org.thoughtcrime.securesms.webrtc.audio.SignalAudioManager
@@ -35,15 +37,6 @@ class CallViewModel @Inject constructor(private val callManager: CallManager): V
     val fullscreenRenderer: SurfaceViewRenderer?
     get() = callManager.fullscreenRenderer
 
-    private var _videoEnabled: Boolean = false
-
-    val videoEnabled: Boolean
-        get() = _videoEnabled
-
-    private var _remoteVideoEnabled: Boolean = false
-
-    private var _videoViewSwapped: Boolean = false
-
     private var _microphoneEnabled: Boolean = true
 
     val microphoneEnabled: Boolean
@@ -63,15 +56,8 @@ class CallViewModel @Inject constructor(private val callManager: CallManager): V
         get() = callManager.audioEvents.map { it.isEnabled }
             .onEach { _microphoneEnabled = it }
 
-    val localVideoEnabledState
-        get() = callManager.videoEvents
-                .map { it.isEnabled }
-                .onEach { _videoEnabled = it }
-
-    val remoteVideoEnabledState
-        get() = callManager.remoteVideoEvents
-                .map { it.isEnabled }
-                .onEach { _remoteVideoEnabled = it }
+    val videoState: StateFlow<VideoState>
+        get() = callManager.videoState
 
     var deviceOrientation: Orientation = Orientation.UNKNOWN
         set(value) {
@@ -91,11 +77,7 @@ class CallViewModel @Inject constructor(private val callManager: CallManager): V
     val callStartTime: Long
         get() = callManager.callStartTime
 
-    /**
-     * Toggles the video swapped state, and return the value post toggle
-     */
-    fun toggleVideoSwap(): Boolean {
-        _videoViewSwapped = !_videoViewSwapped
-        return _videoViewSwapped
+    fun swapVideos() {
+       callManager.swapVideos()
     }
 }

app/src/main/java/org/thoughtcrime/securesms/webrtc/VideoState.kt

@@ -0,0 +1,17 @@
+package org.thoughtcrime.securesms.webrtc
+
+data class VideoState (
+    val swapped: Boolean,
+    val userVideoEnabled: Boolean,
+    val remoteVideoEnabled: Boolean
+){
+    fun showFloatingVideo(): Boolean {
+        return userVideoEnabled && !swapped ||
+                remoteVideoEnabled && swapped
+    }
+
+    fun showFullscreenVideo(): Boolean {
+        return userVideoEnabled && swapped ||
+                remoteVideoEnabled && !swapped
+    }
+}

app/src/main/res/layout/activity_webrtc.xml

@@ -20,7 +20,7 @@
         <FrameLayout
             android:id="@+id/fullscreen_renderer"
             android:layout_width="match_parent"
-            android:layout_height="match_parent"
+            android:layout_height="wrap_content"
             android:layout_gravity="center"/>
     </FrameLayout>
     <ImageView
@@ -132,7 +132,8 @@
             android:elevation="8dp"
             android:id="@+id/floating_renderer"
             android:layout_width="match_parent"
-            android:layout_height="match_parent"/>
+            android:layout_height="wrap_content"
+            android:layout_gravity="center"/>
         <com.github.ybq.android.spinkit.SpinKitView
             android:id="@+id/local_loading_view"
             style="@style/SpinKitView.Large.ThreeBounce"
@@ -142,19 +143,20 @@
             android:layout_gravity="center"
             tools:visibility="visible"
             android:visibility="gone" />
-        <ImageView
-            android:id="@+id/swap_view_icon"
-            android:src="@drawable/ic_baseline_screen_rotation_alt_24"
-            app:layout_constraintEnd_toEndOf="parent"
-            app:layout_constraintTop_toTopOf="parent"
-            app:tint="?android:textColorPrimary"
-            android:layout_marginTop="@dimen/very_small_spacing"
-            android:layout_marginEnd="@dimen/very_small_spacing"
-            android:layout_gravity="end"
-            android:layout_width="14dp"
-            android:layout_height="14dp"/>
+
     </FrameLayout>
 
+    <ImageView
+        android:id="@+id/swap_view_icon"
+        android:src="@drawable/ic_baseline_screen_rotation_alt_24"
+        app:layout_constraintTop_toTopOf="@id/floating_renderer_container"
+        app:layout_constraintEnd_toEndOf="@id/floating_renderer_container"
+        app:tint="?android:textColorPrimary"
+        android:layout_marginTop="@dimen/very_small_spacing"
+        android:layout_marginEnd="@dimen/very_small_spacing"
+        android:layout_width="14dp"
+        android:layout_height="14dp"/>
+
     <ImageView
         android:id="@+id/endCallButton"
         android:background="@drawable/circle_tintable"

app/src/main/res/values/dimens.xml

@@ -32,6 +32,7 @@
     <dimen name="fake_chat_view_height">250dp</dimen>
     <dimen name="setting_button_height">64dp</dimen>
     <dimen name="dialog_corner_radius">8dp</dimen>
+    <dimen name="video_inset_radius">11dp</dimen>
     <dimen name="dialog_button_corner_radius">4dp</dimen>
     <dimen name="pn_option_corner_radius">8dp</dimen>
     <dimen name="path_status_view_size">8dp</dimen>