diff --git a/app/src/main/java/org/thoughtcrime/securesms/calls/WebRtcCallActivity.kt b/app/src/main/java/org/thoughtcrime/securesms/calls/WebRtcCallActivity.kt index 522ef10140..8d14d6a6aa 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/calls/WebRtcCallActivity.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/calls/WebRtcCallActivity.kt @@ -5,6 +5,7 @@ import android.content.BroadcastReceiver import android.content.Context import android.content.Intent import android.content.IntentFilter +import android.graphics.Outline import android.hardware.Sensor import android.hardware.SensorEvent import android.hardware.SensorEventListener @@ -14,7 +15,11 @@ import android.os.Build import android.os.Bundle import android.provider.Settings import android.view.MenuItem +import android.view.View +import android.view.ViewGroup +import android.view.ViewOutlineProvider import android.view.WindowManager +import android.widget.FrameLayout import androidx.activity.viewModels import androidx.core.content.ContextCompat import androidx.core.view.isVisible @@ -49,6 +54,7 @@ import org.thoughtcrime.securesms.webrtc.CallViewModel.State.CALL_RINGING import org.thoughtcrime.securesms.webrtc.Orientation import org.thoughtcrime.securesms.webrtc.audio.SignalAudioManager.AudioDevice.EARPIECE import org.thoughtcrime.securesms.webrtc.audio.SignalAudioManager.AudioDevice.SPEAKER_PHONE +import org.webrtc.RendererCommon import kotlin.math.asin @AndroidEntryPoint @@ -137,9 +143,7 @@ class WebRtcCallActivity : PassphraseRequiredActionBarActivity(), SensorEventLis } binding.floatingRendererContainer.setOnClickListener { - val swapVideoViewIntent = - WebRtcCallService.swapVideoViews(this, viewModel.toggleVideoSwap()) - startService(swapVideoViewIntent) + viewModel.swapVideos() } binding.microphoneButton.setOnClickListener { @@ -180,7 +184,7 @@ class WebRtcCallActivity : PassphraseRequiredActionBarActivity(), SensorEventLis Permissions.with(this) .request(Manifest.permission.CAMERA) .onAllGranted { - val intent = WebRtcCallService.cameraEnabled(this, !viewModel.videoEnabled) + val intent = WebRtcCallService.cameraEnabled(this, !viewModel.videoState.value.userVideoEnabled) startService(intent) } .execute() @@ -197,6 +201,26 @@ class WebRtcCallActivity : PassphraseRequiredActionBarActivity(), SensorEventLis onBackPressed() } + clipFloatingInsets() + } + + /** + * Makes sure the floating video inset has clipped rounded corners, included with the video stream itself + */ + private fun clipFloatingInsets() { + // clip the video inset with rounded corners + val videoInsetProvider = object : ViewOutlineProvider() { + override fun getOutline(view: View, outline: Outline) { + // all corners + outline.setRoundRect( + 0, 0, view.width, view.height, + resources.getDimensionPixelSize(R.dimen.video_inset_radius).toFloat() + ) + } + } + + binding.floatingRendererContainer.outlineProvider = videoInsetProvider + binding.floatingRendererContainer.clipToOutline = true } //Function to check if Android System Auto-rotate is on or off @@ -216,7 +240,11 @@ class WebRtcCallActivity : PassphraseRequiredActionBarActivity(), SensorEventLis override fun onPause() { super.onPause() - sensorManager.unregisterListener(this) + try { + sensorManager.unregisterListener(this) + } catch (e: Exception) { + // the unregister can throw if the activity dies too quickly and the sensorManager is not initialised yet + } } override fun onSensorChanged(event: SensorEvent) { @@ -394,33 +422,37 @@ class WebRtcCallActivity : PassphraseRequiredActionBarActivity(), SensorEventLis } } + // handle video windows launch { - viewModel.localVideoEnabledState.collect { isEnabled -> + viewModel.videoState.collect { state -> binding.floatingRenderer.removeAllViews() - if (isEnabled) { - viewModel.floatingRenderer?.let { surfaceView -> - surfaceView.setZOrderOnTop(true) - binding.floatingRenderer.addView(surfaceView) - } - } - - binding.floatingRenderer.isVisible = isEnabled - binding.enableCameraButton.isSelected = isEnabled - //binding.swapViewIcon.bringToFront() - } - } - - launch { - viewModel.remoteVideoEnabledState.collect { isEnabled -> binding.fullscreenRenderer.removeAllViews() - if (isEnabled) { + + // handle fullscreen video window + if(state.showFullscreenVideo()){ viewModel.fullscreenRenderer?.let { surfaceView -> binding.fullscreenRenderer.addView(surfaceView) + binding.fullscreenRenderer.isVisible = true + binding.remoteRecipient.isVisible = false } + } else { + binding.fullscreenRenderer.isVisible = false + binding.remoteRecipient.isVisible = true } - binding.fullscreenRenderer.isVisible = isEnabled - binding.remoteRecipient.isVisible = !isEnabled - //binding.swapViewIcon.bringToFront() + + // handle floating video window + if(state.showFloatingVideo()){ + viewModel.floatingRenderer?.let { surfaceView -> + binding.floatingRenderer.addView(surfaceView) + binding.floatingRenderer.isVisible = true + binding.swapViewIcon.bringToFront() + } + } else { + binding.floatingRenderer.isVisible = false + } + + // handle buttons + binding.enableCameraButton.isSelected = state.userVideoEnabled } } } diff --git a/app/src/main/java/org/thoughtcrime/securesms/service/WebRtcCallService.kt b/app/src/main/java/org/thoughtcrime/securesms/service/WebRtcCallService.kt index ef2ebf9382..36106123a8 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/service/WebRtcCallService.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/service/WebRtcCallService.kt @@ -62,7 +62,6 @@ class WebRtcCallService : LifecycleService(), CallManager.WebRtcListener { const val ACTION_LOCAL_HANGUP = "LOCAL_HANGUP" const val ACTION_SET_MUTE_AUDIO = "SET_MUTE_AUDIO" const val ACTION_SET_MUTE_VIDEO = "SET_MUTE_VIDEO" - const val ACTION_SWAP_VIDEO_VIEW = "SWAP_VIDEO_VIEW" const val ACTION_FLIP_CAMERA = "FLIP_CAMERA" const val ACTION_UPDATE_AUDIO = "UPDATE_AUDIO" const val ACTION_WIRED_HEADSET_CHANGE = "WIRED_HEADSET_CHANGE" @@ -110,11 +109,6 @@ class WebRtcCallService : LifecycleService(), CallManager.WebRtcListener { fun acceptCallIntent(context: Context) = Intent(context, WebRtcCallService::class.java) .setAction(ACTION_ANSWER_CALL) - fun swapVideoViews(context: Context, swapped: Boolean) = - Intent(context, WebRtcCallService::class.java) - .setAction(ACTION_SWAP_VIDEO_VIEW) - .putExtra(EXTRA_SWAPPED, swapped) - fun microphoneIntent(context: Context, enabled: Boolean) = Intent(context, WebRtcCallService::class.java) .setAction(ACTION_SET_MUTE_AUDIO) @@ -299,7 +293,6 @@ class WebRtcCallService : LifecycleService(), CallManager.WebRtcListener { action == ACTION_DENY_CALL -> handleDenyCall(intent) action == ACTION_LOCAL_HANGUP -> handleLocalHangup(intent) action == ACTION_REMOTE_HANGUP -> handleRemoteHangup(intent) - action == ACTION_SWAP_VIDEO_VIEW ->handleSwapVideoView(intent) action == ACTION_SET_MUTE_AUDIO -> handleSetMuteAudio(intent) action == ACTION_SET_MUTE_VIDEO -> handleSetMuteVideo(intent) action == ACTION_FLIP_CAMERA -> handleSetCameraFlip(intent) @@ -602,11 +595,6 @@ class WebRtcCallService : LifecycleService(), CallManager.WebRtcListener { onHangup() } - private fun handleSwapVideoView(intent: Intent) { - val swapped = intent.getBooleanExtra(EXTRA_SWAPPED, false) - callManager.handleSwapVideoView(swapped) - } - private fun handleSetMuteAudio(intent: Intent) { val muted = intent.getBooleanExtra(EXTRA_MUTE, false) callManager.handleSetMuteAudio(muted) diff --git a/app/src/main/java/org/thoughtcrime/securesms/webrtc/CallManager.kt b/app/src/main/java/org/thoughtcrime/securesms/webrtc/CallManager.kt index 3c057c2a71..6a46722f18 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/webrtc/CallManager.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/webrtc/CallManager.kt @@ -6,6 +6,7 @@ import android.telephony.TelephonyManager import androidx.core.content.ContextCompat import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.asSharedFlow +import kotlinx.coroutines.flow.asStateFlow import kotlinx.serialization.json.Json import kotlinx.serialization.json.JsonObject import kotlinx.serialization.json.JsonPrimitive @@ -51,6 +52,7 @@ import org.webrtc.MediaStream import org.webrtc.PeerConnection import org.webrtc.PeerConnection.IceConnectionState import org.webrtc.PeerConnectionFactory +import org.webrtc.RendererCommon import org.webrtc.RtpReceiver import org.webrtc.SessionDescription import org.webrtc.SurfaceViewRenderer @@ -67,7 +69,6 @@ class CallManager( SignalAudioManager.EventListener, CameraEventListener, DataChannel.Observer { sealed class StateEvent { - data class VideoSwapped(val isSwapped: Boolean): StateEvent() data class AudioEnabled(val isEnabled: Boolean): StateEvent() data class VideoEnabled(val isEnabled: Boolean): StateEvent() data class CallStateUpdate(val state: CallState): StateEvent() @@ -106,10 +107,15 @@ class CallManager( private val _audioEvents = MutableStateFlow(AudioEnabled(false)) val audioEvents = _audioEvents.asSharedFlow() - private val _videoEvents = MutableStateFlow(VideoEnabled(false)) - val videoEvents = _videoEvents.asSharedFlow() - private val _remoteVideoEvents = MutableStateFlow(VideoEnabled(false)) - val remoteVideoEvents = _remoteVideoEvents.asSharedFlow() + + private val _videoState: MutableStateFlow = MutableStateFlow( + VideoState( + swapped = false, + userVideoEnabled = false, + remoteVideoEnabled = false + ) + ) + val videoState = _videoState private val stateProcessor = StateProcessor(CallState.Idle) @@ -221,8 +227,10 @@ class CallManager( val base = EglBase.create() eglBase = base floatingRenderer = SurfaceViewRenderer(context) + floatingRenderer?.setScalingType(RendererCommon.ScalingType.SCALE_ASPECT_FIT) fullscreenRenderer = SurfaceViewRenderer(context) + fullscreenRenderer?.setScalingType(RendererCommon.ScalingType.SCALE_ASPECT_FIT) remoteRotationSink = RemoteRotationVideoProxySink() @@ -363,7 +371,8 @@ class CallManager( val byteArray = ByteArray(buffer.data.remaining()) { buffer.data[it] } val json = Json.parseToJsonElement(byteArray.decodeToString()) as JsonObject if (json.containsKey("video")) { - _remoteVideoEvents.value = VideoEnabled((json["video"] as JsonPrimitive).boolean) + _videoState.value = _videoState.value.copy(remoteVideoEnabled = (json["video"] as JsonPrimitive).boolean) + handleMirroring() } else if (json.containsKey("hangup")) { peerConnectionObservers.forEach(WebRtcListener::onHangup) } @@ -399,8 +408,11 @@ class CallManager( pendingOffer = null callStartTime = -1 _audioEvents.value = AudioEnabled(false) - _videoEvents.value = VideoEnabled(false) - _remoteVideoEvents.value = VideoEnabled(false) + _videoState.value = VideoState( + swapped = false, + userVideoEnabled = false, + remoteVideoEnabled = false + ) pendingOutgoingIceUpdates.clear() pendingIncomingIceUpdates.clear() } @@ -411,7 +423,7 @@ class CallManager( // If the camera we've switched to is the front one then mirror it to match what someone // would see when looking in the mirror rather than the left<-->right flipped version. - handleUserMirroring() + handleMirroring() } fun onPreOffer(callId: UUID, recipient: Recipient, onSuccess: () -> Unit) { @@ -609,10 +621,19 @@ class CallManager( } } - fun handleSwapVideoView(swapped: Boolean) { - videoSwapped = swapped + fun swapVideos() { + videoSwapped = !videoSwapped - if (!swapped) { + // update the state + _videoState.value = _videoState.value.copy(swapped = videoSwapped) + handleMirroring() + +//todo TOM received rotated video shouldn't be full scale + //todo TOM make sure the swap icon is visible + //todo TOM Should we show the 'no video' inset straight away? + //todo TOM ios rotates the controls in landscape ( just the buttons though, not the whole ui??) + + if (!videoSwapped) { peerConnection?.rotationVideoSink?.apply { setSink(floatingRenderer) } @@ -633,18 +654,31 @@ class CallManager( */ private fun getUserRenderer() = if(videoSwapped) fullscreenRenderer else floatingRenderer + /** + * Returns the renderer currently showing the contact's video, not the user's + */ + private fun getRemoteRenderer() = if(videoSwapped) floatingRenderer else fullscreenRenderer + /** * Makes sure the user's renderer applies mirroring if necessary */ - private fun handleUserMirroring() = getUserRenderer()?.setMirror(isCameraFrontFacing()) + private fun handleMirroring() { + val videoState = _videoState.value + + // if we have user video and the camera is front facing, make sure to mirror stream + if(videoState.userVideoEnabled) { + getUserRenderer()?.setMirror(isCameraFrontFacing()) + } + + // the remote video is never mirrored + if(videoState.remoteVideoEnabled){ + getRemoteRenderer()?.setMirror(false) + } + } fun handleSetMuteVideo(muted: Boolean, lockManager: LockManager) { - _videoEvents.value = VideoEnabled(!muted) - - // if we have video and the camera is not front facing, make sure to mirror stream - if(!muted){ - handleUserMirroring() - } + _videoState.value = _videoState.value.copy(userVideoEnabled = !muted) + handleMirroring() val connection = peerConnection ?: return connection.setVideoEnabled(!muted) @@ -761,7 +795,7 @@ class CallManager( connection.setCommunicationMode() setAudioEnabled(true) dataChannel?.let { channel -> - val toSend = if (!_videoEvents.value.isEnabled) VIDEO_DISABLED_JSON else VIDEO_ENABLED_JSON + val toSend = if (!_videoState.value.userVideoEnabled) VIDEO_DISABLED_JSON else VIDEO_ENABLED_JSON val buffer = DataChannel.Buffer(ByteBuffer.wrap(toSend.toString().encodeToByteArray()), false) channel.send(buffer) } @@ -790,7 +824,7 @@ class CallManager( fun isInitiator(): Boolean = peerConnection?.isInitiator() == true - fun isCameraFrontFacing() = localCameraState.activeDirection == CameraState.Direction.FRONT + fun isCameraFrontFacing() = localCameraState.activeDirection != CameraState.Direction.BACK interface WebRtcListener: PeerConnection.Observer { fun onHangup() diff --git a/app/src/main/java/org/thoughtcrime/securesms/webrtc/CallViewModel.kt b/app/src/main/java/org/thoughtcrime/securesms/webrtc/CallViewModel.kt index e2e82a418a..f49e2d3333 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/webrtc/CallViewModel.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/webrtc/CallViewModel.kt @@ -2,6 +2,8 @@ package org.thoughtcrime.securesms.webrtc import androidx.lifecycle.ViewModel import dagger.hilt.android.lifecycle.HiltViewModel +import kotlinx.coroutines.flow.SharedFlow +import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.map import kotlinx.coroutines.flow.onEach import org.thoughtcrime.securesms.webrtc.audio.SignalAudioManager @@ -35,15 +37,6 @@ class CallViewModel @Inject constructor(private val callManager: CallManager): V val fullscreenRenderer: SurfaceViewRenderer? get() = callManager.fullscreenRenderer - private var _videoEnabled: Boolean = false - - val videoEnabled: Boolean - get() = _videoEnabled - - private var _remoteVideoEnabled: Boolean = false - - private var _videoViewSwapped: Boolean = false - private var _microphoneEnabled: Boolean = true val microphoneEnabled: Boolean @@ -63,15 +56,8 @@ class CallViewModel @Inject constructor(private val callManager: CallManager): V get() = callManager.audioEvents.map { it.isEnabled } .onEach { _microphoneEnabled = it } - val localVideoEnabledState - get() = callManager.videoEvents - .map { it.isEnabled } - .onEach { _videoEnabled = it } - - val remoteVideoEnabledState - get() = callManager.remoteVideoEvents - .map { it.isEnabled } - .onEach { _remoteVideoEnabled = it } + val videoState: StateFlow + get() = callManager.videoState var deviceOrientation: Orientation = Orientation.UNKNOWN set(value) { @@ -91,11 +77,7 @@ class CallViewModel @Inject constructor(private val callManager: CallManager): V val callStartTime: Long get() = callManager.callStartTime - /** - * Toggles the video swapped state, and return the value post toggle - */ - fun toggleVideoSwap(): Boolean { - _videoViewSwapped = !_videoViewSwapped - return _videoViewSwapped + fun swapVideos() { + callManager.swapVideos() } } \ No newline at end of file diff --git a/app/src/main/java/org/thoughtcrime/securesms/webrtc/VideoState.kt b/app/src/main/java/org/thoughtcrime/securesms/webrtc/VideoState.kt new file mode 100644 index 0000000000..55bb04038a --- /dev/null +++ b/app/src/main/java/org/thoughtcrime/securesms/webrtc/VideoState.kt @@ -0,0 +1,17 @@ +package org.thoughtcrime.securesms.webrtc + +data class VideoState ( + val swapped: Boolean, + val userVideoEnabled: Boolean, + val remoteVideoEnabled: Boolean +){ + fun showFloatingVideo(): Boolean { + return userVideoEnabled && !swapped || + remoteVideoEnabled && swapped + } + + fun showFullscreenVideo(): Boolean { + return userVideoEnabled && swapped || + remoteVideoEnabled && !swapped + } +} \ No newline at end of file diff --git a/app/src/main/res/layout/activity_webrtc.xml b/app/src/main/res/layout/activity_webrtc.xml index 3aea4ffa11..c285616a11 100644 --- a/app/src/main/res/layout/activity_webrtc.xml +++ b/app/src/main/res/layout/activity_webrtc.xml @@ -20,7 +20,7 @@ + android:layout_height="wrap_content" + android:layout_gravity="center"/> - + + + 250dp 64dp 8dp + 11dp 4dp 8dp 8dp