feat: call establishing and displaying

2025-08-24 22:17:25 +00:00 · 2021-11-10 11:57:03 +11:00
parent 2e973146a3
commit 99b6a38b90
9 changed files with 153 additions and 61 deletions
--- a/app/src/main/java/org/thoughtcrime/securesms/calls/WebRtcCallActivity.kt
+++ b/app/src/main/java/org/thoughtcrime/securesms/calls/WebRtcCallActivity.kt
@@ -11,8 +11,13 @@ import android.view.MenuItem
 import android.view.WindowManager
 import androidx.activity.viewModels
 import androidx.core.content.ContextCompat
+import androidx.core.view.isVisible
 import androidx.lifecycle.lifecycleScope
+import com.jakewharton.rxbinding3.view.clicks
 import dagger.hilt.android.AndroidEntryPoint
+import kotlinx.android.synthetic.main.activity_webrtc_tests.*
+import kotlinx.coroutines.Job
+import kotlinx.coroutines.flow.collect
 import kotlinx.coroutines.launch
 import network.loki.messenger.R
 import org.session.libsession.utilities.Address
@@ -39,13 +44,13 @@ class WebRtcCallActivity: PassphraseRequiredActionBarActivity() {

    private val viewModel by viewModels<CallViewModel>()

-    private val acceptedCallMessageHashes = mutableSetOf<Int>()
-
    private val candidates: MutableList<IceCandidate> = mutableListOf()

    private lateinit var callAddress: Address
    private lateinit var callId: UUID

+    private var uiJob: Job? = null
+
    override fun onOptionsItemSelected(item: MenuItem): Boolean {
        if (item.itemId == android.R.id.home) {
            finish()
@@ -70,10 +75,6 @@ class WebRtcCallActivity: PassphraseRequiredActionBarActivity() {
            }
            .execute()

-        lifecycleScope.launch {
-            // repeat on start or something
-        }
-
        if (intent.action == ACTION_ANSWER) {
            // answer via ViewModel
            val answerIntent = WebRtcCallService.acceptCallIntent(this)
@@ -85,6 +86,20 @@ class WebRtcCallActivity: PassphraseRequiredActionBarActivity() {
                finish()
            }
        },IntentFilter(ACTION_END))
+
+        enableCameraButton.setOnClickListener {
+            startService(WebRtcCallService.cameraEnabled(this, true))
+        }
+
+        switchCameraButton.setOnClickListener {
+            startService(WebRtcCallService.flipCamera(this))
+        }
+
+        endCallButton.setOnClickListener {
+            startService(WebRtcCallService.hangupIntent(this))
+        }
+
+
    }

    private fun initializeResources() {
@@ -95,4 +110,29 @@ class WebRtcCallActivity: PassphraseRequiredActionBarActivity() {

    }

+    override fun onStart() {
+        super.onStart()
+
+        uiJob = lifecycleScope.launch {
+
+            viewModel.callState.collect { state ->
+                if (state == CallViewModel.State.CALL_CONNECTED) {
+                    // call connected, render the surfaces
+                    remote_renderer.removeAllViews()
+                    local_renderer.removeAllViews()
+                    viewModel.remoteRenderer?.let { remote_renderer.addView(it) }
+                    viewModel.localRenderer?.let { local_renderer.addView(it) }
+                }
+            }
+
+            viewModel.remoteVideoEnabledState.collect {
+
+            }
+        }
+    }
+
+    override fun onStop() {
+        super.onStop()
+        uiJob?.cancel()
+    }
 }
--- a/app/src/main/java/org/thoughtcrime/securesms/notifications/NotificationChannels.java
+++ b/app/src/main/java/org/thoughtcrime/securesms/notifications/NotificationChannels.java
@@ -427,7 +427,7 @@ public class NotificationChannels {
    notificationManager.createNotificationChannelGroup(messagesGroup);

    NotificationChannel messages     = new NotificationChannel(getMessagesChannel(context), context.getString(R.string.NotificationChannel_messages), NotificationManager.IMPORTANCE_HIGH);
-    NotificationChannel calls        = new NotificationChannel(CALLS, context.getString(R.string.NotificationChannel_calls), NotificationManager.IMPORTANCE_LOW);
+    NotificationChannel calls        = new NotificationChannel(CALLS, context.getString(R.string.NotificationChannel_calls), NotificationManager.IMPORTANCE_DEFAULT);
    NotificationChannel failures     = new NotificationChannel(FAILURES, context.getString(R.string.NotificationChannel_failures), NotificationManager.IMPORTANCE_HIGH);
    NotificationChannel backups      = new NotificationChannel(BACKUPS, context.getString(R.string.NotificationChannel_backups), NotificationManager.IMPORTANCE_LOW);
    NotificationChannel lockedStatus = new NotificationChannel(LOCKED_STATUS, context.getString(R.string.NotificationChannel_locked_status), NotificationManager.IMPORTANCE_LOW);
--- a/app/src/main/java/org/thoughtcrime/securesms/service/WebRtcCallService.kt
+++ b/app/src/main/java/org/thoughtcrime/securesms/service/WebRtcCallService.kt
@@ -17,6 +17,7 @@ import org.session.libsession.messaging.messages.control.CallMessage
 import org.session.libsession.messaging.sending_receiving.MessageSender
 import org.session.libsession.utilities.Address
 import org.session.libsession.utilities.FutureTaskListener
+import org.session.libsession.utilities.TextSecurePreferences
 import org.session.libsession.utilities.Util
 import org.session.libsession.utilities.recipients.Recipient
 import org.session.libsignal.utilities.Log
@@ -42,8 +43,6 @@ import javax.inject.Inject
@AndroidEntryPoint
 class WebRtcCallService: Service(), PeerConnection.Observer {

-    @Inject lateinit var callManager: CallManager
-
    companion object {

        private val TAG = Log.tag(WebRtcCallService::class.java)
@@ -86,6 +85,13 @@ class WebRtcCallService: Service(), PeerConnection.Observer {

        const val INVALID_NOTIFICATION_ID = -1

+        fun cameraEnabled(context: Context, enabled: Boolean) = Intent(context, WebRtcCallService::class.java)
+                .setAction(ACTION_SET_MUTE_VIDEO)
+                .putExtra(EXTRA_MUTE, !enabled)
+
+        fun flipCamera(context: Context) = Intent(context, WebRtcCallService::class.java)
+                .setAction(ACTION_FLIP_CAMERA)
+
        fun acceptCallIntent(context: Context) = Intent(context, WebRtcCallService::class.java).setAction(ACTION_ANSWER_CALL)

        fun createCall(context: Context, recipient: Recipient) = Intent(context, WebRtcCallService::class.java)
@@ -139,6 +145,8 @@ class WebRtcCallService: Service(), PeerConnection.Observer {
        }
    }

+    @Inject lateinit var callManager: CallManager
+
    private var lastNotificationId: Int = INVALID_NOTIFICATION_ID
    private var lastNotification: Notification? = null

@@ -152,6 +160,7 @@ class WebRtcCallService: Service(), PeerConnection.Observer {
    private var callReceiver: IncomingPstnCallReceiver? = null
    private var wiredHeadsetStateReceiver: WiredHeadsetStateReceiver? = null
    private var uncaughtExceptionHandlerManager: UncaughtExceptionHandlerManager? = null
+    private var powerButtonReceiver: PowerButtonReceiver? = null

    @Synchronized
    private fun terminate() {
@@ -277,6 +286,10 @@ class WebRtcCallService: Service(), PeerConnection.Observer {
        callManager.onIncomingRing(offer, callId, recipient, timestamp)
        callManager.clearPendingIceUpdates()
        callManager.postConnectionEvent(STATE_LOCAL_RINGING)
+        if (TextSecurePreferences.isCallNotificationsEnabled(this)) {
+            callManager.startIncomingRinger()
+        }
+        registerPowerButtonReceiver()
    }

    private fun handleOutgoingCall(intent: Intent) {
@@ -344,6 +357,7 @@ class WebRtcCallService: Service(), PeerConnection.Observer {

        timeoutExecutor.schedule(TimeoutRunnable(callId, this), 5, TimeUnit.MINUTES)

+        callManager.initializeAudioForCall()
        callManager.initializeVideo(this)

        val expectedState = callManager.currentConnectionState
@@ -373,12 +387,7 @@ class WebRtcCallService: Service(), PeerConnection.Observer {
            return
        }

-        if (callManager.callNotSetup()) {
-            throw AssertionError("assert")
-        }
-
        callManager.handleDenyCall()
-
        // DatabaseComponent.get(this).smsDatabase().insertMissedCall(recipient)
        terminate()
    }
@@ -471,10 +480,17 @@ class WebRtcCallService: Service(), PeerConnection.Observer {
    }

    private fun handleIceConnected(intent: Intent) {
-        if (callManager.currentConnectionState == STATE_ANSWERING) {
-            val recipient = callManager.recipient ?: return
+        val recipient = callManager.recipient ?: return
+        if (callManager.currentConnectionState in arrayOf(STATE_ANSWERING, STATE_LOCAL_RINGING)) {
            callManager.postConnectionEvent(STATE_CONNECTED)
+            callManager.postViewModelState(CallViewModel.State.CALL_CONNECTED)
+        } else {
+            Log.w(TAG, "Got ice connected out of state")
        }
+
+        setCallInProgressNotification(TYPE_ESTABLISHED, recipient)
+
+        callManager.startCommunication(lockManager)
    }

    private fun handleCallConnected(intent: Intent) {
@@ -485,6 +501,14 @@ class WebRtcCallService: Service(), PeerConnection.Observer {

    }

+    private fun registerPowerButtonReceiver() {
+        if (powerButtonReceiver == null) {
+            powerButtonReceiver = PowerButtonReceiver()
+
+            registerReceiver(powerButtonReceiver, IntentFilter(Intent.ACTION_SCREEN_OFF))
+        }
+    }
+



@@ -643,7 +667,9 @@ class WebRtcCallService: Service(), PeerConnection.Observer {

    override fun onDataChannel(p0: DataChannel?) {}

-    override fun onRenegotiationNeeded() {}
+    override fun onRenegotiationNeeded() {
+        Log.w(TAG,"onRenegotiationNeeded was called!")
+    }

    override fun onAddTrack(p0: RtpReceiver?, p1: Array<out MediaStream>?) {}
 }
--- a/app/src/main/java/org/thoughtcrime/securesms/util/CallNotificationBuilder.kt
+++ b/app/src/main/java/org/thoughtcrime/securesms/util/CallNotificationBuilder.kt
@@ -58,6 +58,7 @@ class CallNotificationBuilder {
                            R.drawable.ic_phone_grey600_32dp,
                            R.string.NotificationBarManager__answer_call
                    ))
+                    builder.priority = NotificationCompat.PRIORITY_HIGH
                }
                TYPE_OUTGOING_RINGING -> {
                    builder.setContentText(context.getString(R.string.NotificationBarManager__establishing_signal_call))
--- a/app/src/main/java/org/thoughtcrime/securesms/webrtc/CallManager.kt
+++ b/app/src/main/java/org/thoughtcrime/securesms/webrtc/CallManager.kt
@@ -110,8 +110,8 @@ class CallManager(context: Context, audioManager: AudioManagerCompat): PeerConne

    private val outgoingIceDebouncer = Debouncer(2_000L)

-    private var localRenderer: SurfaceViewRenderer? = null
-    private var remoteRenderer: SurfaceViewRenderer? = null
+    var localRenderer: SurfaceViewRenderer? = null
+    var remoteRenderer: SurfaceViewRenderer? = null
    private var peerConnectionFactory: PeerConnectionFactory? = null

    fun clearPendingIceUpdates() {
@@ -175,7 +175,7 @@ class CallManager(context: Context, audioManager: AudioManagerCompat): PeerConne
            peerConnectionFactory = PeerConnectionFactory.builder()
                    .setOptions(object: PeerConnectionFactory.Options() {
                        init {
-                            networkIgnoreMask = 1 shl 4
+//                            networkIgnoreMask = 1 shl 4
                        }
                    })
                    .setVideoEncoderFactory(encoderFactory)
@@ -199,7 +199,7 @@ class CallManager(context: Context, audioManager: AudioManagerCompat): PeerConne
    fun setVideoEnabled(isEnabled: Boolean) {
        currentConnectionState.withState(*(CONNECTED_STATES + PENDING_CONNECTION_STATES)) {
            peerConnection?.setVideoEnabled(isEnabled)
-            _audioEvents.value = StateEvent.AudioEnabled(true)
+            _videoEvents.value = StateEvent.VideoEnabled(true)
        }
    }

@@ -290,7 +290,9 @@ class CallManager(context: Context, audioManager: AudioManagerCompat): PeerConne

    override fun onMessage(buffer: DataChannel.Buffer?) {
        Log.i(TAG,"onMessage...")
-        TODO("interpret the data channel buffer and check for signals")
+        buffer ?: return
+
+        Log.i(TAG,"received: ${buffer.data}")
    }

    override fun onAudioDeviceChanged(activeDevice: AudioDevice, devices: Set<AudioDevice>) {
@@ -329,6 +331,10 @@ class CallManager(context: Context, audioManager: AudioManagerCompat): PeerConne
        localCameraState = newCameraState
    }

+    fun enableLocalCamera() {
+        setVideoEnabled(true)
+    }
+
    fun onIncomingRing(offer: String, callId: UUID, recipient: Recipient, callTime: Long) {
        if (currentConnectionState != CallState.STATE_IDLE) return

@@ -357,6 +363,7 @@ class CallManager(context: Context, audioManager: AudioManagerCompat): PeerConne
        peerConnection = connection
        localCameraState = connection.getCameraState()
        val dataChannel = connection.createDataChannel(DATA_CHANNEL_NAME)
+        this.dataChannel = dataChannel
        dataChannel.registerObserver(this)
        connection.setRemoteDescription(SessionDescription(SessionDescription.Type.OFFER, offer))
        val answer = connection.createAnswer(MediaConstraints())
@@ -401,6 +408,7 @@ class CallManager(context: Context, audioManager: AudioManagerCompat): PeerConne
        localCameraState = connection.getCameraState()
        val dataChannel = connection.createDataChannel(DATA_CHANNEL_NAME)
        dataChannel.registerObserver(this)
+        this.dataChannel = dataChannel
        val offer = connection.createOffer(MediaConstraints())
        connection.setLocalDescription(offer)

@@ -536,4 +544,18 @@ class CallManager(context: Context, audioManager: AudioManagerCompat): PeerConne
        signalAudioManager.handleCommand(AudioManagerCommand.Shutdown)
    }

+    fun startIncomingRinger() {
+        signalAudioManager.handleCommand(AudioManagerCommand.StartIncomingRinger(true))
+    }
+
+    fun startCommunication(lockManager: LockManager) {
+        signalAudioManager.handleCommand(AudioManagerCommand.Start)
+        val connection = peerConnection ?: return
+        if (localCameraState.enabled) lockManager.updatePhoneState(LockManager.PhoneState.IN_VIDEO)
+        else lockManager.updatePhoneState(LockManager.PhoneState.IN_CALL)
+        connection.setCommunicationMode()
+        connection.setAudioEnabled(_audioEvents.value.isEnabled)
+        connection.setVideoEnabled(localCameraState.enabled)
+    }
+
 }
--- a/app/src/main/java/org/thoughtcrime/securesms/webrtc/CallViewModel.kt
+++ b/app/src/main/java/org/thoughtcrime/securesms/webrtc/CallViewModel.kt
@@ -3,11 +3,18 @@ package org.thoughtcrime.securesms.webrtc
 import androidx.lifecycle.ViewModel
 import dagger.hilt.android.lifecycle.HiltViewModel
 import kotlinx.coroutines.flow.map
+import org.webrtc.SurfaceViewRenderer
 import javax.inject.Inject

@HiltViewModel
 class CallViewModel @Inject constructor(private val callManager: CallManager): ViewModel() {

+    val localRenderer: SurfaceViewRenderer?
+    get() = callManager.localRenderer
+
+    val remoteRenderer: SurfaceViewRenderer?
+    get() = callManager.remoteRenderer
+
    enum class State {
        CALL_PENDING,

--- a/app/src/main/java/org/thoughtcrime/securesms/webrtc/PeerConnectionWrapper.kt
+++ b/app/src/main/java/org/thoughtcrime/securesms/webrtc/PeerConnectionWrapper.kt
@@ -83,6 +83,8 @@ class PeerConnectionWrapper(context: Context,
    fun createDataChannel(channelName: String): DataChannel {
        val dataChannelConfiguration = DataChannel.Init().apply {
            ordered = true
+            negotiated = true
+            id = 548
        }
        return peerConnection.createDataChannel(channelName, dataChannelConfiguration)
    }
@@ -220,6 +222,11 @@ class PeerConnectionWrapper(context: Context,
        }
    }

+    fun setCommunicationMode() {
+        peerConnection.setAudioPlayout(true)
+        peerConnection.setAudioRecording(true)
+    }
+
    fun setAudioEnabled(isEnabled: Boolean) {
        audioTrack.setEnabled(isEnabled)
    }
--- a/app/src/main/res/layout/activity_webrtc_tests.xml
+++ b/app/src/main/res/layout/activity_webrtc_tests.xml
@@ -6,44 +6,10 @@
    android:layout_height="match_parent"
    xmlns:tools="http://schemas.android.com/tools">

-    <TextView
-        android:padding="@dimen/small_spacing"
-        android:elevation="8dp"
-        tools:visibility="visible"
-        android:textColor="@color/white"
-        tools:text="relay"
-        android:visibility="gone"
-        android:background="@drawable/pill"
-        android:backgroundTint="@color/black"
-        android:layout_marginTop="@dimen/medium_spacing"
-        android:id="@+id/local_candidate_info"
-        app:layout_constraintEnd_toStartOf="@id/remote_candidate_info"
-        android:layout_width="wrap_content"
-        android:layout_height="wrap_content"
-        app:layout_constraintTop_toTopOf="parent"
-        app:layout_constraintStart_toStartOf="parent"/>
-
-    <TextView
-        android:padding="@dimen/small_spacing"
-        android:elevation="8dp"
-        tools:visibility="visible"
-        tools:text="relay"
-        android:visibility="gone"
-        android:textColor="@color/white"
-        android:background="@drawable/pill"
-        android:backgroundTint="@color/black"
-        android:layout_marginTop="@dimen/medium_spacing"
-        android:id="@+id/remote_candidate_info"
-        app:layout_constraintStart_toEndOf="@id/local_candidate_info"
-        android:layout_width="wrap_content"
-        android:layout_height="wrap_content"
-        app:layout_constraintTop_toTopOf="parent"
-        app:layout_constraintEnd_toEndOf="parent"/>
-
    <FrameLayout
        android:layout_width="match_parent"
        android:layout_height="match_parent">
-        <org.webrtc.SurfaceViewRenderer
+        <FrameLayout
            android:id="@+id/remote_renderer"
            android:layout_width="match_parent"
            android:layout_height="match_parent"
@@ -69,7 +35,8 @@
        app:layout_constraintWidth_percent="0.2"
        android:layout_height="0dp"
        android:layout_width="0dp">
-        <org.webrtc.SurfaceViewRenderer
+        <FrameLayout
+            android:elevation="8dp"
            android:id="@+id/local_renderer"
            android:layout_width="match_parent"
            android:layout_height="match_parent"/>
@@ -85,7 +52,7 @@
    </FrameLayout>

    <ImageView
-        android:id="@+id/end_call_button"
+        android:id="@+id/endCallButton"
        android:background="@drawable/circle_tintable"
        android:src="@drawable/ic_baseline_call_end_24"
        android:padding="@dimen/small_spacing"
@@ -100,7 +67,7 @@
        />

    <ImageView
-        android:id="@+id/switch_camera_button"
+        android:id="@+id/switchCameraButton"
        android:background="@drawable/circle_tintable"
        android:src="@drawable/ic_baseline_flip_camera_android_24"
        android:padding="@dimen/small_spacing"
@@ -116,7 +83,23 @@
        />

    <ImageView
-        android:id="@+id/switch_audio_button"
+        android:id="@+id/enableCameraButton"
+        android:background="@drawable/circle_tintable"
+        android:src="@drawable/ic_baseline_photo_camera_48"
+        android:padding="@dimen/small_spacing"
+        app:tint="@color/unimportant"
+        android:backgroundTint="@color/unimportant_button_background"
+        android:layout_width="@dimen/large_button_height"
+        android:layout_height="@dimen/large_button_height"
+        app:layout_constraintBottom_toBottomOf="parent"
+        app:layout_constraintStart_toStartOf="parent"
+        app:layout_constraintEnd_toEndOf="parent"
+        android:layout_marginBottom="@dimen/large_spacing"
+        app:layout_constraintHorizontal_bias="0.2"
+        />
+
+    <ImageView
+        android:id="@+id/speakerPhoneButton"
        android:background="@drawable/circle_tintable"
        android:src="@drawable/ic_audio_light"
        android:padding="@dimen/small_spacing"