From 00f06ab034f79791aa3cd2fcd272d05413bbe0a3 Mon Sep 17 00:00:00 2001 From: Harris Date: Wed, 18 May 2022 10:20:57 +1000 Subject: [PATCH] Namespace retrieval and storage with auth (#880) * feat: add migration and fork info for upcoming auth and closed group retrieval updates * feat: add closed group poller calls and include namespace to parse raw messages function * feat: add DB upgrades and queries for namespaces * fix: fix the polling for post-HF signatures and group messages * fix: realise we need a compound key for namespaces in received hashes, test explicitly setting namespace * feat: add setForkInfo implementation * refactor: include default fork info command on create, refactor migration to use new table since we can't add constraints in alter for PK, replace `lastHash` with `last_hash` in case that fixes paging * refactor: include namespace and use when statement for closed group polling * refactor: revert to main net * refactor: use namespace constants * refactor: revert to testnet and log the poll result * fix: use or to log either poller * fix: revert to default network and add more logging, only set the latest fork info if it is an increment * build: update minor version * refactor: use single target snode and namespace list for message sending * fix: link previews and expiring messages in closed groups --- app/build.gradle | 4 +- .../v2/menus/ConversationMenuHelper.kt | 2 +- .../v2/messages/VisibleMessageContentView.kt | 2 +- .../v2/utilities/MentionManagerUtilities.kt | 3 - .../securesms/database/LokiAPIDatabase.kt | 127 ++++++++++----- .../database/helpers/SQLCipherOpenHelper.java | 14 +- .../sending_receiving/MessageSender.kt | 43 ++++-- .../pollers/ClosedGroupPollerV2.kt | 29 +++- .../libsession/snode/OnionRequestAPI.kt | 36 ++++- .../org/session/libsession/snode/SnodeAPI.kt | 144 ++++++++++++------ .../session/libsession/snode/SnodeMessage.kt | 5 +- .../database/LokiAPIDatabaseProtocol.kt | 14 +- .../session/libsignal/utilities/ForkInfo.kt | 20 +++ .../session/libsignal/utilities/Namespace.kt | 7 + 14 files changed, 325 insertions(+), 125 deletions(-) create mode 100644 libsignal/src/main/java/org/session/libsignal/utilities/ForkInfo.kt create mode 100644 libsignal/src/main/java/org/session/libsignal/utilities/Namespace.kt diff --git a/app/build.gradle b/app/build.gradle index 18370f9cf5..4407a79b99 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -158,8 +158,8 @@ dependencies { testImplementation 'org.robolectric:shadows-multidex:4.4' } -def canonicalVersionCode = 272 -def canonicalVersionName = "1.12.15" +def canonicalVersionCode = 273 +def canonicalVersionName = "1.13.0" def postFixSize = 10 def abiPostFix = ['armeabi-v7a' : 1, diff --git a/app/src/main/java/org/thoughtcrime/securesms/conversation/v2/menus/ConversationMenuHelper.kt b/app/src/main/java/org/thoughtcrime/securesms/conversation/v2/menus/ConversationMenuHelper.kt index 5669cafa4f..27f5b3dbee 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/conversation/v2/menus/ConversationMenuHelper.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/conversation/v2/menus/ConversationMenuHelper.kt @@ -68,7 +68,7 @@ object ConversationMenuHelper { // Base menu (options that should always be present) inflater.inflate(R.menu.menu_conversation, menu) // Expiring messages - if (!isOpenGroup && thread.hasApprovedMe()) { + if (!isOpenGroup && (thread.hasApprovedMe() || thread.isClosedGroupRecipient)) { if (thread.expireMessages > 0) { inflater.inflate(R.menu.menu_conversation_expiration_on, menu) val item = menu.findItem(R.id.menu_expiring_messages) diff --git a/app/src/main/java/org/thoughtcrime/securesms/conversation/v2/messages/VisibleMessageContentView.kt b/app/src/main/java/org/thoughtcrime/securesms/conversation/v2/messages/VisibleMessageContentView.kt index 3172c0c63f..96dc9a2065 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/conversation/v2/messages/VisibleMessageContentView.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/conversation/v2/messages/VisibleMessageContentView.kt @@ -98,7 +98,7 @@ class VisibleMessageContentView : LinearLayout { linkPreviewLayout.width = if (mediaThumbnailMessage) 0 else ViewGroup.LayoutParams.WRAP_CONTENT binding.linkPreviewView.layoutParams = linkPreviewLayout - binding.untrustedView.isVisible = !contactIsTrusted && message is MmsMessageRecord && message.quote == null + binding.untrustedView.isVisible = !contactIsTrusted && message is MmsMessageRecord && message.quote == null && message.linkPreviews.isEmpty() binding.voiceMessageView.isVisible = contactIsTrusted && message is MmsMessageRecord && message.slideDeck.audioSlide != null binding.documentView.isVisible = contactIsTrusted && message is MmsMessageRecord && message.slideDeck.documentSlide != null binding.albumThumbnailView.isVisible = mediaThumbnailMessage diff --git a/app/src/main/java/org/thoughtcrime/securesms/conversation/v2/utilities/MentionManagerUtilities.kt b/app/src/main/java/org/thoughtcrime/securesms/conversation/v2/utilities/MentionManagerUtilities.kt index 48ce85de19..ee1c7257c2 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/conversation/v2/utilities/MentionManagerUtilities.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/conversation/v2/utilities/MentionManagerUtilities.kt @@ -9,9 +9,6 @@ import org.thoughtcrime.securesms.dependencies.DatabaseComponent object MentionManagerUtilities { fun populateUserPublicKeyCacheIfNeeded(threadID: Long, context: Context) { - // exit early if we need to - if (MentionsManager.userPublicKeyCache[threadID] != null) return - val result = mutableSetOf() val recipient = DatabaseComponent.get(context).threadDatabase().getRecipientForThreadId(threadID) ?: return if (recipient.address.isClosedGroup) { diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/LokiAPIDatabase.kt b/app/src/main/java/org/thoughtcrime/securesms/database/LokiAPIDatabase.kt index 7f32fab1d1..826c39ae8a 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/database/LokiAPIDatabase.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/database/LokiAPIDatabase.kt @@ -7,28 +7,24 @@ import org.session.libsignal.crypto.ecc.DjbECPrivateKey import org.session.libsignal.crypto.ecc.DjbECPublicKey import org.session.libsignal.crypto.ecc.ECKeyPair import org.session.libsignal.database.LokiAPIDatabaseProtocol -import org.session.libsignal.utilities.* +import org.session.libsignal.utilities.ForkInfo +import org.session.libsignal.utilities.Hex +import org.session.libsignal.utilities.Log +import org.session.libsignal.utilities.PublicKeyValidation +import org.session.libsignal.utilities.Snode +import org.session.libsignal.utilities.removing05PrefixIfNeeded +import org.session.libsignal.utilities.toHexString import org.thoughtcrime.securesms.crypto.IdentityKeyUtil -import org.thoughtcrime.securesms.database.* import org.thoughtcrime.securesms.database.helpers.SQLCipherOpenHelper -import org.thoughtcrime.securesms.util.* -import java.util.* -import kotlin.Array -import kotlin.Boolean -import kotlin.Int -import kotlin.Long -import kotlin.Pair -import kotlin.String -import kotlin.arrayOf -import kotlin.to +import java.util.Date class LokiAPIDatabase(context: Context, helper: SQLCipherOpenHelper) : Database(context, helper), LokiAPIDatabaseProtocol { companion object { // Shared - private val publicKey = "public_key" - private val timestamp = "timestamp" - private val snode = "snode" + private const val publicKey = "public_key" + private const val timestamp = "timestamp" + private const val snode = "snode" // Snode pool public val snodePoolTable = "loki_snode_pool_cache" private val dummyKey = "dummy_key" @@ -44,15 +40,19 @@ class LokiAPIDatabase(context: Context, helper: SQLCipherOpenHelper) : Database( private val swarm = "swarm" @JvmStatic val createSwarmTableCommand = "CREATE TABLE $swarmTable ($swarmPublicKey TEXT PRIMARY KEY, $swarm TEXT);" // Last message hash values - private val lastMessageHashValueTable2 = "last_message_hash_value_table" - private val lastMessageHashValue = "last_message_hash_value" + private const val legacyLastMessageHashValueTable2 = "last_message_hash_value_table" + private const val lastMessageHashValueTable2 = "session_last_message_hash_value_table" + private const val lastMessageHashValue = "last_message_hash_value" + private const val lastMessageHashNamespace = "last_message_namespace" @JvmStatic val createLastMessageHashValueTable2Command - = "CREATE TABLE $lastMessageHashValueTable2 ($snode TEXT, $publicKey TEXT, $lastMessageHashValue TEXT, PRIMARY KEY ($snode, $publicKey));" + = "CREATE TABLE $legacyLastMessageHashValueTable2 ($snode TEXT, $publicKey TEXT, $lastMessageHashValue TEXT, PRIMARY KEY ($snode, $publicKey));" // Received message hash values - private val receivedMessageHashValuesTable3 = "received_message_hash_values_table_3" - private val receivedMessageHashValues = "received_message_hash_values" + private const val legacyReceivedMessageHashValuesTable3 = "received_message_hash_values_table_3" + private const val receivedMessageHashValuesTable = "session_received_message_hash_values_table" + private const val receivedMessageHashValues = "received_message_hash_values" + private const val receivedMessageHashNamespace = "received_message_namespace" @JvmStatic val createReceivedMessageHashValuesTable3Command - = "CREATE TABLE $receivedMessageHashValuesTable3 ($publicKey STRING PRIMARY KEY, $receivedMessageHashValues TEXT);" + = "CREATE TABLE $legacyReceivedMessageHashValuesTable3 ($publicKey STRING PRIMARY KEY, $receivedMessageHashValues TEXT);" // Open group auth tokens private val openGroupAuthTokenTable = "loki_api_group_chat_auth_token_database" private val server = "server" @@ -97,6 +97,31 @@ class LokiAPIDatabase(context: Context, helper: SQLCipherOpenHelper) : Database( public val groupPublicKey = "group_public_key" @JvmStatic val createClosedGroupPublicKeysTable = "CREATE TABLE $closedGroupPublicKeysTable ($groupPublicKey STRING PRIMARY KEY)" + // Hard fork service node info + const val FORK_INFO_TABLE = "fork_info" + const val DUMMY_KEY = "dummy_key" + const val DUMMY_VALUE = "1" + const val HF_VALUE = "hf_value" + const val SF_VALUE = "sf_value" + const val CREATE_FORK_INFO_TABLE_COMMAND = "CREATE TABLE $FORK_INFO_TABLE ($DUMMY_KEY INTEGER PRIMARY KEY, $HF_VALUE INTEGER, $SF_VALUE INTEGER);" + const val CREATE_DEFAULT_FORK_INFO_COMMAND = "INSERT INTO $FORK_INFO_TABLE ($DUMMY_KEY, $HF_VALUE, $SF_VALUE) VALUES ($DUMMY_VALUE, 18, 1);" + + const val UPDATE_HASHES_INCLUDE_NAMESPACE_COMMAND = """ + CREATE TABLE IF NOT EXISTS $lastMessageHashValueTable2( + $snode TEXT, $publicKey TEXT, $lastMessageHashValue TEXT, $lastMessageHashNamespace INTEGER DEFAULT 0, PRIMARY KEY ($snode, $publicKey, $lastMessageHashNamespace) + ); + INSERT INTO $lastMessageHashValueTable2($snode, $publicKey, $lastMessageHashValue) SELECT $snode, $publicKey, $lastMessageHashValue FROM $legacyLastMessageHashValueTable2); + DROP TABLE $legacyLastMessageHashValueTable2; + """ + const val UPDATE_RECEIVED_INCLUDE_NAMESPACE_COMMAND = """ + CREATE TABLE IF NOT EXISTS $receivedMessageHashValuesTable( + $publicKey STRING, $receivedMessageHashValues TEXT, $receivedMessageHashNamespace INTEGER DEFAULT 0, PRIMARY KEY ($publicKey, $receivedMessageHashNamespace) + ); + + INSERT INTO $receivedMessageHashValuesTable($publicKey, $receivedMessageHashValues) SELECT $publicKey, $receivedMessageHashValues FROM $legacyReceivedMessageHashValuesTable3; + + DROP TABLE $legacyReceivedMessageHashValuesTable3; + """ // region Deprecated private val deviceLinkCache = "loki_pairing_authorisation_cache" @@ -232,36 +257,45 @@ class LokiAPIDatabase(context: Context, helper: SQLCipherOpenHelper) : Database( database.insertOrUpdate(swarmTable, row, "${Companion.swarmPublicKey} = ?", wrap(publicKey)) } - override fun getLastMessageHashValue(snode: Snode, publicKey: String): String? { + override fun getLastMessageHashValue(snode: Snode, publicKey: String, namespace: Int): String? { val database = databaseHelper.readableDatabase - val query = "${Companion.snode} = ? AND ${Companion.publicKey} = ?" - return database.get(lastMessageHashValueTable2, query, arrayOf( snode.toString(), publicKey )) { cursor -> + val query = "${Companion.snode} = ? AND ${Companion.publicKey} = ? AND $lastMessageHashNamespace = ?" + return database.get(lastMessageHashValueTable2, query, arrayOf( snode.toString(), publicKey, namespace.toString() )) { cursor -> cursor.getString(cursor.getColumnIndexOrThrow(lastMessageHashValue)) } } - override fun setLastMessageHashValue(snode: Snode, publicKey: String, newValue: String) { + override fun setLastMessageHashValue(snode: Snode, publicKey: String, newValue: String, namespace: Int) { val database = databaseHelper.writableDatabase - val row = wrap(mapOf( Companion.snode to snode.toString(), Companion.publicKey to publicKey, lastMessageHashValue to newValue )) - val query = "${Companion.snode} = ? AND ${Companion.publicKey} = ?" - database.insertOrUpdate(lastMessageHashValueTable2, row, query, arrayOf( snode.toString(), publicKey )) + val row = wrap(mapOf( + Companion.snode to snode.toString(), + Companion.publicKey to publicKey, + lastMessageHashValue to newValue, + lastMessageHashNamespace to namespace.toString() + )) + val query = "${Companion.snode} = ? AND ${Companion.publicKey} = ? AND $lastMessageHashNamespace = ?" + database.insertOrUpdate(lastMessageHashValueTable2, row, query, arrayOf( snode.toString(), publicKey, namespace.toString() )) } - override fun getReceivedMessageHashValues(publicKey: String): Set? { + override fun getReceivedMessageHashValues(publicKey: String, namespace: Int): Set? { val database = databaseHelper.readableDatabase - val query = "${Companion.publicKey} = ?" - return database.get(receivedMessageHashValuesTable3, query, arrayOf( publicKey )) { cursor -> + val query = "${Companion.publicKey} = ? AND ${Companion.receivedMessageHashNamespace} = ?" + return database.get(receivedMessageHashValuesTable, query, arrayOf( publicKey, namespace.toString() )) { cursor -> val receivedMessageHashValuesAsString = cursor.getString(cursor.getColumnIndexOrThrow(Companion.receivedMessageHashValues)) receivedMessageHashValuesAsString.split("-").toSet() } } - override fun setReceivedMessageHashValues(publicKey: String, newValue: Set) { + override fun setReceivedMessageHashValues(publicKey: String, newValue: Set, namespace: Int) { val database = databaseHelper.writableDatabase val receivedMessageHashValuesAsString = newValue.joinToString("-") - val row = wrap(mapOf( Companion.publicKey to publicKey, Companion.receivedMessageHashValues to receivedMessageHashValuesAsString )) - val query = "${Companion.publicKey} = ?" - database.insertOrUpdate(receivedMessageHashValuesTable3, row, query, arrayOf( publicKey )) + val row = wrap(mapOf( + Companion.publicKey to publicKey, + Companion.receivedMessageHashValues to receivedMessageHashValuesAsString, + Companion.receivedMessageHashNamespace to namespace.toString() + )) + val query = "${Companion.publicKey} = ? AND $receivedMessageHashNamespace = ?" + database.insertOrUpdate(receivedMessageHashValuesTable, row, query, arrayOf( publicKey, namespace.toString() )) } override fun getAuthToken(server: String): String? { @@ -441,6 +475,29 @@ class LokiAPIDatabase(context: Context, helper: SQLCipherOpenHelper) : Database( val database = databaseHelper.writableDatabase database.delete(closedGroupPublicKeysTable, "${Companion.groupPublicKey} = ?", wrap(groupPublicKey)) } + + override fun getForkInfo(): ForkInfo { + val database = databaseHelper.readableDatabase + val queryCursor = database.query(FORK_INFO_TABLE, arrayOf(HF_VALUE, SF_VALUE), "$DUMMY_KEY = $DUMMY_VALUE", null, null, null, null) + val forkInfo = queryCursor.use { cursor -> + if (!cursor.moveToNext()) { + ForkInfo(18, 1) // no HF info, none set will at least be the version + } else { + ForkInfo(cursor.getInt(0), cursor.getInt(1)) + } + } + return forkInfo + } + + override fun setForkInfo(forkInfo: ForkInfo) { + val database = databaseHelper.writableDatabase + val query = "$DUMMY_KEY = $DUMMY_VALUE" + val contentValues = ContentValues(3) + contentValues.put(DUMMY_KEY, DUMMY_VALUE) + contentValues.put(HF_VALUE, forkInfo.hf) + contentValues.put(SF_VALUE, forkInfo.sf) + database.insertOrUpdate(FORK_INFO_TABLE, contentValues, query, emptyArray()) + } } // region Convenience diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/helpers/SQLCipherOpenHelper.java b/app/src/main/java/org/thoughtcrime/securesms/database/helpers/SQLCipherOpenHelper.java index 8d62125b44..8d3fcb472a 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/database/helpers/SQLCipherOpenHelper.java +++ b/app/src/main/java/org/thoughtcrime/securesms/database/helpers/SQLCipherOpenHelper.java @@ -64,9 +64,10 @@ public class SQLCipherOpenHelper extends SQLiteOpenHelper { private static final int lokiV30 = 51; private static final int lokiV31 = 52; private static final int lokiV32 = 53; + private static final int lokiV33 = 54; // Loki - onUpgrade(...) must be updated to use Loki version numbers if Signal makes any database changes - private static final int DATABASE_VERSION = lokiV32; + private static final int DATABASE_VERSION = lokiV33; private static final String DATABASE_NAME = "signal.db"; private final Context context; @@ -143,6 +144,10 @@ public class SQLCipherOpenHelper extends SQLiteOpenHelper { db.execSQL(RecipientDatabase.getCreateApprovedCommand()); db.execSQL(RecipientDatabase.getCreateApprovedMeCommand()); db.execSQL(MmsDatabase.getCreateMessageRequestResponseCommand()); + db.execSQL(LokiAPIDatabase.CREATE_FORK_INFO_TABLE_COMMAND); + db.execSQL(LokiAPIDatabase.CREATE_DEFAULT_FORK_INFO_COMMAND); + db.execSQL(LokiAPIDatabase.UPDATE_HASHES_INCLUDE_NAMESPACE_COMMAND); + db.execSQL(LokiAPIDatabase.UPDATE_RECEIVED_INCLUDE_NAMESPACE_COMMAND); executeStatements(db, SmsDatabase.CREATE_INDEXS); executeStatements(db, MmsDatabase.CREATE_INDEXS); @@ -337,6 +342,13 @@ public class SQLCipherOpenHelper extends SQLiteOpenHelper { db.execSQL(RecipientDatabase.getUpdateApprovedSelectConversations()); } + if (oldVersion < lokiV33) { + db.execSQL(LokiAPIDatabase.CREATE_FORK_INFO_TABLE_COMMAND); + db.execSQL(LokiAPIDatabase.CREATE_DEFAULT_FORK_INFO_COMMAND); + db.execSQL(LokiAPIDatabase.UPDATE_HASHES_INCLUDE_NAMESPACE_COMMAND); + db.execSQL(LokiAPIDatabase.UPDATE_RECEIVED_INCLUDE_NAMESPACE_COMMAND); + } + db.setTransactionSuccessful(); } finally { db.endTransaction(); diff --git a/libsession/src/main/java/org/session/libsession/messaging/sending_receiving/MessageSender.kt b/libsession/src/main/java/org/session/libsession/messaging/sending_receiving/MessageSender.kt index 277c0c3a9d..75b6e68d83 100644 --- a/libsession/src/main/java/org/session/libsession/messaging/sending_receiving/MessageSender.kt +++ b/libsession/src/main/java/org/session/libsession/messaging/sending_receiving/MessageSender.kt @@ -8,9 +8,17 @@ import org.session.libsession.messaging.jobs.MessageSendJob import org.session.libsession.messaging.jobs.NotifyPNServerJob import org.session.libsession.messaging.messages.Destination import org.session.libsession.messaging.messages.Message -import org.session.libsession.messaging.messages.control.* -import org.session.libsession.messaging.messages.visible.* -import org.session.libsession.messaging.open_groups.* +import org.session.libsession.messaging.messages.control.CallMessage +import org.session.libsession.messaging.messages.control.ClosedGroupControlMessage +import org.session.libsession.messaging.messages.control.ConfigurationMessage +import org.session.libsession.messaging.messages.control.ExpirationTimerUpdate +import org.session.libsession.messaging.messages.control.UnsendRequest +import org.session.libsession.messaging.messages.visible.LinkPreview +import org.session.libsession.messaging.messages.visible.Profile +import org.session.libsession.messaging.messages.visible.Quote +import org.session.libsession.messaging.messages.visible.VisibleMessage +import org.session.libsession.messaging.open_groups.OpenGroupAPIV2 +import org.session.libsession.messaging.open_groups.OpenGroupMessageV2 import org.session.libsession.messaging.utilities.MessageWrapper import org.session.libsession.snode.RawResponsePromise import org.session.libsession.snode.SnodeAPI @@ -22,8 +30,11 @@ import org.session.libsession.utilities.SSKEnvironment import org.session.libsignal.crypto.PushTransportDetails import org.session.libsignal.protos.SignalServiceProtos import org.session.libsignal.utilities.Base64 -import org.session.libsignal.utilities.Log +import org.session.libsignal.utilities.Namespace +import org.session.libsignal.utilities.defaultRequiresAuth +import org.session.libsignal.utilities.hasNamespaces import org.session.libsignal.utilities.hexEncodedPublicKey +import java.util.concurrent.atomic.AtomicInteger import org.session.libsession.messaging.sending_receiving.attachments.Attachment as SignalAttachment import org.session.libsession.messaging.sending_receiving.link_preview.LinkPreview as SignalLinkPreview import org.session.libsession.messaging.sending_receiving.quotes.QuoteModel as SignalQuote @@ -125,6 +136,15 @@ object MessageSender { // Wrap the result val kind: SignalServiceProtos.Envelope.Type val senderPublicKey: String + // TODO: this might change in future for config messages + val forkInfo = SnodeAPI.forkInfo + val namespaces: List = when { + destination is Destination.ClosedGroup + && forkInfo.defaultRequiresAuth() -> listOf(Namespace.UNAUTHENTICATED_CLOSED_GROUP) + destination is Destination.ClosedGroup + && forkInfo.hasNamespaces() -> listOf(Namespace.UNAUTHENTICATED_CLOSED_GROUP, Namespace.DEFAULT) + else -> listOf(Namespace.DEFAULT) + } when (destination) { is Destination.Contact -> { kind = SignalServiceProtos.Envelope.Type.SESSION_MESSAGE @@ -148,11 +168,11 @@ object MessageSender { if (destination is Destination.Contact && message is VisibleMessage && !isSelfSend) { SnodeModule.shared.broadcaster.broadcast("sendingMessage", message.sentTimestamp!!) } - SnodeAPI.sendMessage(snodeMessage).success { promises: Set -> + namespaces.map { namespace -> SnodeAPI.sendMessage(snodeMessage, requiresAuth = false, namespace = namespace) }.let { promises -> var isSuccess = false val promiseCount = promises.size - var errorCount = 0 - promises.iterator().forEach { promise: RawResponsePromise -> + var errorCount = AtomicInteger(0) + promises.forEach { promise: RawResponsePromise -> promise.success { if (isSuccess) { return@success } // Succeed as soon as the first promise succeeds isSuccess = true @@ -162,7 +182,7 @@ object MessageSender { val hash = it["hash"] as? String message.serverHash = hash handleSuccessfulMessageSend(message, destination, isSyncMessage) - var shouldNotify = ((message is VisibleMessage || message is UnsendRequest || message is CallMessage) && !isSyncMessage) + val shouldNotify = ((message is VisibleMessage || message is UnsendRequest || message is CallMessage) && !isSyncMessage) /* if (message is ClosedGroupControlMessage && message.kind is ClosedGroupControlMessage.Kind.New) { shouldNotify = true @@ -175,14 +195,11 @@ object MessageSender { deferred.resolve(Unit) } promise.fail { - errorCount += 1 - if (errorCount != promiseCount) { return@fail } // Only error out if all promises failed + errorCount.getAndIncrement() + if (errorCount.get() != promiseCount) { return@fail } // Only error out if all promises failed handleFailure(it) } } - }.fail { - Log.d("Loki", "Couldn't send message due to error: $it.") - handleFailure(it) } } catch (exception: Exception) { handleFailure(exception) diff --git a/libsession/src/main/java/org/session/libsession/messaging/sending_receiving/pollers/ClosedGroupPollerV2.kt b/libsession/src/main/java/org/session/libsession/messaging/sending_receiving/pollers/ClosedGroupPollerV2.kt index 1c15c65f8e..df6b9bb52f 100644 --- a/libsession/src/main/java/org/session/libsession/messaging/sending_receiving/pollers/ClosedGroupPollerV2.kt +++ b/libsession/src/main/java/org/session/libsession/messaging/sending_receiving/pollers/ClosedGroupPollerV2.kt @@ -3,16 +3,20 @@ package org.session.libsession.messaging.sending_receiving.pollers import nl.komponents.kovenant.Promise import nl.komponents.kovenant.functional.bind import nl.komponents.kovenant.functional.map +import nl.komponents.kovenant.task import org.session.libsession.messaging.MessagingModuleConfiguration import org.session.libsession.messaging.jobs.BatchMessageReceiveJob import org.session.libsession.messaging.jobs.JobQueue -import org.session.libsession.messaging.jobs.MessageReceiveJob import org.session.libsession.messaging.jobs.MessageReceiveParameters import org.session.libsession.snode.SnodeAPI import org.session.libsession.utilities.GroupUtil import org.session.libsignal.crypto.getRandomElementOrNull import org.session.libsignal.utilities.Log -import java.util.* +import org.session.libsignal.utilities.Namespace +import org.session.libsignal.utilities.defaultRequiresAuth +import org.session.libsignal.utilities.hasNamespaces +import java.text.DateFormat +import java.util.Date import java.util.concurrent.Executors import java.util.concurrent.ScheduledFuture import java.util.concurrent.TimeUnit @@ -98,7 +102,26 @@ class ClosedGroupPollerV2 { val promise = SnodeAPI.getSwarm(groupPublicKey).bind { swarm -> val snode = swarm.getRandomElementOrNull() ?: throw InsufficientSnodesException() // Should be cryptographically secure if (!isPolling(groupPublicKey)) { throw PollingCanceledException() } - SnodeAPI.getRawMessages(snode, groupPublicKey).map { SnodeAPI.parseRawMessagesResponse(it, snode, groupPublicKey) } + val currentForkInfo = SnodeAPI.forkInfo + when { + currentForkInfo.defaultRequiresAuth() -> SnodeAPI.getRawMessages(snode, groupPublicKey, requiresAuth = false, namespace = Namespace.UNAUTHENTICATED_CLOSED_GROUP) + .map { SnodeAPI.parseRawMessagesResponse(it, snode, groupPublicKey, Namespace.UNAUTHENTICATED_CLOSED_GROUP) } + currentForkInfo.hasNamespaces() -> task { + val unAuthed = SnodeAPI.getRawMessages(snode, groupPublicKey, requiresAuth = false, namespace = Namespace.UNAUTHENTICATED_CLOSED_GROUP) + .map { SnodeAPI.parseRawMessagesResponse(it, snode, groupPublicKey, Namespace.UNAUTHENTICATED_CLOSED_GROUP) } + val default = SnodeAPI.getRawMessages(snode, groupPublicKey, requiresAuth = false, namespace = Namespace.DEFAULT) + .map { SnodeAPI.parseRawMessagesResponse(it, snode, groupPublicKey, Namespace.DEFAULT) } + val unAuthedResult = unAuthed.get() + val defaultResult = default.get() + val format = DateFormat.getTimeInstance() + if (unAuthedResult.isNotEmpty() || defaultResult.isNotEmpty()) { + Log.d("Poller", "@${format.format(Date())}Polled ${unAuthedResult.size} from -10, ${defaultResult.size} from 0") + } + unAuthedResult + defaultResult + } + else -> SnodeAPI.getRawMessages(snode, groupPublicKey, requiresAuth = false, namespace = Namespace.DEFAULT) + .map { SnodeAPI.parseRawMessagesResponse(it, snode, groupPublicKey) } + } } promise.success { envelopes -> if (!isPolling(groupPublicKey)) { return@success } diff --git a/libsession/src/main/java/org/session/libsession/snode/OnionRequestAPI.kt b/libsession/src/main/java/org/session/libsession/snode/OnionRequestAPI.kt index 5a5256c10f..e90c06ac12 100644 --- a/libsession/src/main/java/org/session/libsession/snode/OnionRequestAPI.kt +++ b/libsession/src/main/java/org/session/libsession/snode/OnionRequestAPI.kt @@ -8,20 +8,24 @@ import nl.komponents.kovenant.functional.map import okhttp3.Request import org.session.libsession.messaging.file_server.FileServerAPIV2 import org.session.libsession.utilities.AESGCM -import org.session.libsignal.utilities.Log -import org.session.libsignal.utilities.Base64 -import org.session.libsignal.utilities.* -import org.session.libsignal.utilities.Snode import org.session.libsession.utilities.AESGCM.EncryptionResult import org.session.libsession.utilities.getBodyForOnionRequest import org.session.libsession.utilities.getHeadersForOnionRequest import org.session.libsignal.crypto.getRandomElement import org.session.libsignal.crypto.getRandomElementOrNull -import org.session.libsignal.utilities.Broadcaster -import org.session.libsignal.utilities.HTTP import org.session.libsignal.database.LokiAPIDatabaseProtocol -import java.util.* -import kotlin.math.abs +import org.session.libsignal.utilities.Base64 +import org.session.libsignal.utilities.Broadcaster +import org.session.libsignal.utilities.ForkInfo +import org.session.libsignal.utilities.HTTP +import org.session.libsignal.utilities.JsonUtil +import org.session.libsignal.utilities.Log +import org.session.libsignal.utilities.Snode +import org.session.libsignal.utilities.ThreadUtils +import org.session.libsignal.utilities.recover +import org.session.libsignal.utilities.toHexString +import java.util.Date +import kotlin.collections.set private typealias Path = List @@ -356,6 +360,22 @@ object OnionRequestAPI { val offset = timestamp - Date().time SnodeAPI.clockOffset = offset } + if (body.containsKey("hf")) { + @Suppress("UNCHECKED_CAST") + val currentHf = body["hf"] as List + if (currentHf.size < 2) { + Log.e("Loki", "Response contains fork information but doesn't have a hard and soft number") + } else { + val hf = currentHf[0] + val sf = currentHf[1] + val newForkInfo = ForkInfo(hf, sf) + if (newForkInfo > SnodeAPI.forkInfo) { + SnodeAPI.forkInfo = ForkInfo(hf,sf) + } else if (newForkInfo < SnodeAPI.forkInfo) { + Log.w("Loki", "Got a new snode info fork version that was $newForkInfo, less than current known ${SnodeAPI.forkInfo}") + } + } + } if (statusCode != 200) { val exception = HTTPRequestFailedAtDestinationException(statusCode, body, destination.description) return@queue deferred.reject(exception) diff --git a/libsession/src/main/java/org/session/libsession/snode/SnodeAPI.kt b/libsession/src/main/java/org/session/libsession/snode/SnodeAPI.kt index dfbc6c34da..ffc04886aa 100644 --- a/libsession/src/main/java/org/session/libsession/snode/SnodeAPI.kt +++ b/libsession/src/main/java/org/session/libsession/snode/SnodeAPI.kt @@ -11,19 +11,33 @@ import com.goterl.lazysodium.interfaces.PwHash import com.goterl.lazysodium.interfaces.SecretBox import com.goterl.lazysodium.interfaces.Sign import com.goterl.lazysodium.utils.Key -import nl.komponents.kovenant.* +import nl.komponents.kovenant.Promise +import nl.komponents.kovenant.all +import nl.komponents.kovenant.deferred import nl.komponents.kovenant.functional.bind import nl.komponents.kovenant.functional.map +import nl.komponents.kovenant.task import org.session.libsession.messaging.MessagingModuleConfiguration import org.session.libsession.messaging.utilities.MessageWrapper import org.session.libsignal.crypto.getRandomElement import org.session.libsignal.database.LokiAPIDatabaseProtocol import org.session.libsignal.protos.SignalServiceProtos -import org.session.libsignal.utilities.* import org.session.libsignal.utilities.Base64 +import org.session.libsignal.utilities.Broadcaster +import org.session.libsignal.utilities.HTTP +import org.session.libsignal.utilities.Hex +import org.session.libsignal.utilities.Log +import org.session.libsignal.utilities.Snode +import org.session.libsignal.utilities.ThreadUtils +import org.session.libsignal.utilities.prettifiedDescription +import org.session.libsignal.utilities.retryIfNeeded import java.security.SecureRandom -import java.util.* -import kotlin.Pair +import java.util.Date +import java.util.Locale +import kotlin.collections.component1 +import kotlin.collections.component2 +import kotlin.collections.set +import kotlin.properties.Delegates.observable object SnodeAPI { private val sodium by lazy { LazySodiumAndroid(SodiumAndroid()) } @@ -41,6 +55,12 @@ object SnodeAPI { * user's clock is incorrect. */ internal var clockOffset = 0L + internal var forkInfo by observable(database.getForkInfo()) { _, oldValue, newValue -> + if (newValue > oldValue) { + Log.d("Loki", "Setting new fork info new: $newValue, old: $oldValue") + database.setForkInfo(newValue) + } + } // Settings private val maxRetryCount = 6 @@ -55,11 +75,10 @@ object SnodeAPI { setOf( "https://storage.seed1.loki.network:$seedNodePort", "https://storage.seed3.loki.network:$seedNodePort", "https://public.loki.foundation:$seedNodePort" ) } } - private val snodeFailureThreshold = 3 - private val targetSwarmSnodeCount = 2 - private val useOnionRequests = true + private const val snodeFailureThreshold = 3 + private const val useOnionRequests = true - internal val useTestnet = false + const val useTestnet = false // Error internal sealed class Error(val description: String) : Exception(description) { @@ -254,11 +273,6 @@ object SnodeAPI { return promise } - fun getTargetSnodes(publicKey: String): Promise, Exception> { - // SecureRandom() should be cryptographically secure - return getSwarm(publicKey).map { it.shuffled(SecureRandom()).take(targetSwarmSnodeCount) } - } - fun getSwarm(publicKey: String): Promise, Exception> { val cachedSwarm = database.getSwarm(publicKey) if (cachedSwarm != null && cachedSwarm.size >= minimumSwarmSnodeCount) { @@ -266,7 +280,7 @@ object SnodeAPI { cachedSwarmCopy.addAll(cachedSwarm) return task { cachedSwarmCopy } } else { - val parameters = mapOf( "pubKey" to if (useTestnet) publicKey.removing05PrefixIfNeeded() else publicKey ) + val parameters = mapOf( "pubKey" to publicKey ) return getRandomSnode().bind { invoke(Snode.Method.GetSwarm, it, publicKey, parameters) }.map { @@ -277,28 +291,39 @@ object SnodeAPI { } } - fun getRawMessages(snode: Snode, publicKey: String): RawResponsePromise { -// val userED25519KeyPair = MessagingModuleConfiguration.shared.getUserED25519KeyPair() ?: return Promise.ofFail(Error.NoKeyPair) + fun getRawMessages(snode: Snode, publicKey: String, requiresAuth: Boolean = true, namespace: Int = 0): RawResponsePromise { + val userED25519KeyPair = MessagingModuleConfiguration.shared.getUserED25519KeyPair() ?: return Promise.ofFail(Error.NoKeyPair) // Get last message hash - val lastHashValue = database.getLastMessageHashValue(snode, publicKey) ?: "" - // Construct signature -// val timestamp = Date().time + SnodeAPI.clockOffset -// val ed25519PublicKey = userED25519KeyPair.publicKey.asHexString -// val verificationData = "retrieve$timestamp".toByteArray() -// val signature = ByteArray(Sign.BYTES) -// try { -// sodium.cryptoSignDetached(signature, verificationData, verificationData.size.toLong(), userED25519KeyPair.secretKey.asBytes) -// } catch (exception: Exception) { -// return Promise.ofFail(Error.SigningFailed) -// } - // Make the request - val parameters = mapOf( - "pubKey" to if (useTestnet) publicKey.removing05PrefixIfNeeded() else publicKey, - "lastHash" to lastHashValue, -// "timestamp" to timestamp, -// "pubkey_ed25519" to ed25519PublicKey, -// "signature" to Base64.encodeBytes(signature) + val lastHashValue = database.getLastMessageHashValue(snode, publicKey, namespace) ?: "" + val parameters = mutableMapOf( + "pubKey" to publicKey, + "last_hash" to lastHashValue, ) + // Construct signature + if (requiresAuth) { + val timestamp = Date().time + SnodeAPI.clockOffset + val ed25519PublicKey = userED25519KeyPair.publicKey.asHexString + val signature = ByteArray(Sign.BYTES) + val verificationData = + if (namespace != 0) "retrieve$namespace$timestamp".toByteArray() + else "retrieve$timestamp".toByteArray() + try { + sodium.cryptoSignDetached(signature, verificationData, verificationData.size.toLong(), userED25519KeyPair.secretKey.asBytes) + } catch (exception: Exception) { + return Promise.ofFail(Error.SigningFailed) + } + parameters["timestamp"] = timestamp + parameters["pubkey_ed25519"] = ed25519PublicKey + parameters["signature"] = Base64.encodeBytes(signature) + } + + // If the namespace is default (0) here it will be implicitly read as 0 on the storage server + // we only need to specify it explicitly if we want to (in future) or if it is non-zero + if (namespace != 0) { + parameters["namespace"] = namespace + } + + // Make the request return invoke(Snode.Method.GetMessages, snode, publicKey, parameters) } @@ -317,14 +342,35 @@ object SnodeAPI { } } - fun sendMessage(message: SnodeMessage): Promise, Exception> { - val destination = if (useTestnet) message.recipient.removing05PrefixIfNeeded() else message.recipient + fun sendMessage(message: SnodeMessage, requiresAuth: Boolean = false, namespace: Int = 0): RawResponsePromise { + val destination = message.recipient return retryIfNeeded(maxRetryCount) { - getTargetSnodes(destination).map { swarm -> - swarm.map { snode -> - val parameters = message.toJSON() - invoke(Snode.Method.SendMessage, snode, destination, parameters) - }.toSet() + val module = MessagingModuleConfiguration.shared + val userED25519KeyPair = module.getUserED25519KeyPair() ?: return@retryIfNeeded Promise.ofFail(Error.NoKeyPair) + val parameters = message.toJSON().toMutableMap() + // Construct signature + if (requiresAuth) { + val sigTimestamp = System.currentTimeMillis() + SnodeAPI.clockOffset + val ed25519PublicKey = userED25519KeyPair.publicKey.asHexString + val signature = ByteArray(Sign.BYTES) + // assume namespace here is non-zero, as zero namespace doesn't require auth + val verificationData = "store$namespace$sigTimestamp".toByteArray() + try { + sodium.cryptoSignDetached(signature, verificationData, verificationData.size.toLong(), userED25519KeyPair.secretKey.asBytes) + } catch (exception: Exception) { + return@retryIfNeeded Promise.ofFail(Error.SigningFailed) + } + parameters["sig_timestamp"] = sigTimestamp + parameters["pubkey_ed25519"] = ed25519PublicKey + parameters["signature"] = Base64.encodeBytes(signature) + } + // If the namespace is default (0) here it will be implicitly read as 0 on the storage server + // we only need to specify it explicitly if we want to (in future) or if it is non-zero + if (namespace != 0) { + parameters["namespace"] = namespace + } + getSingleTargetSnode(destination).bind { snode -> + invoke(Snode.Method.SendMessage, snode, destination, parameters) } } } @@ -426,29 +472,29 @@ object SnodeAPI { } } - fun parseRawMessagesResponse(rawResponse: RawResponse, snode: Snode, publicKey: String): List> { + fun parseRawMessagesResponse(rawResponse: RawResponse, snode: Snode, publicKey: String, namespace: Int = 0): List> { val messages = rawResponse["messages"] as? List<*> return if (messages != null) { - updateLastMessageHashValueIfPossible(snode, publicKey, messages) - val newRawMessages = removeDuplicates(publicKey, messages) + updateLastMessageHashValueIfPossible(snode, publicKey, messages, namespace) + val newRawMessages = removeDuplicates(publicKey, messages, namespace) return parseEnvelopes(newRawMessages); } else { listOf() } } - private fun updateLastMessageHashValueIfPossible(snode: Snode, publicKey: String, rawMessages: List<*>) { + private fun updateLastMessageHashValueIfPossible(snode: Snode, publicKey: String, rawMessages: List<*>, namespace: Int) { val lastMessageAsJSON = rawMessages.lastOrNull() as? Map<*, *> val hashValue = lastMessageAsJSON?.get("hash") as? String if (hashValue != null) { - database.setLastMessageHashValue(snode, publicKey, hashValue) + database.setLastMessageHashValue(snode, publicKey, hashValue, namespace) } else if (rawMessages.isNotEmpty()) { Log.d("Loki", "Failed to update last message hash value from: ${rawMessages.prettifiedDescription()}.") } } - private fun removeDuplicates(publicKey: String, rawMessages: List<*>): List<*> { - val receivedMessageHashValues = database.getReceivedMessageHashValues(publicKey)?.toMutableSet() ?: mutableSetOf() + private fun removeDuplicates(publicKey: String, rawMessages: List<*>, namespace: Int): List<*> { + val receivedMessageHashValues = database.getReceivedMessageHashValues(publicKey, namespace)?.toMutableSet() ?: mutableSetOf() val result = rawMessages.filter { rawMessage -> val rawMessageAsJSON = rawMessage as? Map<*, *> val hashValue = rawMessageAsJSON?.get("hash") as? String @@ -461,7 +507,7 @@ object SnodeAPI { false } } - database.setReceivedMessageHashValues(publicKey, receivedMessageHashValues) + database.setReceivedMessageHashValues(publicKey, receivedMessageHashValues, namespace) return result } diff --git a/libsession/src/main/java/org/session/libsession/snode/SnodeMessage.kt b/libsession/src/main/java/org/session/libsession/snode/SnodeMessage.kt index 0a4def6f5c..31ced16209 100644 --- a/libsession/src/main/java/org/session/libsession/snode/SnodeMessage.kt +++ b/libsession/src/main/java/org/session/libsession/snode/SnodeMessage.kt @@ -1,7 +1,5 @@ package org.session.libsession.snode -import org.session.libsignal.utilities.removing05PrefixIfNeeded - data class SnodeMessage( /** * The hex encoded public key of the recipient. @@ -25,11 +23,10 @@ data class SnodeMessage( internal fun toJSON(): Map { return mapOf( - "pubKey" to if (SnodeAPI.useTestnet) recipient.removing05PrefixIfNeeded() else recipient, + "pubKey" to recipient, "data" to data, "ttl" to ttl.toString(), "timestamp" to timestamp.toString(), - "nonce" to "" ) } } diff --git a/libsignal/src/main/java/org/session/libsignal/database/LokiAPIDatabaseProtocol.kt b/libsignal/src/main/java/org/session/libsignal/database/LokiAPIDatabaseProtocol.kt index 2390b29768..1bf093128d 100644 --- a/libsignal/src/main/java/org/session/libsignal/database/LokiAPIDatabaseProtocol.kt +++ b/libsignal/src/main/java/org/session/libsignal/database/LokiAPIDatabaseProtocol.kt @@ -1,8 +1,9 @@ package org.session.libsignal.database import org.session.libsignal.crypto.ecc.ECKeyPair +import org.session.libsignal.utilities.ForkInfo import org.session.libsignal.utilities.Snode -import java.util.* +import java.util.Date interface LokiAPIDatabaseProtocol { @@ -13,10 +14,10 @@ interface LokiAPIDatabaseProtocol { fun setOnionRequestPaths(newValue: List>) fun getSwarm(publicKey: String): Set? fun setSwarm(publicKey: String, newValue: Set) - fun getLastMessageHashValue(snode: Snode, publicKey: String): String? - fun setLastMessageHashValue(snode: Snode, publicKey: String, newValue: String) - fun getReceivedMessageHashValues(publicKey: String): Set? - fun setReceivedMessageHashValues(publicKey: String, newValue: Set) + fun getLastMessageHashValue(snode: Snode, publicKey: String, namespace: Int): String? + fun setLastMessageHashValue(snode: Snode, publicKey: String, newValue: String, namespace: Int) + fun getReceivedMessageHashValues(publicKey: String, namespace: Int): Set? + fun setReceivedMessageHashValues(publicKey: String, newValue: Set, namespace: Int) fun getAuthToken(server: String): String? fun setAuthToken(server: String, newValue: String?) fun setUserCount(group: Long, server: String, newValue: Int) @@ -33,4 +34,7 @@ interface LokiAPIDatabaseProtocol { fun getClosedGroupEncryptionKeyPairs(groupPublicKey: String): List fun getLatestClosedGroupEncryptionKeyPair(groupPublicKey: String): ECKeyPair? fun isClosedGroup(groupPublicKey: String): Boolean + fun getForkInfo(): ForkInfo + fun setForkInfo(forkInfo: ForkInfo) + } diff --git a/libsignal/src/main/java/org/session/libsignal/utilities/ForkInfo.kt b/libsignal/src/main/java/org/session/libsignal/utilities/ForkInfo.kt new file mode 100644 index 0000000000..10ebdf533e --- /dev/null +++ b/libsignal/src/main/java/org/session/libsignal/utilities/ForkInfo.kt @@ -0,0 +1,20 @@ +package org.session.libsignal.utilities + +data class ForkInfo(val hf: Int, val sf: Int) { + companion object { + const val DEFAULT_HF = 18 + const val DEFAULT_SF = 1 + val DEFAULT = ForkInfo(DEFAULT_HF, DEFAULT_SF) + val baseTable = arrayOf(10,100,1000,10000,100000) + } + + operator fun compareTo(other: ForkInfo): Int { + val base = baseTable.first { it > sf && it > other.sf } + return (hf*base - other.hf*base) + (sf - other.sf) + } + +} + +// add info here for when various features are active +fun ForkInfo.hasNamespaces() = hf >= 19 +fun ForkInfo.defaultRequiresAuth() = hf >= 19 && sf >= 1 \ No newline at end of file diff --git a/libsignal/src/main/java/org/session/libsignal/utilities/Namespace.kt b/libsignal/src/main/java/org/session/libsignal/utilities/Namespace.kt new file mode 100644 index 0000000000..1c635d9934 --- /dev/null +++ b/libsignal/src/main/java/org/session/libsignal/utilities/Namespace.kt @@ -0,0 +1,7 @@ +package org.session.libsignal.utilities + +object Namespace { + const val DEFAULT = 0 + const val UNAUTHENTICATED_CLOSED_GROUP = -10 + const val CONFIGURATION = 5 +} \ No newline at end of file