Namespace retrieval and storage with auth (#880)

* feat: add migration and fork info for upcoming auth and closed group retrieval updates

* feat: add closed group poller calls and include namespace to parse raw messages function

* feat: add DB upgrades and queries for namespaces

* fix: fix the polling for post-HF signatures and group messages

* fix: realise we need a compound key for namespaces in received hashes, test explicitly setting namespace

* feat: add setForkInfo implementation

* refactor: include default fork info command on create, refactor migration to use new table since we can't add constraints in alter for PK, replace `lastHash` with `last_hash` in case that fixes paging

* refactor: include namespace and use when statement for closed group polling

* refactor: revert to main net

* refactor: use namespace constants

* refactor: revert to testnet and log the poll result

* fix: use or to log either poller

* fix: revert to default network and add more logging, only set the latest fork info if it is an increment

* build: update minor version

* refactor: use single target snode and namespace list for message sending

* fix: link previews and expiring messages in closed groups

app/build.gradle

@@ -158,8 +158,8 @@ dependencies {
     testImplementation 'org.robolectric:shadows-multidex:4.4'
 }
 
-def canonicalVersionCode = 272
+def canonicalVersionCode = 273
-def canonicalVersionName = "1.12.15"
+def canonicalVersionName = "1.13.0"
 
 def postFixSize = 10
 def abiPostFix = ['armeabi-v7a' : 1,

app/src/main/java/org/thoughtcrime/securesms/conversation/v2/menus/ConversationMenuHelper.kt

@@ -68,7 +68,7 @@ object ConversationMenuHelper {
         // Base menu (options that should always be present)
         inflater.inflate(R.menu.menu_conversation, menu)
         // Expiring messages
-        if (!isOpenGroup && thread.hasApprovedMe()) {
+        if (!isOpenGroup && (thread.hasApprovedMe() || thread.isClosedGroupRecipient)) {
             if (thread.expireMessages > 0) {
                 inflater.inflate(R.menu.menu_conversation_expiration_on, menu)
                 val item = menu.findItem(R.id.menu_expiring_messages)

app/src/main/java/org/thoughtcrime/securesms/conversation/v2/messages/VisibleMessageContentView.kt

@@ -98,7 +98,7 @@ class VisibleMessageContentView : LinearLayout {
         linkPreviewLayout.width = if (mediaThumbnailMessage) 0 else ViewGroup.LayoutParams.WRAP_CONTENT
         binding.linkPreviewView.layoutParams = linkPreviewLayout
 
-        binding.untrustedView.isVisible = !contactIsTrusted && message is MmsMessageRecord && message.quote == null
+        binding.untrustedView.isVisible = !contactIsTrusted && message is MmsMessageRecord && message.quote == null && message.linkPreviews.isEmpty()
         binding.voiceMessageView.isVisible = contactIsTrusted && message is MmsMessageRecord && message.slideDeck.audioSlide != null
         binding.documentView.isVisible = contactIsTrusted && message is MmsMessageRecord && message.slideDeck.documentSlide != null
         binding.albumThumbnailView.isVisible = mediaThumbnailMessage

app/src/main/java/org/thoughtcrime/securesms/conversation/v2/utilities/MentionManagerUtilities.kt

@@ -9,9 +9,6 @@ import org.thoughtcrime.securesms.dependencies.DatabaseComponent
 object MentionManagerUtilities {
 
     fun populateUserPublicKeyCacheIfNeeded(threadID: Long, context: Context) {
-        // exit early if we need to
-        if (MentionsManager.userPublicKeyCache[threadID] != null) return
-
         val result = mutableSetOf<String>()
         val recipient = DatabaseComponent.get(context).threadDatabase().getRecipientForThreadId(threadID) ?: return
         if (recipient.address.isClosedGroup) {

app/src/main/java/org/thoughtcrime/securesms/database/LokiAPIDatabase.kt

@@ -7,28 +7,24 @@ import org.session.libsignal.crypto.ecc.DjbECPrivateKey
 import org.session.libsignal.crypto.ecc.DjbECPublicKey
 import org.session.libsignal.crypto.ecc.ECKeyPair
 import org.session.libsignal.database.LokiAPIDatabaseProtocol
-import org.session.libsignal.utilities.*
+import org.session.libsignal.utilities.ForkInfo
+import org.session.libsignal.utilities.Hex
+import org.session.libsignal.utilities.Log
+import org.session.libsignal.utilities.PublicKeyValidation
+import org.session.libsignal.utilities.Snode
+import org.session.libsignal.utilities.removing05PrefixIfNeeded
+import org.session.libsignal.utilities.toHexString
 import org.thoughtcrime.securesms.crypto.IdentityKeyUtil
-import org.thoughtcrime.securesms.database.*
 import org.thoughtcrime.securesms.database.helpers.SQLCipherOpenHelper
-import org.thoughtcrime.securesms.util.*
+import java.util.Date
-import java.util.*
-import kotlin.Array
-import kotlin.Boolean
-import kotlin.Int
-import kotlin.Long
-import kotlin.Pair
-import kotlin.String
-import kotlin.arrayOf
-import kotlin.to
 
 class LokiAPIDatabase(context: Context, helper: SQLCipherOpenHelper) : Database(context, helper), LokiAPIDatabaseProtocol {
 
     companion object {
         // Shared
-        private val publicKey = "public_key"
+        private const val publicKey = "public_key"
-        private val timestamp = "timestamp"
+        private const val timestamp = "timestamp"
-        private val snode = "snode"
+        private const val snode = "snode"
         // Snode pool
         public val snodePoolTable = "loki_snode_pool_cache"
         private val dummyKey = "dummy_key"
@@ -44,15 +40,19 @@ class LokiAPIDatabase(context: Context, helper: SQLCipherOpenHelper) : Database(
         private val swarm = "swarm"
         @JvmStatic val createSwarmTableCommand = "CREATE TABLE $swarmTable ($swarmPublicKey TEXT PRIMARY KEY, $swarm TEXT);"
         // Last message hash values
-        private val lastMessageHashValueTable2 = "last_message_hash_value_table"
+        private const val legacyLastMessageHashValueTable2 = "last_message_hash_value_table"
-        private val lastMessageHashValue = "last_message_hash_value"
+        private const val lastMessageHashValueTable2 = "session_last_message_hash_value_table"
+        private const val lastMessageHashValue = "last_message_hash_value"
+        private const val lastMessageHashNamespace = "last_message_namespace"
         @JvmStatic val createLastMessageHashValueTable2Command
-            = "CREATE TABLE $lastMessageHashValueTable2 ($snode TEXT, $publicKey TEXT, $lastMessageHashValue TEXT, PRIMARY KEY ($snode, $publicKey));"
+            = "CREATE TABLE $legacyLastMessageHashValueTable2 ($snode TEXT, $publicKey TEXT, $lastMessageHashValue TEXT, PRIMARY KEY ($snode, $publicKey));"
         // Received message hash values
-        private val receivedMessageHashValuesTable3 = "received_message_hash_values_table_3"
+        private const val legacyReceivedMessageHashValuesTable3 = "received_message_hash_values_table_3"
-        private val receivedMessageHashValues = "received_message_hash_values"
+        private const val receivedMessageHashValuesTable = "session_received_message_hash_values_table"
+        private const val receivedMessageHashValues = "received_message_hash_values"
+        private const val receivedMessageHashNamespace = "received_message_namespace"
         @JvmStatic val createReceivedMessageHashValuesTable3Command
-            = "CREATE TABLE $receivedMessageHashValuesTable3 ($publicKey STRING PRIMARY KEY, $receivedMessageHashValues TEXT);"
+            = "CREATE TABLE $legacyReceivedMessageHashValuesTable3 ($publicKey STRING PRIMARY KEY, $receivedMessageHashValues TEXT);"
         // Open group auth tokens
         private val openGroupAuthTokenTable = "loki_api_group_chat_auth_token_database"
         private val server = "server"
@@ -97,6 +97,31 @@ class LokiAPIDatabase(context: Context, helper: SQLCipherOpenHelper) : Database(
         public val groupPublicKey = "group_public_key"
         @JvmStatic
         val createClosedGroupPublicKeysTable = "CREATE TABLE $closedGroupPublicKeysTable ($groupPublicKey STRING PRIMARY KEY)"
+        // Hard fork service node info
+        const val FORK_INFO_TABLE = "fork_info"
+        const val DUMMY_KEY = "dummy_key"
+        const val DUMMY_VALUE = "1"
+        const val HF_VALUE = "hf_value"
+        const val SF_VALUE = "sf_value"
+        const val CREATE_FORK_INFO_TABLE_COMMAND = "CREATE TABLE $FORK_INFO_TABLE ($DUMMY_KEY INTEGER PRIMARY KEY, $HF_VALUE INTEGER, $SF_VALUE INTEGER);"
+        const val CREATE_DEFAULT_FORK_INFO_COMMAND = "INSERT INTO $FORK_INFO_TABLE ($DUMMY_KEY, $HF_VALUE, $SF_VALUE) VALUES ($DUMMY_VALUE, 18, 1);"
+
+        const val UPDATE_HASHES_INCLUDE_NAMESPACE_COMMAND = """
+            CREATE TABLE IF NOT EXISTS $lastMessageHashValueTable2(
+                $snode TEXT, $publicKey TEXT, $lastMessageHashValue TEXT, $lastMessageHashNamespace INTEGER DEFAULT 0, PRIMARY KEY ($snode, $publicKey, $lastMessageHashNamespace)
+            );
+            INSERT INTO $lastMessageHashValueTable2($snode, $publicKey, $lastMessageHashValue) SELECT $snode, $publicKey, $lastMessageHashValue FROM $legacyLastMessageHashValueTable2);
+            DROP TABLE $legacyLastMessageHashValueTable2;
+        """
+        const val UPDATE_RECEIVED_INCLUDE_NAMESPACE_COMMAND = """
+            CREATE TABLE IF NOT EXISTS $receivedMessageHashValuesTable(
+                $publicKey STRING, $receivedMessageHashValues TEXT, $receivedMessageHashNamespace INTEGER DEFAULT 0, PRIMARY KEY ($publicKey, $receivedMessageHashNamespace)
+            );
+            
+            INSERT INTO $receivedMessageHashValuesTable($publicKey, $receivedMessageHashValues) SELECT $publicKey, $receivedMessageHashValues FROM $legacyReceivedMessageHashValuesTable3;
+            
+            DROP TABLE $legacyReceivedMessageHashValuesTable3;
+        """
 
         // region Deprecated
         private val deviceLinkCache = "loki_pairing_authorisation_cache"
@@ -232,36 +257,45 @@ class LokiAPIDatabase(context: Context, helper: SQLCipherOpenHelper) : Database(
         database.insertOrUpdate(swarmTable, row, "${Companion.swarmPublicKey} = ?", wrap(publicKey))
     }
 
-    override fun getLastMessageHashValue(snode: Snode, publicKey: String): String? {
+    override fun getLastMessageHashValue(snode: Snode, publicKey: String, namespace: Int): String? {
         val database = databaseHelper.readableDatabase
-        val query = "${Companion.snode} = ? AND ${Companion.publicKey} = ?"
+        val query = "${Companion.snode} = ? AND ${Companion.publicKey} = ? AND $lastMessageHashNamespace = ?"
-        return database.get(lastMessageHashValueTable2, query, arrayOf( snode.toString(), publicKey )) { cursor ->
+        return database.get(lastMessageHashValueTable2, query, arrayOf( snode.toString(), publicKey, namespace.toString() )) { cursor ->
             cursor.getString(cursor.getColumnIndexOrThrow(lastMessageHashValue))
         }
     }
 
-    override fun setLastMessageHashValue(snode: Snode, publicKey: String, newValue: String) {
+    override fun setLastMessageHashValue(snode: Snode, publicKey: String, newValue: String, namespace: Int) {
         val database = databaseHelper.writableDatabase
-        val row = wrap(mapOf( Companion.snode to snode.toString(), Companion.publicKey to publicKey, lastMessageHashValue to newValue ))
+        val row = wrap(mapOf(
-        val query = "${Companion.snode} = ? AND ${Companion.publicKey} = ?"
+            Companion.snode to snode.toString(),
-        database.insertOrUpdate(lastMessageHashValueTable2, row, query, arrayOf( snode.toString(), publicKey ))
+            Companion.publicKey to publicKey,
+            lastMessageHashValue to newValue,
+            lastMessageHashNamespace to namespace.toString()
+        ))
+        val query = "${Companion.snode} = ? AND ${Companion.publicKey} = ? AND $lastMessageHashNamespace = ?"
+        database.insertOrUpdate(lastMessageHashValueTable2, row, query, arrayOf( snode.toString(), publicKey, namespace.toString() ))
     }
 
-    override fun getReceivedMessageHashValues(publicKey: String): Set<String>? {
+    override fun getReceivedMessageHashValues(publicKey: String, namespace: Int): Set<String>? {
         val database = databaseHelper.readableDatabase
-        val query = "${Companion.publicKey} = ?"
+        val query = "${Companion.publicKey} = ? AND ${Companion.receivedMessageHashNamespace} = ?"
-        return database.get(receivedMessageHashValuesTable3, query, arrayOf( publicKey )) { cursor ->
+        return database.get(receivedMessageHashValuesTable, query, arrayOf( publicKey, namespace.toString() )) { cursor ->
             val receivedMessageHashValuesAsString = cursor.getString(cursor.getColumnIndexOrThrow(Companion.receivedMessageHashValues))
             receivedMessageHashValuesAsString.split("-").toSet()
         }
     }
 
-    override fun setReceivedMessageHashValues(publicKey: String, newValue: Set<String>) {
+    override fun setReceivedMessageHashValues(publicKey: String, newValue: Set<String>, namespace: Int) {
         val database = databaseHelper.writableDatabase
         val receivedMessageHashValuesAsString = newValue.joinToString("-")
-        val row = wrap(mapOf( Companion.publicKey to publicKey, Companion.receivedMessageHashValues to receivedMessageHashValuesAsString ))
+        val row = wrap(mapOf(
-        val query = "${Companion.publicKey} = ?"
+            Companion.publicKey to publicKey,
-        database.insertOrUpdate(receivedMessageHashValuesTable3, row, query, arrayOf( publicKey ))
+            Companion.receivedMessageHashValues to receivedMessageHashValuesAsString,
+            Companion.receivedMessageHashNamespace to namespace.toString()
+        ))
+        val query = "${Companion.publicKey} = ? AND $receivedMessageHashNamespace = ?"
+        database.insertOrUpdate(receivedMessageHashValuesTable, row, query, arrayOf( publicKey, namespace.toString() ))
     }
 
     override fun getAuthToken(server: String): String? {
@@ -441,6 +475,29 @@ class LokiAPIDatabase(context: Context, helper: SQLCipherOpenHelper) : Database(
         val database = databaseHelper.writableDatabase
         database.delete(closedGroupPublicKeysTable, "${Companion.groupPublicKey} = ?", wrap(groupPublicKey))
     }
+
+    override fun getForkInfo(): ForkInfo {
+        val database = databaseHelper.readableDatabase
+        val queryCursor = database.query(FORK_INFO_TABLE, arrayOf(HF_VALUE, SF_VALUE), "$DUMMY_KEY = $DUMMY_VALUE", null, null, null, null)
+        val forkInfo = queryCursor.use { cursor ->
+            if (!cursor.moveToNext()) {
+                ForkInfo(18, 1) // no HF info, none set will at least be the version
+            } else {
+                ForkInfo(cursor.getInt(0), cursor.getInt(1))
+            }
+        }
+        return forkInfo
+    }
+
+    override fun setForkInfo(forkInfo: ForkInfo) {
+        val database = databaseHelper.writableDatabase
+        val query = "$DUMMY_KEY = $DUMMY_VALUE"
+        val contentValues = ContentValues(3)
+        contentValues.put(DUMMY_KEY, DUMMY_VALUE)
+        contentValues.put(HF_VALUE, forkInfo.hf)
+        contentValues.put(SF_VALUE, forkInfo.sf)
+        database.insertOrUpdate(FORK_INFO_TABLE, contentValues, query, emptyArray())
+    }
 }
 
 // region Convenience

app/src/main/java/org/thoughtcrime/securesms/database/helpers/SQLCipherOpenHelper.java

@@ -64,9 +64,10 @@ public class SQLCipherOpenHelper extends SQLiteOpenHelper {
   private static final int lokiV30                          = 51;
   private static final int lokiV31                          = 52;
   private static final int lokiV32                          = 53;
+  private static final int lokiV33                          = 54;
 
   // Loki - onUpgrade(...) must be updated to use Loki version numbers if Signal makes any database changes
-  private static final int    DATABASE_VERSION = lokiV32;
+  private static final int    DATABASE_VERSION = lokiV33;
   private static final String DATABASE_NAME    = "signal.db";
 
   private final Context        context;
@@ -143,6 +144,10 @@ public class SQLCipherOpenHelper extends SQLiteOpenHelper {
     db.execSQL(RecipientDatabase.getCreateApprovedCommand());
     db.execSQL(RecipientDatabase.getCreateApprovedMeCommand());
     db.execSQL(MmsDatabase.getCreateMessageRequestResponseCommand());
+    db.execSQL(LokiAPIDatabase.CREATE_FORK_INFO_TABLE_COMMAND);
+    db.execSQL(LokiAPIDatabase.CREATE_DEFAULT_FORK_INFO_COMMAND);
+    db.execSQL(LokiAPIDatabase.UPDATE_HASHES_INCLUDE_NAMESPACE_COMMAND);
+    db.execSQL(LokiAPIDatabase.UPDATE_RECEIVED_INCLUDE_NAMESPACE_COMMAND);
 
     executeStatements(db, SmsDatabase.CREATE_INDEXS);
     executeStatements(db, MmsDatabase.CREATE_INDEXS);
@@ -337,6 +342,13 @@ public class SQLCipherOpenHelper extends SQLiteOpenHelper {
         db.execSQL(RecipientDatabase.getUpdateApprovedSelectConversations());
       }
 
+      if (oldVersion < lokiV33) {
+        db.execSQL(LokiAPIDatabase.CREATE_FORK_INFO_TABLE_COMMAND);
+        db.execSQL(LokiAPIDatabase.CREATE_DEFAULT_FORK_INFO_COMMAND);
+        db.execSQL(LokiAPIDatabase.UPDATE_HASHES_INCLUDE_NAMESPACE_COMMAND);
+        db.execSQL(LokiAPIDatabase.UPDATE_RECEIVED_INCLUDE_NAMESPACE_COMMAND);
+      }
+
       db.setTransactionSuccessful();
     } finally {
       db.endTransaction();

libsession/src/main/java/org/session/libsession/messaging/sending_receiving/MessageSender.kt

@@ -8,9 +8,17 @@ import org.session.libsession.messaging.jobs.MessageSendJob
 import org.session.libsession.messaging.jobs.NotifyPNServerJob
 import org.session.libsession.messaging.messages.Destination
 import org.session.libsession.messaging.messages.Message
-import org.session.libsession.messaging.messages.control.*
+import org.session.libsession.messaging.messages.control.CallMessage
-import org.session.libsession.messaging.messages.visible.*
+import org.session.libsession.messaging.messages.control.ClosedGroupControlMessage
-import org.session.libsession.messaging.open_groups.*
+import org.session.libsession.messaging.messages.control.ConfigurationMessage
+import org.session.libsession.messaging.messages.control.ExpirationTimerUpdate
+import org.session.libsession.messaging.messages.control.UnsendRequest
+import org.session.libsession.messaging.messages.visible.LinkPreview
+import org.session.libsession.messaging.messages.visible.Profile
+import org.session.libsession.messaging.messages.visible.Quote
+import org.session.libsession.messaging.messages.visible.VisibleMessage
+import org.session.libsession.messaging.open_groups.OpenGroupAPIV2
+import org.session.libsession.messaging.open_groups.OpenGroupMessageV2
 import org.session.libsession.messaging.utilities.MessageWrapper
 import org.session.libsession.snode.RawResponsePromise
 import org.session.libsession.snode.SnodeAPI
@@ -22,8 +30,11 @@ import org.session.libsession.utilities.SSKEnvironment
 import org.session.libsignal.crypto.PushTransportDetails
 import org.session.libsignal.protos.SignalServiceProtos
 import org.session.libsignal.utilities.Base64
-import org.session.libsignal.utilities.Log
+import org.session.libsignal.utilities.Namespace
+import org.session.libsignal.utilities.defaultRequiresAuth
+import org.session.libsignal.utilities.hasNamespaces
 import org.session.libsignal.utilities.hexEncodedPublicKey
+import java.util.concurrent.atomic.AtomicInteger
 import org.session.libsession.messaging.sending_receiving.attachments.Attachment as SignalAttachment
 import org.session.libsession.messaging.sending_receiving.link_preview.LinkPreview as SignalLinkPreview
 import org.session.libsession.messaging.sending_receiving.quotes.QuoteModel as SignalQuote
@@ -125,6 +136,15 @@ object MessageSender {
             // Wrap the result
             val kind: SignalServiceProtos.Envelope.Type
             val senderPublicKey: String
+            // TODO: this might change in future for config messages
+            val forkInfo = SnodeAPI.forkInfo
+            val namespaces: List<Int> = when {
+                destination is Destination.ClosedGroup
+                        && forkInfo.defaultRequiresAuth() -> listOf(Namespace.UNAUTHENTICATED_CLOSED_GROUP)
+                destination is Destination.ClosedGroup
+                        && forkInfo.hasNamespaces() -> listOf(Namespace.UNAUTHENTICATED_CLOSED_GROUP, Namespace.DEFAULT)
+                else -> listOf(Namespace.DEFAULT)
+            }
             when (destination) {
                 is Destination.Contact -> {
                     kind = SignalServiceProtos.Envelope.Type.SESSION_MESSAGE
@@ -148,11 +168,11 @@ object MessageSender {
             if (destination is Destination.Contact && message is VisibleMessage && !isSelfSend) {
                 SnodeModule.shared.broadcaster.broadcast("sendingMessage", message.sentTimestamp!!)
             }
-            SnodeAPI.sendMessage(snodeMessage).success { promises: Set<RawResponsePromise> ->
+            namespaces.map { namespace -> SnodeAPI.sendMessage(snodeMessage, requiresAuth = false, namespace = namespace) }.let { promises ->
                 var isSuccess = false
                 val promiseCount = promises.size
-                var errorCount = 0
+                var errorCount = AtomicInteger(0)
-                promises.iterator().forEach { promise: RawResponsePromise ->
+                promises.forEach { promise: RawResponsePromise ->
                     promise.success {
                         if (isSuccess) { return@success } // Succeed as soon as the first promise succeeds
                         isSuccess = true
@@ -162,7 +182,7 @@ object MessageSender {
                         val hash = it["hash"] as? String
                         message.serverHash = hash
                         handleSuccessfulMessageSend(message, destination, isSyncMessage)
-                        var shouldNotify = ((message is VisibleMessage || message is UnsendRequest || message is CallMessage) && !isSyncMessage)
+                        val shouldNotify = ((message is VisibleMessage || message is UnsendRequest || message is CallMessage) && !isSyncMessage)
                         /*
                         if (message is ClosedGroupControlMessage && message.kind is ClosedGroupControlMessage.Kind.New) {
                             shouldNotify = true
@@ -175,14 +195,11 @@ object MessageSender {
                         deferred.resolve(Unit)
                     }
                     promise.fail {
-                        errorCount += 1
+                        errorCount.getAndIncrement()
-                        if (errorCount != promiseCount) { return@fail } // Only error out if all promises failed
+                        if (errorCount.get() != promiseCount) { return@fail } // Only error out if all promises failed
                         handleFailure(it)
                     }
                 }
-            }.fail {
-                Log.d("Loki", "Couldn't send message due to error: $it.")
-                handleFailure(it)
             }
         } catch (exception: Exception) {
             handleFailure(exception)

libsession/src/main/java/org/session/libsession/messaging/sending_receiving/pollers/ClosedGroupPollerV2.kt

@@ -3,16 +3,20 @@ package org.session.libsession.messaging.sending_receiving.pollers
 import nl.komponents.kovenant.Promise
 import nl.komponents.kovenant.functional.bind
 import nl.komponents.kovenant.functional.map
+import nl.komponents.kovenant.task
 import org.session.libsession.messaging.MessagingModuleConfiguration
 import org.session.libsession.messaging.jobs.BatchMessageReceiveJob
 import org.session.libsession.messaging.jobs.JobQueue
-import org.session.libsession.messaging.jobs.MessageReceiveJob
 import org.session.libsession.messaging.jobs.MessageReceiveParameters
 import org.session.libsession.snode.SnodeAPI
 import org.session.libsession.utilities.GroupUtil
 import org.session.libsignal.crypto.getRandomElementOrNull
 import org.session.libsignal.utilities.Log
-import java.util.*
+import org.session.libsignal.utilities.Namespace
+import org.session.libsignal.utilities.defaultRequiresAuth
+import org.session.libsignal.utilities.hasNamespaces
+import java.text.DateFormat
+import java.util.Date
 import java.util.concurrent.Executors
 import java.util.concurrent.ScheduledFuture
 import java.util.concurrent.TimeUnit
@@ -98,7 +102,26 @@ class ClosedGroupPollerV2 {
         val promise = SnodeAPI.getSwarm(groupPublicKey).bind { swarm ->
             val snode = swarm.getRandomElementOrNull() ?: throw InsufficientSnodesException() // Should be cryptographically secure
             if (!isPolling(groupPublicKey)) { throw PollingCanceledException() }
-            SnodeAPI.getRawMessages(snode, groupPublicKey).map { SnodeAPI.parseRawMessagesResponse(it, snode, groupPublicKey) }
+            val currentForkInfo = SnodeAPI.forkInfo
+            when {
+                currentForkInfo.defaultRequiresAuth() -> SnodeAPI.getRawMessages(snode, groupPublicKey, requiresAuth = false, namespace = Namespace.UNAUTHENTICATED_CLOSED_GROUP)
+                    .map { SnodeAPI.parseRawMessagesResponse(it, snode, groupPublicKey, Namespace.UNAUTHENTICATED_CLOSED_GROUP) }
+                currentForkInfo.hasNamespaces() -> task {
+                    val unAuthed = SnodeAPI.getRawMessages(snode, groupPublicKey, requiresAuth = false, namespace = Namespace.UNAUTHENTICATED_CLOSED_GROUP)
+                        .map { SnodeAPI.parseRawMessagesResponse(it, snode, groupPublicKey, Namespace.UNAUTHENTICATED_CLOSED_GROUP) }
+                    val default = SnodeAPI.getRawMessages(snode, groupPublicKey, requiresAuth = false, namespace = Namespace.DEFAULT)
+                        .map { SnodeAPI.parseRawMessagesResponse(it, snode, groupPublicKey, Namespace.DEFAULT) }
+                    val unAuthedResult = unAuthed.get()
+                    val defaultResult = default.get()
+                    val format = DateFormat.getTimeInstance()
+                    if (unAuthedResult.isNotEmpty() || defaultResult.isNotEmpty()) {
+                        Log.d("Poller", "@${format.format(Date())}Polled ${unAuthedResult.size} from -10, ${defaultResult.size} from 0")
+                    }
+                    unAuthedResult + defaultResult
+                }
+                else -> SnodeAPI.getRawMessages(snode, groupPublicKey, requiresAuth = false, namespace = Namespace.DEFAULT)
+                    .map { SnodeAPI.parseRawMessagesResponse(it, snode, groupPublicKey) }
+            }
         }
         promise.success { envelopes ->
             if (!isPolling(groupPublicKey)) { return@success }

libsession/src/main/java/org/session/libsession/snode/OnionRequestAPI.kt

@@ -8,20 +8,24 @@ import nl.komponents.kovenant.functional.map
 import okhttp3.Request
 import org.session.libsession.messaging.file_server.FileServerAPIV2
 import org.session.libsession.utilities.AESGCM
-import org.session.libsignal.utilities.Log
-import org.session.libsignal.utilities.Base64
-import org.session.libsignal.utilities.*
-import org.session.libsignal.utilities.Snode
 import org.session.libsession.utilities.AESGCM.EncryptionResult
 import org.session.libsession.utilities.getBodyForOnionRequest
 import org.session.libsession.utilities.getHeadersForOnionRequest
 import org.session.libsignal.crypto.getRandomElement
 import org.session.libsignal.crypto.getRandomElementOrNull
-import org.session.libsignal.utilities.Broadcaster
-import org.session.libsignal.utilities.HTTP
 import org.session.libsignal.database.LokiAPIDatabaseProtocol
-import java.util.*
+import org.session.libsignal.utilities.Base64
-import kotlin.math.abs
+import org.session.libsignal.utilities.Broadcaster
+import org.session.libsignal.utilities.ForkInfo
+import org.session.libsignal.utilities.HTTP
+import org.session.libsignal.utilities.JsonUtil
+import org.session.libsignal.utilities.Log
+import org.session.libsignal.utilities.Snode
+import org.session.libsignal.utilities.ThreadUtils
+import org.session.libsignal.utilities.recover
+import org.session.libsignal.utilities.toHexString
+import java.util.Date
+import kotlin.collections.set
 
 private typealias Path = List<Snode>
 
@@ -356,6 +360,22 @@ object OnionRequestAPI {
                                     val offset = timestamp - Date().time
                                     SnodeAPI.clockOffset = offset
                                 }
+                                if (body.containsKey("hf")) {
+                                    @Suppress("UNCHECKED_CAST")
+                                    val currentHf = body["hf"] as List<Int>
+                                    if (currentHf.size < 2) {
+                                        Log.e("Loki", "Response contains fork information but doesn't have a hard and soft number")
+                                    } else {
+                                        val hf = currentHf[0]
+                                        val sf = currentHf[1]
+                                        val newForkInfo = ForkInfo(hf, sf)
+                                        if (newForkInfo > SnodeAPI.forkInfo) {
+                                            SnodeAPI.forkInfo = ForkInfo(hf,sf)
+                                        } else if (newForkInfo < SnodeAPI.forkInfo) {
+                                            Log.w("Loki", "Got a new snode info fork version that was $newForkInfo, less than current known ${SnodeAPI.forkInfo}")
+                                        }
+                                    }
+                                }
                                 if (statusCode != 200) {
                                     val exception = HTTPRequestFailedAtDestinationException(statusCode, body, destination.description)
                                     return@queue deferred.reject(exception)

libsession/src/main/java/org/session/libsession/snode/SnodeAPI.kt

@@ -11,19 +11,33 @@ import com.goterl.lazysodium.interfaces.PwHash
 import com.goterl.lazysodium.interfaces.SecretBox
 import com.goterl.lazysodium.interfaces.Sign
 import com.goterl.lazysodium.utils.Key
-import nl.komponents.kovenant.*
+import nl.komponents.kovenant.Promise
+import nl.komponents.kovenant.all
+import nl.komponents.kovenant.deferred
 import nl.komponents.kovenant.functional.bind
 import nl.komponents.kovenant.functional.map
+import nl.komponents.kovenant.task
 import org.session.libsession.messaging.MessagingModuleConfiguration
 import org.session.libsession.messaging.utilities.MessageWrapper
 import org.session.libsignal.crypto.getRandomElement
 import org.session.libsignal.database.LokiAPIDatabaseProtocol
 import org.session.libsignal.protos.SignalServiceProtos
-import org.session.libsignal.utilities.*
 import org.session.libsignal.utilities.Base64
+import org.session.libsignal.utilities.Broadcaster
+import org.session.libsignal.utilities.HTTP
+import org.session.libsignal.utilities.Hex
+import org.session.libsignal.utilities.Log
+import org.session.libsignal.utilities.Snode
+import org.session.libsignal.utilities.ThreadUtils
+import org.session.libsignal.utilities.prettifiedDescription
+import org.session.libsignal.utilities.retryIfNeeded
 import java.security.SecureRandom
-import java.util.*
+import java.util.Date
-import kotlin.Pair
+import java.util.Locale
+import kotlin.collections.component1
+import kotlin.collections.component2
+import kotlin.collections.set
+import kotlin.properties.Delegates.observable
 
 object SnodeAPI {
     private val sodium by lazy { LazySodiumAndroid(SodiumAndroid()) }
@@ -41,6 +55,12 @@ object SnodeAPI {
      * user's clock is incorrect.
      */
     internal var clockOffset = 0L
+    internal var forkInfo by observable(database.getForkInfo()) { _, oldValue, newValue ->
+        if (newValue > oldValue) {
+            Log.d("Loki", "Setting new fork info new: $newValue, old: $oldValue")
+            database.setForkInfo(newValue)
+        }
+    }
 
     // Settings
     private val maxRetryCount = 6
@@ -55,11 +75,10 @@ object SnodeAPI {
             setOf( "https://storage.seed1.loki.network:$seedNodePort", "https://storage.seed3.loki.network:$seedNodePort", "https://public.loki.foundation:$seedNodePort" )
         }
     }
-    private val snodeFailureThreshold = 3
+    private const val snodeFailureThreshold = 3
-    private val targetSwarmSnodeCount = 2
+    private const val useOnionRequests = true
-    private val useOnionRequests = true
 
-    internal val useTestnet = false
+    const val useTestnet = false
 
     // Error
     internal sealed class Error(val description: String) : Exception(description) {
@@ -254,11 +273,6 @@ object SnodeAPI {
         return promise
     }
 
-    fun getTargetSnodes(publicKey: String): Promise<List<Snode>, Exception> {
-        // SecureRandom() should be cryptographically secure
-        return getSwarm(publicKey).map { it.shuffled(SecureRandom()).take(targetSwarmSnodeCount) }
-    }
-
     fun getSwarm(publicKey: String): Promise<Set<Snode>, Exception> {
         val cachedSwarm = database.getSwarm(publicKey)
         if (cachedSwarm != null && cachedSwarm.size >= minimumSwarmSnodeCount) {
@@ -266,7 +280,7 @@ object SnodeAPI {
             cachedSwarmCopy.addAll(cachedSwarm)
             return task { cachedSwarmCopy }
         } else {
-            val parameters = mapOf( "pubKey" to if (useTestnet) publicKey.removing05PrefixIfNeeded() else publicKey )
+            val parameters = mapOf( "pubKey" to publicKey )
             return getRandomSnode().bind {
                 invoke(Snode.Method.GetSwarm, it, publicKey, parameters)
             }.map {
@@ -277,28 +291,39 @@ object SnodeAPI {
         }
     }
 
-    fun getRawMessages(snode: Snode, publicKey: String): RawResponsePromise {
+    fun getRawMessages(snode: Snode, publicKey: String, requiresAuth: Boolean = true, namespace: Int = 0): RawResponsePromise {
-//        val userED25519KeyPair = MessagingModuleConfiguration.shared.getUserED25519KeyPair() ?: return Promise.ofFail(Error.NoKeyPair)
+        val userED25519KeyPair = MessagingModuleConfiguration.shared.getUserED25519KeyPair() ?: return Promise.ofFail(Error.NoKeyPair)
         // Get last message hash
-        val lastHashValue = database.getLastMessageHashValue(snode, publicKey) ?: ""
+        val lastHashValue = database.getLastMessageHashValue(snode, publicKey, namespace) ?: ""
-        // Construct signature
+        val parameters = mutableMapOf<String,Any>(
-//        val timestamp = Date().time + SnodeAPI.clockOffset
+            "pubKey" to publicKey,
-//        val ed25519PublicKey = userED25519KeyPair.publicKey.asHexString
+            "last_hash" to lastHashValue,
-//        val verificationData = "retrieve$timestamp".toByteArray()
-//        val signature = ByteArray(Sign.BYTES)
-//        try {
-//            sodium.cryptoSignDetached(signature, verificationData, verificationData.size.toLong(), userED25519KeyPair.secretKey.asBytes)
-//        } catch (exception: Exception) {
-//            return Promise.ofFail(Error.SigningFailed)
-//        }
-        // Make the request
-        val parameters = mapOf(
-            "pubKey" to if (useTestnet) publicKey.removing05PrefixIfNeeded() else publicKey,
-            "lastHash" to lastHashValue,
-//            "timestamp" to timestamp,
-//            "pubkey_ed25519" to ed25519PublicKey,
-//            "signature" to Base64.encodeBytes(signature)
         )
+        // Construct signature
+        if (requiresAuth) {
+            val timestamp = Date().time + SnodeAPI.clockOffset
+            val ed25519PublicKey = userED25519KeyPair.publicKey.asHexString
+            val signature = ByteArray(Sign.BYTES)
+            val verificationData =
+                if (namespace != 0) "retrieve$namespace$timestamp".toByteArray()
+                else "retrieve$timestamp".toByteArray()
+            try {
+                sodium.cryptoSignDetached(signature, verificationData, verificationData.size.toLong(), userED25519KeyPair.secretKey.asBytes)
+            } catch (exception: Exception) {
+                return Promise.ofFail(Error.SigningFailed)
+            }
+            parameters["timestamp"] = timestamp
+            parameters["pubkey_ed25519"] = ed25519PublicKey
+            parameters["signature"] = Base64.encodeBytes(signature)
+        }
+
+        // If the namespace is default (0) here it will be implicitly read as 0 on the storage server
+        // we only need to specify it explicitly if we want to (in future) or if it is non-zero
+        if (namespace != 0) {
+            parameters["namespace"] = namespace
+        }
+
+        // Make the request
         return invoke(Snode.Method.GetMessages, snode, publicKey, parameters)
     }
 
@@ -317,14 +342,35 @@ object SnodeAPI {
         }
     }
 
-    fun sendMessage(message: SnodeMessage): Promise<Set<RawResponsePromise>, Exception> {
+    fun sendMessage(message: SnodeMessage, requiresAuth: Boolean = false, namespace: Int = 0): RawResponsePromise {
-        val destination = if (useTestnet) message.recipient.removing05PrefixIfNeeded() else message.recipient
+        val destination = message.recipient
         return retryIfNeeded(maxRetryCount) {
-            getTargetSnodes(destination).map { swarm ->
+            val module = MessagingModuleConfiguration.shared
-                swarm.map { snode ->
+            val userED25519KeyPair = module.getUserED25519KeyPair() ?: return@retryIfNeeded Promise.ofFail(Error.NoKeyPair)
-                    val parameters = message.toJSON()
+            val parameters = message.toJSON().toMutableMap<String,Any>()
-                    invoke(Snode.Method.SendMessage, snode, destination, parameters)
+            // Construct signature
-                }.toSet()
+            if (requiresAuth) {
+                val sigTimestamp = System.currentTimeMillis() + SnodeAPI.clockOffset
+                val ed25519PublicKey = userED25519KeyPair.publicKey.asHexString
+                val signature = ByteArray(Sign.BYTES)
+                // assume namespace here is non-zero, as zero namespace doesn't require auth
+                val verificationData = "store$namespace$sigTimestamp".toByteArray()
+                try {
+                    sodium.cryptoSignDetached(signature, verificationData, verificationData.size.toLong(), userED25519KeyPair.secretKey.asBytes)
+                } catch (exception: Exception) {
+                    return@retryIfNeeded Promise.ofFail(Error.SigningFailed)
+                }
+                parameters["sig_timestamp"] = sigTimestamp
+                parameters["pubkey_ed25519"] = ed25519PublicKey
+                parameters["signature"] = Base64.encodeBytes(signature)
+            }
+            // If the namespace is default (0) here it will be implicitly read as 0 on the storage server
+            // we only need to specify it explicitly if we want to (in future) or if it is non-zero
+            if (namespace != 0) {
+                parameters["namespace"] = namespace
+            }
+            getSingleTargetSnode(destination).bind { snode ->
+                invoke(Snode.Method.SendMessage, snode, destination, parameters)
             }
         }
     }
@@ -426,29 +472,29 @@ object SnodeAPI {
         }
     }
 
-    fun parseRawMessagesResponse(rawResponse: RawResponse, snode: Snode, publicKey: String): List<Pair<SignalServiceProtos.Envelope, String?>> {
+    fun parseRawMessagesResponse(rawResponse: RawResponse, snode: Snode, publicKey: String, namespace: Int = 0): List<Pair<SignalServiceProtos.Envelope, String?>> {
         val messages = rawResponse["messages"] as? List<*>
         return if (messages != null) {
-            updateLastMessageHashValueIfPossible(snode, publicKey, messages)
+            updateLastMessageHashValueIfPossible(snode, publicKey, messages, namespace)
-            val newRawMessages = removeDuplicates(publicKey, messages)
+            val newRawMessages = removeDuplicates(publicKey, messages, namespace)
             return parseEnvelopes(newRawMessages);
         } else {
             listOf()
         }
     }
 
-    private fun updateLastMessageHashValueIfPossible(snode: Snode, publicKey: String, rawMessages: List<*>) {
+    private fun updateLastMessageHashValueIfPossible(snode: Snode, publicKey: String, rawMessages: List<*>, namespace: Int) {
         val lastMessageAsJSON = rawMessages.lastOrNull() as? Map<*, *>
         val hashValue = lastMessageAsJSON?.get("hash") as? String
         if (hashValue != null) {
-            database.setLastMessageHashValue(snode, publicKey, hashValue)
+            database.setLastMessageHashValue(snode, publicKey, hashValue, namespace)
         } else if (rawMessages.isNotEmpty()) {
             Log.d("Loki", "Failed to update last message hash value from: ${rawMessages.prettifiedDescription()}.")
         }
     }
 
-    private fun removeDuplicates(publicKey: String, rawMessages: List<*>): List<*> {
+    private fun removeDuplicates(publicKey: String, rawMessages: List<*>, namespace: Int): List<*> {
-        val receivedMessageHashValues = database.getReceivedMessageHashValues(publicKey)?.toMutableSet() ?: mutableSetOf()
+        val receivedMessageHashValues = database.getReceivedMessageHashValues(publicKey, namespace)?.toMutableSet() ?: mutableSetOf()
         val result = rawMessages.filter { rawMessage ->
             val rawMessageAsJSON = rawMessage as? Map<*, *>
             val hashValue = rawMessageAsJSON?.get("hash") as? String
@@ -461,7 +507,7 @@ object SnodeAPI {
                 false
             }
         }
-        database.setReceivedMessageHashValues(publicKey, receivedMessageHashValues)
+        database.setReceivedMessageHashValues(publicKey, receivedMessageHashValues, namespace)
         return result
     }
 

libsession/src/main/java/org/session/libsession/snode/SnodeMessage.kt

@@ -1,7 +1,5 @@
 package org.session.libsession.snode
 
-import org.session.libsignal.utilities.removing05PrefixIfNeeded
-
 data class SnodeMessage(
     /**
      * The hex encoded public key of the recipient.
@@ -25,11 +23,10 @@ data class SnodeMessage(
 
     internal fun toJSON(): Map<String, String> {
         return mapOf(
-            "pubKey" to if (SnodeAPI.useTestnet) recipient.removing05PrefixIfNeeded() else recipient,
+            "pubKey" to recipient,
             "data" to data,
             "ttl" to ttl.toString(),
             "timestamp" to timestamp.toString(),
-            "nonce" to ""
         )
     }
 }

libsignal/src/main/java/org/session/libsignal/database/LokiAPIDatabaseProtocol.kt

@@ -1,8 +1,9 @@
 package org.session.libsignal.database
 
 import org.session.libsignal.crypto.ecc.ECKeyPair
+import org.session.libsignal.utilities.ForkInfo
 import org.session.libsignal.utilities.Snode
-import java.util.*
+import java.util.Date
 
 interface LokiAPIDatabaseProtocol {
 
@@ -13,10 +14,10 @@ interface LokiAPIDatabaseProtocol {
     fun setOnionRequestPaths(newValue: List<List<Snode>>)
     fun getSwarm(publicKey: String): Set<Snode>?
     fun setSwarm(publicKey: String, newValue: Set<Snode>)
-    fun getLastMessageHashValue(snode: Snode, publicKey: String): String?
+    fun getLastMessageHashValue(snode: Snode, publicKey: String, namespace: Int): String?
-    fun setLastMessageHashValue(snode: Snode, publicKey: String, newValue: String)
+    fun setLastMessageHashValue(snode: Snode, publicKey: String, newValue: String, namespace: Int)
-    fun getReceivedMessageHashValues(publicKey: String): Set<String>?
+    fun getReceivedMessageHashValues(publicKey: String, namespace: Int): Set<String>?
-    fun setReceivedMessageHashValues(publicKey: String, newValue: Set<String>)
+    fun setReceivedMessageHashValues(publicKey: String, newValue: Set<String>, namespace: Int)
     fun getAuthToken(server: String): String?
     fun setAuthToken(server: String, newValue: String?)
     fun setUserCount(group: Long, server: String, newValue: Int)
@@ -33,4 +34,7 @@ interface LokiAPIDatabaseProtocol {
     fun getClosedGroupEncryptionKeyPairs(groupPublicKey: String): List<ECKeyPair>
     fun getLatestClosedGroupEncryptionKeyPair(groupPublicKey: String): ECKeyPair?
     fun isClosedGroup(groupPublicKey: String): Boolean
+    fun getForkInfo(): ForkInfo
+    fun setForkInfo(forkInfo: ForkInfo)
+
 }

libsignal/src/main/java/org/session/libsignal/utilities/ForkInfo.kt

@@ -0,0 +1,20 @@
+package org.session.libsignal.utilities
+
+data class ForkInfo(val hf: Int, val sf: Int) {
+    companion object {
+        const val DEFAULT_HF = 18
+        const val DEFAULT_SF = 1
+        val DEFAULT = ForkInfo(DEFAULT_HF, DEFAULT_SF)
+        val baseTable = arrayOf(10,100,1000,10000,100000)
+    }
+
+    operator fun compareTo(other: ForkInfo): Int {
+        val base = baseTable.first { it > sf && it > other.sf }
+        return (hf*base - other.hf*base) + (sf - other.sf)
+    }
+
+}
+
+// add info here for when various features are active
+fun ForkInfo.hasNamespaces() = hf >= 19
+fun ForkInfo.defaultRequiresAuth() = hf >= 19 && sf >= 1

libsignal/src/main/java/org/session/libsignal/utilities/Namespace.kt

@@ -0,0 +1,7 @@
+package org.session.libsignal.utilities
+
+object Namespace {
+    const val DEFAULT = 0
+    const val UNAUTHENTICATED_CLOSED_GROUP = -10
+    const val CONFIGURATION = 5
+}