From b93ec3be04f3370ac5b7d285295a0e142038fe82 Mon Sep 17 00:00:00 2001 From: bemusementpark Date: Fri, 2 Aug 2024 12:22:25 +0930 Subject: [PATCH 01/12] Optimise Snode and Snode.Version --- .../securesms/database/LokiAPIDatabase.kt | 35 ++---------- .../org/session/libsession/snode/SnodeAPI.kt | 34 +++++------- .../org/session/libsession/utilities/Util.kt | 33 ++---------- .../org/session/libsignal/utilities/Snode.kt | 54 ++++++++++++++++++- 4 files changed, 73 insertions(+), 83 deletions(-) diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/LokiAPIDatabase.kt b/app/src/main/java/org/thoughtcrime/securesms/database/LokiAPIDatabase.kt index f1f999242c..ff22ae14e3 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/database/LokiAPIDatabase.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/database/LokiAPIDatabase.kt @@ -166,8 +166,6 @@ class LokiAPIDatabase(context: Context, helper: SQLCipherOpenHelper) : Database( const val RESET_SEQ_NO = "UPDATE $lastMessageServerIDTable SET $lastMessageServerID = 0;" - const val EMPTY_VERSION = "0.0.0" - // endregion } @@ -175,15 +173,7 @@ class LokiAPIDatabase(context: Context, helper: SQLCipherOpenHelper) : Database( val database = databaseHelper.readableDatabase return database.get(snodePoolTable, "${Companion.dummyKey} = ?", wrap("dummy_key")) { cursor -> val snodePoolAsString = cursor.getString(cursor.getColumnIndexOrThrow(snodePool)) - snodePoolAsString.split(", ").mapNotNull { snodeAsString -> - val components = snodeAsString.split("-") - val address = components[0] - val port = components.getOrNull(1)?.toIntOrNull() ?: return@mapNotNull null - val ed25519Key = components.getOrNull(2) ?: return@mapNotNull null - val x25519Key = components.getOrNull(3) ?: return@mapNotNull null - val version = components.getOrNull(4) ?: EMPTY_VERSION - Snode(address, port, Snode.KeySet(ed25519Key, x25519Key), version) - } + snodePoolAsString.split(", ").mapNotNull(::Snode) }?.toSet() ?: setOf() } @@ -231,18 +221,7 @@ class LokiAPIDatabase(context: Context, helper: SQLCipherOpenHelper) : Database( val database = databaseHelper.readableDatabase fun get(indexPath: String): Snode? { return database.get(onionRequestPathTable, "${Companion.indexPath} = ?", wrap(indexPath)) { cursor -> - val snodeAsString = cursor.getString(cursor.getColumnIndexOrThrow(snode)) - val components = snodeAsString.split("-") - val address = components[0] - val port = components.getOrNull(1)?.toIntOrNull() - val ed25519Key = components.getOrNull(2) - val x25519Key = components.getOrNull(3) - val version = components.getOrNull(4) ?: EMPTY_VERSION - if (port != null && ed25519Key != null && x25519Key != null) { - Snode(address, port, Snode.KeySet(ed25519Key, x25519Key), version) - } else { - null - } + Snode(cursor.getString(cursor.getColumnIndexOrThrow(snode))) } } val result = mutableListOf>() @@ -276,15 +255,7 @@ class LokiAPIDatabase(context: Context, helper: SQLCipherOpenHelper) : Database( val database = databaseHelper.readableDatabase return database.get(swarmTable, "${Companion.swarmPublicKey} = ?", wrap(publicKey)) { cursor -> val swarmAsString = cursor.getString(cursor.getColumnIndexOrThrow(swarm)) - swarmAsString.split(", ").mapNotNull { targetAsString -> - val components = targetAsString.split("-") - val address = components[0] - val port = components.getOrNull(1)?.toIntOrNull() ?: return@mapNotNull null - val ed25519Key = components.getOrNull(2) ?: return@mapNotNull null - val x25519Key = components.getOrNull(3) ?: return@mapNotNull null - val version = components.getOrNull(4) ?: EMPTY_VERSION - Snode(address, port, Snode.KeySet(ed25519Key, x25519Key), version) - } + swarmAsString.split(", ").mapNotNull(::Snode) }?.toSet() } diff --git a/libsession/src/main/java/org/session/libsession/snode/SnodeAPI.kt b/libsession/src/main/java/org/session/libsession/snode/SnodeAPI.kt index 034ef0e7d2..9ceefe8386 100644 --- a/libsession/src/main/java/org/session/libsession/snode/SnodeAPI.kt +++ b/libsession/src/main/java/org/session/libsession/snode/SnodeAPI.kt @@ -18,6 +18,7 @@ import nl.komponents.kovenant.task import org.session.libsession.messaging.MessagingModuleConfiguration import org.session.libsession.messaging.utilities.MessageWrapper import org.session.libsession.messaging.utilities.SodiumUtilities.sodium +import org.session.libsession.utilities.toByteArray import org.session.libsignal.crypto.getRandomElement import org.session.libsignal.database.LokiAPIDatabaseProtocol import org.session.libsignal.protos.SignalServiceProtos @@ -94,8 +95,6 @@ object SnodeAPI { const val KEY_ED25519 = "pubkey_ed25519" const val KEY_VERSION = "storage_server_version" - const val EMPTY_VERSION = "0.0.0" - // Error sealed class Error(val description: String) : Exception(description) { object Generic : Error("An error occurred.") @@ -191,7 +190,7 @@ object SnodeAPI { val x25519Key = rawSnodeAsJSON?.get(KEY_X25519) as? String val version = (rawSnodeAsJSON?.get(KEY_VERSION) as? ArrayList<*>) ?.filterIsInstance() // get the array as Integers - ?.joinToString(separator = ".") // turn it int a version string + ?.let(Snode::Version) // turn it int a version if (address != null && port != null && ed25519Key != null && x25519Key != null && address != "0.0.0.0" && version != null) { @@ -696,7 +695,7 @@ object SnodeAPI { getSingleTargetSnode(publicKey).bind { snode -> retryIfNeeded(maxRetryCount) { val signature = ByteArray(Sign.BYTES) - val verificationData = (Snode.Method.DeleteMessage.rawValue + serverHashes.fold("") { a, v -> a + v }).toByteArray() + val verificationData = sequenceOf(Snode.Method.DeleteMessage.rawValue).plus(serverHashes).toByteArray() sodium.cryptoSignDetached(signature, verificationData, verificationData.size.toLong(), userED25519KeyPair.secretKey.asBytes) val deleteMessageParams = mapOf( "pubkey" to userPublicKey, @@ -719,7 +718,7 @@ object SnodeAPI { val signature = json["signature"] as String val snodePublicKey = Key.fromHexString(hexSnodePublicKey) // The signature looks like ( PUBKEY_HEX || RMSG[0] || ... || RMSG[N] || DMSG[0] || ... || DMSG[M] ) - val message = (userPublicKey + serverHashes.fold("") { a, v -> a + v } + hashes.fold("") { a, v -> a + v }).toByteArray() + val message = sequenceOf(userPublicKey).plus(serverHashes).plus(hashes).toByteArray() sodium.cryptoSignVerifyDetached(Base64.decode(signature), message, message.size, snodePublicKey.asBytes) } } @@ -733,11 +732,10 @@ object SnodeAPI { } // Parsing - private fun parseSnodes(rawResponse: Any): List { - val json = rawResponse as? Map<*, *> - val rawSnodes = json?.get("snodes") as? List<*> - if (rawSnodes != null) { - return rawSnodes.mapNotNull { rawSnode -> + private fun parseSnodes(rawResponse: Any): List = + (rawResponse as? Map<*, *>) + ?.run { get("snodes") as? List<*> } + ?.mapNotNull { rawSnode -> val rawSnodeAsJSON = rawSnode as? Map<*, *> val address = rawSnodeAsJSON?.get("ip") as? String val portAsString = rawSnodeAsJSON?.get("port") as? String @@ -746,17 +744,12 @@ object SnodeAPI { val x25519Key = rawSnodeAsJSON?.get(KEY_X25519) as? String if (address != null && port != null && ed25519Key != null && x25519Key != null && address != "0.0.0.0") { - Snode("https://$address", port, Snode.KeySet(ed25519Key, x25519Key), EMPTY_VERSION) + Snode("https://$address", port, Snode.KeySet(ed25519Key, x25519Key), Snode.Version.ZERO) } else { Log.d("Loki", "Failed to parse snode from: ${rawSnode?.prettifiedDescription()}.") null } - } - } else { - Log.d("Loki", "Failed to parse snodes from: ${rawResponse.prettifiedDescription()}.") - return listOf() - } - } + } ?: listOf().also { Log.d("Loki", "Failed to parse snodes from: ${rawResponse.prettifiedDescription()}.") } fun deleteAllMessages(): Promise, Exception> { return retryIfNeeded(maxRetryCount) { @@ -796,8 +789,7 @@ object SnodeAPI { getSingleTargetSnode(userPublicKey).bind { snode -> retryIfNeeded(maxRetryCount) { // "expire" || expiry || messages[0] || ... || messages[N] - val verificationData = - (Snode.Method.Expire.rawValue + updatedExpiryMsWithNetworkOffset + serverHashes.fold("") { a, v -> a + v }).toByteArray() + val verificationData = sequenceOf(Snode.Method.Expire.rawValue, "$updatedExpiryMsWithNetworkOffset").plus(serverHashes).toByteArray() val signature = ByteArray(Sign.BYTES) sodium.cryptoSignDetached( signature, @@ -828,7 +820,7 @@ object SnodeAPI { val signature = json["signature"] as String val snodePublicKey = Key.fromHexString(hexSnodePublicKey) // The signature looks like ( PUBKEY_HEX || RMSG[0] || ... || RMSG[N] || DMSG[0] || ... || DMSG[M] ) - val message = (userPublicKey + serverHashes.fold("") { a, v -> a + v } + hashes.fold("") { a, v -> a + v }).toByteArray() + val message = sequenceOf(userPublicKey).plus(serverHashes).plus(hashes).toByteArray() if (sodium.cryptoSignVerifyDetached(Base64.decode(signature), message, message.size, snodePublicKey.asBytes)) { hashes to expiryApplied } else listOf() to 0L @@ -922,7 +914,7 @@ object SnodeAPI { val signature = json["signature"] as String val snodePublicKey = Key.fromHexString(hexSnodePublicKey) // The signature looks like ( PUBKEY_HEX || TIMESTAMP || DELETEDHASH[0] || ... || DELETEDHASH[N] ) - val message = (userPublicKey + timestamp.toString() + hashes.joinToString(separator = "")).toByteArray() + val message = sequenceOf(userPublicKey, "$timestamp").plus(hashes).toByteArray() sodium.cryptoSignVerifyDetached(Base64.decode(signature), message, message.size, snodePublicKey.asBytes) } } diff --git a/libsession/src/main/java/org/session/libsession/utilities/Util.kt b/libsession/src/main/java/org/session/libsession/utilities/Util.kt index 929f53e305..d47754b7ed 100644 --- a/libsession/src/main/java/org/session/libsession/utilities/Util.kt +++ b/libsession/src/main/java/org/session/libsession/utilities/Util.kt @@ -366,34 +366,6 @@ object Util { val digitGroups = (Math.log10(sizeBytes.toDouble()) / Math.log10(1024.0)).toInt() return DecimalFormat("#,##0.#").format(sizeBytes / Math.pow(1024.0, digitGroups.toDouble())) + " " + units[digitGroups] } - - /** - * Compares two version strings (for example "1.8.0") - * - * @param version1 the first version string to compare. - * @param version2 the second version string to compare. - * @return an integer indicating the result of the comparison: - * - 0 if the versions are equal - * - a positive number if version1 is greater than version2 - * - a negative number if version1 is less than version2 - */ - @JvmStatic - fun compareVersions(version1: String, version2: String): Int { - val parts1 = version1.split(".").map { it.toIntOrNull() ?: 0 } - val parts2 = version2.split(".").map { it.toIntOrNull() ?: 0 } - - val maxLength = maxOf(parts1.size, parts2.size) - val paddedParts1 = parts1 + List(maxLength - parts1.size) { 0 } - val paddedParts2 = parts2 + List(maxLength - parts2.size) { 0 } - - for (i in 0 until maxLength) { - val compare = paddedParts1[i].compareTo(paddedParts2[i]) - if (compare != 0) { - return compare - } - } - return 0 - } } fun T.runIf(condition: Boolean, block: T.() -> R): R where T: R = if (condition) block() else this @@ -440,3 +412,8 @@ fun Iterable.associateByNotNull( inline fun Iterable.groupByNotNull(keySelector: (E) -> K?): Map> = LinkedHashMap>().also { forEach { e -> keySelector(e)?.let { k -> it.getOrPut(k) { mutableListOf() } += e } } } + +fun Sequence.toByteArray(): ByteArray = ByteArrayOutputStream().use { output -> + forEach { it.byteInputStream().use { input -> input.copyTo(output) } } + output.toByteArray() +} diff --git a/libsignal/src/main/java/org/session/libsignal/utilities/Snode.kt b/libsignal/src/main/java/org/session/libsignal/utilities/Snode.kt index f6b11754ad..cc123a8527 100644 --- a/libsignal/src/main/java/org/session/libsignal/utilities/Snode.kt +++ b/libsignal/src/main/java/org/session/libsignal/utilities/Snode.kt @@ -1,9 +1,21 @@ package org.session.libsignal.utilities -class Snode(val address: String, val port: Int, val publicKeySet: KeySet?, val version: String) { +import android.annotation.SuppressLint + +fun Snode(string: String): Snode? { + val components = string.split("-") + val address = components[0] + val port = components.getOrNull(1)?.toIntOrNull() ?: return null + val ed25519Key = components.getOrNull(2) ?: return null + val x25519Key = components.getOrNull(3) ?: return null + val version = components.getOrNull(4)?.let(Snode::Version) ?: Snode.Version.ZERO + return Snode(address, port, Snode.KeySet(ed25519Key, x25519Key), version) +} + +class Snode(val address: String, val port: Int, val publicKeySet: KeySet?, val version: Version) { val ip: String get() = address.removePrefix("https://") - public enum class Method(val rawValue: String) { + enum class Method(val rawValue: String) { GetSwarm("get_snodes_for_pubkey"), Retrieve("retrieve"), SendMessage("store"), @@ -32,4 +44,42 @@ class Snode(val address: String, val port: Int, val publicKeySet: KeySet?, val v } override fun toString(): String { return "$address:$port" } + + companion object { + private val CACHE = mutableMapOf() + + @SuppressLint("NotConstructor") + fun Version(value: String) = CACHE.getOrElse(value) { + Snode.Version(value) + } + } + + @JvmInline + value class Version(val value: ULong) { + companion object { + val ZERO = Version(0UL) + private const val MASK_BITS = 16 + private const val MASK = 0xFFFFUL + + private fun Sequence.foldToVersionAsULong() = take(4).foldIndexed(0UL) { i, acc, it -> + it and MASK shl (3 - i) * MASK_BITS or acc + } + } + + constructor(parts: List): this( + parts.asSequence() + .map { it.toByte().toULong() } + .foldToVersionAsULong() + ) + + constructor(value: Int): this(value.toULong()) + + internal constructor(value: String): this( + value.splitToSequence(".") + .map { it.toULongOrNull() ?: 0UL } + .foldToVersionAsULong() + ) + + operator fun compareTo(other: Version): Int = value.compareTo(other.value) + } } From 482f169df1f5fde73624624de86504f92ccd83a1 Mon Sep 17 00:00:00 2001 From: bemusementpark Date: Sat, 3 Aug 2024 02:46:42 +0930 Subject: [PATCH 02/12] Refactor SnodeApi --- .../messaging/jobs/ConfigurationSyncJob.kt | 1 - .../org/session/libsession/snode/SnodeAPI.kt | 726 +++++++----------- .../session/libsignal/utilities/Base64.java | 4 +- 3 files changed, 268 insertions(+), 463 deletions(-) diff --git a/libsession/src/main/java/org/session/libsession/messaging/jobs/ConfigurationSyncJob.kt b/libsession/src/main/java/org/session/libsession/messaging/jobs/ConfigurationSyncJob.kt index 4a3299d197..a9f076b106 100644 --- a/libsession/src/main/java/org/session/libsession/messaging/jobs/ConfigurationSyncJob.kt +++ b/libsession/src/main/java/org/session/libsession/messaging/jobs/ConfigurationSyncJob.kt @@ -63,7 +63,6 @@ data class ConfigurationSyncJob(val destination: Destination): Job { // return a list of batch request objects val snodeMessage = MessageSender.buildConfigMessageToSnode(destination.destinationPublicKey(), message) val authenticated = SnodeAPI.buildAuthenticatedStoreBatchInfo( - destination.destinationPublicKey(), config.configNamespace(), snodeMessage ) ?: return@map null // this entry will be null otherwise diff --git a/libsession/src/main/java/org/session/libsession/snode/SnodeAPI.kt b/libsession/src/main/java/org/session/libsession/snode/SnodeAPI.kt index 9ceefe8386..8e19234b0d 100644 --- a/libsession/src/main/java/org/session/libsession/snode/SnodeAPI.kt +++ b/libsession/src/main/java/org/session/libsession/snode/SnodeAPI.kt @@ -9,9 +9,9 @@ import com.goterl.lazysodium.interfaces.PwHash import com.goterl.lazysodium.interfaces.SecretBox import com.goterl.lazysodium.interfaces.Sign import com.goterl.lazysodium.utils.Key +import com.goterl.lazysodium.utils.KeyPair import nl.komponents.kovenant.Promise import nl.komponents.kovenant.all -import nl.komponents.kovenant.deferred import nl.komponents.kovenant.functional.bind import nl.komponents.kovenant.functional.map import nl.komponents.kovenant.task @@ -30,7 +30,6 @@ import org.session.libsignal.utilities.JsonUtil import org.session.libsignal.utilities.Log import org.session.libsignal.utilities.Namespace import org.session.libsignal.utilities.Snode -import org.session.libsignal.utilities.ThreadUtils import org.session.libsignal.utilities.prettifiedDescription import org.session.libsignal.utilities.retryIfNeeded import java.security.SecureRandom @@ -46,7 +45,7 @@ object SnodeAPI { private val broadcaster: Broadcaster get() = SnodeModule.shared.broadcaster - internal var snodeFailureCount: MutableMap = mutableMapOf() + private var snodeFailureCount: MutableMap = mutableMapOf() internal var snodePool: Set get() = database.getSnodePool() set(newValue) { database.setSnodePool(newValue) } @@ -57,7 +56,7 @@ object SnodeAPI { internal var clockOffset = 0L @JvmStatic - public val nowWithOffset + val nowWithOffset get() = System.currentTimeMillis() + clockOffset internal var forkInfo by observable(database.getForkInfo()) { _, oldValue, newValue -> @@ -68,32 +67,32 @@ object SnodeAPI { } // Settings - private val maxRetryCount = 6 - private val minimumSnodePoolCount = 12 - private val minimumSwarmSnodeCount = 3 + private const val maxRetryCount = 6 + private const val minimumSnodePoolCount = 12 + private const val minimumSwarmSnodeCount = 3 // Use port 4433 if the API level can handle the network security configuration and enforce pinned certificates private val seedNodePort = if (Build.VERSION.SDK_INT < Build.VERSION_CODES.N) 443 else 4443 - private val seedNodePool by lazy { - if (useTestnet) { - setOf( "http://public.loki.foundation:38157" ) - } else { - setOf( - "https://seed1.getsession.org:$seedNodePort", - "https://seed2.getsession.org:$seedNodePort", - "https://seed3.getsession.org:$seedNodePort", - ) - } - } + private const val snodeFailureThreshold = 3 private const val useOnionRequests = true - const val useTestnet = false + private const val useTestnet = false - const val KEY_IP = "public_ip" - const val KEY_PORT = "storage_port" - const val KEY_X25519 = "pubkey_x25519" - const val KEY_ED25519 = "pubkey_ed25519" - const val KEY_VERSION = "storage_server_version" + private val seedNodePool = if (useTestnet) { + setOf( "http://public.loki.foundation:38157" ) + } else { + setOf( + "https://seed1.getsession.org:$seedNodePort", + "https://seed2.getsession.org:$seedNodePort", + "https://seed3.getsession.org:$seedNodePort", + ) + } + + private const val KEY_IP = "public_ip" + private const val KEY_PORT = "storage_port" + private const val KEY_X25519 = "pubkey_x25519" + private const val KEY_ED25519 = "pubkey_ed25519" + private const val KEY_VERSION = "storage_server_version" // Error sealed class Error(val description: String) : Exception(description) { @@ -122,39 +121,28 @@ object SnodeAPI { parameters: Map, publicKey: String? = null, version: Version = Version.V3 - ): RawResponsePromise { - val url = "${snode.address}:${snode.port}/storage_rpc/v1" - val deferred = deferred, Exception>() - if (useOnionRequests) { - OnionRequestAPI.sendOnionRequest(method, parameters, snode, version, publicKey).map { - val body = it.body ?: throw Error.Generic - deferred.resolve(JsonUtil.fromJson(body, Map::class.java)) - }.fail { deferred.reject(it) } - } else { - ThreadUtils.queue { - val payload = mapOf( "method" to method.rawValue, "params" to parameters ) - try { - val response = HTTP.execute(HTTP.Verb.POST, url, payload).toString() - val json = JsonUtil.fromJson(response, Map::class.java) - deferred.resolve(json) - } catch (exception: Exception) { - val httpRequestFailedException = exception as? HTTP.HTTPRequestFailedException - if (httpRequestFailedException != null) { - val error = handleSnodeError(httpRequestFailedException.statusCode, httpRequestFailedException.json, snode, publicKey) - if (error != null) { return@queue deferred.reject(exception) } - } - Log.d("Loki", "Unhandled exception: $exception.") - deferred.reject(exception) + ): RawResponsePromise = if (useOnionRequests) OnionRequestAPI.sendOnionRequest(method, parameters, snode, version, publicKey).map { + val body = it.body ?: throw Error.Generic + JsonUtil.fromJson(body, Map::class.java) + } else task { + val payload = mapOf( "method" to method.rawValue, "params" to parameters ) + try { + val url = "${snode.address}:${snode.port}/storage_rpc/v1" + val response = HTTP.execute(HTTP.Verb.POST, url, payload).toString() + JsonUtil.fromJson(response, Map::class.java) + } catch (exception: Exception) { + (exception as? HTTP.HTTPRequestFailedException)?.run { + handleSnodeError(statusCode, json, snode, publicKey) + // TODO Check if we meant to throw the error returned by handleSnodeError + throw exception } + Log.d("Loki", "Unhandled exception: $exception.") + throw exception } } - return deferred.promise - } - internal fun getRandomSnode(): Promise { - val snodePool = this.snodePool - - if (snodePool.count() < minimumSnodePoolCount) { + internal fun getRandomSnode(): Promise = + snodePool.takeIf { it.size >= minimumSnodePoolCount }?.let { Promise.of(it.getRandomElement()) } ?: task { val target = seedNodePool.random() val url = "$target/json_rpc" Log.d("Loki", "Populating snode pool using: $target.") @@ -169,73 +157,48 @@ object SnodeAPI { ) ) ) - val deferred = deferred() - deferred() - ThreadUtils.queue { - try { - val response = HTTP.execute(HTTP.Verb.POST, url, parameters, useSeedNodeConnection = true) - val json = try { - JsonUtil.fromJson(response, Map::class.java) - } catch (exception: Exception) { - mapOf( "result" to response.toString()) - } - val intermediate = json["result"] as? Map<*, *> - val rawSnodes = intermediate?.get("service_node_states") as? List<*> - if (rawSnodes != null) { - val snodePool = rawSnodes.mapNotNull { rawSnode -> - val rawSnodeAsJSON = rawSnode as? Map<*, *> - val address = rawSnodeAsJSON?.get(KEY_IP) as? String - val port = rawSnodeAsJSON?.get(KEY_PORT) as? Int - val ed25519Key = rawSnodeAsJSON?.get(KEY_ED25519) as? String - val x25519Key = rawSnodeAsJSON?.get(KEY_X25519) as? String - val version = (rawSnodeAsJSON?.get(KEY_VERSION) as? ArrayList<*>) - ?.filterIsInstance() // get the array as Integers - ?.let(Snode::Version) // turn it int a version - - if (address != null && port != null && ed25519Key != null && x25519Key != null - && address != "0.0.0.0" && version != null) { - Snode( - address = "https://$address", - port = port, - publicKeySet = Snode.KeySet(ed25519Key, x25519Key), - version = version - ) - } else { - Log.d("Loki", "Failed to parse: ${rawSnode?.prettifiedDescription()}.") - null - } - }.toMutableSet() - Log.d("Loki", "Persisting snode pool to database.") - this.snodePool = snodePool - try { - deferred.resolve(snodePool.getRandomElement()) - } catch (exception: Exception) { - Log.d("Loki", "Got an empty snode pool from: $target.") - deferred.reject(SnodeAPI.Error.Generic) - } - } else { - Log.d("Loki", "Failed to update snode pool from: ${(rawSnodes as List<*>?)?.prettifiedDescription()}.") - deferred.reject(SnodeAPI.Error.Generic) - } - } catch (exception: Exception) { - deferred.reject(exception) - } + val response = HTTP.execute(HTTP.Verb.POST, url, parameters, useSeedNodeConnection = true) + val json = try { + JsonUtil.fromJson(response, Map::class.java) + } catch (exception: Exception) { + mapOf( "result" to response.toString()) } - return deferred.promise - } else { - return Promise.of(snodePool.getRandomElement()) - } - } + val intermediate = json["result"] as? Map<*, *> ?: throw Error.Generic + .also { Log.d("Loki", "Failed to update snode pool, intermediate was null.") } + val rawSnodes = intermediate["service_node_states"] as? List<*> ?: throw Error.Generic + .also { Log.d("Loki", "Failed to update snode pool, rawSnodes was null.") } - private fun extractVersionString(jsonVersion: String): String{ - return jsonVersion.removeSurrounding("[", "]").split(", ").joinToString(separator = ".") + rawSnodes.asSequence().mapNotNull { it as? Map<*, *> }.mapNotNull { rawSnode -> + createSnode( + address = rawSnode[KEY_IP] as? String, + port = rawSnode[KEY_PORT] as? Int, + ed25519Key = rawSnode[KEY_ED25519] as? String, + x25519Key = rawSnode[KEY_X25519] as? String, + version = (rawSnode[KEY_VERSION] as? List<*>) + ?.filterIsInstance() + ?.let(Snode::Version) + ).also { if (it == null) Log.d("Loki", "Failed to parse: ${rawSnode.prettifiedDescription()}.") } + }.toSet().also { + Log.d("Loki", "Persisting snode pool to database.") + this.snodePool = it + }.runCatching { getRandomElement() }.onFailure { + Log.d("Loki", "Got an empty snode pool from: $target.") + throw SnodeAPI.Error.Generic + }.getOrThrow() + } + + private fun createSnode(address: String?, port: Int?, ed25519Key: String?, x25519Key: String?, version: Snode.Version? = Snode.Version.ZERO): Snode? { + return Snode( + address?.takeUnless { it == "0.0.0.0" }?.let { "https://$it" } ?: return null, + port ?: return null, + Snode.KeySet(ed25519Key ?: return null, x25519Key ?: return null), + version ?: return null + ) } internal fun dropSnodeFromSwarmIfNeeded(snode: Snode, publicKey: String) { - val swarm = database.getSwarm(publicKey)?.toMutableSet() - if (swarm != null && swarm.contains(snode)) { - swarm.remove(snode) - database.setSwarm(publicKey, swarm) + database.getSwarm(publicKey)?.takeIf { snode in it }?.let { + database.setSwarm(publicKey, it - snode) } } @@ -246,8 +209,6 @@ object SnodeAPI { // Public API fun getAccountID(onsName: String): Promise { - val deferred = deferred() - val promise = deferred.promise val validationCount = 3 val accountIDByteCount = 33 // Hash the ONS name using BLAKE2b @@ -255,96 +216,79 @@ object SnodeAPI { val nameAsData = onsName.toByteArray() val nameHash = ByteArray(GenericHash.BYTES) if (!sodium.cryptoGenericHash(nameHash, nameHash.size, nameAsData, nameAsData.size.toLong())) { - deferred.reject(Error.HashingFailed) - return promise + throw Error.HashingFailed } val base64EncodedNameHash = Base64.encodeBytes(nameHash) // Ask 3 different snodes for the Account ID associated with the given name hash val parameters = mapOf( - "endpoint" to "ons_resolve", - "params" to mapOf( "type" to 0, "name_hash" to base64EncodedNameHash ) + "endpoint" to "ons_resolve", + "params" to mapOf( "type" to 0, "name_hash" to base64EncodedNameHash ) ) - val promises = (1..validationCount).map { + val promises = List(validationCount) { getRandomSnode().bind { snode -> retryIfNeeded(maxRetryCount) { invoke(Snode.Method.OxenDaemonRPCCall, snode, parameters) } } } - all(promises).success { results -> + return all(promises).map { results -> val accountIDs = mutableListOf() for (json in results) { val intermediate = json["result"] as? Map<*, *> - val hexEncodedCiphertext = intermediate?.get("encrypted_value") as? String - if (hexEncodedCiphertext != null) { - val ciphertext = Hex.fromStringCondensed(hexEncodedCiphertext) - val isArgon2Based = (intermediate["nonce"] == null) - if (isArgon2Based) { - // Handle old Argon2-based encryption used before HF16 - val salt = ByteArray(PwHash.SALTBYTES) - val key: ByteArray - val nonce = ByteArray(SecretBox.NONCEBYTES) - val accountIDAsData = ByteArray(accountIDByteCount) - try { - key = Key.fromHexString(sodium.cryptoPwHash(onsName, SecretBox.KEYBYTES, salt, PwHash.OPSLIMIT_MODERATE, PwHash.MEMLIMIT_MODERATE, PwHash.Alg.PWHASH_ALG_ARGON2ID13)).asBytes - } catch (e: SodiumException) { - deferred.reject(Error.HashingFailed) - return@success - } - if (!sodium.cryptoSecretBoxOpenEasy(accountIDAsData, ciphertext, ciphertext.size.toLong(), nonce, key)) { - deferred.reject(Error.DecryptionFailed) - return@success - } - accountIDs.add(Hex.toStringCondensed(accountIDAsData)) - } else { - val hexEncodedNonce = intermediate["nonce"] as? String - if (hexEncodedNonce == null) { - deferred.reject(Error.Generic) - return@success - } - val nonce = Hex.fromStringCondensed(hexEncodedNonce) - val key = ByteArray(GenericHash.BYTES) - if (!sodium.cryptoGenericHash(key, key.size, nameAsData, nameAsData.size.toLong(), nameHash, nameHash.size)) { - deferred.reject(Error.HashingFailed) - return@success - } - val accountIDAsData = ByteArray(accountIDByteCount) - if (!sodium.cryptoAeadXChaCha20Poly1305IetfDecrypt(accountIDAsData, null, null, ciphertext, ciphertext.size.toLong(), null, 0, nonce, key)) { - deferred.reject(Error.DecryptionFailed) - return@success - } - accountIDs.add(Hex.toStringCondensed(accountIDAsData)) + val hexEncodedCiphertext = intermediate?.get("encrypted_value") as? String ?: throw Error.Generic + val ciphertext = Hex.fromStringCondensed(hexEncodedCiphertext) + val isArgon2Based = (intermediate["nonce"] == null) + if (isArgon2Based) { + // Handle old Argon2-based encryption used before HF16 + val salt = ByteArray(PwHash.SALTBYTES) + val nonce = ByteArray(SecretBox.NONCEBYTES) + val accountIDAsData = ByteArray(accountIDByteCount) + val key = try { + Key.fromHexString(sodium.cryptoPwHash(onsName, SecretBox.KEYBYTES, salt, PwHash.OPSLIMIT_MODERATE, PwHash.MEMLIMIT_MODERATE, PwHash.Alg.PWHASH_ALG_ARGON2ID13)).asBytes + } catch (e: SodiumException) { + throw Error.HashingFailed } + if (!sodium.cryptoSecretBoxOpenEasy(accountIDAsData, ciphertext, ciphertext.size.toLong(), nonce, key)) { + throw Error.DecryptionFailed + } + accountIDs.add(Hex.toStringCondensed(accountIDAsData)) } else { - deferred.reject(Error.Generic) - return@success + val hexEncodedNonce = intermediate["nonce"] as? String ?: throw Error.Generic + val nonce = Hex.fromStringCondensed(hexEncodedNonce) + val key = ByteArray(GenericHash.BYTES) + if (!sodium.cryptoGenericHash(key, key.size, nameAsData, nameAsData.size.toLong(), nameHash, nameHash.size)) { + throw Error.HashingFailed + } + val accountIDAsData = ByteArray(accountIDByteCount) + if (!sodium.cryptoAeadXChaCha20Poly1305IetfDecrypt(accountIDAsData, null, null, ciphertext, ciphertext.size.toLong(), null, 0, nonce, key)) { + throw Error.DecryptionFailed + } + accountIDs.add(Hex.toStringCondensed(accountIDAsData)) } } - if (accountIDs.size == validationCount && accountIDs.toSet().size == 1) { - deferred.resolve(accountIDs.first()) - } else { - deferred.reject(Error.ValidationFailed) - } + accountIDs.takeIf { it.size == validationCount && it.toSet().size == 1 }?.first() + ?: throw Error.ValidationFailed } - return promise } - fun getSwarm(publicKey: String): Promise, Exception> { - val cachedSwarm = database.getSwarm(publicKey) - return if (cachedSwarm != null && cachedSwarm.size >= minimumSwarmSnodeCount) { - val cachedSwarmCopy = mutableSetOf() // Workaround for a Kotlin compiler issue - cachedSwarmCopy.addAll(cachedSwarm) - task { cachedSwarmCopy } - } else { - val parameters = mapOf( "pubKey" to publicKey ) - getRandomSnode().bind { - invoke(Snode.Method.GetSwarm, it, parameters, publicKey) + fun getSwarm(publicKey: String): Promise, Exception> = + database.getSwarm(publicKey)?.takeIf { it.size >= minimumSwarmSnodeCount }?.let(Promise.Companion::of) + ?: getRandomSnode().bind { + invoke(Snode.Method.GetSwarm, it, parameters = mapOf( "pubKey" to publicKey ), publicKey) }.map { parseSnodes(it).toSet() }.success { database.setSwarm(publicKey, it) } - } + + private fun signAndEncode(data: ByteArray, userED25519KeyPair: KeyPair) = sign(data, userED25519KeyPair).let(Base64::encodeBytes) + private fun sign(data: ByteArray, userED25519KeyPair: KeyPair): ByteArray = ByteArray(Sign.BYTES).also { + sodium.cryptoSignDetached( + it, + data, + data.size.toLong(), + userED25519KeyPair.secretKey.asBytes + ) } fun getRawMessages(snode: Snode, publicKey: String, requiresAuth: Boolean = true, namespace: Int = 0): RawResponsePromise { @@ -365,23 +309,19 @@ object SnodeAPI { } val timestamp = System.currentTimeMillis() + clockOffset val ed25519PublicKey = userED25519KeyPair.publicKey.asHexString - val signature = ByteArray(Sign.BYTES) - val verificationData = - if (namespace != 0) "retrieve$namespace$timestamp".toByteArray() - else "retrieve$timestamp".toByteArray() - try { - sodium.cryptoSignDetached( - signature, - verificationData, - verificationData.size.toLong(), - userED25519KeyPair.secretKey.asBytes - ) + val verificationData = buildString { + append("retrieve") + if (namespace != 0) append(namespace) + append(timestamp) + }.toByteArray() + val signature = try { + signAndEncode(verificationData, userED25519KeyPair) } catch (exception: Exception) { return Promise.ofFail(Error.SigningFailed) } parameters["timestamp"] = timestamp parameters["pubkey_ed25519"] = ed25519PublicKey - parameters["signature"] = Base64.encodeBytes(signature) + parameters["signature"] = signature } // If the namespace is default (0) here it will be implicitly read as 0 on the storage server @@ -394,42 +334,34 @@ object SnodeAPI { return invoke(Snode.Method.Retrieve, snode, parameters, publicKey) } - fun buildAuthenticatedStoreBatchInfo(publicKey: String, namespace: Int, message: SnodeMessage): SnodeBatchRequestInfo? { - val params = mutableMapOf() - // load the message data params into the sub request - // currently loads: - // pubKey - // data - // ttl - // timestamp - params.putAll(message.toJSON()) - params["namespace"] = namespace - + fun buildAuthenticatedStoreBatchInfo(namespace: Int, message: SnodeMessage): SnodeBatchRequestInfo? { // used for sig generation since it is also the value used in timestamp parameter val messageTimestamp = message.timestamp - val userEd25519KeyPair = try { - MessagingModuleConfiguration.shared.getUserED25519KeyPair() ?: return null + val userED25519KeyPair = runCatching { MessagingModuleConfiguration.shared.getUserED25519KeyPair() }.getOrNull() ?: return null + + val verificationData = "store$namespace$messageTimestamp".toByteArray() + val signature = try { + signAndEncode(verificationData, userED25519KeyPair) } catch (e: Exception) { + Log.e("Loki", "Signing data failed with user secret key", e) return null } - val ed25519PublicKey = userEd25519KeyPair.publicKey.asHexString - val signature = ByteArray(Sign.BYTES) - val verificationData = "store$namespace$messageTimestamp".toByteArray() - try { - sodium.cryptoSignDetached( - signature, - verificationData, - verificationData.size.toLong(), - userEd25519KeyPair.secretKey.asBytes - ) - } catch (e: Exception) { - Log.e("Loki", "Signing data failed with user secret key", e) + val params = buildMap { + // load the message data params into the sub request + // currently loads: + // pubKey + // data + // ttl + // timestamp + putAll(message.toJSON()) + this["namespace"] = namespace + // timestamp already set + this["pubkey_ed25519"] = userED25519KeyPair.publicKey.asHexString + this["signature"] = signature } - // timestamp already set - params["pubkey_ed25519"] = ed25519PublicKey - params["signature"] = Base64.encodeBytes(signature) + return SnodeBatchRequestInfo( Snode.Method.SendMessage.rawValue, params, @@ -444,32 +376,26 @@ object SnodeAPI { * @param required indicates that *at least one* message in the list is deleted from the server, otherwise it will return 404 */ fun buildAuthenticatedDeleteBatchInfo(publicKey: String, messageHashes: List, required: Boolean = false): SnodeBatchRequestInfo? { - val params = mutableMapOf( - "pubkey" to publicKey, - "required" to required, // could be omitted technically but explicit here - "messages" to messageHashes - ) val userEd25519KeyPair = try { MessagingModuleConfiguration.shared.getUserED25519KeyPair() ?: return null } catch (e: Exception) { return null } val ed25519PublicKey = userEd25519KeyPair.publicKey.asHexString - val signature = ByteArray(Sign.BYTES) - val verificationData = "delete${messageHashes.joinToString("")}".toByteArray() - try { - sodium.cryptoSignDetached( - signature, - verificationData, - verificationData.size.toLong(), - userEd25519KeyPair.secretKey.asBytes - ) + val verificationData = sequenceOf("delete").plus(messageHashes).toByteArray() + val signature = try { + signAndEncode(verificationData, userEd25519KeyPair) } catch (e: Exception) { Log.e("Loki", "Signing data failed with user secret key", e) return null } - params["pubkey_ed25519"] = ed25519PublicKey - params["signature"] = Base64.encodeBytes(signature) + val params = buildMap { + this["pubkey"] = publicKey + this["required"] = required // could be omitted technically but explicit here + this["messages"] = messageHashes + this["pubkey_ed25519"] = ed25519PublicKey + this["signature"] = signature + } return SnodeBatchRequestInfo( Snode.Method.DeleteMessage.rawValue, params, @@ -479,39 +405,25 @@ object SnodeAPI { fun buildAuthenticatedRetrieveBatchRequest(snode: Snode, publicKey: String, namespace: Int = 0, maxSize: Int? = null): SnodeBatchRequestInfo? { val lastHashValue = database.getLastMessageHashValue(snode, publicKey, namespace) ?: "" - val params = mutableMapOf( - "pubkey" to publicKey, - "last_hash" to lastHashValue, - ) - val userEd25519KeyPair = try { - MessagingModuleConfiguration.shared.getUserED25519KeyPair() ?: return null - } catch (e: Exception) { - return null - } + val userEd25519KeyPair = runCatching { MessagingModuleConfiguration.shared.getUserED25519KeyPair() }.getOrNull() ?: return null val ed25519PublicKey = userEd25519KeyPair.publicKey.asHexString val timestamp = System.currentTimeMillis() + clockOffset - val signature = ByteArray(Sign.BYTES) val verificationData = if (namespace == 0) "retrieve$timestamp".toByteArray() else "retrieve$namespace$timestamp".toByteArray() - try { - sodium.cryptoSignDetached( - signature, - verificationData, - verificationData.size.toLong(), - userEd25519KeyPair.secretKey.asBytes - ) + val signature = try { + signAndEncode(verificationData, userEd25519KeyPair) } catch (e: Exception) { Log.e("Loki", "Signing data failed with user secret key", e) return null } - params["timestamp"] = timestamp - params["pubkey_ed25519"] = ed25519PublicKey - params["signature"] = Base64.encodeBytes(signature) - if (namespace != 0) { - params["namespace"] = namespace - } - if (maxSize != null) { - params["max_size"] = maxSize + val params = buildMap { + this["pubkey"] = publicKey + this["last_hash"] = lastHashValue + this["timestamp"] = timestamp + this["pubkey_ed25519"] = ed25519PublicKey + this["signature"] = signature + if (namespace != 0) this["namespace"] = namespace + if (maxSize != null) this["max_size"] = maxSize } return SnodeBatchRequestInfo( Snode.Method.Retrieve.rawValue, @@ -535,13 +447,12 @@ object SnodeAPI { } fun getRawBatchResponse(snode: Snode, publicKey: String, requests: List, sequence: Boolean = false): RawResponsePromise { - val parameters = mutableMapOf( - "requests" to requests - ) + val parameters = buildMap { this["requests"] = requests } return invoke(if (sequence) Snode.Method.Sequence else Snode.Method.Batch, snode, parameters, publicKey).success { rawResponses -> - val responseList = (rawResponses["results"] as List) - responseList.forEachIndexed { index, response -> - if (response["code"] as? Int != 200) { + rawResponses["results"].let { it as List } + .asSequence() + .filter { it["code"] as? Int != 200 } + .forEach { response -> Log.w("Loki", "response code was not 200") handleSnodeError( response["code"] as? Int ?: 0, @@ -550,7 +461,6 @@ object SnodeAPI { publicKey ) } - } } } @@ -562,14 +472,8 @@ object SnodeAPI { val signData = "${Snode.Method.GetExpiries.rawValue}$timestamp${hashes.joinToString(separator = "")}".toByteArray() val ed25519PublicKey = userEd25519KeyPair.publicKey.asHexString - val signature = ByteArray(Sign.BYTES) - try { - sodium.cryptoSignDetached( - signature, - signData, - signData.size.toLong(), - userEd25519KeyPair.secretKey.asBytes - ) + val signature = try { + signAndEncode(signData, userEd25519KeyPair) } catch (e: Exception) { Log.e("Loki", "Signing data failed with user secret key", e) return@retryIfNeeded Promise.ofFail(e) @@ -579,7 +483,7 @@ object SnodeAPI { "messages" to hashes, "timestamp" to timestamp, "pubkey_ed25519" to ed25519PublicKey, - "signature" to Base64.encodeBytes(signature) + "signature" to signature ) getSingleTargetSnode(publicKey) bind { snode -> invoke(Snode.Method.GetExpiries, snode, params, publicKey) @@ -587,8 +491,8 @@ object SnodeAPI { } } - fun alterTtl(messageHashes: List, newExpiry: Long, publicKey: String, extend: Boolean = false, shorten: Boolean = false): RawResponsePromise { - return retryIfNeeded(maxRetryCount) { + fun alterTtl(messageHashes: List, newExpiry: Long, publicKey: String, extend: Boolean = false, shorten: Boolean = false): RawResponsePromise = + retryIfNeeded(maxRetryCount) { val params = buildAlterTtlParams(messageHashes, newExpiry, publicKey, extend, shorten) ?: return@retryIfNeeded Promise.ofFail( Exception("Couldn't build signed params for alterTtl request for newExpiry=$newExpiry, extend=$extend, shorten=$shorten") @@ -597,111 +501,96 @@ object SnodeAPI { invoke(Snode.Method.Expire, snode, params, publicKey) } } - } private fun buildAlterTtlParams( // TODO: in future this will probably need to use the closed group subkeys / admin keys for group swarms messageHashes: List, newExpiry: Long, publicKey: String, extend: Boolean = false, - shorten: Boolean = false): Map? { + shorten: Boolean = false + ): Map? { val userEd25519KeyPair = MessagingModuleConfiguration.shared.getUserED25519KeyPair() ?: return null - val params = mutableMapOf( - "expiry" to newExpiry, - "messages" to messageHashes, - ) - if (extend) { - params["extend"] = true - } else if (shorten) { - params["shorten"] = true - } + val shortenOrExtend = if (extend) "extend" else if (shorten) "shorten" else "" val signData = "${Snode.Method.Expire.rawValue}$shortenOrExtend$newExpiry${messageHashes.joinToString(separator = "")}".toByteArray() - val ed25519PublicKey = userEd25519KeyPair.publicKey.asHexString - val signature = ByteArray(Sign.BYTES) - try { - sodium.cryptoSignDetached( - signature, - signData, - signData.size.toLong(), - userEd25519KeyPair.secretKey.asBytes - ) + val signature = try { + signAndEncode(signData, userEd25519KeyPair) } catch (e: Exception) { Log.e("Loki", "Signing data failed with user secret key", e) return null } - params["pubkey"] = publicKey - params["pubkey_ed25519"] = ed25519PublicKey - params["signature"] = Base64.encodeBytes(signature) - return params - } - - fun getMessages(publicKey: String): MessageListPromise { - return retryIfNeeded(maxRetryCount) { - getSingleTargetSnode(publicKey).bind { snode -> - getRawMessages(snode, publicKey).map { parseRawMessagesResponse(it, snode, publicKey) } + return buildMap { + this["expiry"] = newExpiry + this["messages"] = messageHashes + when { + extend -> this["extend"] = true + shorten -> this["shorten"] = true } + this["pubkey"] = publicKey + this["pubkey_ed25519"] = userEd25519KeyPair.publicKey.asHexString + this["signature"] = signature } } - private fun getNetworkTime(snode: Snode): Promise, Exception> { - return invoke(Snode.Method.Info, snode, emptyMap()).map { rawResponse -> + fun getMessages(publicKey: String): MessageListPromise = retryIfNeeded(maxRetryCount) { + getSingleTargetSnode(publicKey).bind { snode -> + getRawMessages(snode, publicKey).map { parseRawMessagesResponse(it, snode, publicKey) } + } + } + + private fun getNetworkTime(snode: Snode): Promise, Exception> = + invoke(Snode.Method.Info, snode, emptyMap()).map { rawResponse -> val timestamp = rawResponse["timestamp"] as? Long ?: -1 snode to timestamp } - } - fun sendMessage(message: SnodeMessage, requiresAuth: Boolean = false, namespace: Int = 0): RawResponsePromise { - val destination = message.recipient - return retryIfNeeded(maxRetryCount) { + fun sendMessage(message: SnodeMessage, requiresAuth: Boolean = false, namespace: Int = 0): RawResponsePromise = + retryIfNeeded(maxRetryCount) { val module = MessagingModuleConfiguration.shared val userED25519KeyPair = module.getUserED25519KeyPair() ?: return@retryIfNeeded Promise.ofFail(Error.NoKeyPair) - val parameters = message.toJSON().toMutableMap() + val parameters = message.toJSON().toMutableMap() // Construct signature if (requiresAuth) { val sigTimestamp = nowWithOffset val ed25519PublicKey = userED25519KeyPair.publicKey.asHexString - val signature = ByteArray(Sign.BYTES) // assume namespace here is non-zero, as zero namespace doesn't require auth val verificationData = "store$namespace$sigTimestamp".toByteArray() - try { - sodium.cryptoSignDetached(signature, verificationData, verificationData.size.toLong(), userED25519KeyPair.secretKey.asBytes) + val signature = try { + signAndEncode(verificationData, userED25519KeyPair) } catch (exception: Exception) { return@retryIfNeeded Promise.ofFail(Error.SigningFailed) } parameters["sig_timestamp"] = sigTimestamp parameters["pubkey_ed25519"] = ed25519PublicKey - parameters["signature"] = Base64.encodeBytes(signature) + parameters["signature"] = signature } // If the namespace is default (0) here it will be implicitly read as 0 on the storage server // we only need to specify it explicitly if we want to (in future) or if it is non-zero if (namespace != 0) { parameters["namespace"] = namespace } + val destination = message.recipient getSingleTargetSnode(destination).bind { snode -> invoke(Snode.Method.SendMessage, snode, parameters, destination) } } - } - fun deleteMessage(publicKey: String, serverHashes: List): Promise, Exception> { - return retryIfNeeded(maxRetryCount) { + fun deleteMessage(publicKey: String, serverHashes: List): Promise, Exception> = + retryIfNeeded(maxRetryCount) { val module = MessagingModuleConfiguration.shared val userED25519KeyPair = module.getUserED25519KeyPair() ?: return@retryIfNeeded Promise.ofFail(Error.NoKeyPair) val userPublicKey = module.storage.getUserPublicKey() ?: return@retryIfNeeded Promise.ofFail(Error.NoKeyPair) getSingleTargetSnode(publicKey).bind { snode -> retryIfNeeded(maxRetryCount) { - val signature = ByteArray(Sign.BYTES) val verificationData = sequenceOf(Snode.Method.DeleteMessage.rawValue).plus(serverHashes).toByteArray() - sodium.cryptoSignDetached(signature, verificationData, verificationData.size.toLong(), userED25519KeyPair.secretKey.asBytes) val deleteMessageParams = mapOf( "pubkey" to userPublicKey, "pubkey_ed25519" to userED25519KeyPair.publicKey.asHexString, "messages" to serverHashes, - "signature" to Base64.encodeBytes(signature) + "signature" to signAndEncode(verificationData, userED25519KeyPair) ) invoke(Snode.Method.DeleteMessage, snode, deleteMessageParams, publicKey).map { rawResponse -> val swarms = rawResponse["swarm"] as? Map ?: return@map mapOf() @@ -729,44 +618,36 @@ object SnodeAPI { } } } - } // Parsing private fun parseSnodes(rawResponse: Any): List = (rawResponse as? Map<*, *>) ?.run { get("snodes") as? List<*> } - ?.mapNotNull { rawSnode -> - val rawSnodeAsJSON = rawSnode as? Map<*, *> - val address = rawSnodeAsJSON?.get("ip") as? String - val portAsString = rawSnodeAsJSON?.get("port") as? String - val port = portAsString?.toInt() - val ed25519Key = rawSnodeAsJSON?.get(KEY_ED25519) as? String - val x25519Key = rawSnodeAsJSON?.get(KEY_X25519) as? String + ?.asSequence() + ?.mapNotNull { it as? Map<*, *> } + ?.mapNotNull { + createSnode( + address = it["ip"] as? String, + port = (it["port"] as? String)?.toInt(), + ed25519Key = it[KEY_ED25519] as? String, + x25519Key = it[KEY_X25519] as? String + ).apply { if (this == null) Log.d("Loki", "Failed to parse snode from: ${it.prettifiedDescription()}.") } + }?.toList() ?: listOf().also { Log.d("Loki", "Failed to parse snodes from: ${rawResponse.prettifiedDescription()}.") } - if (address != null && port != null && ed25519Key != null && x25519Key != null && address != "0.0.0.0") { - Snode("https://$address", port, Snode.KeySet(ed25519Key, x25519Key), Snode.Version.ZERO) - } else { - Log.d("Loki", "Failed to parse snode from: ${rawSnode?.prettifiedDescription()}.") - null - } - } ?: listOf().also { Log.d("Loki", "Failed to parse snodes from: ${rawResponse.prettifiedDescription()}.") } - - fun deleteAllMessages(): Promise, Exception> { - return retryIfNeeded(maxRetryCount) { + fun deleteAllMessages(): Promise, Exception> = + retryIfNeeded(maxRetryCount) { val module = MessagingModuleConfiguration.shared val userED25519KeyPair = module.getUserED25519KeyPair() ?: return@retryIfNeeded Promise.ofFail(Error.NoKeyPair) val userPublicKey = module.storage.getUserPublicKey() ?: return@retryIfNeeded Promise.ofFail(Error.NoKeyPair) getSingleTargetSnode(userPublicKey).bind { snode -> retryIfNeeded(maxRetryCount) { getNetworkTime(snode).bind { (_, timestamp) -> - val signature = ByteArray(Sign.BYTES) val verificationData = (Snode.Method.DeleteAll.rawValue + Namespace.ALL + timestamp.toString()).toByteArray() - sodium.cryptoSignDetached(signature, verificationData, verificationData.size.toLong(), userED25519KeyPair.secretKey.asBytes) val deleteMessageParams = mapOf( "pubkey" to userPublicKey, "pubkey_ed25519" to userED25519KeyPair.publicKey.asHexString, "timestamp" to timestamp, - "signature" to Base64.encodeBytes(signature), + "signature" to signAndEncode(verificationData, userED25519KeyPair), "namespace" to Namespace.ALL, ) invoke(Snode.Method.DeleteAll, snode, deleteMessageParams, userPublicKey).map { @@ -778,75 +659,14 @@ object SnodeAPI { } } } - } - fun updateExpiry(updatedExpiryMs: Long, serverHashes: List): Promise, Long>>, Exception> { - return retryIfNeeded(maxRetryCount) { - val module = MessagingModuleConfiguration.shared - val userED25519KeyPair = module.getUserED25519KeyPair() ?: return@retryIfNeeded Promise.ofFail(Error.NoKeyPair) - val userPublicKey = module.storage.getUserPublicKey() ?: return@retryIfNeeded Promise.ofFail(Error.NoKeyPair) - val updatedExpiryMsWithNetworkOffset = updatedExpiryMs + clockOffset - getSingleTargetSnode(userPublicKey).bind { snode -> - retryIfNeeded(maxRetryCount) { - // "expire" || expiry || messages[0] || ... || messages[N] - val verificationData = sequenceOf(Snode.Method.Expire.rawValue, "$updatedExpiryMsWithNetworkOffset").plus(serverHashes).toByteArray() - val signature = ByteArray(Sign.BYTES) - sodium.cryptoSignDetached( - signature, - verificationData, - verificationData.size.toLong(), - userED25519KeyPair.secretKey.asBytes - ) - val params = mapOf( - "pubkey" to userPublicKey, - "pubkey_ed25519" to userED25519KeyPair.publicKey.asHexString, - "expiry" to updatedExpiryMs, - "messages" to serverHashes, - "signature" to Base64.encodeBytes(signature) - ) - invoke(Snode.Method.Expire, snode, params, userPublicKey).map { rawResponse -> - val swarms = rawResponse["swarm"] as? Map ?: return@map mapOf() - val result = swarms.mapNotNull { (hexSnodePublicKey, rawJSON) -> - val json = rawJSON as? Map ?: return@mapNotNull null - val isFailed = json["failed"] as? Boolean ?: false - val statusCode = json["code"] as? String - val reason = json["reason"] as? String - hexSnodePublicKey to if (isFailed) { - Log.e("Loki", "Failed to update expiry for: $hexSnodePublicKey due to error: $reason ($statusCode).") - listOf() to 0L - } else { - val hashes = json["updated"] as List - val expiryApplied = json["expiry"] as Long - val signature = json["signature"] as String - val snodePublicKey = Key.fromHexString(hexSnodePublicKey) - // The signature looks like ( PUBKEY_HEX || RMSG[0] || ... || RMSG[N] || DMSG[0] || ... || DMSG[M] ) - val message = sequenceOf(userPublicKey).plus(serverHashes).plus(hashes).toByteArray() - if (sodium.cryptoSignVerifyDetached(Base64.decode(signature), message, message.size, snodePublicKey.asBytes)) { - hashes to expiryApplied - } else listOf() to 0L - } - } - return@map result.toMap() - }.fail { e -> - Log.e("Loki", "Failed to update expiry", e) - } - } - } - } - } - - fun parseRawMessagesResponse(rawResponse: RawResponse, snode: Snode, publicKey: String, namespace: Int = 0, updateLatestHash: Boolean = true, updateStoredHashes: Boolean = true): List> { - val messages = rawResponse["messages"] as? List<*> - return if (messages != null) { + fun parseRawMessagesResponse(rawResponse: RawResponse, snode: Snode, publicKey: String, namespace: Int = 0, updateLatestHash: Boolean = true, updateStoredHashes: Boolean = true): List> = + (rawResponse["messages"] as? List<*>)?.let { messages -> if (updateLatestHash) { updateLastMessageHashValueIfPossible(snode, publicKey, messages, namespace) } - val newRawMessages = removeDuplicates(publicKey, messages, namespace, updateStoredHashes) - return parseEnvelopes(newRawMessages) - } else { - listOf() - } - } + removeDuplicates(publicKey, messages, namespace, updateStoredHashes).let(::parseEnvelopes) + } ?: listOf() fun updateLastMessageHashValueIfPossible(snode: Snode, publicKey: String, rawMessages: List<*>, namespace: Int) { val lastMessageAsJSON = rawMessages.lastOrNull() as? Map<*, *> @@ -859,34 +679,33 @@ object SnodeAPI { } fun removeDuplicates(publicKey: String, rawMessages: List<*>, namespace: Int, updateStoredHashes: Boolean): List<*> { - val originalMessageHashValues = database.getReceivedMessageHashValues(publicKey, namespace)?.toMutableSet() ?: mutableSetOf() + val originalMessageHashValues = database.getReceivedMessageHashValues(publicKey, namespace) ?: emptySet() val receivedMessageHashValues = originalMessageHashValues.toMutableSet() val result = rawMessages.filter { rawMessage -> - val rawMessageAsJSON = rawMessage as? Map<*, *> - val hashValue = rawMessageAsJSON?.get("hash") as? String - if (hashValue != null) { - val isDuplicate = receivedMessageHashValues.contains(hashValue) - receivedMessageHashValues.add(hashValue) - !isDuplicate - } else { - Log.d("Loki", "Missing hash value for message: ${rawMessage?.prettifiedDescription()}.") - false - } + (rawMessage as? Map<*, *>) + ?.let { it["hash"] as? String } + ?.let { receivedMessageHashValues.add(it) } + ?: false.also { Log.d("Loki", "Missing hash value for message: ${rawMessage?.prettifiedDescription()}.") } } - if (originalMessageHashValues != receivedMessageHashValues && updateStoredHashes) { + if (updateStoredHashes && originalMessageHashValues.containsAll(receivedMessageHashValues)) { database.setReceivedMessageHashValues(publicKey, receivedMessageHashValues, namespace) } return result } - private fun parseEnvelopes(rawMessages: List<*>): List> { - return rawMessages.mapNotNull { rawMessage -> + private fun parseEnvelopes(rawMessages: List<*>): List> = + rawMessages.mapNotNull { rawMessage -> val rawMessageAsJSON = rawMessage as? Map<*, *> val base64EncodedData = rawMessageAsJSON?.get("data") as? String val data = base64EncodedData?.let { Base64.decode(it) } + + data?.runCatching(MessageWrapper::unwrap) + ?.map { it to rawMessageAsJSON["hash"] as? String } + ?.onFailure { Log.d("Loki", "Failed to unwrap data for message: ${rawMessage.prettifiedDescription()}.") } + if (data != null) { try { - Pair(MessageWrapper.unwrap(data), rawMessageAsJSON.get("hash") as? String) + MessageWrapper.unwrap(data) to rawMessageAsJSON["hash"] as? String } catch (e: Exception) { Log.d("Loki", "Failed to unwrap data for message: ${rawMessage.prettifiedDescription()}.") null @@ -896,12 +715,11 @@ object SnodeAPI { null } } - } @Suppress("UNCHECKED_CAST") private fun parseDeletions(userPublicKey: String, timestamp: Long, rawResponse: RawResponse): Map { val swarms = rawResponse["swarm"] as? Map ?: return mapOf() - val result = swarms.mapNotNull { (hexSnodePublicKey, rawJSON) -> + return swarms.mapNotNull { (hexSnodePublicKey, rawJSON) -> val json = rawJSON as? Map ?: return@mapNotNull null val isFailed = json["failed"] as? Boolean ?: false val statusCode = json["code"] as? String @@ -917,14 +735,13 @@ object SnodeAPI { val message = sequenceOf(userPublicKey, "$timestamp").plus(hashes).toByteArray() sodium.cryptoSignVerifyDetached(Base64.decode(signature), message, message.size, snodePublicKey.asBytes) } - } - return result.toMap() + }.toMap() } // endregion // Error Handling - internal fun handleSnodeError(statusCode: Int, json: Map<*, *>?, snode: Snode, publicKey: String? = null): Exception? { + internal fun handleSnodeError(statusCode: Int, json: Map<*, *>?, snode: Snode, publicKey: String? = null): Throwable? = runCatching { fun handleBadSnode() { val oldFailureCount = snodeFailureCount[snode] ?: 0 val newFailureCount = oldFailureCount + 1 @@ -932,56 +749,43 @@ object SnodeAPI { Log.d("Loki", "Couldn't reach snode at $snode; setting failure count to $newFailureCount.") if (newFailureCount >= snodeFailureThreshold) { Log.d("Loki", "Failure threshold reached for: $snode; dropping it.") - if (publicKey != null) { - dropSnodeFromSwarmIfNeeded(snode, publicKey) - } - snodePool = snodePool.toMutableSet().minus(snode).toSet() + publicKey?.let { dropSnodeFromSwarmIfNeeded(snode, it) } + snodePool -= snode Log.d("Loki", "Snode pool count: ${snodePool.count()}.") snodeFailureCount[snode] = 0 } } when (statusCode) { - 400, 500, 502, 503 -> { // Usually indicates that the snode isn't up to date - handleBadSnode() - } + // Usually indicates that the snode isn't up to date + 400, 500, 502, 503 -> handleBadSnode() 406 -> { Log.d("Loki", "The user's clock is out of sync with the service node network.") broadcaster.broadcast("clockOutOfSync") - return Error.ClockOutOfSync + throw Error.ClockOutOfSync } 421 -> { // The snode isn't associated with the given public key anymore if (publicKey != null) { - fun invalidateSwarm() { - Log.d("Loki", "Invalidating swarm for: $publicKey.") - dropSnodeFromSwarmIfNeeded(snode, publicKey) - } - if (json != null) { - val snodes = parseSnodes(json) - if (snodes.isNotEmpty()) { - database.setSwarm(publicKey, snodes.toSet()) - } else { - invalidateSwarm() + json?.let(::parseSnodes) + ?.takeIf { it.isNotEmpty() } + ?.let { database.setSwarm(publicKey, it.toSet()) } + ?: run { + Log.d("Loki", "Invalidating swarm for: $publicKey.") + dropSnodeFromSwarmIfNeeded(snode, publicKey) } - } else { - invalidateSwarm() - } - } else { - Log.d("Loki", "Got a 421 without an associated public key.") - } + } else Log.d("Loki", "Got a 421 without an associated public key.") } 404 -> { Log.d("Loki", "404, probably no file found") - return Error.Generic + throw Error.Generic } else -> { handleBadSnode() Log.d("Loki", "Unhandled response code: ${statusCode}.") - return Error.Generic + throw Error.Generic } } - return null - } + }.exceptionOrNull() } // Type Aliases diff --git a/libsignal/src/main/java/org/session/libsignal/utilities/Base64.java b/libsignal/src/main/java/org/session/libsignal/utilities/Base64.java index 3ff38f76d3..35ec22e0e0 100644 --- a/libsignal/src/main/java/org/session/libsignal/utilities/Base64.java +++ b/libsignal/src/main/java/org/session/libsignal/utilities/Base64.java @@ -1,5 +1,7 @@ package org.session.libsignal.utilities; +import androidx.annotation.NonNull; + /** *

Encodes and decodes to and from Base64 notation.

*

Homepage: http://iharder.net/base64.

@@ -714,7 +716,7 @@ public class Base64 * @throws NullPointerException if source array is null * @since 1.4 */ - public static String encodeBytes( byte[] source ) { + public static String encodeBytes(@NonNull byte[] source ) { // Since we're not going to have the GZIP encoding turned on, // we're not going to have an java.io.IOException thrown, so // we should not force the user to have to catch it. From c1d40cdbe73605c02a4718d0600cb419177cd5b5 Mon Sep 17 00:00:00 2001 From: bemusementpark Date: Sat, 3 Aug 2024 20:19:29 +0930 Subject: [PATCH 03/12] Optimise SnodesAPI --- .../org/session/libsession/snode/SnodeAPI.kt | 349 ++++++++---------- .../org/session/libsession/utilities/Util.kt | 21 ++ 2 files changed, 181 insertions(+), 189 deletions(-) diff --git a/libsession/src/main/java/org/session/libsession/snode/SnodeAPI.kt b/libsession/src/main/java/org/session/libsession/snode/SnodeAPI.kt index 8e19234b0d..6d1ffaa5c0 100644 --- a/libsession/src/main/java/org/session/libsession/snode/SnodeAPI.kt +++ b/libsession/src/main/java/org/session/libsession/snode/SnodeAPI.kt @@ -18,6 +18,8 @@ import nl.komponents.kovenant.task import org.session.libsession.messaging.MessagingModuleConfiguration import org.session.libsession.messaging.utilities.MessageWrapper import org.session.libsession.messaging.utilities.SodiumUtilities.sodium +import org.session.libsession.utilities.buildMutableMap +import org.session.libsession.utilities.mapValuesNotNull import org.session.libsession.utilities.toByteArray import org.session.libsignal.crypto.getRandomElement import org.session.libsignal.database.LokiAPIDatabaseProtocol @@ -73,20 +75,18 @@ object SnodeAPI { // Use port 4433 if the API level can handle the network security configuration and enforce pinned certificates private val seedNodePort = if (Build.VERSION.SDK_INT < Build.VERSION_CODES.N) 443 else 4443 - private const val snodeFailureThreshold = 3 - private const val useOnionRequests = true - private const val useTestnet = false - private val seedNodePool = if (useTestnet) { - setOf( "http://public.loki.foundation:38157" ) - } else { - setOf( - "https://seed1.getsession.org:$seedNodePort", - "https://seed2.getsession.org:$seedNodePort", - "https://seed3.getsession.org:$seedNodePort", - ) - } + private val seedNodePool = if (useTestnet) setOf( + "http://public.loki.foundation:38157" + ) else setOf( + "https://seed1.getsession.org:$seedNodePort", + "https://seed2.getsession.org:$seedNodePort", + "https://seed3.getsession.org:$seedNodePort", + ) + + private const val snodeFailureThreshold = 3 + private const val useOnionRequests = true private const val KEY_IP = "public_ip" private const val KEY_PORT = "storage_port" @@ -121,48 +121,45 @@ object SnodeAPI { parameters: Map, publicKey: String? = null, version: Version = Version.V3 - ): RawResponsePromise = if (useOnionRequests) OnionRequestAPI.sendOnionRequest(method, parameters, snode, version, publicKey).map { - val body = it.body ?: throw Error.Generic - JsonUtil.fromJson(body, Map::class.java) - } else task { - val payload = mapOf( "method" to method.rawValue, "params" to parameters ) - try { - val url = "${snode.address}:${snode.port}/storage_rpc/v1" - val response = HTTP.execute(HTTP.Verb.POST, url, payload).toString() - JsonUtil.fromJson(response, Map::class.java) - } catch (exception: Exception) { - (exception as? HTTP.HTTPRequestFailedException)?.run { - handleSnodeError(statusCode, json, snode, publicKey) - // TODO Check if we meant to throw the error returned by handleSnodeError - throw exception + ): RawResponsePromise = when { + useOnionRequests -> OnionRequestAPI.sendOnionRequest(method, parameters, snode, version, publicKey).map { + JsonUtil.fromJson(it.body ?: throw Error.Generic, Map::class.java) + } + else -> task { + HTTP.execute( + HTTP.Verb.POST, + url = "${snode.address}:${snode.port}/storage_rpc/v1", + parameters = buildMap { + this["method"] = method.rawValue + this["params"] = parameters } - Log.d("Loki", "Unhandled exception: $exception.") - throw exception + ).toString().let { + JsonUtil.fromJson(it, Map::class.java) + } + }.fail { e -> + when (e) { + is HTTP.HTTPRequestFailedException -> handleSnodeError(e.statusCode, e.json, snode, publicKey) + else -> Log.d("Loki", "Unhandled exception: $e.") } } + } + + private val GET_RANDOM_SNODE_PARAMS = buildMap { + this["method"] = "get_n_service_nodes" + this["params"] = buildMap { + this["active_only"] = true + this["fields"] = sequenceOf(KEY_IP, KEY_PORT, KEY_X25519, KEY_ED25519, KEY_VERSION).associateWith { true } + } + } internal fun getRandomSnode(): Promise = - snodePool.takeIf { it.size >= minimumSnodePoolCount }?.let { Promise.of(it.getRandomElement()) } ?: task { + snodePool.takeIf { it.size >= minimumSnodePoolCount }?.getRandomElement()?.let { Promise.of(it) } ?: task { val target = seedNodePool.random() - val url = "$target/json_rpc" Log.d("Loki", "Populating snode pool using: $target.") - val parameters = mapOf( - "method" to "get_n_service_nodes", - "params" to mapOf( - "active_only" to true, - "fields" to mapOf( - KEY_IP to true, KEY_PORT to true, - KEY_X25519 to true, KEY_ED25519 to true, - KEY_VERSION to true - ) - ) - ) - val response = HTTP.execute(HTTP.Verb.POST, url, parameters, useSeedNodeConnection = true) - val json = try { - JsonUtil.fromJson(response, Map::class.java) - } catch (exception: Exception) { - mapOf( "result" to response.toString()) - } + val url = "$target/json_rpc" + val response = HTTP.execute(HTTP.Verb.POST, url, GET_RANDOM_SNODE_PARAMS, useSeedNodeConnection = true) + val json = runCatching { JsonUtil.fromJson(response, Map::class.java) }.getOrNull() + ?: buildMap { this["result"] = response.toString() } val intermediate = json["result"] as? Map<*, *> ?: throw Error.Generic .also { Log.d("Loki", "Failed to update snode pool, intermediate was null.") } val rawSnodes = intermediate["service_node_states"] as? List<*> ?: throw Error.Generic @@ -180,7 +177,7 @@ object SnodeAPI { ).also { if (it == null) Log.d("Loki", "Failed to parse: ${rawSnode.prettifiedDescription()}.") } }.toSet().also { Log.d("Loki", "Persisting snode pool to database.") - this.snodePool = it + snodePool = it }.runCatching { getRandomElement() }.onFailure { Log.d("Loki", "Got an empty snode pool from: $target.") throw SnodeAPI.Error.Generic @@ -220,10 +217,13 @@ object SnodeAPI { } val base64EncodedNameHash = Base64.encodeBytes(nameHash) // Ask 3 different snodes for the Account ID associated with the given name hash - val parameters = mapOf( - "endpoint" to "ons_resolve", - "params" to mapOf( "type" to 0, "name_hash" to base64EncodedNameHash ) - ) + val parameters = buildMap { + this["endpoint"] = "ons_resolve" + this["params"] = buildMap { + this["type"] = 0 + this["name_hash"] = base64EncodedNameHash + } + } val promises = List(validationCount) { getRandomSnode().bind { snode -> retryIfNeeded(maxRetryCount) { @@ -232,10 +232,9 @@ object SnodeAPI { } } return all(promises).map { results -> - val accountIDs = mutableListOf() - for (json in results) { - val intermediate = json["result"] as? Map<*, *> - val hexEncodedCiphertext = intermediate?.get("encrypted_value") as? String ?: throw Error.Generic + results.map { json -> + val intermediate = json["result"] as? Map<*, *> ?: throw Error.Generic + val hexEncodedCiphertext = intermediate["encrypted_value"] as? String ?: throw Error.Generic val ciphertext = Hex.fromStringCondensed(hexEncodedCiphertext) val isArgon2Based = (intermediate["nonce"] == null) if (isArgon2Based) { @@ -251,7 +250,7 @@ object SnodeAPI { if (!sodium.cryptoSecretBoxOpenEasy(accountIDAsData, ciphertext, ciphertext.size.toLong(), nonce, key)) { throw Error.DecryptionFailed } - accountIDs.add(Hex.toStringCondensed(accountIDAsData)) + Hex.toStringCondensed(accountIDAsData) } else { val hexEncodedNonce = intermediate["nonce"] as? String ?: throw Error.Generic val nonce = Hex.fromStringCondensed(hexEncodedNonce) @@ -263,10 +262,9 @@ object SnodeAPI { if (!sodium.cryptoAeadXChaCha20Poly1305IetfDecrypt(accountIDAsData, null, null, ciphertext, ciphertext.size.toLong(), null, 0, nonce, key)) { throw Error.DecryptionFailed } - accountIDs.add(Hex.toStringCondensed(accountIDAsData)) + Hex.toStringCondensed(accountIDAsData) } - } - accountIDs.takeIf { it.size == validationCount && it.toSet().size == 1 }?.first() + }.takeIf { it.size == validationCount && it.toSet().size == 1 }?.first() ?: throw Error.ValidationFailed } } @@ -274,30 +272,31 @@ object SnodeAPI { fun getSwarm(publicKey: String): Promise, Exception> = database.getSwarm(publicKey)?.takeIf { it.size >= minimumSwarmSnodeCount }?.let(Promise.Companion::of) ?: getRandomSnode().bind { - invoke(Snode.Method.GetSwarm, it, parameters = mapOf( "pubKey" to publicKey ), publicKey) + invoke(Snode.Method.GetSwarm, it, parameters = buildMap { this["pubKey"] = publicKey }, publicKey) }.map { parseSnodes(it).toSet() }.success { database.setSwarm(publicKey, it) } - private fun signAndEncode(data: ByteArray, userED25519KeyPair: KeyPair) = sign(data, userED25519KeyPair).let(Base64::encodeBytes) + private fun signAndEncodeCatching(data: ByteArray, userED25519KeyPair: KeyPair): Result = + runCatching { signAndEncode(data, userED25519KeyPair) } + private fun signAndEncode(data: ByteArray, userED25519KeyPair: KeyPair): String = + sign(data, userED25519KeyPair).let(Base64::encodeBytes) private fun sign(data: ByteArray, userED25519KeyPair: KeyPair): ByteArray = ByteArray(Sign.BYTES).also { - sodium.cryptoSignDetached( - it, - data, - data.size.toLong(), - userED25519KeyPair.secretKey.asBytes - ) + sodium.cryptoSignDetached(it, data, data.size.toLong(), userED25519KeyPair.secretKey.asBytes) } fun getRawMessages(snode: Snode, publicKey: String, requiresAuth: Boolean = true, namespace: Int = 0): RawResponsePromise { // Get last message hash val lastHashValue = database.getLastMessageHashValue(snode, publicKey, namespace) ?: "" - val parameters = mutableMapOf( - "pubKey" to publicKey, - "last_hash" to lastHashValue, - ) + val parameters = buildMutableMap { + this["pubKey"] = publicKey + this["last_hash"] = lastHashValue + // If the namespace is default (0) here it will be implicitly read as 0 on the storage server + // we only need to specify it explicitly if we want to (in future) or if it is non-zero + namespace.takeIf { it != 0 }?.let { this["namespace"] = it } + } // Construct signature if (requiresAuth) { val userED25519KeyPair = try { @@ -311,23 +310,13 @@ object SnodeAPI { val ed25519PublicKey = userED25519KeyPair.publicKey.asHexString val verificationData = buildString { append("retrieve") - if (namespace != 0) append(namespace) + namespace.takeIf { it != 0 }?.let(::append) append(timestamp) }.toByteArray() - val signature = try { - signAndEncode(verificationData, userED25519KeyPair) - } catch (exception: Exception) { - return Promise.ofFail(Error.SigningFailed) - } + parameters["signature"] = signAndEncodeCatching(verificationData, userED25519KeyPair).getOrNull() + ?: return Promise.ofFail(Error.SigningFailed) parameters["timestamp"] = timestamp parameters["pubkey_ed25519"] = ed25519PublicKey - parameters["signature"] = signature - } - - // If the namespace is default (0) here it will be implicitly read as 0 on the storage server - // we only need to specify it explicitly if we want to (in future) or if it is non-zero - if (namespace != 0) { - parameters["namespace"] = namespace } // Make the request @@ -341,11 +330,8 @@ object SnodeAPI { val userED25519KeyPair = runCatching { MessagingModuleConfiguration.shared.getUserED25519KeyPair() }.getOrNull() ?: return null val verificationData = "store$namespace$messageTimestamp".toByteArray() - val signature = try { - signAndEncode(verificationData, userED25519KeyPair) - } catch (e: Exception) { - Log.e("Loki", "Signing data failed with user secret key", e) - return null + val signature = signAndEncodeCatching(verificationData, userED25519KeyPair).run { + getOrNull() ?: return null.also { Log.e("Loki", "Signing data failed with user secret key", exceptionOrNull()) } } val params = buildMap { @@ -478,13 +464,13 @@ object SnodeAPI { Log.e("Loki", "Signing data failed with user secret key", e) return@retryIfNeeded Promise.ofFail(e) } - val params = mapOf( - "pubkey" to publicKey, - "messages" to hashes, - "timestamp" to timestamp, - "pubkey_ed25519" to ed25519PublicKey, - "signature" to signature - ) + val params = buildMap { + this["pubkey"] = publicKey + this["messages"] = hashes + this["timestamp"] = timestamp + this["pubkey_ed25519"] = ed25519PublicKey + this["signature"] = signature + } getSingleTargetSnode(publicKey) bind { snode -> invoke(Snode.Method.GetExpiries, snode, params, publicKey) } @@ -578,7 +564,7 @@ object SnodeAPI { } } - fun deleteMessage(publicKey: String, serverHashes: List): Promise, Exception> = + fun deleteMessage(publicKey: String, serverHashes: List): Promise, Exception> = retryIfNeeded(maxRetryCount) { val module = MessagingModuleConfiguration.shared val userED25519KeyPair = module.getUserED25519KeyPair() ?: return@retryIfNeeded Promise.ofFail(Error.NoKeyPair) @@ -586,35 +572,40 @@ object SnodeAPI { getSingleTargetSnode(publicKey).bind { snode -> retryIfNeeded(maxRetryCount) { val verificationData = sequenceOf(Snode.Method.DeleteMessage.rawValue).plus(serverHashes).toByteArray() - val deleteMessageParams = mapOf( - "pubkey" to userPublicKey, - "pubkey_ed25519" to userED25519KeyPair.publicKey.asHexString, - "messages" to serverHashes, - "signature" to signAndEncode(verificationData, userED25519KeyPair) - ) + val deleteMessageParams = buildMap { + this["pubkey"] = userPublicKey + this["pubkey_ed25519"] = userED25519KeyPair.publicKey.asHexString + this["messages"] = serverHashes + this["signature"] = signAndEncode(verificationData, userED25519KeyPair) + } invoke(Snode.Method.DeleteMessage, snode, deleteMessageParams, publicKey).map { rawResponse -> val swarms = rawResponse["swarm"] as? Map ?: return@map mapOf() - val result = swarms.mapNotNull { (hexSnodePublicKey, rawJSON) -> - val json = rawJSON as? Map ?: return@mapNotNull null - val isFailed = json["failed"] as? Boolean ?: false - val statusCode = json["code"] as? String - val reason = json["reason"] as? String - hexSnodePublicKey to if (isFailed) { - Log.e("Loki", "Failed to delete messages from: $hexSnodePublicKey due to error: $reason ($statusCode).") - false - } else { - val hashes = json["deleted"] as List // Hashes of deleted messages - val signature = json["signature"] as String - val snodePublicKey = Key.fromHexString(hexSnodePublicKey) - // The signature looks like ( PUBKEY_HEX || RMSG[0] || ... || RMSG[N] || DMSG[0] || ... || DMSG[M] ) - val message = sequenceOf(userPublicKey).plus(serverHashes).plus(hashes).toByteArray() - sodium.cryptoSignVerifyDetached(Base64.decode(signature), message, message.size, snodePublicKey.asBytes) + swarms.mapValuesNotNull { (hexSnodePublicKey, rawJSON) -> + (rawJSON as? Map)?.let { json -> + val isFailed = json["failed"] as? Boolean ?: false + val statusCode = json["code"] as? String + val reason = json["reason"] as? String + + if (isFailed) { + Log.e("Loki", "Failed to delete messages from: $hexSnodePublicKey due to error: $reason ($statusCode).") + false + } else { + // Hashes of deleted messages + val hashes = json["deleted"] as List + val signature = json["signature"] as String + val snodePublicKey = Key.fromHexString(hexSnodePublicKey) + // The signature looks like ( PUBKEY_HEX || RMSG[0] || ... || RMSG[N] || DMSG[0] || ... || DMSG[M] ) + val message = sequenceOf(userPublicKey).plus(serverHashes).plus(hashes).toByteArray() + sodium.cryptoSignVerifyDetached( + Base64.decode(signature), + message, + message.size, + snodePublicKey.asBytes + ) + } } } - return@map result.toMap() - }.fail { e -> - Log.e("Loki", "Failed to delete messages", e) - } + }.fail { e -> Log.e("Loki", "Failed to delete messages", e) } } } } @@ -643,18 +634,16 @@ object SnodeAPI { retryIfNeeded(maxRetryCount) { getNetworkTime(snode).bind { (_, timestamp) -> val verificationData = (Snode.Method.DeleteAll.rawValue + Namespace.ALL + timestamp.toString()).toByteArray() - val deleteMessageParams = mapOf( - "pubkey" to userPublicKey, - "pubkey_ed25519" to userED25519KeyPair.publicKey.asHexString, - "timestamp" to timestamp, - "signature" to signAndEncode(verificationData, userED25519KeyPair), - "namespace" to Namespace.ALL, - ) - invoke(Snode.Method.DeleteAll, snode, deleteMessageParams, userPublicKey).map { - rawResponse -> parseDeletions(userPublicKey, timestamp, rawResponse) - }.fail { e -> - Log.e("Loki", "Failed to clear data", e) + val deleteMessageParams = buildMap { + this["pubkey"] = userPublicKey + this["pubkey_ed25519"] = userED25519KeyPair.publicKey.asHexString + this["timestamp"] = timestamp + this["signature"] = signAndEncode(verificationData, userED25519KeyPair) + this["namespace"] = Namespace.ALL } + invoke(Snode.Method.DeleteAll, snode, deleteMessageParams, userPublicKey) + .map { rawResponse -> parseDeletions(userPublicKey, timestamp, rawResponse) } + .fail { e -> Log.e("Loki", "Failed to clear data", e) } } } } @@ -662,69 +651,53 @@ object SnodeAPI { fun parseRawMessagesResponse(rawResponse: RawResponse, snode: Snode, publicKey: String, namespace: Int = 0, updateLatestHash: Boolean = true, updateStoredHashes: Boolean = true): List> = (rawResponse["messages"] as? List<*>)?.let { messages -> - if (updateLatestHash) { - updateLastMessageHashValueIfPossible(snode, publicKey, messages, namespace) - } + if (updateLatestHash) updateLastMessageHashValueIfPossible(snode, publicKey, messages, namespace) removeDuplicates(publicKey, messages, namespace, updateStoredHashes).let(::parseEnvelopes) } ?: listOf() fun updateLastMessageHashValueIfPossible(snode: Snode, publicKey: String, rawMessages: List<*>, namespace: Int) { val lastMessageAsJSON = rawMessages.lastOrNull() as? Map<*, *> val hashValue = lastMessageAsJSON?.get("hash") as? String - if (hashValue != null) { - database.setLastMessageHashValue(snode, publicKey, hashValue, namespace) - } else if (rawMessages.isNotEmpty()) { - Log.d("Loki", "Failed to update last message hash value from: ${rawMessages.prettifiedDescription()}.") + when { + hashValue != null -> database.setLastMessageHashValue(snode, publicKey, hashValue, namespace) + rawMessages.isNotEmpty() -> Log.d("Loki", "Failed to update last message hash value from: ${rawMessages.prettifiedDescription()}.") } } fun removeDuplicates(publicKey: String, rawMessages: List<*>, namespace: Int, updateStoredHashes: Boolean): List<*> { val originalMessageHashValues = database.getReceivedMessageHashValues(publicKey, namespace) ?: emptySet() val receivedMessageHashValues = originalMessageHashValues.toMutableSet() - val result = rawMessages.filter { rawMessage -> + return rawMessages.filter { rawMessage -> (rawMessage as? Map<*, *>) ?.let { it["hash"] as? String } ?.let { receivedMessageHashValues.add(it) } ?: false.also { Log.d("Loki", "Missing hash value for message: ${rawMessage?.prettifiedDescription()}.") } - } - if (updateStoredHashes && originalMessageHashValues.containsAll(receivedMessageHashValues)) { - database.setReceivedMessageHashValues(publicKey, receivedMessageHashValues, namespace) - } - return result - } - - private fun parseEnvelopes(rawMessages: List<*>): List> = - rawMessages.mapNotNull { rawMessage -> - val rawMessageAsJSON = rawMessage as? Map<*, *> - val base64EncodedData = rawMessageAsJSON?.get("data") as? String - val data = base64EncodedData?.let { Base64.decode(it) } - - data?.runCatching(MessageWrapper::unwrap) - ?.map { it to rawMessageAsJSON["hash"] as? String } - ?.onFailure { Log.d("Loki", "Failed to unwrap data for message: ${rawMessage.prettifiedDescription()}.") } - - if (data != null) { - try { - MessageWrapper.unwrap(data) to rawMessageAsJSON["hash"] as? String - } catch (e: Exception) { - Log.d("Loki", "Failed to unwrap data for message: ${rawMessage.prettifiedDescription()}.") - null - } - } else { - Log.d("Loki", "Failed to decode data for message: ${rawMessage?.prettifiedDescription()}.") - null + }.also { + if (updateStoredHashes && originalMessageHashValues.containsAll(receivedMessageHashValues)) { + database.setReceivedMessageHashValues(publicKey, receivedMessageHashValues, namespace) } } + } + + private fun parseEnvelopes(rawMessages: List<*>): List> = rawMessages.mapNotNull { rawMessage -> + val rawMessageAsJSON = rawMessage as? Map<*, *> + val base64EncodedData = rawMessageAsJSON?.get("data") as? String + val data = base64EncodedData?.let { Base64.decode(it) } + + data ?: Log.d("Loki", "Failed to decode data for message: ${rawMessage?.prettifiedDescription()}.") + + data?.runCatching { MessageWrapper.unwrap(this) to rawMessageAsJSON["hash"] as? String } + ?.onFailure { Log.d("Loki", "Failed to unwrap data for message: ${rawMessage.prettifiedDescription()}.") } + ?.getOrNull() + } @Suppress("UNCHECKED_CAST") - private fun parseDeletions(userPublicKey: String, timestamp: Long, rawResponse: RawResponse): Map { - val swarms = rawResponse["swarm"] as? Map ?: return mapOf() - return swarms.mapNotNull { (hexSnodePublicKey, rawJSON) -> - val json = rawJSON as? Map ?: return@mapNotNull null - val isFailed = json["failed"] as? Boolean ?: false - val statusCode = json["code"] as? String - val reason = json["reason"] as? String - hexSnodePublicKey to if (isFailed) { + private fun parseDeletions(userPublicKey: String, timestamp: Long, rawResponse: RawResponse): Map = + (rawResponse["swarm"] as? Map)?.mapValuesNotNull { (hexSnodePublicKey, rawJSON) -> + val json = rawJSON as? Map ?: return@mapValuesNotNull null + if (json["failed"] as? Boolean == true) { + val reason = json["reason"] as? String + val statusCode = json["code"] as? String Log.e("Loki", "Failed to delete all messages from: $hexSnodePublicKey due to error: $reason ($statusCode).") false } else { @@ -735,8 +708,7 @@ object SnodeAPI { val message = sequenceOf(userPublicKey, "$timestamp").plus(hashes).toByteArray() sodium.cryptoSignVerifyDetached(Base64.decode(signature), message, message.size, snodePublicKey.asBytes) } - }.toMap() - } + } ?: mapOf() // endregion @@ -752,7 +724,7 @@ object SnodeAPI { publicKey?.let { dropSnodeFromSwarmIfNeeded(snode, it) } snodePool -= snode Log.d("Loki", "Snode pool count: ${snodePool.count()}.") - snodeFailureCount[snode] = 0 + snodeFailureCount.remove(snode) } } when (statusCode) { @@ -765,15 +737,14 @@ object SnodeAPI { } 421 -> { // The snode isn't associated with the given public key anymore - if (publicKey != null) { - json?.let(::parseSnodes) - ?.takeIf { it.isNotEmpty() } - ?.let { database.setSwarm(publicKey, it.toSet()) } - ?: run { - Log.d("Loki", "Invalidating swarm for: $publicKey.") - dropSnodeFromSwarmIfNeeded(snode, publicKey) - } - } else Log.d("Loki", "Got a 421 without an associated public key.") + if (publicKey == null) Log.d("Loki", "Got a 421 without an associated public key.") + else json?.let(::parseSnodes) + ?.takeIf { it.isNotEmpty() } + ?.let { database.setSwarm(publicKey, it.toSet()) } + ?: run { + Log.d("Loki", "Invalidating swarm for: $publicKey.") + dropSnodeFromSwarmIfNeeded(snode, publicKey) + } } 404 -> { Log.d("Loki", "404, probably no file found") diff --git a/libsession/src/main/java/org/session/libsession/utilities/Util.kt b/libsession/src/main/java/org/session/libsession/utilities/Util.kt index d47754b7ed..bc533d235f 100644 --- a/libsession/src/main/java/org/session/libsession/utilities/Util.kt +++ b/libsession/src/main/java/org/session/libsession/utilities/Util.kt @@ -402,6 +402,12 @@ fun Iterable.associateByNotNull( for(e in this) { it[keySelector(e) ?: continue] = valueTransform(e) ?: continue } } +fun Map.mapValuesNotNull( + valueTransform: (Map.Entry) -> W? +): Map = mutableMapOf().also { + for(e in this) { it[e.key] = valueTransform(e) ?: continue } +} + /** * Groups elements of the original collection by the key returned by the given [keySelector] function * applied to each element and returns a map where each group key is associated with a list of @@ -413,6 +419,21 @@ inline fun Iterable.groupByNotNull(keySelector: (E) -> K?): Map keySelector(e)?.let { k -> it.getOrPut(k) { mutableListOf() } += e } } } +/** + * Analogous to [buildMap], this function creates a [MutableMap] and populates it using the given [action]. + */ +inline fun buildMutableMap(action: MutableMap.() -> Unit): MutableMap = + mutableMapOf().apply(action) + +/** + * Converts a list of Pairs into a Map, filtering out any Pairs where the value is null. + * + * @param pairs The list of Pairs to convert. + * @return A Map with non-null values. + */ +fun Iterable>.toMapNotNull(): Map = + associateByNotNull(Pair::first, Pair::second) + fun Sequence.toByteArray(): ByteArray = ByteArrayOutputStream().use { output -> forEach { it.byteInputStream().use { input -> input.copyTo(output) } } output.toByteArray() From 3c8302f7a4ce226096b2cac79b0c81d11fee949a Mon Sep 17 00:00:00 2001 From: bemusementpark Date: Sat, 3 Aug 2024 21:19:31 +0930 Subject: [PATCH 04/12] Optimise SnodeAPI further --- .../java/org/session/libsession/snode/SnodeAPI.kt | 11 +++++------ .../java/org/session/libsignal/utilities/Snode.kt | 6 +++--- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/libsession/src/main/java/org/session/libsession/snode/SnodeAPI.kt b/libsession/src/main/java/org/session/libsession/snode/SnodeAPI.kt index 6d1ffaa5c0..4d59669d1c 100644 --- a/libsession/src/main/java/org/session/libsession/snode/SnodeAPI.kt +++ b/libsession/src/main/java/org/session/libsession/snode/SnodeAPI.kt @@ -455,7 +455,7 @@ object SnodeAPI { val hashes = messageHashes.takeIf { it.size != 1 } ?: (messageHashes + "///////////////////////////////////////////") // TODO remove this when bug is fixed on nodes. return retryIfNeeded(maxRetryCount) { val timestamp = System.currentTimeMillis() + clockOffset - val signData = "${Snode.Method.GetExpiries.rawValue}$timestamp${hashes.joinToString(separator = "")}".toByteArray() + val signData = sequenceOf(Snode.Method.GetExpiries.rawValue).plus(timestamp.toString()).plus(hashes).toByteArray() val ed25519PublicKey = userEd25519KeyPair.publicKey.asHexString val signature = try { @@ -499,7 +499,7 @@ object SnodeAPI { val shortenOrExtend = if (extend) "extend" else if (shorten) "shorten" else "" - val signData = "${Snode.Method.Expire.rawValue}$shortenOrExtend$newExpiry${messageHashes.joinToString(separator = "")}".toByteArray() + val signData = sequenceOf(Snode.Method.Expire.rawValue).plus(shortenOrExtend).plus(newExpiry.toString()).plus(messageHashes).toByteArray() val signature = try { signAndEncode(signData, userEd25519KeyPair) @@ -633,7 +633,7 @@ object SnodeAPI { getSingleTargetSnode(userPublicKey).bind { snode -> retryIfNeeded(maxRetryCount) { getNetworkTime(snode).bind { (_, timestamp) -> - val verificationData = (Snode.Method.DeleteAll.rawValue + Namespace.ALL + timestamp.toString()).toByteArray() + val verificationData = sequenceOf(Snode.Method.DeleteAll.rawValue, Namespace.ALL, timestamp.toString()).toByteArray() val deleteMessageParams = buildMap { this["pubkey"] = userPublicKey this["pubkey_ed25519"] = userED25519KeyPair.publicKey.asHexString @@ -722,9 +722,8 @@ object SnodeAPI { if (newFailureCount >= snodeFailureThreshold) { Log.d("Loki", "Failure threshold reached for: $snode; dropping it.") publicKey?.let { dropSnodeFromSwarmIfNeeded(snode, it) } - snodePool -= snode - Log.d("Loki", "Snode pool count: ${snodePool.count()}.") - snodeFailureCount.remove(snode) + snodePool = (snodePool - snode).also { Log.d("Loki", "Snode pool count: ${it.count()}.") } + snodeFailureCount -= snode } } when (statusCode) { diff --git a/libsignal/src/main/java/org/session/libsignal/utilities/Snode.kt b/libsignal/src/main/java/org/session/libsignal/utilities/Snode.kt index cc123a8527..f918dbbf73 100644 --- a/libsignal/src/main/java/org/session/libsignal/utilities/Snode.kt +++ b/libsignal/src/main/java/org/session/libsignal/utilities/Snode.kt @@ -52,6 +52,8 @@ class Snode(val address: String, val port: Int, val publicKeySet: KeySet?, val v fun Version(value: String) = CACHE.getOrElse(value) { Snode.Version(value) } + + fun Version(parts: List) = Version(parts.joinToString(".")) } @JvmInline @@ -66,14 +68,12 @@ class Snode(val address: String, val port: Int, val publicKeySet: KeySet?, val v } } - constructor(parts: List): this( + internal constructor(parts: List): this( parts.asSequence() .map { it.toByte().toULong() } .foldToVersionAsULong() ) - constructor(value: Int): this(value.toULong()) - internal constructor(value: String): this( value.splitToSequence(".") .map { it.toULongOrNull() ?: 0UL } From 6e1ed8cc117eb5706c8c4d97a33f1926af33777a Mon Sep 17 00:00:00 2001 From: bemusementpark Date: Sat, 3 Aug 2024 21:41:45 +0930 Subject: [PATCH 05/12] Add SnodeTest --- .../session/libsignal/utilities/SnodeTest.kt | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 app/src/test/java/org/session/libsignal/utilities/SnodeTest.kt diff --git a/app/src/test/java/org/session/libsignal/utilities/SnodeTest.kt b/app/src/test/java/org/session/libsignal/utilities/SnodeTest.kt new file mode 100644 index 0000000000..d778db6519 --- /dev/null +++ b/app/src/test/java/org/session/libsignal/utilities/SnodeTest.kt @@ -0,0 +1,53 @@ +package org.session.libsignal.utilities + +import org.hamcrest.MatcherAssert.assertThat +import org.hamcrest.core.IsEqual.equalTo +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.Parameterized + +@RunWith(Parameterized::class) +class SnodeVersionTest( + private val v1: String, + private val v2: String, + private val expectedEqual: Boolean, + private val expectedLessThan: Boolean +) { + companion object { + @JvmStatic + @Parameterized.Parameters(name = "{index}: testVersion({0},{1}) = (equalTo: {2}, lessThan: {3})") + fun data(): Collection> = listOf( + arrayOf("1", "1", true, false), + arrayOf("1", "2", false, true), + arrayOf("2", "1", false, false), + arrayOf("1.0", "1", true, false), + arrayOf("1.0", "1.0.0", true, false), + arrayOf("1.0", "1.0.0.0", true, false), + arrayOf("1.0", "1.0.0.0.0.0", true, false), + arrayOf("2.0", "1.2", false, false), + arrayOf("1.0.0.0", "1.0.0.1", false, true), + // Snode.Version only considers the first 4 integers, so these are equal + arrayOf("1.0.0.0", "1.0.0.0.1", true, false), + arrayOf("1.0.0.1", "1.0.0.1", true, false), + arrayOf("12345.12345.12345.12345", "12345.12345.12345.12345", true, false), + arrayOf("11111.11111.11111.11111", "11111.11111.11111.99999", false, true), + arrayOf("11111.11111.11111.11111", "11111.11111.99999.99999", false, true), + arrayOf("11111.11111.11111.11111", "11111.99999.99999.99999", false, true), + arrayOf("11111.11111.11111.11111", "99999.99999.99999.99999", false, true), + ) + } + + @Test + fun testVersionEqual() { + val version1 = Snode.Version(v1) + val version2 = Snode.Version(v2) + assertThat(version1 == version2, equalTo(expectedEqual)) + } + + @Test + fun testVersionOnePartLessThan() { + val version1 = Snode.Version(v1) + val version2 = Snode.Version(v2) + assertThat(version1 < version2, equalTo(expectedLessThan)) + } +} \ No newline at end of file From 541766099680dbf80c60767813bf8fc2accf9dde Mon Sep 17 00:00:00 2001 From: bemusementpark Date: Sat, 3 Aug 2024 21:52:01 +0930 Subject: [PATCH 06/12] Improve Version to cap parts at 16-bits rather than masking them --- .../java/org/session/libsignal/utilities/SnodeTest.kt | 9 ++++----- .../main/java/org/session/libsignal/utilities/Snode.kt | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/app/src/test/java/org/session/libsignal/utilities/SnodeTest.kt b/app/src/test/java/org/session/libsignal/utilities/SnodeTest.kt index d778db6519..aed54fd0d3 100644 --- a/app/src/test/java/org/session/libsignal/utilities/SnodeTest.kt +++ b/app/src/test/java/org/session/libsignal/utilities/SnodeTest.kt @@ -29,11 +29,10 @@ class SnodeVersionTest( // Snode.Version only considers the first 4 integers, so these are equal arrayOf("1.0.0.0", "1.0.0.0.1", true, false), arrayOf("1.0.0.1", "1.0.0.1", true, false), - arrayOf("12345.12345.12345.12345", "12345.12345.12345.12345", true, false), - arrayOf("11111.11111.11111.11111", "11111.11111.11111.99999", false, true), - arrayOf("11111.11111.11111.11111", "11111.11111.99999.99999", false, true), - arrayOf("11111.11111.11111.11111", "11111.99999.99999.99999", false, true), - arrayOf("11111.11111.11111.11111", "99999.99999.99999.99999", false, true), + // parts can be up to 16 bits, around 65,535 + arrayOf("65535.65535.65535.65535", "65535.65535.65535.65535", true, false), + // values higher than this are coerced to 65535 (: + arrayOf("65535.65535.65535.65535", "65535.65535.65535.99999", true, false), ) } diff --git a/libsignal/src/main/java/org/session/libsignal/utilities/Snode.kt b/libsignal/src/main/java/org/session/libsignal/utilities/Snode.kt index f918dbbf73..86f3e26b64 100644 --- a/libsignal/src/main/java/org/session/libsignal/utilities/Snode.kt +++ b/libsignal/src/main/java/org/session/libsignal/utilities/Snode.kt @@ -64,7 +64,7 @@ class Snode(val address: String, val port: Int, val publicKeySet: KeySet?, val v private const val MASK = 0xFFFFUL private fun Sequence.foldToVersionAsULong() = take(4).foldIndexed(0UL) { i, acc, it -> - it and MASK shl (3 - i) * MASK_BITS or acc + it.coerceAtMost(MASK) shl (3 - i) * MASK_BITS or acc } } From 2125502e771e8ff35948e6f0e63a5cf402a3712c Mon Sep 17 00:00:00 2001 From: bemusementpark Date: Sat, 3 Aug 2024 22:08:17 +0930 Subject: [PATCH 07/12] Refactor a few MessagingModuleConfiguration function calls --- .../org/session/libsession/snode/SnodeAPI.kt | 34 ++++++++----------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/libsession/src/main/java/org/session/libsession/snode/SnodeAPI.kt b/libsession/src/main/java/org/session/libsession/snode/SnodeAPI.kt index 4d59669d1c..4a2a9f2134 100644 --- a/libsession/src/main/java/org/session/libsession/snode/SnodeAPI.kt +++ b/libsession/src/main/java/org/session/libsession/snode/SnodeAPI.kt @@ -287,6 +287,10 @@ object SnodeAPI { sodium.cryptoSignDetached(it, data, data.size.toLong(), userED25519KeyPair.secretKey.asBytes) } + private fun getUserED25519KeyPairCatchingOrNull() = runCatching { MessagingModuleConfiguration.shared.getUserED25519KeyPair() }.getOrNull() + private fun getUserED25519KeyPair(): KeyPair? = MessagingModuleConfiguration.shared.getUserED25519KeyPair() + private fun getUserPublicKey() = MessagingModuleConfiguration.shared.storage.getUserPublicKey() + fun getRawMessages(snode: Snode, publicKey: String, requiresAuth: Boolean = true, namespace: Int = 0): RawResponsePromise { // Get last message hash val lastHashValue = database.getLastMessageHashValue(snode, publicKey, namespace) ?: "" @@ -300,8 +304,7 @@ object SnodeAPI { // Construct signature if (requiresAuth) { val userED25519KeyPair = try { - MessagingModuleConfiguration.shared.getUserED25519KeyPair() - ?: return Promise.ofFail(Error.NoKeyPair) + getUserED25519KeyPair() ?: return Promise.ofFail(Error.NoKeyPair) } catch (e: Exception) { Log.e("Loki", "Error getting KeyPair", e) return Promise.ofFail(Error.NoKeyPair) @@ -327,7 +330,7 @@ object SnodeAPI { // used for sig generation since it is also the value used in timestamp parameter val messageTimestamp = message.timestamp - val userED25519KeyPair = runCatching { MessagingModuleConfiguration.shared.getUserED25519KeyPair() }.getOrNull() ?: return null + val userED25519KeyPair = getUserED25519KeyPairCatchingOrNull() ?: return null val verificationData = "store$namespace$messageTimestamp".toByteArray() val signature = signAndEncodeCatching(verificationData, userED25519KeyPair).run { @@ -362,11 +365,7 @@ object SnodeAPI { * @param required indicates that *at least one* message in the list is deleted from the server, otherwise it will return 404 */ fun buildAuthenticatedDeleteBatchInfo(publicKey: String, messageHashes: List, required: Boolean = false): SnodeBatchRequestInfo? { - val userEd25519KeyPair = try { - MessagingModuleConfiguration.shared.getUserED25519KeyPair() ?: return null - } catch (e: Exception) { - return null - } + val userEd25519KeyPair = getUserED25519KeyPairCatchingOrNull() ?: return null val ed25519PublicKey = userEd25519KeyPair.publicKey.asHexString val verificationData = sequenceOf("delete").plus(messageHashes).toByteArray() val signature = try { @@ -391,7 +390,7 @@ object SnodeAPI { fun buildAuthenticatedRetrieveBatchRequest(snode: Snode, publicKey: String, namespace: Int = 0, maxSize: Int? = null): SnodeBatchRequestInfo? { val lastHashValue = database.getLastMessageHashValue(snode, publicKey, namespace) ?: "" - val userEd25519KeyPair = runCatching { MessagingModuleConfiguration.shared.getUserED25519KeyPair() }.getOrNull() ?: return null + val userEd25519KeyPair = getUserED25519KeyPairCatchingOrNull() ?: return null val ed25519PublicKey = userEd25519KeyPair.publicKey.asHexString val timestamp = System.currentTimeMillis() + clockOffset val verificationData = if (namespace == 0) "retrieve$timestamp".toByteArray() @@ -451,7 +450,7 @@ object SnodeAPI { } fun getExpiries(messageHashes: List, publicKey: String) : RawResponsePromise { - val userEd25519KeyPair = MessagingModuleConfiguration.shared.getUserED25519KeyPair() ?: return Promise.ofFail(NullPointerException("No user key pair")) + val userEd25519KeyPair = getUserED25519KeyPairCatchingOrNull() ?: return Promise.ofFail(NullPointerException("No user key pair")) val hashes = messageHashes.takeIf { it.size != 1 } ?: (messageHashes + "///////////////////////////////////////////") // TODO remove this when bug is fixed on nodes. return retryIfNeeded(maxRetryCount) { val timestamp = System.currentTimeMillis() + clockOffset @@ -495,7 +494,7 @@ object SnodeAPI { extend: Boolean = false, shorten: Boolean = false ): Map? { - val userEd25519KeyPair = MessagingModuleConfiguration.shared.getUserED25519KeyPair() ?: return null + val userEd25519KeyPair = getUserED25519KeyPairCatchingOrNull() ?: return null val shortenOrExtend = if (extend) "extend" else if (shorten) "shorten" else "" @@ -535,8 +534,7 @@ object SnodeAPI { fun sendMessage(message: SnodeMessage, requiresAuth: Boolean = false, namespace: Int = 0): RawResponsePromise = retryIfNeeded(maxRetryCount) { - val module = MessagingModuleConfiguration.shared - val userED25519KeyPair = module.getUserED25519KeyPair() ?: return@retryIfNeeded Promise.ofFail(Error.NoKeyPair) + val userED25519KeyPair = getUserED25519KeyPair() ?: return@retryIfNeeded Promise.ofFail(Error.NoKeyPair) val parameters = message.toJSON().toMutableMap() // Construct signature if (requiresAuth) { @@ -566,9 +564,8 @@ object SnodeAPI { fun deleteMessage(publicKey: String, serverHashes: List): Promise, Exception> = retryIfNeeded(maxRetryCount) { - val module = MessagingModuleConfiguration.shared - val userED25519KeyPair = module.getUserED25519KeyPair() ?: return@retryIfNeeded Promise.ofFail(Error.NoKeyPair) - val userPublicKey = module.storage.getUserPublicKey() ?: return@retryIfNeeded Promise.ofFail(Error.NoKeyPair) + val userED25519KeyPair = getUserED25519KeyPair() ?: return@retryIfNeeded Promise.ofFail(Error.NoKeyPair) + val userPublicKey = getUserPublicKey() ?: return@retryIfNeeded Promise.ofFail(Error.NoKeyPair) getSingleTargetSnode(publicKey).bind { snode -> retryIfNeeded(maxRetryCount) { val verificationData = sequenceOf(Snode.Method.DeleteMessage.rawValue).plus(serverHashes).toByteArray() @@ -627,9 +624,8 @@ object SnodeAPI { fun deleteAllMessages(): Promise, Exception> = retryIfNeeded(maxRetryCount) { - val module = MessagingModuleConfiguration.shared - val userED25519KeyPair = module.getUserED25519KeyPair() ?: return@retryIfNeeded Promise.ofFail(Error.NoKeyPair) - val userPublicKey = module.storage.getUserPublicKey() ?: return@retryIfNeeded Promise.ofFail(Error.NoKeyPair) + val userED25519KeyPair = getUserED25519KeyPair() ?: return@retryIfNeeded Promise.ofFail(Error.NoKeyPair) + val userPublicKey = getUserPublicKey() ?: return@retryIfNeeded Promise.ofFail(Error.NoKeyPair) getSingleTargetSnode(userPublicKey).bind { snode -> retryIfNeeded(maxRetryCount) { getNetworkTime(snode).bind { (_, timestamp) -> From 61cb602e63be3ee7093c035c91adfa0ea03f184b Mon Sep 17 00:00:00 2001 From: bemusementpark Date: Sat, 3 Aug 2024 22:49:23 +0930 Subject: [PATCH 08/12] Simplify and document some functions --- .../org/session/libsession/snode/SnodeAPI.kt | 5 +-- .../org/session/libsignal/utilities/Snode.kt | 35 ++++++------------- 2 files changed, 12 insertions(+), 28 deletions(-) diff --git a/libsession/src/main/java/org/session/libsession/snode/SnodeAPI.kt b/libsession/src/main/java/org/session/libsession/snode/SnodeAPI.kt index 4a2a9f2134..9c584ba0fb 100644 --- a/libsession/src/main/java/org/session/libsession/snode/SnodeAPI.kt +++ b/libsession/src/main/java/org/session/libsession/snode/SnodeAPI.kt @@ -736,10 +736,7 @@ object SnodeAPI { else json?.let(::parseSnodes) ?.takeIf { it.isNotEmpty() } ?.let { database.setSwarm(publicKey, it.toSet()) } - ?: run { - Log.d("Loki", "Invalidating swarm for: $publicKey.") - dropSnodeFromSwarmIfNeeded(snode, publicKey) - } + ?: dropSnodeFromSwarmIfNeeded(snode, publicKey).also { Log.d("Loki", "Invalidating swarm for: $publicKey.") } } 404 -> { Log.d("Loki", "404, probably no file found") diff --git a/libsignal/src/main/java/org/session/libsignal/utilities/Snode.kt b/libsignal/src/main/java/org/session/libsignal/utilities/Snode.kt index 86f3e26b64..58dcbe8a37 100644 --- a/libsignal/src/main/java/org/session/libsignal/utilities/Snode.kt +++ b/libsignal/src/main/java/org/session/libsignal/utilities/Snode.kt @@ -2,6 +2,9 @@ package org.session.libsignal.utilities import android.annotation.SuppressLint +/** + * Create a Snode from a "-" delimited String if valid, null otherwise. + */ fun Snode(string: String): Snode? { val components = string.split("-") val address = components[0] @@ -31,25 +34,16 @@ class Snode(val address: String, val port: Int, val publicKeySet: KeySet?, val v data class KeySet(val ed25519Key: String, val x25519Key: String) - override fun equals(other: Any?): Boolean { - return if (other is Snode) { - address == other.address && port == other.port - } else { - false - } - } - - override fun hashCode(): Int { - return address.hashCode() xor port.hashCode() - } - - override fun toString(): String { return "$address:$port" } + override fun equals(other: Any?) = other is Snode && address == other.address && port == other.port + override fun hashCode(): Int = address.hashCode() xor port.hashCode() + override fun toString(): String = "$address:$port" companion object { private val CACHE = mutableMapOf() @SuppressLint("NotConstructor") fun Version(value: String) = CACHE.getOrElse(value) { + // internal constructor takes precedence Snode.Version(value) } @@ -62,22 +56,15 @@ class Snode(val address: String, val port: Int, val publicKeySet: KeySet?, val v val ZERO = Version(0UL) private const val MASK_BITS = 16 private const val MASK = 0xFFFFUL - - private fun Sequence.foldToVersionAsULong() = take(4).foldIndexed(0UL) { i, acc, it -> - it.coerceAtMost(MASK) shl (3 - i) * MASK_BITS or acc - } } - internal constructor(parts: List): this( - parts.asSequence() - .map { it.toByte().toULong() } - .foldToVersionAsULong() - ) - internal constructor(value: String): this( value.splitToSequence(".") + .take(4) .map { it.toULongOrNull() ?: 0UL } - .foldToVersionAsULong() + .foldIndexed(0UL) { i, acc, it -> + it.coerceAtMost(MASK) shl (3 - i) * MASK_BITS or acc + } ) operator fun compareTo(other: Version): Int = value.compareTo(other.value) From fa0abef243e5301c81ab564ff7076ffeeac18e5d Mon Sep 17 00:00:00 2001 From: bemusementpark Date: Mon, 5 Aug 2024 09:59:02 +0930 Subject: [PATCH 09/12] Fix Snode Version CACHE usage --- .../src/main/java/org/session/libsignal/utilities/Snode.kt | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/libsignal/src/main/java/org/session/libsignal/utilities/Snode.kt b/libsignal/src/main/java/org/session/libsignal/utilities/Snode.kt index 58dcbe8a37..288c3fcf1d 100644 --- a/libsignal/src/main/java/org/session/libsignal/utilities/Snode.kt +++ b/libsignal/src/main/java/org/session/libsignal/utilities/Snode.kt @@ -42,10 +42,7 @@ class Snode(val address: String, val port: Int, val publicKeySet: KeySet?, val v private val CACHE = mutableMapOf() @SuppressLint("NotConstructor") - fun Version(value: String) = CACHE.getOrElse(value) { - // internal constructor takes precedence - Snode.Version(value) - } + fun Version(value: String) = CACHE[value] ?: Snode.Version(value).also { CACHE[value] = it } fun Version(parts: List) = Version(parts.joinToString(".")) } From f9ace6a9b92c8df4b0a7346f297c860327b31230 Mon Sep 17 00:00:00 2001 From: bemusementpark Date: Mon, 5 Aug 2024 10:26:52 +0930 Subject: [PATCH 10/12] Add LruCache and thread-safety --- .../src/main/java/org/session/libsignal/utilities/Snode.kt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/libsignal/src/main/java/org/session/libsignal/utilities/Snode.kt b/libsignal/src/main/java/org/session/libsignal/utilities/Snode.kt index 288c3fcf1d..675fe55f85 100644 --- a/libsignal/src/main/java/org/session/libsignal/utilities/Snode.kt +++ b/libsignal/src/main/java/org/session/libsignal/utilities/Snode.kt @@ -1,6 +1,7 @@ package org.session.libsignal.utilities import android.annotation.SuppressLint +import android.util.LruCache /** * Create a Snode from a "-" delimited String if valid, null otherwise. @@ -39,10 +40,11 @@ class Snode(val address: String, val port: Int, val publicKeySet: KeySet?, val v override fun toString(): String = "$address:$port" companion object { - private val CACHE = mutableMapOf() + private val CACHE = LruCache(100) @SuppressLint("NotConstructor") - fun Version(value: String) = CACHE[value] ?: Snode.Version(value).also { CACHE[value] = it } + @Synchronized + fun Version(value: String) = CACHE[value] ?: Snode.Version(value).also { CACHE.put(value, it) } fun Version(parts: List) = Version(parts.joinToString(".")) } From 2960eddd8532d8c2387d7e3ef29f3624b5dca3b7 Mon Sep 17 00:00:00 2001 From: bemusementpark Date: Mon, 5 Aug 2024 13:46:55 +0930 Subject: [PATCH 11/12] Fix removeDuplicates --- .../sending_receiving/pollers/Poller.kt | 3 +- .../org/session/libsession/snode/SnodeAPI.kt | 36 ++++++++++--------- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/libsession/src/main/java/org/session/libsession/messaging/sending_receiving/pollers/Poller.kt b/libsession/src/main/java/org/session/libsession/messaging/sending_receiving/pollers/Poller.kt index 2b7c8159ea..f0caa9d314 100644 --- a/libsession/src/main/java/org/session/libsession/messaging/sending_receiving/pollers/Poller.kt +++ b/libsession/src/main/java/org/session/libsession/messaging/sending_receiving/pollers/Poller.kt @@ -142,8 +142,7 @@ class Poller(private val configFactory: ConfigFactoryProtocol, debounceTimer: Ti val messages = rawMessages["messages"] as? List<*> val processed = if (!messages.isNullOrEmpty()) { SnodeAPI.updateLastMessageHashValueIfPossible(snode, userPublicKey, messages, namespace) - SnodeAPI.removeDuplicates(userPublicKey, messages, namespace, true).mapNotNull { messageBody -> - val rawMessageAsJSON = messageBody as? Map<*, *> ?: return@mapNotNull null + SnodeAPI.removeDuplicates(userPublicKey, messages, namespace, true).mapNotNull { rawMessageAsJSON -> val hashValue = rawMessageAsJSON["hash"] as? String ?: return@mapNotNull null val b64EncodedBody = rawMessageAsJSON["data"] as? String ?: return@mapNotNull null val timestamp = rawMessageAsJSON["t"] as? Long ?: SnodeAPI.nowWithOffset diff --git a/libsession/src/main/java/org/session/libsession/snode/SnodeAPI.kt b/libsession/src/main/java/org/session/libsession/snode/SnodeAPI.kt index 9c584ba0fb..72b144be03 100644 --- a/libsession/src/main/java/org/session/libsession/snode/SnodeAPI.kt +++ b/libsession/src/main/java/org/session/libsession/snode/SnodeAPI.kt @@ -660,29 +660,33 @@ object SnodeAPI { } } - fun removeDuplicates(publicKey: String, rawMessages: List<*>, namespace: Int, updateStoredHashes: Boolean): List<*> { - val originalMessageHashValues = database.getReceivedMessageHashValues(publicKey, namespace) ?: emptySet() - val receivedMessageHashValues = originalMessageHashValues.toMutableSet() - return rawMessages.filter { rawMessage -> - (rawMessage as? Map<*, *>) - ?.let { it["hash"] as? String } - ?.let { receivedMessageHashValues.add(it) } - ?: false.also { Log.d("Loki", "Missing hash value for message: ${rawMessage?.prettifiedDescription()}.") } + /** + * + * + * TODO Use a db transaction, synchronizing is sufficient for now because + * database#setReceivedMessageHashValues is only called here. + */ + @Synchronized + fun removeDuplicates(publicKey: String, rawMessages: List<*>, namespace: Int, updateStoredHashes: Boolean): List> { + val hashValues = database.getReceivedMessageHashValues(publicKey, namespace)?.toMutableSet() ?: mutableSetOf() + return rawMessages.filterIsInstance>().filter { rawMessage -> + val hash = rawMessage["hash"] as? String + hash ?: Log.d("Loki", "Missing hash value for message: ${rawMessage.prettifiedDescription()}.") + hash?.let(hashValues::add) == true }.also { - if (updateStoredHashes && originalMessageHashValues.containsAll(receivedMessageHashValues)) { - database.setReceivedMessageHashValues(publicKey, receivedMessageHashValues, namespace) + if (updateStoredHashes && it.isNotEmpty()) { + database.setReceivedMessageHashValues(publicKey, hashValues, namespace) } } } - private fun parseEnvelopes(rawMessages: List<*>): List> = rawMessages.mapNotNull { rawMessage -> - val rawMessageAsJSON = rawMessage as? Map<*, *> - val base64EncodedData = rawMessageAsJSON?.get("data") as? String - val data = base64EncodedData?.let { Base64.decode(it) } + private fun parseEnvelopes(rawMessages: List>): List> = rawMessages.mapNotNull { rawMessage -> + val base64EncodedData = rawMessage["data"] as? String + val data = base64EncodedData?.let(Base64::decode) - data ?: Log.d("Loki", "Failed to decode data for message: ${rawMessage?.prettifiedDescription()}.") + data ?: Log.d("Loki", "Failed to decode data for message: ${rawMessage.prettifiedDescription()}.") - data?.runCatching { MessageWrapper.unwrap(this) to rawMessageAsJSON["hash"] as? String } + data?.runCatching { MessageWrapper.unwrap(this) to rawMessage["hash"] as? String } ?.onFailure { Log.d("Loki", "Failed to unwrap data for message: ${rawMessage.prettifiedDescription()}.") } ?.getOrNull() } From 8a9faa182d43322fa418e555461cc355b43280d7 Mon Sep 17 00:00:00 2001 From: bemusementpark Date: Mon, 5 Aug 2024 16:26:20 +0930 Subject: [PATCH 12/12] Fix SnodeAPI error thrown outside of Promise --- .../src/main/java/org/session/libsession/snode/SnodeAPI.kt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/libsession/src/main/java/org/session/libsession/snode/SnodeAPI.kt b/libsession/src/main/java/org/session/libsession/snode/SnodeAPI.kt index 2f6cf141d1..099b2541ab 100644 --- a/libsession/src/main/java/org/session/libsession/snode/SnodeAPI.kt +++ b/libsession/src/main/java/org/session/libsession/snode/SnodeAPI.kt @@ -15,6 +15,7 @@ import nl.komponents.kovenant.all import nl.komponents.kovenant.functional.bind import nl.komponents.kovenant.functional.map import nl.komponents.kovenant.task +import nl.komponents.kovenant.unwrap import org.session.libsession.messaging.MessagingModuleConfiguration import org.session.libsession.messaging.utilities.MessageWrapper import org.session.libsession.messaging.utilities.SodiumUtilities.sodium @@ -202,7 +203,7 @@ object SnodeAPI { } // Public API - fun getAccountID(onsName: String): Promise { + fun getAccountID(onsName: String): Promise = task { val validationCount = 3 val accountIDByteCount = 33 // Hash the ONS name using BLAKE2b @@ -228,7 +229,7 @@ object SnodeAPI { } } } - return all(promises).map { results -> + all(promises).map { results -> results.map { json -> val intermediate = json["result"] as? Map<*, *> ?: throw Error.Generic val hexEncodedCiphertext = intermediate["encrypted_value"] as? String ?: throw Error.Generic @@ -264,7 +265,7 @@ object SnodeAPI { }.takeIf { it.size == validationCount && it.toSet().size == 1 }?.first() ?: throw Error.ValidationFailed } - } + }.unwrap() fun getSwarm(publicKey: String): Promise, Exception> = database.getSwarm(publicKey)?.takeIf { it.size >= minimumSwarmSnodeCount }?.let(Promise.Companion::of)