From e8bac5005ee02caec8100cbf717167e572f52167 Mon Sep 17 00:00:00 2001 From: jubb Date: Wed, 5 May 2021 17:29:27 +1000 Subject: [PATCH] feat: file server v2 and syncing open groups v2 in config messages --- .../securesms/database/Storage.kt | 20 +++- .../libsession/messaging/StorageProtocol.kt | 2 +- .../messaging/file_server/FileServerAPIV2.kt | 108 ++++++++++++++++++ .../messaging/jobs/AttachmentDownloadJob.kt | 7 +- .../messages/control/ConfigurationMessage.kt | 7 +- .../messaging/open_groups/OpenGroupAPIV2.kt | 19 +-- .../messaging/open_groups/OpenGroupV2.kt | 2 + .../sending_receiving/MessageSender.kt | 2 +- .../ReceivedMessageHandler.kt | 6 +- .../pollers/OpenGroupV2Poller.kt | 1 + .../messaging/utilities/DotNetAPI.kt | 2 +- .../libsession/utilities/DownloadUtilities.kt | 100 +++++++++------- 12 files changed, 211 insertions(+), 65 deletions(-) create mode 100644 libsession/src/main/java/org/session/libsession/messaging/file_server/FileServerAPIV2.kt diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/Storage.kt b/app/src/main/java/org/thoughtcrime/securesms/database/Storage.kt index 1c63437a73..da8db56761 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/database/Storage.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/database/Storage.kt @@ -2,6 +2,7 @@ package org.thoughtcrime.securesms.database import android.content.Context import android.net.Uri +import okhttp3.HttpUrl import org.session.libsession.messaging.StorageProtocol import org.session.libsession.messaging.jobs.AttachmentUploadJob import org.session.libsession.messaging.jobs.Job @@ -543,8 +544,23 @@ class Storage(context: Context, helper: SQLCipherOpenHelper) : Database(context, return DatabaseFactory.getLokiThreadDatabase(context).getAllV2OpenGroups() } - override fun addOpenGroup(server: String, channel: Long) { - OpenGroupUtilities.addGroup(context, server, channel) + override fun addOpenGroup(serverUrl: String, channel: Long) { + val httpUrl = HttpUrl.parse(serverUrl) ?: return + if (httpUrl.queryParameterNames().contains("public_key")) { + // open group v2 + val server = HttpUrl.Builder().scheme(httpUrl.scheme()).host(httpUrl.host()).apply { + if (httpUrl.port() != 80 || httpUrl.port() != 443) { + // non-standard port, add to server + this.port(httpUrl.port()) + } + }.build() + val room = httpUrl.pathSegments().firstOrNull() ?: return + val publicKey = httpUrl.queryParameter("public_key") ?: return + + OpenGroupUtilities.addGroup(context, server.toString().removeSuffix("/"), room, publicKey) + } else { + OpenGroupUtilities.addGroup(context, serverUrl, channel) + } } override fun getAllGroups(): List { diff --git a/libsession/src/main/java/org/session/libsession/messaging/StorageProtocol.kt b/libsession/src/main/java/org/session/libsession/messaging/StorageProtocol.kt index 906153d2c6..c443b2f551 100644 --- a/libsession/src/main/java/org/session/libsession/messaging/StorageProtocol.kt +++ b/libsession/src/main/java/org/session/libsession/messaging/StorageProtocol.kt @@ -63,7 +63,7 @@ interface StorageProtocol { // Open Groups fun getThreadID(openGroupID: String): String? - fun addOpenGroup(server: String, channel: Long) + fun addOpenGroup(serverUrl: String, channel: Long) fun setOpenGroupServerMessageID(messageID: Long, serverID: Long, threadID: Long, isSms: Boolean) fun getQuoteServerID(quoteID: Long, publicKey: String): Long? diff --git a/libsession/src/main/java/org/session/libsession/messaging/file_server/FileServerAPIV2.kt b/libsession/src/main/java/org/session/libsession/messaging/file_server/FileServerAPIV2.kt new file mode 100644 index 0000000000..5e3fa6ff7a --- /dev/null +++ b/libsession/src/main/java/org/session/libsession/messaging/file_server/FileServerAPIV2.kt @@ -0,0 +1,108 @@ +package org.session.libsession.messaging.file_server + +import nl.komponents.kovenant.Promise +import nl.komponents.kovenant.functional.bind +import nl.komponents.kovenant.functional.map +import okhttp3.Headers +import okhttp3.HttpUrl +import okhttp3.MediaType +import okhttp3.RequestBody +import org.session.libsession.messaging.MessagingModuleConfiguration +import org.session.libsession.messaging.open_groups.OpenGroupAPIV2 +import org.session.libsession.snode.OnionRequestAPI +import org.session.libsignal.service.loki.HTTP +import org.session.libsignal.service.loki.utilities.retryIfNeeded +import org.session.libsignal.utilities.Base64 +import org.session.libsignal.utilities.JsonUtil +import org.session.libsignal.utilities.logging.Log + +object FileServerAPIV2 { + + const val DEFAULT_SERVER = "http://88.99.175.227" + private const val DEFAULT_SERVER_PUBLIC_KEY = "7cb31905b55cd5580c686911debf672577b3fb0bff81df4ce2d5c4cb3a7aaa69" + + sealed class Error : Exception() { + object PARSING_FAILED : Error() + object INVALID_URL : Error() + + fun errorDescription() = when (this) { + PARSING_FAILED -> "Invalid response." + INVALID_URL -> "Invalid URL." + } + + } + + data class Request( + val verb: HTTP.Verb, + val endpoint: String, + val queryParameters: Map = mapOf(), + val parameters: Any? = null, + val headers: Map = mapOf(), + // Always `true` under normal circumstances. You might want to disable + // this when running over Lokinet. + val useOnionRouting: Boolean = true + ) + + private fun createBody(parameters: Any?): RequestBody? { + if (parameters == null) return null + + val parametersAsJSON = JsonUtil.toJson(parameters) + return RequestBody.create(MediaType.get("application/json"), parametersAsJSON) + } + + private fun send(request: Request): Promise, Exception> { + val parsed = HttpUrl.parse(DEFAULT_SERVER) ?: return Promise.ofFail(OpenGroupAPIV2.Error.INVALID_URL) + val urlBuilder = HttpUrl.Builder() + .scheme(parsed.scheme()) + .host(parsed.host()) + .port(parsed.port()) + .addPathSegments(request.endpoint) + + if (request.verb == HTTP.Verb.GET) { + for ((key, value) in request.queryParameters) { + urlBuilder.addQueryParameter(key, value) + } + } + + val requestBuilder = okhttp3.Request.Builder() + .url(urlBuilder.build()) + .headers(Headers.of(request.headers)) + when (request.verb) { + HTTP.Verb.GET -> requestBuilder.get() + HTTP.Verb.PUT -> requestBuilder.put(createBody(request.parameters)!!) + HTTP.Verb.POST -> requestBuilder.post(createBody(request.parameters)!!) + HTTP.Verb.DELETE -> requestBuilder.delete(createBody(request.parameters)) + } + + if (request.useOnionRouting) { + val publicKey = MessagingModuleConfiguration.shared.storage.getOpenGroupPublicKey(DEFAULT_SERVER) + ?: return Promise.ofFail(OpenGroupAPIV2.Error.NO_PUBLIC_KEY) + return OnionRequestAPI.sendOnionRequest(requestBuilder.build(), DEFAULT_SERVER, publicKey) + .fail { e -> + Log.e("Loki", "FileServerV2 failed with error",e) + } + } else { + return Promise.ofFail(IllegalStateException("It's currently not allowed to send non onion routed requests.")) + } + + } + + // region Sending + fun upload(file: ByteArray): Promise { + val base64EncodedFile = Base64.encodeBytes(file) + val parameters = mapOf("file" to base64EncodedFile) + val request = Request(verb = HTTP.Verb.POST, endpoint = "files", parameters = parameters) + return send(request).map { json -> + json["result"] as? Long ?: throw OpenGroupAPIV2.Error.PARSING_FAILED + } + } + + fun download(file: Long): Promise { + val request = Request(verb = HTTP.Verb.GET, endpoint = "files/$file") + return send(request).map { json -> + val base64EncodedFile = json["result"] as? String ?: throw Error.PARSING_FAILED + Base64.decode(base64EncodedFile) ?: throw Error.PARSING_FAILED + } + } + +} \ No newline at end of file diff --git a/libsession/src/main/java/org/session/libsession/messaging/jobs/AttachmentDownloadJob.kt b/libsession/src/main/java/org/session/libsession/messaging/jobs/AttachmentDownloadJob.kt index 3298f9d3dc..6855b08b02 100644 --- a/libsession/src/main/java/org/session/libsession/messaging/jobs/AttachmentDownloadJob.kt +++ b/libsession/src/main/java/org/session/libsession/messaging/jobs/AttachmentDownloadJob.kt @@ -3,9 +3,11 @@ package org.session.libsession.messaging.jobs import okhttp3.HttpUrl import org.session.libsession.messaging.MessagingModuleConfiguration import org.session.libsession.messaging.file_server.FileServerAPI +import org.session.libsession.messaging.file_server.FileServerAPIV2 import org.session.libsession.messaging.open_groups.OpenGroupAPIV2 import org.session.libsession.messaging.sending_receiving.attachments.AttachmentState import org.session.libsession.messaging.utilities.DotNetAPI +import org.session.libsession.utilities.DownloadUtilities import org.session.libsignal.service.api.crypto.AttachmentCipherInputStream import org.session.libsignal.utilities.Base64 import org.session.libsignal.utilities.logging.Log @@ -58,10 +60,7 @@ class AttachmentDownloadJob(val attachmentID: Long, val databaseMessageID: Long) val openGroupV2 = MessagingModuleConfiguration.shared.storage.getV2OpenGroup(threadId.toString()) val stream = if (openGroupV2 == null) { - FileServerAPI.shared.downloadFile(tempFile, attachment.url, null) - - - + DownloadUtilities.downloadFile(tempFile, attachment.url, FileServerAPI.maxFileSize, null) // Assume we're retrieving an attachment for an open group server if the digest is not set if (attachment.digest?.size ?: 0 == 0 || attachment.key.isNullOrEmpty()) FileInputStream(tempFile) else AttachmentCipherInputStream.createForAttachment(tempFile, attachment.size, Base64.decode(attachment.key), attachment.digest) diff --git a/libsession/src/main/java/org/session/libsession/messaging/messages/control/ConfigurationMessage.kt b/libsession/src/main/java/org/session/libsession/messaging/messages/control/ConfigurationMessage.kt index 29aa13e586..310ec0c019 100644 --- a/libsession/src/main/java/org/session/libsession/messaging/messages/control/ConfigurationMessage.kt +++ b/libsession/src/main/java/org/session/libsession/messaging/messages/control/ConfigurationMessage.kt @@ -113,8 +113,11 @@ class ConfigurationMessage(var closedGroups: List, var openGroups: } if (groupRecord.isOpenGroup) { val threadID = storage.getThreadID(groupRecord.encodedId) ?: continue - val openGroup = storage.getOpenGroup(threadID) ?: continue - openGroups.add(openGroup.server) + val openGroup = storage.getOpenGroup(threadID) + val openGroupV2 = storage.getV2OpenGroup(threadID) + + val shareUrl = openGroup?.server ?: openGroupV2?.toJoinUrl() ?: continue + openGroups.add(shareUrl) } } diff --git a/libsession/src/main/java/org/session/libsession/messaging/open_groups/OpenGroupAPIV2.kt b/libsession/src/main/java/org/session/libsession/messaging/open_groups/OpenGroupAPIV2.kt index 3dab0e9b9f..7714a66e5d 100644 --- a/libsession/src/main/java/org/session/libsession/messaging/open_groups/OpenGroupAPIV2.kt +++ b/libsession/src/main/java/org/session/libsession/messaging/open_groups/OpenGroupAPIV2.kt @@ -45,6 +45,16 @@ object OpenGroupAPIV2 { object SIGNING_FAILED : Error() object INVALID_URL : Error() object NO_PUBLIC_KEY : Error() + + fun errorDescription() = when (this) { + Error.GENERIC -> "An error occurred." + Error.PARSING_FAILED -> "Invalid response." + Error.DECRYPTION_FAILED -> "Couldn't decrypt response." + Error.SIGNING_FAILED -> "Couldn't sign message." + Error.INVALID_URL -> "Invalid URL." + Error.NO_PUBLIC_KEY -> "Couldn't find server public key." + } + } data class DefaultGroup(val id: String, @@ -493,13 +503,4 @@ object OpenGroupAPIV2 { } // endregion -} - -fun Error.errorDescription() = when (this) { - Error.GENERIC -> "An error occurred." - Error.PARSING_FAILED -> "Invalid response." - Error.DECRYPTION_FAILED -> "Couldn't decrypt response." - Error.SIGNING_FAILED -> "Couldn't sign message." - Error.INVALID_URL -> "Invalid URL." - Error.NO_PUBLIC_KEY -> "Couldn't find server public key." } \ No newline at end of file diff --git a/libsession/src/main/java/org/session/libsession/messaging/open_groups/OpenGroupV2.kt b/libsession/src/main/java/org/session/libsession/messaging/open_groups/OpenGroupV2.kt index 2f9fd01394..561989e318 100644 --- a/libsession/src/main/java/org/session/libsession/messaging/open_groups/OpenGroupV2.kt +++ b/libsession/src/main/java/org/session/libsession/messaging/open_groups/OpenGroupV2.kt @@ -39,6 +39,8 @@ data class OpenGroupV2( } + fun toJoinUrl(): String = "$server/$id?public_key=$publicKey" + fun toJson(): Map = mapOf( "room" to room, "server" to server, diff --git a/libsession/src/main/java/org/session/libsession/messaging/sending_receiving/MessageSender.kt b/libsession/src/main/java/org/session/libsession/messaging/sending_receiving/MessageSender.kt index 84afeff63b..7344478ae7 100644 --- a/libsession/src/main/java/org/session/libsession/messaging/sending_receiving/MessageSender.kt +++ b/libsession/src/main/java/org/session/libsession/messaging/sending_receiving/MessageSender.kt @@ -287,7 +287,7 @@ object MessageSender { val userPublicKey = storage.getUserPublicKey()!! val messageId = storage.getMessageIdInDatabase(message.sentTimestamp!!, message.sender?:userPublicKey) ?: return // Ignore future self-sends - storage.addReceivedMessageTimestamp(message.sentTimestamp!!) +// storage.addReceivedMessageTimestamp(message.sentTimestamp!!) // Track the open group server message ID if (message.openGroupServerMessageID != null && destination is Destination.OpenGroupV2) { val encoded = GroupUtil.getEncodedOpenGroupID("${destination.server}.${destination.room}".toByteArray()) diff --git a/libsession/src/main/java/org/session/libsession/messaging/sending_receiving/ReceivedMessageHandler.kt b/libsession/src/main/java/org/session/libsession/messaging/sending_receiving/ReceivedMessageHandler.kt index 49ead00424..2a0b13ae3e 100644 --- a/libsession/src/main/java/org/session/libsession/messaging/sending_receiving/ReceivedMessageHandler.kt +++ b/libsession/src/main/java/org/session/libsession/messaging/sending_receiving/ReceivedMessageHandler.kt @@ -1,6 +1,7 @@ package org.session.libsession.messaging.sending_receiving import android.text.TextUtils +import okhttp3.HttpUrl import org.session.libsession.messaging.MessagingModuleConfiguration import org.session.libsession.messaging.jobs.AttachmentDownloadJob import org.session.libsession.messaging.jobs.JobQueue @@ -125,10 +126,9 @@ private fun handleConfigurationMessage(message: ConfigurationMessage) { handleNewClosedGroup(message.sender!!, message.sentTimestamp!!, closeGroup.publicKey, closeGroup.name, closeGroup.encryptionKeyPair!!, closeGroup.members, closeGroup.admins, message.sentTimestamp!!) } val allOpenGroups = storage.getAllOpenGroups().map { it.value.server } - val allV2OpenGroups = storage.getAllV2OpenGroups().map { it.value.server } + val allV2OpenGroups = storage.getAllV2OpenGroups().map { it.value.toJoinUrl() } for (openGroup in message.openGroups) { - if (allOpenGroups.contains(openGroup)) continue - // TODO: add in v2 + if (allOpenGroups.contains(openGroup) || allV2OpenGroups.contains(openGroup)) continue storage.addOpenGroup(openGroup, 1) } if (message.displayName.isNotEmpty()) { diff --git a/libsession/src/main/java/org/session/libsession/messaging/sending_receiving/pollers/OpenGroupV2Poller.kt b/libsession/src/main/java/org/session/libsession/messaging/sending_receiving/pollers/OpenGroupV2Poller.kt index 311e213592..e5e9f94377 100644 --- a/libsession/src/main/java/org/session/libsession/messaging/sending_receiving/pollers/OpenGroupV2Poller.kt +++ b/libsession/src/main/java/org/session/libsession/messaging/sending_receiving/pollers/OpenGroupV2Poller.kt @@ -97,6 +97,7 @@ class OpenGroupV2Poller(private val openGroups: List, private val e builder.source = senderPublicKey builder.sourceDevice = 1 builder.content = message.toProto().toByteString() + builder.serverTimestamp = message.serverID ?: 0 builder.timestamp = message.sentTimestamp val envelope = builder.build() val job = MessageReceiveJob(envelope.toByteArray(), isBackgroundPoll, message.serverID, serverRoomId) diff --git a/libsession/src/main/java/org/session/libsession/messaging/utilities/DotNetAPI.kt b/libsession/src/main/java/org/session/libsession/messaging/utilities/DotNetAPI.kt index e93ed936e4..069929ac28 100644 --- a/libsession/src/main/java/org/session/libsession/messaging/utilities/DotNetAPI.kt +++ b/libsession/src/main/java/org/session/libsession/messaging/utilities/DotNetAPI.kt @@ -203,7 +203,7 @@ open class DotNetAPI { /** * Blocks the calling thread. */ - fun downloadFile(outputStream: OutputStream, url: String, listener: SignalServiceAttachment.ProgressListener?) { + private fun downloadFile(outputStream: OutputStream, url: String, listener: SignalServiceAttachment.ProgressListener?) { // We need to throw a PushNetworkException or NonSuccessfulResponseCodeException // because the underlying Signal logic requires these to work correctly val oldPrefixedHost = "https://" + HttpUrl.get(url).host() diff --git a/libsession/src/main/java/org/session/libsession/utilities/DownloadUtilities.kt b/libsession/src/main/java/org/session/libsession/utilities/DownloadUtilities.kt index 1c0e29f045..9ea2d8e3a5 100644 --- a/libsession/src/main/java/org/session/libsession/utilities/DownloadUtilities.kt +++ b/libsession/src/main/java/org/session/libsession/utilities/DownloadUtilities.kt @@ -3,7 +3,9 @@ package org.session.libsession.utilities import okhttp3.HttpUrl import okhttp3.Request import org.session.libsession.messaging.file_server.FileServerAPI +import org.session.libsession.messaging.file_server.FileServerAPIV2 import org.session.libsession.snode.OnionRequestAPI +import org.session.libsignal.service.api.crypto.AttachmentCipherInputStream import org.session.libsignal.utilities.logging.Log import org.session.libsignal.service.api.messages.SignalServiceAttachment import org.session.libsignal.service.api.push.exceptions.NonSuccessfulResponseCodeException @@ -39,50 +41,64 @@ object DownloadUtilities { */ @JvmStatic fun downloadFile(outputStream: OutputStream, url: String, maxSize: Int, listener: SignalServiceAttachment.ProgressListener?) { - // We need to throw a PushNetworkException or NonSuccessfulResponseCodeException - // because the underlying Signal logic requires these to work correctly - val oldPrefixedHost = "https://" + HttpUrl.get(url).host() - var newPrefixedHost = oldPrefixedHost - if (oldPrefixedHost.contains(FileServerAPI.fileStorageBucketURL)) { - newPrefixedHost = FileServerAPI.shared.server - } - // Edge case that needs to work: https://file-static.lokinet.org/i1pNmpInq3w9gF3TP8TFCa1rSo38J6UM - // → https://file.getsession.org/loki/v1/f/XLxogNXVEIWHk14NVCDeppzTujPHxu35 - val fileID = url.substringAfter(oldPrefixedHost).substringAfter("/f/") - val sanitizedURL = "$newPrefixedHost/loki/v1/f/$fileID" - val request = Request.Builder().url(sanitizedURL).get() - try { - val serverPublicKey = if (newPrefixedHost.contains(FileServerAPI.shared.server)) FileServerAPI.fileServerPublicKey - else FileServerAPI.shared.getPublicKeyForOpenGroupServer(newPrefixedHost).get() - val json = OnionRequestAPI.sendOnionRequest(request.build(), newPrefixedHost, serverPublicKey, isJSONRequired = false).get() - val result = json["result"] as? String - if (result == null) { - Log.d("Loki", "Couldn't parse attachment from: $json.") - throw PushNetworkException("Missing response body.") - } - val body = Base64.decode(result) - if (body.size > maxSize) { - Log.d("Loki", "Attachment size limit exceeded.") - throw PushNetworkException("Max response size exceeded.") - } - body.inputStream().use { input -> - val buffer = ByteArray(32768) - var count = 0 - var bytes = input.read(buffer) - while (bytes >= 0) { - outputStream.write(buffer, 0, bytes) - count += bytes - if (count > maxSize) { - Log.d("Loki", "Attachment size limit exceeded.") - throw PushNetworkException("Max response size exceeded.") - } - listener?.onAttachmentProgress(body.size.toLong(), count.toLong()) - bytes = input.read(buffer) + + if (url.contains(FileServerAPIV2.DEFAULT_SERVER)) { + val httpUrl = HttpUrl.parse(url)!! + val fileId = httpUrl.pathSegments().last() + try { + FileServerAPIV2.download(fileId.toLong()).get().let { + outputStream.write(it) } + } catch (e: Exception) { + Log.e("Loki", "Couln't download attachment due to error",e) + throw e + } + } else { + // We need to throw a PushNetworkException or NonSuccessfulResponseCodeException + // because the underlying Signal logic requires these to work correctly + val oldPrefixedHost = "https://" + HttpUrl.get(url).host() + var newPrefixedHost = oldPrefixedHost + if (oldPrefixedHost.contains(FileServerAPI.fileStorageBucketURL)) { + newPrefixedHost = FileServerAPI.shared.server + } + // Edge case that needs to work: https://file-static.lokinet.org/i1pNmpInq3w9gF3TP8TFCa1rSo38J6UM + // → https://file.getsession.org/loki/v1/f/XLxogNXVEIWHk14NVCDeppzTujPHxu35 + val fileID = url.substringAfter(oldPrefixedHost).substringAfter("/f/") + val sanitizedURL = "$newPrefixedHost/loki/v1/f/$fileID" + val request = Request.Builder().url(sanitizedURL).get() + try { + val serverPublicKey = if (newPrefixedHost.contains(FileServerAPI.shared.server)) FileServerAPI.fileServerPublicKey + else FileServerAPI.shared.getPublicKeyForOpenGroupServer(newPrefixedHost).get() + val json = OnionRequestAPI.sendOnionRequest(request.build(), newPrefixedHost, serverPublicKey, isJSONRequired = false).get() + val result = json["result"] as? String + if (result == null) { + Log.d("Loki", "Couldn't parse attachment from: $json.") + throw PushNetworkException("Missing response body.") + } + val body = Base64.decode(result) + if (body.size > maxSize) { + Log.d("Loki", "Attachment size limit exceeded.") + throw PushNetworkException("Max response size exceeded.") + } + body.inputStream().use { input -> + val buffer = ByteArray(32768) + var count = 0 + var bytes = input.read(buffer) + while (bytes >= 0) { + outputStream.write(buffer, 0, bytes) + count += bytes + if (count > maxSize) { + Log.d("Loki", "Attachment size limit exceeded.") + throw PushNetworkException("Max response size exceeded.") + } + listener?.onAttachmentProgress(body.size.toLong(), count.toLong()) + bytes = input.read(buffer) + } + } + } catch (e: Exception) { + Log.e("Loki", "Couldn't download attachment due to error", e) + throw if (e is NonSuccessfulResponseCodeException) e else PushNetworkException(e) } - } catch (e: Exception) { - Log.d("Loki", "Couldn't download attachment due to error: $e.") - throw if (e is NonSuccessfulResponseCodeException) e else PushNetworkException(e) } } } \ No newline at end of file