Migrate old official open group locations for polling and adding (#932)

* feat: adding in first part of open group migrations and tests for migration logic / helpers

* feat: test code and migration logic for open groups in the case of no conflicts

* feat: add in extra test cases and refactor code for migrator

* refactor: migrate open group join URLs and references to server in adding new open groups to catch legacy and re-write it

* refactor: joining open groups using OpenGroupUrlParser.kt now

* fix: add in compile issues for renamed OpenGroupApi.kt from OpenGroupV2

* fix: prevent duplicates of http/https for new open group DNS and prevent adding new groups based on public key

* fix: room and server swapped parameters

* fix: replace default server for config messages

* fix: actually using public key to de-dupe didn't work for rooms

* build: bump version code and name
This commit is contained in:
Harris 2022-08-22 09:27:35 +10:00 committed by GitHub
parent 5469f232a0
commit d6d0c52745
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 627 additions and 79 deletions

View File

@ -159,8 +159,8 @@ dependencies {
testImplementation 'org.robolectric:shadows-multidex:4.4'
}
def canonicalVersionCode = 293
def canonicalVersionName = "1.14.1"
def canonicalVersionCode = 294
def canonicalVersionName = "1.14.2"
def postFixSize = 10
def abiPostFix = ['armeabi-v7a' : 1,

View File

@ -58,6 +58,7 @@ import org.thoughtcrime.securesms.database.Storage;
import org.thoughtcrime.securesms.dependencies.DatabaseComponent;
import org.thoughtcrime.securesms.dependencies.DatabaseModule;
import org.thoughtcrime.securesms.groups.OpenGroupManager;
import org.thoughtcrime.securesms.groups.OpenGroupMigrator;
import org.thoughtcrime.securesms.home.HomeActivity;
import org.thoughtcrime.securesms.jobmanager.JobManager;
import org.thoughtcrime.securesms.jobmanager.impl.JsonDataSerializer;
@ -191,6 +192,9 @@ public class ApplicationContext extends Application implements DefaultLifecycleO
storage,
messageDataProvider,
()-> KeyPairUtilities.INSTANCE.getUserED25519KeyPair(this));
// migrate session open group data
OpenGroupMigrator.migrate(getDatabaseComponent());
// end migration
callMessageProcessor = new CallMessageProcessor(this, textSecurePreferences, ProcessLifecycleOwner.get().getLifecycle(), storage);
Log.i(TAG, "onCreate()");
startKovenant();

View File

@ -40,7 +40,7 @@ class JoinOpenGroupDialog(private val name: String, private val url: String) : B
ThreadUtils.queue {
try {
OpenGroupManager.add(openGroup.server, openGroup.room, openGroup.serverPublicKey, activity)
MessagingModuleConfiguration.shared.storage.onOpenGroupAdded(url)
MessagingModuleConfiguration.shared.storage.onOpenGroupAdded(openGroup.server)
ConfigurationMessageUtilities.forceSyncConfigurationNowIfNeeded(activity)
} catch (e: Exception) {
Toast.makeText(activity, R.string.activity_join_public_chat_error, Toast.LENGTH_SHORT).show()

View File

@ -14,6 +14,7 @@ import com.annimon.stream.Stream;
import net.sqlcipher.database.SQLiteDatabase;
import org.jetbrains.annotations.NotNull;
import org.session.libsession.utilities.Address;
import org.session.libsession.utilities.GroupRecord;
import org.session.libsession.utilities.TextSecurePreferences;
@ -441,7 +442,15 @@ public class GroupDatabase extends Database implements LokiOpenGroupDatabaseProt
}
}
public static class Reader implements Closeable {
public void migrateEncodedGroup(@NotNull String legacyEncodedGroupId, @NotNull String newEncodedGroupId) {
String query = GROUP_ID+" = ?";
ContentValues contentValues = new ContentValues(1);
contentValues.put(GROUP_ID, newEncodedGroupId);
SQLiteDatabase db = databaseHelper.getWritableDatabase();
db.update(TABLE_NAME, contentValues, query, new String[]{legacyEncodedGroupId});
}
public static class Reader implements Closeable {
private final Cursor cursor;

View File

@ -380,18 +380,29 @@ class LokiAPIDatabase(context: Context, helper: SQLCipherOpenHelper) : Database(
database.delete(lastDeletionServerIDTable, "$lastDeletionServerIDTableIndex = ?", wrap(index))
}
fun removeLastDeletionServerID(group: Long, server: String) {
override fun migrateLegacyOpenGroup(legacyServerId: String, newServerId: String) {
val database = databaseHelper.writableDatabase
val index = "$server.$group"
database.delete(lastDeletionServerIDTable,"$lastDeletionServerIDTableIndex = ?", wrap(index))
}
fun getUserCount(group: Long, server: String): Int? {
val database = databaseHelper.readableDatabase
val index = "$server.$group"
return database.get(userCountTable, "$publicChatID = ?", wrap(index)) { cursor ->
cursor.getInt(userCount)
}?.toInt()
database.beginTransaction()
val authRow = wrap(mapOf(server to newServerId))
database.update(openGroupAuthTokenTable, authRow, "$server = ?", wrap(legacyServerId))
val lastMessageRow = wrap(mapOf(lastMessageServerIDTableIndex to newServerId))
database.update(lastMessageServerIDTable, lastMessageRow,
"$lastMessageServerIDTableIndex = ?", wrap(legacyServerId))
val lastDeletionRow = wrap(mapOf(lastDeletionServerIDTableIndex to newServerId))
database.update(
lastDeletionServerIDTable, lastDeletionRow,
"$lastDeletionServerIDTableIndex = ?", wrap(legacyServerId))
val userCountRow = wrap(mapOf(publicChatID to newServerId))
database.update(
userCountTable, userCountRow,
"$publicChatID = ?", wrap(legacyServerId)
)
val publicKeyRow = wrap(mapOf(server to newServerId))
database.update(
openGroupPublicKeyTable, publicKeyRow,
"$server = ?", wrap(legacyServerId)
)
database.endTransaction()
}
fun getUserCount(room: String, server: String): Int? {
@ -402,13 +413,6 @@ class LokiAPIDatabase(context: Context, helper: SQLCipherOpenHelper) : Database(
}?.toInt()
}
override fun setUserCount(group: Long, server: String, newValue: Int) {
val database = databaseHelper.writableDatabase
val index = "$server.$group"
val row = wrap(mapOf( publicChatID to index, Companion.userCount to newValue.toString() ))
database.insertOrUpdate(userCountTable, row, "$publicChatID = ?", wrap(index))
}
override fun setUserCount(room: String, server: String, newValue: Int) {
val database = databaseHelper.writableDatabase
val index = "$server.$room"

View File

@ -177,4 +177,12 @@ class LokiMessageDatabase(context: Context, helper: SQLCipherOpenHelper) : Datab
val database = databaseHelper.writableDatabase
database.delete(messageHashTable, "${Companion.messageID} = ?", arrayOf(messageID.toString()))
}
fun migrateThreadId(legacyThreadId: Long, newThreadId: Long) {
val database = databaseHelper.writableDatabase
val contentValues = ContentValues(1)
contentValues.put(threadID, newThreadId)
database.update(messageThreadMappingTable, contentValues, "$threadID = ?", arrayOf(legacyThreadId.toString()))
}
}

View File

@ -7,15 +7,14 @@ import android.text.TextUtils;
import net.sqlcipher.database.SQLiteDatabase;
import org.session.libsession.utilities.Address;
import org.session.libsession.utilities.Document;
import org.session.libsession.utilities.IdentityKeyMismatch;
import org.session.libsession.utilities.IdentityKeyMismatchList;
import org.thoughtcrime.securesms.database.helpers.SQLCipherOpenHelper;
import org.session.libsignal.utilities.Log;
import org.session.libsignal.crypto.IdentityKey;
import org.session.libsession.utilities.Address;
import org.session.libsignal.utilities.JsonUtil;
import org.session.libsignal.utilities.Log;
import org.thoughtcrime.securesms.database.helpers.SQLCipherOpenHelper;
import java.io.IOException;
import java.util.ArrayList;
@ -159,6 +158,15 @@ public abstract class MessagingDatabase extends Database implements MmsSmsColumn
}
}
public void migrateThreadId(long oldThreadId, long newThreadId) {
SQLiteDatabase db = databaseHelper.getWritableDatabase();
String where = THREAD_ID+" = ?";
String[] args = new String[]{oldThreadId+""};
ContentValues contentValues = new ContentValues();
contentValues.put(THREAD_ID, newThreadId);
db.update(getTableName(), contentValues, where, args);
}
public static class SyncMessageId {
private final Address address;

View File

@ -567,9 +567,8 @@ class Storage(context: Context, helper: SQLCipherOpenHelper) : Database(context,
OpenGroupManager.addOpenGroup(urlAsString, context)
}
override fun onOpenGroupAdded(urlAsString: String) {
val server = OpenGroup.getServer(urlAsString)
OpenGroupManager.restartPollerForServer(server.toString().removeSuffix("/"))
override fun onOpenGroupAdded(server: String) {
OpenGroupManager.restartPollerForServer(server.removeSuffix("/"))
}
override fun hasBackgroundGroupAddJob(groupJoinUrl: String): Boolean {

View File

@ -34,6 +34,7 @@ import com.annimon.stream.Stream;
import net.sqlcipher.database.SQLiteDatabase;
import org.jetbrains.annotations.NotNull;
import org.session.libsession.utilities.Address;
import org.session.libsession.utilities.Contact;
import org.session.libsession.utilities.DelimiterUtil;
@ -56,12 +57,15 @@ import org.thoughtcrime.securesms.database.model.MessageRecord;
import org.thoughtcrime.securesms.database.model.MmsMessageRecord;
import org.thoughtcrime.securesms.database.model.ThreadRecord;
import org.thoughtcrime.securesms.dependencies.DatabaseComponent;
import org.thoughtcrime.securesms.groups.OpenGroupMigrator;
import org.thoughtcrime.securesms.mms.Slide;
import org.thoughtcrime.securesms.mms.SlideDeck;
import org.thoughtcrime.securesms.notifications.MarkReadReceiver;
import org.thoughtcrime.securesms.util.SessionMetaProtocol;
import java.io.Closeable;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
@ -765,7 +769,85 @@ public class ThreadDatabase extends Database {
return query;
}
public interface ProgressListener {
@NotNull
public List<ThreadRecord> getHttpOxenOpenGroups() {
String where = TABLE_NAME+"."+ADDRESS+" LIKE ?";
String selection = OpenGroupMigrator.HTTP_PREFIX+OpenGroupMigrator.OPEN_GET_SESSION_TRAILING_DOT_ENCODED +"%";
SQLiteDatabase db = databaseHelper.getReadableDatabase();
String query = createQuery(where, 0);
Cursor cursor = db.rawQuery(query, new String[]{selection});
if (cursor == null) {
return Collections.emptyList();
}
List<ThreadRecord> threads = new ArrayList<>();
try {
Reader reader = readerFor(cursor);
ThreadRecord record;
while ((record = reader.getNext()) != null) {
threads.add(record);
}
} finally {
cursor.close();
}
return threads;
}
@NotNull
public List<ThreadRecord> getLegacyOxenOpenGroups() {
String where = TABLE_NAME+"."+ADDRESS+" LIKE ?";
String selection = OpenGroupMigrator.LEGACY_GROUP_ENCODED_ID+"%";
SQLiteDatabase db = databaseHelper.getReadableDatabase();
String query = createQuery(where, 0);
Cursor cursor = db.rawQuery(query, new String[]{selection});
if (cursor == null) {
return Collections.emptyList();
}
List<ThreadRecord> threads = new ArrayList<>();
try {
Reader reader = readerFor(cursor);
ThreadRecord record;
while ((record = reader.getNext()) != null) {
threads.add(record);
}
} finally {
cursor.close();
}
return threads;
}
@NotNull
public List<ThreadRecord> getHttpsOxenOpenGroups() {
String where = TABLE_NAME+"."+ADDRESS+" LIKE ?";
String selection = OpenGroupMigrator.NEW_GROUP_ENCODED_ID+"%";
SQLiteDatabase db = databaseHelper.getReadableDatabase();
String query = createQuery(where, 0);
Cursor cursor = db.rawQuery(query, new String[]{selection});
if (cursor == null) {
return Collections.emptyList();
}
List<ThreadRecord> threads = new ArrayList<>();
try {
Reader reader = readerFor(cursor);
ThreadRecord record;
while ((record = reader.getNext()) != null) {
threads.add(record);
}
} finally {
cursor.close();
}
return threads;
}
public void migrateEncodedGroup(long threadId, @NotNull String newEncodedGroupId) {
ContentValues contentValues = new ContentValues(1);
contentValues.put(ADDRESS, newEncodedGroupId);
SQLiteDatabase db = databaseHelper.getWritableDatabase();
db.update(TABLE_NAME, contentValues, ID_WHERE, new String[] {threadId+""});
}
public interface ProgressListener {
void onProgress(int complete, int total);
}

View File

@ -24,14 +24,14 @@ import kotlinx.coroutines.withContext
import network.loki.messenger.R
import network.loki.messenger.databinding.ActivityJoinPublicChatBinding
import network.loki.messenger.databinding.FragmentEnterChatUrlBinding
import okhttp3.HttpUrl
import org.session.libsession.messaging.MessagingModuleConfiguration
import org.session.libsession.messaging.open_groups.OpenGroupApi.DefaultGroup
import org.session.libsession.utilities.Address
import org.session.libsession.utilities.GroupUtil
import org.session.libsession.utilities.OpenGroupUrlParser
import org.session.libsession.utilities.OpenGroupUrlParser.Error
import org.session.libsession.utilities.recipients.Recipient
import org.session.libsignal.utilities.Log
import org.session.libsignal.utilities.PublicKeyValidation
import org.thoughtcrime.securesms.BaseActionBarActivity
import org.thoughtcrime.securesms.PassphraseRequiredActionBarActivity
import org.thoughtcrime.securesms.conversation.v2.ConversationActivityV2
@ -83,32 +83,27 @@ class JoinPublicChatActivity : PassphraseRequiredActionBarActivity(), ScanQRCode
fun joinPublicChatIfPossible(url: String) {
// Add "http" if not entered explicitly
val stringWithExplicitScheme = if (!url.startsWith("http")) "http://$url" else url
val url = HttpUrl.parse(stringWithExplicitScheme) ?: return Toast.makeText(this,R.string.invalid_url, Toast.LENGTH_SHORT).show()
val room = url.pathSegments().firstOrNull()
val publicKey = url.queryParameter("public_key")
val isV2OpenGroup = !room.isNullOrEmpty()
if (isV2OpenGroup && (publicKey == null || !PublicKeyValidation.isValid(publicKey, 64,false))) {
return Toast.makeText(this, R.string.invalid_public_key, Toast.LENGTH_SHORT).show()
val openGroup = try {
OpenGroupUrlParser.parseUrl(url)
} catch (e: Error) {
when (e) {
is Error.MalformedURL -> return Toast.makeText(this, R.string.activity_join_public_chat_error, Toast.LENGTH_SHORT).show()
is Error.InvalidPublicKey -> return Toast.makeText(this, R.string.invalid_public_key, Toast.LENGTH_SHORT).show()
is Error.NoPublicKey -> return Toast.makeText(this, R.string.invalid_public_key, Toast.LENGTH_SHORT).show()
is Error.NoRoom -> return Toast.makeText(this, R.string.activity_join_public_chat_error, Toast.LENGTH_SHORT).show()
}
}
showLoader()
lifecycleScope.launch(Dispatchers.IO) {
try {
val (threadID, groupID) = if (isV2OpenGroup) {
val server = HttpUrl.Builder().scheme(url.scheme()).host(url.host()).apply {
if (url.port() != 80 || url.port() != 443) { this.port(url.port()) } // Non-standard port; add to server
}.build()
val sanitizedServer = openGroup.server.removeSuffix("/")
val openGroupID = "$sanitizedServer.${openGroup.room}"
OpenGroupManager.add(sanitizedServer, openGroup.room, openGroup.serverPublicKey, this@JoinPublicChatActivity)
val storage = MessagingModuleConfiguration.shared.storage
storage.onOpenGroupAdded(sanitizedServer)
val threadID = GroupManager.getOpenGroupThreadID(openGroupID, this@JoinPublicChatActivity)
val groupID = GroupUtil.getEncodedOpenGroupID(openGroupID.toByteArray())
val sanitizedServer = server.toString().removeSuffix("/")
val openGroupID = "$sanitizedServer.${room!!}"
OpenGroupManager.add(sanitizedServer, room, publicKey!!, this@JoinPublicChatActivity)
MessagingModuleConfiguration.shared.storage.onOpenGroupAdded(stringWithExplicitScheme)
val threadID = GroupManager.getOpenGroupThreadID(openGroupID, this@JoinPublicChatActivity)
val groupID = GroupUtil.getEncodedOpenGroupID(openGroupID.toByteArray())
threadID to groupID
} else {
throw Exception("No longer supported.")
}
ConfigurationMessageUtilities.forceSyncConfigurationNowIfNeeded(this@JoinPublicChatActivity)
withContext(Dispatchers.Main) {
val recipient = Recipient.from(this@JoinPublicChatActivity, Address.fromSerialized(groupID), false)

View File

@ -8,6 +8,7 @@ import org.session.libsession.messaging.open_groups.GroupMemberRole
import org.session.libsession.messaging.open_groups.OpenGroup
import org.session.libsession.messaging.open_groups.OpenGroupApi
import org.session.libsession.messaging.sending_receiving.pollers.OpenGroupPoller
import org.session.libsignal.utilities.Log
import org.session.libsignal.utilities.ThreadUtils
import org.thoughtcrime.securesms.dependencies.DatabaseComponent
import java.util.concurrent.Executors
@ -71,7 +72,7 @@ object OpenGroupManager {
storage.removeLastInboxMessageId(server)
storage.removeLastOutboxMessageId(server)
// Store the public key
storage.setOpenGroupPublicKey(server,publicKey)
storage.setOpenGroupPublicKey(server, publicKey)
// Get capabilities
val capabilities = OpenGroupApi.getCapabilities(server).get()
storage.setServerCapabilities(server, capabilities.capabilities)
@ -92,6 +93,7 @@ object OpenGroupManager {
pollers[server]?.stop()
pollers[server]?.startIfNeeded() ?: run {
val poller = OpenGroupPoller(server, executorService)
Log.d("Loki", "Starting poller for open group: $server")
pollers[server] = poller
poller.startIfNeeded()
}
@ -133,7 +135,7 @@ object OpenGroupManager {
val server = OpenGroup.getServer(urlAsString)
val room = url.pathSegments().firstOrNull() ?: return
val publicKey = url.queryParameter("public_key") ?: return
add(server.toString().removeSuffix("/"), room, publicKey, context)
add(server.toString().removeSuffix("/"), room, publicKey, context) // assume migrated from calling function
}
fun updateOpenGroup(openGroup: OpenGroup, context: Context) {

View File

@ -0,0 +1,139 @@
package org.thoughtcrime.securesms.groups
import androidx.annotation.VisibleForTesting
import org.session.libsession.messaging.open_groups.OpenGroupApi
import org.session.libsession.utilities.recipients.Recipient
import org.session.libsignal.utilities.Hex
import org.thoughtcrime.securesms.database.model.ThreadRecord
import org.thoughtcrime.securesms.dependencies.DatabaseComponent
object OpenGroupMigrator {
const val HTTP_PREFIX = "__loki_public_chat_group__!687474703a2f2f"
private const val HTTPS_PREFIX = "__loki_public_chat_group__!68747470733a2f2f"
const val OPEN_GET_SESSION_TRAILING_DOT_ENCODED = "6f70656e2e67657473657373696f6e2e6f72672e"
const val LEGACY_GROUP_ENCODED_ID = "__loki_public_chat_group__!687474703a2f2f3131362e3230332e37302e33332e" // old IP based toByteArray()
const val NEW_GROUP_ENCODED_ID = "__loki_public_chat_group__!68747470733a2f2f6f70656e2e67657473657373696f6e2e6f72672e" // new URL based toByteArray()
data class OpenGroupMapping(val stub: String, val legacyThreadId: Long, val newThreadId: Long?)
@VisibleForTesting
fun Recipient.roomStub(): String? {
if (!isOpenGroupRecipient) return null
val serialized = address.serialize()
if (serialized.startsWith(LEGACY_GROUP_ENCODED_ID)) {
return serialized.replace(LEGACY_GROUP_ENCODED_ID,"")
} else if (serialized.startsWith(NEW_GROUP_ENCODED_ID)) {
return serialized.replace(NEW_GROUP_ENCODED_ID,"")
} else if (serialized.startsWith(HTTP_PREFIX + OPEN_GET_SESSION_TRAILING_DOT_ENCODED)) {
return serialized.replace(HTTP_PREFIX + OPEN_GET_SESSION_TRAILING_DOT_ENCODED, "")
}
return null
}
@VisibleForTesting
fun getExistingMappings(legacy: List<ThreadRecord>, new: List<ThreadRecord>): List<OpenGroupMapping> {
val legacyStubsMapping = legacy.mapNotNull { thread ->
val stub = thread.recipient.roomStub()
stub?.let { it to thread.threadId }
}
val newStubsMapping = new.mapNotNull { thread ->
val stub = thread.recipient.roomStub()
stub?.let { it to thread.threadId }
}
return legacyStubsMapping.map { (legacyEncodedStub, legacyId) ->
// get 'new' open group thread ID if stubs match
OpenGroupMapping(
legacyEncodedStub,
legacyId,
newStubsMapping.firstOrNull { (newEncodedStub, _) -> newEncodedStub == legacyEncodedStub }?.second
)
}
}
@JvmStatic
fun migrate(databaseComponent: DatabaseComponent) {
// migrate thread db
val threadDb = databaseComponent.threadDatabase()
val legacyOpenGroups = threadDb.legacyOxenOpenGroups
val httpBasedNewGroups = threadDb.httpOxenOpenGroups
if (legacyOpenGroups.isEmpty() && httpBasedNewGroups.isEmpty()) return // no need to migrate
val newOpenGroups = threadDb.httpsOxenOpenGroups
val firstStepMigration = getExistingMappings(legacyOpenGroups, newOpenGroups)
val secondStepMigration = getExistingMappings(httpBasedNewGroups, newOpenGroups)
val groupDb = databaseComponent.groupDatabase()
val lokiApiDb = databaseComponent.lokiAPIDatabase()
val smsDb = databaseComponent.smsDatabase()
val mmsDb = databaseComponent.mmsDatabase()
val lokiMessageDatabase = databaseComponent.lokiMessageDatabase()
val lokiThreadDatabase = databaseComponent.lokiThreadDatabase()
firstStepMigration.forEach { (stub, old, new) ->
val legacyEncodedGroupId = LEGACY_GROUP_ENCODED_ID+stub
if (new == null) {
val newEncodedGroupId = NEW_GROUP_ENCODED_ID+stub
// migrate thread and group encoded values
threadDb.migrateEncodedGroup(old, newEncodedGroupId)
groupDb.migrateEncodedGroup(legacyEncodedGroupId, newEncodedGroupId)
// migrate Loki API DB values
// decode the hex to bytes, decode byte array to string i.e. "oxen" or "session"
val decodedStub = Hex.fromStringCondensed(stub).decodeToString()
val legacyLokiServerId = "${OpenGroupApi.legacyDefaultServer}.$decodedStub"
val newLokiServerId = "${OpenGroupApi.defaultServer}.$decodedStub"
lokiApiDb.migrateLegacyOpenGroup(legacyLokiServerId, newLokiServerId)
// migrate loki thread db server info
val oldServerInfo = lokiThreadDatabase.getOpenGroupChat(old)
val newServerInfo = oldServerInfo!!.copy(server = OpenGroupApi.defaultServer, id = newLokiServerId)
lokiThreadDatabase.setOpenGroupChat(newServerInfo, old)
} else {
// has a legacy and a new one
// migrate SMS and MMS tables
smsDb.migrateThreadId(old, new)
mmsDb.migrateThreadId(old, new)
lokiMessageDatabase.migrateThreadId(old, new)
// delete group for legacy ID
groupDb.delete(legacyEncodedGroupId)
// delete thread for legacy ID
threadDb.deleteConversation(old)
lokiThreadDatabase.removeOpenGroupChat(old)
}
// maybe migrate jobs here
}
secondStepMigration.forEach { (stub, old, new) ->
val legacyEncodedGroupId = HTTP_PREFIX + OPEN_GET_SESSION_TRAILING_DOT_ENCODED + stub
if (new == null) {
val newEncodedGroupId = NEW_GROUP_ENCODED_ID+stub
// migrate thread and group encoded values
threadDb.migrateEncodedGroup(old, newEncodedGroupId)
groupDb.migrateEncodedGroup(legacyEncodedGroupId, newEncodedGroupId)
// migrate Loki API DB values
// decode the hex to bytes, decode byte array to string i.e. "oxen" or "session"
val decodedStub = Hex.fromStringCondensed(stub).decodeToString()
val legacyLokiServerId = "${OpenGroupApi.httpDefaultServer}.$decodedStub"
val newLokiServerId = "${OpenGroupApi.defaultServer}.$decodedStub"
lokiApiDb.migrateLegacyOpenGroup(legacyLokiServerId, newLokiServerId)
// migrate loki thread db server info
val oldServerInfo = lokiThreadDatabase.getOpenGroupChat(old)
val newServerInfo = oldServerInfo!!.copy(server = OpenGroupApi.defaultServer, id = newLokiServerId)
lokiThreadDatabase.setOpenGroupChat(newServerInfo, old)
} else {
// has a legacy and a new one
// migrate SMS and MMS tables
smsDb.migrateThreadId(old, new)
mmsDb.migrateThreadId(old, new)
lokiMessageDatabase.migrateThreadId(old, new)
// delete group for legacy ID
groupDb.delete(legacyEncodedGroupId)
// delete thread for legacy ID
threadDb.deleteConversation(old)
lokiThreadDatabase.removeOpenGroupChat(old)
}
// maybe migrate jobs here
}
}
}

View File

@ -124,10 +124,10 @@
android:id="@+id/snippetTextView"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:maxLines="1"
android:ellipsize="end"
android:textSize="@dimen/medium_font_size"
android:maxLines="1"
android:textColor="@color/text"
android:textSize="@dimen/medium_font_size"
tools:text="Sorry, gotta go fight crime again" />
<org.thoughtcrime.securesms.conversation.v2.components.TypingIndicatorView

View File

@ -0,0 +1,281 @@
package org.thoughtcrime.securesms.util
import org.junit.Assert.assertEquals
import org.junit.Assert.assertTrue
import org.junit.Test
import org.mockito.kotlin.KStubbing
import org.mockito.kotlin.argumentCaptor
import org.mockito.kotlin.doAnswer
import org.mockito.kotlin.doReturn
import org.mockito.kotlin.eq
import org.mockito.kotlin.mock
import org.mockito.kotlin.verify
import org.mockito.kotlin.verifyNoMoreInteractions
import org.session.libsession.messaging.open_groups.OpenGroup
import org.session.libsession.messaging.open_groups.OpenGroupApi
import org.session.libsession.utilities.Address
import org.session.libsession.utilities.recipients.Recipient
import org.thoughtcrime.securesms.database.GroupDatabase
import org.thoughtcrime.securesms.database.LokiAPIDatabase
import org.thoughtcrime.securesms.database.LokiMessageDatabase
import org.thoughtcrime.securesms.database.LokiThreadDatabase
import org.thoughtcrime.securesms.database.MmsDatabase
import org.thoughtcrime.securesms.database.SmsDatabase
import org.thoughtcrime.securesms.database.ThreadDatabase
import org.thoughtcrime.securesms.database.model.ThreadRecord
import org.thoughtcrime.securesms.dependencies.DatabaseComponent
import org.thoughtcrime.securesms.groups.OpenGroupMigrator
import org.thoughtcrime.securesms.groups.OpenGroupMigrator.OpenGroupMapping
import org.thoughtcrime.securesms.groups.OpenGroupMigrator.roomStub
class OpenGroupMigrationTests {
companion object {
const val EXAMPLE_LEGACY_ENCODED_OPEN_GROUP = "__loki_public_chat_group__!687474703a2f2f3131362e3230332e37302e33332e6f78656e"
const val EXAMPLE_NEW_ENCODED_OPEN_GROUP = "__loki_public_chat_group__!68747470733a2f2f6f70656e2e67657473657373696f6e2e6f72672e6f78656e"
const val OXEN_STUB_HEX = "6f78656e"
const val EXAMPLE_LEGACY_SERVER_ID = "http://116.203.70.33.oxen"
const val EXAMPLE_NEW_SERVER_ID = "https://open.getsession.org.oxen"
const val LEGACY_THREAD_ID = 1L
const val NEW_THREAD_ID = 2L
}
private fun legacyOpenGroupRecipient(additionalMocks: ((KStubbing<Recipient>) -> Unit) ? = null) = mock<Recipient> {
on { address } doReturn Address.fromSerialized(EXAMPLE_LEGACY_ENCODED_OPEN_GROUP)
on { isOpenGroupRecipient } doReturn true
additionalMocks?.let { it(this) }
}
private fun newOpenGroupRecipient(additionalMocks: ((KStubbing<Recipient>) -> Unit) ? = null) = mock<Recipient> {
on { address } doReturn Address.fromSerialized(EXAMPLE_NEW_ENCODED_OPEN_GROUP)
on { isOpenGroupRecipient } doReturn true
additionalMocks?.let { it(this) }
}
private fun legacyThreadRecord(additionalRecipientMocks: ((KStubbing<Recipient>) -> Unit) ? = null, additionalThreadMocks: ((KStubbing<ThreadRecord>) -> Unit)? = null) = mock<ThreadRecord> {
val returnedRecipient = legacyOpenGroupRecipient(additionalRecipientMocks)
on { recipient } doReturn returnedRecipient
on { threadId } doReturn LEGACY_THREAD_ID
}
private fun newThreadRecord(additionalRecipientMocks: ((KStubbing<Recipient>) -> Unit)? = null, additionalThreadMocks: ((KStubbing<ThreadRecord>) -> Unit)? = null) = mock<ThreadRecord> {
val returnedRecipient = newOpenGroupRecipient(additionalRecipientMocks)
on { recipient } doReturn returnedRecipient
on { threadId } doReturn NEW_THREAD_ID
}
@Test
fun `it should generate the correct room stubs for legacy groups`() {
val mockRecipient = legacyOpenGroupRecipient()
assertEquals(OXEN_STUB_HEX, mockRecipient.roomStub())
}
@Test
fun `it should generate the correct room stubs for new groups`() {
val mockNewRecipient = newOpenGroupRecipient()
assertEquals(OXEN_STUB_HEX, mockNewRecipient.roomStub())
}
@Test
fun `it should return correct mappings`() {
val legacyThread = legacyThreadRecord()
val newThread = newThreadRecord()
val expectedMapping = listOf(
OpenGroupMapping(OXEN_STUB_HEX, LEGACY_THREAD_ID, NEW_THREAD_ID)
)
assertTrue(expectedMapping.containsAll(OpenGroupMigrator.getExistingMappings(listOf(legacyThread), listOf(newThread))))
}
@Test
fun `it should return no mappings if there are no legacy open groups`() {
val mappings = OpenGroupMigrator.getExistingMappings(listOf(), listOf())
assertTrue(mappings.isEmpty())
}
@Test
fun `it should return no mappings if there are only new open groups`() {
val newThread = newThreadRecord()
val mappings = OpenGroupMigrator.getExistingMappings(emptyList(), listOf(newThread))
assertTrue(mappings.isEmpty())
}
@Test
fun `it should return null new thread in mappings if there are only legacy open groups`() {
val legacyThread = legacyThreadRecord()
val mappings = OpenGroupMigrator.getExistingMappings(listOf(legacyThread), emptyList())
val expectedMappings = listOf(
OpenGroupMapping(OXEN_STUB_HEX, LEGACY_THREAD_ID, null)
)
assertTrue(expectedMappings.containsAll(mappings))
}
@Test
fun `test migration thread DB calls legacy and returns if no legacy official groups`() {
val mockedThreadDb = mock<ThreadDatabase> {
on { legacyOxenOpenGroups } doReturn emptyList()
}
val mockedDbComponent = mock<DatabaseComponent> {
on { threadDatabase() } doReturn mockedThreadDb
}
OpenGroupMigrator.migrate(mockedDbComponent)
verify(mockedDbComponent).threadDatabase()
verify(mockedThreadDb).legacyOxenOpenGroups
verifyNoMoreInteractions(mockedThreadDb)
}
@Test
fun `it should migrate on thread, group and loki dbs with correct values for legacy only migration`() {
// mock threadDB
val capturedThreadId = argumentCaptor<Long>()
val capturedNewEncoded = argumentCaptor<String>()
val mockedThreadDb = mock<ThreadDatabase> {
val legacyThreadRecord = legacyThreadRecord()
on { legacyOxenOpenGroups } doReturn listOf(legacyThreadRecord)
on { httpsOxenOpenGroups } doReturn emptyList()
on { migrateEncodedGroup(capturedThreadId.capture(), capturedNewEncoded.capture()) } doAnswer {}
}
// mock groupDB
val capturedGroupLegacyEncoded = argumentCaptor<String>()
val capturedGroupNewEncoded = argumentCaptor<String>()
val mockedGroupDb = mock<GroupDatabase> {
on {
migrateEncodedGroup(
capturedGroupLegacyEncoded.capture(),
capturedGroupNewEncoded.capture()
)
} doAnswer {}
}
// mock LokiAPIDB
val capturedLokiLegacyGroup = argumentCaptor<String>()
val capturedLokiNewGroup = argumentCaptor<String>()
val mockedLokiApi = mock<LokiAPIDatabase> {
on { migrateLegacyOpenGroup(capturedLokiLegacyGroup.capture(), capturedLokiNewGroup.capture()) } doAnswer {}
}
val pubKey = OpenGroupApi.defaultServerPublicKey
val room = "oxen"
val legacyServer = OpenGroupApi.legacyDefaultServer
val newServer = OpenGroupApi.defaultServer
val lokiThreadOpenGroup = argumentCaptor<OpenGroup>()
val mockedLokiThreadDb = mock<LokiThreadDatabase> {
on { getOpenGroupChat(eq(LEGACY_THREAD_ID)) } doReturn OpenGroup(legacyServer, room, "Oxen", 0, pubKey)
on { setOpenGroupChat(lokiThreadOpenGroup.capture(), eq(LEGACY_THREAD_ID)) } doAnswer {}
}
val mockedDbComponent = mock<DatabaseComponent> {
on { threadDatabase() } doReturn mockedThreadDb
on { groupDatabase() } doReturn mockedGroupDb
on { lokiAPIDatabase() } doReturn mockedLokiApi
on { lokiThreadDatabase() } doReturn mockedLokiThreadDb
}
OpenGroupMigrator.migrate(mockedDbComponent)
// expect threadDB migration to reflect new thread values:
// thread ID = 1, encoded ID = new encoded ID
assertEquals(LEGACY_THREAD_ID, capturedThreadId.firstValue)
assertEquals(EXAMPLE_NEW_ENCODED_OPEN_GROUP, capturedNewEncoded.firstValue)
// expect groupDB migration to reflect new thread values:
// legacy encoded ID, new encoded ID
assertEquals(EXAMPLE_LEGACY_ENCODED_OPEN_GROUP, capturedGroupLegacyEncoded.firstValue)
assertEquals(EXAMPLE_NEW_ENCODED_OPEN_GROUP, capturedGroupNewEncoded.firstValue)
// expect Loki API DB migration to reflect new thread values:
assertEquals("${OpenGroupApi.legacyDefaultServer}.oxen", capturedLokiLegacyGroup.firstValue)
assertEquals("${OpenGroupApi.defaultServer}.oxen", capturedLokiNewGroup.firstValue)
assertEquals(newServer, lokiThreadOpenGroup.firstValue.server)
}
@Test
fun `it should migrate and delete legacy thread with conflicting new and old values`() {
// mock threadDB
val capturedThreadId = argumentCaptor<Long>()
val mockedThreadDb = mock<ThreadDatabase> {
val legacyThreadRecord = legacyThreadRecord()
val newThreadRecord = newThreadRecord()
on { legacyOxenOpenGroups } doReturn listOf(legacyThreadRecord)
on { httpsOxenOpenGroups } doReturn listOf(newThreadRecord)
on { deleteConversation(capturedThreadId.capture()) } doAnswer {}
}
// mock groupDB
val capturedGroupLegacyEncoded = argumentCaptor<String>()
val mockedGroupDb = mock<GroupDatabase> {
on { delete(capturedGroupLegacyEncoded.capture()) } doReturn true
}
// mock LokiAPIDB
val capturedLokiLegacyGroup = argumentCaptor<String>()
val capturedLokiNewGroup = argumentCaptor<String>()
val mockedLokiApi = mock<LokiAPIDatabase> {
on { migrateLegacyOpenGroup(capturedLokiLegacyGroup.capture(), capturedLokiNewGroup.capture()) } doAnswer {}
}
// mock messaging dbs
val migrateMmsFromThreadId = argumentCaptor<Long>()
val migrateMmsToThreadId = argumentCaptor<Long>()
val mockedMmsDb = mock<MmsDatabase> {
on { migrateThreadId(migrateMmsFromThreadId.capture(), migrateMmsToThreadId.capture()) } doAnswer {}
}
val migrateSmsFromThreadId = argumentCaptor<Long>()
val migrateSmsToThreadId = argumentCaptor<Long>()
val mockedSmsDb = mock<SmsDatabase> {
on { migrateThreadId(migrateSmsFromThreadId.capture(), migrateSmsToThreadId.capture()) } doAnswer {}
}
val lokiFromThreadId = argumentCaptor<Long>()
val lokiToThreadId = argumentCaptor<Long>()
val mockedLokiMessageDatabase = mock<LokiMessageDatabase> {
on { migrateThreadId(lokiFromThreadId.capture(), lokiToThreadId.capture()) } doAnswer {}
}
val mockedLokiThreadDb = mock<LokiThreadDatabase> {
on { removeOpenGroupChat(eq(LEGACY_THREAD_ID)) } doAnswer {}
}
val mockedDbComponent = mock<DatabaseComponent> {
on { threadDatabase() } doReturn mockedThreadDb
on { groupDatabase() } doReturn mockedGroupDb
on { lokiAPIDatabase() } doReturn mockedLokiApi
on { mmsDatabase() } doReturn mockedMmsDb
on { smsDatabase() } doReturn mockedSmsDb
on { lokiMessageDatabase() } doReturn mockedLokiMessageDatabase
on { lokiThreadDatabase() } doReturn mockedLokiThreadDb
}
OpenGroupMigrator.migrate(mockedDbComponent)
// should delete thread by thread ID
assertEquals(LEGACY_THREAD_ID, capturedThreadId.firstValue)
// should delete group by legacy encoded ID
assertEquals(EXAMPLE_LEGACY_ENCODED_OPEN_GROUP, capturedGroupLegacyEncoded.firstValue)
// should migrate SMS from legacy thread ID to new thread ID
assertEquals(LEGACY_THREAD_ID, migrateSmsFromThreadId.firstValue)
assertEquals(NEW_THREAD_ID, migrateSmsToThreadId.firstValue)
// should migrate MMS from legacy thread ID to new thread ID
assertEquals(LEGACY_THREAD_ID, migrateMmsFromThreadId.firstValue)
assertEquals(NEW_THREAD_ID, migrateMmsToThreadId.firstValue)
}
}

View File

@ -65,7 +65,7 @@ interface StorageProtocol {
fun updateOpenGroup(openGroup: OpenGroup)
fun getOpenGroup(threadId: Long): OpenGroup?
fun addOpenGroup(urlAsString: String)
fun onOpenGroupAdded(urlAsString: String)
fun onOpenGroupAdded(server: String)
fun hasBackgroundGroupAddJob(groupJoinUrl: String): Boolean
fun setOpenGroupServerMessageID(messageID: Long, serverID: Long, threadID: Long, isSms: Boolean)
fun getOpenGroup(room: String, server: String): OpenGroup?

View File

@ -6,6 +6,7 @@ import org.session.libsession.messaging.open_groups.OpenGroup
import org.session.libsession.messaging.open_groups.OpenGroupApi
import org.session.libsession.messaging.utilities.Data
import org.session.libsession.utilities.GroupUtil
import org.session.libsession.utilities.OpenGroupUrlParser
import org.session.libsignal.utilities.Log
class BackgroundGroupAddJob(val joinUrl: String): Job {
@ -30,31 +31,27 @@ class BackgroundGroupAddJob(val joinUrl: String): Job {
override fun execute() {
try {
val openGroup = OpenGroupUrlParser.parseUrl(joinUrl)
val storage = MessagingModuleConfiguration.shared.storage
val allOpenGroups = storage.getAllOpenGroups().map { it.value.joinURL }
if (allOpenGroups.contains(joinUrl)) {
if (allOpenGroups.contains(openGroup.joinUrl())) {
Log.e("OpenGroupDispatcher", "Failed to add group because", DuplicateGroupException())
delegate?.handleJobFailed(this, DuplicateGroupException())
return
}
// get image
val url = HttpUrl.parse(joinUrl) ?: throw Exception("Group joinUrl isn't valid")
val server = OpenGroup.getServer(joinUrl)
val serverString = server.toString().removeSuffix("/")
val publicKey = url.queryParameter("public_key") ?: throw Exception("Group public key isn't valid")
val room = url.pathSegments().firstOrNull() ?: throw Exception("Group room isn't valid")
storage.setOpenGroupPublicKey(serverString, publicKey)
// get info and auth token
storage.addOpenGroup(joinUrl)
val info = OpenGroupApi.getRoomInfo(room, serverString).get()
storage.setOpenGroupPublicKey(openGroup.server, openGroup.serverPublicKey)
val info = OpenGroupApi.getRoomInfo(openGroup.room, openGroup.server).get()
val imageId = info.imageId
storage.addOpenGroup(openGroup.joinUrl())
if (imageId != null) {
val bytes = OpenGroupApi.downloadOpenGroupProfilePicture(serverString, room, imageId).get()
val groupId = GroupUtil.getEncodedOpenGroupID("$server.$room".toByteArray())
val bytes = OpenGroupApi.downloadOpenGroupProfilePicture(openGroup.server, openGroup.room, imageId).get()
val groupId = GroupUtil.getEncodedOpenGroupID("${openGroup.server}.${openGroup.room}".toByteArray())
storage.updateProfilePicture(groupId, bytes)
storage.updateTimestampUpdated(groupId, System.currentTimeMillis())
}
storage.onOpenGroupAdded(joinUrl)
Log.d(KEY, "onOpenGroupAdded(${openGroup.server})")
storage.onOpenGroupAdded(openGroup.server)
} catch (e: Exception) {
Log.e("OpenGroupDispatcher", "Failed to add group because",e)
delegate?.handleJobFailed(this, e)

View File

@ -55,9 +55,14 @@ object OpenGroupApi {
now - lastOpenDate
}
const val defaultServerPublicKey =
"a03c383cf63c3c4efe67acc52112a6dd734b3a946b9545f488aaa93da7991238"
const val defaultServer = "http://116.203.70.33"
const val defaultServerPublicKey = "a03c383cf63c3c4efe67acc52112a6dd734b3a946b9545f488aaa93da7991238"
const val legacyServerIP = "116.203.70.33"
const val legacyDefaultServer = "http://116.203.70.33" // TODO: migrate all references to use new value
/** For migration purposes only, don't use this value in joining groups */
const val httpDefaultServer = "http://open.getsession.org"
const val defaultServer = "https://open.getsession.org"
sealed class Error(message: String) : Exception(message) {
object Generic : Error("An error occurred.")

View File

@ -0,0 +1,9 @@
package org.session.libsession.messaging.open_groups
fun String.migrateLegacyServerUrl() = if (contains(OpenGroupApi.legacyServerIP)) {
OpenGroupApi.defaultServer
} else if (contains(OpenGroupApi.httpDefaultServer)) {
OpenGroupApi.defaultServer
} else {
this
}

View File

@ -17,6 +17,7 @@ import org.session.libsession.messaging.messages.control.TypingIndicator
import org.session.libsession.messaging.messages.control.UnsendRequest
import org.session.libsession.messaging.messages.visible.Attachment
import org.session.libsession.messaging.messages.visible.VisibleMessage
import org.session.libsession.messaging.open_groups.OpenGroupApi
import org.session.libsession.messaging.sending_receiving.attachments.PointerAttachment
import org.session.libsession.messaging.sending_receiving.data_extraction.DataExtractionNotificationInfoMessage
import org.session.libsession.messaging.sending_receiving.link_preview.LinkPreview
@ -157,7 +158,10 @@ private fun handleConfigurationMessage(message: ConfigurationMessage) {
}
}
val allV2OpenGroups = storage.getAllOpenGroups().map { it.value.joinURL }
for (openGroup in message.openGroups) {
for (openGroup in message.openGroups.map {
it.replace(OpenGroupApi.legacyDefaultServer, OpenGroupApi.defaultServer)
.replace(OpenGroupApi.httpDefaultServer, OpenGroupApi.defaultServer)
}) {
if (allV2OpenGroups.contains(openGroup)) continue
Log.d("OpenGroup", "All open groups doesn't contain $openGroup")
if (!storage.hasBackgroundGroupAddJob(openGroup)) {

View File

@ -1,6 +1,7 @@
package org.session.libsession.utilities
import okhttp3.HttpUrl
import org.session.libsession.messaging.open_groups.migrateLegacyServerUrl
object OpenGroupUrlParser {
@ -20,7 +21,7 @@ object OpenGroupUrlParser {
// If the URL is malformed, throw an exception
val url = HttpUrl.parse(urlWithPrefix) ?: throw Error.MalformedURL
// Parse components
val server = HttpUrl.Builder().scheme(url.scheme()).host(url.host()).port(url.port()).build().toString().removeSuffix(suffix)
val server = HttpUrl.Builder().scheme(url.scheme()).host(url.host()).port(url.port()).build().toString().removeSuffix(suffix).migrateLegacyServerUrl()
val room = url.pathSegments().firstOrNull { !it.isNullOrEmpty() } ?: throw Error.NoRoom
val publicKey = url.queryParameter(queryPrefix) ?: throw Error.NoPublicKey
if (publicKey.length != 64) throw Error.InvalidPublicKey
@ -33,4 +34,6 @@ object OpenGroupUrlParser {
}
}
class V2OpenGroupInfo(val server: String, val room: String, val serverPublicKey: String)
class V2OpenGroupInfo(val server: String, val room: String, val serverPublicKey: String) {
fun joinUrl() = "$server/$room?public_key=$serverPublicKey"
}

View File

@ -20,7 +20,6 @@ interface LokiAPIDatabaseProtocol {
fun setReceivedMessageHashValues(publicKey: String, newValue: Set<String>, namespace: Int)
fun getAuthToken(server: String): String?
fun setAuthToken(server: String, newValue: String?)
fun setUserCount(group: Long, server: String, newValue: Int)
fun setUserCount(room: String, server: String, newValue: Int)
fun getLastMessageServerID(room: String, server: String): Long?
fun setLastMessageServerID(room: String, server: String, newValue: Long)
@ -36,5 +35,5 @@ interface LokiAPIDatabaseProtocol {
fun isClosedGroup(groupPublicKey: String): Boolean
fun getForkInfo(): ForkInfo
fun setForkInfo(forkInfo: ForkInfo)
fun migrateLegacyOpenGroup(legacyServerId: String, newServerId: String)
}