From 66d87533d3ca1b5df465f3443e8bfa57a76708f5 Mon Sep 17 00:00:00 2001 From: ryanzhao Date: Wed, 4 Dec 2019 11:28:12 +1100 Subject: [PATCH 1/3] add cache for Addresses and Messages. make multiple insertion into one transaction. --- .../securesms/database/ThreadDatabase.java | 44 +++++++++++++++++-- .../database/helpers/SQLCipherOpenHelper.java | 8 ++++ .../securesms/loki/LokiMessageDatabase.kt | 1 + .../securesms/loki/LokiPublicChatPoller.kt | 2 + .../securesms/util/TextSecurePreferences.java | 8 ++++ 5 files changed, 60 insertions(+), 3 deletions(-) diff --git a/src/org/thoughtcrime/securesms/database/ThreadDatabase.java b/src/org/thoughtcrime/securesms/database/ThreadDatabase.java index a4d1917174..49a0cc8bfb 100644 --- a/src/org/thoughtcrime/securesms/database/ThreadDatabase.java +++ b/src/org/thoughtcrime/securesms/database/ThreadDatabase.java @@ -50,14 +50,21 @@ import org.whispersystems.libsignal.util.Pair; import org.whispersystems.libsignal.util.guava.Optional; import java.io.Closeable; +import java.util.ArrayList; +import java.util.HashMap; import java.util.LinkedList; import java.util.List; +import java.util.Map; import java.util.Set; public class ThreadDatabase extends Database { private static final String TAG = ThreadDatabase.class.getSimpleName(); + private Map addressCache = new HashMap<>(); + private Map> messageCache = new HashMap<>(); + private Map recipientCache = new HashMap<>(); + public static final String TABLE_NAME = "thread"; public static final String ID = "_id"; public static final String DATE = "date"; @@ -150,11 +157,34 @@ public class ThreadDatabase extends Database { contentValues.put(ARCHIVED, 0); } - SQLiteDatabase db = databaseHelper.getWritableDatabase(); - db.update(TABLE_NAME, contentValues, ID + " = ?", new String[] {threadId + ""}); - notifyConversationListListeners(); + if (!messageCache.containsKey(threadId)) { + messageCache.put(threadId, new ArrayList<>()); + } + messageCache.get(threadId).add(contentValues); + + if (!recipientCache.containsKey(threadId)) { + recipientCache.put(threadId, getRecipientForThreadId(threadId).getName().replaceAll("\\s*", "")); + } + String key = recipientCache.get(threadId); + int newMessagesNumber = TextSecurePreferences.getNewMessagesNumber(this.context, key); + if (newMessagesNumber == 0 || newMessagesNumber == messageCache.get(threadId).size()) { + SQLiteDatabase db = databaseHelper.getWritableDatabase(); + db.beginTransactionNonExclusive(); + try { + for (ContentValues contentValue : messageCache.get(threadId)) { + db.update(TABLE_NAME, contentValue, ID + " = ?", new String[] {threadId + ""}); + } + TextSecurePreferences.setNewMessagesNumber(this.context, key, TextSecurePreferences.getNewMessagesNumber(this.context, key) - newMessagesNumber); + messageCache.get(threadId).clear(); + db.setTransactionSuccessful(); + notifyConversationListListeners(); + } finally { + db.endTransaction(); + } + } } + public void updateSnippet(long threadId, String snippet, @Nullable Uri attachment, long date, long type, boolean unarchive) { ContentValues contentValues = new ContentValues(4); @@ -528,6 +558,13 @@ public class ThreadDatabase extends Database { } public @Nullable Recipient getRecipientForThreadId(long threadId) { + // Loki - Cache the address. + // Don't know if this will affect any other signal code + // Don't know if it is necessary to add some cache time + if (addressCache.containsKey(threadId) && addressCache.get(threadId) != null) { + return Recipient.from(context, addressCache.get(threadId), false); + } + SQLiteDatabase db = databaseHelper.getReadableDatabase(); Cursor cursor = null; @@ -536,6 +573,7 @@ public class ThreadDatabase extends Database { if (cursor != null && cursor.moveToFirst()) { Address address = Address.fromSerialized(cursor.getString(cursor.getColumnIndexOrThrow(ADDRESS))); + addressCache.put(threadId, address); return Recipient.from(context, address, false); } } finally { diff --git a/src/org/thoughtcrime/securesms/database/helpers/SQLCipherOpenHelper.java b/src/org/thoughtcrime/securesms/database/helpers/SQLCipherOpenHelper.java index 6715feb5ee..ff11e5bbb6 100644 --- a/src/org/thoughtcrime/securesms/database/helpers/SQLCipherOpenHelper.java +++ b/src/org/thoughtcrime/securesms/database/helpers/SQLCipherOpenHelper.java @@ -166,6 +166,14 @@ public class SQLCipherOpenHelper extends SQLiteOpenHelper { } } + @Override + public void onConfigure(SQLiteDatabase db) { + super.onConfigure(db); + //Loki: Enable Write Ahead Logging Mode, increase the cache size + db.enableWriteAheadLogging(); + db.execSQL("PRAGMA cache_size = 10000"); + } + @Override public void onUpgrade(SQLiteDatabase db, int oldVersion, int newVersion) { Log.i(TAG, "Upgrading database: " + oldVersion + ", " + newVersion); diff --git a/src/org/thoughtcrime/securesms/loki/LokiMessageDatabase.kt b/src/org/thoughtcrime/securesms/loki/LokiMessageDatabase.kt index 817b76f5dd..0fe322a4f6 100644 --- a/src/org/thoughtcrime/securesms/loki/LokiMessageDatabase.kt +++ b/src/org/thoughtcrime/securesms/loki/LokiMessageDatabase.kt @@ -6,6 +6,7 @@ import org.thoughtcrime.securesms.database.Address import org.thoughtcrime.securesms.database.Database import org.thoughtcrime.securesms.database.DatabaseFactory import org.thoughtcrime.securesms.database.helpers.SQLCipherOpenHelper +import org.thoughtcrime.securesms.logging.Log import org.whispersystems.signalservice.loki.messaging.LokiMessageDatabaseProtocol import org.whispersystems.signalservice.loki.messaging.LokiMessageFriendRequestStatus diff --git a/src/org/thoughtcrime/securesms/loki/LokiPublicChatPoller.kt b/src/org/thoughtcrime/securesms/loki/LokiPublicChatPoller.kt index b1af5cc08c..e5dc2a75f8 100644 --- a/src/org/thoughtcrime/securesms/loki/LokiPublicChatPoller.kt +++ b/src/org/thoughtcrime/securesms/loki/LokiPublicChatPoller.kt @@ -246,6 +246,8 @@ class LokiPublicChatPoller(private val context: Context, private val group: Loki displayNameUpdatees = displayNameUpdatees.union(newDisplayNameUpdatees) }.successBackground { messages -> // Process messages in the background + val key = group.displayName.replace(Regex("\\s*"), "") + TextSecurePreferences.setNewMessagesNumber(this.context, key, TextSecurePreferences.getNewMessagesNumber(this.context, key) + messages.size) messages.forEach { message -> if (userDevices.contains(message.hexEncodedPublicKey)) { processOutgoingMessage(message) diff --git a/src/org/thoughtcrime/securesms/util/TextSecurePreferences.java b/src/org/thoughtcrime/securesms/util/TextSecurePreferences.java index 00f334bc1a..f9a3c174c7 100644 --- a/src/org/thoughtcrime/securesms/util/TextSecurePreferences.java +++ b/src/org/thoughtcrime/securesms/util/TextSecurePreferences.java @@ -185,6 +185,14 @@ public class TextSecurePreferences { private static final String MEDIA_KEYBOARD_MODE = "pref_media_keyboard_mode"; + public static void setNewMessagesNumber(@NonNull Context context, String key, int value) { + setIntegerPrefrence(context, key, value); + } + + public static int getNewMessagesNumber(@NonNull Context context, String key) { + return getIntegerPreference(context, key, 0); + } + public static boolean isScreenLockEnabled(@NonNull Context context) { return getBooleanPreference(context, SCREEN_LOCK, false); } From 3d574be93177d9efcdd3aaac1245d778e36fd2a7 Mon Sep 17 00:00:00 2001 From: Mikunj Date: Thu, 5 Dec 2019 10:51:54 +1100 Subject: [PATCH 2/3] Clean. Removed recipient cache as signal already had that. Removed message cache as it messes with the ui (public chat conversations pop in with all the messages) --- .../securesms/database/ThreadDatabase.java | 37 ++++--------------- .../securesms/jobs/PushDecryptJob.java | 2 +- .../securesms/loki/LokiAPIDatabase.kt | 4 +- .../securesms/loki/LokiPublicChatPoller.kt | 3 -- .../securesms/util/TextSecurePreferences.java | 8 ---- 5 files changed, 11 insertions(+), 43 deletions(-) diff --git a/src/org/thoughtcrime/securesms/database/ThreadDatabase.java b/src/org/thoughtcrime/securesms/database/ThreadDatabase.java index 49a0cc8bfb..60a304d459 100644 --- a/src/org/thoughtcrime/securesms/database/ThreadDatabase.java +++ b/src/org/thoughtcrime/securesms/database/ThreadDatabase.java @@ -62,8 +62,6 @@ public class ThreadDatabase extends Database { private static final String TAG = ThreadDatabase.class.getSimpleName(); private Map addressCache = new HashMap<>(); - private Map> messageCache = new HashMap<>(); - private Map recipientCache = new HashMap<>(); public static final String TABLE_NAME = "thread"; public static final String ID = "_id"; @@ -157,31 +155,9 @@ public class ThreadDatabase extends Database { contentValues.put(ARCHIVED, 0); } - if (!messageCache.containsKey(threadId)) { - messageCache.put(threadId, new ArrayList<>()); - } - messageCache.get(threadId).add(contentValues); - - if (!recipientCache.containsKey(threadId)) { - recipientCache.put(threadId, getRecipientForThreadId(threadId).getName().replaceAll("\\s*", "")); - } - String key = recipientCache.get(threadId); - int newMessagesNumber = TextSecurePreferences.getNewMessagesNumber(this.context, key); - if (newMessagesNumber == 0 || newMessagesNumber == messageCache.get(threadId).size()) { - SQLiteDatabase db = databaseHelper.getWritableDatabase(); - db.beginTransactionNonExclusive(); - try { - for (ContentValues contentValue : messageCache.get(threadId)) { - db.update(TABLE_NAME, contentValue, ID + " = ?", new String[] {threadId + ""}); - } - TextSecurePreferences.setNewMessagesNumber(this.context, key, TextSecurePreferences.getNewMessagesNumber(this.context, key) - newMessagesNumber); - messageCache.get(threadId).clear(); - db.setTransactionSuccessful(); - notifyConversationListListeners(); - } finally { - db.endTransaction(); - } - } + SQLiteDatabase db = databaseHelper.getWritableDatabase(); + db.update(TABLE_NAME, contentValues, ID + " = ?", new String[] {threadId + ""}); + notifyConversationListListeners(); } @@ -207,6 +183,7 @@ public class ThreadDatabase extends Database { private void deleteThread(long threadId) { SQLiteDatabase db = databaseHelper.getWritableDatabase(); db.delete(TABLE_NAME, ID_WHERE, new String[] {threadId + ""}); + addressCache.remove(threadId); notifyConversationListListeners(); } @@ -221,12 +198,16 @@ public class ThreadDatabase extends Database { where = where.substring(0, where.length() - 4); db.delete(TABLE_NAME, where, null); + for (long threadId: threadIds) { + addressCache.remove(threadId); + } notifyConversationListListeners(); } private void deleteAllThreads() { SQLiteDatabase db = databaseHelper.getWritableDatabase(); db.delete(TABLE_NAME, null, null); + addressCache.clear(); notifyConversationListListeners(); } @@ -559,8 +540,6 @@ public class ThreadDatabase extends Database { public @Nullable Recipient getRecipientForThreadId(long threadId) { // Loki - Cache the address. - // Don't know if this will affect any other signal code - // Don't know if it is necessary to add some cache time if (addressCache.containsKey(threadId) && addressCache.get(threadId) != null) { return Recipient.from(context, addressCache.get(threadId), false); } diff --git a/src/org/thoughtcrime/securesms/jobs/PushDecryptJob.java b/src/org/thoughtcrime/securesms/jobs/PushDecryptJob.java index fee75e62b5..2b29b7916b 100644 --- a/src/org/thoughtcrime/securesms/jobs/PushDecryptJob.java +++ b/src/org/thoughtcrime/securesms/jobs/PushDecryptJob.java @@ -1712,7 +1712,7 @@ public class PushDecryptJob extends BaseJob implements InjectableType { } return Recipient.from(context, Address.fromSerialized(publicKey), false); } catch (Exception e) { - Log.d("Loki", "Failed to get primary device public key for message. " + e.getMessage()); + Log.d("Loki", "Failed to get primary device public key for " + pubKey + ". " + e.getMessage()); return Recipient.from(context, Address.fromSerialized(pubKey), false); } } diff --git a/src/org/thoughtcrime/securesms/loki/LokiAPIDatabase.kt b/src/org/thoughtcrime/securesms/loki/LokiAPIDatabase.kt index 85d8f38408..91526991e5 100644 --- a/src/org/thoughtcrime/securesms/loki/LokiAPIDatabase.kt +++ b/src/org/thoughtcrime/securesms/loki/LokiAPIDatabase.kt @@ -186,12 +186,12 @@ class LokiAPIDatabase(context: Context, helper: SQLCipherOpenHelper) : Database( } override fun removePairingAuthorisations(hexEncodedPublicKey: String) { - val database = databaseHelper.readableDatabase + val database = databaseHelper.writableDatabase database.delete(pairingAuthorisationCache, "$primaryDevicePublicKey = ? OR $secondaryDevicePublicKey = ?", arrayOf( hexEncodedPublicKey, hexEncodedPublicKey )) } fun removePairingAuthorisation(primaryDevicePublicKey: String, secondaryDevicePublicKey: String) { - val database = databaseHelper.readableDatabase + val database = databaseHelper.writableDatabase database.delete(pairingAuthorisationCache, "${Companion.primaryDevicePublicKey} = ? OR ${Companion.secondaryDevicePublicKey} = ?", arrayOf( primaryDevicePublicKey, secondaryDevicePublicKey )) } } diff --git a/src/org/thoughtcrime/securesms/loki/LokiPublicChatPoller.kt b/src/org/thoughtcrime/securesms/loki/LokiPublicChatPoller.kt index e5dc2a75f8..9c21983fbe 100644 --- a/src/org/thoughtcrime/securesms/loki/LokiPublicChatPoller.kt +++ b/src/org/thoughtcrime/securesms/loki/LokiPublicChatPoller.kt @@ -10,7 +10,6 @@ import org.thoughtcrime.securesms.ApplicationContext import org.thoughtcrime.securesms.crypto.IdentityKeyUtil import org.thoughtcrime.securesms.database.Address import org.thoughtcrime.securesms.database.DatabaseFactory -import org.thoughtcrime.securesms.database.RecipientDatabase import org.thoughtcrime.securesms.jobs.PushDecryptJob import org.thoughtcrime.securesms.jobs.RetrieveProfileAvatarJob import org.thoughtcrime.securesms.recipients.Recipient @@ -246,8 +245,6 @@ class LokiPublicChatPoller(private val context: Context, private val group: Loki displayNameUpdatees = displayNameUpdatees.union(newDisplayNameUpdatees) }.successBackground { messages -> // Process messages in the background - val key = group.displayName.replace(Regex("\\s*"), "") - TextSecurePreferences.setNewMessagesNumber(this.context, key, TextSecurePreferences.getNewMessagesNumber(this.context, key) + messages.size) messages.forEach { message -> if (userDevices.contains(message.hexEncodedPublicKey)) { processOutgoingMessage(message) diff --git a/src/org/thoughtcrime/securesms/util/TextSecurePreferences.java b/src/org/thoughtcrime/securesms/util/TextSecurePreferences.java index f9a3c174c7..00f334bc1a 100644 --- a/src/org/thoughtcrime/securesms/util/TextSecurePreferences.java +++ b/src/org/thoughtcrime/securesms/util/TextSecurePreferences.java @@ -185,14 +185,6 @@ public class TextSecurePreferences { private static final String MEDIA_KEYBOARD_MODE = "pref_media_keyboard_mode"; - public static void setNewMessagesNumber(@NonNull Context context, String key, int value) { - setIntegerPrefrence(context, key, value); - } - - public static int getNewMessagesNumber(@NonNull Context context, String key) { - return getIntegerPreference(context, key, 0); - } - public static boolean isScreenLockEnabled(@NonNull Context context) { return getBooleanPreference(context, SCREEN_LOCK, false); } From 338149b7a485f3d633ca79c5825eba4bc5c1de39 Mon Sep 17 00:00:00 2001 From: Mikunj Date: Mon, 9 Dec 2019 13:55:18 +1100 Subject: [PATCH 3/3] Clean up my code. --- src/org/thoughtcrime/securesms/database/ThreadDatabase.java | 2 -- .../securesms/database/helpers/SQLCipherOpenHelper.java | 3 ++- src/org/thoughtcrime/securesms/loki/LokiMessageDatabase.kt | 1 - 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/org/thoughtcrime/securesms/database/ThreadDatabase.java b/src/org/thoughtcrime/securesms/database/ThreadDatabase.java index 60a304d459..261abbdb4f 100644 --- a/src/org/thoughtcrime/securesms/database/ThreadDatabase.java +++ b/src/org/thoughtcrime/securesms/database/ThreadDatabase.java @@ -50,7 +50,6 @@ import org.whispersystems.libsignal.util.Pair; import org.whispersystems.libsignal.util.guava.Optional; import java.io.Closeable; -import java.util.ArrayList; import java.util.HashMap; import java.util.LinkedList; import java.util.List; @@ -160,7 +159,6 @@ public class ThreadDatabase extends Database { notifyConversationListListeners(); } - public void updateSnippet(long threadId, String snippet, @Nullable Uri attachment, long date, long type, boolean unarchive) { ContentValues contentValues = new ContentValues(4); diff --git a/src/org/thoughtcrime/securesms/database/helpers/SQLCipherOpenHelper.java b/src/org/thoughtcrime/securesms/database/helpers/SQLCipherOpenHelper.java index ff11e5bbb6..2cf4269ca9 100644 --- a/src/org/thoughtcrime/securesms/database/helpers/SQLCipherOpenHelper.java +++ b/src/org/thoughtcrime/securesms/database/helpers/SQLCipherOpenHelper.java @@ -169,7 +169,8 @@ public class SQLCipherOpenHelper extends SQLiteOpenHelper { @Override public void onConfigure(SQLiteDatabase db) { super.onConfigure(db); - //Loki: Enable Write Ahead Logging Mode, increase the cache size + // Loki: Enable Write Ahead Logging Mode, increase the cache size + // This should be disabled if we ever run into serious race condition bugs db.enableWriteAheadLogging(); db.execSQL("PRAGMA cache_size = 10000"); } diff --git a/src/org/thoughtcrime/securesms/loki/LokiMessageDatabase.kt b/src/org/thoughtcrime/securesms/loki/LokiMessageDatabase.kt index 266fe1199c..f25f39e955 100644 --- a/src/org/thoughtcrime/securesms/loki/LokiMessageDatabase.kt +++ b/src/org/thoughtcrime/securesms/loki/LokiMessageDatabase.kt @@ -6,7 +6,6 @@ import org.thoughtcrime.securesms.database.Address import org.thoughtcrime.securesms.database.Database import org.thoughtcrime.securesms.database.DatabaseFactory import org.thoughtcrime.securesms.database.helpers.SQLCipherOpenHelper -import org.thoughtcrime.securesms.logging.Log import org.whispersystems.signalservice.loki.messaging.LokiMessageDatabaseProtocol import org.whispersystems.signalservice.loki.messaging.LokiMessageFriendRequestStatus