Implemented full-text search.

You can now use the search bar on the conversation list to find
conversations, messages, and contacts.

src/org/thoughtcrime/securesms/database/CursorList.java

@@ -0,0 +1,191 @@
+package org.thoughtcrime.securesms.database;
+
+import android.database.Cursor;
+import android.database.MatrixCursor;
+import android.support.annotation.NonNull;
+
+import java.io.Closeable;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.ListIterator;
+
+/**
+ * A list backed by a {@link Cursor} that retrieves models using a provided {@link ModelBuilder}.
+ * Allows you to abstract away the use of a {@link Cursor} while still getting the benefits of a
+ * {@link Cursor} (e.g. windowing).
+ *
+ * The one special consideration that must be made is that because this contains a cursor, you must
+ * call {@link #close()} when you are finished with it.
+ *
+ * Given that this is cursor-backed, it is effectively immutable.
+ */
+public class CursorList<T> implements List<T>, Closeable {
+
+  private static final Cursor EMPTY_CURSOR = new MatrixCursor(new String[] { "a" }, 0);
+
+  private final Cursor          cursor;
+  private final ModelBuilder<T> modelBuilder;
+
+  public CursorList(@NonNull Cursor cursor, @NonNull ModelBuilder<T> modelBuilder) {
+    this.cursor       = cursor;
+    this.modelBuilder = modelBuilder;
+
+    this.cursor.moveToFirst();
+  }
+
+  public static <T> CursorList<T> emptyList() {
+    //noinspection ConstantConditions,unchecked
+    return (CursorList<T>) new CursorList(EMPTY_CURSOR, null);
+  }
+
+  @Override
+  public int size() {
+    return cursor.getCount();
+  }
+
+  @Override
+  public boolean isEmpty() {
+    return size() == 0;
+  }
+
+  @Override
+  public boolean contains(Object o) {
+    throw new UnsupportedOperationException();
+  }
+
+  @NonNull
+  @Override
+  public Iterator<T> iterator() {
+    return new Iterator<T>() {
+      @Override
+      public boolean hasNext() {
+        return cursor.getCount() > 0 && !cursor.isLast();
+      }
+
+      @Override
+      public T next() {
+        T model = modelBuilder.build(cursor);
+        cursor.moveToNext();
+        return model;
+      }
+    };
+  }
+
+  @NonNull
+  @Override
+  public Object[] toArray() {
+    Object[] out = new Object[size()];
+    for (int i = 0; i < cursor.getCount(); i++) {
+      cursor.moveToPosition(i);
+      out[i] = modelBuilder.build(cursor);
+    }
+    return out;
+  }
+
+  @Override
+  public boolean add(T o) {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public boolean remove(Object o) {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public boolean addAll(@NonNull Collection collection) {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public boolean addAll(int i, @NonNull Collection collection) {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public void clear() {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public T get(int i) {
+    cursor.moveToPosition(i);
+    return modelBuilder.build(cursor);
+  }
+
+  @Override
+  public T set(int i, T o) {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public void add(int i, T o) {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public T remove(int i) {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public int indexOf(Object o) {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public int lastIndexOf(Object o) {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public ListIterator<T> listIterator() {
+    throw new UnsupportedOperationException();
+  }
+
+  @NonNull
+  @Override
+  public ListIterator<T> listIterator(int i) {
+    throw new UnsupportedOperationException();
+  }
+
+  @NonNull
+  @Override
+  public List<T> subList(int i, int i1) {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public boolean retainAll(@NonNull Collection collection) {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public boolean removeAll(@NonNull Collection collection) {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public boolean containsAll(@NonNull Collection collection) {
+    throw new UnsupportedOperationException();
+  }
+
+  @NonNull
+  @Override
+  public T[] toArray(@NonNull Object[] objects) {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public void close() {
+    if (cursor != null) {
+      cursor.close();
+    }
+  }
+
+  public interface ModelBuilder<T> {
+    T build(@NonNull Cursor cursor);
+  }
+}
+

src/org/thoughtcrime/securesms/database/DatabaseFactory.java

@@ -56,6 +56,7 @@ public class DatabaseFactory {
   private final OneTimePreKeyDatabase preKeyDatabase;
   private final SignedPreKeyDatabase  signedPreKeyDatabase;
   private final SessionDatabase       sessionDatabase;
+  private final SearchDatabase        searchDatabase;
 
   public static DatabaseFactory getInstance(Context context) {
     synchronized (lock) {
@@ -130,6 +131,10 @@ public class DatabaseFactory {
     return getInstance(context).sessionDatabase;
   }
 
+  public static SearchDatabase getSearchDatabase(Context context) {
+    return getInstance(context).searchDatabase;
+  }
+
   public static SQLiteDatabase getBackupDatabase(Context context) {
     return getInstance(context).databaseHelper.getReadableDatabase();
   }
@@ -162,6 +167,7 @@ public class DatabaseFactory {
     this.preKeyDatabase       = new OneTimePreKeyDatabase(context, databaseHelper);
     this.signedPreKeyDatabase = new SignedPreKeyDatabase(context, databaseHelper);
     this.sessionDatabase      = new SessionDatabase(context, databaseHelper);
+    this.searchDatabase       = new SearchDatabase(context, databaseHelper);
   }
 
   public void onApplicationLevelUpgrade(@NonNull Context context, @NonNull MasterSecret masterSecret,

src/org/thoughtcrime/securesms/database/MmsSmsDatabase.java

@@ -93,18 +93,19 @@ public class MmsSmsDatabase extends Database {
     return null;
   }
 
-  public Cursor getConversation(long threadId, long limit) {
+  public Cursor getConversation(long threadId, long offset, long limit) {
     String order     = MmsSmsColumns.NORMALIZED_DATE_RECEIVED + " DESC";
     String selection = MmsSmsColumns.THREAD_ID + " = " + threadId;
+    String limitStr  = limit > 0 || offset > 0 ? offset + ", " + limit : null;
 
-    Cursor cursor = queryTables(PROJECTION, selection, order, limit > 0 ? String.valueOf(limit) : null);
+    Cursor cursor = queryTables(PROJECTION, selection, order, limitStr);
     setNotifyConverationListeners(cursor, threadId);
 
     return cursor;
   }
 
   public Cursor getConversation(long threadId) {
-    return getConversation(threadId, 0);
+    return getConversation(threadId, 0, 0);
   }
 
   public Cursor getIdentityConflictMessagesForThread(long threadId) {
@@ -179,6 +180,26 @@ public class MmsSmsDatabase extends Database {
     return -1;
   }
 
+  /**
+   * Retrieves the position of the message with the provided timestamp in the query results you'd
+   * get from calling {@link #getConversation(long)}.
+   *
+   * Note: This could give back incorrect results in the situation where multiple messages have the
+   * same received timestamp. However, because this was designed to determine where to scroll to,
+   * you'll still wind up in about the right spot.
+   */
+  public int getMessagePositionInConversation(long threadId, long receivedTimestamp) {
+    String order     = MmsSmsColumns.NORMALIZED_DATE_RECEIVED + " DESC";
+    String selection = MmsSmsColumns.THREAD_ID + " = " + threadId + " AND " + MmsSmsColumns.NORMALIZED_DATE_RECEIVED + " > " + receivedTimestamp;
+
+    try (Cursor cursor = queryTables(new String[]{ "COUNT(*)" }, selection, order, null)) {
+      if (cursor != null && cursor.moveToFirst()) {
+        return cursor.getInt(0);
+      }
+    }
+    return -1;
+  }
+
   private Cursor queryTables(String[] projection, String selection, String order, String limit) {
     String[] mmsProjection = {MmsDatabase.DATE_SENT + " AS " + MmsSmsColumns.NORMALIZED_DATE_SENT,
                               MmsDatabase.DATE_RECEIVED + " AS " + MmsSmsColumns.NORMALIZED_DATE_RECEIVED,

src/org/thoughtcrime/securesms/database/SearchDatabase.java

@@ -0,0 +1,89 @@
+package org.thoughtcrime.securesms.database;
+
+import android.content.Context;
+import android.support.annotation.NonNull;
+import android.util.Log;
+
+import net.sqlcipher.Cursor;
+import net.sqlcipher.database.SQLiteDatabase;
+
+import org.thoughtcrime.securesms.database.helpers.SQLCipherOpenHelper;
+
+/**
+ * Contains all databases necessary for full-text search (FTS).
+ */
+public class SearchDatabase extends Database {
+
+  public static final String SMS_FTS_TABLE_NAME = "sms_fts";
+  public static final String MMS_FTS_TABLE_NAME = "mms_fts";
+
+  public static final String ID      = "rowid";
+  public static final String BODY    = MmsSmsColumns.BODY;
+  public static final String RANK    = "rank";
+  public static final String SNIPPET = "snippet";
+
+  public static final String[] CREATE_TABLE = {
+      "CREATE VIRTUAL TABLE " + SMS_FTS_TABLE_NAME + " USING fts5(" + BODY + ", content=" + SmsDatabase.TABLE_NAME + ", content_rowid=" + SmsDatabase.ID + ");",
+
+      "CREATE TRIGGER sms_ai AFTER INSERT ON " + SmsDatabase.TABLE_NAME + " BEGIN\n" +
+          "  INSERT INTO " + SMS_FTS_TABLE_NAME + "(" + ID + ", " + BODY + ") VALUES (new." + SmsDatabase.ID + ", new." + SmsDatabase.BODY + ");\n" +
+          "END;\n",
+      "CREATE TRIGGER sms_ad AFTER DELETE ON " + SmsDatabase.TABLE_NAME + " BEGIN\n" +
+          "  INSERT INTO " + SMS_FTS_TABLE_NAME + "(" + SMS_FTS_TABLE_NAME + ", " + ID + ", " + BODY + ") VALUES('delete', old." + SmsDatabase.ID + ", old." + SmsDatabase.BODY + ");\n" +
+          "END;\n",
+      "CREATE TRIGGER sms_au AFTER UPDATE ON " + SmsDatabase.TABLE_NAME + " BEGIN\n" +
+          "  INSERT INTO " + SMS_FTS_TABLE_NAME + "(" + SMS_FTS_TABLE_NAME + ", " + ID + ", " + BODY + ") VALUES('delete', old." + SmsDatabase.ID + ", old." + SmsDatabase.BODY + ");\n" +
+          "  INSERT INTO " + SMS_FTS_TABLE_NAME + "(" + ID + ", " + BODY + ") VALUES(new." + SmsDatabase.ID + ", new." + SmsDatabase.BODY + ");\n" +
+          "END;",
+
+
+      "CREATE VIRTUAL TABLE " + MMS_FTS_TABLE_NAME + " USING fts5(" + BODY + ", content=" + MmsDatabase.TABLE_NAME + ", content_rowid=" + MmsDatabase.ID + ");",
+
+      "CREATE TRIGGER mms_ai AFTER INSERT ON " + MmsDatabase.TABLE_NAME + " BEGIN\n" +
+          "  INSERT INTO " + MMS_FTS_TABLE_NAME + "(" + ID + ", " + BODY + ") VALUES (new." + MmsDatabase.ID + ", new." + MmsDatabase.BODY + ");\n" +
+          "END;\n",
+      "CREATE TRIGGER mms_ad AFTER DELETE ON " + MmsDatabase.TABLE_NAME + " BEGIN\n" +
+          "  INSERT INTO " + MMS_FTS_TABLE_NAME + "(" + MMS_FTS_TABLE_NAME + ", " + ID + ", " + BODY + ") VALUES('delete', old." + MmsDatabase.ID + ", old." + MmsDatabase.BODY + ");\n" +
+          "END;\n",
+      "CREATE TRIGGER mms_au AFTER UPDATE ON " + MmsDatabase.TABLE_NAME + " BEGIN\n" +
+          "  INSERT INTO " + MMS_FTS_TABLE_NAME + "(" + MMS_FTS_TABLE_NAME + ", " + ID + ", " + BODY + ") VALUES('delete', old." + MmsDatabase.ID + ", old." + MmsDatabase.BODY + ");\n" +
+          "  INSERT INTO " + MMS_FTS_TABLE_NAME + "(" + ID + ", " + BODY + ") VALUES (new." + MmsDatabase.ID + ", new." + MmsDatabase.BODY + ");\n" +
+          "END;"
+  };
+
+  private static final String MESSAGES_QUERY =
+      "SELECT " +
+        MmsSmsColumns.ADDRESS + ", " +
+        "snippet(" + SMS_FTS_TABLE_NAME + ", -1, '', '', '...', 7) AS " + SNIPPET + ", " +
+        SmsDatabase.DATE_RECEIVED + " AS " + MmsSmsColumns.NORMALIZED_DATE_RECEIVED + ", " +
+        MmsSmsColumns.THREAD_ID + ", " +
+        "bm25(" + SMS_FTS_TABLE_NAME + ") AS " + RANK + " " +
+      "FROM " + SmsDatabase.TABLE_NAME + " " +
+      "INNER JOIN " + SMS_FTS_TABLE_NAME + " ON " + SMS_FTS_TABLE_NAME + "." + ID + " = " + SmsDatabase.TABLE_NAME + "." + SmsDatabase.ID + " " +
+      "WHERE " + SMS_FTS_TABLE_NAME + " MATCH ? " +
+      "UNION ALL " +
+      "SELECT " +
+        MmsSmsColumns.ADDRESS + ", " +
+        "snippet(" + MMS_FTS_TABLE_NAME + ", -1, '', '', '...', 7) AS " + SNIPPET + ", " +
+        MmsDatabase.DATE_RECEIVED + " AS " + MmsSmsColumns.NORMALIZED_DATE_RECEIVED + ", " +
+        MmsSmsColumns.THREAD_ID + ", " +
+        "bm25(" + MMS_FTS_TABLE_NAME + ") AS " + RANK + " " +
+      "FROM " + MmsDatabase.TABLE_NAME + " " +
+      "INNER JOIN " + MMS_FTS_TABLE_NAME + " ON " + MMS_FTS_TABLE_NAME + "." + ID + " = " + MmsDatabase.TABLE_NAME + "." + MmsDatabase.ID + " " +
+      "WHERE " + MMS_FTS_TABLE_NAME + " MATCH ? " +
+      "ORDER BY rank " +
+      "LIMIT 500";
+
+  public SearchDatabase(@NonNull Context context, @NonNull SQLCipherOpenHelper databaseHelper) {
+    super(context, databaseHelper);
+  }
+
+  public Cursor queryMessages(@NonNull String query) {
+    SQLiteDatabase db = databaseHelper.getReadableDatabase();
+
+    String prefixQuery = query + '*';
+
+    return db.rawQuery(MESSAGES_QUERY, new String[] { prefixQuery, prefixQuery });
+  }
+}
+

src/org/thoughtcrime/securesms/database/ThreadDatabase.java

@@ -46,6 +46,7 @@ import org.thoughtcrime.securesms.util.Util;
 import org.whispersystems.libsignal.util.Pair;
 import org.whispersystems.libsignal.util.guava.Optional;
 
+import java.io.Closeable;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Set;
@@ -625,7 +626,7 @@ public class ThreadDatabase extends Database {
     public static final int INBOX_ZERO   = 4;
   }
 
-  public class Reader {
+  public class Reader implements Closeable {
 
     private final Cursor cursor;
 
@@ -692,8 +693,11 @@ public class ThreadDatabase extends Database {
       }
     }
 
+    @Override
     public void close() {
-      cursor.close();
+      if (cursor != null) {
+        cursor.close();
+      }
     }
   }
 }

src/org/thoughtcrime/securesms/database/helpers/SQLCipherOpenHelper.java

@@ -4,6 +4,7 @@ package org.thoughtcrime.securesms.database.helpers;
 import android.content.ContentValues;
 import android.content.Context;
 import android.database.Cursor;
+import android.os.SystemClock;
 import android.support.annotation.NonNull;
 import android.util.Log;
 
@@ -23,6 +24,7 @@ import org.thoughtcrime.securesms.database.MmsDatabase;
 import org.thoughtcrime.securesms.database.OneTimePreKeyDatabase;
 import org.thoughtcrime.securesms.database.PushDatabase;
 import org.thoughtcrime.securesms.database.RecipientDatabase;
+import org.thoughtcrime.securesms.database.SearchDatabase;
 import org.thoughtcrime.securesms.database.SessionDatabase;
 import org.thoughtcrime.securesms.database.SignedPreKeyDatabase;
 import org.thoughtcrime.securesms.database.SmsDatabase;
@@ -45,8 +47,9 @@ public class SQLCipherOpenHelper extends SQLiteOpenHelper {
   private static final int ATTACHMENT_DIMENSIONS            = 6;
   private static final int QUOTED_REPLIES                   = 7;
   private static final int SHARED_CONTACTS                  = 8;
+  private static final int FULL_TEXT_SEARCH                 = 9;
 
-  private static final int    DATABASE_VERSION = 8;
+  private static final int    DATABASE_VERSION = 9;
   private static final String DATABASE_NAME    = "signal.db";
 
   private final Context        context;
@@ -86,6 +89,9 @@ public class SQLCipherOpenHelper extends SQLiteOpenHelper {
     db.execSQL(OneTimePreKeyDatabase.CREATE_TABLE);
     db.execSQL(SignedPreKeyDatabase.CREATE_TABLE);
     db.execSQL(SessionDatabase.CREATE_TABLE);
+    for (String sql : SearchDatabase.CREATE_TABLE) {
+      db.execSQL(sql);
+    }
 
     executeStatements(db, SmsDatabase.CREATE_INDEXS);
     executeStatements(db, MmsDatabase.CREATE_INDEXS);
@@ -182,6 +188,28 @@ public class SQLCipherOpenHelper extends SQLiteOpenHelper {
         db.execSQL("ALTER TABLE mms ADD COLUMN shared_contacts TEXT");
       }
 
+      if (oldVersion < FULL_TEXT_SEARCH) {
+        for (String sql : SearchDatabase.CREATE_TABLE) {
+          db.execSQL(sql);
+        }
+
+        Log.i(TAG, "Beginning to build search index.");
+        long start = SystemClock.elapsedRealtime();
+
+        db.execSQL("INSERT INTO " + SearchDatabase.SMS_FTS_TABLE_NAME + " (rowid, " + SearchDatabase.BODY + ") " +
+            "SELECT " + SmsDatabase.ID + " , " + SmsDatabase.BODY + " FROM " + SmsDatabase.TABLE_NAME);
+
+        long smsFinished = SystemClock.elapsedRealtime();
+        Log.i(TAG, "Indexing SMS completed in " + (smsFinished - start) + " ms");
+
+        db.execSQL("INSERT INTO " + SearchDatabase.MMS_FTS_TABLE_NAME + " (rowid, " + SearchDatabase.BODY + ") " +
+            "SELECT " + MmsDatabase.ID + " , " + MmsDatabase.BODY + " FROM " + MmsDatabase.TABLE_NAME);
+
+        long mmsFinished = SystemClock.elapsedRealtime();
+        Log.i(TAG, "Indexing MMS completed in " + (mmsFinished - smsFinished) + " ms");
+        Log.i(TAG, "Indexing finished. Total time: " + (mmsFinished - start) + " ms");
+      }
+
       db.setTransactionSuccessful();
     } finally {
       db.endTransaction();

src/org/thoughtcrime/securesms/database/loaders/ConversationLoader.java

@@ -9,13 +9,15 @@ import org.whispersystems.libsignal.util.Pair;
 
 public class ConversationLoader extends AbstractCursorLoader {
   private final long    threadId;
-  private       long    limit;
+  private       int     offset;
+  private       int     limit;
   private       long    lastSeen;
   private       boolean hasSent;
 
-  public ConversationLoader(Context context, long threadId, long limit, long lastSeen) {
+  public ConversationLoader(Context context, long threadId, int offset, int limit, long lastSeen) {
     super(context);
     this.threadId = threadId;
+    this.offset   = offset;
     this.limit    = limit;
     this.lastSeen = lastSeen;
     this.hasSent  = true;
@@ -25,6 +27,14 @@ public class ConversationLoader extends AbstractCursorLoader {
     return limit > 0;
   }
 
+  public boolean hasOffset() {
+    return offset > 0;
+  }
+
+  public int getOffset() {
+    return offset;
+  }
+
   public long getLastSeen() {
     return lastSeen;
   }
@@ -43,6 +53,6 @@ public class ConversationLoader extends AbstractCursorLoader {
       this.lastSeen = lastSeenAndHasSent.first();
     }
 
-    return DatabaseFactory.getMmsSmsDatabase(context).getConversation(threadId, limit);
+    return DatabaseFactory.getMmsSmsDatabase(context).getConversation(threadId, offset, limit);
   }
 }