Fix backup/import issue with expiring messages.

There was an issue where we were backing up group receipts and attachments
that were for expiring messages (which are already excluded from the backup).

This commit excludes these items from the backup, and for backups made
before this change, this commit also deletes these invalid entries at
the end of the restore process.

We also do a little database migration to cleanup any bad state that may
have been imported in the past.

src/org/thoughtcrime/securesms/DatabaseUpgradeActivity.java

@@ -82,6 +82,7 @@ public class DatabaseUpgradeActivity extends BaseActivity {
   public static final int REMOVE_JOURNAL                       = 353;
   public static final int REMOVE_CACHE                         = 354;
   public static final int FULL_TEXT_SEARCH                     = 358;
+  public static final int BAD_IMPORT_CLEANUP                   = 373;
 
   private static final SortedSet<Integer> UPGRADE_VERSIONS = new TreeSet<Integer>() {{
     add(NO_MORE_KEY_EXCHANGE_PREFIX_VERSION);
@@ -103,6 +104,7 @@ public class DatabaseUpgradeActivity extends BaseActivity {
     add(SQLCIPHER_COMPLETE);
     add(REMOVE_CACHE);
     add(FULL_TEXT_SEARCH);
+    add(BAD_IMPORT_CLEANUP);
   }};
 
   private MasterSecret masterSecret;

src/org/thoughtcrime/securesms/backup/FullBackupExporter.java

@@ -21,6 +21,7 @@ import org.thoughtcrime.securesms.crypto.ClassicDecryptingPartInputStream;
 import org.thoughtcrime.securesms.crypto.IdentityKeyUtil;
 import org.thoughtcrime.securesms.crypto.ModernDecryptingPartInputStream;
 import org.thoughtcrime.securesms.database.AttachmentDatabase;
+import org.thoughtcrime.securesms.database.GroupReceiptDatabase;
 import org.thoughtcrime.securesms.database.MmsDatabase;
 import org.thoughtcrime.securesms.database.MmsSmsColumns;
 import org.thoughtcrime.securesms.database.OneTimePreKeyDatabase;
@@ -75,8 +76,10 @@ public class FullBackupExporter extends FullBackupBase {
     for (String table : tables) {
       if (table.equals(SmsDatabase.TABLE_NAME) || table.equals(MmsDatabase.TABLE_NAME)) {
         count = exportTable(table, input, outputStream, cursor -> cursor.getInt(cursor.getColumnIndexOrThrow(MmsSmsColumns.EXPIRES_IN)) <= 0, null, count);
+      } else if (table.equals(GroupReceiptDatabase.TABLE_NAME)) {
+        count = exportTable(table, input, outputStream, cursor -> isForNonExpiringMessage(input, cursor.getLong(cursor.getColumnIndexOrThrow(GroupReceiptDatabase.MMS_ID))), null, count);
       } else if (table.equals(AttachmentDatabase.TABLE_NAME)) {
-        count = exportTable(table, input, outputStream, null, cursor -> exportAttachment(attachmentSecret, cursor, outputStream), count);
+        count = exportTable(table, input, outputStream, cursor -> isForNonExpiringMessage(input, cursor.getLong(cursor.getColumnIndexOrThrow(AttachmentDatabase.MMS_ID))), cursor -> exportAttachment(attachmentSecret, cursor, outputStream), count);
       } else if (!table.equals(SignedPreKeyDatabase.TABLE_NAME)       &&
                  !table.equals(OneTimePreKeyDatabase.TABLE_NAME)      &&
                  !table.equals(SessionDatabase.TABLE_NAME)            &&
@@ -229,6 +232,21 @@ public class FullBackupExporter extends FullBackupBase {
     return result;
   }
 
+  private static boolean isForNonExpiringMessage(@NonNull SQLiteDatabase db, long mmsId) {
+    String[] columns = new String[] { MmsDatabase.EXPIRES_IN };
+    String   where   = MmsDatabase.ID + " = ?";
+    String[] args    = new String[] { String.valueOf(mmsId) };
+
+    try (Cursor mmsCursor = db.query(MmsDatabase.TABLE_NAME, columns, where, args, null, null, null)) {
+      if (mmsCursor != null && mmsCursor.moveToFirst()) {
+        return mmsCursor.getLong(0) == 0;
+      }
+    }
+
+    return false;
+  }
+
+
   private static class BackupFrameOutputStream extends BackupStream {
 
     private final OutputStream outputStream;
@@ -358,9 +376,9 @@ public class FullBackupExporter extends FullBackupBase {
       }
     }
 
+
     public void close() throws IOException {
       outputStream.close();
     }
-
   }
 }

src/org/thoughtcrime/securesms/backup/FullBackupImporter.java

@@ -13,6 +13,7 @@ import android.util.Pair;
 import net.sqlcipher.database.SQLiteDatabase;
 
 import org.greenrobot.eventbus.EventBus;
+import org.thoughtcrime.securesms.attachments.AttachmentId;
 import org.thoughtcrime.securesms.backup.BackupProtos.Attachment;
 import org.thoughtcrime.securesms.backup.BackupProtos.BackupFrame;
 import org.thoughtcrime.securesms.backup.BackupProtos.DatabaseVersion;
@@ -22,7 +23,12 @@ import org.thoughtcrime.securesms.crypto.AttachmentSecret;
 import org.thoughtcrime.securesms.crypto.ModernEncryptingPartOutputStream;
 import org.thoughtcrime.securesms.database.Address;
 import org.thoughtcrime.securesms.database.AttachmentDatabase;
+import org.thoughtcrime.securesms.database.DatabaseFactory;
+import org.thoughtcrime.securesms.database.GroupReceiptDatabase;
+import org.thoughtcrime.securesms.database.MmsDatabase;
+import org.thoughtcrime.securesms.database.MmsSmsColumns;
 import org.thoughtcrime.securesms.database.SearchDatabase;
+import org.thoughtcrime.securesms.database.ThreadDatabase;
 import org.thoughtcrime.securesms.profiles.AvatarHelper;
 import org.thoughtcrime.securesms.util.Conversions;
 import org.thoughtcrime.securesms.util.Util;
@@ -80,6 +86,8 @@ public class FullBackupImporter extends FullBackupBase {
         else if (frame.hasAvatar())     processAvatar(context, frame.getAvatar(), inputStream);
       }
 
+      trimEntriesForExpiredMessages(context, db);
+
       db.setTransactionSuccessful();
     } finally {
       db.endTransaction();
@@ -158,6 +166,27 @@ public class FullBackupImporter extends FullBackupBase {
     }
   }
 
+  private static void trimEntriesForExpiredMessages(@NonNull Context context, @NonNull SQLiteDatabase db) {
+    String trimmedCondition = " NOT IN (SELECT " + MmsDatabase.ID + " FROM " + MmsDatabase.TABLE_NAME + ")";
+
+    db.delete(GroupReceiptDatabase.TABLE_NAME, GroupReceiptDatabase.MMS_ID + trimmedCondition, null);
+
+    String[] columns = new String[] { AttachmentDatabase.ROW_ID, AttachmentDatabase.UNIQUE_ID };
+    String   where   = AttachmentDatabase.MMS_ID + trimmedCondition;
+
+    try (Cursor cursor = db.query(AttachmentDatabase.TABLE_NAME, columns, where, null, null, null, null)) {
+      while (cursor != null && cursor.moveToNext()) {
+        DatabaseFactory.getAttachmentDatabase(context).deleteAttachment(new AttachmentId(cursor.getLong(0), cursor.getLong(1)));
+      }
+    }
+
+    try (Cursor cursor = db.query(ThreadDatabase.TABLE_NAME, new String[] { ThreadDatabase.ID }, ThreadDatabase.EXPIRES_IN + " > 0", null, null, null, null)) {
+      while (cursor != null && cursor.moveToNext()) {
+        DatabaseFactory.getThreadDatabase(context).update(cursor.getLong(0), false);
+      }
+    }
+  }
+
   private static class BackupRecordInputStream extends BackupStream {
 
     private final InputStream in;

src/org/thoughtcrime/securesms/database/AttachmentDatabase.java

@@ -76,7 +76,7 @@ public class AttachmentDatabase extends Database {
   public  static final String TABLE_NAME             = "part";
   public  static final String ROW_ID                 = "_id";
           static final String ATTACHMENT_JSON_ALIAS  = "attachment_json";
-          static final String MMS_ID                 = "mid";
+  public  static final String MMS_ID                 = "mid";
           static final String CONTENT_TYPE           = "ct";
           static final String NAME                   = "name";
           static final String CONTENT_DISPOSITION    = "cd";

src/org/thoughtcrime/securesms/database/GroupReceiptDatabase.java

@@ -18,7 +18,7 @@ public class GroupReceiptDatabase extends Database {
   public  static final String TABLE_NAME = "group_receipts";
 
   private static final String ID        = "_id";
-  private static final String MMS_ID    = "mms_id";
+  public  static final String MMS_ID    = "mms_id";
   private static final String ADDRESS   = "address";
   private static final String STATUS    = "status";
   private static final String TIMESTAMP = "timestamp";

src/org/thoughtcrime/securesms/database/ThreadDatabase.java

@@ -55,7 +55,7 @@ public class ThreadDatabase extends Database {
 
   private static final String TAG = ThreadDatabase.class.getSimpleName();
 
-          static final String TABLE_NAME             = "thread";
+  public  static final String TABLE_NAME             = "thread";
   public  static final String ID                     = "_id";
   public  static final String DATE                   = "date";
   public  static final String MESSAGE_COUNT          = "message_count";

src/org/thoughtcrime/securesms/database/helpers/SQLCipherOpenHelper.java

@@ -6,6 +6,7 @@ import android.content.Context;
 import android.database.Cursor;
 import android.os.SystemClock;
 import android.support.annotation.NonNull;
+import android.text.TextUtils;
 import android.util.Log;
 
 import net.sqlcipher.database.SQLiteDatabase;
@@ -48,8 +49,9 @@ public class SQLCipherOpenHelper extends SQLiteOpenHelper {
   private static final int QUOTED_REPLIES                   = 7;
   private static final int SHARED_CONTACTS                  = 8;
   private static final int FULL_TEXT_SEARCH                 = 9;
+  private static final int BAD_IMPORT_CLEANUP               = 10;
 
-  private static final int    DATABASE_VERSION = 9;
+  private static final int    DATABASE_VERSION = 10;
   private static final String DATABASE_NAME    = "signal.db";
 
   private final Context        context;
@@ -210,6 +212,31 @@ public class SQLCipherOpenHelper extends SQLiteOpenHelper {
         Log.i(TAG, "Indexing finished. Total time: " + (mmsFinished - start) + " ms");
       }
 
+      if (oldVersion < BAD_IMPORT_CLEANUP) {
+        String trimmedCondition = " NOT IN (SELECT _id FROM mms)";
+
+        db.delete("group_receipts", "mms_id" + trimmedCondition, null);
+
+        String[] columns = new String[] { "_id", "unique_id", "_data", "thumbnail"};
+
+        try (Cursor cursor = db.query("part", columns, "mid" + trimmedCondition, null, null, null, null)) {
+          while (cursor != null && cursor.moveToNext()) {
+            db.delete("part", "_id = ? AND unique_id = ?", new String[] { String.valueOf(cursor.getLong(0)), String.valueOf(cursor.getLong(1)) });
+
+            String data      = cursor.getString(2);
+            String thumbnail = cursor.getString(3);
+
+            if (!TextUtils.isEmpty(data)) {
+              new File(data).delete();
+            }
+
+            if (!TextUtils.isEmpty(thumbnail)) {
+              new File(thumbnail).delete();
+            }
+          }
+        }
+      }
+
       db.setTransactionSuccessful();
     } finally {
       db.endTransaction();