thumbnail generation and disk caching

// FREEBIE

res/values/dimens.xml

@@ -12,4 +12,5 @@
     <dimen name="conversation_item_corner_radius">3dp</dimen>
     <dimen name="conversation_item_drop_shadow_dist">2dp</dimen>
     <dimen name="contact_selection_photo_size">50dp</dimen>
+    <dimen name="thumbnail_max_size">230dp</dimen>
 </resources>

src/org/thoughtcrime/securesms/database/DatabaseFactory.java

@@ -58,8 +58,8 @@ public class DatabaseFactory {
   private static final int INTRODUCED_PUSH_FIX_VERSION       = 12;
   private static final int INTRODUCED_DELIVERY_RECEIPTS      = 13;
   private static final int INTRODUCED_PART_DATA_SIZE_VERSION = 14;
-  private static final int DATABASE_VERSION                  = 14;
+  private static final int INTRODUCED_THUMBNAILS_VERSION     = 15;
-
+  private static final int DATABASE_VERSION                  = 15;
 
   private static final String DATABASE_NAME    = "messages.db";
   private static final Object lock             = new Object();
@@ -705,6 +705,11 @@ public class DatabaseFactory {
         db.execSQL("ALTER TABLE part ADD COLUMN data_size INTEGER DEFAULT 0;");
       }
 
+      if (oldVersion < INTRODUCED_THUMBNAILS_VERSION) {
+        db.execSQL("ALTER TABLE part ADD COLUMN thumbnail TEXT");
+        db.execSQL("ALTER TABLE part ADD COLUMN aspect_ratio REAL");
+      }
+
       db.setTransactionSuccessful();
       db.endTransaction();
     }

src/org/thoughtcrime/securesms/database/PartDatabase.java

@@ -26,9 +26,11 @@ import android.text.TextUtils;
 import android.util.Log;
 import android.util.Pair;
 
+import org.thoughtcrime.securesms.ApplicationContext;
 import org.thoughtcrime.securesms.crypto.DecryptingPartInputStream;
 import org.thoughtcrime.securesms.crypto.EncryptingPartOutputStream;
 import org.thoughtcrime.securesms.crypto.MasterSecret;
+import org.thoughtcrime.securesms.jobs.ThumbnailGenerateJob;
 import org.thoughtcrime.securesms.mms.PartAuthority;
 import org.thoughtcrime.securesms.util.Util;
 
@@ -66,6 +68,8 @@ public class PartDatabase extends Database {
   private static final String DATA                    = "_data";
   private static final String PENDING_PUSH_ATTACHMENT = "pending_push";
   private static final String SIZE                    = "data_size";
+  private static final String THUMBNAIL               = "thumbnail";
+  private static final String ASPECT_RATIO            = "aspect_ratio";
 
   public static final String CREATE_TABLE = "CREATE TABLE " + TABLE_NAME + " (" + ID + " INTEGER PRIMARY KEY, " +
     MMS_ID + " INTEGER, " + SEQUENCE + " INTEGER DEFAULT 0, "                        +
@@ -73,7 +77,8 @@ public class PartDatabase extends Database {
     CONTENT_DISPOSITION + " TEXT, " + FILENAME + " TEXT, " + CONTENT_ID + " TEXT, "  +
     CONTENT_LOCATION + " TEXT, " + CONTENT_TYPE_START + " INTEGER, "                 +
     CONTENT_TYPE_TYPE + " TEXT, " + ENCRYPTED + " INTEGER, "                         +
-    PENDING_PUSH_ATTACHMENT + " INTEGER, "+ DATA + " TEXT, " + SIZE + " INTEGER);";
+    PENDING_PUSH_ATTACHMENT + " INTEGER, "+ DATA + " TEXT, " + SIZE + " INTEGER, "   +
+    THUMBNAIL + " TEXT, " + ASPECT_RATIO + " REAL);";
 
   public static final String[] CREATE_INDEXS = {
     "CREATE INDEX IF NOT EXISTS part_mms_id_index ON " + TABLE_NAME + " (" + MMS_ID + ");",
@@ -90,6 +95,12 @@ public class PartDatabase extends Database {
     return getDataStream(masterSecret, partId, DATA);
   }
 
+  public InputStream getThumbnailStream(MasterSecret masterSecret, long partId)
+      throws FileNotFoundException
+  {
+    return getDataStream(masterSecret, partId, THUMBNAIL);
+  }
+
   public void updateFailedDownloadedPart(long messageId, long partId, PduPart part)
       throws MmsException
   {
@@ -144,20 +155,26 @@ public class PartDatabase extends Database {
     }
   }
 
+  @SuppressWarnings("ResultOfMethodCallIgnored")
   public void deleteParts(long mmsId) {
     SQLiteDatabase database = databaseHelper.getWritableDatabase();
     Cursor cursor           = null;
 
     try {
-      cursor = database.query(TABLE_NAME, new String[] {DATA}, MMS_ID + " = ?",
+      cursor = database.query(TABLE_NAME, new String[] {DATA, THUMBNAIL}, MMS_ID + " = ?",
                               new String[] {mmsId+""}, null, null, null);
 
       while (cursor != null && cursor.moveToNext()) {
         String data      = cursor.getString(0);
+        String thumbnail = cursor.getString(1);
 
         if (!TextUtils.isEmpty(data)) {
           new File(data).delete();
         }
+
+        if (!TextUtils.isEmpty(thumbnail)) {
+          new File(thumbnail).delete();
+        }
       }
     } finally {
       if (cursor != null)
@@ -167,6 +184,7 @@ public class PartDatabase extends Database {
     database.delete(TABLE_NAME, MMS_ID + " = ?", new String[] {mmsId+""});
   }
 
+  @SuppressWarnings("ResultOfMethodCallIgnored")
   public void deleteAllParts() {
     SQLiteDatabase database = databaseHelper.getWritableDatabase();
     database.delete(TABLE_NAME, null, null);
@@ -232,6 +250,12 @@ public class PartDatabase extends Database {
     if (!cursor.isNull(pendingPushColumn))
       part.setPendingPush(cursor.getInt(pendingPushColumn) == 1);
 
+    int thumbnailColumn = cursor.getColumnIndexOrThrow(THUMBNAIL);
+
+    if (!cursor.isNull(thumbnailColumn))
+      part.setThumbnailUri(ContentUris.withAppendedId(PartAuthority.THUMB_CONTENT_URI,
+                                                      cursor.getLong(cursor.getColumnIndexOrThrow(ID))));
+
     int sizeColumn = cursor.getColumnIndexOrThrow(SIZE);
 
     if (!cursor.isNull(sizeColumn))
@@ -357,7 +381,6 @@ public class PartDatabase extends Database {
 
   private PduPart getPart(Cursor cursor) {
     PduPart part   = new PduPart();
-    String dataLocation = cursor.getString(cursor.getColumnIndexOrThrow(DATA));
     long    partId = cursor.getLong(cursor.getColumnIndexOrThrow(ID));
 
     getPartValues(part, cursor);
@@ -385,7 +408,11 @@ public class PartDatabase extends Database {
       contentValues.put(SIZE, partData.second);
     }
 
-    return database.insert(TABLE_NAME, null, contentValues);
+    long partId = database.insert(TABLE_NAME, null, contentValues);
+
+    ApplicationContext.getInstance(context).getJobManager().add(new ThumbnailGenerateJob(context, partId));
+
+    return partId;
   }
 
   public void updateDownloadedPart(MasterSecret masterSecret, long messageId,
@@ -407,6 +434,24 @@ public class PartDatabase extends Database {
 
     database.update(TABLE_NAME, values, ID_WHERE, new String[] {partId+""});
 
+    ApplicationContext.getInstance(context).getJobManager().add(new ThumbnailGenerateJob(context, partId));
+
     notifyConversationListeners(DatabaseFactory.getMmsDatabase(context).getThreadIdForMessage(messageId));
   }
+
+  public void updatePartThumbnail(MasterSecret masterSecret, long partId, PduPart part, InputStream in, float aspectRatio)
+      throws MmsException
+  {
+    Log.w(TAG, "updating part thumbnail for #" + partId);
+
+    Pair<File, Long> thumbnailFile = writePartData(masterSecret, part, in);
+
+    SQLiteDatabase database = databaseHelper.getWritableDatabase();
+    ContentValues  values   = new ContentValues(2);
+
+    values.put(THUMBNAIL, thumbnailFile.first.getAbsolutePath());
+    values.put(ASPECT_RATIO, aspectRatio);
+
+    database.update(TABLE_NAME, values, ID_WHERE, new String[] {partId + ""});
+  }
 }

src/org/thoughtcrime/securesms/jobs/ThumbnailGenerateJob.java

@@ -0,0 +1,125 @@
+package org.thoughtcrime.securesms.jobs;
+
+import android.content.Context;
+import android.graphics.Bitmap;
+import android.graphics.Bitmap.CompressFormat;
+import android.util.Log;
+import android.util.Pair;
+
+import org.thoughtcrime.securesms.R;
+import org.thoughtcrime.securesms.crypto.AsymmetricMasterCipher;
+import org.thoughtcrime.securesms.crypto.AsymmetricMasterSecret;
+import org.thoughtcrime.securesms.crypto.MasterSecret;
+import org.thoughtcrime.securesms.crypto.MasterSecretUtil;
+import org.thoughtcrime.securesms.crypto.SecurityEvent;
+import org.thoughtcrime.securesms.crypto.SmsCipher;
+import org.thoughtcrime.securesms.crypto.storage.TextSecureAxolotlStore;
+import org.thoughtcrime.securesms.database.DatabaseFactory;
+import org.thoughtcrime.securesms.database.EncryptingSmsDatabase;
+import org.thoughtcrime.securesms.database.NoSuchMessageException;
+import org.thoughtcrime.securesms.database.PartDatabase;
+import org.thoughtcrime.securesms.database.model.SmsMessageRecord;
+import org.thoughtcrime.securesms.jobs.requirements.MasterSecretRequirement;
+import org.thoughtcrime.securesms.notifications.MessageNotifier;
+import org.thoughtcrime.securesms.service.KeyCachingService;
+import org.thoughtcrime.securesms.sms.IncomingEncryptedMessage;
+import org.thoughtcrime.securesms.sms.IncomingEndSessionMessage;
+import org.thoughtcrime.securesms.sms.IncomingKeyExchangeMessage;
+import org.thoughtcrime.securesms.sms.IncomingPreKeyBundleMessage;
+import org.thoughtcrime.securesms.sms.IncomingTextMessage;
+import org.thoughtcrime.securesms.sms.MessageSender;
+import org.thoughtcrime.securesms.sms.OutgoingKeyExchangeMessage;
+import org.thoughtcrime.securesms.util.BitmapDecodingException;
+import org.thoughtcrime.securesms.util.BitmapUtil;
+import org.thoughtcrime.securesms.util.TextSecurePreferences;
+import org.whispersystems.jobqueue.JobParameters;
+import org.whispersystems.libaxolotl.DuplicateMessageException;
+import org.whispersystems.libaxolotl.InvalidMessageException;
+import org.whispersystems.libaxolotl.InvalidVersionException;
+import org.whispersystems.libaxolotl.LegacyMessageException;
+import org.whispersystems.libaxolotl.NoSessionException;
+import org.whispersystems.libaxolotl.StaleKeyExchangeException;
+import org.whispersystems.libaxolotl.UntrustedIdentityException;
+import org.whispersystems.libaxolotl.util.guava.Optional;
+import org.whispersystems.textsecure.api.messages.TextSecureGroup;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+
+import ws.com.google.android.mms.ContentType;
+import ws.com.google.android.mms.MmsException;
+import ws.com.google.android.mms.pdu.PduPart;
+
+public class ThumbnailGenerateJob extends MasterSecretJob {
+
+  private static final String TAG = ThumbnailGenerateJob.class.getSimpleName();
+
+  private final long partId;
+
+  public ThumbnailGenerateJob(Context context, long partId) {
+    super(context, JobParameters.newBuilder()
+                                .withRequirement(new MasterSecretRequirement(context))
+                                .create());
+
+    this.partId = partId;
+  }
+
+  @Override
+  public void onAdded() { }
+
+  @Override
+  public void onRun(MasterSecret masterSecret) throws MmsException {
+    PartDatabase database = DatabaseFactory.getPartDatabase(context);
+    PduPart part = database.getPart(partId);
+
+    if (part.getThumbnailUri() != null) {
+      return;
+    }
+
+    long startMillis = System.currentTimeMillis();
+    Bitmap thumbnail = generateThumbnailForPart(masterSecret, part);
+
+    if (thumbnail != null) {
+      ByteArrayOutputStream thumbnailBytes = new ByteArrayOutputStream();
+      thumbnail.compress(CompressFormat.JPEG, 85, thumbnailBytes);
+
+      float aspectRatio = (float)thumbnail.getWidth() / (float)thumbnail.getHeight();
+      Log.w(TAG, String.format("generated thumbnail for part #%d, %dx%d (%.3f:1) in %dms",
+                               partId,
+                               thumbnail.getWidth(),
+                               thumbnail.getHeight(),
+                               aspectRatio, System.currentTimeMillis() - startMillis));
+      database.updatePartThumbnail(masterSecret, partId, part, new ByteArrayInputStream(thumbnailBytes.toByteArray()), aspectRatio);
+    } else {
+      Log.w(TAG, "thumbnail not generated");
+    }
+  }
+
+  private Bitmap generateThumbnailForPart(MasterSecret masterSecret, PduPart part) {
+    String contentType = new String(part.getContentType());
+
+    if      (ContentType.isImageType(contentType)) return generateImageThumbnail(masterSecret, part);
+    else                                           return null;
+  }
+
+  private Bitmap generateImageThumbnail(MasterSecret masterSecret, PduPart part) {
+    try {
+      int maxSize = context.getResources().getDimensionPixelSize(R.dimen.thumbnail_max_size);
+      return BitmapUtil.createScaledBitmap(context, masterSecret, part.getDataUri(), maxSize, maxSize);
+    } catch (FileNotFoundException | BitmapDecodingException e) {
+      Log.w(TAG, e);
+      return null;
+    }
+  }
+
+  @Override
+  public boolean onShouldRetryThrowable(Exception exception) {
+    return false;
+  }
+
+  @Override
+  public void onCanceled() { }
+}

src/org/thoughtcrime/securesms/mms/AttachmentManager.java

@@ -20,7 +20,9 @@ import android.app.Activity;
 import android.content.ActivityNotFoundException;
 import android.content.Context;
 import android.content.Intent;
+import android.graphics.drawable.Drawable;
 import android.net.Uri;
+import android.os.AsyncTask;
 import android.os.Build;
 import android.util.Log;
 import android.provider.ContactsContract;
@@ -74,13 +76,24 @@ public class AttachmentManager {
     setMedia(new AudioSlide(context, audio));
   }
 
-  public void setMedia(Slide slide, int thumbnailWidth, int thumbnailHeight) {
+  public void setMedia(final Slide slide, final int thumbnailWidth, final int thumbnailHeight) {
     slideDeck.clear();
     slideDeck.addSlide(slide);
-    thumbnail.setImageDrawable(slide.getThumbnail(thumbnailWidth, thumbnailHeight));
+    new AsyncTask<Void,Void,Drawable>() {
+
+      @Override
+      protected Drawable doInBackground(Void... params) {
+        return slide.getThumbnail(thumbnailWidth, thumbnailHeight);
+      }
+
+      @Override
+      protected void onPostExecute(Drawable drawable) {
+        thumbnail.setImageDrawable(drawable);
         attachmentView.setVisibility(View.VISIBLE);
         attachmentListener.onAttachmentChanged();
       }
+    }.execute();
+  }
 
   public void setMedia(Slide slide) {
     setMedia(slide, thumbnail.getWidth(), thumbnail.getHeight());

src/org/thoughtcrime/securesms/mms/ImageSlide.java

@@ -17,6 +17,8 @@
 package org.thoughtcrime.securesms.mms;
 
 import android.content.Context;
+import android.graphics.Bitmap;
+import android.graphics.BitmapFactory;
 import android.graphics.Color;
 import android.graphics.drawable.AnimationDrawable;
 import android.graphics.drawable.BitmapDrawable;
@@ -41,7 +43,6 @@ import org.thoughtcrime.securesms.crypto.MasterSecret;
 
 import java.io.FileNotFoundException;
 import java.io.IOException;
-import java.io.InputStream;
 import java.lang.ref.SoftReference;
 import java.lang.ref.WeakReference;
 import java.util.Collections;
@@ -51,6 +52,7 @@ import ws.com.google.android.mms.ContentType;
 import ws.com.google.android.mms.pdu.PduPart;
 
 public class ImageSlide extends Slide {
+  private static final String TAG = ImageSlide.class.getSimpleName();
 
   private static final int MAX_CACHE_SIZE = 10;
   private static final Map<Uri, SoftReference<Drawable>> thumbnailCache =
@@ -77,8 +79,15 @@ public class ImageSlide extends Slide {
     }
 
     try {
-      thumbnail = new BitmapDrawable(context.getResources(),
+      Bitmap thumbnailBitmap;
-                                     BitmapUtil.createScaledBitmap(context, masterSecret, getUri(), maxWidth, maxHeight));
+      long startDecode = System.currentTimeMillis();
+      Log.w(TAG, (part.getThumbnailUri() == null ? "generating" : "fetching pre-generated") + " thumbnail");
+      if (part.getThumbnailUri() != null) thumbnailBitmap = BitmapFactory.decodeStream(PartAuthority.getPartStream(context, masterSecret, part.getThumbnailUri()));
+      else                                thumbnailBitmap = BitmapUtil.createScaledBitmap(context, masterSecret, getUri(), maxWidth, maxHeight);
+
+      Log.w(TAG, "thumbnail decode/generate time: " + (System.currentTimeMillis() - startDecode) + "ms");
+
+      thumbnail = new BitmapDrawable(context.getResources(), thumbnailBitmap);
       thumbnailCache.put(part.getDataUri(), new SoftReference<>(thumbnail));
 
       return thumbnail;

src/org/thoughtcrime/securesms/mms/PartAuthority.java

@@ -16,15 +16,20 @@ import java.io.InputStream;
 public class PartAuthority {
 
   private static final String PART_URI_STRING   = "content://org.thoughtcrime.securesms/part";
+  private static final String THUMB_URI_STRING  = "content://org.thoughtcrime.securesms/thumb";
+
   public  static final Uri    PART_CONTENT_URI  = Uri.parse(PART_URI_STRING);
+  public  static final Uri    THUMB_CONTENT_URI = Uri.parse(THUMB_URI_STRING);
 
   private static final int PART_ROW  = 1;
+  private static final int THUMB_ROW = 2;
 
   private static final UriMatcher uriMatcher;
 
   static {
     uriMatcher = new UriMatcher(UriMatcher.NO_MATCH);
     uriMatcher.addURI("org.thoughtcrime.securesms", "part/#", PART_ROW);
+    uriMatcher.addURI("org.thoughtcrime.securesms", "thumb/#", THUMB_ROW);
   }
 
   public static InputStream getPartStream(Context context, MasterSecret masterSecret, Uri uri)
@@ -35,6 +40,7 @@ public class PartAuthority {
 
     switch (match) {
       case PART_ROW:  return partDatabase.getPartStream(masterSecret, ContentUris.parseId(uri));
+      case THUMB_ROW: return partDatabase.getThumbnailStream(masterSecret, ContentUris.parseId(uri));
       default:        return context.getContentResolver().openInputStream(uri);
     }
   }

src/org/thoughtcrime/securesms/mms/Slide.java

@@ -16,7 +16,6 @@
  */
 package org.thoughtcrime.securesms.mms;
 
-import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
 
@@ -25,10 +24,7 @@ import org.w3c.dom.smil.SMILDocument;
 import org.w3c.dom.smil.SMILMediaElement;
 import org.w3c.dom.smil.SMILRegionElement;
 import org.thoughtcrime.securesms.crypto.MasterSecret;
-import org.thoughtcrime.securesms.database.DatabaseFactory;
-import org.thoughtcrime.securesms.providers.PartProvider;
 
-import android.content.ContentUris;
 import android.content.Context;
 import android.graphics.Bitmap;
 import android.graphics.drawable.Drawable;
@@ -56,10 +52,6 @@ public abstract class Slide {
     this.masterSecret = masterSecret;
   }
 
-  public InputStream getPartDataInputStream() throws FileNotFoundException {
-    return PartAuthority.getPartStream(context, masterSecret, part.getDataUri());
-  }
-
   protected byte[] getPartData() {
     try {
       if (part.getData() != null)

src/org/thoughtcrime/securesms/mms/TextSlide.java

@@ -19,6 +19,7 @@ package org.thoughtcrime.securesms.mms;
 import android.content.Context;
 import android.net.Uri;
 import android.util.Log;
+import android.widget.ImageView;
 
 import org.thoughtcrime.securesms.util.SmilUtil;
 import org.w3c.dom.smil.SMILDocument;

src/org/thoughtcrime/securesms/util/BitmapUtil.java

@@ -5,12 +5,19 @@ import android.graphics.Bitmap;
 import android.graphics.BitmapFactory;
 import android.graphics.Canvas;
 import android.graphics.Matrix;
+import android.graphics.Color;
 import android.graphics.Paint;
 import android.graphics.PorterDuff;
+import android.graphics.PorterDuff.Mode;
 import android.graphics.PorterDuffXfermode;
 import android.graphics.Rect;
+import android.graphics.RectF;
 import android.net.Uri;
 import android.util.Log;
+import android.util.Pair;
+
+import org.thoughtcrime.securesms.crypto.MasterSecret;
+import org.thoughtcrime.securesms.mms.PartAuthority;
 
 import java.io.BufferedInputStream;
 import java.io.ByteArrayOutputStream;
@@ -151,11 +158,14 @@ public class BitmapUtil {
         aspectWidth = (aspectHeight / options.outHeight) * options.outWidth;
       }
 
+      final int fineWidth  = Math.round(aspectWidth);
+      final int fineHeight = Math.round(aspectHeight);
+
       Log.w(TAG, "fine scale " + options.outWidth + "x" + options.outHeight +
-                 " => " + aspectWidth + "x" + aspectHeight);
+                 " => " + fineWidth + "x" + fineHeight);
       Bitmap scaledThumbnail = null;
       try {
-        scaledThumbnail = Bitmap.createScaledBitmap(roughThumbnail, (int) aspectWidth, (int) aspectHeight, true);
+        scaledThumbnail = Bitmap.createScaledBitmap(roughThumbnail, fineWidth, fineHeight, true);
       } finally {
         if (roughThumbnail != scaledThumbnail) roughThumbnail.recycle();
       }

src/ws/com/google/android/mms/pdu/PduPart.java

@@ -19,9 +19,13 @@ package ws.com.google.android.mms.pdu;
 
 import android.net.Uri;
 
+import org.thoughtcrime.securesms.util.Util;
+
 import java.util.HashMap;
 import java.util.Map;
 
+import ws.com.google.android.mms.ContentType;
+
 /**
  * The pdu part.
  */
@@ -121,6 +125,7 @@ public class PduPart {
 
      private static final String TAG = "PduPart";
 
+     private Uri     thumbnailUri;
      private boolean isEncrypted;
      private boolean isPendingPush;
      private long    dataSize;
@@ -157,6 +162,14 @@ public class PduPart {
        return isPendingPush;
      }
 
+     public void setThumbnailUri(Uri thumbnailUri) {
+       this.thumbnailUri = thumbnailUri;
+     }
+
+     public Uri getThumbnailUri() {
+       return this.thumbnailUri;
+     }
+
      /**
       * Set part data. The data are stored as byte array.
       *
@@ -318,7 +331,7 @@ public class PduPart {
      /**
       *  Set Content-Type value.
       *
-      *  @param value the value
+      *  @param contentType the value
       *  @throws NullPointerException if the value is null.
       */
      public void setContentType(byte[] contentType) {
@@ -341,7 +354,7 @@ public class PduPart {
      /**
       * Set Content-Transfer-Encoding value
       *
-      * @param contentId the content-id value
+      * @param contentTransferEncoding the value
       * @throws NullPointerException if the value is null.
       */
      public void setContentTransferEncoding(byte[] contentTransferEncoding) {