Merge pull request #376 from RyanRory/link-preview-for-any-site

Link Previews for Any Site

src/org/thoughtcrime/securesms/conversation/ConversationActivity.java

@@ -2370,7 +2370,7 @@ public class ConversationActivity extends PassphraseRequiredActionBarActivity
                                        recipient.getAddress().isEmail()               ||
                                        inputPanel.getQuote().isPresent()              ||
                                        linkPreviewViewModel.hasLinkPreview()          ||
-                                       LinkPreviewUtil.isWhitelistedMediaUrl(message) || // Loki - Send GIFs as media messages
+                                       LinkPreviewUtil.isValidMediaUrl(message) || // Loki - Send GIFs as media messages
                                        needsSplit;
 
       Log.i(TAG, "isManual Selection: " + sendButton.isManualSelection());

src/org/thoughtcrime/securesms/conversation/ConversationItem.java

@@ -500,7 +500,7 @@ public class ConversationItem extends TapJackingProofLinearLayout
   private void adjustMarginsIfNeeded(MessageRecord messageRecord) {
     LinearLayout.LayoutParams bodyTextLayoutParams = (LinearLayout.LayoutParams)bodyText.getLayoutParams();
     bodyTextLayoutParams.topMargin = 0;
-    if (hasOnlyThumbnail(messageRecord)) {
+    if (hasOnlyThumbnail(messageRecord) || hasLinkPreview(messageRecord)) {
       int topPadding = 0;
       if (groupSenderHolder.getVisibility() == VISIBLE) {
         topPadding = (int)getResources().getDimension(R.dimen.medium_spacing);

src/org/thoughtcrime/securesms/jobs/PushDecryptJob.java

@@ -1368,7 +1368,7 @@ public class PushDecryptJob extends BaseJob implements InjectableType {
       Optional<String>     title         = Optional.fromNullable(preview.getTitle());
       boolean              hasContent    = !TextUtils.isEmpty(title.or("")) || thumbnail.isPresent();
       boolean              presentInBody = url.isPresent() && Stream.of(LinkPreviewUtil.findWhitelistedUrls(message)).map(Link::getUrl).collect(Collectors.toSet()).contains(url.get());
-      boolean              validDomain   = url.isPresent() && LinkPreviewUtil.isWhitelistedLinkUrl(url.get());
+      boolean              validDomain   = url.isPresent() && LinkPreviewUtil.isValidLinkUrl(url.get());
 
       if (hasContent && presentInBody && validDomain) {
         LinkPreview linkPreview = new LinkPreview(url.get(), title.or(""), thumbnail);

src/org/thoughtcrime/securesms/linkpreview/LinkPreviewDomains.java

@@ -1,50 +0,0 @@
-package org.thoughtcrime.securesms.linkpreview;
-
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.Set;
-
-public class LinkPreviewDomains {
-  public static final String STICKERS = "signal.org";
-
-  public static final Set<String> LINKS = new HashSet<>(Arrays.asList(
-        // YouTube
-      "youtube.com",
-      "www.youtube.com",
-      "m.youtube.com",
-      "youtu.be",
-      // Reddit
-      "reddit.com",
-      "www.reddit.com",
-      "m.reddit.com",
-      // Imgur
-      "imgur.com",
-      "www.imgur.com",
-      "m.imgur.com",
-      // Instagram
-      "instagram.com",
-      "www.instagram.com",
-      "m.instagram.com",
-      // Pinterest
-      "pinterest.com",
-      "www.pinterest.com",
-      "pin.it",
-      // Giphy
-      "giphy.com",
-      "media.giphy.com",
-      "media1.giphy.com",
-      "media2.giphy.com",
-      "media3.giphy.com",
-      "gph.is"
-  ));
-
-  public static final Set<String> IMAGES = new HashSet<>(Arrays.asList(
-      "ytimg.com",
-      "cdninstagram.com",
-      "fbcdn.net",
-      "redd.it",
-      "imgur.com",
-      "pinimg.com",
-      "giphy.com"
-  ));
-}

src/org/thoughtcrime/securesms/linkpreview/LinkPreviewRepository.java

@@ -2,13 +2,16 @@ package org.thoughtcrime.securesms.linkpreview;
 
 import android.content.Context;
 import android.graphics.Bitmap;
+import android.graphics.BitmapFactory;
 import android.net.Uri;
 import androidx.annotation.NonNull;
+import androidx.annotation.Nullable;
 import android.text.Html;
 import android.text.TextUtils;
 
 import com.bumptech.glide.load.engine.DiskCacheStrategy;
 import com.bumptech.glide.request.FutureTarget;
+import com.google.android.gms.common.util.IOUtils;
 
 import org.thoughtcrime.securesms.ApplicationContext;
 import org.thoughtcrime.securesms.attachments.Attachment;
@@ -37,6 +40,7 @@ import org.whispersystems.signalservice.api.messages.SignalServiceStickerManifes
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.io.InputStream;
 import java.util.concurrent.CancellationException;
 import java.util.concurrent.ExecutionException;
 import java.util.regex.Matcher;
@@ -73,7 +77,7 @@ public class LinkPreviewRepository implements InjectableType {
   RequestController getLinkPreview(@NonNull Context context, @NonNull String url, @NonNull Callback<Optional<LinkPreview>> callback) {
     CompositeRequestController compositeController = new CompositeRequestController();
 
-    if (!LinkPreviewUtil.isWhitelistedLinkUrl(url)) {
+    if (!LinkPreviewUtil.isValidLinkUrl(url)) {
       Log.w(TAG, "Tried to get a link preview for a non-whitelisted domain.");
       callback.onComplete(Optional.absent());
       return compositeController;
@@ -137,7 +141,7 @@ public class LinkPreviewRepository implements InjectableType {
         Optional<String>  title    = getProperty(body, "title");
         Optional<String>  imageUrl = getProperty(body, "image");
 
-        if (imageUrl.isPresent() && !LinkPreviewUtil.isWhitelistedMediaUrl(imageUrl.get())) {
+        if (imageUrl.isPresent() && !LinkPreviewUtil.isValidMediaUrl(imageUrl.get())) {
           Log.i(TAG, "Image URL was invalid or for a non-whitelisted domain. Skipping.");
           imageUrl = Optional.absent();
         }
@@ -150,57 +154,49 @@ public class LinkPreviewRepository implements InjectableType {
   }
 
   private @NonNull RequestController fetchThumbnail(@NonNull Context context, @NonNull String imageUrl, @NonNull Callback<Optional<Attachment>> callback) {
-    FutureTarget<Bitmap> bitmapFuture = GlideApp.with(context).asBitmap()
+    Call                  call       = client.newCall(new Request.Builder().url(imageUrl).build());
-                                                              .load(new ChunkedImageUrl(imageUrl))
+    CallRequestController controller = new CallRequestController(call);
-                                                              .skipMemoryCache(true)
-                                                              .diskCacheStrategy(DiskCacheStrategy.NONE)
-                                                              .centerInside()
-                                                              .submit(1024, 1024);
-
-    RequestController controller = () -> bitmapFuture.cancel(false);
 
     SignalExecutors.UNBOUNDED.execute(() -> {
       try {
-        Bitmap                bitmap = bitmapFuture.get();
+        Response response = call.execute();
-        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        if (!response.isSuccessful() || response.body() == null) {
+          return;
+        }
 
-        bitmap.compress(Bitmap.CompressFormat.JPEG, 80, baos);
+        InputStream bodyStream = response.body().byteStream();
+        controller.setStream(bodyStream);
 
-        byte[]               bytes     = baos.toByteArray();
+        byte[]               data      = IOUtils.readInputStreamFully(bodyStream);
-        Uri                  uri       = BlobProvider.getInstance().forData(bytes).createForSingleSessionInMemory();
+        Bitmap               bitmap    = BitmapFactory.decodeByteArray(data, 0, data.length);
-        Optional<Attachment> thumbnail = Optional.of(new UriAttachment(uri,
+        Optional<Attachment> thumbnail = bitmapToAttachment(bitmap, Bitmap.CompressFormat.JPEG, MediaUtil.IMAGE_JPEG);
-                                                                       uri,
+
-                                                                       MediaUtil.IMAGE_JPEG,
+        if (bitmap != null) bitmap.recycle();
-                                                                       AttachmentDatabase.TRANSFER_PROGRESS_STARTED,
-                                                                       bytes.length,
-                                                                       bitmap.getWidth(),
-                                                                       bitmap.getHeight(),
-                                                                       null,
-                                                                       null,
-                                                                       false,
-                                                                       false,
-                                                                       null,
-                                                                       null));
 
         callback.onComplete(thumbnail);
-      } catch (CancellationException | ExecutionException | InterruptedException e) {
+      } catch (IOException e) {
+        Log.w(TAG, "Exception during link preview image retrieval.", e);
         controller.cancel();
         callback.onComplete(Optional.absent());
-      } finally {
-        bitmapFuture.cancel(false);
       }
     });
 
-    return () -> bitmapFuture.cancel(true);
+    return controller;
   }
 
   private @NonNull Optional<String> getProperty(@NonNull String searchText, @NonNull String property) {
     Pattern pattern = Pattern.compile("<\\s*meta\\s+property\\s*=\\s*\"\\s*og:" + property + "\\s*\"\\s+[^>]*content\\s*=\\s*\"(.*?)\"[^>]*/?\\s*>", Pattern.CASE_INSENSITIVE | Pattern.DOTALL);
     Matcher matcher = pattern.matcher(searchText);
-
     if (matcher.find()) {
       String text = Html.fromHtml(matcher.group(1)).toString();
-      return TextUtils.isEmpty(text) ? Optional.absent() : Optional.of(text);
+      if (!TextUtils.isEmpty(text)) { return Optional.of(text); }
+    }
+
+    pattern = Pattern.compile("<\\s*" + property + "[^>]*>(.*?)<\\s*/" + property + "[^>]*>", Pattern.CASE_INSENSITIVE | Pattern.DOTALL);
+    matcher = pattern.matcher(searchText);
+    if (matcher.find()) {
+      String text = Html.fromHtml(matcher.group(1)).toString();
+      if (!TextUtils.isEmpty(text)) { return Optional.of(text); }
     }
 
     return Optional.absent();
@@ -266,6 +262,38 @@ public class LinkPreviewRepository implements InjectableType {
     return () -> Log.i(TAG, "Cancelled sticker pack link preview fetch -- no effect.");
   }
 
+  private static Optional<Attachment> bitmapToAttachment(@Nullable Bitmap bitmap,
+                                                         @NonNull Bitmap.CompressFormat format,
+                                                         @NonNull String contentType)
+  {
+    if (bitmap == null) {
+      return Optional.absent();
+    }
+
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+
+    bitmap.compress(format, 80, baos);
+
+    byte[] bytes = baos.toByteArray();
+    Uri    uri   = BlobProvider.getInstance().forData(bytes).createForSingleSessionInMemory();
+
+    return  Optional.of(new UriAttachment(uri,
+            uri,
+            contentType,
+            AttachmentDatabase.TRANSFER_PROGRESS_STARTED,
+            bytes.length,
+            bitmap.getWidth(),
+            bitmap.getHeight(),
+            null,
+            null,
+            false,
+            false,
+            null,
+            null));
+
+  }
+
+
   private static class Metadata {
     private final Optional<String> title;
     private final Optional<String> imageUrl;

src/org/thoughtcrime/securesms/linkpreview/LinkPreviewUtil.java

@@ -38,14 +38,14 @@ public final class LinkPreviewUtil {
 
     return Stream.of(spannable.getSpans(0, spannable.length(), URLSpan.class))
                  .map(span -> new Link(span.getURL(), spannable.getSpanStart(span)))
-                 .filter(link -> isWhitelistedLinkUrl(link.getUrl()))
+                 .filter(link -> isValidLinkUrl(link.getUrl()))
                  .toList();
   }
 
   /**
-   * @return True if the host is present in the link whitelist.
+   * @return True if the host is valid.
    */
-  public static boolean isWhitelistedLinkUrl(@Nullable String linkUrl) {
+  public static boolean isValidLinkUrl(@Nullable String linkUrl) {
     if (linkUrl == null)                      return false;
     if (StickerUrl.isValidShareLink(linkUrl)) return true;
 
@@ -53,21 +53,19 @@ public final class LinkPreviewUtil {
     return url != null                                   &&
            !TextUtils.isEmpty(url.scheme())              &&
            "https".equals(url.scheme())                  &&
-           LinkPreviewDomains.LINKS.contains(url.host()) &&
            isLegalUrl(linkUrl);
   }
 
   /**
-   * @return True if the top-level domain is present in the media whitelist.
+   * @return True if the top-level domain is valid.
    */
-  public static boolean isWhitelistedMediaUrl(@Nullable String mediaUrl) {
+  public static boolean isValidMediaUrl(@Nullable String mediaUrl) {
     if (mediaUrl == null) return false;
 
     HttpUrl url = HttpUrl.parse(mediaUrl);
     return url != null                                                &&
            !TextUtils.isEmpty(url.scheme())                           &&
            "https".equals(url.scheme())                               &&
-           LinkPreviewDomains.IMAGES.contains(url.topPrivateDomain()) &&
            isLegalUrl(mediaUrl);
   }
 

src/org/thoughtcrime/securesms/net/ContentProxySafetyInterceptor.java

@@ -31,7 +31,7 @@ public class ContentProxySafetyInterceptor implements Interceptor {
       Response response = chain.proceed(chain.request());
 
       if (response.isRedirect()) {
-        if (isWhitelisted(response.header("Location"))) {
+        if (isWhitelisted(response.header("location")) || isWhitelisted(response.header("Location"))) {
           return response;
         } else {
           Log.w(TAG, "Tried to redirect to a non-whitelisted domain!");
@@ -53,6 +53,6 @@ public class ContentProxySafetyInterceptor implements Interceptor {
   }
 
   private static boolean isWhitelisted(@Nullable String url) {
-    return LinkPreviewUtil.isWhitelistedLinkUrl(url) || LinkPreviewUtil.isWhitelistedMediaUrl(url);
+    return LinkPreviewUtil.isValidLinkUrl(url) || LinkPreviewUtil.isValidMediaUrl(url);
   }
 }

src/org/thoughtcrime/securesms/net/ContentProxySelector.java

@@ -1,13 +1,9 @@
 package org.thoughtcrime.securesms.net;
 
 
-import android.os.AsyncTask;
-
-import org.thoughtcrime.securesms.linkpreview.LinkPreviewDomains;
 import org.thoughtcrime.securesms.logging.Log;
 
 import network.loki.messenger.BuildConfig;
-import org.thoughtcrime.securesms.util.Util;
 
 import java.io.IOException;
 import java.net.InetSocketAddress;
@@ -27,8 +23,7 @@ public class ContentProxySelector extends ProxySelector {
 
   private static final Set<String> WHITELISTED_DOMAINS = new HashSet<>();
   static {
-    WHITELISTED_DOMAINS.addAll(LinkPreviewDomains.LINKS);
+    WHITELISTED_DOMAINS.add("giphy.com");
-    WHITELISTED_DOMAINS.addAll(LinkPreviewDomains.IMAGES);
   }
 
   private final List<Proxy> CONTENT = new ArrayList<Proxy>(1) {{