From 5ad7fcf95494f56f7e79ab1fe8590390d70c4c34 Mon Sep 17 00:00:00 2001 From: Ryan ZHAO Date: Fri, 6 Nov 2020 14:49:23 +1100 Subject: [PATCH 1/6] rename whitelisted function --- .../securesms/conversation/ConversationActivity.java | 2 +- .../thoughtcrime/securesms/jobs/PushDecryptJob.java | 2 +- .../securesms/linkpreview/LinkPreviewUtil.java | 12 +++++------- .../securesms/net/ContentProxySafetyInterceptor.java | 2 +- 4 files changed, 8 insertions(+), 10 deletions(-) diff --git a/src/org/thoughtcrime/securesms/conversation/ConversationActivity.java b/src/org/thoughtcrime/securesms/conversation/ConversationActivity.java index 465de91e5f..d9f463844e 100644 --- a/src/org/thoughtcrime/securesms/conversation/ConversationActivity.java +++ b/src/org/thoughtcrime/securesms/conversation/ConversationActivity.java @@ -2370,7 +2370,7 @@ public class ConversationActivity extends PassphraseRequiredActionBarActivity recipient.getAddress().isEmail() || inputPanel.getQuote().isPresent() || linkPreviewViewModel.hasLinkPreview() || - LinkPreviewUtil.isWhitelistedMediaUrl(message) || // Loki - Send GIFs as media messages + LinkPreviewUtil.isValidMediaUrl(message) || // Loki - Send GIFs as media messages needsSplit; Log.i(TAG, "isManual Selection: " + sendButton.isManualSelection()); diff --git a/src/org/thoughtcrime/securesms/jobs/PushDecryptJob.java b/src/org/thoughtcrime/securesms/jobs/PushDecryptJob.java index 84476b944e..8801ce4aa1 100644 --- a/src/org/thoughtcrime/securesms/jobs/PushDecryptJob.java +++ b/src/org/thoughtcrime/securesms/jobs/PushDecryptJob.java @@ -1368,7 +1368,7 @@ public class PushDecryptJob extends BaseJob implements InjectableType { Optional title = Optional.fromNullable(preview.getTitle()); boolean hasContent = !TextUtils.isEmpty(title.or("")) || thumbnail.isPresent(); boolean presentInBody = url.isPresent() && Stream.of(LinkPreviewUtil.findWhitelistedUrls(message)).map(Link::getUrl).collect(Collectors.toSet()).contains(url.get()); - boolean validDomain = url.isPresent() && LinkPreviewUtil.isWhitelistedLinkUrl(url.get()); + boolean validDomain = url.isPresent() && LinkPreviewUtil.isValidLinkUrl(url.get()); if (hasContent && presentInBody && validDomain) { LinkPreview linkPreview = new LinkPreview(url.get(), title.or(""), thumbnail); diff --git a/src/org/thoughtcrime/securesms/linkpreview/LinkPreviewUtil.java b/src/org/thoughtcrime/securesms/linkpreview/LinkPreviewUtil.java index 9139f67ca0..c9ca8e3b6c 100644 --- a/src/org/thoughtcrime/securesms/linkpreview/LinkPreviewUtil.java +++ b/src/org/thoughtcrime/securesms/linkpreview/LinkPreviewUtil.java @@ -38,14 +38,14 @@ public final class LinkPreviewUtil { return Stream.of(spannable.getSpans(0, spannable.length(), URLSpan.class)) .map(span -> new Link(span.getURL(), spannable.getSpanStart(span))) - .filter(link -> isWhitelistedLinkUrl(link.getUrl())) + .filter(link -> isValidLinkUrl(link.getUrl())) .toList(); } /** - * @return True if the host is present in the link whitelist. + * @return True if the host is valid. */ - public static boolean isWhitelistedLinkUrl(@Nullable String linkUrl) { + public static boolean isValidLinkUrl(@Nullable String linkUrl) { if (linkUrl == null) return false; if (StickerUrl.isValidShareLink(linkUrl)) return true; @@ -53,21 +53,19 @@ public final class LinkPreviewUtil { return url != null && !TextUtils.isEmpty(url.scheme()) && "https".equals(url.scheme()) && - LinkPreviewDomains.LINKS.contains(url.host()) && isLegalUrl(linkUrl); } /** - * @return True if the top-level domain is present in the media whitelist. + * @return True if the top-level domain is valid. */ - public static boolean isWhitelistedMediaUrl(@Nullable String mediaUrl) { + public static boolean isValidMediaUrl(@Nullable String mediaUrl) { if (mediaUrl == null) return false; HttpUrl url = HttpUrl.parse(mediaUrl); return url != null && !TextUtils.isEmpty(url.scheme()) && "https".equals(url.scheme()) && - LinkPreviewDomains.IMAGES.contains(url.topPrivateDomain()) && isLegalUrl(mediaUrl); } diff --git a/src/org/thoughtcrime/securesms/net/ContentProxySafetyInterceptor.java b/src/org/thoughtcrime/securesms/net/ContentProxySafetyInterceptor.java index 992500b8c1..dcf6097738 100644 --- a/src/org/thoughtcrime/securesms/net/ContentProxySafetyInterceptor.java +++ b/src/org/thoughtcrime/securesms/net/ContentProxySafetyInterceptor.java @@ -53,6 +53,6 @@ public class ContentProxySafetyInterceptor implements Interceptor { } private static boolean isWhitelisted(@Nullable String url) { - return LinkPreviewUtil.isWhitelistedLinkUrl(url) || LinkPreviewUtil.isWhitelistedMediaUrl(url); + return LinkPreviewUtil.isValidLinkUrl(url) || LinkPreviewUtil.isValidMediaUrl(url); } } From 283e7d62dd78aa7eb4e995a2ea9932ff5462f1a5 Mon Sep 17 00:00:00 2001 From: Ryan ZHAO Date: Fri, 6 Nov 2020 14:49:48 +1100 Subject: [PATCH 2/6] remove unused white list --- .../linkpreview/LinkPreviewDomains.java | 50 ------------------- .../securesms/net/ContentProxySelector.java | 7 +-- 2 files changed, 1 insertion(+), 56 deletions(-) delete mode 100644 src/org/thoughtcrime/securesms/linkpreview/LinkPreviewDomains.java diff --git a/src/org/thoughtcrime/securesms/linkpreview/LinkPreviewDomains.java b/src/org/thoughtcrime/securesms/linkpreview/LinkPreviewDomains.java deleted file mode 100644 index 64baa6d42a..0000000000 --- a/src/org/thoughtcrime/securesms/linkpreview/LinkPreviewDomains.java +++ /dev/null @@ -1,50 +0,0 @@ -package org.thoughtcrime.securesms.linkpreview; - -import java.util.Arrays; -import java.util.HashSet; -import java.util.Set; - -public class LinkPreviewDomains { - public static final String STICKERS = "signal.org"; - - public static final Set LINKS = new HashSet<>(Arrays.asList( - // YouTube - "youtube.com", - "www.youtube.com", - "m.youtube.com", - "youtu.be", - // Reddit - "reddit.com", - "www.reddit.com", - "m.reddit.com", - // Imgur - "imgur.com", - "www.imgur.com", - "m.imgur.com", - // Instagram - "instagram.com", - "www.instagram.com", - "m.instagram.com", - // Pinterest - "pinterest.com", - "www.pinterest.com", - "pin.it", - // Giphy - "giphy.com", - "media.giphy.com", - "media1.giphy.com", - "media2.giphy.com", - "media3.giphy.com", - "gph.is" - )); - - public static final Set IMAGES = new HashSet<>(Arrays.asList( - "ytimg.com", - "cdninstagram.com", - "fbcdn.net", - "redd.it", - "imgur.com", - "pinimg.com", - "giphy.com" - )); -} diff --git a/src/org/thoughtcrime/securesms/net/ContentProxySelector.java b/src/org/thoughtcrime/securesms/net/ContentProxySelector.java index e370042af4..29418f34e3 100644 --- a/src/org/thoughtcrime/securesms/net/ContentProxySelector.java +++ b/src/org/thoughtcrime/securesms/net/ContentProxySelector.java @@ -1,13 +1,9 @@ package org.thoughtcrime.securesms.net; -import android.os.AsyncTask; - -import org.thoughtcrime.securesms.linkpreview.LinkPreviewDomains; import org.thoughtcrime.securesms.logging.Log; import network.loki.messenger.BuildConfig; -import org.thoughtcrime.securesms.util.Util; import java.io.IOException; import java.net.InetSocketAddress; @@ -27,8 +23,7 @@ public class ContentProxySelector extends ProxySelector { private static final Set WHITELISTED_DOMAINS = new HashSet<>(); static { - WHITELISTED_DOMAINS.addAll(LinkPreviewDomains.LINKS); - WHITELISTED_DOMAINS.addAll(LinkPreviewDomains.IMAGES); + WHITELISTED_DOMAINS.add("giphy.com"); } private final List CONTENT = new ArrayList(1) {{ From d669e8a0b10544495bd064a2eaef0b720a587698 Mon Sep 17 00:00:00 2001 From: Ryan ZHAO Date: Fri, 6 Nov 2020 14:50:00 +1100 Subject: [PATCH 3/6] enable link preview for any site --- .../linkpreview/LinkPreviewRepository.java | 86 ++++++++++++------- 1 file changed, 54 insertions(+), 32 deletions(-) diff --git a/src/org/thoughtcrime/securesms/linkpreview/LinkPreviewRepository.java b/src/org/thoughtcrime/securesms/linkpreview/LinkPreviewRepository.java index fc931c0545..5401a992a1 100644 --- a/src/org/thoughtcrime/securesms/linkpreview/LinkPreviewRepository.java +++ b/src/org/thoughtcrime/securesms/linkpreview/LinkPreviewRepository.java @@ -2,13 +2,16 @@ package org.thoughtcrime.securesms.linkpreview; import android.content.Context; import android.graphics.Bitmap; +import android.graphics.BitmapFactory; import android.net.Uri; import androidx.annotation.NonNull; +import androidx.annotation.Nullable; import android.text.Html; import android.text.TextUtils; import com.bumptech.glide.load.engine.DiskCacheStrategy; import com.bumptech.glide.request.FutureTarget; +import com.google.android.gms.common.util.IOUtils; import org.thoughtcrime.securesms.ApplicationContext; import org.thoughtcrime.securesms.attachments.Attachment; @@ -37,6 +40,7 @@ import org.whispersystems.signalservice.api.messages.SignalServiceStickerManifes import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.InputStream; import java.util.concurrent.CancellationException; import java.util.concurrent.ExecutionException; import java.util.regex.Matcher; @@ -73,7 +77,7 @@ public class LinkPreviewRepository implements InjectableType { RequestController getLinkPreview(@NonNull Context context, @NonNull String url, @NonNull Callback> callback) { CompositeRequestController compositeController = new CompositeRequestController(); - if (!LinkPreviewUtil.isWhitelistedLinkUrl(url)) { + if (!LinkPreviewUtil.isValidLinkUrl(url)) { Log.w(TAG, "Tried to get a link preview for a non-whitelisted domain."); callback.onComplete(Optional.absent()); return compositeController; @@ -137,7 +141,7 @@ public class LinkPreviewRepository implements InjectableType { Optional title = getProperty(body, "title"); Optional imageUrl = getProperty(body, "image"); - if (imageUrl.isPresent() && !LinkPreviewUtil.isWhitelistedMediaUrl(imageUrl.get())) { + if (imageUrl.isPresent() && !LinkPreviewUtil.isValidMediaUrl(imageUrl.get())) { Log.i(TAG, "Image URL was invalid or for a non-whitelisted domain. Skipping."); imageUrl = Optional.absent(); } @@ -150,48 +154,34 @@ public class LinkPreviewRepository implements InjectableType { } private @NonNull RequestController fetchThumbnail(@NonNull Context context, @NonNull String imageUrl, @NonNull Callback> callback) { - FutureTarget bitmapFuture = GlideApp.with(context).asBitmap() - .load(new ChunkedImageUrl(imageUrl)) - .skipMemoryCache(true) - .diskCacheStrategy(DiskCacheStrategy.NONE) - .centerInside() - .submit(1024, 1024); - - RequestController controller = () -> bitmapFuture.cancel(false); + Call call = client.newCall(new Request.Builder().url(imageUrl).build()); + CallRequestController controller = new CallRequestController(call); SignalExecutors.UNBOUNDED.execute(() -> { try { - Bitmap bitmap = bitmapFuture.get(); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); + Response response = call.execute(); + if (!response.isSuccessful() || response.body() == null) { + return; + } - bitmap.compress(Bitmap.CompressFormat.JPEG, 80, baos); + InputStream bodyStream = response.body().byteStream(); + controller.setStream(bodyStream); - byte[] bytes = baos.toByteArray(); - Uri uri = BlobProvider.getInstance().forData(bytes).createForSingleSessionInMemory(); - Optional thumbnail = Optional.of(new UriAttachment(uri, - uri, - MediaUtil.IMAGE_JPEG, - AttachmentDatabase.TRANSFER_PROGRESS_STARTED, - bytes.length, - bitmap.getWidth(), - bitmap.getHeight(), - null, - null, - false, - false, - null, - null)); + byte[] data = IOUtils.readInputStreamFully(bodyStream); + Bitmap bitmap = BitmapFactory.decodeByteArray(data, 0, data.length); + Optional thumbnail = bitmapToAttachment(bitmap, Bitmap.CompressFormat.JPEG, MediaUtil.IMAGE_JPEG); + + if (bitmap != null) bitmap.recycle(); callback.onComplete(thumbnail); - } catch (CancellationException | ExecutionException | InterruptedException e) { + } catch (IOException e) { + Log.w(TAG, "Exception during link preview image retrieval.", e); controller.cancel(); callback.onComplete(Optional.absent()); - } finally { - bitmapFuture.cancel(false); } }); - return () -> bitmapFuture.cancel(true); + return controller; } private @NonNull Optional getProperty(@NonNull String searchText, @NonNull String property) { @@ -266,6 +256,38 @@ public class LinkPreviewRepository implements InjectableType { return () -> Log.i(TAG, "Cancelled sticker pack link preview fetch -- no effect."); } + private static Optional bitmapToAttachment(@Nullable Bitmap bitmap, + @NonNull Bitmap.CompressFormat format, + @NonNull String contentType) + { + if (bitmap == null) { + return Optional.absent(); + } + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + + bitmap.compress(format, 80, baos); + + byte[] bytes = baos.toByteArray(); + Uri uri = BlobProvider.getInstance().forData(bytes).createForSingleSessionInMemory(); + + return Optional.of(new UriAttachment(uri, + uri, + contentType, + AttachmentDatabase.TRANSFER_PROGRESS_STARTED, + bytes.length, + bitmap.getWidth(), + bitmap.getHeight(), + null, + null, + false, + false, + null, + null)); + + } + + private static class Metadata { private final Optional title; private final Optional imageUrl; From aabfa5817ce926d6b2427e5fdafc0339fd88f1e2 Mon Sep 17 00:00:00 2001 From: Ryan ZHAO Date: Fri, 6 Nov 2020 16:57:00 +1100 Subject: [PATCH 4/6] fix a UI bug of link preview --- .../thoughtcrime/securesms/conversation/ConversationItem.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/org/thoughtcrime/securesms/conversation/ConversationItem.java b/src/org/thoughtcrime/securesms/conversation/ConversationItem.java index e98ab75a43..612d9d3455 100644 --- a/src/org/thoughtcrime/securesms/conversation/ConversationItem.java +++ b/src/org/thoughtcrime/securesms/conversation/ConversationItem.java @@ -500,7 +500,7 @@ public class ConversationItem extends TapJackingProofLinearLayout private void adjustMarginsIfNeeded(MessageRecord messageRecord) { LinearLayout.LayoutParams bodyTextLayoutParams = (LinearLayout.LayoutParams)bodyText.getLayoutParams(); bodyTextLayoutParams.topMargin = 0; - if (hasOnlyThumbnail(messageRecord)) { + if (hasOnlyThumbnail(messageRecord) || hasLinkPreview(messageRecord)) { int topPadding = 0; if (groupSenderHolder.getVisibility() == VISIBLE) { topPadding = (int)getResources().getDimension(R.dimen.medium_spacing); From 9f495b1ab8fb443ab39f727c160f2ef551f0b118 Mon Sep 17 00:00:00 2001 From: Ryan ZHAO Date: Mon, 16 Nov 2020 17:00:00 +1100 Subject: [PATCH 5/6] ignore case for the key "location" in headers --- .../securesms/net/ContentProxySafetyInterceptor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/org/thoughtcrime/securesms/net/ContentProxySafetyInterceptor.java b/src/org/thoughtcrime/securesms/net/ContentProxySafetyInterceptor.java index dcf6097738..1e89618aee 100644 --- a/src/org/thoughtcrime/securesms/net/ContentProxySafetyInterceptor.java +++ b/src/org/thoughtcrime/securesms/net/ContentProxySafetyInterceptor.java @@ -31,7 +31,7 @@ public class ContentProxySafetyInterceptor implements Interceptor { Response response = chain.proceed(chain.request()); if (response.isRedirect()) { - if (isWhitelisted(response.header("Location"))) { + if (isWhitelisted(response.header("location")) || isWhitelisted(response.header("Location"))) { return response; } else { Log.w(TAG, "Tried to redirect to a non-whitelisted domain!"); From ad016731e4d9ec9a0b6b5d26e8a8d1a111391f22 Mon Sep 17 00:00:00 2001 From: Ryan ZHAO Date: Mon, 16 Nov 2020 17:00:30 +1100 Subject: [PATCH 6/6] more general regex for matching the image and title in a raw html --- .../securesms/linkpreview/LinkPreviewRepository.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/org/thoughtcrime/securesms/linkpreview/LinkPreviewRepository.java b/src/org/thoughtcrime/securesms/linkpreview/LinkPreviewRepository.java index 5401a992a1..9dde7e0fea 100644 --- a/src/org/thoughtcrime/securesms/linkpreview/LinkPreviewRepository.java +++ b/src/org/thoughtcrime/securesms/linkpreview/LinkPreviewRepository.java @@ -187,10 +187,16 @@ public class LinkPreviewRepository implements InjectableType { private @NonNull Optional getProperty(@NonNull String searchText, @NonNull String property) { Pattern pattern = Pattern.compile("<\\s*meta\\s+property\\s*=\\s*\"\\s*og:" + property + "\\s*\"\\s+[^>]*content\\s*=\\s*\"(.*?)\"[^>]*/?\\s*>", Pattern.CASE_INSENSITIVE | Pattern.DOTALL); Matcher matcher = pattern.matcher(searchText); - if (matcher.find()) { String text = Html.fromHtml(matcher.group(1)).toString(); - return TextUtils.isEmpty(text) ? Optional.absent() : Optional.of(text); + if (!TextUtils.isEmpty(text)) { return Optional.of(text); } + } + + pattern = Pattern.compile("<\\s*" + property + "[^>]*>(.*?)<\\s*/" + property + "[^>]*>", Pattern.CASE_INSENSITIVE | Pattern.DOTALL); + matcher = pattern.matcher(searchText); + if (matcher.find()) { + String text = Html.fromHtml(matcher.group(1)).toString(); + if (!TextUtils.isEmpty(text)) { return Optional.of(text); } } return Optional.absent();