From 40fd7ca332d6c87b9f0c2eba76150b24072d6c66 Mon Sep 17 00:00:00 2001 From: Alan Evans Date: Thu, 13 Feb 2020 14:22:21 -0400 Subject: [PATCH] Video trimming behind feature flag. --- .../conversation/ConversationActivity.java | 6 +- .../conversation/ConversationFragment.java | 3 +- .../database/AttachmentDatabase.java | 143 +++++-- .../database/helpers/SQLCipherOpenHelper.java | 7 +- .../jobs/AttachmentCompressionJob.java | 40 +- .../jobs/AttachmentMarkUploadedJob.java | 96 +++++ .../securesms/jobs/JobManagerFactories.java | 3 +- .../mediapreview/MediaPreviewViewModel.java | 10 +- .../ImageEditorModelRenderMediaTransform.java | 56 +++ .../securesms/mediasend/Media.java | 43 ++- .../securesms/mediasend/MediaRepository.java | 54 +-- .../mediasend/MediaSendActivity.java | 24 +- .../mediasend/MediaSendVideoFragment.java | 199 +++++++++- .../mediasend/MediaSendViewModel.java | 27 +- .../securesms/mediasend/MediaTransform.java | 12 + .../mediasend/MediaUploadRepository.java | 8 +- .../mediasend/VideoTrimTransform.java | 33 ++ .../mms/DecryptableStreamLocalUriFetcher.java | 2 +- .../org/thoughtcrime/securesms/mms/Slide.java | 20 +- .../securesms/mms/VideoSlide.java | 9 +- .../securesms/scribbles/VideoEditorHud.java | 130 +++++++ .../securesms/sms/MessageSender.java | 18 +- .../securesms/util/FeatureFlags.java | 8 + .../securesms/util/MediaUtil.java | 4 +- .../video/DecryptableUriVideoInput.java | 54 +++ .../securesms/video/InMemoryTranscoder.java | 33 +- .../securesms/video/VideoPlayer.java | 130 +++++-- .../videoconverter/AudioTrackConverter.java | 3 +- .../video/videoconverter/MediaConverter.java | 81 +--- .../video/videoconverter/OutputSurface.java | 10 +- .../video/videoconverter/TextureRender.java | 15 +- .../video/videoconverter/VideoInput.java | 83 ++++ .../VideoThumbnailsExtractor.java | 182 +++++++++ .../VideoThumbnailsRangeSelectorView.java | 363 ++++++++++++++++++ .../videoconverter/VideoThumbnailsView.java | 228 +++++++++++ .../videoconverter/VideoTrackConverter.java | 2 +- .../drawable/ic_chevron_left_black_8dp.xml | 4 + .../drawable/ic_chevron_right_black_8dp.xml | 4 + .../res/layout/mediasend_video_fragment.xml | 18 +- app/src/main/res/layout/video_editor_hud.xml | 59 +++ app/src/main/res/values/attrs.xml | 10 + 41 files changed, 1966 insertions(+), 268 deletions(-) create mode 100644 app/src/main/java/org/thoughtcrime/securesms/jobs/AttachmentMarkUploadedJob.java create mode 100644 app/src/main/java/org/thoughtcrime/securesms/mediasend/ImageEditorModelRenderMediaTransform.java create mode 100644 app/src/main/java/org/thoughtcrime/securesms/mediasend/MediaTransform.java create mode 100644 app/src/main/java/org/thoughtcrime/securesms/mediasend/VideoTrimTransform.java create mode 100644 app/src/main/java/org/thoughtcrime/securesms/scribbles/VideoEditorHud.java create mode 100644 app/src/main/java/org/thoughtcrime/securesms/video/DecryptableUriVideoInput.java create mode 100644 app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoInput.java create mode 100644 app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoThumbnailsExtractor.java create mode 100644 app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoThumbnailsRangeSelectorView.java create mode 100644 app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoThumbnailsView.java create mode 100644 app/src/main/res/drawable/ic_chevron_left_black_8dp.xml create mode 100644 app/src/main/res/drawable/ic_chevron_right_black_8dp.xml create mode 100644 app/src/main/res/layout/video_editor_hud.xml diff --git a/app/src/main/java/org/thoughtcrime/securesms/conversation/ConversationActivity.java b/app/src/main/java/org/thoughtcrime/securesms/conversation/ConversationActivity.java index 23ddcdcecc..7d5360dc06 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/conversation/ConversationActivity.java +++ b/app/src/main/java/org/thoughtcrime/securesms/conversation/ConversationActivity.java @@ -609,7 +609,7 @@ public class ConversationActivity extends PassphraseRequiredActionBarActivity for (Media mediaItem : result.getNonUploadedMedia()) { if (MediaUtil.isVideoType(mediaItem.getMimeType())) { - slideDeck.addSlide(new VideoSlide(this, mediaItem.getUri(), 0, mediaItem.getCaption().orNull())); + slideDeck.addSlide(new VideoSlide(this, mediaItem.getUri(), 0, mediaItem.getCaption().orNull(), mediaItem.getTransformProperties().orNull())); } else if (MediaUtil.isGif(mediaItem.getMimeType())) { slideDeck.addSlide(new GifSlide(this, mediaItem.getUri(), 0, mediaItem.getWidth(), mediaItem.getHeight(), mediaItem.getCaption().orNull())); } else if (MediaUtil.isImageType(mediaItem.getMimeType())) { @@ -1904,7 +1904,7 @@ public class ConversationActivity extends PassphraseRequiredActionBarActivity openContactShareEditor(uri); return new SettableFuture<>(false); } else if (MediaType.IMAGE.equals(mediaType) || MediaType.GIF.equals(mediaType) || MediaType.VIDEO.equals(mediaType)) { - Media media = new Media(uri, MediaUtil.getMimeType(this, uri), 0, width, height, 0, 0, Optional.absent(), Optional.absent()); + Media media = new Media(uri, MediaUtil.getMimeType(this, uri), 0, width, height, 0, 0, Optional.absent(), Optional.absent(), Optional.absent()); startActivityForResult(MediaSendActivity.buildEditorIntent(ConversationActivity.this, Collections.singletonList(media), recipient.get(), composeText.getTextTrimmed(), sendButton.getSelectedTransport()), MEDIA_SENDER); return new SettableFuture<>(false); } else { @@ -2617,7 +2617,7 @@ public class ConversationActivity extends PassphraseRequiredActionBarActivity private void sendSticker(@NonNull StickerLocator stickerLocator, @NonNull Uri uri, long size, boolean clearCompose) { if (sendButton.getSelectedTransport().isSms()) { - Media media = new Media(uri, MediaUtil.IMAGE_WEBP, System.currentTimeMillis(), StickerSlide.WIDTH, StickerSlide.HEIGHT, size, 0, Optional.absent(), Optional.absent()); + Media media = new Media(uri, MediaUtil.IMAGE_WEBP, System.currentTimeMillis(), StickerSlide.WIDTH, StickerSlide.HEIGHT, size, 0, Optional.absent(), Optional.absent(), Optional.absent()); Intent intent = MediaSendActivity.buildEditorIntent(this, Collections.singletonList(media), recipient.get(), composeText.getTextTrimmed(), sendButton.getSelectedTransport()); startActivityForResult(intent, MEDIA_SENDER); return; diff --git a/app/src/main/java/org/thoughtcrime/securesms/conversation/ConversationFragment.java b/app/src/main/java/org/thoughtcrime/securesms/conversation/ConversationFragment.java index eac0b03a39..d25a1de479 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/conversation/ConversationFragment.java +++ b/app/src/main/java/org/thoughtcrime/securesms/conversation/ConversationFragment.java @@ -605,7 +605,8 @@ public class ConversationFragment extends Fragment attachment.getSize(), 0, Optional.absent(), - Optional.fromNullable(attachment.getCaption()))); + Optional.fromNullable(attachment.getCaption()), + Optional.absent())); } }; diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentDatabase.java b/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentDatabase.java index 45ee683b40..e94c3f97c1 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentDatabase.java +++ b/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentDatabase.java @@ -33,6 +33,7 @@ import androidx.annotation.RequiresApi; import androidx.annotation.VisibleForTesting; import com.bumptech.glide.Glide; +import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import net.sqlcipher.DatabaseUtils; @@ -166,9 +167,12 @@ public class AttachmentDatabase extends Database { "CREATE INDEX IF NOT EXISTS part_mms_id_index ON " + TABLE_NAME + " (" + MMS_ID + ");", "CREATE INDEX IF NOT EXISTS pending_push_index ON " + TABLE_NAME + " (" + TRANSFER_STATE + ");", "CREATE INDEX IF NOT EXISTS part_sticker_pack_id_index ON " + TABLE_NAME + " (" + STICKER_PACK_ID + ");", - "CREATE INDEX IF NOT EXISTS part_data_hash_index ON " + TABLE_NAME + " (" + DATA_HASH + ");" + "CREATE INDEX IF NOT EXISTS part_data_hash_index ON " + TABLE_NAME + " (" + DATA_HASH + ");", + "CREATE INDEX IF NOT EXISTS part_data_index ON " + TABLE_NAME + " (" + DATA + ");" }; + private static final long STANDARD_THUMB_TIME = 1000; + private final ExecutorService thumbnailExecutor = Util.newSingleThreadedLifoExecutor(); private final AttachmentSecret attachmentSecret; @@ -198,7 +202,7 @@ public class AttachmentDatabase extends Database { } try { - InputStream generatedStream = thumbnailExecutor.submit(new ThumbnailFetchCallable(attachmentId)).get(); + InputStream generatedStream = thumbnailExecutor.submit(new ThumbnailFetchCallable(attachmentId, STANDARD_THUMB_TIME)).get(); if (generatedStream == null) throw new FileNotFoundException("No thumbnail stream available: " + attachmentId); else return generatedStream; @@ -525,7 +529,7 @@ public class AttachmentDatabase extends Database { notifyConversationListListeners(); } - thumbnailExecutor.submit(new ThumbnailFetchCallable(attachmentId)); + thumbnailExecutor.submit(new ThumbnailFetchCallable(attachmentId, STANDARD_THUMB_TIME)); } private static @Nullable String getBlurHashStringOrNull(@Nullable BlurHash blurHash) { @@ -671,9 +675,14 @@ public class AttachmentDatabase extends Database { return insertedAttachments; } + /** + * @param onlyModifyThisAttachment If false and more than one attachment shares this file, they will all up updated. + * If true, then guarantees not to affect other attachments. + */ public void updateAttachmentData(@NonNull DatabaseAttachment databaseAttachment, - @NonNull MediaStream mediaStream) - throws MmsException + @NonNull MediaStream mediaStream, + boolean onlyModifyThisAttachment) + throws MmsException, IOException { SQLiteDatabase database = databaseHelper.getWritableDatabase(); DataInfo oldDataInfo = getAttachmentDataFileInfo(databaseAttachment.getAttachmentId(), DATA); @@ -682,7 +691,16 @@ public class AttachmentDatabase extends Database { throw new MmsException("No attachment data found!"); } - DataInfo dataInfo = setAttachmentData(oldDataInfo.file, + File destination = oldDataInfo.file; + + if (onlyModifyThisAttachment) { + if (fileReferencedByMoreThanOneAttachment(destination)) { + Log.i(TAG, "Creating a new file as this one is used by more than one attachment"); + destination = newFile(); + } + } + + DataInfo dataInfo = setAttachmentData(destination, mediaStream.getStream(), false, databaseAttachment.getAttachmentId()); @@ -700,19 +718,37 @@ public class AttachmentDatabase extends Database { Log.i(TAG, "[updateAttachmentData] Updated " + updateCount + " rows."); } + /** + * Returns true if the file referenced by two or more attachments. + * Returns false if the file is referenced by zero or one attachments. + */ + private boolean fileReferencedByMoreThanOneAttachment(@NonNull File file) { + SQLiteDatabase database = databaseHelper.getReadableDatabase(); + String selection = DATA + " = ?"; + String[] args = new String[]{file.getAbsolutePath()}; + + try (Cursor cursor = database.query(TABLE_NAME, null, selection, args, null, null, null, "2")) { + return cursor != null && cursor.moveToFirst() && cursor.moveToNext(); + } + } + public void markAttachmentAsTransformed(@NonNull AttachmentId attachmentId) { + updateAttachmentTransformProperties(attachmentId, TransformProperties.forSkipTransform()); + } + + public void updateAttachmentTransformProperties(@NonNull AttachmentId attachmentId, @NonNull TransformProperties transformProperties) { DataInfo dataInfo = getAttachmentDataFileInfo(attachmentId, DATA); if (dataInfo == null) { - Log.w(TAG, "[markAttachmentAsTransformed] No data info found!"); + Log.w(TAG, "[updateAttachmentTransformProperties] No data info found!"); return; } ContentValues contentValues = new ContentValues(); - contentValues.put(TRANSFORM_PROPERTIES, TransformProperties.forSkipTransform().serialize()); + contentValues.put(TRANSFORM_PROPERTIES, transformProperties.serialize()); int updateCount = updateAttachmentAndMatchingHashes(databaseHelper.getWritableDatabase(), attachmentId, dataInfo.hash, contentValues); - Log.i(TAG, "[markAttachmentAsTransformed] Updated " + updateCount + " rows."); + Log.i(TAG, "[updateAttachmentTransformProperties] Updated " + updateCount + " rows."); } public @NonNull File getOrCreateTransferFile(@NonNull AttachmentId attachmentId) throws IOException { @@ -925,14 +961,18 @@ public class AttachmentDatabase extends Database { throws MmsException { try { - File partsDirectory = context.getDir(DIRECTORY, Context.MODE_PRIVATE); - File dataFile = File.createTempFile("part", ".mms", partsDirectory); + File dataFile = newFile(); return setAttachmentData(dataFile, in, isThumbnail, attachmentId); } catch (IOException e) { throw new MmsException(e); } } + private File newFile() throws IOException { + File partsDirectory = context.getDir(DIRECTORY, Context.MODE_PRIVATE); + return File.createTempFile("part", ".mms", partsDirectory); + } + private @NonNull DataInfo setAttachmentData(@NonNull File destination, @NonNull InputStream in, boolean isThumbnail, @@ -1098,9 +1138,10 @@ public class AttachmentDatabase extends Database { { Log.d(TAG, "Inserting attachment for mms id: " + mmsId); - SQLiteDatabase database = databaseHelper.getWritableDatabase(); - DataInfo dataInfo = null; - long uniqueId = System.currentTimeMillis(); + SQLiteDatabase database = databaseHelper.getWritableDatabase(); + DataInfo dataInfo = null; + long uniqueId = System.currentTimeMillis(); + long thumbnailTimeUs; if (attachment.getDataUri() != null) { dataInfo = setAttachmentData(attachment.getDataUri(), false, null); @@ -1135,8 +1176,15 @@ public class AttachmentDatabase extends Database { contentValues.put(HEIGHT, template.getHeight()); contentValues.put(QUOTE, quote); contentValues.put(CAPTION, attachment.getCaption()); - contentValues.put(BLUR_HASH, getBlurHashStringOrNull(attachment.getBlurHash())); - contentValues.put(TRANSFORM_PROPERTIES, template.getTransformProperties().serialize()); + if (attachment.getTransformProperties().isVideoEdited()) { + contentValues.putNull(BLUR_HASH); + contentValues.put(TRANSFORM_PROPERTIES, attachment.getTransformProperties().serialize()); + thumbnailTimeUs = Math.max(STANDARD_THUMB_TIME, attachment.getTransformProperties().videoTrimStartTimeUs); + } else { + contentValues.put(BLUR_HASH, getBlurHashStringOrNull(attachment.getBlurHash())); + contentValues.put(TRANSFORM_PROPERTIES, template.getTransformProperties().serialize()); + thumbnailTimeUs = STANDARD_THUMB_TIME; + } if (attachment.isSticker()) { contentValues.put(STICKER_PACK_ID, attachment.getSticker().getPackId()); @@ -1148,7 +1196,11 @@ public class AttachmentDatabase extends Database { contentValues.put(DATA, dataInfo.file.getAbsolutePath()); contentValues.put(SIZE, dataInfo.length); contentValues.put(DATA_RANDOM, dataInfo.random); - contentValues.put(DATA_HASH, dataInfo.hash); + if (attachment.getTransformProperties().isVideoEdited()) { + contentValues.putNull(DATA_HASH); + } else { + contentValues.put(DATA_HASH, dataInfo.hash); + } } boolean notifyPacks = attachment.isSticker() && !hasStickerAttachments(); @@ -1170,8 +1222,8 @@ public class AttachmentDatabase extends Database { } if (!hasThumbnail && dataInfo != null) { - if (MediaUtil.hasVideoThumbnail(attachment.getDataUri())) { - Bitmap bitmap = MediaUtil.getVideoThumbnail(context, attachment.getDataUri()); + if (MediaUtil.hasVideoThumbnail(attachment.getDataUri()) && thumbnailTimeUs == STANDARD_THUMB_TIME) { + Bitmap bitmap = MediaUtil.getVideoThumbnail(context, attachment.getDataUri(), thumbnailTimeUs); if (bitmap != null) { try (ThumbnailData thumbnailData = new ThumbnailData(bitmap)) { @@ -1179,11 +1231,11 @@ public class AttachmentDatabase extends Database { } } else { Log.w(TAG, "Retrieving video thumbnail failed, submitting thumbnail generation job..."); - thumbnailExecutor.submit(new ThumbnailFetchCallable(attachmentId)); + thumbnailExecutor.submit(new ThumbnailFetchCallable(attachmentId, thumbnailTimeUs)); } } else { Log.i(TAG, "Submitting thumbnail generation job..."); - thumbnailExecutor.submit(new ThumbnailFetchCallable(attachmentId)); + thumbnailExecutor.submit(new ThumbnailFetchCallable(attachmentId, thumbnailTimeUs)); } } @@ -1241,9 +1293,11 @@ public class AttachmentDatabase extends Database { class ThumbnailFetchCallable implements Callable { private final AttachmentId attachmentId; + private final long timeUs; - ThumbnailFetchCallable(AttachmentId attachmentId) { + ThumbnailFetchCallable(AttachmentId attachmentId, long timeUs) { this.attachmentId = attachmentId; + this.timeUs = timeUs; } @Override @@ -1263,7 +1317,7 @@ public class AttachmentDatabase extends Database { if (MediaUtil.isVideoType(attachment.getContentType())) { - try (ThumbnailData data = generateVideoThumbnail(attachmentId)) { + try (ThumbnailData data = generateVideoThumbnail(attachmentId, timeUs)) { if (data != null) { updateAttachmentThumbnail(attachmentId, data.toDataStream(), data.getAspectRatio()); @@ -1276,7 +1330,7 @@ public class AttachmentDatabase extends Database { return null; } - private ThumbnailData generateVideoThumbnail(AttachmentId attachmentId) throws IOException { + private ThumbnailData generateVideoThumbnail(AttachmentId attachmentId, long timeUs) throws IOException { if (Build.VERSION.SDK_INT < 23) { Log.w(TAG, "Video thumbnails not supported..."); return null; @@ -1288,7 +1342,7 @@ public class AttachmentDatabase extends Database { MediaMetadataRetriever retriever = new MediaMetadataRetriever(); MediaMetadataRetrieverUtil.setDataSource(retriever, dataSource); - Bitmap bitmap = retriever.getFrameAtTime(1000); + Bitmap bitmap = retriever.getFrameAtTime(timeUs); Log.i(TAG, "Generated video thumbnail..."); return bitmap != null ? new ThumbnailData(bitmap) : null; @@ -1325,23 +1379,54 @@ public class AttachmentDatabase extends Database { public static final class TransformProperties { @JsonProperty private final boolean skipTransform; + @JsonProperty private final boolean videoTrim; + @JsonProperty private final long videoTrimStartTimeUs; + @JsonProperty private final long videoTrimEndTimeUs; - public TransformProperties(@JsonProperty("skipTransform") boolean skipTransform) { - this.skipTransform = skipTransform; + @JsonCreator + public TransformProperties(@JsonProperty("skipTransform") boolean skipTransform, + @JsonProperty("videoTrim") boolean videoTrim, + @JsonProperty("videoTrimStartTimeUs") long videoTrimStartTimeUs, + @JsonProperty("videoTrimEndTimeUs") long videoTrimEndTimeUs) + { + this.skipTransform = skipTransform; + this.videoTrim = videoTrim; + this.videoTrimStartTimeUs = videoTrimStartTimeUs; + this.videoTrimEndTimeUs = videoTrimEndTimeUs; } public static @NonNull TransformProperties empty() { - return new TransformProperties(false); + return new TransformProperties(false, false, 0, 0); } public static @NonNull TransformProperties forSkipTransform() { - return new TransformProperties(true); + return new TransformProperties(true, false, 0, 0); + } + + public static @NonNull TransformProperties forVideoTrim(long videoTrimStartTimeUs, long videoTrimEndTimeUs) { + return new TransformProperties(false, true, videoTrimStartTimeUs, videoTrimEndTimeUs); } public boolean shouldSkipTransform() { return skipTransform; } + public boolean isVideoEdited() { + return isVideoTrim(); + } + + public boolean isVideoTrim() { + return videoTrim; + } + + public long getVideoTrimStartTimeUs() { + return videoTrimStartTimeUs; + } + + public long getVideoTrimEndTimeUs() { + return videoTrimEndTimeUs; + } + @NonNull String serialize() { return JsonUtil.toJson(this); } diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/helpers/SQLCipherOpenHelper.java b/app/src/main/java/org/thoughtcrime/securesms/database/helpers/SQLCipherOpenHelper.java index 969bed1282..7fe80af4df 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/database/helpers/SQLCipherOpenHelper.java +++ b/app/src/main/java/org/thoughtcrime/securesms/database/helpers/SQLCipherOpenHelper.java @@ -110,8 +110,9 @@ public class SQLCipherOpenHelper extends SQLiteOpenHelper { private static final int MEGAPHONE_FIRST_APPEARANCE = 46; private static final int PROFILE_KEY_TO_DB = 47; private static final int PROFILE_KEY_CREDENTIALS = 48; + private static final int ATTACHMENT_FILE_INDEX = 49; - private static final int DATABASE_VERSION = 48; + private static final int DATABASE_VERSION = 49; private static final String DATABASE_NAME = "signal.db"; private final Context context; @@ -748,6 +749,10 @@ public class SQLCipherOpenHelper extends SQLiteOpenHelper { db.execSQL("ALTER TABLE recipient ADD COLUMN profile_key_credential TEXT DEFAULT NULL"); } + if (oldVersion < ATTACHMENT_FILE_INDEX) { + db.execSQL("CREATE INDEX IF NOT EXISTS part_data_index ON part (_data)"); + } + db.setTransactionSuccessful(); } finally { db.endTransaction(); diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/AttachmentCompressionJob.java b/app/src/main/java/org/thoughtcrime/securesms/jobs/AttachmentCompressionJob.java index 520124be8d..ef8dc1cc80 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/AttachmentCompressionJob.java +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/AttachmentCompressionJob.java @@ -4,8 +4,6 @@ import android.content.Context; import android.media.MediaDataSource; import androidx.annotation.NonNull; -import androidx.annotation.Nullable; -import androidx.annotation.RequiresApi; import org.greenrobot.eventbus.EventBus; import org.thoughtcrime.securesms.R; @@ -29,7 +27,6 @@ import org.thoughtcrime.securesms.transport.UndeliverableMessageException; import org.thoughtcrime.securesms.util.BitmapDecodingException; import org.thoughtcrime.securesms.util.BitmapUtil; import org.thoughtcrime.securesms.util.MediaUtil; -import org.thoughtcrime.securesms.util.MemoryFileDescriptor; import org.thoughtcrime.securesms.util.MemoryFileDescriptor.MemoryFileException; import org.thoughtcrime.securesms.video.InMemoryTranscoder; import org.thoughtcrime.securesms.video.VideoSizeException; @@ -143,17 +140,20 @@ public final class AttachmentCompressionJob extends BaseJob { throws UndeliverableMessageException { try { - if (MediaUtil.isVideo(attachment) && MediaConstraints.isVideoTranscodeAvailable()) { + if (MediaUtil.isVideo(attachment)) { transcodeVideoIfNeededToDatabase(context, attachmentDatabase, attachment, constraints, EventBus.getDefault(), this::isCanceled); + if (!constraints.isSatisfied(context, attachment)) { + throw new UndeliverableMessageException("Size constraints could not be met on video!"); + } } else if (constraints.isSatisfied(context, attachment)) { if (MediaUtil.isJpeg(attachment)) { MediaStream stripped = getResizedMedia(context, attachment, constraints); - attachmentDatabase.updateAttachmentData(attachment, stripped); + attachmentDatabase.updateAttachmentData(attachment, stripped, false); attachmentDatabase.markAttachmentAsTransformed(attachmentId); } } else if (constraints.canResize(attachment)) { MediaStream resized = getResizedMedia(context, attachment, constraints); - attachmentDatabase.updateAttachmentData(attachment, resized); + attachmentDatabase.updateAttachmentData(attachment, resized, false); attachmentDatabase.markAttachmentAsTransformed(attachmentId); } else { throw new UndeliverableMessageException("Size constraints could not be met!"); @@ -163,7 +163,6 @@ public final class AttachmentCompressionJob extends BaseJob { } } - @RequiresApi(26) private static void transcodeVideoIfNeededToDatabase(@NonNull Context context, @NonNull AttachmentDatabase attachmentDatabase, @NonNull DatabaseAttachment attachment, @@ -172,6 +171,17 @@ public final class AttachmentCompressionJob extends BaseJob { @NonNull InMemoryTranscoder.CancelationSignal cancelationSignal) throws UndeliverableMessageException { + AttachmentDatabase.TransformProperties transformProperties = attachment.getTransformProperties(); + + boolean allowSkipOnFailure = false; + + if (!MediaConstraints.isVideoTranscodeAvailable()) { + if (transformProperties.isVideoEdited()) { + throw new UndeliverableMessageException("Video edited, but transcode is not available"); + } + return; + } + try (NotificationController notification = GenericForegroundService.startForegroundTask(context, context.getString(R.string.AttachmentUploadJob_compressing_video_start))) { notification.setIndeterminateProgress(); @@ -182,10 +192,14 @@ public final class AttachmentCompressionJob extends BaseJob { throw new UndeliverableMessageException("Cannot get media data source for attachment."); } - try (InMemoryTranscoder transcoder = new InMemoryTranscoder(context, dataSource, constraints.getCompressedVideoMaxSize(context))) { + allowSkipOnFailure = !transformProperties.isVideoEdited(); + InMemoryTranscoder.Options options = null; + if (transformProperties.isVideoTrim()) { + options = new InMemoryTranscoder.Options(transformProperties.getVideoTrimStartTimeUs(), transformProperties.getVideoTrimEndTimeUs()); + } + try (InMemoryTranscoder transcoder = new InMemoryTranscoder(context, dataSource, options, constraints.getCompressedVideoMaxSize(context))) { if (transcoder.isTranscodeRequired()) { - MediaStream mediaStream = transcoder.transcode(percent -> { notification.setProgress(100, percent); eventBus.postSticky(new PartProgressEvent(attachment, @@ -194,7 +208,7 @@ public final class AttachmentCompressionJob extends BaseJob { percent)); }, cancelationSignal); - attachmentDatabase.updateAttachmentData(attachment, mediaStream); + attachmentDatabase.updateAttachmentData(attachment, mediaStream, transformProperties.isVideoEdited()); attachmentDatabase.markAttachmentAsTransformed(attachment.getAttachmentId()); } } @@ -203,7 +217,11 @@ public final class AttachmentCompressionJob extends BaseJob { if (attachment.getSize() > constraints.getVideoMaxSize(context)) { throw new UndeliverableMessageException("Duration not found, attachment too large to skip transcode", e); } else { - Log.w(TAG, "Problem with video source, but video small enough to skip transcode", e); + if (allowSkipOnFailure) { + Log.w(TAG, "Problem with video source, but video small enough to skip transcode", e); + } else { + throw new UndeliverableMessageException("Failed to transcode and cannot skip due to editing", e); + } } } catch (IOException | MmsException | VideoSizeException e) { throw new UndeliverableMessageException("Failed to transcode", e); diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/AttachmentMarkUploadedJob.java b/app/src/main/java/org/thoughtcrime/securesms/jobs/AttachmentMarkUploadedJob.java new file mode 100644 index 0000000000..b74e483fdf --- /dev/null +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/AttachmentMarkUploadedJob.java @@ -0,0 +1,96 @@ +package org.thoughtcrime.securesms.jobs; + +import androidx.annotation.NonNull; + +import org.thoughtcrime.securesms.attachments.AttachmentId; +import org.thoughtcrime.securesms.attachments.DatabaseAttachment; +import org.thoughtcrime.securesms.database.AttachmentDatabase; +import org.thoughtcrime.securesms.database.DatabaseFactory; +import org.thoughtcrime.securesms.jobmanager.Data; +import org.thoughtcrime.securesms.jobmanager.Job; +import org.thoughtcrime.securesms.logging.Log; + +import java.io.IOException; +import java.util.concurrent.TimeUnit; + +/** + * Only marks an attachment as uploaded. + */ +public final class AttachmentMarkUploadedJob extends BaseJob { + + public static final String KEY = "AttachmentMarkUploadedJob"; + + @SuppressWarnings("unused") + private static final String TAG = Log.tag(AttachmentMarkUploadedJob.class); + + private static final String KEY_ROW_ID = "row_id"; + private static final String KEY_UNIQUE_ID = "unique_id"; + private static final String KEY_MESSAGE_ID = "message_id"; + + private final AttachmentId attachmentId; + private final long messageId; + + public AttachmentMarkUploadedJob(long messageId, @NonNull AttachmentId attachmentId) { + this(new Parameters.Builder() + .setLifespan(TimeUnit.DAYS.toMillis(1)) + .setMaxAttempts(Parameters.UNLIMITED) + .build(), + messageId, + attachmentId); + } + + private AttachmentMarkUploadedJob(@NonNull Parameters parameters, long messageId, @NonNull AttachmentId attachmentId) { + super(parameters); + this.attachmentId = attachmentId; + this.messageId = messageId; + } + + @Override + public @NonNull Data serialize() { + return new Data.Builder().putLong(KEY_ROW_ID, attachmentId.getRowId()) + .putLong(KEY_UNIQUE_ID, attachmentId.getUniqueId()) + .putLong(KEY_MESSAGE_ID, messageId) + .build(); + } + + @Override + public @NonNull String getFactoryKey() { + return KEY; + } + + @Override + public void onRun() throws Exception { + AttachmentDatabase database = DatabaseFactory.getAttachmentDatabase(context); + DatabaseAttachment databaseAttachment = database.getAttachment(attachmentId); + + if (databaseAttachment == null) { + throw new InvalidAttachmentException("Cannot find the specified attachment."); + } + + database.markAttachmentUploaded(messageId, databaseAttachment); + } + + @Override + public void onFailure() { + } + + @Override + protected boolean onShouldRetry(@NonNull Exception exception) { + return exception instanceof IOException; + } + + private class InvalidAttachmentException extends Exception { + InvalidAttachmentException(String message) { + super(message); + } + } + + public static final class Factory implements Job.Factory { + @Override + public @NonNull AttachmentMarkUploadedJob create(@NonNull Parameters parameters, @NonNull Data data) { + return new AttachmentMarkUploadedJob(parameters, + data.getLong(KEY_MESSAGE_ID), + new AttachmentId(data.getLong(KEY_ROW_ID), data.getLong(KEY_UNIQUE_ID))); + } + } +} diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/JobManagerFactories.java b/app/src/main/java/org/thoughtcrime/securesms/jobs/JobManagerFactories.java index aebbe19d60..ae289774f4 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/JobManagerFactories.java +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/JobManagerFactories.java @@ -26,8 +26,8 @@ import org.thoughtcrime.securesms.migrations.LegacyMigrationJob; import org.thoughtcrime.securesms.migrations.MigrationCompleteJob; import org.thoughtcrime.securesms.migrations.RecipientSearchMigrationJob; import org.thoughtcrime.securesms.migrations.RegistrationPinV2MigrationJob; -import org.thoughtcrime.securesms.migrations.StickerLaunchMigrationJob; import org.thoughtcrime.securesms.migrations.StickerAdditionMigrationJob; +import org.thoughtcrime.securesms.migrations.StickerLaunchMigrationJob; import org.thoughtcrime.securesms.migrations.UuidMigrationJob; import java.util.Arrays; @@ -42,6 +42,7 @@ public final class JobManagerFactories { put(AttachmentCopyJob.KEY, new AttachmentCopyJob.Factory()); put(AttachmentDownloadJob.KEY, new AttachmentDownloadJob.Factory()); put(AttachmentUploadJob.KEY, new AttachmentUploadJob.Factory()); + put(AttachmentMarkUploadedJob.KEY, new AttachmentMarkUploadedJob.Factory()); put(AttachmentCompressionJob.KEY, new AttachmentCompressionJob.Factory()); put(AvatarDownloadJob.KEY, new AvatarDownloadJob.Factory()); put(CleanPreKeysJob.KEY, new CleanPreKeysJob.Factory()); diff --git a/app/src/main/java/org/thoughtcrime/securesms/mediapreview/MediaPreviewViewModel.java b/app/src/main/java/org/thoughtcrime/securesms/mediapreview/MediaPreviewViewModel.java index c6514d458b..be1094eb17 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/mediapreview/MediaPreviewViewModel.java +++ b/app/src/main/java/org/thoughtcrime/securesms/mediapreview/MediaPreviewViewModel.java @@ -1,13 +1,14 @@ package org.thoughtcrime.securesms.mediapreview; -import androidx.lifecycle.LiveData; -import androidx.lifecycle.MutableLiveData; -import androidx.lifecycle.ViewModel; import android.content.Context; import android.database.Cursor; import android.net.Uri; + import androidx.annotation.NonNull; import androidx.annotation.Nullable; +import androidx.lifecycle.LiveData; +import androidx.lifecycle.MutableLiveData; +import androidx.lifecycle.ViewModel; import org.thoughtcrime.securesms.database.MediaDatabase.MediaRecord; import org.thoughtcrime.securesms.mediasend.Media; @@ -112,7 +113,8 @@ public class MediaPreviewViewModel extends ViewModel { mediaRecord.getAttachment().getSize(), 0, Optional.absent(), - Optional.fromNullable(mediaRecord.getAttachment().getCaption())); + Optional.fromNullable(mediaRecord.getAttachment().getCaption()), + Optional.absent()); } public LiveData getPreviewData() { diff --git a/app/src/main/java/org/thoughtcrime/securesms/mediasend/ImageEditorModelRenderMediaTransform.java b/app/src/main/java/org/thoughtcrime/securesms/mediasend/ImageEditorModelRenderMediaTransform.java new file mode 100644 index 0000000000..9c20f81c50 --- /dev/null +++ b/app/src/main/java/org/thoughtcrime/securesms/mediasend/ImageEditorModelRenderMediaTransform.java @@ -0,0 +1,56 @@ +package org.thoughtcrime.securesms.mediasend; + +import android.content.Context; +import android.graphics.Bitmap; +import android.net.Uri; + +import androidx.annotation.NonNull; +import androidx.annotation.WorkerThread; + +import org.thoughtcrime.securesms.imageeditor.model.EditorModel; +import org.thoughtcrime.securesms.logging.Log; +import org.thoughtcrime.securesms.providers.BlobProvider; +import org.thoughtcrime.securesms.util.MediaUtil; +import org.whispersystems.libsignal.util.guava.Optional; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +public final class ImageEditorModelRenderMediaTransform implements MediaTransform { + + private static final String TAG = Log.tag(ImageEditorModelRenderMediaTransform.class); + + private final EditorModel modelToRender; + + ImageEditorModelRenderMediaTransform(@NonNull EditorModel modelToRender) { + this.modelToRender = modelToRender; + } + + @WorkerThread + @Override + public @NonNull Media transform(@NonNull Context context, @NonNull Media media) { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + + Bitmap bitmap = modelToRender.render(context); + try { + bitmap.compress(Bitmap.CompressFormat.JPEG, 80, outputStream); + + Uri uri = BlobProvider.getInstance() + .forData(outputStream.toByteArray()) + .withMimeType(MediaUtil.IMAGE_JPEG) + .createForSingleSessionOnDisk(context); + + return new Media(uri, MediaUtil.IMAGE_JPEG, media.getDate(), bitmap.getWidth(), bitmap.getHeight(), outputStream.size(), 0, media.getBucketId(), media.getCaption(), Optional.absent()); + } catch (IOException e) { + Log.w(TAG, "Failed to render image. Using base image."); + return media; + } finally { + bitmap.recycle(); + try { + outputStream.close(); + } catch (IOException e) { + Log.w(TAG, e); + } + } + } +} diff --git a/app/src/main/java/org/thoughtcrime/securesms/mediasend/Media.java b/app/src/main/java/org/thoughtcrime/securesms/mediasend/Media.java index 7b18bed177..2f8aa4a6e1 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/mediasend/Media.java +++ b/app/src/main/java/org/thoughtcrime/securesms/mediasend/Media.java @@ -3,9 +3,14 @@ package org.thoughtcrime.securesms.mediasend; import android.net.Uri; import android.os.Parcel; import android.os.Parcelable; + import androidx.annotation.NonNull; +import org.thoughtcrime.securesms.database.AttachmentDatabase; import org.whispersystems.libsignal.util.guava.Optional; +import org.whispersystems.signalservice.internal.util.JsonUtil; + +import java.io.IOException; /** * Represents a piece of media that the user has on their device. @@ -22,8 +27,9 @@ public class Media implements Parcelable { private final long size; private final long duration; - private Optional bucketId; - private Optional caption; + private Optional bucketId; + private Optional caption; + private Optional transformProperties; public Media(@NonNull Uri uri, @NonNull String mimeType, @@ -33,17 +39,19 @@ public class Media implements Parcelable { long size, long duration, Optional bucketId, - Optional caption) + Optional caption, + Optional transformProperties) { - this.uri = uri; - this.mimeType = mimeType; - this.date = date; - this.width = width; - this.height = height; - this.size = size; - this.duration = duration; - this.bucketId = bucketId; - this.caption = caption; + this.uri = uri; + this.mimeType = mimeType; + this.date = date; + this.width = width; + this.height = height; + this.size = size; + this.duration = duration; + this.bucketId = bucketId; + this.caption = caption; + this.transformProperties = transformProperties; } protected Media(Parcel in) { @@ -56,6 +64,12 @@ public class Media implements Parcelable { duration = in.readLong(); bucketId = Optional.fromNullable(in.readString()); caption = Optional.fromNullable(in.readString()); + try { + String json = in.readString(); + transformProperties = json == null ? Optional.absent() : Optional.fromNullable(JsonUtil.fromJson(json, AttachmentDatabase.TransformProperties.class)); + } catch (IOException e) { + throw new AssertionError(e); + } } public Uri getUri() { @@ -98,6 +112,10 @@ public class Media implements Parcelable { this.caption = Optional.fromNullable(caption); } + public Optional getTransformProperties() { + return transformProperties; + } + @Override public int describeContents() { return 0; @@ -114,6 +132,7 @@ public class Media implements Parcelable { dest.writeLong(duration); dest.writeString(bucketId.orNull()); dest.writeString(caption.orNull()); + dest.writeString(transformProperties.transform(JsonUtil::toJson).orNull()); } public static final Creator CREATOR = new Creator() { diff --git a/app/src/main/java/org/thoughtcrime/securesms/mediasend/MediaRepository.java b/app/src/main/java/org/thoughtcrime/securesms/mediasend/MediaRepository.java index 6a0f6476a8..806dc82a9e 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/mediasend/MediaRepository.java +++ b/app/src/main/java/org/thoughtcrime/securesms/mediasend/MediaRepository.java @@ -4,31 +4,28 @@ import android.Manifest; import android.annotation.TargetApi; import android.content.Context; import android.database.Cursor; -import android.graphics.Bitmap; import android.net.Uri; import android.os.Environment; import android.provider.MediaStore.Images; import android.provider.MediaStore.Video; import android.provider.OpenableColumns; +import android.util.Pair; + import androidx.annotation.NonNull; import androidx.annotation.Nullable; import androidx.annotation.WorkerThread; -import android.util.Pair; import com.annimon.stream.Stream; import org.thoughtcrime.securesms.R; -import org.thoughtcrime.securesms.imageeditor.model.EditorModel; import org.thoughtcrime.securesms.logging.Log; import org.thoughtcrime.securesms.mms.PartAuthority; import org.thoughtcrime.securesms.permissions.Permissions; -import org.thoughtcrime.securesms.providers.BlobProvider; import org.thoughtcrime.securesms.util.MediaUtil; import org.thoughtcrime.securesms.util.Util; import org.thoughtcrime.securesms.util.concurrent.SignalExecutors; import org.whispersystems.libsignal.util.guava.Optional; -import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; import java.util.ArrayList; @@ -77,12 +74,12 @@ public class MediaRepository { SignalExecutors.BOUNDED.execute(() -> callback.onComplete(getMostRecentItem(context))); } - void renderMedia(@NonNull Context context, - @NonNull List currentMedia, - @NonNull Map modelsToRender, - @NonNull Callback> callback) + static void transformMedia(@NonNull Context context, + @NonNull List currentMedia, + @NonNull Map modelsToTransform, + @NonNull Callback> callback) { - SignalExecutors.BOUNDED.execute(() -> callback.onComplete(renderMedia(context, currentMedia, modelsToRender))); + SignalExecutors.BOUNDED.execute(() -> callback.onComplete(transformMedia(context, currentMedia, modelsToTransform))); } @WorkerThread @@ -220,7 +217,7 @@ public class MediaRepository { long size = cursor.getLong(cursor.getColumnIndexOrThrow(Images.Media.SIZE)); long duration = !isImage ? cursor.getInt(cursor.getColumnIndexOrThrow(Video.Media.DURATION)) : 0; - media.add(new Media(uri, mimetype, date, width, height, size, duration, Optional.of(bucketId), Optional.absent())); + media.add(new Media(uri, mimetype, date, width, height, size, duration, Optional.of(bucketId), Optional.absent(), Optional.absent())); } } @@ -249,35 +246,16 @@ public class MediaRepository { } @WorkerThread - private LinkedHashMap renderMedia(@NonNull Context context, - @NonNull List currentMedia, - @NonNull Map modelsToRender) + private static LinkedHashMap transformMedia(@NonNull Context context, + @NonNull List currentMedia, + @NonNull Map modelsToTransform) { LinkedHashMap updatedMedia = new LinkedHashMap<>(currentMedia.size()); - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); for (Media media : currentMedia) { - EditorModel modelToRender = modelsToRender.get(media); - if (modelToRender != null) { - Bitmap bitmap = modelToRender.render(context); - try { - outputStream.reset(); - bitmap.compress(Bitmap.CompressFormat.JPEG, 80, outputStream); - - Uri uri = BlobProvider.getInstance() - .forData(outputStream.toByteArray()) - .withMimeType(MediaUtil.IMAGE_JPEG) - .createForSingleSessionOnDisk(context); - - Media updated = new Media(uri, MediaUtil.IMAGE_JPEG, media.getDate(), bitmap.getWidth(), bitmap.getHeight(), outputStream.size(), 0, media.getBucketId(), media.getCaption()); - - updatedMedia.put(media, updated); - } catch (IOException e) { - Log.w(TAG, "Failed to render image. Using base image."); - updatedMedia.put(media, media); - } finally { - bitmap.recycle(); - } + MediaTransform transformer = modelsToTransform.get(media); + if (transformer != null) { + updatedMedia.put(media, transformer.transform(context, media)); } else { updatedMedia.put(media, media); } @@ -333,7 +311,7 @@ public class MediaRepository { height = dimens.second; } - return new Media(media.getUri(), media.getMimeType(), media.getDate(), width, height, size, 0, media.getBucketId(), media.getCaption()); + return new Media(media.getUri(), media.getMimeType(), media.getDate(), width, height, size, 0, media.getBucketId(), media.getCaption(), Optional.absent()); } private Media getContentResolverPopulatedMedia(@NonNull Context context, @NonNull Media media) throws IOException { @@ -359,7 +337,7 @@ public class MediaRepository { height = dimens.second; } - return new Media(media.getUri(), media.getMimeType(), media.getDate(), width, height, size, 0, media.getBucketId(), media.getCaption()); + return new Media(media.getUri(), media.getMimeType(), media.getDate(), width, height, size, 0, media.getBucketId(), media.getCaption(), Optional.absent()); } private static class FolderResult { diff --git a/app/src/main/java/org/thoughtcrime/securesms/mediasend/MediaSendActivity.java b/app/src/main/java/org/thoughtcrime/securesms/mediasend/MediaSendActivity.java index 5c0c0b34b5..1cd6c73b21 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/mediasend/MediaSendActivity.java +++ b/app/src/main/java/org/thoughtcrime/securesms/mediasend/MediaSendActivity.java @@ -87,6 +87,7 @@ import java.util.Map; public class MediaSendActivity extends PassphraseRequiredActionBarActivity implements MediaPickerFolderFragment.Controller, MediaPickerItemFragment.Controller, ImageEditorFragment.Controller, + MediaSendVideoFragment.Controller, CameraFragment.Controller, CameraContactSelectionFragment.Controller, ViewTreeObserver.OnGlobalLayoutListener, @@ -346,6 +347,11 @@ public class MediaSendActivity extends PassphraseRequiredActionBarActivity imple navigateToMediaSend(Locale.getDefault()); } + @Override + public void onVideoBeginEdit(@NonNull Uri uri) { + viewModel.onVideoBeginEdit(uri); + } + @Override public void onTouchEventsNeeded(boolean needed) { MediaSendFragment fragment = (MediaSendFragment) getSupportFragmentManager().findFragmentByTag(TAG_SEND); @@ -414,6 +420,7 @@ public class MediaSendActivity extends PassphraseRequiredActionBarActivity imple length, 0, Optional.of(Media.ALL_MEDIA_BUCKET_ID), + Optional.absent(), Optional.absent() ); } catch (IOException e) { @@ -512,7 +519,7 @@ public class MediaSendActivity extends PassphraseRequiredActionBarActivity imple MediaSendFragment fragment = getMediaSendFragment(); if (fragment != null) { - viewModel.onSendClicked(buildModelsToRender(fragment), recipients).observe(this, result -> { + viewModel.onSendClicked(buildModelsToTransform(fragment), recipients).observe(this, result -> { finish(); }); } else { @@ -533,13 +540,13 @@ public class MediaSendActivity extends PassphraseRequiredActionBarActivity imple sendButton.setEnabled(false); - viewModel.onSendClicked(buildModelsToRender(fragment), Collections.emptyList()).observe(this, this::setActivityResultAndFinish); + viewModel.onSendClicked(buildModelsToTransform(fragment), Collections.emptyList()).observe(this, this::setActivityResultAndFinish); } - private Map buildModelsToRender(@NonNull MediaSendFragment fragment) { + private static Map buildModelsToTransform(@NonNull MediaSendFragment fragment) { List mediaList = fragment.getAllMedia(); Map savedState = fragment.getSavedState(); - Map modelsToRender = new HashMap<>(); + Map modelsToRender = new HashMap<>(); for (Media media : mediaList) { Object state = savedState.get(media.getUri()); @@ -547,7 +554,14 @@ public class MediaSendActivity extends PassphraseRequiredActionBarActivity imple if (state instanceof ImageEditorFragment.Data) { EditorModel model = ((ImageEditorFragment.Data) state).readModel(); if (model != null && model.isChanged()) { - modelsToRender.put(media, model); + modelsToRender.put(media, new ImageEditorModelRenderMediaTransform(model)); + } + } + + if (state instanceof MediaSendVideoFragment.Data) { + MediaSendVideoFragment.Data data = (MediaSendVideoFragment.Data) state; + if (data.durationEdited) { + modelsToRender.put(media, new VideoTrimTransform(data)); } } } diff --git a/app/src/main/java/org/thoughtcrime/securesms/mediasend/MediaSendVideoFragment.java b/app/src/main/java/org/thoughtcrime/securesms/mediasend/MediaSendVideoFragment.java index c6b45190f3..7d91173110 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/mediasend/MediaSendVideoFragment.java +++ b/app/src/main/java/org/thoughtcrime/securesms/mediasend/MediaSendVideoFragment.java @@ -1,28 +1,45 @@ package org.thoughtcrime.securesms.mediasend; import android.net.Uri; +import android.os.Build; import android.os.Bundle; -import androidx.annotation.NonNull; -import androidx.annotation.Nullable; -import androidx.fragment.app.Fragment; +import android.os.Handler; import android.view.LayoutInflater; import android.view.View; import android.view.ViewGroup; +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; +import androidx.annotation.RequiresApi; +import androidx.fragment.app.Fragment; + import org.thoughtcrime.securesms.R; import org.thoughtcrime.securesms.logging.Log; +import org.thoughtcrime.securesms.mms.MediaConstraints; import org.thoughtcrime.securesms.mms.VideoSlide; +import org.thoughtcrime.securesms.scribbles.VideoEditorHud; +import org.thoughtcrime.securesms.util.FeatureFlags; +import org.thoughtcrime.securesms.util.Throttler; import org.thoughtcrime.securesms.video.VideoPlayer; import java.io.IOException; -public class MediaSendVideoFragment extends Fragment implements MediaSendPageFragment { +public class MediaSendVideoFragment extends Fragment implements VideoEditorHud.EventListener, + MediaSendPageFragment { - private static final String TAG = MediaSendVideoFragment.class.getSimpleName(); + private static final String TAG = Log.tag(MediaSendVideoFragment.class); private static final String KEY_URI = "uri"; - private Uri uri; + private final Throttler videoScanThrottle = new Throttler(150); + private final Handler handler = new Handler(); + + private Controller controller; + private Data data = new Data(); + private Uri uri; + private VideoPlayer player; + private VideoEditorHud hud; + private Runnable updatePosition; public static MediaSendVideoFragment newInstance(@NonNull Uri uri) { Bundle args = new Bundle(); @@ -34,6 +51,15 @@ public class MediaSendVideoFragment extends Fragment implements MediaSendPageFra return fragment; } + @Override + public void onCreate(@Nullable Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + if (!(getActivity() instanceof Controller)) { + throw new IllegalStateException("Parent activity must implement Controller interface."); + } + controller = (Controller) getActivity(); + } + @Override public @Nullable View onCreateView(@NonNull LayoutInflater inflater, @Nullable ViewGroup container, @Nullable Bundle savedInstanceState) { return inflater.inflate(R.layout.mediasend_video_fragment, container, false); @@ -43,19 +69,50 @@ public class MediaSendVideoFragment extends Fragment implements MediaSendPageFra public void onViewCreated(@NonNull View view, @Nullable Bundle savedInstanceState) { super.onViewCreated(view, savedInstanceState); - uri = getArguments().getParcelable(KEY_URI); + player = view.findViewById(R.id.video_player); + + uri = requireArguments().getParcelable(KEY_URI); VideoSlide slide = new VideoSlide(requireContext(), uri, 0); - ((VideoPlayer) view).setWindow(requireActivity().getWindow()); - ((VideoPlayer) view).setVideoSource(slide, true); + player.setWindow(requireActivity().getWindow()); + player.setVideoSource(slide, true); + + if (FeatureFlags.videoTrimming() && MediaConstraints.isVideoTranscodeAvailable()) { + hud = view.findViewById(R.id.video_editor_hud); + hud.setEventListener(this); + updateHud(data); + if (data.durationEdited) { + player.clip(data.startTimeUs, data.endTimeUs, true); + } + try { + hud.setVideoSource(slide); + hud.setVisibility(View.VISIBLE); + startPositionUpdates(); + } catch (IOException e) { + Log.w(TAG, e); + } + + player.setPlayerCallback(new VideoPlayer.PlayerCallback() { + + @Override + public void onPlaying() { + hud.playing(); + } + + @Override + public void onStopped() { + hud.stopped(); + } + }); + } } @Override public void onDestroyView() { super.onDestroyView(); - if (getView() != null) { - ((VideoPlayer) getView()).cleanup(); + if (player != null) { + player.cleanup(); } } @@ -63,6 +120,32 @@ public class MediaSendVideoFragment extends Fragment implements MediaSendPageFra public void onPause() { super.onPause(); notifyHidden(); + + stopPositionUpdates(); + } + + @Override + public void onResume() { + super.onResume(); + startPositionUpdates(); + } + + private void startPositionUpdates() { + if (hud != null && Build.VERSION.SDK_INT >= 23) { + stopPositionUpdates(); + updatePosition = new Runnable() { + @Override + public void run() { + hud.setPosition(player.getPlaybackPositionUs()); + handler.postDelayed(this, 100); + } + }; + handler.post(updatePosition); + } + } + + private void stopPositionUpdates() { + handler.removeCallbacks(updatePosition); } @Override @@ -84,22 +167,106 @@ public class MediaSendVideoFragment extends Fragment implements MediaSendPageFra @Override public @Nullable View getPlaybackControls() { - VideoPlayer player = (VideoPlayer) getView(); + if (hud != null && hud.getVisibility() == View.VISIBLE) return null; + return player != null ? player.getControlView() : null; } @Override public @Nullable Object saveState() { - return null; + return data; } @Override - public void restoreState(@NonNull Object state) { } + public void restoreState(@NonNull Object state) { + if (state instanceof Data) { + data = (Data) state; + if (Build.VERSION.SDK_INT >= 23) { + updateHud(data); + } + } else { + Log.w(TAG, "Received a bad saved state. Received class: " + state.getClass().getName()); + } + } + + @RequiresApi(api = 23) + private void updateHud(Data data) { + if (hud != null && data.totalDurationUs > 0 && data.durationEdited) { + hud.setDurationRange(data.totalDurationUs, data.startTimeUs, data.endTimeUs); + } + } @Override public void notifyHidden() { - if (getView() != null) { - ((VideoPlayer) getView()).pause(); + if (player != null) { + player.pause(); } } + + @Override + public void onEditVideoDuration(long totalDurationUs, long startTimeUs, long endTimeUs, boolean fromEdited, boolean editingComplete) { + controller.onTouchEventsNeeded(!editingComplete); + + boolean wasEdited = data.durationEdited; + boolean durationEdited = startTimeUs > 0 || endTimeUs < totalDurationUs; + + data.durationEdited = durationEdited; + data.totalDurationUs = totalDurationUs; + data.startTimeUs = startTimeUs; + data.endTimeUs = endTimeUs; + + if (editingComplete) { + videoScanThrottle.clear(); + } + + videoScanThrottle.publish(() -> { + player.pause(); + if (!editingComplete) { + player.removeClip(false); + } + player.setPlaybackPosition(fromEdited || editingComplete ? startTimeUs / 1000 : endTimeUs / 1000); + if (editingComplete) { + if (durationEdited) { + player.clip(startTimeUs, endTimeUs, true); + } else { + player.removeClip(true); + } + } + }); + + if (!wasEdited && durationEdited) { + controller.onVideoBeginEdit(uri); + } + } + + @Override + public void onPlay() { + player.playFromStart(); + } + + @Override + public void onSeek(long position, boolean dragComplete) { + if (dragComplete) { + videoScanThrottle.clear(); + } + + videoScanThrottle.publish(() -> { + player.pause(); + player.setPlaybackPosition(position); + }); + } + + static class Data { + boolean durationEdited; + long totalDurationUs; + long startTimeUs; + long endTimeUs; + } + + public interface Controller { + + void onTouchEventsNeeded(boolean needed); + + void onVideoBeginEdit(@NonNull Uri uri); + } } diff --git a/app/src/main/java/org/thoughtcrime/securesms/mediasend/MediaSendViewModel.java b/app/src/main/java/org/thoughtcrime/securesms/mediasend/MediaSendViewModel.java index 5b8ce62d56..e9630b4f77 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/mediasend/MediaSendViewModel.java +++ b/app/src/main/java/org/thoughtcrime/securesms/mediasend/MediaSendViewModel.java @@ -1,24 +1,22 @@ package org.thoughtcrime.securesms.mediasend; import android.app.Application; +import android.content.Context; +import android.net.Uri; +import android.text.TextUtils; +import androidx.annotation.NonNull; import androidx.annotation.Nullable; import androidx.annotation.WorkerThread; import androidx.lifecycle.LiveData; import androidx.lifecycle.MutableLiveData; import androidx.lifecycle.ViewModel; import androidx.lifecycle.ViewModelProvider; -import android.content.Context; -import android.net.Uri; -import androidx.annotation.NonNull; - -import android.text.TextUtils; import com.annimon.stream.Stream; import org.thoughtcrime.securesms.TransportOption; import org.thoughtcrime.securesms.database.ThreadDatabase; -import org.thoughtcrime.securesms.imageeditor.model.EditorModel; import org.thoughtcrime.securesms.logging.Log; import org.thoughtcrime.securesms.mms.MediaConstraints; import org.thoughtcrime.securesms.mms.OutgoingMediaMessage; @@ -34,7 +32,6 @@ import org.thoughtcrime.securesms.util.MessageUtil; import org.thoughtcrime.securesms.util.SingleLiveEvent; import org.thoughtcrime.securesms.util.TextSecurePreferences; import org.thoughtcrime.securesms.util.Util; -import org.whispersystems.libsignal.util.Pair; import org.whispersystems.libsignal.util.guava.Optional; import org.whispersystems.libsignal.util.guava.Preconditions; @@ -303,7 +300,7 @@ class MediaSendViewModel extends ViewModel { captionVisible = false; List uncaptioned = Stream.of(getSelectedMediaOrDefault()) - .map(m -> new Media(m.getUri(), m.getMimeType(), m.getDate(), m.getWidth(), m.getHeight(), m.getSize(), m.getDuration(), m.getBucketId(), Optional.absent())) + .map(m -> new Media(m.getUri(), m.getMimeType(), m.getDate(), m.getWidth(), m.getHeight(), m.getSize(), m.getDuration(), m.getBucketId(), Optional.absent(), Optional.absent())) .toList(); selectedMedia.setValue(uncaptioned); @@ -405,6 +402,10 @@ class MediaSendViewModel extends ViewModel { hudState.setValue(buildHudState()); } + void onVideoBeginEdit(@NonNull Uri uri) { + cancelUpload(new Media(uri, "", 0, 0, 0, 0, 0, Optional.absent(), Optional.absent(), Optional.absent())); + } + void onMediaCaptured(@NonNull Media media) { lastCameraCapture = Optional.of(media); @@ -449,7 +450,7 @@ class MediaSendViewModel extends ViewModel { savedDrawState.putAll(state); } - @NonNull LiveData onSendClicked(Map modelsToRender, @NonNull List recipients) { + @NonNull LiveData onSendClicked(Map modelsToTransform, @NonNull List recipients) { if (isSms && recipients.size() > 0) { throw new IllegalStateException("Provided recipients to send to, but this is SMS!"); } @@ -463,9 +464,13 @@ class MediaSendViewModel extends ViewModel { Util.runOnMainDelayed(dialogRunnable, 250); - repository.renderMedia(application, initialMedia, modelsToRender, (oldToNew) -> { + MediaRepository.transformMedia(application, initialMedia, modelsToTransform, (oldToNew) -> { List updatedMedia = new ArrayList<>(oldToNew.values()); + for (Media media : updatedMedia){ + Log.w(TAG, media.getUri().toString() + " : " + media.getTransformProperties().transform(t->"" + t.isVideoTrim()).or("null")); + } + if (isSms || MessageSender.isLocalSelfSend(application, recipient, isSms)) { Log.i(TAG, "SMS or local self-send. Skipping pre-upload."); result.postValue(MediaSendActivityResult.forTraditionalSend(updatedMedia, trimmedBody, transport, isViewOnce())); @@ -477,7 +482,7 @@ class MediaSendViewModel extends ViewModel { if (splitMessage.getTextSlide().isPresent()) { Slide slide = splitMessage.getTextSlide().get(); - uploadRepository.startUpload(new Media(Objects.requireNonNull(slide.getUri()), slide.getContentType(), System.currentTimeMillis(), 0, 0, slide.getFileSize(), 0, Optional.absent(), Optional.absent()), recipient); + uploadRepository.startUpload(new Media(Objects.requireNonNull(slide.getUri()), slide.getContentType(), System.currentTimeMillis(), 0, 0, slide.getFileSize(), 0, Optional.absent(), Optional.absent(), Optional.absent()), recipient); } uploadRepository.applyMediaUpdates(oldToNew, recipient); diff --git a/app/src/main/java/org/thoughtcrime/securesms/mediasend/MediaTransform.java b/app/src/main/java/org/thoughtcrime/securesms/mediasend/MediaTransform.java new file mode 100644 index 0000000000..34a3e67ab9 --- /dev/null +++ b/app/src/main/java/org/thoughtcrime/securesms/mediasend/MediaTransform.java @@ -0,0 +1,12 @@ +package org.thoughtcrime.securesms.mediasend; + +import android.content.Context; + +import androidx.annotation.NonNull; +import androidx.annotation.WorkerThread; + +public interface MediaTransform { + + @WorkerThread + @NonNull Media transform(@NonNull Context context, @NonNull Media media); +} diff --git a/app/src/main/java/org/thoughtcrime/securesms/mediasend/MediaUploadRepository.java b/app/src/main/java/org/thoughtcrime/securesms/mediasend/MediaUploadRepository.java index 79c2201b5c..4e1e9318b7 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/mediasend/MediaUploadRepository.java +++ b/app/src/main/java/org/thoughtcrime/securesms/mediasend/MediaUploadRepository.java @@ -78,7 +78,9 @@ class MediaUploadRepository { void applyMediaUpdates(@NonNull Map oldToNew, @Nullable Recipient recipient) { executor.execute(() -> { for (Map.Entry entry : oldToNew.entrySet()) { - if (!entry.getKey().equals(entry.getValue()) || !uploadResults.containsKey(entry.getValue())) { + + boolean same = entry.getKey().equals(entry.getValue()) && (!entry.getValue().getTransformProperties().isPresent() || !entry.getValue().getTransformProperties().get().isVideoEdited()); + if (!same || !uploadResults.containsKey(entry.getValue())) { cancelUploadInternal(entry.getKey()); uploadMediaInternal(entry.getValue(), recipient); } @@ -187,9 +189,9 @@ class MediaUploadRepository { } } - private static @NonNull Attachment asAttachment(@NonNull Context context, @NonNull Media media) { + public static @NonNull Attachment asAttachment(@NonNull Context context, @NonNull Media media) { if (MediaUtil.isVideoType(media.getMimeType())) { - return new VideoSlide(context, media.getUri(), 0, media.getCaption().orNull()).asAttachment(); + return new VideoSlide(context, media.getUri(), 0, media.getCaption().orNull(), media.getTransformProperties().orNull()).asAttachment(); } else if (MediaUtil.isGif(media.getMimeType())) { return new GifSlide(context, media.getUri(), 0, media.getWidth(), media.getHeight(), media.getCaption().orNull()).asAttachment(); } else if (MediaUtil.isImageType(media.getMimeType())) { diff --git a/app/src/main/java/org/thoughtcrime/securesms/mediasend/VideoTrimTransform.java b/app/src/main/java/org/thoughtcrime/securesms/mediasend/VideoTrimTransform.java new file mode 100644 index 0000000000..1f769d7ebb --- /dev/null +++ b/app/src/main/java/org/thoughtcrime/securesms/mediasend/VideoTrimTransform.java @@ -0,0 +1,33 @@ +package org.thoughtcrime.securesms.mediasend; + +import android.content.Context; + +import androidx.annotation.NonNull; +import androidx.annotation.WorkerThread; + +import org.thoughtcrime.securesms.database.AttachmentDatabase; +import org.whispersystems.libsignal.util.guava.Optional; + +public final class VideoTrimTransform implements MediaTransform { + + private final MediaSendVideoFragment.Data data; + + VideoTrimTransform(@NonNull MediaSendVideoFragment.Data data) { + this.data = data; + } + + @WorkerThread + @Override + public @NonNull Media transform(@NonNull Context context, @NonNull Media media) { + return new Media(media.getUri(), + media.getMimeType(), + media.getDate(), + media.getWidth(), + media.getHeight(), + media.getSize(), + media.getDuration(), + media.getBucketId(), + media.getCaption(), + Optional.of(new AttachmentDatabase.TransformProperties(false, data.durationEdited, data.startTimeUs, data.endTimeUs))); + } +} diff --git a/app/src/main/java/org/thoughtcrime/securesms/mms/DecryptableStreamLocalUriFetcher.java b/app/src/main/java/org/thoughtcrime/securesms/mms/DecryptableStreamLocalUriFetcher.java index b6fc1ac1a5..9559962c9f 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/mms/DecryptableStreamLocalUriFetcher.java +++ b/app/src/main/java/org/thoughtcrime/securesms/mms/DecryptableStreamLocalUriFetcher.java @@ -30,7 +30,7 @@ class DecryptableStreamLocalUriFetcher extends StreamLocalUriFetcher { @Override protected InputStream loadResource(Uri uri, ContentResolver contentResolver) throws FileNotFoundException { if (MediaUtil.hasVideoThumbnail(uri)) { - Bitmap thumbnail = MediaUtil.getVideoThumbnail(context, uri); + Bitmap thumbnail = MediaUtil.getVideoThumbnail(context, uri, 1000); if (thumbnail != null) { ByteArrayOutputStream baos = new ByteArrayOutputStream(); diff --git a/app/src/main/java/org/thoughtcrime/securesms/mms/Slide.java b/app/src/main/java/org/thoughtcrime/securesms/mms/Slide.java index 5febeece34..49f39079cd 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/mms/Slide.java +++ b/app/src/main/java/org/thoughtcrime/securesms/mms/Slide.java @@ -157,6 +157,24 @@ public abstract class Slide { @Nullable BlurHash blurHash, boolean voiceNote, boolean quote) + { + return constructAttachmentFromUri(context, uri, defaultMime, size, width, height, hasThumbnail, fileName, caption, stickerLocator, blurHash, voiceNote, quote, null); + } + + protected static Attachment constructAttachmentFromUri(@NonNull Context context, + @NonNull Uri uri, + @NonNull String defaultMime, + long size, + int width, + int height, + boolean hasThumbnail, + @Nullable String fileName, + @Nullable String caption, + @Nullable StickerLocator stickerLocator, + @Nullable BlurHash blurHash, + boolean voiceNote, + boolean quote, + @Nullable AttachmentDatabase.TransformProperties transformProperties) { String resolvedType = Optional.fromNullable(MediaUtil.getMimeType(context, uri)).or(defaultMime); String fastPreflightId = String.valueOf(new SecureRandom().nextLong()); @@ -174,7 +192,7 @@ public abstract class Slide { caption, stickerLocator, blurHash, - null); + transformProperties); } public @NonNull Optional getFileType(@NonNull Context context) { diff --git a/app/src/main/java/org/thoughtcrime/securesms/mms/VideoSlide.java b/app/src/main/java/org/thoughtcrime/securesms/mms/VideoSlide.java index 509b381fc5..191ab5f547 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/mms/VideoSlide.java +++ b/app/src/main/java/org/thoughtcrime/securesms/mms/VideoSlide.java @@ -19,24 +19,25 @@ package org.thoughtcrime.securesms.mms; import android.content.Context; import android.content.res.Resources.Theme; import android.net.Uri; + import androidx.annotation.DrawableRes; import androidx.annotation.NonNull; import androidx.annotation.Nullable; import org.thoughtcrime.securesms.R; import org.thoughtcrime.securesms.attachments.Attachment; -import org.thoughtcrime.securesms.blurhash.BlurHash; +import org.thoughtcrime.securesms.database.AttachmentDatabase; import org.thoughtcrime.securesms.util.MediaUtil; import org.thoughtcrime.securesms.util.ResUtil; public class VideoSlide extends Slide { public VideoSlide(Context context, Uri uri, long dataSize) { - this(context, uri, dataSize, null); + this(context, uri, dataSize, null, null); } - public VideoSlide(Context context, Uri uri, long dataSize, @Nullable String caption) { - super(context, constructAttachmentFromUri(context, uri, MediaUtil.VIDEO_UNSPECIFIED, dataSize, 0, 0, MediaUtil.hasVideoThumbnail(uri), null, caption, null, null, false, false)); + public VideoSlide(Context context, Uri uri, long dataSize, @Nullable String caption, @Nullable AttachmentDatabase.TransformProperties transformProperties) { + super(context, constructAttachmentFromUri(context, uri, MediaUtil.VIDEO_UNSPECIFIED, dataSize, 0, 0, MediaUtil.hasVideoThumbnail(uri), null, caption, null, null, false, false, transformProperties)); } public VideoSlide(Context context, Attachment attachment) { diff --git a/app/src/main/java/org/thoughtcrime/securesms/scribbles/VideoEditorHud.java b/app/src/main/java/org/thoughtcrime/securesms/scribbles/VideoEditorHud.java new file mode 100644 index 0000000000..ebb78d369e --- /dev/null +++ b/app/src/main/java/org/thoughtcrime/securesms/scribbles/VideoEditorHud.java @@ -0,0 +1,130 @@ +package org.thoughtcrime.securesms.scribbles; + +import android.content.Context; +import android.net.Uri; +import android.util.AttributeSet; +import android.view.View; +import android.widget.LinearLayout; + +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; +import androidx.annotation.RequiresApi; + +import org.thoughtcrime.securesms.R; +import org.thoughtcrime.securesms.logging.Log; +import org.thoughtcrime.securesms.mms.VideoSlide; +import org.thoughtcrime.securesms.video.DecryptableUriVideoInput; +import org.thoughtcrime.securesms.video.videoconverter.VideoThumbnailsRangeSelectorView; + +import java.io.IOException; + +/** + * The HUD (heads-up display) that contains all of the tools for editing video. + */ +public final class VideoEditorHud extends LinearLayout { + + @SuppressWarnings("unused") + private static final String TAG = Log.tag(VideoEditorHud.class); + + private VideoThumbnailsRangeSelectorView videoTimeLine; + private EventListener eventListener; + private View playOverlay; + + public VideoEditorHud(@NonNull Context context) { + super(context); + initialize(); + } + + public VideoEditorHud(@NonNull Context context, @Nullable AttributeSet attrs) { + super(context, attrs); + initialize(); + } + + public VideoEditorHud(@NonNull Context context, @Nullable AttributeSet attrs, int defStyleAttr) { + super(context, attrs, defStyleAttr); + initialize(); + } + + private void initialize() { + View root = inflate(getContext(), R.layout.video_editor_hud, this); + setOrientation(VERTICAL); + + videoTimeLine = root.findViewById(R.id.video_timeline); + playOverlay = root.findViewById(R.id.play_overlay); + + playOverlay.setOnClickListener(v -> eventListener.onPlay()); + } + + public void setEventListener(EventListener eventListener) { + this.eventListener = eventListener; + } + + @RequiresApi(api = 23) + public void setVideoSource(VideoSlide slide) throws IOException { + Uri uri = slide.getUri(); + + if (uri == null || !slide.hasVideo()) { + return; + } + + videoTimeLine.setInput(DecryptableUriVideoInput.createForUri(getContext(), uri)); + + videoTimeLine.setOnRangeChangeListener(new VideoThumbnailsRangeSelectorView.OnRangeChangeListener() { + + @Override + public void onPositionDrag(long position) { + if (eventListener != null) { + eventListener.onSeek(position, false); + } + } + + @Override + public void onEndPositionDrag(long position) { + if (eventListener != null) { + eventListener.onSeek(position, true); + } + } + + @Override + public void onRangeDrag(long minValue, long maxValue, long duration, VideoThumbnailsRangeSelectorView.Thumb thumb) { + if (eventListener != null) { + eventListener.onEditVideoDuration(duration, minValue, maxValue, thumb == VideoThumbnailsRangeSelectorView.Thumb.MIN, false); + } + } + + @Override + public void onRangeDragEnd(long minValue, long maxValue, long duration, VideoThumbnailsRangeSelectorView.Thumb thumb) { + if (eventListener != null) { + eventListener.onEditVideoDuration(duration, minValue, maxValue, thumb == VideoThumbnailsRangeSelectorView.Thumb.MIN, true); + } + } + }); + } + + public void playing() { + playOverlay.setVisibility(INVISIBLE); + } + + public void stopped() { + playOverlay.setVisibility(VISIBLE); + } + + @RequiresApi(api = 23) + public void setDurationRange(long totalDuration, long fromDuration, long toDuration) { + videoTimeLine.setRange(fromDuration, toDuration); + } + + @RequiresApi(api = 23) + public void setPosition(long playbackPositionUs) { + videoTimeLine.setActualPosition(playbackPositionUs); + } + + public interface EventListener { + + void onEditVideoDuration(long totalDurationUs, long startTimeUs, long endTimeUs, boolean fromEdited, boolean editingComplete); + + void onPlay(); + + void onSeek(long position, boolean dragComplete); + } +} diff --git a/app/src/main/java/org/thoughtcrime/securesms/sms/MessageSender.java b/app/src/main/java/org/thoughtcrime/securesms/sms/MessageSender.java index f28277b411..4e34ace138 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/sms/MessageSender.java +++ b/app/src/main/java/org/thoughtcrime/securesms/sms/MessageSender.java @@ -46,8 +46,9 @@ import org.thoughtcrime.securesms.database.model.SmsMessageRecord; import org.thoughtcrime.securesms.dependencies.ApplicationDependencies; import org.thoughtcrime.securesms.jobmanager.Job; import org.thoughtcrime.securesms.jobmanager.JobManager; -import org.thoughtcrime.securesms.jobs.AttachmentCopyJob; import org.thoughtcrime.securesms.jobs.AttachmentCompressionJob; +import org.thoughtcrime.securesms.jobs.AttachmentCopyJob; +import org.thoughtcrime.securesms.jobs.AttachmentMarkUploadedJob; import org.thoughtcrime.securesms.jobs.AttachmentUploadJob; import org.thoughtcrime.securesms.jobs.MmsSendJob; import org.thoughtcrime.securesms.jobs.PushGroupSendJob; @@ -437,15 +438,22 @@ public class MessageSender { private static void sendLocalMediaSelf(Context context, long messageId) { try { ExpiringMessageManager expirationManager = ApplicationContext.getInstance(context).getExpiringMessageManager(); - AttachmentDatabase attachmentDatabase = DatabaseFactory.getAttachmentDatabase(context); MmsDatabase mmsDatabase = DatabaseFactory.getMmsDatabase(context); MmsSmsDatabase mmsSmsDatabase = DatabaseFactory.getMmsSmsDatabase(context); OutgoingMediaMessage message = mmsDatabase.getOutgoingMessage(messageId); SyncMessageId syncId = new SyncMessageId(Recipient.self().getId(), message.getSentTimeMillis()); - for (Attachment attachment : message.getAttachments()) { - attachmentDatabase.markAttachmentUploaded(messageId, attachment); - } + List compressionJobs = Stream.of(message.getAttachments()) + .map(a -> AttachmentCompressionJob.fromAttachment((DatabaseAttachment) a, false, -1)) + .toList(); + + List fakeUploadJobs = Stream.of(message.getAttachments()) + .map(a -> new AttachmentMarkUploadedJob(messageId, ((DatabaseAttachment) a).getAttachmentId())) + .toList(); + + ApplicationDependencies.getJobManager().startChain(compressionJobs) + .then(fakeUploadJobs) + .enqueue(); mmsDatabase.markAsSent(messageId, true); mmsDatabase.markUnidentified(messageId, true); diff --git a/app/src/main/java/org/thoughtcrime/securesms/util/FeatureFlags.java b/app/src/main/java/org/thoughtcrime/securesms/util/FeatureFlags.java index a6b7f92cc7..88e1c07e8a 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/util/FeatureFlags.java +++ b/app/src/main/java/org/thoughtcrime/securesms/util/FeatureFlags.java @@ -54,6 +54,7 @@ public final class FeatureFlags { private static final String PINS_FOR_ALL = generateKey("pinsForAll"); private static final String PINS_MEGAPHONE_KILL_SWITCH = generateKey("pinsMegaphoneKillSwitch"); private static final String PROFILE_NAMES_MEGAPHONE_ENABLED = generateKey("profileNamesMegaphoneEnabled"); + private static final String VIDEO_TRIMMING = generateKey("videoTrimming"); /** * We will only store remote values for flags in this set. If you want a flag to be controllable @@ -61,6 +62,7 @@ public final class FeatureFlags { */ private static final Set REMOTE_CAPABLE = Sets.newHashSet( + VIDEO_TRIMMING, PINS_FOR_ALL, PINS_MEGAPHONE_KILL_SWITCH, PROFILE_NAMES_MEGAPHONE_ENABLED @@ -84,6 +86,7 @@ public final class FeatureFlags { * more burden on the reader to ensure that the app experience remains consistent. */ private static final Set HOT_SWAPPABLE = Sets.newHashSet( + VIDEO_TRIMMING, PINS_MEGAPHONE_KILL_SWITCH ); @@ -174,6 +177,11 @@ public final class FeatureFlags { TextSecurePreferences.getFirstInstallVersion(ApplicationDependencies.getApplication()) < 600; } + /** Allow trimming videos. */ + public static boolean videoTrimming() { + return getValue(VIDEO_TRIMMING, false); + } + /** Only for rendering debug info. */ public static synchronized @NonNull Map getMemoryValues() { return new TreeMap<>(REMOTE_VALUES); diff --git a/app/src/main/java/org/thoughtcrime/securesms/util/MediaUtil.java b/app/src/main/java/org/thoughtcrime/securesms/util/MediaUtil.java index 1a61a92a77..3304025cba 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/util/MediaUtil.java +++ b/app/src/main/java/org/thoughtcrime/securesms/util/MediaUtil.java @@ -300,7 +300,7 @@ public class MediaUtil { } @WorkerThread - public static @Nullable Bitmap getVideoThumbnail(Context context, Uri uri) { + public static @Nullable Bitmap getVideoThumbnail(Context context, Uri uri, long timeUs) { if ("com.android.providers.media.documents".equals(uri.getAuthority())) { long videoId = Long.parseLong(uri.getLastPathSegment().split(":")[1]); @@ -327,7 +327,7 @@ public class MediaUtil { MediaMetadataRetriever mediaMetadataRetriever = new MediaMetadataRetriever(); MediaMetadataRetrieverUtil.setDataSource(mediaMetadataRetriever, mediaDataSource); - return mediaMetadataRetriever.getFrameAtTime(1000); + return mediaMetadataRetriever.getFrameAtTime(timeUs); } catch (IOException e) { Log.w(TAG, "failed to get thumbnail for video blob uri: " + uri, e); return null; diff --git a/app/src/main/java/org/thoughtcrime/securesms/video/DecryptableUriVideoInput.java b/app/src/main/java/org/thoughtcrime/securesms/video/DecryptableUriVideoInput.java new file mode 100644 index 0000000000..92a33a80d4 --- /dev/null +++ b/app/src/main/java/org/thoughtcrime/securesms/video/DecryptableUriVideoInput.java @@ -0,0 +1,54 @@ +package org.thoughtcrime.securesms.video; + +import android.content.Context; +import android.media.MediaDataSource; +import android.net.Uri; + +import androidx.annotation.NonNull; +import androidx.annotation.RequiresApi; + +import org.thoughtcrime.securesms.attachments.AttachmentId; +import org.thoughtcrime.securesms.database.DatabaseFactory; +import org.thoughtcrime.securesms.mms.PartAuthority; +import org.thoughtcrime.securesms.mms.PartUriParser; +import org.thoughtcrime.securesms.providers.BlobProvider; +import org.thoughtcrime.securesms.video.videoconverter.VideoInput; + +import java.io.IOException; + +@RequiresApi(api = 23) +public final class DecryptableUriVideoInput { + + private DecryptableUriVideoInput() { + } + + public static VideoInput createForUri(@NonNull Context context, @NonNull Uri uri) throws IOException { + + if (BlobProvider.isAuthority(uri)) { + return new VideoInput.MediaDataSourceVideoInput(BlobProvider.getInstance().getMediaDataSource(context, uri)); + } + + if (PartAuthority.isLocalUri(uri)) { + return createForAttachmentUri(context, uri); + } + + return new VideoInput.UriVideoInput(context, uri); + } + + private static VideoInput createForAttachmentUri(@NonNull Context context, @NonNull Uri uri) { + AttachmentId partId = new PartUriParser(uri).getPartId(); + + if (!partId.isValid()) { + throw new AssertionError(); + } + + MediaDataSource mediaDataSource = DatabaseFactory.getAttachmentDatabase(context) + .mediaDataSourceFor(partId); + + if (mediaDataSource == null) { + throw new AssertionError(); + } + + return new VideoInput.MediaDataSourceVideoInput(mediaDataSource); + } +} diff --git a/app/src/main/java/org/thoughtcrime/securesms/video/InMemoryTranscoder.java b/app/src/main/java/org/thoughtcrime/securesms/video/InMemoryTranscoder.java index 5fd4010603..62ea89230a 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/video/InMemoryTranscoder.java +++ b/app/src/main/java/org/thoughtcrime/securesms/video/InMemoryTranscoder.java @@ -15,6 +15,7 @@ import org.thoughtcrime.securesms.mms.MediaStream; import org.thoughtcrime.securesms.util.MemoryFileDescriptor; import org.thoughtcrime.securesms.video.videoconverter.EncodingException; import org.thoughtcrime.securesms.video.videoconverter.MediaConverter; +import org.thoughtcrime.securesms.video.videoconverter.VideoInput; import java.io.Closeable; import java.io.FileDescriptor; @@ -46,15 +47,17 @@ public final class InMemoryTranscoder implements Closeable { private final boolean transcodeRequired; private final long fileSizeEstimate; private final int outputFormat; + private final @Nullable Options options; private @Nullable MemoryFileDescriptor memoryFile; /** * @param upperSizeLimit A upper size to transcode to. The actual output size can be up to 10% smaller. */ - public InMemoryTranscoder(@NonNull Context context, @NonNull MediaDataSource dataSource, long upperSizeLimit) throws IOException, VideoSourceException { + public InMemoryTranscoder(@NonNull Context context, @NonNull MediaDataSource dataSource, @Nullable Options options, long upperSizeLimit) throws IOException, VideoSourceException { this.context = context; this.dataSource = dataSource; + this.options = options; final MediaMetadataRetriever mediaMetadataRetriever = new MediaMetadataRetriever(); try { @@ -72,9 +75,9 @@ public final class InMemoryTranscoder implements Closeable { this.targetVideoBitRate = getTargetVideoBitRate(upperSizeLimitWithMargin, duration); this.upperSizeLimit = upperSizeLimit; - this.transcodeRequired = inputBitRate >= targetVideoBitRate * 1.2 || inSize > upperSizeLimit || containsLocation(mediaMetadataRetriever); + this.transcodeRequired = inputBitRate >= targetVideoBitRate * 1.2 || inSize > upperSizeLimit || containsLocation(mediaMetadataRetriever) || options != null; if (!transcodeRequired) { - Log.i(TAG, "Video is within 20% of target bitrate, below the size limit and contained no location metadata."); + Log.i(TAG, "Video is within 20% of target bitrate, below the size limit, contained no location metadata or custom options."); } this.fileSizeEstimate = (targetVideoBitRate + AUDIO_BITRATE) * duration / 8000; @@ -84,7 +87,8 @@ public final class InMemoryTranscoder implements Closeable { : OUTPUT_FORMAT; } - public @NonNull MediaStream transcode(@NonNull Progress progress, @Nullable CancelationSignal cancelationSignal) + public @NonNull MediaStream transcode(@NonNull Progress progress, + @Nullable CancelationSignal cancelationSignal) throws IOException, EncodingException, VideoSizeException { if (memoryFile != null) throw new AssertionError("Not expecting to reuse transcoder"); @@ -125,12 +129,21 @@ public final class InMemoryTranscoder implements Closeable { final MediaConverter converter = new MediaConverter(); - converter.setInput(dataSource); + converter.setInput(new VideoInput.MediaDataSourceVideoInput(dataSource)); converter.setOutput(memoryFileFileDescriptor); converter.setVideoResolution(outputFormat); converter.setVideoBitrate(targetVideoBitRate); converter.setAudioBitrate(AUDIO_BITRATE); + if (options != null) { + if (options.endTimeUs > 0) { + long timeFrom = options.startTimeUs / 1000; + long timeTo = options.endTimeUs / 1000; + converter.setTimeRange(timeFrom, timeTo); + Log.i(TAG, String.format(Locale.US, "Trimming:\nTotal duration: %d\nKeeping: %d..%d\nFinal duration:(%d)", duration, timeFrom, timeTo, timeTo - timeFrom)); + } + } + converter.setListener(percent -> { progress.onProgress(percent); return cancelationSignal != null && cancelationSignal.isCanceled(); @@ -219,4 +232,14 @@ public final class InMemoryTranscoder implements Closeable { public interface CancelationSignal { boolean isCanceled(); } + + public final static class Options { + final long startTimeUs; + final long endTimeUs; + + public Options(long startTimeUs, long endTimeUs) { + this.startTimeUs = startTimeUs; + this.endTimeUs = endTimeUs; + } + } } diff --git a/app/src/main/java/org/thoughtcrime/securesms/video/VideoPlayer.java b/app/src/main/java/org/thoughtcrime/securesms/video/VideoPlayer.java index 0d38a618a8..491e251a70 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/video/VideoPlayer.java +++ b/app/src/main/java/org/thoughtcrime/securesms/video/VideoPlayer.java @@ -17,21 +17,24 @@ package org.thoughtcrime.securesms.video; import android.content.Context; -import androidx.annotation.NonNull; -import androidx.annotation.Nullable; import android.util.AttributeSet; import android.view.View; import android.view.Window; import android.view.WindowManager; import android.widget.FrameLayout; +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; + import com.google.android.exoplayer2.DefaultLoadControl; +import com.google.android.exoplayer2.DefaultRenderersFactory; import com.google.android.exoplayer2.ExoPlayerFactory; import com.google.android.exoplayer2.LoadControl; import com.google.android.exoplayer2.Player; import com.google.android.exoplayer2.SimpleExoPlayer; import com.google.android.exoplayer2.extractor.DefaultExtractorsFactory; import com.google.android.exoplayer2.extractor.ExtractorsFactory; +import com.google.android.exoplayer2.source.ClippingMediaSource; import com.google.android.exoplayer2.source.ExtractorMediaSource; import com.google.android.exoplayer2.source.MediaSource; import com.google.android.exoplayer2.trackselection.AdaptiveTrackSelection; @@ -40,25 +43,30 @@ import com.google.android.exoplayer2.trackselection.TrackSelection; import com.google.android.exoplayer2.trackselection.TrackSelector; import com.google.android.exoplayer2.ui.PlayerControlView; import com.google.android.exoplayer2.ui.PlayerView; -import com.google.android.exoplayer2.upstream.BandwidthMeter; -import com.google.android.exoplayer2.upstream.DefaultBandwidthMeter; import com.google.android.exoplayer2.upstream.DefaultDataSourceFactory; import org.thoughtcrime.securesms.R; +import org.thoughtcrime.securesms.logging.Log; import org.thoughtcrime.securesms.mms.VideoSlide; import org.thoughtcrime.securesms.util.ViewUtil; import org.thoughtcrime.securesms.video.exo.AttachmentDataSourceFactory; +import java.util.concurrent.TimeUnit; + public class VideoPlayer extends FrameLayout { - private static final String TAG = VideoPlayer.class.getSimpleName(); + @SuppressWarnings("unused") + private static final String TAG = Log.tag(VideoPlayer.class); - private final PlayerView exoView; + private final PlayerView exoView; + private final PlayerControlView exoControls; - private SimpleExoPlayer exoPlayer; - private PlayerControlView exoControls; - private Window window; - private PlayerStateCallback playerStateCallback; + private SimpleExoPlayer exoPlayer; + private Window window; + private PlayerStateCallback playerStateCallback; + private PlayerCallback playerCallback; + private boolean clipped; + private long clippedStartUs; public VideoPlayer(Context context) { this(context, null); @@ -73,29 +81,49 @@ public class VideoPlayer extends FrameLayout { inflate(context, R.layout.video_player, this); - this.exoView = ViewUtil.findById(this, R.id.video_view); + this.exoView = ViewUtil.findById(this, R.id.video_view); this.exoControls = new PlayerControlView(getContext()); this.exoControls.setShowTimeoutMs(-1); } - public void setVideoSource(@NonNull VideoSlide videoSource, boolean autoplay) { - BandwidthMeter bandwidthMeter = new DefaultBandwidthMeter(); - TrackSelection.Factory videoTrackSelectionFactory = new AdaptiveTrackSelection.Factory(bandwidthMeter); - TrackSelector trackSelector = new DefaultTrackSelector(videoTrackSelectionFactory); - LoadControl loadControl = new DefaultLoadControl(); + private CreateMediaSource createMediaSource; - exoPlayer = ExoPlayerFactory.newSimpleInstance(getContext(), trackSelector, loadControl); + public void setVideoSource(@NonNull VideoSlide videoSource, boolean autoplay) { + Context context = getContext(); + DefaultRenderersFactory renderersFactory = new DefaultRenderersFactory(context); + TrackSelection.Factory videoTrackSelectionFactory = new AdaptiveTrackSelection.Factory(); + TrackSelector trackSelector = new DefaultTrackSelector(videoTrackSelectionFactory); + LoadControl loadControl = new DefaultLoadControl(); + + exoPlayer = ExoPlayerFactory.newSimpleInstance(context, renderersFactory, trackSelector, loadControl); exoPlayer.addListener(new ExoPlayerListener(window, playerStateCallback)); + exoPlayer.addListener(new Player.DefaultEventListener() { + @Override + public void onPlayerStateChanged(boolean playWhenReady, int playbackState) { + if (playerCallback != null) { + switch (playbackState) { + case Player.STATE_READY: + if (playWhenReady) playerCallback.onPlaying(); + break; + case Player.STATE_ENDED: + playerCallback.onStopped(); + break; + } + } + } + }); exoView.setPlayer(exoPlayer); exoControls.setPlayer(exoPlayer); - DefaultDataSourceFactory defaultDataSourceFactory = new DefaultDataSourceFactory(getContext(), "GenericUserAgent", null); - AttachmentDataSourceFactory attachmentDataSourceFactory = new AttachmentDataSourceFactory(getContext(), defaultDataSourceFactory, null); + DefaultDataSourceFactory defaultDataSourceFactory = new DefaultDataSourceFactory(context, "GenericUserAgent", null); + AttachmentDataSourceFactory attachmentDataSourceFactory = new AttachmentDataSourceFactory(context, defaultDataSourceFactory, null); ExtractorsFactory extractorsFactory = new DefaultExtractorsFactory(); - MediaSource mediaSource = new ExtractorMediaSource(videoSource.getUri(), attachmentDataSourceFactory, extractorsFactory, null, null); + createMediaSource = () -> new ExtractorMediaSource.Factory(attachmentDataSourceFactory) + .setExtractorsFactory(extractorsFactory) + .createMediaSource(videoSource.getUri()); - exoPlayer.prepare(mediaSource); + exoPlayer.prepare(createMediaSource.create()); exoPlayer.setPlayWhenReady(autoplay); } @@ -142,6 +170,40 @@ public class VideoPlayer extends FrameLayout { return 0L; } + public long getPlaybackPositionUs() { + if (this.exoPlayer != null) { + return TimeUnit.MILLISECONDS.toMicros(this.exoPlayer.getCurrentPosition()) + clippedStartUs; + } + return 0L; + } + + public void setPlaybackPosition(long positionMs) { + if (this.exoPlayer != null) { + this.exoPlayer.seekTo(positionMs); + } + } + + public void clip(long fromUs, long toUs, boolean playWhenReady) { + if (this.exoPlayer != null && createMediaSource != null) { + MediaSource clippedMediaSource = new ClippingMediaSource(createMediaSource.create(), fromUs, toUs); + exoPlayer.prepare(clippedMediaSource); + exoPlayer.setPlayWhenReady(playWhenReady); + clipped = true; + clippedStartUs = fromUs; + } + } + + public void removeClip(boolean playWhenReady) { + if (exoPlayer != null && createMediaSource != null) { + if (clipped) { + exoPlayer.prepare(createMediaSource.create()); + clipped = false; + clippedStartUs = 0; + } + exoPlayer.setPlayWhenReady(playWhenReady); + } + } + public void setWindow(@Nullable Window window) { this.window = window; } @@ -150,12 +212,23 @@ public class VideoPlayer extends FrameLayout { this.playerStateCallback = playerStateCallback; } + public void setPlayerCallback(PlayerCallback playerCallback) { + this.playerCallback = playerCallback; + } + + public void playFromStart() { + if (exoPlayer != null) { + exoPlayer.setPlayWhenReady(true); + exoPlayer.seekTo(0); + } + } + private static class ExoPlayerListener extends Player.DefaultEventListener { - private final Window window; + private final Window window; private final PlayerStateCallback playerStateCallback; ExoPlayerListener(Window window, PlayerStateCallback playerStateCallback) { - this.window = window; + this.window = window; this.playerStateCallback = playerStateCallback; } @@ -188,4 +261,15 @@ public class VideoPlayer extends FrameLayout { public interface PlayerStateCallback { void onPlayerReady(); } + + public interface PlayerCallback { + + void onPlaying(); + + void onStopped(); + } + + private interface CreateMediaSource { + MediaSource create(); + } } diff --git a/app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/AudioTrackConverter.java b/app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/AudioTrackConverter.java index b47b6f8ae9..1f49c10c93 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/AudioTrackConverter.java +++ b/app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/AudioTrackConverter.java @@ -62,7 +62,7 @@ final class AudioTrackConverter { static @Nullable AudioTrackConverter create( - final @NonNull MediaConverter.Input input, + final @NonNull VideoInput input, final long timeFrom, final long timeTo, final int audioBitrate) throws IOException { @@ -106,6 +106,7 @@ final class AudioTrackConverter { inputAudioFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT)); outputAudioFormat.setInteger(MediaFormat.KEY_BIT_RATE, audioBitrate); outputAudioFormat.setInteger(MediaFormat.KEY_AAC_PROFILE, OUTPUT_AUDIO_AAC_PROFILE); + outputAudioFormat.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, 16 * 1024); // Create a MediaCodec for the desired codec, then configure it as an encoder with // our desired properties. Request a Surface to use for input. diff --git a/app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/MediaConverter.java b/app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/MediaConverter.java index f3a74dfcfd..aeacf8dbf0 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/MediaConverter.java +++ b/app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/MediaConverter.java @@ -18,13 +18,9 @@ package org.thoughtcrime.securesms.video.videoconverter; -import android.content.Context; import android.media.MediaCodecInfo; import android.media.MediaCodecList; -import android.media.MediaDataSource; -import android.media.MediaExtractor; import android.media.MediaFormat; -import android.net.Uri; import androidx.annotation.NonNull; import androidx.annotation.Nullable; @@ -53,7 +49,7 @@ public final class MediaConverter { public static final String VIDEO_CODEC_H264 = "video/avc"; public static final String VIDEO_CODEC_H265 = "video/hevc"; - private Input mInput; + private VideoInput mInput; private Output mOutput; private long mTimeFrom; @@ -73,20 +69,8 @@ public final class MediaConverter { public MediaConverter() { } - @SuppressWarnings("unused") - public void setInput(final @NonNull File file) { - mInput = new FileInput(file); - } - - @SuppressWarnings("unused") - public void setInput(final @NonNull Context context, final @NonNull Uri uri) { - mInput = new UriInput(context, uri); - } - - @RequiresApi(23) - @SuppressWarnings("unused") - public void setInput(final @NonNull MediaDataSource mediaDataSource) { - mInput = new MediaDataSourceInput(mediaDataSource); + public void setInput(final @NonNull VideoInput videoInput) { + mInput = videoInput; } @SuppressWarnings("unused") @@ -312,65 +296,6 @@ public final class MediaConverter { return null; } - interface Input { - @NonNull - MediaExtractor createExtractor() throws IOException; - } - - private static class FileInput implements Input { - - final File file; - - FileInput(final @NonNull File file) { - this.file = file; - } - - @Override - public @NonNull - MediaExtractor createExtractor() throws IOException { - final MediaExtractor extractor = new MediaExtractor(); - extractor.setDataSource(file.getAbsolutePath()); - return extractor; - } - } - - private static class UriInput implements Input { - - final Uri uri; - final Context context; - - UriInput(final @NonNull Context context, final @NonNull Uri uri) { - this.uri = uri; - this.context = context; - } - - @Override - public @NonNull - MediaExtractor createExtractor() throws IOException { - final MediaExtractor extractor = new MediaExtractor(); - extractor.setDataSource(context, uri, null); - return extractor; - } - } - - @RequiresApi(23) - private static class MediaDataSourceInput implements Input { - - private final MediaDataSource mediaDataSource; - - MediaDataSourceInput(final @NonNull MediaDataSource mediaDataSource) { - this.mediaDataSource = mediaDataSource; - } - - @Override - public @NonNull - MediaExtractor createExtractor() throws IOException { - final MediaExtractor extractor = new MediaExtractor(); - extractor.setDataSource(mediaDataSource); - return extractor; - } - } - interface Output { @NonNull Muxer createMuxer() throws IOException; diff --git a/app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/OutputSurface.java b/app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/OutputSurface.java index 53f329eb91..1b704bd6ab 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/OutputSurface.java +++ b/app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/OutputSurface.java @@ -69,7 +69,7 @@ final class OutputSurface implements SurfaceTexture.OnFrameAvailableListener { * EGL context and surface will be made current. Creates a Surface that can be passed * to MediaCodec.configure(). */ - OutputSurface(int width, int height) throws TranscodingException { + OutputSurface(int width, int height, boolean flipX) throws TranscodingException { if (width <= 0 || height <= 0) { throw new IllegalArgumentException(); } @@ -77,7 +77,7 @@ final class OutputSurface implements SurfaceTexture.OnFrameAvailableListener { eglSetup(width, height); makeCurrent(); - setup(); + setup(flipX); } /** @@ -85,15 +85,15 @@ final class OutputSurface implements SurfaceTexture.OnFrameAvailableListener { * passed to MediaCodec.configure(). */ OutputSurface() throws TranscodingException { - setup(); + setup(false); } /** * Creates instances of TextureRender and SurfaceTexture, and a Surface associated * with the SurfaceTexture. */ - private void setup() throws TranscodingException { - mTextureRender = new TextureRender(); + private void setup(boolean flipX) throws TranscodingException { + mTextureRender = new TextureRender(flipX); mTextureRender.surfaceCreated(); // Even if we don't access the SurfaceTexture after the constructor returns, we diff --git a/app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/TextureRender.java b/app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/TextureRender.java index 1de9b2b1b9..8eeced5b28 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/TextureRender.java +++ b/app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/TextureRender.java @@ -47,6 +47,14 @@ final class TextureRender { 1.0f, 1.0f, 0, 1.f, 1.f, }; + private final float[] mTriangleVerticesDataFlippedX = { + // X, Y, Z, U, V + -1.0f, -1.0f, 0, 1.f, 0.f, + 1.0f, -1.0f, 0, 0.f, 0.f, + -1.0f, 1.0f, 0, 1.f, 1.f, + 1.0f, 1.0f, 0, 0.f, 1.f, + }; + private final FloatBuffer mTriangleVertices; private static final String VERTEX_SHADER = @@ -79,11 +87,12 @@ final class TextureRender { private int maPositionHandle; private int maTextureHandle; - TextureRender() { + TextureRender(boolean flipX) { + float[] verticesData = flipX ? mTriangleVerticesDataFlippedX : mTriangleVerticesData; mTriangleVertices = ByteBuffer.allocateDirect( - mTriangleVerticesData.length * FLOAT_SIZE_BYTES) + verticesData.length * FLOAT_SIZE_BYTES) .order(ByteOrder.nativeOrder()).asFloatBuffer(); - mTriangleVertices.put(mTriangleVerticesData).position(0); + mTriangleVertices.put(verticesData).position(0); Matrix.setIdentityM(mSTMatrix, 0); } diff --git a/app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoInput.java b/app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoInput.java new file mode 100644 index 0000000000..a63b6ac7b1 --- /dev/null +++ b/app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoInput.java @@ -0,0 +1,83 @@ +package org.thoughtcrime.securesms.video.videoconverter; + +import android.content.Context; +import android.media.MediaDataSource; +import android.media.MediaExtractor; +import android.net.Uri; + +import androidx.annotation.NonNull; +import androidx.annotation.RequiresApi; + +import java.io.Closeable; +import java.io.File; +import java.io.IOException; + +public abstract class VideoInput implements Closeable { + + @NonNull + abstract MediaExtractor createExtractor() throws IOException; + + public static class FileVideoInput extends VideoInput { + + final File file; + + public FileVideoInput(final @NonNull File file) { + this.file = file; + } + + @Override + public @NonNull MediaExtractor createExtractor() throws IOException { + final MediaExtractor extractor = new MediaExtractor(); + extractor.setDataSource(file.getAbsolutePath()); + return extractor; + } + + @Override + public void close() { + } + } + + public static class UriVideoInput extends VideoInput { + + final Uri uri; + final Context context; + + public UriVideoInput(final @NonNull Context context, final @NonNull Uri uri) { + this.uri = uri; + this.context = context; + } + + @Override + public @NonNull MediaExtractor createExtractor() throws IOException { + final MediaExtractor extractor = new MediaExtractor(); + extractor.setDataSource(context, uri, null); + return extractor; + } + + @Override + public void close() { + } + } + + @RequiresApi(23) + public static class MediaDataSourceVideoInput extends VideoInput { + + private final MediaDataSource mediaDataSource; + + public MediaDataSourceVideoInput(final @NonNull MediaDataSource mediaDataSource) { + this.mediaDataSource = mediaDataSource; + } + + @Override + public @NonNull MediaExtractor createExtractor() throws IOException { + final MediaExtractor extractor = new MediaExtractor(); + extractor.setDataSource(mediaDataSource); + return extractor; + } + + @Override + public void close() throws IOException { + mediaDataSource.close(); + } + } +} diff --git a/app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoThumbnailsExtractor.java b/app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoThumbnailsExtractor.java new file mode 100644 index 0000000000..a739ced8b6 --- /dev/null +++ b/app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoThumbnailsExtractor.java @@ -0,0 +1,182 @@ +package org.thoughtcrime.securesms.video.videoconverter; + +import android.graphics.Bitmap; +import android.media.MediaCodec; +import android.media.MediaExtractor; +import android.media.MediaFormat; +import android.opengl.GLES20; + +import androidx.annotation.NonNull; +import androidx.annotation.RequiresApi; + +import org.thoughtcrime.securesms.logging.Log; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +@RequiresApi(api = 23) +final class VideoThumbnailsExtractor { + + private static final String TAG = Log.tag(VideoThumbnailsExtractor.class); + + interface Callback { + void durationKnown(long duration); + + boolean publishProgress(int index, Bitmap thumbnail); + + void failed(); + } + + static void extractThumbnails(final @NonNull VideoInput input, + final int thumbnailCount, + final int thumbnailResolution, + final @NonNull Callback callback) + { + MediaExtractor extractor = null; + MediaCodec decoder = null; + OutputSurface outputSurface = null; + try { + extractor = input.createExtractor(); + MediaFormat mediaFormat = null; + for (int index = 0; index < extractor.getTrackCount(); ++index) { + if (extractor.getTrackFormat(index).getString(MediaFormat.KEY_MIME).startsWith("video/")) { + extractor.selectTrack(index); + mediaFormat = extractor.getTrackFormat(index); + break; + } + } + if (mediaFormat != null) { + final String mime = mediaFormat.getString(MediaFormat.KEY_MIME); + final int rotation = mediaFormat.containsKey(MediaFormat.KEY_ROTATION) ? mediaFormat.getInteger(MediaFormat.KEY_ROTATION) : 0; + final int width = mediaFormat.getInteger(MediaFormat.KEY_WIDTH); + final int height = mediaFormat.getInteger(MediaFormat.KEY_HEIGHT); + final int outputWidth; + final int outputHeight; + + if (width < height) { + outputWidth = thumbnailResolution; + outputHeight = height * outputWidth / width; + } else { + outputHeight = thumbnailResolution; + outputWidth = width * outputHeight / height; + } + + final int outputWidthRotated; + final int outputHeightRotated; + + if ((rotation % 180 == 90)) { + //noinspection SuspiciousNameCombination + outputWidthRotated = outputHeight; + //noinspection SuspiciousNameCombination + outputHeightRotated = outputWidth; + } else { + outputWidthRotated = outputWidth; + outputHeightRotated = outputHeight; + } + + Log.i(TAG, "video: " + width + "x" + height + " " + rotation); + Log.i(TAG, "output: " + outputWidthRotated + "x" + outputHeightRotated); + + outputSurface = new OutputSurface(outputWidthRotated, outputHeightRotated, true); + + decoder = MediaCodec.createDecoderByType(mime); + decoder.configure(mediaFormat, outputSurface.getSurface(), null, 0); + decoder.start(); + + long duration = mediaFormat.getLong(MediaFormat.KEY_DURATION); + callback.durationKnown(duration); + + doExtract(extractor, decoder, outputSurface, outputWidthRotated, outputHeightRotated, duration, thumbnailCount, callback); + } + } catch (IOException | TranscodingException e) { + Log.w(TAG, e); + callback.failed(); + } finally { + if (outputSurface != null) { + outputSurface.release(); + } + if (decoder != null) { + decoder.stop(); + decoder.release(); + } + if (extractor != null) { + extractor.release(); + } + } + } + + private static void doExtract(final @NonNull MediaExtractor extractor, + final @NonNull MediaCodec decoder, + final @NonNull OutputSurface outputSurface, + final int outputWidth, int outputHeight, long duration, int thumbnailCount, + final @NonNull Callback callback) + throws TranscodingException + { + + final int TIMEOUT_USEC = 10000; + final ByteBuffer[] decoderInputBuffers = decoder.getInputBuffers(); + final MediaCodec.BufferInfo info = new MediaCodec.BufferInfo(); + + int samplesExtracted = 0; + int thumbnailsCreated = 0; + + Log.i(TAG, "doExtract started"); + final ByteBuffer pixelBuf = ByteBuffer.allocateDirect(outputWidth * outputHeight * 4); + pixelBuf.order(ByteOrder.LITTLE_ENDIAN); + + boolean outputDone = false; + boolean inputDone = false; + while (!outputDone) { + if (!inputDone) { + int inputBufIndex = decoder.dequeueInputBuffer(TIMEOUT_USEC); + if (inputBufIndex >= 0) { + final ByteBuffer inputBuf = decoderInputBuffers[inputBufIndex]; + final int sampleSize = extractor.readSampleData(inputBuf, 0); + if (sampleSize < 0 || samplesExtracted >= thumbnailCount) { + decoder.queueInputBuffer(inputBufIndex, 0, 0, 0L, MediaCodec.BUFFER_FLAG_END_OF_STREAM); + inputDone = true; + Log.i(TAG, "input done"); + } else { + final long presentationTimeUs = extractor.getSampleTime(); + decoder.queueInputBuffer(inputBufIndex, 0, sampleSize, presentationTimeUs, 0 /*flags*/); + samplesExtracted++; + extractor.seekTo(duration * samplesExtracted / thumbnailCount, MediaExtractor.SEEK_TO_CLOSEST_SYNC); + Log.i(TAG, "seek to " + duration * samplesExtracted / thumbnailCount + ", actual " + extractor.getSampleTime()); + } + } + } + + int outputBufIndex = decoder.dequeueOutputBuffer(info, TIMEOUT_USEC); + if (outputBufIndex >= 0) { + if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { + outputDone = true; + } + + final boolean shouldRender = (info.size != 0) /*&& (info.presentationTimeUs >= duration * decodeCount / thumbnailCount)*/; + + decoder.releaseOutputBuffer(outputBufIndex, shouldRender); + if (shouldRender) { + outputSurface.awaitNewImage(); + outputSurface.drawImage(); + + if (thumbnailsCreated < thumbnailCount) { + pixelBuf.rewind(); + GLES20.glReadPixels(0, 0, outputWidth, outputHeight, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, pixelBuf); + + final Bitmap bitmap = Bitmap.createBitmap(outputWidth, outputHeight, Bitmap.Config.ARGB_8888); + pixelBuf.rewind(); + bitmap.copyPixelsFromBuffer(pixelBuf); + + if (!callback.publishProgress(thumbnailsCreated, bitmap)) { + break; + } + Log.i(TAG, "publishProgress for frame " + thumbnailsCreated + " at " + info.presentationTimeUs + " (target " + duration * thumbnailsCreated / thumbnailCount + ")"); + } + thumbnailsCreated++; + } + } + } + Log.i(TAG, "doExtract finished"); + } +} diff --git a/app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoThumbnailsRangeSelectorView.java b/app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoThumbnailsRangeSelectorView.java new file mode 100644 index 0000000000..e1b327fedd --- /dev/null +++ b/app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoThumbnailsRangeSelectorView.java @@ -0,0 +1,363 @@ +package org.thoughtcrime.securesms.video.videoconverter; + +import android.content.Context; +import android.content.res.TypedArray; +import android.graphics.Canvas; +import android.graphics.Paint; +import android.graphics.Rect; +import android.graphics.drawable.Drawable; +import android.util.AttributeSet; +import android.view.MotionEvent; + +import androidx.annotation.ColorInt; +import androidx.annotation.Nullable; +import androidx.annotation.Px; +import androidx.annotation.RequiresApi; +import androidx.vectordrawable.graphics.drawable.VectorDrawableCompat; + +import org.thoughtcrime.securesms.R; + +import java.util.concurrent.TimeUnit; + +@RequiresApi(api = 23) +public final class VideoThumbnailsRangeSelectorView extends VideoThumbnailsView { + + private static final long MINIMUM_SELECTABLE_RANGE = TimeUnit.MILLISECONDS.toMicros(500); + + private final Paint paint = new Paint(Paint.ANTI_ALIAS_FLAG); + private final Paint paintGrey = new Paint(Paint.ANTI_ALIAS_FLAG); + private final Rect tempDrawRect = new Rect(); + private Drawable chevronLeft; + private Drawable chevronRight; + + @Px private int left; + @Px private int right; + @Px private int cursor; + private Long minValue; + private Long maxValue; + private Long externalMinValue; + private Long externalMaxValue; + private float xDown; + private long downCursor; + private long downMin; + private long downMax; + private Thumb dragThumb; + private OnRangeChangeListener onRangeChangeListener; + + @Px private int thumbSizePixels; + @Px private int thumbTouchRadius; + @Px private int cursorPixels; + @ColorInt private int cursorColor; + @ColorInt private int thumbColor; + @ColorInt private int thumbColorEdited; + private long actualPosition; + private long dragPosition; + + public VideoThumbnailsRangeSelectorView(final Context context) { + super(context); + init(null); + } + + public VideoThumbnailsRangeSelectorView(final Context context, final @Nullable AttributeSet attrs) { + super(context, attrs); + init(attrs); + } + + public VideoThumbnailsRangeSelectorView(final Context context, final @Nullable AttributeSet attrs, final int defStyleAttr) { + super(context, attrs, defStyleAttr); + init(attrs); + } + + private void init(final @Nullable AttributeSet attrs) { + if (attrs != null) { + TypedArray typedArray = getContext().getTheme().obtainStyledAttributes(attrs, R.styleable.VideoThumbnailsRangeSelectorView, 0, 0); + + thumbSizePixels = typedArray.getDimensionPixelSize(R.styleable.VideoThumbnailsRangeSelectorView_thumbWidth, 1); + cursorPixels = typedArray.getDimensionPixelSize(R.styleable.VideoThumbnailsRangeSelectorView_cursorWidth, 1); + thumbColor = typedArray.getColor(R.styleable.VideoThumbnailsRangeSelectorView_thumbColor, 0xffff0000); + thumbColorEdited = typedArray.getColor(R.styleable.VideoThumbnailsRangeSelectorView_thumbColorEdited, thumbColor); + cursorColor = typedArray.getColor(R.styleable.VideoThumbnailsRangeSelectorView_cursorColor, thumbColor); + thumbTouchRadius = typedArray.getDimensionPixelSize(R.styleable.VideoThumbnailsRangeSelectorView_thumbTouchRadius, 50); + } + + chevronLeft = VectorDrawableCompat.create(getResources(), R.drawable.ic_chevron_left_black_8dp, null); + chevronRight = VectorDrawableCompat.create(getResources(), R.drawable.ic_chevron_right_black_8dp, null); + + paintGrey.setColor(0x7f000000); + paintGrey.setStyle(Paint.Style.FILL_AND_STROKE); + paintGrey.setStrokeWidth(1); + + paint.setStrokeWidth(2); + } + + @Override + protected void afterDurationChange(long duration) { + super.afterDurationChange(duration); + + if (maxValue != null && duration < maxValue) { + maxValue = duration; + } + + if (minValue != null && duration < minValue) { + minValue = duration; + } + + if (duration > 0) { + if (externalMinValue != null) { + setMinMax(externalMinValue, getMaxValue(), Thumb.MIN); + externalMinValue = null; + } + + if (externalMaxValue != null) { + setMinMax(getMinValue(), externalMaxValue, Thumb.MAX); + externalMaxValue = null; + } + } + + invalidate(); + } + + public void setOnRangeChangeListener(OnRangeChangeListener onRangeChangeListener) { + this.onRangeChangeListener = onRangeChangeListener; + } + + public void setActualPosition(long position) { + if (this.actualPosition != position) { + this.actualPosition = position; + invalidate(); + } + } + + private void setDragPosition(long position) { + if (this.dragPosition != position) { + this.dragPosition = Math.max(getMinValue(), Math.min(getMaxValue(), position)); + invalidate(); + } + } + + @Override + protected void onDraw(final Canvas canvas) { + super.onDraw(canvas); + + canvas.translate(getPaddingLeft(), getPaddingTop()); + int drawableWidth = getDrawableWidth(); + int drawableHeight = getDrawableHeight(); + + long duration = getDuration(); + + long min = getMinValue(); + long max = getMaxValue(); + + boolean edited = min != 0 || max != duration; + + long drawPosAt = dragThumb == Thumb.POSITION ? dragPosition : actualPosition; + + left = duration != 0 ? (int) ((min * drawableWidth) / duration) : 0; + right = duration != 0 ? (int) ((max * drawableWidth) / duration) : drawableWidth; + cursor = duration != 0 ? (int) ((drawPosAt * drawableWidth) / duration) : drawableWidth; + + // draw greyed out areas + tempDrawRect.set(0, 0, left - 1, drawableHeight); + canvas.drawRect(tempDrawRect, paintGrey); + tempDrawRect.set(right + 1, 0, drawableWidth, drawableHeight); + canvas.drawRect(tempDrawRect, paintGrey); + + // draw area rectangle + paint.setStyle(Paint.Style.STROKE); + tempDrawRect.set(left, 0, right, drawableHeight); + paint.setColor(edited ? thumbColorEdited : thumbColor); + canvas.drawRect(tempDrawRect, paint); + + // draw thumb rectangles + paint.setStyle(Paint.Style.FILL_AND_STROKE); + tempDrawRect.set(left, 0, left + thumbSizePixels, drawableHeight); + canvas.drawRect(tempDrawRect, paint); + tempDrawRect.set(right - thumbSizePixels, 0, right, drawableHeight); + canvas.drawRect(tempDrawRect, paint); + + int arrowSize = Math.min(drawableHeight, thumbSizePixels * 2); + chevronLeft .setBounds(0, 0, arrowSize, arrowSize); + chevronRight.setBounds(0, 0, arrowSize, arrowSize); + + float dy = (drawableHeight - arrowSize) / 2f; + float arrowPaddingX = (thumbSizePixels - arrowSize) / 2f; + + // draw left thumb chevron + canvas.save(); + canvas.translate(left + arrowPaddingX, dy); + chevronLeft.draw(canvas); + canvas.restore(); + + // draw right thumb chevron + canvas.save(); + canvas.translate(right - thumbSizePixels + arrowPaddingX, dy); + chevronRight.draw(canvas); + canvas.restore(); + + // draw current position marker + if (left <= cursor && cursor <= right && dragThumb != Thumb.MIN && dragThumb != Thumb.MAX) { + canvas.translate(cursorPixels / 2, 0); + tempDrawRect.set(cursor, 0, cursor + cursorPixels, drawableHeight); + paint.setColor(cursorColor); + canvas.drawRect(tempDrawRect, paint); + } + } + + public long getMinValue() { + return minValue == null ? 0 : minValue; + } + + public long getMaxValue() { + return maxValue == null ? getDuration() : maxValue; + } + + private boolean setMinValue(long minValue) { + if (this.minValue == null || this.minValue != minValue) { + return setMinMax(minValue, getMaxValue(), Thumb.MIN); + } else{ + return false; + } + } + + public boolean setMaxValue(long maxValue) { + if (this.maxValue == null || this.maxValue != maxValue) { + return setMinMax(getMinValue(), maxValue, Thumb.MAX); + } else{ + return false; + } + } + + private boolean setMinMax(long newMin, long newMax, Thumb thumb) { + final long currentMin = getMinValue(); + final long currentMax = getMaxValue(); + final long duration = getDuration(); + + final long minDiff = Math.max(MINIMUM_SELECTABLE_RANGE, pixelToDuration(thumbSizePixels * 2.5f)); + + if (thumb == Thumb.MIN) { + newMin = clamp(newMin, 0, currentMax - minDiff); + } else { + newMax = clamp(newMax, currentMin + minDiff, duration); + } + + if (newMin != currentMin || newMax != currentMax) { + this.minValue = newMin; + this.maxValue = newMax; + invalidate(); + return true; + } + return false; + } + + private static long clamp(long value, long min, long max) { + return Math.min(Math.max(min, value), max); + } + + @Override + public boolean onTouchEvent(MotionEvent event) { + int actionMasked = event.getActionMasked(); + if (actionMasked == MotionEvent.ACTION_DOWN) { + xDown = event.getX(); + downCursor = actualPosition; + downMin = getMinValue(); + downMax = getMaxValue(); + dragThumb = closestThumb(event.getX()); + return dragThumb != null; + } + + if (actionMasked == MotionEvent.ACTION_MOVE) { + boolean changed = false; + long delta = pixelToDuration(event.getX() - xDown); + switch (dragThumb) { + case POSITION: + setDragPosition(downCursor + delta); + changed = true; + break; + case MIN: + changed = setMinValue(downMin + delta); + break; + case MAX: + changed = setMaxValue(downMax + delta); + break; + } + if (changed && onRangeChangeListener != null) { + if (dragThumb == Thumb.POSITION) { + onRangeChangeListener.onPositionDrag(dragPosition); + } else { + onRangeChangeListener.onRangeDrag(getMinValue(), getMaxValue(), getDuration(), dragThumb); + } + } + return true; + } + + if (actionMasked == MotionEvent.ACTION_UP) { + if (onRangeChangeListener != null) { + if (dragThumb == Thumb.POSITION) { + onRangeChangeListener.onEndPositionDrag(dragPosition); + } else { + onRangeChangeListener.onRangeDragEnd(getMinValue(), getMaxValue(), getDuration(), dragThumb); + } + dragThumb = null; + invalidate(); + } + return true; + } + + if (actionMasked == MotionEvent.ACTION_CANCEL) { + dragThumb = null; + } + + return true; + } + + private @Nullable Thumb closestThumb(@Px float x) { + float midPoint = (right + left) / 2f; + Thumb possibleThumb = x < midPoint ? Thumb.MIN : Thumb.MAX; + int possibleThumbX = x < midPoint ? left : right; + + if (Math.abs(x - possibleThumbX) < thumbTouchRadius) { + return possibleThumb; + } + + return null; + } + + private long pixelToDuration(float pixel) { + return (long) (pixel / getDrawableWidth() * getDuration()); + } + + private int getDrawableWidth() { + return getWidth() - getPaddingLeft() - getPaddingRight(); + } + + private int getDrawableHeight() { + return getHeight() - getPaddingBottom() - getPaddingTop(); + } + + public void setRange(long minValue, long maxValue) { + if (getDuration() > 0) { + setMinMax(minValue, maxValue, Thumb.MIN); + setMinMax(minValue, maxValue, Thumb.MAX); + } else { + externalMinValue = minValue; + externalMaxValue = maxValue; + } + } + + public enum Thumb { + MIN, + MAX, + POSITION + } + + public interface OnRangeChangeListener { + + void onPositionDrag(long position); + + void onEndPositionDrag(long position); + + void onRangeDrag(long minValue, long maxValue, long duration, Thumb thumb); + + void onRangeDragEnd(long minValue, long maxValue, long duration, Thumb thumb); + } +} diff --git a/app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoThumbnailsView.java b/app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoThumbnailsView.java new file mode 100644 index 0000000000..5ae54266ee --- /dev/null +++ b/app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoThumbnailsView.java @@ -0,0 +1,228 @@ +package org.thoughtcrime.securesms.video.videoconverter; + +import android.content.Context; +import android.graphics.Bitmap; +import android.graphics.Canvas; +import android.graphics.Paint; +import android.graphics.Rect; +import android.graphics.RectF; +import android.os.AsyncTask; +import android.util.AttributeSet; +import android.view.View; + +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; +import androidx.annotation.RequiresApi; + +import org.thoughtcrime.securesms.logging.Log; + +import java.io.IOException; +import java.lang.ref.WeakReference; +import java.util.ArrayList; +import java.util.Arrays; + +@RequiresApi(api = 23) +public class VideoThumbnailsView extends View { + + private static final String TAG = Log.tag(VideoThumbnailsView.class); + + private VideoInput input; + private ArrayList thumbnails; + private AsyncTask thumbnailsTask; + private OnDurationListener durationListener; + + private final Paint paint = new Paint(Paint.ANTI_ALIAS_FLAG); + private final RectF tempRect = new RectF(); + private final Rect drawRect = new Rect(); + private final Rect tempDrawRect = new Rect(); + private long duration = 0; + + public VideoThumbnailsView(final Context context) { + super(context); + } + + public VideoThumbnailsView(final Context context, final @Nullable AttributeSet attrs) { + super(context, attrs); + } + + public VideoThumbnailsView(final Context context, final @Nullable AttributeSet attrs, final int defStyleAttr) { + super(context, attrs, defStyleAttr); + } + + public void setInput(VideoInput input) { + this.input = input; + thumbnails = null; + if (thumbnailsTask != null) { + thumbnailsTask.cancel(true); + thumbnailsTask = null; + } + invalidate(); + } + + @Override + protected void onDetachedFromWindow() { + super.onDetachedFromWindow(); + + thumbnails = null; + if (thumbnailsTask != null) { + thumbnailsTask.cancel(true); + thumbnailsTask = null; + } + + if (input != null) { + try { + input.close(); + } catch (IOException e) { + Log.w(TAG, e); + } + } + } + + @Override + protected void onDraw(final Canvas canvas) { + super.onDraw(canvas); + + if (input == null) { + return; + } + + tempDrawRect.set(getPaddingLeft(), getPaddingTop(), getWidth() - getPaddingRight(), getHeight() - getPaddingBottom()); + + if (!drawRect.equals(tempDrawRect)) { + drawRect.set(tempDrawRect); + thumbnails = null; + if (thumbnailsTask != null) { + thumbnailsTask.cancel(true); + thumbnailsTask = null; + } + } + + if (thumbnails == null) { + if (thumbnailsTask == null) { + final int thumbnailCount = drawRect.width() / drawRect.height(); + final float thumbnailWidth = (float) drawRect.width() / thumbnailCount; + final float thumbnailHeight = drawRect.height(); + + thumbnails = new ArrayList<>(thumbnailCount); + thumbnailsTask = new ThumbnailsTask(this, input, thumbnailWidth, thumbnailHeight, thumbnailCount); + thumbnailsTask.execute(); + } + } else { + final int thumbnailCount = drawRect.width() / drawRect.height(); + final float thumbnailWidth = (float) drawRect.width() / thumbnailCount; + final float thumbnailHeight = drawRect.height(); + + tempRect.top = drawRect.top; + tempRect.bottom = drawRect.bottom; + + for (int i = 0; i < thumbnails.size(); i++) { + tempRect.left = drawRect.left + i * thumbnailWidth; + tempRect.right = tempRect.left + thumbnailWidth; + + final Bitmap thumbnailBitmap = thumbnails.get(i); + if (thumbnailBitmap != null) { + canvas.save(); + canvas.rotate(180, tempRect.centerX(), tempRect.centerY()); + tempDrawRect.set(0, 0, thumbnailBitmap.getWidth(), thumbnailBitmap.getHeight()); + if (tempDrawRect.width() * thumbnailHeight > tempDrawRect.height() * thumbnailWidth) { + float w = tempDrawRect.height() * thumbnailWidth / thumbnailHeight; + tempDrawRect.left = tempDrawRect.centerX() - (int) (w / 2); + tempDrawRect.right = tempDrawRect.left + (int) w; + } else { + float h = tempDrawRect.width() * thumbnailHeight / thumbnailWidth; + tempDrawRect.top = tempDrawRect.centerY() - (int) (h / 2); + tempDrawRect.bottom = tempDrawRect.top + (int) h; + } + canvas.drawBitmap(thumbnailBitmap, tempDrawRect, tempRect, paint); + canvas.restore(); + } + } + } + } + + public void setDurationListener(OnDurationListener durationListener) { + this.durationListener = durationListener; + } + + private void setDuration(long duration) { + if (durationListener != null) { + durationListener.onDurationKnown(duration); + } + if (this.duration != duration) { + this.duration = duration; + afterDurationChange(duration); + } + } + + protected void afterDurationChange(long duration) { + } + + protected long getDuration() { + return duration; + } + + private static class ThumbnailsTask extends AsyncTask { + + final WeakReference viewReference; + final VideoInput input; + final float thumbnailWidth; + final float thumbnailHeight; + final int thumbnailCount; + long duration; + + ThumbnailsTask(final @NonNull VideoThumbnailsView view, final @NonNull VideoInput input, final float thumbnailWidth, final float thumbnailHeight, final int thumbnailCount) { + viewReference = new WeakReference<>(view); + this.input = input; + this.thumbnailWidth = thumbnailWidth; + this.thumbnailHeight = thumbnailHeight; + this.thumbnailCount = thumbnailCount; + } + + @Override + protected Void doInBackground(Void... params) { + Log.i(TAG, "generate " + thumbnailCount + " thumbnails " + thumbnailWidth + "x" + thumbnailHeight); + VideoThumbnailsExtractor.extractThumbnails(input, thumbnailCount, (int) thumbnailHeight, new VideoThumbnailsExtractor.Callback() { + + @Override + public void durationKnown(long duration) { + ThumbnailsTask.this.duration = duration; + } + + @Override + public boolean publishProgress(int index, Bitmap thumbnail) { + ThumbnailsTask.this.publishProgress(thumbnail); + return !isCancelled(); + } + + @Override + public void failed() { + Log.w(TAG, "Thumbnail extraction failed"); + } + }); + return null; + } + + @Override + protected void onProgressUpdate(Bitmap... values) { + final VideoThumbnailsView view = viewReference.get(); + if (view != null) { + view.thumbnails.addAll(Arrays.asList(values)); + view.invalidate(); + } + } + + @Override + protected void onPostExecute(Void result) { + final VideoThumbnailsView view = viewReference.get(); + if (view != null) { + view.setDuration(duration); + view.invalidate(); + Log.i(TAG, "onPostExecute, we have " + view.thumbnails.size() + " thumbs"); + } + } + } + + public interface OnDurationListener { + void onDurationKnown(long duration); + } +} diff --git a/app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoTrackConverter.java b/app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoTrackConverter.java index 6a6a2a3e3e..8b09b9a17a 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoTrackConverter.java +++ b/app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/VideoTrackConverter.java @@ -66,7 +66,7 @@ final class VideoTrackConverter { @RequiresApi(23) static @Nullable VideoTrackConverter create( - final @NonNull MediaConverter.Input input, + final @NonNull VideoInput input, final long timeFrom, final long timeTo, final int videoResolution, diff --git a/app/src/main/res/drawable/ic_chevron_left_black_8dp.xml b/app/src/main/res/drawable/ic_chevron_left_black_8dp.xml new file mode 100644 index 0000000000..d26121fdc3 --- /dev/null +++ b/app/src/main/res/drawable/ic_chevron_left_black_8dp.xml @@ -0,0 +1,4 @@ + + + diff --git a/app/src/main/res/drawable/ic_chevron_right_black_8dp.xml b/app/src/main/res/drawable/ic_chevron_right_black_8dp.xml new file mode 100644 index 0000000000..602d546943 --- /dev/null +++ b/app/src/main/res/drawable/ic_chevron_right_black_8dp.xml @@ -0,0 +1,4 @@ + + + diff --git a/app/src/main/res/layout/mediasend_video_fragment.xml b/app/src/main/res/layout/mediasend_video_fragment.xml index 23af3bacae..55f6c852fd 100644 --- a/app/src/main/res/layout/mediasend_video_fragment.xml +++ b/app/src/main/res/layout/mediasend_video_fragment.xml @@ -1,8 +1,18 @@ - - \ No newline at end of file + + + + + \ No newline at end of file diff --git a/app/src/main/res/layout/video_editor_hud.xml b/app/src/main/res/layout/video_editor_hud.xml new file mode 100644 index 0000000000..3537f493ba --- /dev/null +++ b/app/src/main/res/layout/video_editor_hud.xml @@ -0,0 +1,59 @@ + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/app/src/main/res/values/attrs.xml b/app/src/main/res/values/attrs.xml index f193692025..539b46051d 100644 --- a/app/src/main/res/values/attrs.xml +++ b/app/src/main/res/values/attrs.xml @@ -458,4 +458,14 @@ + + + + + + + + + +