TheArchive/session-android
1
0
Fork 0
mirror of https://github.com/oxen-io/session-android.git synced 2025-08-25 18:29:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

Custom streaming video muxer.

Browse Source
This commit is contained in:
Alan Evans
2021-01-05 19:13:38 -04:00
parent 6080e1f338
commit b4c2e21415
37 changed files with 2018 additions and 167 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
app/build.gradle
View File

@@ -337,6 +337,8 @@ dependencies {
implementation project(':libsignal-service')
implementation project(':paging')
implementation project(':core-util')
implementation project(':video')
implementation 'org.signal:zkgroup-android:0.7.0'
implementation 'org.whispersystems:signal-client-android:0.1.5'
implementation 'com.google.protobuf:protobuf-javalite:3.10.0'

4
app/src/main/java/org/thoughtcrime/securesms/database/AttachmentDatabase.java
View File

@@ -749,7 +749,7 @@ public class AttachmentDatabase extends Database {
}
/**
* @param onlyModifyThisAttachment If false and more than one attachment shares this file, they will all up updated.
* @param onlyModifyThisAttachment If false and more than one attachment shares this file, they will all be updated.
* If true, then guarantees not to affect other attachments.
*/
public void updateAttachmentData(@NonNull DatabaseAttachment databaseAttachment,
@@ -1030,7 +1030,7 @@ public class AttachmentDatabase extends Database {
}
}
private File newFile() throws IOException {
public File newFile() throws IOException {
File partsDirectory = context.getDir(DIRECTORY, Context.MODE_PRIVATE);
return File.createTempFile("part", ".mms", partsDirectory);
}

94
app/src/main/java/org/thoughtcrime/securesms/jobs/AttachmentCompressionJob.java
View File

@@ -5,12 +5,18 @@ import android.media.MediaDataSource;
import androidx.annotation.NonNull;
import com.google.android.exoplayer2.util.MimeTypes;
import org.greenrobot.eventbus.EventBus;
import org.signal.core.util.logging.Log;
import org.thoughtcrime.securesms.R;
import org.thoughtcrime.securesms.attachments.Attachment;
import org.thoughtcrime.securesms.attachments.AttachmentId;
import org.thoughtcrime.securesms.attachments.DatabaseAttachment;
import org.thoughtcrime.securesms.crypto.AttachmentSecret;
import org.thoughtcrime.securesms.crypto.AttachmentSecretProvider;
import org.thoughtcrime.securesms.crypto.ModernDecryptingPartInputStream;
import org.thoughtcrime.securesms.crypto.ModernEncryptingPartOutputStream;
import org.thoughtcrime.securesms.database.AttachmentDatabase;
import org.thoughtcrime.securesms.database.DatabaseFactory;
import org.thoughtcrime.securesms.events.PartProgressEvent;
@@ -26,15 +32,22 @@ import org.thoughtcrime.securesms.service.NotificationController;
import org.thoughtcrime.securesms.transport.UndeliverableMessageException;
import org.thoughtcrime.securesms.util.BitmapDecodingException;
import org.thoughtcrime.securesms.util.BitmapUtil;
import org.thoughtcrime.securesms.util.FeatureFlags;
import org.thoughtcrime.securesms.util.MediaUtil;
import org.thoughtcrime.securesms.util.MemoryFileDescriptor;
import org.thoughtcrime.securesms.util.MemoryFileDescriptor.MemoryFileException;
import org.thoughtcrime.securesms.video.InMemoryTranscoder;
import org.thoughtcrime.securesms.video.VideoSizeException;
import org.thoughtcrime.securesms.video.StreamingTranscoder;
import org.thoughtcrime.securesms.video.TranscoderCancelationSignal;
import org.thoughtcrime.securesms.video.TranscoderOptions;
import org.thoughtcrime.securesms.video.VideoSourceException;
import org.thoughtcrime.securesms.video.videoconverter.EncodingException;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
import java.util.Objects;
import java.util.concurrent.TimeUnit;
public final class AttachmentCompressionJob extends BaseJob {
@@ -177,7 +190,7 @@ public final class AttachmentCompressionJob extends BaseJob {
@NonNull DatabaseAttachment attachment,
@NonNull MediaConstraints constraints,
@NonNull EventBus eventBus,
@NonNull InMemoryTranscoder.CancelationSignal cancelationSignal)
@NonNull TranscoderCancelationSignal cancelationSignal)
throws UndeliverableMessageException
{
AttachmentDatabase.TransformProperties transformProperties = attachment.getTransformProperties();
@@ -196,34 +209,73 @@ public final class AttachmentCompressionJob extends BaseJob {
notification.setIndeterminateProgress();
try (MediaDataSource dataSource = attachmentDatabase.mediaDataSourceFor(attachment.getAttachmentId())) {
if (dataSource == null) {
throw new UndeliverableMessageException("Cannot get media data source for attachment.");
}
allowSkipOnFailure = !transformProperties.isVideoEdited();
InMemoryTranscoder.Options options = null;
TranscoderOptions options = null;
if (transformProperties.isVideoTrim()) {
options = new InMemoryTranscoder.Options(transformProperties.getVideoTrimStartTimeUs(), transformProperties.getVideoTrimEndTimeUs());
options = new TranscoderOptions(transformProperties.getVideoTrimStartTimeUs(), transformProperties.getVideoTrimEndTimeUs());
}
try (InMemoryTranscoder transcoder = new InMemoryTranscoder(context, dataSource, options, constraints.getCompressedVideoMaxSize(context))) {
if (transcoder.isTranscodeRequired()) {
MediaStream mediaStream = transcoder.transcode(percent -> {
notification.setProgress(100, percent);
eventBus.postSticky(new PartProgressEvent(attachment,
PartProgressEvent.Type.COMPRESSION,
100,
percent));
}, cancelationSignal);
if (FeatureFlags.useStreamingVideoMuxer() || !MemoryFileDescriptor.supported()) {
StreamingTranscoder transcoder = new StreamingTranscoder(dataSource, options, constraints.getCompressedVideoMaxSize(context));
attachmentDatabase.updateAttachmentData(attachment, mediaStream, transformProperties.isVideoEdited());
attachmentDatabase.markAttachmentAsTransformed(attachment.getAttachmentId());
DatabaseAttachment updatedAttachment = attachmentDatabase.getAttachment(attachment.getAttachmentId());
if (updatedAttachment == null) {
throw new AssertionError();
if (transcoder.isTranscodeRequired()) {
Log.i(TAG, "Compressing with streaming muxer");
AttachmentSecret attachmentSecret = AttachmentSecretProvider.getInstance(context).getOrCreateAttachmentSecret();
File file = DatabaseFactory.getAttachmentDatabase(context)
.newFile();
file.deleteOnExit();
try {
try (OutputStream outputStream = ModernEncryptingPartOutputStream.createFor(attachmentSecret, file, true).second) {
transcoder.transcode(percent -> {
notification.setProgress(100, percent);
eventBus.postSticky(new PartProgressEvent(attachment,
PartProgressEvent.Type.COMPRESSION,
100,
percent));
}, outputStream, cancelationSignal);
}
MediaStream mediaStream = new MediaStream(ModernDecryptingPartInputStream.createFor(attachmentSecret, file, 0), MimeTypes.VIDEO_MP4, 0, 0);
attachmentDatabase.updateAttachmentData(attachment, mediaStream, transformProperties.isVideoEdited());
} finally {
if (!file.delete()) {
Log.w(TAG, "Failed to delete temp file");
}
}
attachmentDatabase.markAttachmentAsTransformed(attachment.getAttachmentId());
return Objects.requireNonNull(attachmentDatabase.getAttachment(attachment.getAttachmentId()));
} else {
Log.i(TAG, "Transcode was not required");
}
} else {
try (InMemoryTranscoder transcoder = new InMemoryTranscoder(context, dataSource, options, constraints.getCompressedVideoMaxSize(context))) {
if (transcoder.isTranscodeRequired()) {
Log.i(TAG, "Compressing with android in-memory muxer");
MediaStream mediaStream = transcoder.transcode(percent -> {
notification.setProgress(100, percent);
eventBus.postSticky(new PartProgressEvent(attachment,
PartProgressEvent.Type.COMPRESSION,
100,
percent));
}, cancelationSignal);
attachmentDatabase.updateAttachmentData(attachment, mediaStream, transformProperties.isVideoEdited());
attachmentDatabase.markAttachmentAsTransformed(attachment.getAttachmentId());
return Objects.requireNonNull(attachmentDatabase.getAttachment(attachment.getAttachmentId()));
} else {
Log.i(TAG, "Transcode was not required (in-memory transcoder)");
}
return updatedAttachment;
}
}
}
@@ -237,7 +289,7 @@ public final class AttachmentCompressionJob extends BaseJob {
throw new UndeliverableMessageException("Failed to transcode and cannot skip due to editing", e);
}
}
} catch (IOException | MmsException | VideoSizeException e) {
} catch (IOException | MmsException e) {
throw new UndeliverableMessageException("Failed to transcode", e);
}
return attachment;

3
app/src/main/java/org/thoughtcrime/securesms/mms/MediaConstraints.java
View File

@@ -11,6 +11,7 @@ import org.signal.core.util.logging.Log;
import org.thoughtcrime.securesms.attachments.Attachment;
import org.thoughtcrime.securesms.util.BitmapDecodingException;
import org.thoughtcrime.securesms.util.BitmapUtil;
import org.thoughtcrime.securesms.util.FeatureFlags;
import org.thoughtcrime.securesms.util.MediaUtil;
import org.thoughtcrime.securesms.util.MemoryFileDescriptor;
@@ -76,6 +77,6 @@ public abstract class MediaConstraints {
}
public static boolean isVideoTranscodeAvailable() {
return Build.VERSION.SDK_INT >= 26 && MemoryFileDescriptor.supported();
return Build.VERSION.SDK_INT >= 26 && (FeatureFlags.useStreamingVideoMuxer() || MemoryFileDescriptor.supported());
}
}

9
app/src/main/java/org/thoughtcrime/securesms/util/FeatureFlags.java
View File

@@ -65,6 +65,7 @@ public final class FeatureFlags {
private static final String GV1_FORCED_MIGRATE = "android.groupsV1Migration.forced";
private static final String GV1_MIGRATION_JOB = "android.groupsV1Migration.job";
private static final String SEND_VIEWED_RECEIPTS = "android.sendViewedReceipts";
private static final String DISABLE_CUSTOM_VIDEO_MUXER = "android.disableCustomVideoMuxer";
/**
* We will only store remote values for flags in this set. If you want a flag to be controllable
@@ -108,7 +109,8 @@ public final class FeatureFlags {
VERIFY_V2,
CLIENT_EXPIRATION,
GROUP_CALLING,
GV1_MIGRATION_JOB
GV1_MIGRATION_JOB,
DISABLE_CUSTOM_VIDEO_MUXER
);
/**
@@ -253,6 +255,11 @@ public final class FeatureFlags {
return getBoolean(SEND_VIEWED_RECEIPTS, false);
}
/** Whether to use the custom streaming muxer or built in android muxer. */
public static boolean useStreamingVideoMuxer() {
return !getBoolean(DISABLE_CUSTOM_VIDEO_MUXER, false);
}
/** Only for rendering debug info. */
public static synchronized @NonNull Map<String, Object> getMemoryValues() {
return new TreeMap<>(REMOTE_VALUES);

20
app/src/main/java/org/thoughtcrime/securesms/video/InMemoryTranscoder.java
View File

@@ -39,14 +39,14 @@ public final class InMemoryTranscoder implements Closeable {
private final long memoryFileEstimate;
private final boolean transcodeRequired;
private final long fileSizeEstimate;
private final @Nullable Options options;
private final @Nullable TranscoderOptions options;
private @Nullable MemoryFileDescriptor memoryFile;
/**
* @param upperSizeLimit A upper size to transcode to. The actual output size can be up to 10% smaller.
*/
public InMemoryTranscoder(@NonNull Context context, @NonNull MediaDataSource dataSource, @Nullable Options options, long upperSizeLimit) throws IOException, VideoSourceException {
public InMemoryTranscoder(@NonNull Context context, @NonNull MediaDataSource dataSource, @Nullable TranscoderOptions options, long upperSizeLimit) throws IOException, VideoSourceException {
this.context = context;
this.dataSource = dataSource;
this.options = options;
@@ -75,7 +75,7 @@ public final class InMemoryTranscoder implements Closeable {
}
public @NonNull MediaStream transcode(@NonNull Progress progress,
@Nullable CancelationSignal cancelationSignal)
@Nullable TranscoderCancelationSignal cancelationSignal)
throws IOException, EncodingException, VideoSizeException
{
if (memoryFile != null) throw new AssertionError("Not expecting to reuse transcoder");
@@ -202,18 +202,4 @@ public final class InMemoryTranscoder implements Closeable {
public interface Progress {
void onProgress(int percent);
}
public interface CancelationSignal {
boolean isCanceled();
}
public final static class Options {
final long startTimeUs;
final long endTimeUs;
public Options(long startTimeUs, long endTimeUs) {
this.startTimeUs = startTimeUs;
this.endTimeUs = endTimeUs;
}
}
}

214
app/src/main/java/org/thoughtcrime/securesms/video/StreamingTranscoder.java Normal file
View File

@@ -0,0 +1,214 @@
package org.thoughtcrime.securesms.video;
import android.media.MediaDataSource;
import android.media.MediaMetadataRetriever;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import androidx.annotation.RequiresApi;
import org.signal.core.util.logging.Log;
import org.thoughtcrime.securesms.media.MediaInput;
import org.thoughtcrime.securesms.video.videoconverter.EncodingException;
import org.thoughtcrime.securesms.video.videoconverter.MediaConverter;
import java.io.FilterOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.text.NumberFormat;
import java.util.Locale;
@RequiresApi(26)
public final class StreamingTranscoder {
private static final String TAG = Log.tag(StreamingTranscoder.class);
private final MediaDataSource dataSource;
private final long upperSizeLimit;
private final long inSize;
private final long duration;
private final int inputBitRate;
private final VideoBitRateCalculator.Quality targetQuality;
private final long memoryFileEstimate;
private final boolean transcodeRequired;
private final long fileSizeEstimate;
private final @Nullable TranscoderOptions options;
/**
* @param upperSizeLimit A upper size to transcode to. The actual output size can be up to 10% smaller.
*/
public StreamingTranscoder(@NonNull MediaDataSource dataSource,
@Nullable TranscoderOptions options,
long upperSizeLimit)
throws IOException, VideoSourceException
{
this.dataSource = dataSource;
this.options = options;
final MediaMetadataRetriever mediaMetadataRetriever = new MediaMetadataRetriever();
try {
mediaMetadataRetriever.setDataSource(dataSource);
} catch (RuntimeException e) {
Log.w(TAG, "Unable to read datasource", e);
throw new VideoSourceException("Unable to read datasource", e);
}
this.inSize = dataSource.getSize();
this.duration = getDuration(mediaMetadataRetriever);
this.inputBitRate = VideoBitRateCalculator.bitRate(inSize, duration);
this.targetQuality = new VideoBitRateCalculator(upperSizeLimit).getTargetQuality(duration, inputBitRate);
this.upperSizeLimit = upperSizeLimit;
this.transcodeRequired = inputBitRate >= targetQuality.getTargetTotalBitRate() * 1.2 || inSize > upperSizeLimit || containsLocation(mediaMetadataRetriever) || options != null;
if (!transcodeRequired) {
Log.i(TAG, "Video is within 20% of target bitrate, below the size limit, contained no location metadata or custom options.");
}
this.fileSizeEstimate = targetQuality.getFileSizeEstimate();
this.memoryFileEstimate = (long) (fileSizeEstimate * 1.1);
}
public void transcode(@NonNull Progress progress,
@NonNull OutputStream stream,
@Nullable TranscoderCancelationSignal cancelationSignal)
throws IOException, EncodingException
{
float durationSec = duration / 1000f;
NumberFormat numberFormat = NumberFormat.getInstance(Locale.US);
Log.i(TAG, String.format(Locale.US,
"Transcoding:\n" +
"Target bitrate : %s + %s = %s\n" +
"Target format : %dp\n" +
"Video duration : %.1fs\n" +
"Size limit : %s kB\n" +
"Estimate : %s kB\n" +
"Input size : %s kB\n" +
"Input bitrate : %s bps",
numberFormat.format(targetQuality.getTargetVideoBitRate()),
numberFormat.format(targetQuality.getTargetAudioBitRate()),
numberFormat.format(targetQuality.getTargetTotalBitRate()),
targetQuality.getOutputResolution(),
durationSec,
numberFormat.format(upperSizeLimit / 1024),
numberFormat.format(fileSizeEstimate / 1024),
numberFormat.format(inSize / 1024),
numberFormat.format(inputBitRate)));
if (fileSizeEstimate > upperSizeLimit) {
throw new VideoSizeException("Size constraints could not be met!");
}
final long startTime = System.currentTimeMillis();
final MediaConverter converter = new MediaConverter();
final LimitedSizeOutputStream limitedSizeOutputStream = new LimitedSizeOutputStream(stream, upperSizeLimit);
converter.setInput(new MediaInput.MediaDataSourceMediaInput(dataSource));
converter.setOutput(limitedSizeOutputStream);
converter.setVideoResolution(targetQuality.getOutputResolution());
converter.setVideoBitrate(targetQuality.getTargetVideoBitRate());
converter.setAudioBitrate(targetQuality.getTargetAudioBitRate());
if (options != null) {
if (options.endTimeUs > 0) {
long timeFrom = options.startTimeUs / 1000;
long timeTo = options.endTimeUs / 1000;
converter.setTimeRange(timeFrom, timeTo);
Log.i(TAG, String.format(Locale.US, "Trimming:\nTotal duration: %d\nKeeping: %d..%d\nFinal duration:(%d)", duration, timeFrom, timeTo, timeTo - timeFrom));
}
}
converter.setListener(percent -> {
progress.onProgress(percent);
return cancelationSignal != null && cancelationSignal.isCanceled();
});
converter.convert();
long outSize = limitedSizeOutputStream.written;
float encodeDurationSec = (System.currentTimeMillis() - startTime) / 1000f;
Log.i(TAG, String.format(Locale.US,
"Transcoding complete:\n" +
"Transcode time : %.1fs (%.1fx)\n" +
"Output size : %s kB\n" +
" of Original : %.1f%%\n" +
" of Estimate : %.1f%%\n" +
" of Memory : %.1f%%\n" +
"Output bitrate : %s bps",
encodeDurationSec,
durationSec / encodeDurationSec,
numberFormat.format(outSize / 1024),
(outSize * 100d) / inSize,
(outSize * 100d) / fileSizeEstimate,
(outSize * 100d) / memoryFileEstimate,
numberFormat.format(VideoBitRateCalculator.bitRate(outSize, duration))));
if (outSize > upperSizeLimit) {
throw new VideoSizeException("Size constraints could not be met!");
}
stream.flush();
}
public boolean isTranscodeRequired() {
return transcodeRequired;
}
private static long getDuration(MediaMetadataRetriever mediaMetadataRetriever) throws VideoSourceException {
String durationString = mediaMetadataRetriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_DURATION);
if (durationString == null) {
throw new VideoSourceException("Cannot determine duration of video, null meta data");
}
try {
long duration = Long.parseLong(durationString);
if (duration <= 0) {
throw new VideoSourceException("Cannot determine duration of video, meta data: " + durationString);
}
return duration;
} catch (NumberFormatException e) {
throw new VideoSourceException("Cannot determine duration of video, meta data: " + durationString, e);
}
}
private static boolean containsLocation(MediaMetadataRetriever mediaMetadataRetriever) {
String locationString = mediaMetadataRetriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_LOCATION);
return locationString != null;
}
public interface Progress {
void onProgress(int percent);
}
private static class LimitedSizeOutputStream extends FilterOutputStream {
private final long sizeLimit;
private long written;
LimitedSizeOutputStream(@NonNull OutputStream inner, long sizeLimit) {
super(inner);
this.sizeLimit = sizeLimit;
}
@Override public void write(int b) throws IOException {
incWritten(1);
out.write(b);
}
@Override public void write(byte[] b, int off, int len) throws IOException {
incWritten(len);
out.write(b, off, len);
}
private void incWritten(int len) throws IOException {
long newWritten = written + len;
if (newWritten > sizeLimit) {
Log.w(TAG, String.format(Locale.US, "File size limit hit. Wrote %d, tried to write %d more. Limit is %d", written, len, sizeLimit));
throw new VideoSizeException("File size limit hit");
}
written = newWritten;
}
}
}

5
app/src/main/java/org/thoughtcrime/securesms/video/TranscoderCancelationSignal.java Normal file
View File

@@ -0,0 +1,5 @@
package org.thoughtcrime.securesms.video;
public interface TranscoderCancelationSignal {
boolean isCanceled();
}

11
app/src/main/java/org/thoughtcrime/securesms/video/TranscoderOptions.java Normal file
View File

@@ -0,0 +1,11 @@
package org.thoughtcrime.securesms.video;
public final class TranscoderOptions {
final long startTimeUs;
final long endTimeUs;
public TranscoderOptions(long startTimeUs, long endTimeUs) {
this.startTimeUs = startTimeUs;
this.endTimeUs = endTimeUs;
}
}

4
app/src/main/java/org/thoughtcrime/securesms/video/VideoSizeException.java
View File

@@ -1,6 +1,8 @@
package org.thoughtcrime.securesms.video;
public final class VideoSizeException extends Exception {
import java.io.IOException;
public final class VideoSizeException extends IOException {
VideoSizeException(String message) {
super(message);

54
app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/AndroidMuxer.java
View File

@@ -1,54 +0,0 @@
package org.thoughtcrime.securesms.video.videoconverter;
import android.media.MediaCodec;
import android.media.MediaFormat;
import android.media.MediaMuxer;
import androidx.annotation.NonNull;
import androidx.annotation.RequiresApi;
import java.io.File;
import java.io.FileDescriptor;
import java.io.IOException;
import java.nio.ByteBuffer;
public final class AndroidMuxer implements Muxer {
private final MediaMuxer muxer;
AndroidMuxer(final @NonNull File file) throws IOException {
muxer = new MediaMuxer(file.getAbsolutePath(), MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
}
@RequiresApi(26)
AndroidMuxer(final @NonNull FileDescriptor fileDescriptor) throws IOException {
muxer = new MediaMuxer(fileDescriptor, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
}
@Override
public void start() {
muxer.start();
}
@Override
public void stop() {
muxer.stop();
}
@Override
public int addTrack(final @NonNull MediaFormat format) {
return muxer.addTrack(format);
}
@Override
public void writeSampleData(final int trackIndex, final @NonNull ByteBuffer byteBuf, final @NonNull MediaCodec.BufferInfo bufferInfo) {
muxer.writeSampleData(trackIndex, byteBuf, bufferInfo);
}
@Override
public void release() {
muxer.release();
}
}

11
app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/EncodingException.java
View File

@@ -1,11 +0,0 @@
package org.thoughtcrime.securesms.video.videoconverter;
public final class EncodingException extends Exception {
EncodingException(String message) {
super(message);
}
EncodingException(String message, Exception inner) {
super(message, inner);
}
}

187
app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/InputSurface.java
View File

@@ -1,187 +0,0 @@
/*
* Copyright (C) 2013 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* This file has been modified by Signal.
*/
package org.thoughtcrime.securesms.video.videoconverter;
import android.opengl.EGL14;
import android.opengl.EGLConfig;
import android.opengl.EGLContext;
import android.opengl.EGLDisplay;
import android.opengl.EGLExt;
import android.opengl.EGLSurface;
import android.view.Surface;
import org.signal.core.util.logging.Log;
/**
* Holds state associated with a Surface used for MediaCodec encoder input.
* <p>
* The constructor takes a Surface obtained from MediaCodec.createInputSurface(), and uses that
* to create an EGL window surface. Calls to eglSwapBuffers() cause a frame of data to be sent
* to the video encoder.
*/
final class InputSurface {
private static final String TAG = "InputSurface";
private static final boolean VERBOSE = false;
private static final int EGL_RECORDABLE_ANDROID = 0x3142;
private static final int EGL_OPENGL_ES2_BIT = 4;
private EGLDisplay mEGLDisplay;
private EGLContext mEGLContext;
private EGLSurface mEGLSurface;
private Surface mSurface;
/**
* Creates an InputSurface from a Surface.
*/
InputSurface(Surface surface) throws TranscodingException {
if (surface == null) {
throw new NullPointerException();
}
mSurface = surface;
eglSetup();
}
/**
* Prepares EGL. We want a GLES 2.0 context and a surface that supports recording.
*/
private void eglSetup() throws TranscodingException {
mEGLDisplay = EGL14.eglGetDisplay(EGL14.EGL_DEFAULT_DISPLAY);
if (mEGLDisplay == EGL14.EGL_NO_DISPLAY) {
throw new TranscodingException("unable to get EGL14 display");
}
int[] version = new int[2];
if (!EGL14.eglInitialize(mEGLDisplay, version, 0, version, 1)) {
mEGLDisplay = null;
throw new TranscodingException("unable to initialize EGL14");
}
// Configure EGL for pbuffer and OpenGL ES 2.0. We want enough RGB bits
// to be able to tell if the frame is reasonable.
int[] attribList = {
EGL14.EGL_RED_SIZE, 8,
EGL14.EGL_GREEN_SIZE, 8,
EGL14.EGL_BLUE_SIZE, 8,
EGL14.EGL_RENDERABLE_TYPE, EGL_OPENGL_ES2_BIT,
EGL_RECORDABLE_ANDROID, 1,
EGL14.EGL_NONE
};
EGLConfig[] configs = new EGLConfig[1];
int[] numConfigs = new int[1];
if (!EGL14.eglChooseConfig(mEGLDisplay, attribList, 0, configs, 0, configs.length,
numConfigs, 0)) {
throw new TranscodingException("unable to find RGB888+recordable ES2 EGL config");
}
// Configure context for OpenGL ES 2.0.
int[] attrib_list = {
EGL14.EGL_CONTEXT_CLIENT_VERSION, 2,
EGL14.EGL_NONE
};
mEGLContext = EGL14.eglCreateContext(mEGLDisplay, configs[0], EGL14.EGL_NO_CONTEXT,
attrib_list, 0);
checkEglError("eglCreateContext");
if (mEGLContext == null) {
throw new TranscodingException("null context");
}
// Create a window surface, and attach it to the Surface we received.
int[] surfaceAttribs = {
EGL14.EGL_NONE
};
mEGLSurface = EGL14.eglCreateWindowSurface(mEGLDisplay, configs[0], mSurface,
surfaceAttribs, 0);
checkEglError("eglCreateWindowSurface");
if (mEGLSurface == null) {
throw new TranscodingException("surface was null");
}
}
/**
* Discard all resources held by this class, notably the EGL context. Also releases the
* Surface that was passed to our constructor.
*/
public void release() {
if (EGL14.eglGetCurrentContext().equals(mEGLContext)) {
// Clear the current context and surface to ensure they are discarded immediately.
EGL14.eglMakeCurrent(mEGLDisplay, EGL14.EGL_NO_SURFACE, EGL14.EGL_NO_SURFACE,
EGL14.EGL_NO_CONTEXT);
}
EGL14.eglDestroySurface(mEGLDisplay, mEGLSurface);
EGL14.eglDestroyContext(mEGLDisplay, mEGLContext);
//EGL14.eglTerminate(mEGLDisplay);
mSurface.release();
// null everything out so future attempts to use this object will cause an NPE
mEGLDisplay = null;
mEGLContext = null;
mEGLSurface = null;
mSurface = null;
}
/**
* Makes our EGL context and surface current.
*/
void makeCurrent() throws TranscodingException {
if (!EGL14.eglMakeCurrent(mEGLDisplay, mEGLSurface, mEGLSurface, mEGLContext)) {
throw new TranscodingException("eglMakeCurrent failed");
}
}
/**
* Calls eglSwapBuffers. Use this to "publish" the current frame.
*/
boolean swapBuffers() {
return EGL14.eglSwapBuffers(mEGLDisplay, mEGLSurface);
}
/**
* Returns the Surface that the MediaCodec receives buffers from.
*/
public Surface getSurface() {
return mSurface;
}
/**
* Sends the presentation time stamp to EGL. Time is expressed in nanoseconds.
*/
void setPresentationTime(long nsecs) {
EGLExt.eglPresentationTimeANDROID(mEGLDisplay, mEGLSurface, nsecs);
}
/**
* Checks for EGL errors.
*/
private static void checkEglError(String msg) throws TranscodingException {
boolean failed = false;
int error;
while ((error = EGL14.eglGetError()) != EGL14.EGL_SUCCESS) {
Log.e(TAG, msg + ": EGL error: 0x" + Integer.toHexString(error));
failed = true;
}
if (failed) {
throw new TranscodingException("EGL error encountered (see log)");
}
}
}

20
app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/MediaConverter.java
View File

@@ -30,11 +30,13 @@ import androidx.annotation.WorkerThread;
import org.signal.core.util.logging.Log;
import org.thoughtcrime.securesms.media.MediaInput;
import org.thoughtcrime.securesms.video.videoconverter.muxer.StreamingMuxer;
import java.io.File;
import java.io.FileDescriptor;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.OutputStream;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
@@ -85,6 +87,10 @@ public final class MediaConverter {
mOutput = new FileDescriptorOutput(fileDescriptor);
}
public void setOutput(final @NonNull OutputStream stream) {
mOutput = new StreamOutput(stream);
}
@SuppressWarnings("unused")
public void setTimeRange(long timeFrom, long timeTo) {
mTimeFrom = timeFrom;
@@ -332,4 +338,18 @@ public final class MediaConverter {
return new AndroidMuxer(fileDescriptor);
}
}
private static class StreamOutput implements Output {
final OutputStream outputStream;
StreamOutput(final @NonNull OutputStream outputStream) {
this.outputStream = outputStream;
}
@Override
public @NonNull Muxer createMuxer() {
return new StreamingMuxer(outputStream);
}
}
}

22
app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/Muxer.java
View File

@@ -1,22 +0,0 @@
package org.thoughtcrime.securesms.video.videoconverter;
import android.media.MediaCodec;
import android.media.MediaFormat;
import androidx.annotation.NonNull;
import java.io.IOException;
import java.nio.ByteBuffer;
public interface Muxer {
void start() throws IOException;
void stop() throws IOException;
int addTrack(@NonNull MediaFormat format) throws IOException;
void writeSampleData(int trackIndex, @NonNull ByteBuffer byteBuf, @NonNull MediaCodec.BufferInfo bufferInfo) throws IOException;
void release();
}

303
app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/OutputSurface.java
View File

@@ -1,303 +0,0 @@
/*
* Copyright (C) 2013 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* This file has been modified by Signal.
*/
package org.thoughtcrime.securesms.video.videoconverter;
import android.graphics.SurfaceTexture;
import android.opengl.EGL14;
import android.view.Surface;
import org.signal.core.util.logging.Log;
import javax.microedition.khronos.egl.EGL10;
import javax.microedition.khronos.egl.EGLConfig;
import javax.microedition.khronos.egl.EGLContext;
import javax.microedition.khronos.egl.EGLDisplay;
import javax.microedition.khronos.egl.EGLSurface;
/**
* Holds state associated with a Surface used for MediaCodec decoder output.
* <p>
* The (width,height) constructor for this class will prepare GL, create a SurfaceTexture,
* and then create a Surface for that SurfaceTexture. The Surface can be passed to
* MediaCodec.configure() to receive decoder output. When a frame arrives, we latch the
* texture with updateTexImage, then render the texture with GL to a pbuffer.
* <p>
* The no-arg constructor skips the GL preparation step and doesn't allocate a pbuffer.
* Instead, it just creates the Surface and SurfaceTexture, and when a frame arrives
* we just draw it on whatever surface is current.
* <p>
* By default, the Surface will be using a BufferQueue in asynchronous mode, so we
* can potentially drop frames.
*/
final class OutputSurface implements SurfaceTexture.OnFrameAvailableListener {
private static final String TAG = "OutputSurface";
private static final boolean VERBOSE = false;
private static final int EGL_OPENGL_ES2_BIT = 4;
private EGL10 mEGL;
private EGLDisplay mEGLDisplay;
private EGLContext mEGLContext;
private EGLSurface mEGLSurface;
private SurfaceTexture mSurfaceTexture;
private Surface mSurface;
private final Object mFrameSyncObject = new Object(); // guards mFrameAvailable
private boolean mFrameAvailable;
private TextureRender mTextureRender;
/**
* Creates an OutputSurface backed by a pbuffer with the specifed dimensions. The new
* EGL context and surface will be made current. Creates a Surface that can be passed
* to MediaCodec.configure().
*/
OutputSurface(int width, int height, boolean flipX) throws TranscodingException {
if (width <= 0 || height <= 0) {
throw new IllegalArgumentException();
}
eglSetup(width, height);
makeCurrent();
setup(flipX);
}
/**
* Creates an OutputSurface using the current EGL context. Creates a Surface that can be
* passed to MediaCodec.configure().
*/
OutputSurface() throws TranscodingException {
setup(false);
}
/**
* Creates instances of TextureRender and SurfaceTexture, and a Surface associated
* with the SurfaceTexture.
*/
private void setup(boolean flipX) throws TranscodingException {
mTextureRender = new TextureRender(flipX);
mTextureRender.surfaceCreated();
// Even if we don't access the SurfaceTexture after the constructor returns, we
// still need to keep a reference to it. The Surface doesn't retain a reference
// at the Java level, so if we don't either then the object can get GCed, which
// causes the native finalizer to run.
if (VERBOSE) Log.d(TAG, "textureID=" + mTextureRender.getTextureId());
mSurfaceTexture = new SurfaceTexture(mTextureRender.getTextureId());
// This doesn't work if OutputSurface is created on the thread that CTS started for
// these test cases.
//
// The CTS-created thread has a Looper, and the SurfaceTexture constructor will
// create a Handler that uses it. The "frame available" message is delivered
// there, but since we're not a Looper-based thread we'll never see it. For
// this to do anything useful, OutputSurface must be created on a thread without
// a Looper, so that SurfaceTexture uses the main application Looper instead.
//
// Java language note: passing "this" out of a constructor is generally unwise,
// but we should be able to get away with it here.
mSurfaceTexture.setOnFrameAvailableListener(this);
mSurface = new Surface(mSurfaceTexture);
}
/**
* Prepares EGL. We want a GLES 2.0 context and a surface that supports pbuffer.
*/
private void eglSetup(int width, int height) throws TranscodingException {
mEGL = (EGL10)EGLContext.getEGL();
mEGLDisplay = mEGL.eglGetDisplay(EGL10.EGL_DEFAULT_DISPLAY);
if (!mEGL.eglInitialize(mEGLDisplay, null)) {
throw new TranscodingException("unable to initialize EGL10");
}
// Configure EGL for pbuffer and OpenGL ES 2.0. We want enough RGB bits
// to be able to tell if the frame is reasonable.
int[] attribList = {
EGL10.EGL_RED_SIZE, 8,
EGL10.EGL_GREEN_SIZE, 8,
EGL10.EGL_BLUE_SIZE, 8,
EGL10.EGL_SURFACE_TYPE, EGL10.EGL_PBUFFER_BIT,
EGL10.EGL_RENDERABLE_TYPE, EGL_OPENGL_ES2_BIT,
EGL10.EGL_NONE
};
EGLConfig[] configs = new EGLConfig[1];
int[] numConfigs = new int[1];
if (!mEGL.eglChooseConfig(mEGLDisplay, attribList, configs, 1, numConfigs)) {
throw new TranscodingException("unable to find RGB888+pbuffer EGL config");
}
// Configure context for OpenGL ES 2.0.
int[] attrib_list = {
EGL14.EGL_CONTEXT_CLIENT_VERSION, 2,
EGL10.EGL_NONE
};
mEGLContext = mEGL.eglCreateContext(mEGLDisplay, configs[0], EGL10.EGL_NO_CONTEXT,
attrib_list);
checkEglError("eglCreateContext");
if (mEGLContext == null) {
throw new TranscodingException("null context");
}
// Create a pbuffer surface. By using this for output, we can use glReadPixels
// to test values in the output.
int[] surfaceAttribs = {
EGL10.EGL_WIDTH, width,
EGL10.EGL_HEIGHT, height,
EGL10.EGL_NONE
};
mEGLSurface = mEGL.eglCreatePbufferSurface(mEGLDisplay, configs[0], surfaceAttribs);
checkEglError("eglCreatePbufferSurface");
if (mEGLSurface == null) {
throw new TranscodingException("surface was null");
}
}
/**
* Discard all resources held by this class, notably the EGL context.
*/
public void release() {
if (mEGL != null) {
if (mEGL.eglGetCurrentContext().equals(mEGLContext)) {
// Clear the current context and surface to ensure they are discarded immediately.
mEGL.eglMakeCurrent(mEGLDisplay, EGL10.EGL_NO_SURFACE, EGL10.EGL_NO_SURFACE,
EGL10.EGL_NO_CONTEXT);
}
mEGL.eglDestroySurface(mEGLDisplay, mEGLSurface);
mEGL.eglDestroyContext(mEGLDisplay, mEGLContext);
//mEGL.eglTerminate(mEGLDisplay);
}
mSurface.release();
// this causes a bunch of warnings that appear harmless but might confuse someone:
// W BufferQueue: [unnamed-3997-2] cancelBuffer: BufferQueue has been abandoned!
//mSurfaceTexture.release();
// null everything out so future attempts to use this object will cause an NPE
mEGLDisplay = null;
mEGLContext = null;
mEGLSurface = null;
mEGL = null;
mTextureRender = null;
mSurface = null;
mSurfaceTexture = null;
}
/**
* Makes our EGL context and surface current.
*/
private void makeCurrent() throws TranscodingException {
if (mEGL == null) {
throw new TranscodingException("not configured for makeCurrent");
}
checkEglError("before makeCurrent");
if (!mEGL.eglMakeCurrent(mEGLDisplay, mEGLSurface, mEGLSurface, mEGLContext)) {
throw new TranscodingException("eglMakeCurrent failed");
}
}
/**
* Returns the Surface that we draw onto.
*/
public Surface getSurface() {
return mSurface;
}
/**
* Replaces the fragment shader.
*/
void changeFragmentShader(String fragmentShader) throws TranscodingException {
mTextureRender.changeFragmentShader(fragmentShader);
}
/**
* Latches the next buffer into the texture. Must be called from the thread that created
* the OutputSurface object, after the onFrameAvailable callback has signaled that new
* data is available.
*/
void awaitNewImage() throws TranscodingException {
final int TIMEOUT_MS = 750;
synchronized (mFrameSyncObject) {
final long expireTime = System.currentTimeMillis() + TIMEOUT_MS;
while (!mFrameAvailable) {
try {
// Wait for onFrameAvailable() to signal us. Use a timeout to avoid
// stalling the test if it doesn't arrive.
mFrameSyncObject.wait(TIMEOUT_MS);
if (!mFrameAvailable && System.currentTimeMillis() > expireTime) {
throw new TranscodingException("Surface frame wait timed out");
}
} catch (InterruptedException ie) {
// shouldn't happen
throw new TranscodingException(ie);
}
}
mFrameAvailable = false;
}
// Latch the data.
TextureRender.checkGlError("before updateTexImage");
mSurfaceTexture.updateTexImage();
}
/**
* Draws the data from SurfaceTexture onto the current EGL surface.
*/
void drawImage() throws TranscodingException {
mTextureRender.drawFrame(mSurfaceTexture);
}
@Override
public void onFrameAvailable(SurfaceTexture st) {
if (VERBOSE) Log.d(TAG, "new frame available");
synchronized (mFrameSyncObject) {
if (mFrameAvailable) {
try {
throw new TranscodingException("mFrameAvailable already set, frame could be dropped");
} catch (TranscodingException e) {
e.printStackTrace();
}
}
mFrameAvailable = true;
mFrameSyncObject.notifyAll();
}
}
/**
* Checks for EGL errors.
*/
private void checkEglError(String msg) throws TranscodingException {
boolean failed = false;
int error;
while ((error = mEGL.eglGetError()) != EGL10.EGL_SUCCESS) {
Log.e(TAG, msg + ": EGL error: 0x" + Integer.toHexString(error));
failed = true;
}
if (failed) {
throw new TranscodingException("EGL error encountered (see log)");
}
}
}

10
app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/Preconditions.java
View File

@@ -1,10 +0,0 @@
package org.thoughtcrime.securesms.video.videoconverter;
final class Preconditions {
static void checkState(final Object errorMessage, final boolean expression) {
if (!expression) {
throw new IllegalStateException(String.valueOf(errorMessage));
}
}
}

258
app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/TextureRender.java
View File

@@ -1,258 +0,0 @@
/*
* Copyright (C) 2013 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* This file has been modified by Signal.
*/
package org.thoughtcrime.securesms.video.videoconverter;
import android.graphics.SurfaceTexture;
import android.opengl.GLES11Ext;
import android.opengl.GLES20;
import android.opengl.Matrix;
import org.signal.core.util.logging.Log;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.FloatBuffer;
/**
* Code for rendering a texture onto a surface using OpenGL ES 2.0.
*/
final class TextureRender {
private static final String TAG = "TextureRender";
private static final int FLOAT_SIZE_BYTES = 4;
private static final int TRIANGLE_VERTICES_DATA_STRIDE_BYTES = 5 * FLOAT_SIZE_BYTES;
private static final int TRIANGLE_VERTICES_DATA_POS_OFFSET = 0;
private static final int TRIANGLE_VERTICES_DATA_UV_OFFSET = 3;
private final float[] mTriangleVerticesData = {
// X, Y, Z, U, V
-1.0f, -1.0f, 0, 0.f, 0.f,
1.0f, -1.0f, 0, 1.f, 0.f,
-1.0f, 1.0f, 0, 0.f, 1.f,
1.0f, 1.0f, 0, 1.f, 1.f,
};
private final float[] mTriangleVerticesDataFlippedX = {
// X, Y, Z, U, V
-1.0f, -1.0f, 0, 1.f, 0.f,
1.0f, -1.0f, 0, 0.f, 0.f,
-1.0f, 1.0f, 0, 1.f, 1.f,
1.0f, 1.0f, 0, 0.f, 1.f,
};
private final FloatBuffer mTriangleVertices;
private static final String VERTEX_SHADER =
"uniform mat4 uMVPMatrix;\n" +
"uniform mat4 uSTMatrix;\n" +
"attribute vec4 aPosition;\n" +
"attribute vec4 aTextureCoord;\n" +
"varying vec2 vTextureCoord;\n" +
"void main() {\n" +
" gl_Position = uMVPMatrix * aPosition;\n" +
" vTextureCoord = (uSTMatrix * aTextureCoord).xy;\n" +
"}\n";
private static final String FRAGMENT_SHADER =
"#extension GL_OES_EGL_image_external : require\n" +
"precision mediump float;\n" + // highp here doesn't seem to matter
"varying vec2 vTextureCoord;\n" +
"uniform samplerExternalOES sTexture;\n" +
"void main() {\n" +
" gl_FragColor = texture2D(sTexture, vTextureCoord);\n" +
"}\n";
private final float[] mMVPMatrix = new float[16];
private final float[] mSTMatrix = new float[16];
private int mProgram;
private int mTextureID = -12345;
private int muMVPMatrixHandle;
private int muSTMatrixHandle;
private int maPositionHandle;
private int maTextureHandle;
TextureRender(boolean flipX) {
float[] verticesData = flipX ? mTriangleVerticesDataFlippedX : mTriangleVerticesData;
mTriangleVertices = ByteBuffer.allocateDirect(
verticesData.length * FLOAT_SIZE_BYTES)
.order(ByteOrder.nativeOrder()).asFloatBuffer();
mTriangleVertices.put(verticesData).position(0);
Matrix.setIdentityM(mSTMatrix, 0);
}
int getTextureId() {
return mTextureID;
}
void drawFrame(SurfaceTexture st) throws TranscodingException {
checkGlError("onDrawFrame start");
st.getTransformMatrix(mSTMatrix);
GLES20.glClearColor(0.0f, 1.0f, 0.0f, 1.0f);
GLES20.glClear(GLES20.GL_DEPTH_BUFFER_BIT | GLES20.GL_COLOR_BUFFER_BIT);
GLES20.glUseProgram(mProgram);
checkGlError("glUseProgram");
GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, mTextureID);
mTriangleVertices.position(TRIANGLE_VERTICES_DATA_POS_OFFSET);
GLES20.glVertexAttribPointer(maPositionHandle, 3, GLES20.GL_FLOAT, false,
TRIANGLE_VERTICES_DATA_STRIDE_BYTES, mTriangleVertices);
checkGlError("glVertexAttribPointer maPosition");
GLES20.glEnableVertexAttribArray(maPositionHandle);
checkGlError("glEnableVertexAttribArray maPositionHandle");
mTriangleVertices.position(TRIANGLE_VERTICES_DATA_UV_OFFSET);
GLES20.glVertexAttribPointer(maTextureHandle, 2, GLES20.GL_FLOAT, false,
TRIANGLE_VERTICES_DATA_STRIDE_BYTES, mTriangleVertices);
checkGlError("glVertexAttribPointer maTextureHandle");
GLES20.glEnableVertexAttribArray(maTextureHandle);
checkGlError("glEnableVertexAttribArray maTextureHandle");
Matrix.setIdentityM(mMVPMatrix, 0);
GLES20.glUniformMatrix4fv(muMVPMatrixHandle, 1, false, mMVPMatrix, 0);
GLES20.glUniformMatrix4fv(muSTMatrixHandle, 1, false, mSTMatrix, 0);
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4);
checkGlError("glDrawArrays");
GLES20.glFinish();
}
/**
* Initializes GL state. Call this after the EGL surface has been created and made current.
*/
void surfaceCreated() throws TranscodingException {
mProgram = createProgram(VERTEX_SHADER, FRAGMENT_SHADER);
if (mProgram == 0) {
throw new TranscodingException("failed creating program");
}
maPositionHandle = GLES20.glGetAttribLocation(mProgram, "aPosition");
checkGlError("glGetAttribLocation aPosition");
if (maPositionHandle == -1) {
throw new TranscodingException("Could not get attrib location for aPosition");
}
maTextureHandle = GLES20.glGetAttribLocation(mProgram, "aTextureCoord");
checkGlError("glGetAttribLocation aTextureCoord");
if (maTextureHandle == -1) {
throw new TranscodingException("Could not get attrib location for aTextureCoord");
}
muMVPMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uMVPMatrix");
checkGlError("glGetUniformLocation uMVPMatrix");
if (muMVPMatrixHandle == -1) {
throw new TranscodingException("Could not get attrib location for uMVPMatrix");
}
muSTMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uSTMatrix");
checkGlError("glGetUniformLocation uSTMatrix");
if (muSTMatrixHandle == -1) {
throw new TranscodingException("Could not get attrib location for uSTMatrix");
}
int[] textures = new int[1];
GLES20.glGenTextures(1, textures, 0);
mTextureID = textures[0];
GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, mTextureID);
checkGlError("glBindTexture mTextureID");
GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MIN_FILTER,
GLES20.GL_LINEAR);
GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MAG_FILTER,
GLES20.GL_LINEAR);
GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_S,
GLES20.GL_CLAMP_TO_EDGE);
GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_T,
GLES20.GL_CLAMP_TO_EDGE);
checkGlError("glTexParameter");
}
/**
* Replaces the fragment shader.
*/
public void changeFragmentShader(String fragmentShader) throws TranscodingException {
GLES20.glDeleteProgram(mProgram);
mProgram = createProgram(VERTEX_SHADER, fragmentShader);
if (mProgram == 0) {
throw new TranscodingException("failed creating program");
}
}
private static int loadShader(int shaderType, String source) throws TranscodingException {
int shader = GLES20.glCreateShader(shaderType);
checkGlError("glCreateShader type=" + shaderType);
GLES20.glShaderSource(shader, source);
GLES20.glCompileShader(shader);
int[] compiled = new int[1];
GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, compiled, 0);
if (compiled[0] == 0) {
Log.e(TAG, "Could not compile shader " + shaderType + ":");
Log.e(TAG, " " + GLES20.glGetShaderInfoLog(shader));
GLES20.glDeleteShader(shader);
shader = 0;
}
return shader;
}
private int createProgram(String vertexSource, String fragmentSource) throws TranscodingException {
int vertexShader = loadShader(GLES20.GL_VERTEX_SHADER, vertexSource);
if (vertexShader == 0) {
return 0;
}
int pixelShader = loadShader(GLES20.GL_FRAGMENT_SHADER, fragmentSource);
if (pixelShader == 0) {
return 0;
}
int program = GLES20.glCreateProgram();
checkGlError("glCreateProgram");
if (program == 0) {
Log.e(TAG, "Could not create program");
}
GLES20.glAttachShader(program, vertexShader);
checkGlError("glAttachShader");
GLES20.glAttachShader(program, pixelShader);
checkGlError("glAttachShader");
GLES20.glLinkProgram(program);
int[] linkStatus = new int[1];
GLES20.glGetProgramiv(program, GLES20.GL_LINK_STATUS, linkStatus, 0);
if (linkStatus[0] != GLES20.GL_TRUE) {
Log.e(TAG, "Could not link program: ");
Log.e(TAG, GLES20.glGetProgramInfoLog(program));
GLES20.glDeleteProgram(program);
program = 0;
}
return program;
}
static void checkGlError(String msg) throws TranscodingException {
boolean failed = false;
int error;
while ((error = GLES20.glGetError()) != GLES20.GL_NO_ERROR) {
Log.e(TAG, msg + ": GLES20 error: 0x" + Integer.toHexString(error));
failed = true;
}
if (failed) {
throw new TranscodingException("GLES20 error encountered (see log)");
}
}
}

12
app/src/main/java/org/thoughtcrime/securesms/video/videoconverter/TranscodingException.java
View File

@@ -1,12 +0,0 @@
package org.thoughtcrime.securesms.video.videoconverter;
final class TranscodingException extends Exception {
TranscodingException(String message) {
super(message);
}
TranscodingException(Throwable inner) {
super(inner);
}
}

12
app/witness-verifications.gradle
View File

@@ -423,6 +423,15 @@ dependencyVerification {
['org.jsoup:jsoup:1.8.3',
'abeaf34795a4de70f72aed6de5966d2955ec7eb348eeb813324f23c999575473'],
['org.mp4parser:isoparser:1.9.39',
'a3a7172648f1ac4b2a369ecca2861317e472179c842a5217b08643ba0a1dfa12'],
['org.mp4parser:muxer:1.9.39',
'4befe68d411cd889628b53bab211d395899a9ce893ae6766ec2f4fefec5b7835'],
['org.mp4parser:streaming:1.9.39',
'da5151cfc3bf491d550fb9127bba22736f4b7416058d58a1a5fcfdfa3673876d'],
['org.signal:aesgcmprovider:0.0.3',
'6eb4422e8a618b3b76cb2096a3619d251f9e27989dc68307a1e5414c3710f2d1'],
@@ -441,6 +450,9 @@ dependencyVerification {
['org.signal:zkgroup-java:0.7.0',
'd0099eedd60d6f7d4df5b288175e5d585228ed8897789926bdab69bf8c05659f'],
['org.slf4j:slf4j-api:1.7.24',
'baf3c7fe15fefeaf9e5b000d94547379dc48370f22a8797e239c127e7d7756ec'],
['org.threeten:threetenbp:1.3.6',
'f4c23ffaaed717c3b99c003e0ee02d6d66377fd47d866fec7d971bd8644fc1a7'],