From daace9bd1a4770ba6aff03975ddc639a1fbf958d Mon Sep 17 00:00:00 2001 From: Alan Evans Date: Thu, 28 May 2020 16:50:27 -0300 Subject: [PATCH] Audio wave forms on voice notes. --- .../securesms/audio/AudioWaveForm.java | 232 ++++++++++++++++++ .../securesms/components/AudioView.java | 36 +++ .../components/WaveFormSeekBarView.java | 155 ++++++++++++ app/src/main/res/layout/audio_view.xml | 25 +- .../res/layout/conversation_item_received.xml | 4 +- .../conversation_item_received_audio.xml | 4 +- .../res/layout/conversation_item_sent.xml | 4 +- .../layout/conversation_item_sent_audio.xml | 4 +- app/src/main/res/values/attrs.xml | 2 + app/src/main/res/values/dimens.xml | 4 + app/src/main/res/values/strings.xml | 3 + 11 files changed, 464 insertions(+), 9 deletions(-) create mode 100644 app/src/main/java/org/thoughtcrime/securesms/audio/AudioWaveForm.java create mode 100644 app/src/main/java/org/thoughtcrime/securesms/components/WaveFormSeekBarView.java diff --git a/app/src/main/java/org/thoughtcrime/securesms/audio/AudioWaveForm.java b/app/src/main/java/org/thoughtcrime/securesms/audio/AudioWaveForm.java new file mode 100644 index 0000000000..65bb9b9a3d --- /dev/null +++ b/app/src/main/java/org/thoughtcrime/securesms/audio/AudioWaveForm.java @@ -0,0 +1,232 @@ +package org.thoughtcrime.securesms.audio; + +import android.content.Context; +import android.media.MediaCodec; +import android.media.MediaExtractor; +import android.media.MediaFormat; +import android.net.Uri; +import android.os.Build; +import android.util.LruCache; + +import androidx.annotation.AnyThread; +import androidx.annotation.NonNull; +import androidx.annotation.RequiresApi; +import androidx.annotation.WorkerThread; +import androidx.core.util.Consumer; + +import org.thoughtcrime.securesms.logging.Log; +import org.thoughtcrime.securesms.media.DecryptableUriMediaInput; +import org.thoughtcrime.securesms.media.MediaInput; +import org.thoughtcrime.securesms.mms.AudioSlide; +import org.thoughtcrime.securesms.util.Util; +import org.thoughtcrime.securesms.util.concurrent.SignalExecutors; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Locale; +import java.util.concurrent.Executor; +import java.util.concurrent.TimeUnit; + +@RequiresApi(api = Build.VERSION_CODES.M) +public final class AudioWaveForm { + + private static final String TAG = Log.tag(AudioWaveForm.class); + + private static final int BARS = 46; + private static final int SAMPLES_PER_BAR = 4; + + private final Context context; + private final AudioSlide slide; + + public AudioWaveForm(@NonNull Context context, @NonNull AudioSlide slide) { + this.context = context.getApplicationContext(); + this.slide = slide; + } + + private static final LruCache WAVE_FORM_CACHE = new LruCache<>(200); + private static final Executor AUDIO_DECODER_EXECUTOR = SignalExecutors.BOUNDED; + + @AnyThread + public void generateWaveForm(@NonNull Consumer onSuccess, @NonNull Consumer onFailure) { + AUDIO_DECODER_EXECUTOR.execute(() -> { + try { + long startTime = System.currentTimeMillis(); + Uri uri = slide.getUri(); + if (uri == null) { + Util.runOnMain(() -> onFailure.accept(null)); + return; + } + + AudioFileInfo cached = WAVE_FORM_CACHE.get(uri); + if (cached != null) { + Util.runOnMain(() -> onSuccess.accept(cached)); + return; + } + + AudioFileInfo fileInfo = generateWaveForm(uri); + WAVE_FORM_CACHE.put(uri, fileInfo); + + Log.i(TAG, String.format(Locale.US, "Audio wave form generation time %d ms", System.currentTimeMillis() - startTime)); + + Util.runOnMain(() -> onSuccess.accept(fileInfo)); + } catch (IOException e) { + Log.e(TAG, "", e); + onFailure.accept(e); + } + }); + } + + /** + * Based on decode sample from: + *

+ * https://android.googlesource.com/platform/cts/+/jb-mr2-release/tests/tests/media/src/android/media/cts/DecoderTest.java + */ + @WorkerThread + @RequiresApi(api = 23) + private AudioFileInfo generateWaveForm(@NonNull Uri uri) throws IOException { + try (MediaInput dataSource = DecryptableUriMediaInput.createForUri(context, uri)) { + long[] wave = new long[BARS]; + int[] waveSamples = new int[BARS]; + int[] inputSamples = new int[BARS * SAMPLES_PER_BAR]; + + MediaExtractor extractor = dataSource.createExtractor(); + MediaFormat format = extractor.getTrackFormat(0); + long totalDurationUs = format.getLong(MediaFormat.KEY_DURATION); + String mime = requireAudio(format.getString(MediaFormat.KEY_MIME)); + MediaCodec codec = MediaCodec.createDecoderByType(mime); + + codec.configure(format, null, null, 0); + codec.start(); + + ByteBuffer[] codecInputBuffers = codec.getInputBuffers(); + ByteBuffer[] codecOutputBuffers = codec.getOutputBuffers(); + + extractor.selectTrack(0); + + long kTimeOutUs = 5000; + MediaCodec.BufferInfo info = new MediaCodec.BufferInfo(); + boolean sawInputEOS = false; + boolean sawOutputEOS = false; + int noOutputCounter = 0; + + while (!sawOutputEOS && noOutputCounter < 50) { + noOutputCounter++; + if (!sawInputEOS) { + int inputBufIndex = codec.dequeueInputBuffer(kTimeOutUs); + if (inputBufIndex >= 0) { + ByteBuffer dstBuf = codecInputBuffers[inputBufIndex]; + int sampleSize = extractor.readSampleData(dstBuf, 0); + long presentationTimeUs = 0; + + if (sampleSize < 0) { + sawInputEOS = true; + sampleSize = 0; + } else { + presentationTimeUs = extractor.getSampleTime(); + } + + codec.queueInputBuffer( + inputBufIndex, + 0, + sampleSize, + presentationTimeUs, + sawInputEOS ? MediaCodec.BUFFER_FLAG_END_OF_STREAM : 0); + + if (!sawInputEOS) { + int barSampleIndex = (int) (SAMPLES_PER_BAR * (wave.length * extractor.getSampleTime()) / totalDurationUs); + inputSamples[barSampleIndex]++; + sawInputEOS = !extractor.advance(); + if (inputSamples[barSampleIndex] > 0) { + int nextBarSampleIndex = (int) (SAMPLES_PER_BAR * (wave.length * extractor.getSampleTime()) / totalDurationUs); + while (!sawInputEOS && nextBarSampleIndex == barSampleIndex) { + sawInputEOS = !extractor.advance(); + if (!sawInputEOS) { + nextBarSampleIndex = (int) (SAMPLES_PER_BAR * (wave.length * extractor.getSampleTime()) / totalDurationUs); + } + } + } + } + } + } + + int outputBufferIndex; + do { + outputBufferIndex = codec.dequeueOutputBuffer(info, kTimeOutUs); + if (outputBufferIndex >= 0) { + if (info.size > 0) { + noOutputCounter = 0; + } + + ByteBuffer buf = codecOutputBuffers[outputBufferIndex]; + int barIndex = (int) ((wave.length * info.presentationTimeUs) / totalDurationUs) - 1; + long total = 0; + for (int i = 0; i < info.size; i += 2 * 4) { + short aShort = buf.getShort(i); + total += Math.abs(aShort); + } + if (barIndex > 0) { + wave[barIndex] += total; + waveSamples[barIndex] += info.size / 2; + } + codec.releaseOutputBuffer(outputBufferIndex, false); + if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { + Log.d(TAG, "saw output EOS."); + sawOutputEOS = true; + } + } else if (outputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { + codecOutputBuffers = codec.getOutputBuffers(); + Log.d(TAG, "output buffers have changed."); + } else if (outputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { + MediaFormat oformat = codec.getOutputFormat(); + Log.d(TAG, "output format has changed to " + oformat); + } else { + Log.d(TAG, "dequeueOutputBuffer returned " + outputBufferIndex); + } + } while (outputBufferIndex >= 0); + } + + codec.stop(); + codec.release(); + extractor.release(); + + float[] floats = new float[AudioWaveForm.BARS]; + float max = 0; + for (int i = 0; i < AudioWaveForm.BARS; i++) { + floats[i] = wave[i] / (float) waveSamples[i]; + if (floats[i] > max) { + max = floats[i]; + } + } + for (int i = 0; i < AudioWaveForm.BARS; i++) { + floats[i] /= max; + } + return new AudioFileInfo(totalDurationUs, floats); + } + } + + private static @NonNull String requireAudio(@NonNull String mime) { + if (!mime.startsWith("audio/")) { + throw new AssertionError(); + } + + return mime; + } + + public static class AudioFileInfo { + private final long durationUs; + private final float[] waveForm; + + private AudioFileInfo(long durationUs, float[] waveForm) { + this.durationUs = durationUs; + this.waveForm = waveForm; + } + + public long getDuration(@NonNull TimeUnit timeUnit) { + return timeUnit.convert(durationUs, TimeUnit.MICROSECONDS); + } + + public float[] getWaveForm() { + return waveForm; + } + } +} diff --git a/app/src/main/java/org/thoughtcrime/securesms/components/AudioView.java b/app/src/main/java/org/thoughtcrime/securesms/components/AudioView.java index c494a71215..44ae1fe3cb 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/components/AudioView.java +++ b/app/src/main/java/org/thoughtcrime/securesms/components/AudioView.java @@ -14,6 +14,7 @@ import android.widget.ImageView; import android.widget.SeekBar; import android.widget.TextView; +import androidx.annotation.ColorInt; import androidx.annotation.NonNull; import androidx.annotation.Nullable; @@ -29,6 +30,7 @@ import org.greenrobot.eventbus.Subscribe; import org.greenrobot.eventbus.ThreadMode; import org.thoughtcrime.securesms.R; import org.thoughtcrime.securesms.audio.AudioSlidePlayer; +import org.thoughtcrime.securesms.audio.AudioWaveForm; import org.thoughtcrime.securesms.database.AttachmentDatabase; import org.thoughtcrime.securesms.events.PartProgressEvent; import org.thoughtcrime.securesms.logging.Log; @@ -57,6 +59,10 @@ public final class AudioView extends FrameLayout implements AudioSlidePlayer.Lis private final boolean autoRewind; @Nullable private final TextView timestamp; + @Nullable private final TextView duration; + + @ColorInt private final int waveFormPlayedBarsColor; + @ColorInt private final int waveFormUnplayedBarsColor; @Nullable private SlideClickListener downloadListener; @Nullable private AudioSlidePlayer audioSlidePlayer; @@ -91,6 +97,7 @@ public final class AudioView extends FrameLayout implements AudioSlidePlayer.Lis this.circleProgress = findViewById(R.id.circle_progress); this.seekBar = findViewById(R.id.seek); this.timestamp = findViewById(R.id.timestamp); + this.duration = findViewById(R.id.duration); lottieDirection = REVERSE; this.playPauseButton.setOnClickListener(new PlayPauseClickedListener()); @@ -98,6 +105,10 @@ public final class AudioView extends FrameLayout implements AudioSlidePlayer.Lis setTint(typedArray.getColor(R.styleable.AudioView_foregroundTintColor, Color.WHITE), typedArray.getColor(R.styleable.AudioView_backgroundTintColor, Color.WHITE)); + + this.waveFormPlayedBarsColor = typedArray.getColor(R.styleable.AudioView_waveformPlayedBarsColor, Color.WHITE); + this.waveFormUnplayedBarsColor = typedArray.getColor(R.styleable.AudioView_waveformUnplayedBarsColor, Color.WHITE); + container.setBackgroundColor(typedArray.getColor(R.styleable.AudioView_widgetBackground, Color.TRANSPARENT)); } finally { if (typedArray != null) { @@ -141,6 +152,28 @@ public final class AudioView extends FrameLayout implements AudioSlidePlayer.Lis } this.audioSlidePlayer = AudioSlidePlayer.createFor(getContext(), audio, this); + + if (seekBar instanceof WaveFormSeekBarView) { + WaveFormSeekBarView waveFormView = (WaveFormSeekBarView) seekBar; + waveFormView.setColors(waveFormPlayedBarsColor, waveFormUnplayedBarsColor); + if (android.os.Build.VERSION.SDK_INT >= 23) { + new AudioWaveForm(getContext(), audio).generateWaveForm( + data -> { + waveFormView.setWaveData(data.getWaveForm()); + if (duration != null) { + long durationSecs = data.getDuration(TimeUnit.SECONDS); + duration.setText(getContext().getResources().getString(R.string.AudioView_duration, durationSecs / 60, durationSecs % 60)); + duration.setVisibility(VISIBLE); + } + }, + e -> waveFormView.setWaveMode(false)); + } else { + waveFormView.setWaveMode(false); + if (duration != null) { + duration.setVisibility(GONE); + } + } + } } public void cleanup() { @@ -232,6 +265,9 @@ public final class AudioView extends FrameLayout implements AudioSlidePlayer.Lis if (this.timestamp != null) { this.timestamp.setTextColor(foregroundTint); } + if (this.duration != null) { + this.duration.setTextColor(foregroundTint); + } this.seekBar.getProgressDrawable().setColorFilter(foregroundTint, PorterDuff.Mode.SRC_IN); this.seekBar.getThumb().setColorFilter(foregroundTint, PorterDuff.Mode.SRC_IN); } diff --git a/app/src/main/java/org/thoughtcrime/securesms/components/WaveFormSeekBarView.java b/app/src/main/java/org/thoughtcrime/securesms/components/WaveFormSeekBarView.java new file mode 100644 index 0000000000..46f40505ec --- /dev/null +++ b/app/src/main/java/org/thoughtcrime/securesms/components/WaveFormSeekBarView.java @@ -0,0 +1,155 @@ +package org.thoughtcrime.securesms.components; + +import android.content.Context; +import android.graphics.Canvas; +import android.graphics.Paint; +import android.graphics.drawable.Drawable; +import android.util.AttributeSet; +import android.view.animation.Interpolator; +import android.view.animation.OvershootInterpolator; + +import androidx.annotation.ColorInt; +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; +import androidx.annotation.Px; +import androidx.appcompat.widget.AppCompatSeekBar; + +import org.thoughtcrime.securesms.R; + +import java.util.Arrays; + +public final class WaveFormSeekBarView extends AppCompatSeekBar { + + private static final int ANIM_DURATION = 450; + private static final int ANIM_BAR_OFF_SET_DURATION = 12; + + private final Interpolator overshoot = new OvershootInterpolator(); + private final Paint paint = new Paint(); + private float[] data = new float[0]; + private long dataSetTime; + private Drawable progressDrawable; + private boolean waveMode; + + @ColorInt private int playedBarColor = 0xffffffff; + @ColorInt private int unplayedBarColor = 0x7fffffff; + @Px private int barWidth; + + public WaveFormSeekBarView(Context context) { + super(context); + init(); + } + + public WaveFormSeekBarView(Context context, @Nullable AttributeSet attrs) { + super(context, attrs); + init(); + } + + public WaveFormSeekBarView(Context context, @Nullable AttributeSet attrs, int defStyleAttr) { + super(context, attrs, defStyleAttr); + init(); + } + + private void init() { + setWillNotDraw(false); + + paint.setStrokeCap(Paint.Cap.ROUND); + paint.setAntiAlias(true); + + progressDrawable = super.getProgressDrawable(); + + if (isInEditMode()) { + setWaveData(sinusoidalExampleData()); + dataSetTime = 0; + } + + barWidth = getResources().getDimensionPixelSize(R.dimen.wave_form_bar_width); + } + + public void setColors(@ColorInt int playedBarColor, @ColorInt int unplayedBarColor) { + this.playedBarColor = playedBarColor; + this.unplayedBarColor = unplayedBarColor; + invalidate(); + } + + @Override + public void setProgressDrawable(Drawable progressDrawable) { + this.progressDrawable = progressDrawable; + if (!waveMode) { + super.setProgressDrawable(progressDrawable); + } + } + + @Override + public Drawable getProgressDrawable() { + return progressDrawable; + } + + public void setWaveData(@NonNull float[] data) { + if (!Arrays.equals(data, this.data)) { + this.data = data; + this.dataSetTime = System.currentTimeMillis(); + setWaveMode(data.length > 0); + invalidate(); + } + } + + public void setWaveMode(boolean waveMode) { + this.waveMode = waveMode; + super.setProgressDrawable(this.waveMode ? null : progressDrawable); + invalidate(); + } + + @Override + protected void onDraw(Canvas canvas) { + drawWave(canvas); + super.onDraw(canvas); + } + + private void drawWave(Canvas canvas) { + paint.setStrokeWidth(barWidth); + + int usableHeight = getHeight() - getPaddingTop() - getPaddingBottom(); + int usableWidth = getWidth() - getPaddingLeft() - getPaddingRight(); + float midpoint = usableHeight / 2f; + float maxHeight = usableHeight / 2f - barWidth; + float barGap = (usableWidth - data.length * barWidth) / (float) (data.length - 1); + + boolean hasMoreFrames = false; + + canvas.save(); + canvas.translate(getPaddingLeft(), getPaddingTop()); + + for (int bar = 0; bar < data.length; bar++) { + float x = bar * (barWidth + barGap) + barWidth / 2f; + float y = data[bar] * maxHeight; + float progress = x / usableWidth; + + paint.setColor(progress * getMax() < getProgress() ? playedBarColor : unplayedBarColor); + + long time = System.currentTimeMillis() - bar * ANIM_BAR_OFF_SET_DURATION - dataSetTime; + float timeX = Math.max(0, Math.min(1, time / (float) ANIM_DURATION)); + float interpolatedTime = overshoot.getInterpolation(timeX); + float interpolatedY = y * interpolatedTime; + + canvas.drawLine(x, midpoint - interpolatedY, x, midpoint + interpolatedY, paint); + + if (time < ANIM_DURATION) { + hasMoreFrames = true; + } + } + + canvas.restore(); + + if (hasMoreFrames) { + invalidate(); + } + } + + private static float[] sinusoidalExampleData() { + float[] data = new float[21]; + for (int i = 0; i < data.length; i++) { + data[i] = (float) Math.sin(i / (float) (data.length - 1) * 2 * Math.PI); + } + return data; + } +} diff --git a/app/src/main/res/layout/audio_view.xml b/app/src/main/res/layout/audio_view.xml index 907d95c319..69f7138426 100644 --- a/app/src/main/res/layout/audio_view.xml +++ b/app/src/main/res/layout/audio_view.xml @@ -17,11 +17,30 @@ - + + + android:layout_gravity="center_vertical" + android:fontFamily="sans-serif-light" + android:textAppearance="?android:attr/textAppearanceMedium" + android:textColor="?conversation_item_sent_text_secondary_color" + android:textSize="@dimen/conversation_item_date_text_size" + android:visibility="gone" + tools:text="00:30" + tools:visibility="visible" /> diff --git a/app/src/main/res/layout/conversation_item_received.xml b/app/src/main/res/layout/conversation_item_received.xml index 46fe18a0e5..8231ea930b 100644 --- a/app/src/main/res/layout/conversation_item_received.xml +++ b/app/src/main/res/layout/conversation_item_received.xml @@ -149,9 +149,9 @@ diff --git a/app/src/main/res/layout/conversation_item_received_audio.xml b/app/src/main/res/layout/conversation_item_received_audio.xml index 10c2eb0db4..df968566bb 100644 --- a/app/src/main/res/layout/conversation_item_received_audio.xml +++ b/app/src/main/res/layout/conversation_item_received_audio.xml @@ -4,9 +4,11 @@ xmlns:app="http://schemas.android.com/apk/res-auto" xmlns:tools="http://schemas.android.com/tools" android:id="@+id/audio_view" - android:layout_width="210dp" + android:layout_width="@dimen/message_audio_width" android:layout_height="wrap_content" android:visibility="gone" app:foregroundTintColor="@color/white" app:backgroundTintColor="@color/blue_500" + app:waveformPlayedBarsColor="@color/core_white" + app:waveformUnplayedBarsColor="@color/transparent_white_40" tools:visibility="visible"/> diff --git a/app/src/main/res/layout/conversation_item_sent.xml b/app/src/main/res/layout/conversation_item_sent.xml index 3ddfd3af12..0725815006 100644 --- a/app/src/main/res/layout/conversation_item_sent.xml +++ b/app/src/main/res/layout/conversation_item_sent.xml @@ -86,10 +86,10 @@ diff --git a/app/src/main/res/layout/conversation_item_sent_audio.xml b/app/src/main/res/layout/conversation_item_sent_audio.xml index d62fb22f7e..8436056ba8 100644 --- a/app/src/main/res/layout/conversation_item_sent_audio.xml +++ b/app/src/main/res/layout/conversation_item_sent_audio.xml @@ -3,8 +3,10 @@ xmlns:android="http://schemas.android.com/apk/res/android" xmlns:app="http://schemas.android.com/apk/res-auto" android:id="@+id/audio_view" - android:layout_width="210dp" + android:layout_width="@dimen/message_audio_width" android:layout_height="wrap_content" app:foregroundTintColor="@color/grey_500" app:backgroundTintColor="@color/white" + app:waveformPlayedBarsColor="@color/core_ultramarine_light" + app:waveformUnplayedBarsColor="@color/core_grey_25" android:visibility="gone"/> diff --git a/app/src/main/res/values/attrs.xml b/app/src/main/res/values/attrs.xml index d06300613e..f03266ef02 100644 --- a/app/src/main/res/values/attrs.xml +++ b/app/src/main/res/values/attrs.xml @@ -339,6 +339,8 @@ + + diff --git a/app/src/main/res/values/dimens.xml b/app/src/main/res/values/dimens.xml index 0592d9e044..2a0929b790 100644 --- a/app/src/main/res/values/dimens.xml +++ b/app/src/main/res/values/dimens.xml @@ -34,6 +34,7 @@ 1.5dp 12dp 6dp + 12dp 6dp 32dp 8dp @@ -45,6 +46,7 @@ 100dp 320dp 175dp + 260dp -37dp 32dp @@ -162,4 +164,6 @@ 12dp 52dp 16dp + + 2dp diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index b2bb10591b..bf226317e2 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -1166,6 +1166,9 @@ %1$02d:%2$02d + + %1$02d:%2$02d + Bad encrypted message Message encrypted for non-existing session