TheArchive/session-android
1
0
Fork 0
mirror of https://github.com/oxen-io/session-android.git synced 2025-12-31 12:26:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

Show remaining time on wave form view and cache wave form in database.

Browse Source
This commit is contained in:
Alan Evans
2020-06-01 18:10:10 -03:00
committed by Greyson Parrelli
parent e01838e996
commit 3fec23fd36
33 changed files with 357 additions and 162 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
app/src/main/java/org/thoughtcrime/securesms/audio/AudioHash.java Normal file
View File

@@ -0,0 +1,57 @@
package org.thoughtcrime.securesms.audio;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import org.thoughtcrime.securesms.database.model.databaseprotos.AudioWaveFormData;
import org.whispersystems.util.Base64;
import java.io.IOException;
/**
* An AudioHash is a compact string representation of the wave form and duration for an audio file.
*/
public final class AudioHash {
@NonNull private final String hash;
@NonNull private final AudioWaveFormData audioWaveForm;
private AudioHash(@NonNull String hash, @NonNull AudioWaveFormData audioWaveForm) {
this.hash = hash;
this.audioWaveForm = audioWaveForm;
}
public AudioHash(@NonNull AudioWaveFormData audioWaveForm) {
this(Base64.encodeBytes(audioWaveForm.toByteArray()), audioWaveForm);
}
public static @Nullable AudioHash parseOrNull(@Nullable String hash) {
if (hash == null) return null;
try {
return new AudioHash(hash, AudioWaveFormData.parseFrom(Base64.decode(hash)));
} catch (IOException e) {
return null;
}
}
@NonNull AudioWaveFormData getAudioWaveForm() {
return audioWaveForm;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
AudioHash other = (AudioHash) o;
return hash.equals(other.hash);
}
@Override
public int hashCode() {
return hash.hashCode();
}
public @NonNull String getHash() {
return hash;
}
}

178
app/src/main/java/org/thoughtcrime/securesms/audio/AudioWaveForm.java
View File

@@ -14,11 +14,18 @@ import androidx.annotation.RequiresApi;
import androidx.annotation.WorkerThread;
import androidx.core.util.Consumer;
import com.google.protobuf.ByteString;
import org.thoughtcrime.securesms.attachments.Attachment;
import org.thoughtcrime.securesms.attachments.DatabaseAttachment;
import org.thoughtcrime.securesms.database.DatabaseFactory;
import org.thoughtcrime.securesms.database.model.databaseprotos.AudioWaveFormData;
import org.thoughtcrime.securesms.logging.Log;
import org.thoughtcrime.securesms.media.DecryptableUriMediaInput;
import org.thoughtcrime.securesms.media.MediaInput;
import org.thoughtcrime.securesms.mms.AudioSlide;
import org.thoughtcrime.securesms.util.Util;
import org.thoughtcrime.securesms.util.concurrent.SerialExecutor;
import org.thoughtcrime.securesms.util.concurrent.SignalExecutors;
import java.io.IOException;
@@ -32,7 +39,7 @@ public final class AudioWaveForm {
private static final String TAG = Log.tag(AudioWaveForm.class);
private static final int BARS = 46;
private static final int BAR_COUNT = 46;
private static final int SAMPLES_PER_BAR = 4;
private final Context context;
@@ -43,34 +50,68 @@ public final class AudioWaveForm {
this.slide = slide;
}
private static final LruCache<Uri, AudioFileInfo> WAVE_FORM_CACHE = new LruCache<>(200);
private static final Executor AUDIO_DECODER_EXECUTOR = SignalExecutors.BOUNDED;
private static final LruCache<String, AudioFileInfo> WAVE_FORM_CACHE = new LruCache<>(200);
private static final Executor AUDIO_DECODER_EXECUTOR = new SerialExecutor(SignalExecutors.BOUNDED);
@AnyThread
public void generateWaveForm(@NonNull Consumer<AudioFileInfo> onSuccess, @NonNull Consumer<IOException> onFailure) {
public void getWaveForm(@NonNull Consumer<AudioFileInfo> onSuccess, @NonNull Consumer<IOException> onFailure) {
Uri uri = slide.getUri();
Attachment attachment = slide.asAttachment();
if (uri == null) {
Log.w(TAG, "No uri");
Util.runOnMain(() -> onFailure.accept(null));
return;
}
if (!(attachment instanceof DatabaseAttachment)) {
Log.i(TAG, "Not yet in database");
Util.runOnMain(() -> onFailure.accept(null));
return;
}
String cacheKey = uri.toString();
AudioFileInfo cached = WAVE_FORM_CACHE.get(cacheKey);
if (cached != null) {
Log.i(TAG, "Loaded wave form from cache " + cacheKey);
Util.runOnMain(() -> onSuccess.accept(cached));
return;
}
AUDIO_DECODER_EXECUTOR.execute(() -> {
AudioFileInfo cachedInExecutor = WAVE_FORM_CACHE.get(cacheKey);
if (cachedInExecutor != null) {
Log.i(TAG, "Loaded wave form from cache inside executor" + cacheKey);
Util.runOnMain(() -> onSuccess.accept(cachedInExecutor));
return;
}
AudioHash audioHash = attachment.getAudioHash();
if (audioHash != null) {
AudioFileInfo audioFileInfo = AudioFileInfo.fromDatabaseProtobuf(audioHash.getAudioWaveForm());
if (audioFileInfo.waveForm.length != BAR_COUNT) {
Log.w(TAG, "Wave form from database does not match bar count, regenerating " + cacheKey);
} else {
WAVE_FORM_CACHE.put(cacheKey, audioFileInfo);
Log.i(TAG, "Loaded wave form from DB " + cacheKey);
Util.runOnMain(() -> onSuccess.accept(audioFileInfo));
return;
}
}
try {
long startTime = System.currentTimeMillis();
Uri uri = slide.getUri();
if (uri == null) {
Util.runOnMain(() -> onFailure.accept(null));
return;
}
DatabaseAttachment dbAttachment = (DatabaseAttachment) attachment;
long startTime = System.currentTimeMillis();
AudioFileInfo fileInfo = generateWaveForm(uri);
AudioFileInfo cached = WAVE_FORM_CACHE.get(uri);
if (cached != null) {
Util.runOnMain(() -> onSuccess.accept(cached));
return;
}
Log.i(TAG, String.format(Locale.US, "Audio wave form generation time %d ms (%s)", System.currentTimeMillis() - startTime, cacheKey));
AudioFileInfo fileInfo = generateWaveForm(uri);
WAVE_FORM_CACHE.put(uri, fileInfo);
Log.i(TAG, String.format(Locale.US, "Audio wave form generation time %d ms", System.currentTimeMillis() - startTime));
DatabaseFactory.getAttachmentDatabase(context).writeAudioHash(dbAttachment.getAttachmentId(), fileInfo.toDatabaseProtobuf());
WAVE_FORM_CACHE.put(cacheKey, fileInfo);
Util.runOnMain(() -> onSuccess.accept(fileInfo));
} catch (IOException e) {
Log.e(TAG, "", e);
Log.w(TAG, "Failed to create audio wave form for " + cacheKey, e);
onFailure.accept(e);
}
});
@@ -83,17 +124,36 @@ public final class AudioWaveForm {
*/
@WorkerThread
@RequiresApi(api = 23)
private AudioFileInfo generateWaveForm(@NonNull Uri uri) throws IOException {
private @NonNull AudioFileInfo generateWaveForm(@NonNull Uri uri) throws IOException {
try (MediaInput dataSource = DecryptableUriMediaInput.createForUri(context, uri)) {
long[] wave = new long[BARS];
int[] waveSamples = new int[BARS];
int[] inputSamples = new int[BARS * SAMPLES_PER_BAR];
long[] wave = new long[BAR_COUNT];
int[] waveSamples = new int[BAR_COUNT];
int[] inputSamples = new int[BAR_COUNT * SAMPLES_PER_BAR];
MediaExtractor extractor = dataSource.createExtractor();
MediaFormat format = extractor.getTrackFormat(0);
long totalDurationUs = format.getLong(MediaFormat.KEY_DURATION);
String mime = requireAudio(format.getString(MediaFormat.KEY_MIME));
MediaCodec codec = MediaCodec.createDecoderByType(mime);
MediaExtractor extractor = dataSource.createExtractor();
if (extractor.getTrackCount() == 0) {
throw new IOException("No audio track");
}
MediaFormat format = extractor.getTrackFormat(0);
if (!format.containsKey(MediaFormat.KEY_DURATION)) {
throw new IOException("Unknown duration");
}
long totalDurationUs = format.getLong(MediaFormat.KEY_DURATION);
String mime = format.getString(MediaFormat.KEY_MIME);
if (!mime.startsWith("audio/")) {
throw new IOException("Mime not audio");
}
MediaCodec codec = MediaCodec.createDecoderByType(mime);
if (totalDurationUs == 0) {
throw new IOException("Zero duration");
}
codec.configure(format, null, null, 0);
codec.start();
@@ -158,29 +218,24 @@ public final class AudioWaveForm {
}
ByteBuffer buf = codecOutputBuffers[outputBufferIndex];
int barIndex = (int) ((wave.length * info.presentationTimeUs) / totalDurationUs) - 1;
int barIndex = (int) ((wave.length * info.presentationTimeUs) / totalDurationUs);
long total = 0;
for (int i = 0; i < info.size; i += 2 * 4) {
short aShort = buf.getShort(i);
total += Math.abs(aShort);
}
if (barIndex > 0) {
if (barIndex >= 0 && barIndex < wave.length) {
wave[barIndex] += total;
waveSamples[barIndex] += info.size / 2;
}
codec.releaseOutputBuffer(outputBufferIndex, false);
if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
Log.d(TAG, "saw output EOS.");
sawOutputEOS = true;
}
} else if (outputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
codecOutputBuffers = codec.getOutputBuffers();
Log.d(TAG, "output buffers have changed.");
} else if (outputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
MediaFormat oformat = codec.getOutputFormat();
Log.d(TAG, "output format has changed to " + oformat);
} else {
Log.d(TAG, "dequeueOutputBuffer returned " + outputBufferIndex);
Log.d(TAG, "output format has changed to " + codec.getOutputFormat());
}
} while (outputBufferIndex >= 0);
}
@@ -189,36 +244,46 @@ public final class AudioWaveForm {
codec.release();
extractor.release();
float[] floats = new float[AudioWaveForm.BARS];
float max = 0;
for (int i = 0; i < AudioWaveForm.BARS; i++) {
float[] floats = new float[BAR_COUNT];
byte[] bytes = new byte[BAR_COUNT];
float max = 0;
for (int i = 0; i < BAR_COUNT; i++) {
if (waveSamples[i] == 0) continue;
floats[i] = wave[i] / (float) waveSamples[i];
if (floats[i] > max) {
max = floats[i];
}
}
for (int i = 0; i < AudioWaveForm.BARS; i++) {
floats[i] /= max;
for (int i = 0; i < BAR_COUNT; i++) {
float normalized = floats[i] / max;
bytes[i] = (byte) (255 * normalized);
}
return new AudioFileInfo(totalDurationUs, floats);
}
}
private static @NonNull String requireAudio(@NonNull String mime) {
if (!mime.startsWith("audio/")) {
throw new AssertionError();
return new AudioFileInfo(totalDurationUs, bytes);
}
return mime;
}
public static class AudioFileInfo {
private final long durationUs;
private final byte[] waveFormBytes;
private final float[] waveForm;
private AudioFileInfo(long durationUs, float[] waveForm) {
this.durationUs = durationUs;
this.waveForm = waveForm;
private static @NonNull AudioFileInfo fromDatabaseProtobuf(@NonNull AudioWaveFormData audioWaveForm) {
return new AudioFileInfo(audioWaveForm.getDurationUs(), audioWaveForm.getWaveForm().toByteArray());
}
private AudioFileInfo(long durationUs, byte[] waveFormBytes) {
this.durationUs = durationUs;
this.waveFormBytes = waveFormBytes;
this.waveForm = new float[waveFormBytes.length];
for (int i = 0; i < waveFormBytes.length; i++) {
int unsigned = waveFormBytes[i] & 0xff;
this.waveForm[i] = unsigned / 255f;
}
}
public long getDuration(@NonNull TimeUnit timeUnit) {
@@ -228,5 +293,12 @@ public final class AudioWaveForm {
public float[] getWaveForm() {
return waveForm;
}
private @NonNull AudioWaveFormData toDatabaseProtobuf() {
return AudioWaveFormData.newBuilder()
.setDurationUs(durationUs)
.setWaveForm(ByteString.copyFrom(waveFormBytes))
.build();
}
}
}