TheArchive/session-android
1
0
Fork 0
mirror of https://github.com/oxen-io/session-android.git synced 2024-11-27 20:15:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

Manually encode voice messages to AAC with ADTS headers

Browse Source 
Should resolve issues with platforms that don't support AMR (!).

Fixes #4640
Fixes #4652
Fixes #4647
// FREEBIE
This commit is contained in:
Moxie Marlinspike 2015-11-22 10:44:44 -08:00
parent a0beb7f0e0
commit fc5777e904
7 changed files with 309 additions and 161 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
src/org/thoughtcrime/securesms/ConversationActivity.java
View File

@ -59,6 +59,7 @@ import com.google.protobuf.ByteString;
import org.thoughtcrime.redphone.RedPhone;
import org.thoughtcrime.redphone.RedPhoneService;
import org.thoughtcrime.securesms.TransportOptions.OnTransportChangedListener;
import org.thoughtcrime.securesms.audio.AudioCodec;
import org.thoughtcrime.securesms.audio.AudioRecorder;
import org.thoughtcrime.securesms.audio.AudioSlidePlayer;
import org.thoughtcrime.securesms.color.MaterialColor;
@ -138,6 +139,8 @@ import java.security.SecureRandom;
import java.util.List;
import java.util.concurrent.ExecutionException;
import ws.com.google.android.mms.ContentType;
import static org.thoughtcrime.securesms.TransportOption.Type;
import static org.thoughtcrime.securesms.database.GroupDatabase.GroupRecord;
import static org.whispersystems.textsecure.internal.push.TextSecureProtos.GroupContext;
@ -1372,14 +1375,10 @@ public class ConversationActivity extends PassphraseRequiredActionBarActivity
@Override
public void onRecorderStarted() {
try {
Vibrator vibrator = (Vibrator)getSystemService(Context.VIBRATOR_SERVICE);
vibrator.vibrate(20);
audioRecorder.startRecording();
} catch (IOException e) {
Log.w(TAG, e);
}
}
@Override
@ -1393,7 +1392,7 @@ public class ConversationActivity extends PassphraseRequiredActionBarActivity
public void onSuccess(final @NonNull Pair<Uri, Long> result) {
try {
boolean forceSms = sendButton.isManualSelection() && sendButton.getSelectedTransport().isSms();
AudioSlide audioSlide = new AudioSlide(ConversationActivity.this, result.first, result.second);
AudioSlide audioSlide = new AudioSlide(ConversationActivity.this, result.first, result.second, ContentType.AUDIO_AAC);
SlideDeck slideDeck = new SlideDeck();
slideDeck.addSlide(audioSlide);
@ -1409,7 +1408,7 @@ public class ConversationActivity extends PassphraseRequiredActionBarActivity
}.execute();
}
});
} catch (IOException | InvalidMessageException e) {
} catch (InvalidMessageException e) {
Log.w(TAG, e);
Toast.makeText(ConversationActivity.this, R.string.ConversationActivity_error_sending_voice_message, Toast.LENGTH_LONG).show();
}

198
src/org/thoughtcrime/securesms/audio/AudioCodec.java Normal file
View File

@ -0,0 +1,198 @@
package org.thoughtcrime.securesms.audio;
import android.annotation.TargetApi;
import android.media.AudioFormat;
import android.media.AudioRecord;
import android.media.MediaCodec;
import android.media.MediaCodecInfo;
import android.media.MediaFormat;
import android.media.MediaRecorder;
import android.os.Build;
import android.util.Log;
import org.thoughtcrime.securesms.util.Util;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.ByteBuffer;
@TargetApi(Build.VERSION_CODES.JELLY_BEAN)
public class AudioCodec {
private static final String TAG = AudioCodec.class.getSimpleName();
private static final int SAMPLE_RATE = 44100;
private static final int SAMPLE_RATE_INDEX = 4;
private static final int CHANNELS = 1;
private static final int BIT_RATE = 32000;
private final int bufferSize;
private final MediaCodec mediaCodec;
private final AudioRecord audioRecord;
private boolean running = true;
private boolean finished = false;
public AudioCodec() throws IOException {
this.bufferSize = AudioRecord.getMinBufferSize(SAMPLE_RATE, AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT);
this.audioRecord = createAudioRecord(this.bufferSize);
this.mediaCodec = createMediaCodec(this.bufferSize);
this.mediaCodec.start();
try {
audioRecord.startRecording();
} catch (Exception e) {
Log.w(TAG, e);
mediaCodec.release();
throw new IOException(e);
}
}
public synchronized void stop() {
running = false;
while (!finished) Util.wait(this, 0);
}
public void start(final OutputStream outputStream) {
new Thread(new Runnable() {
@Override
public void run() {
MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo();
byte[] audioRecordData = new byte[bufferSize];
ByteBuffer[] codecInputBuffers = mediaCodec.getInputBuffers();
ByteBuffer[] codecOutputBuffers = mediaCodec.getOutputBuffers();
try {
while (true) {
boolean running = isRunning();
handleCodecInput(audioRecord, audioRecordData, mediaCodec, codecInputBuffers, running);
handleCodecOutput(mediaCodec, codecOutputBuffers, bufferInfo, outputStream);
if (!running) break;
}
} catch (IOException e) {
Log.w(TAG, e);
} finally {
mediaCodec.stop();
audioRecord.stop();
mediaCodec.release();
audioRecord.release();
Util.close(outputStream);
setFinished();
}
}
}, AudioCodec.class.getSimpleName()).start();
}
private synchronized boolean isRunning() {
return running;
}
private synchronized void setFinished() {
finished = true;
notifyAll();
}
private void handleCodecInput(AudioRecord audioRecord, byte[] audioRecordData,
MediaCodec mediaCodec, ByteBuffer[] codecInputBuffers,
boolean running)
{
int length = audioRecord.read(audioRecordData, 0, audioRecordData.length);
int codecInputBufferIndex = mediaCodec.dequeueInputBuffer(10 * 1000);
if (codecInputBufferIndex >= 0) {
ByteBuffer codecBuffer = codecInputBuffers[codecInputBufferIndex];
codecBuffer.clear();
codecBuffer.put(audioRecordData);
mediaCodec.queueInputBuffer(codecInputBufferIndex, 0, length, 0, running ? 0 : MediaCodec.BUFFER_FLAG_END_OF_STREAM);
}
}
private void handleCodecOutput(MediaCodec mediaCodec,
ByteBuffer[] codecOutputBuffers,
MediaCodec.BufferInfo bufferInfo,
OutputStream outputStream)
throws IOException
{
int codecOutputBufferIndex = mediaCodec.dequeueOutputBuffer(bufferInfo, 0);
while (codecOutputBufferIndex != MediaCodec.INFO_TRY_AGAIN_LATER) {
if (codecOutputBufferIndex >= 0) {
ByteBuffer encoderOutputBuffer = codecOutputBuffers[codecOutputBufferIndex];
encoderOutputBuffer.position(bufferInfo.offset);
encoderOutputBuffer.limit(bufferInfo.offset + bufferInfo.size);
if ((bufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != MediaCodec.BUFFER_FLAG_CODEC_CONFIG) {
byte[] header = createAdtsHeader(bufferInfo.size - bufferInfo.offset);
outputStream.write(header);
byte[] data = new byte[encoderOutputBuffer.remaining()];
encoderOutputBuffer.get(data);
outputStream.write(data);
}
encoderOutputBuffer.clear();
mediaCodec.releaseOutputBuffer(codecOutputBufferIndex, false);
} else if (codecOutputBufferIndex== MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
codecOutputBuffers = mediaCodec.getOutputBuffers();
}
codecOutputBufferIndex = mediaCodec.dequeueOutputBuffer(bufferInfo, 0);
}
}
private byte[] createAdtsHeader(int length) {
int frameLength = length + 7;
byte[] adtsHeader = new byte[7];
adtsHeader[0] = (byte) 0xFF; // Sync Word
adtsHeader[1] = (byte) 0xF1; // MPEG-4, Layer (0), No CRC
adtsHeader[2] = (byte) ((MediaCodecInfo.CodecProfileLevel.AACObjectLC - 1) << 6);
adtsHeader[2] |= (((byte) SAMPLE_RATE_INDEX) << 2);
adtsHeader[2] |= (((byte) CHANNELS) >> 2);
adtsHeader[3] = (byte) (((CHANNELS & 3) << 6) | ((frameLength >> 11) & 0x03));
adtsHeader[4] = (byte) ((frameLength >> 3) & 0xFF);
adtsHeader[5] = (byte) (((frameLength & 0x07) << 5) | 0x1f);
adtsHeader[6] = (byte) 0xFC;
return adtsHeader;
}
private AudioRecord createAudioRecord(int bufferSize) {
return new AudioRecord(MediaRecorder.AudioSource.MIC, SAMPLE_RATE,
AudioFormat.CHANNEL_IN_MONO,
AudioFormat.ENCODING_PCM_16BIT, bufferSize * 10);
}
private MediaCodec createMediaCodec(int bufferSize) throws IOException {
MediaCodec mediaCodec = MediaCodec.createEncoderByType("audio/mp4a-latm");
MediaFormat mediaFormat = new MediaFormat();
mediaFormat.setString(MediaFormat.KEY_MIME, "audio/mp4a-latm");
mediaFormat.setInteger(MediaFormat.KEY_SAMPLE_RATE, SAMPLE_RATE);
mediaFormat.setInteger(MediaFormat.KEY_CHANNEL_COUNT, CHANNELS);
mediaFormat.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, bufferSize);
mediaFormat.setInteger(MediaFormat.KEY_BIT_RATE, BIT_RATE);
mediaFormat.setInteger(MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC);
try {
mediaCodec.configure(mediaFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
} catch (Exception e) {
Log.w(TAG, e);
mediaCodec.release();
throw new IOException(e);
}
return mediaCodec;
}
}

177
src/org/thoughtcrime/securesms/audio/AudioRecorder.java
View File

@ -1,36 +1,38 @@
package org.thoughtcrime.securesms.audio;
import android.annotation.TargetApi;
import android.content.Context;
import android.media.MediaRecorder;
import android.net.Uri;
import android.os.Build;
import android.os.ParcelFileDescriptor;
import android.support.annotation.NonNull;
import android.util.Log;
import android.util.Pair;
import org.thoughtcrime.securesms.ApplicationContext;
import org.thoughtcrime.securesms.crypto.MasterSecret;
import org.thoughtcrime.securesms.providers.PersistentBlobProvider;
import org.thoughtcrime.securesms.util.MediaUtil;
import org.thoughtcrime.securesms.util.ThreadUtil;
import org.thoughtcrime.securesms.util.Util;
import org.thoughtcrime.securesms.util.concurrent.ListenableFuture;
import org.thoughtcrime.securesms.util.concurrent.SettableFuture;
import org.whispersystems.jobqueue.Job;
import org.whispersystems.jobqueue.JobParameters;
import java.io.IOException;
import java.util.concurrent.ExecutorService;
@TargetApi(Build.VERSION_CODES.JELLY_BEAN)
public class AudioRecorder {
private static final String TAG = AudioRecorder.class.getSimpleName();
private static final ExecutorService executor = ThreadUtil.newDynamicSingleThreadedExecutor();
private final Context context;
private final MasterSecret masterSecret;
private final PersistentBlobProvider blobProvider;
private MediaRecorder mediaRecorder;
private AudioCodec audioCodec;
private Uri captureUri;
private ParcelFileDescriptor fd;
public AudioRecorder(@NonNull Context context, @NonNull MasterSecret masterSecret) {
this.context = context;
@ -38,95 +40,63 @@ public class AudioRecorder {
this.blobProvider = PersistentBlobProvider.getInstance(context.getApplicationContext());
}
public void startRecording() throws IOException {
public void startRecording() {
Log.w(TAG, "startRecording()");
ApplicationContext.getInstance(context)
.getJobManager()
.add(new StartRecordingJob());
executor.execute(new Runnable() {
@Override
public void run() {
Log.w(TAG, "Running startRecording() + " + Thread.currentThread().getId());
try {
if (audioCodec != null) {
throw new AssertionError("We can only record once at a time.");
}
ParcelFileDescriptor fds[] = ParcelFileDescriptor.createPipe();
captureUri = blobProvider.create(masterSecret, new ParcelFileDescriptor.AutoCloseInputStream(fds[0]));
audioCodec = new AudioCodec();
audioCodec.start(new ParcelFileDescriptor.AutoCloseOutputStream(fds[1]));
} catch (IOException e) {
Log.w(TAG, e);
}
}
});
}
public @NonNull ListenableFuture<Pair<Uri, Long>> stopRecording() {
Log.w(TAG, "stopRecording()");
StopRecordingJob stopRecordingJob = new StopRecordingJob();
ApplicationContext.getInstance(context)
.getJobManager()
.add(stopRecordingJob);
return stopRecordingJob.getFuture();
}
private class StopRecordingJob extends Job {
private final SettableFuture<Pair<Uri, Long>> future = new SettableFuture<>();
public StopRecordingJob() {
super(JobParameters.newBuilder()
.withGroupId(AudioRecorder.class.getSimpleName())
.create());
}
public ListenableFuture<Pair<Uri, Long>> getFuture() {
return future;
}
final SettableFuture<Pair<Uri, Long>> future = new SettableFuture<>();
executor.execute(new Runnable() {
@Override
public void onAdded() {}
@Override
public void onRun() {
if (mediaRecorder == null) {
sendToFuture(new IOException("MediaRecorder was never initialized successfully!"));
public void run() {
if (audioCodec == null) {
sendToFuture(future, new IOException("MediaRecorder was never initialized successfully!"));
return;
}
try {
mediaRecorder.stop();
} catch (Exception e) {
Log.w(TAG, e);
}
try {
fd.close();
} catch (IOException e) {
Log.w("AudioRecorder", e);
}
mediaRecorder.release();
mediaRecorder = null;
audioCodec.stop();
try {
long size = MediaUtil.getMediaSize(context, masterSecret, captureUri);
sendToFuture(new Pair<>(captureUri, size));
sendToFuture(future, new Pair<>(captureUri, size));
} catch (IOException ioe) {
Log.w(TAG, ioe);
sendToFuture(ioe);
sendToFuture(future, ioe);
}
audioCodec = null;
captureUri = null;
fd = null;
}
@Override
public boolean onShouldRetry(Exception e) {
return false;
}
@Override
public void onCanceled() {}
private void sendToFuture(final @NonNull Pair<Uri, Long> result) {
Util.runOnMain(new Runnable() {
@Override
public void run() {
future.set(result);
}
});
return future;
}
private void sendToFuture(final @NonNull Exception exception) {
private <T> void sendToFuture(final SettableFuture<T> future, final Exception exception) {
Util.runOnMain(new Runnable() {
@Override
public void run() {
@ -134,68 +104,13 @@ public class AudioRecorder {
}
});
}
}
private class StartRecordingJob extends Job {
public StartRecordingJob() {
super(JobParameters.newBuilder()
.withGroupId(AudioRecorder.class.getSimpleName())
.create());
}
private <T> void sendToFuture(final SettableFuture<T> future, final T result) {
Util.runOnMain(new Runnable() {
@Override
public void onAdded() {}
@Override
public void onRun() throws Exception {
if (mediaRecorder != null) {
throw new AssertionError("We can only record once at a time.");
public void run() {
future.set(result);
}
ParcelFileDescriptor fds[] = ParcelFileDescriptor.createPipe();
fd = fds[1];
captureUri = blobProvider.create(masterSecret, new ParcelFileDescriptor.AutoCloseInputStream(fds[0]));
mediaRecorder = new MediaRecorder();
mediaRecorder.setAudioSource(MediaRecorder.AudioSource.MIC);
mediaRecorder.setOutputFormat(MediaRecorder.OutputFormat.RAW_AMR);
mediaRecorder.setAudioEncoder(MediaRecorder.AudioEncoder.AMR_NB);
mediaRecorder.setOutputFile(fds[1].getFileDescriptor());
mediaRecorder.prepare();
try {
mediaRecorder.start();
} catch (Exception e) {
Log.w(TAG, e);
throw new IOException(e);
});
}
}
@Override
public boolean onShouldRetry(Exception e) {
return false;
}
@Override
public void onCanceled() {
try {
if (fd != null) {
fd.close();
}
if (captureUri != null) {
blobProvider.delete(captureUri);
}
fd = null;
mediaRecorder = null;
captureUri = null;
} catch (IOException e) {
Log.w(TAG, e);
}
}
}
}

2
src/org/thoughtcrime/securesms/components/InputPanel.java
View File

@ -72,7 +72,7 @@ public class InputPanel extends LinearLayout implements MicrophoneRecorderView.L
this.microphoneRecorderView = ViewUtil.findById(this, R.id.recorder_view);
this.microphoneRecorderView.setListener(this);
if (Build.VERSION.SDK_INT < 14) {
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.JELLY_BEAN) {
this.microphoneRecorderView.setVisibility(View.GONE);
this.microphoneRecorderView.setClickable(false);
}

6
src/org/thoughtcrime/securesms/mms/AudioSlide.java
View File

@ -25,6 +25,8 @@ import android.support.annotation.Nullable;
import org.thoughtcrime.securesms.R;
import org.thoughtcrime.securesms.attachments.Attachment;
import org.thoughtcrime.securesms.attachments.UriAttachment;
import org.thoughtcrime.securesms.database.AttachmentDatabase;
import org.thoughtcrime.securesms.util.ResUtil;
import java.io.IOException;
@ -38,6 +40,10 @@ public class AudioSlide extends Slide {
super(context, constructAttachmentFromUri(context, uri, ContentType.AUDIO_UNSPECIFIED, dataSize));
}
public AudioSlide(Context context, Uri uri, long dataSize, String contentType) {
super(context, new UriAttachment(uri, contentType, AttachmentDatabase.TRANSFER_PROGRESS_STARTED, dataSize));
}
public AudioSlide(Context context, Attachment attachment) {
super(context, attachment);
}

19
src/org/thoughtcrime/securesms/util/ThreadUtil.java Normal file
View File

@ -0,0 +1,19 @@
package org.thoughtcrime.securesms.util;
import java.util.concurrent.Executor;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
public class ThreadUtil {
public static ExecutorService newDynamicSingleThreadedExecutor() {
ThreadPoolExecutor executor = new ThreadPoolExecutor(1, 1, 60, TimeUnit.SECONDS,
new LinkedBlockingQueue<Runnable>());
executor.allowCoreThreadTimeOut(true);
return executor;
}
}

11
src/org/thoughtcrime/securesms/util/Util.java
View File

@ -35,6 +35,7 @@ import android.text.Spannable;
import android.text.SpannableString;
import android.text.TextUtils;
import android.text.style.StyleSpan;
import android.util.Log;
import android.widget.EditText;
import org.thoughtcrime.securesms.BuildConfig;
@ -64,6 +65,8 @@ import ws.com.google.android.mms.pdu.CharacterSets;
import ws.com.google.android.mms.pdu.EncodedStringValue;
public class Util {
private static final String TAG = Util.class.getSimpleName();
public static Handler handler = new Handler(Looper.getMainLooper());
public static String join(String[] list, String delimiter) {
@ -158,6 +161,14 @@ public class Util {
}
}
public static void close(OutputStream out) {
try {
out.close();
} catch (IOException e) {
Log.w(TAG, e);
}
}
public static String canonicalizeNumber(Context context, String number)
throws InvalidNumberException
{