kopia lustrzana https://github.com/ryukoposting/Signal-Android
Improve UX around seeing audio wave forms.
- Attempts to generate the wave form on download instead on display - Allows multi-threaded generation of wave forms instead of serial executormain
Cody Henthorne
GitHub
rodzic
d4ce8458a4
commit
9610339f38
|
|
@ -41,6 +41,11 @@
|
|||
</value>
|
||||
</option>
|
||||
</JavaCodeStyleSettings>
|
||||
<JetCodeStyleSettings>
|
||||
<option name="PACKAGES_TO_USE_STAR_IMPORTS">
|
||||
<value />
|
||||
</option>
|
||||
</JetCodeStyleSettings>
|
||||
<codeStyleSettings language="HTML">
|
||||
<indentOptions>
|
||||
<option name="INDENT_SIZE" value="2" />
|
||||
|
|
|
|||
|
|
@ -0,0 +1,45 @@
|
|||
package org.thoughtcrime.securesms.audio;
|
||||
|
||||
import androidx.annotation.NonNull;
|
||||
|
||||
import com.google.protobuf.ByteString;
|
||||
|
||||
import org.thoughtcrime.securesms.database.model.databaseprotos.AudioWaveFormData;
|
||||
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
public class AudioFileInfo {
|
||||
private final long durationUs;
|
||||
private final byte[] waveFormBytes;
|
||||
private final float[] waveForm;
|
||||
|
||||
public static @NonNull AudioFileInfo fromDatabaseProtobuf(@NonNull AudioWaveFormData audioWaveForm) {
|
||||
return new AudioFileInfo(audioWaveForm.getDurationUs(), audioWaveForm.getWaveForm().toByteArray());
|
||||
}
|
||||
|
||||
AudioFileInfo(long durationUs, byte[] waveFormBytes) {
|
||||
this.durationUs = durationUs;
|
||||
this.waveFormBytes = waveFormBytes;
|
||||
this.waveForm = new float[waveFormBytes.length];
|
||||
|
||||
for (int i = 0; i < waveFormBytes.length; i++) {
|
||||
int unsigned = waveFormBytes[i] & 0xff;
|
||||
this.waveForm[i] = unsigned / 255f;
|
||||
}
|
||||
}
|
||||
|
||||
public long getDuration(@NonNull TimeUnit timeUnit) {
|
||||
return timeUnit.convert(durationUs, TimeUnit.MICROSECONDS);
|
||||
}
|
||||
|
||||
public float[] getWaveForm() {
|
||||
return waveForm;
|
||||
}
|
||||
|
||||
public @NonNull AudioWaveFormData toDatabaseProtobuf() {
|
||||
return AudioWaveFormData.newBuilder()
|
||||
.setDurationUs(durationUs)
|
||||
.setWaveForm(ByteString.copyFrom(waveFormBytes))
|
||||
.build();
|
||||
}
|
||||
}
|
||||
|
|
@ -1,325 +0,0 @@
|
|||
package org.thoughtcrime.securesms.audio;
|
||||
|
||||
import android.content.Context;
|
||||
import android.media.MediaCodec;
|
||||
import android.media.MediaExtractor;
|
||||
import android.media.MediaFormat;
|
||||
import android.net.Uri;
|
||||
import android.os.Build;
|
||||
import android.util.LruCache;
|
||||
|
||||
import androidx.annotation.AnyThread;
|
||||
import androidx.annotation.NonNull;
|
||||
import androidx.annotation.RequiresApi;
|
||||
import androidx.annotation.WorkerThread;
|
||||
import androidx.core.util.Consumer;
|
||||
|
||||
import com.google.protobuf.ByteString;
|
||||
|
||||
import org.signal.core.util.ThreadUtil;
|
||||
import org.signal.core.util.concurrent.SignalExecutors;
|
||||
import org.signal.core.util.logging.Log;
|
||||
import org.thoughtcrime.securesms.attachments.Attachment;
|
||||
import org.thoughtcrime.securesms.attachments.DatabaseAttachment;
|
||||
import org.thoughtcrime.securesms.database.AttachmentTable;
|
||||
import org.thoughtcrime.securesms.database.SignalDatabase;
|
||||
import org.thoughtcrime.securesms.database.model.databaseprotos.AudioWaveFormData;
|
||||
import org.thoughtcrime.securesms.media.DecryptableUriMediaInput;
|
||||
import org.thoughtcrime.securesms.media.MediaInput;
|
||||
import org.thoughtcrime.securesms.mms.AudioSlide;
|
||||
import org.thoughtcrime.securesms.util.concurrent.SerialExecutor;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.Locale;
|
||||
import java.util.concurrent.Executor;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
@RequiresApi(api = Build.VERSION_CODES.M)
|
||||
public final class AudioWaveForm {
|
||||
|
||||
private static final String TAG = Log.tag(AudioWaveForm.class);
|
||||
|
||||
private static final int BAR_COUNT = 46;
|
||||
private static final int SAMPLES_PER_BAR = 4;
|
||||
|
||||
private final Context context;
|
||||
private final AudioSlide slide;
|
||||
|
||||
public AudioWaveForm(@NonNull Context context, @NonNull AudioSlide slide) {
|
||||
this.context = context.getApplicationContext();
|
||||
this.slide = slide;
|
||||
}
|
||||
|
||||
private static final LruCache<String, AudioFileInfo> WAVE_FORM_CACHE = new LruCache<>(200);
|
||||
private static final Executor AUDIO_DECODER_EXECUTOR = new SerialExecutor(SignalExecutors.BOUNDED);
|
||||
|
||||
@AnyThread
|
||||
public void getWaveForm(@NonNull Consumer<AudioFileInfo> onSuccess, @NonNull Runnable onFailure) {
|
||||
Uri uri = slide.getUri();
|
||||
Attachment attachment = slide.asAttachment();
|
||||
|
||||
if (uri == null) {
|
||||
Log.w(TAG, "No uri");
|
||||
ThreadUtil.runOnMain(onFailure);
|
||||
return;
|
||||
}
|
||||
|
||||
String cacheKey = uri.toString();
|
||||
AudioFileInfo cached = WAVE_FORM_CACHE.get(cacheKey);
|
||||
if (cached != null) {
|
||||
Log.i(TAG, "Loaded wave form from cache " + cacheKey);
|
||||
ThreadUtil.runOnMain(() -> onSuccess.accept(cached));
|
||||
return;
|
||||
}
|
||||
|
||||
AUDIO_DECODER_EXECUTOR.execute(() -> {
|
||||
AudioFileInfo cachedInExecutor = WAVE_FORM_CACHE.get(cacheKey);
|
||||
if (cachedInExecutor != null) {
|
||||
Log.i(TAG, "Loaded wave form from cache inside executor" + cacheKey);
|
||||
ThreadUtil.runOnMain(() -> onSuccess.accept(cachedInExecutor));
|
||||
return;
|
||||
}
|
||||
|
||||
AudioHash audioHash = attachment.getAudioHash();
|
||||
if (audioHash != null) {
|
||||
AudioFileInfo audioFileInfo = AudioFileInfo.fromDatabaseProtobuf(audioHash.getAudioWaveForm());
|
||||
if (audioFileInfo.waveForm.length == 0) {
|
||||
Log.w(TAG, "Recovering from a wave form generation error " + cacheKey);
|
||||
ThreadUtil.runOnMain(onFailure);
|
||||
return;
|
||||
} else if (audioFileInfo.waveForm.length != BAR_COUNT) {
|
||||
Log.w(TAG, "Wave form from database does not match bar count, regenerating " + cacheKey);
|
||||
} else {
|
||||
WAVE_FORM_CACHE.put(cacheKey, audioFileInfo);
|
||||
Log.i(TAG, "Loaded wave form from DB " + cacheKey);
|
||||
ThreadUtil.runOnMain(() -> onSuccess.accept(audioFileInfo));
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if (attachment instanceof DatabaseAttachment) {
|
||||
try {
|
||||
AttachmentTable attachmentDatabase = SignalDatabase.attachments();
|
||||
DatabaseAttachment dbAttachment = (DatabaseAttachment) attachment;
|
||||
long startTime = System.currentTimeMillis();
|
||||
|
||||
attachmentDatabase.writeAudioHash(dbAttachment.getAttachmentId(), AudioWaveFormData.getDefaultInstance());
|
||||
|
||||
Log.i(TAG, String.format("Starting wave form generation (%s)", cacheKey));
|
||||
|
||||
AudioFileInfo fileInfo = generateWaveForm(uri);
|
||||
|
||||
Log.i(TAG, String.format(Locale.US, "Audio wave form generation time %d ms (%s)", System.currentTimeMillis() - startTime, cacheKey));
|
||||
|
||||
attachmentDatabase.writeAudioHash(dbAttachment.getAttachmentId(), fileInfo.toDatabaseProtobuf());
|
||||
|
||||
WAVE_FORM_CACHE.put(cacheKey, fileInfo);
|
||||
ThreadUtil.runOnMain(() -> onSuccess.accept(fileInfo));
|
||||
} catch (Throwable e) {
|
||||
Log.w(TAG, "Failed to create audio wave form for " + cacheKey, e);
|
||||
ThreadUtil.runOnMain(onFailure);
|
||||
}
|
||||
} else {
|
||||
try {
|
||||
Log.i(TAG, "Not in database and not cached. Generating wave form on-the-fly.");
|
||||
|
||||
long startTime = System.currentTimeMillis();
|
||||
|
||||
Log.i(TAG, String.format("Starting wave form generation (%s)", cacheKey));
|
||||
|
||||
AudioFileInfo fileInfo = generateWaveForm(uri);
|
||||
|
||||
Log.i(TAG, String.format(Locale.US, "Audio wave form generation time %d ms (%s)", System.currentTimeMillis() - startTime, cacheKey));
|
||||
|
||||
WAVE_FORM_CACHE.put(cacheKey, fileInfo);
|
||||
ThreadUtil.runOnMain(() -> onSuccess.accept(fileInfo));
|
||||
} catch (IOException e) {
|
||||
Log.w(TAG, "Failed to create audio wave form for " + cacheKey, e);
|
||||
ThreadUtil.runOnMain(onFailure);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Based on decode sample from:
|
||||
* <p>
|
||||
* https://android.googlesource.com/platform/cts/+/jb-mr2-release/tests/tests/media/src/android/media/cts/DecoderTest.java
|
||||
*/
|
||||
@WorkerThread
|
||||
@RequiresApi(api = 23)
|
||||
private @NonNull AudioFileInfo generateWaveForm(@NonNull Uri uri) throws IOException {
|
||||
try (MediaInput dataSource = DecryptableUriMediaInput.createForUri(context, uri)) {
|
||||
long[] wave = new long[BAR_COUNT];
|
||||
int[] waveSamples = new int[BAR_COUNT];
|
||||
|
||||
MediaExtractor extractor = dataSource.createExtractor();
|
||||
|
||||
if (extractor.getTrackCount() == 0) {
|
||||
throw new IOException("No audio track");
|
||||
}
|
||||
|
||||
MediaFormat format = extractor.getTrackFormat(0);
|
||||
|
||||
if (!format.containsKey(MediaFormat.KEY_DURATION)) {
|
||||
throw new IOException("Unknown duration");
|
||||
}
|
||||
|
||||
long totalDurationUs = format.getLong(MediaFormat.KEY_DURATION);
|
||||
String mime = format.getString(MediaFormat.KEY_MIME);
|
||||
|
||||
if (!mime.startsWith("audio/")) {
|
||||
throw new IOException("Mime not audio");
|
||||
}
|
||||
|
||||
MediaCodec codec = MediaCodec.createDecoderByType(mime);
|
||||
|
||||
if (totalDurationUs == 0) {
|
||||
throw new IOException("Zero duration");
|
||||
}
|
||||
|
||||
codec.configure(format, null, null, 0);
|
||||
codec.start();
|
||||
|
||||
ByteBuffer[] codecInputBuffers = codec.getInputBuffers();
|
||||
ByteBuffer[] codecOutputBuffers = codec.getOutputBuffers();
|
||||
|
||||
extractor.selectTrack(0);
|
||||
|
||||
long kTimeOutUs = 5000;
|
||||
MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
|
||||
boolean sawInputEOS = false;
|
||||
boolean sawOutputEOS = false;
|
||||
int noOutputCounter = 0;
|
||||
|
||||
while (!sawOutputEOS && noOutputCounter < 50) {
|
||||
noOutputCounter++;
|
||||
if (!sawInputEOS) {
|
||||
int inputBufIndex = codec.dequeueInputBuffer(kTimeOutUs);
|
||||
if (inputBufIndex >= 0) {
|
||||
ByteBuffer dstBuf = codecInputBuffers[inputBufIndex];
|
||||
int sampleSize = extractor.readSampleData(dstBuf, 0);
|
||||
long presentationTimeUs = 0;
|
||||
|
||||
if (sampleSize < 0) {
|
||||
sawInputEOS = true;
|
||||
sampleSize = 0;
|
||||
} else {
|
||||
presentationTimeUs = extractor.getSampleTime();
|
||||
}
|
||||
|
||||
codec.queueInputBuffer(
|
||||
inputBufIndex,
|
||||
0,
|
||||
sampleSize,
|
||||
presentationTimeUs,
|
||||
sawInputEOS ? MediaCodec.BUFFER_FLAG_END_OF_STREAM : 0);
|
||||
|
||||
if (!sawInputEOS) {
|
||||
int barSampleIndex = (int) (SAMPLES_PER_BAR * (wave.length * extractor.getSampleTime()) / totalDurationUs);
|
||||
sawInputEOS = !extractor.advance();
|
||||
int nextBarSampleIndex = (int) (SAMPLES_PER_BAR * (wave.length * extractor.getSampleTime()) / totalDurationUs);
|
||||
while (!sawInputEOS && nextBarSampleIndex == barSampleIndex) {
|
||||
sawInputEOS = !extractor.advance();
|
||||
if (!sawInputEOS) {
|
||||
nextBarSampleIndex = (int) (SAMPLES_PER_BAR * (wave.length * extractor.getSampleTime()) / totalDurationUs);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int outputBufferIndex;
|
||||
do {
|
||||
outputBufferIndex = codec.dequeueOutputBuffer(info, kTimeOutUs);
|
||||
if (outputBufferIndex >= 0) {
|
||||
if (info.size > 0) {
|
||||
noOutputCounter = 0;
|
||||
}
|
||||
|
||||
ByteBuffer buf = codecOutputBuffers[outputBufferIndex];
|
||||
int barIndex = (int) ((wave.length * info.presentationTimeUs) / totalDurationUs);
|
||||
long total = 0;
|
||||
for (int i = 0; i < info.size; i += 2 * 4) {
|
||||
short aShort = buf.getShort(i);
|
||||
total += Math.abs(aShort);
|
||||
}
|
||||
if (barIndex >= 0 && barIndex < wave.length) {
|
||||
wave[barIndex] += total;
|
||||
waveSamples[barIndex] += info.size / 2;
|
||||
}
|
||||
codec.releaseOutputBuffer(outputBufferIndex, false);
|
||||
if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
|
||||
sawOutputEOS = true;
|
||||
}
|
||||
} else if (outputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
|
||||
codecOutputBuffers = codec.getOutputBuffers();
|
||||
} else if (outputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
|
||||
Log.d(TAG, "output format has changed to " + codec.getOutputFormat());
|
||||
}
|
||||
} while (outputBufferIndex >= 0);
|
||||
}
|
||||
|
||||
codec.stop();
|
||||
codec.release();
|
||||
extractor.release();
|
||||
|
||||
float[] floats = new float[BAR_COUNT];
|
||||
byte[] bytes = new byte[BAR_COUNT];
|
||||
float max = 0;
|
||||
|
||||
for (int i = 0; i < BAR_COUNT; i++) {
|
||||
if (waveSamples[i] == 0) continue;
|
||||
|
||||
floats[i] = wave[i] / (float) waveSamples[i];
|
||||
if (floats[i] > max) {
|
||||
max = floats[i];
|
||||
}
|
||||
}
|
||||
|
||||
for (int i = 0; i < BAR_COUNT; i++) {
|
||||
float normalized = floats[i] / max;
|
||||
bytes[i] = (byte) (255 * normalized);
|
||||
}
|
||||
|
||||
return new AudioFileInfo(totalDurationUs, bytes);
|
||||
}
|
||||
}
|
||||
|
||||
public static class AudioFileInfo {
|
||||
private final long durationUs;
|
||||
private final byte[] waveFormBytes;
|
||||
private final float[] waveForm;
|
||||
|
||||
private static @NonNull AudioFileInfo fromDatabaseProtobuf(@NonNull AudioWaveFormData audioWaveForm) {
|
||||
return new AudioFileInfo(audioWaveForm.getDurationUs(), audioWaveForm.getWaveForm().toByteArray());
|
||||
}
|
||||
|
||||
private AudioFileInfo(long durationUs, byte[] waveFormBytes) {
|
||||
this.durationUs = durationUs;
|
||||
this.waveFormBytes = waveFormBytes;
|
||||
this.waveForm = new float[waveFormBytes.length];
|
||||
|
||||
for (int i = 0; i < waveFormBytes.length; i++) {
|
||||
int unsigned = waveFormBytes[i] & 0xff;
|
||||
this.waveForm[i] = unsigned / 255f;
|
||||
}
|
||||
}
|
||||
|
||||
public long getDuration(@NonNull TimeUnit timeUnit) {
|
||||
return timeUnit.convert(durationUs, TimeUnit.MICROSECONDS);
|
||||
}
|
||||
|
||||
public float[] getWaveForm() {
|
||||
return waveForm;
|
||||
}
|
||||
|
||||
private @NonNull AudioWaveFormData toDatabaseProtobuf() {
|
||||
return AudioWaveFormData.newBuilder()
|
||||
.setDurationUs(durationUs)
|
||||
.setWaveForm(ByteString.copyFrom(waveFormBytes))
|
||||
.build();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,173 @@
|
|||
package org.thoughtcrime.securesms.audio;
|
||||
|
||||
import android.content.Context;
|
||||
import android.media.MediaCodec;
|
||||
import android.media.MediaExtractor;
|
||||
import android.media.MediaFormat;
|
||||
import android.net.Uri;
|
||||
|
||||
import androidx.annotation.NonNull;
|
||||
import androidx.annotation.RequiresApi;
|
||||
import androidx.annotation.WorkerThread;
|
||||
|
||||
import org.signal.core.util.logging.Log;
|
||||
import org.thoughtcrime.securesms.media.DecryptableUriMediaInput;
|
||||
import org.thoughtcrime.securesms.media.MediaInput;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.ByteBuffer;
|
||||
|
||||
@RequiresApi(api = 23)
|
||||
public final class AudioWaveFormGenerator {
|
||||
|
||||
private static final String TAG = Log.tag(AudioWaveFormGenerator.class);
|
||||
|
||||
public static final int BAR_COUNT = 46;
|
||||
private static final int SAMPLES_PER_BAR = 4;
|
||||
|
||||
private AudioWaveFormGenerator() {}
|
||||
|
||||
/**
|
||||
* Based on decode sample from:
|
||||
* <p>
|
||||
* https://android.googlesource.com/platform/cts/+/jb-mr2-release/tests/tests/media/src/android/media/cts/DecoderTest.java
|
||||
*/
|
||||
@WorkerThread
|
||||
public static @NonNull AudioFileInfo generateWaveForm(@NonNull Context context, @NonNull Uri uri) throws IOException {
|
||||
try (MediaInput dataSource = DecryptableUriMediaInput.createForUri(context, uri)) {
|
||||
long[] wave = new long[BAR_COUNT];
|
||||
int[] waveSamples = new int[BAR_COUNT];
|
||||
|
||||
MediaExtractor extractor = dataSource.createExtractor();
|
||||
|
||||
if (extractor.getTrackCount() == 0) {
|
||||
throw new IOException("No audio track");
|
||||
}
|
||||
|
||||
MediaFormat format = extractor.getTrackFormat(0);
|
||||
|
||||
if (!format.containsKey(MediaFormat.KEY_DURATION)) {
|
||||
throw new IOException("Unknown duration");
|
||||
}
|
||||
|
||||
long totalDurationUs = format.getLong(MediaFormat.KEY_DURATION);
|
||||
String mime = format.getString(MediaFormat.KEY_MIME);
|
||||
|
||||
if (!mime.startsWith("audio/")) {
|
||||
throw new IOException("Mime not audio");
|
||||
}
|
||||
|
||||
MediaCodec codec = MediaCodec.createDecoderByType(mime);
|
||||
|
||||
if (totalDurationUs == 0) {
|
||||
throw new IOException("Zero duration");
|
||||
}
|
||||
|
||||
codec.configure(format, null, null, 0);
|
||||
codec.start();
|
||||
|
||||
ByteBuffer[] codecInputBuffers = codec.getInputBuffers();
|
||||
ByteBuffer[] codecOutputBuffers = codec.getOutputBuffers();
|
||||
|
||||
extractor.selectTrack(0);
|
||||
|
||||
long kTimeOutUs = 5000;
|
||||
MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
|
||||
boolean sawInputEOS = false;
|
||||
boolean sawOutputEOS = false;
|
||||
int noOutputCounter = 0;
|
||||
|
||||
while (!sawOutputEOS && noOutputCounter < 50) {
|
||||
noOutputCounter++;
|
||||
if (!sawInputEOS) {
|
||||
int inputBufIndex = codec.dequeueInputBuffer(kTimeOutUs);
|
||||
if (inputBufIndex >= 0) {
|
||||
ByteBuffer dstBuf = codecInputBuffers[inputBufIndex];
|
||||
int sampleSize = extractor.readSampleData(dstBuf, 0);
|
||||
long presentationTimeUs = 0;
|
||||
|
||||
if (sampleSize < 0) {
|
||||
sawInputEOS = true;
|
||||
sampleSize = 0;
|
||||
} else {
|
||||
presentationTimeUs = extractor.getSampleTime();
|
||||
}
|
||||
|
||||
codec.queueInputBuffer(
|
||||
inputBufIndex,
|
||||
0,
|
||||
sampleSize,
|
||||
presentationTimeUs,
|
||||
sawInputEOS ? MediaCodec.BUFFER_FLAG_END_OF_STREAM : 0);
|
||||
|
||||
if (!sawInputEOS) {
|
||||
int barSampleIndex = (int) (SAMPLES_PER_BAR * (wave.length * extractor.getSampleTime()) / totalDurationUs);
|
||||
sawInputEOS = !extractor.advance();
|
||||
int nextBarSampleIndex = (int) (SAMPLES_PER_BAR * (wave.length * extractor.getSampleTime()) / totalDurationUs);
|
||||
while (!sawInputEOS && nextBarSampleIndex == barSampleIndex) {
|
||||
sawInputEOS = !extractor.advance();
|
||||
if (!sawInputEOS) {
|
||||
nextBarSampleIndex = (int) (SAMPLES_PER_BAR * (wave.length * extractor.getSampleTime()) / totalDurationUs);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int outputBufferIndex;
|
||||
do {
|
||||
outputBufferIndex = codec.dequeueOutputBuffer(info, kTimeOutUs);
|
||||
if (outputBufferIndex >= 0) {
|
||||
if (info.size > 0) {
|
||||
noOutputCounter = 0;
|
||||
}
|
||||
|
||||
ByteBuffer buf = codecOutputBuffers[outputBufferIndex];
|
||||
int barIndex = (int) ((wave.length * info.presentationTimeUs) / totalDurationUs);
|
||||
long total = 0;
|
||||
for (int i = 0; i < info.size; i += 2 * 4) {
|
||||
short aShort = buf.getShort(i);
|
||||
total += Math.abs(aShort);
|
||||
}
|
||||
if (barIndex >= 0 && barIndex < wave.length) {
|
||||
wave[barIndex] += total;
|
||||
waveSamples[barIndex] += info.size / 2;
|
||||
}
|
||||
codec.releaseOutputBuffer(outputBufferIndex, false);
|
||||
if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
|
||||
sawOutputEOS = true;
|
||||
}
|
||||
} else if (outputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
|
||||
codecOutputBuffers = codec.getOutputBuffers();
|
||||
} else if (outputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
|
||||
Log.d(TAG, "output format has changed to " + codec.getOutputFormat());
|
||||
}
|
||||
} while (outputBufferIndex >= 0);
|
||||
}
|
||||
|
||||
codec.stop();
|
||||
codec.release();
|
||||
extractor.release();
|
||||
|
||||
float[] floats = new float[BAR_COUNT];
|
||||
byte[] bytes = new byte[BAR_COUNT];
|
||||
float max = 0;
|
||||
|
||||
for (int i = 0; i < BAR_COUNT; i++) {
|
||||
if (waveSamples[i] == 0) continue;
|
||||
|
||||
floats[i] = wave[i] / (float) waveSamples[i];
|
||||
if (floats[i] > max) {
|
||||
max = floats[i];
|
||||
}
|
||||
}
|
||||
|
||||
for (int i = 0; i < BAR_COUNT; i++) {
|
||||
float normalized = floats[i] / max;
|
||||
bytes[i] = (byte) (255 * normalized);
|
||||
}
|
||||
|
||||
return new AudioFileInfo(totalDurationUs, bytes);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,152 @@
|
|||
package org.thoughtcrime.securesms.audio
|
||||
|
||||
import android.content.Context
|
||||
import android.net.Uri
|
||||
import android.util.LruCache
|
||||
import androidx.annotation.AnyThread
|
||||
import androidx.annotation.RequiresApi
|
||||
import io.reactivex.rxjava3.core.Single
|
||||
import io.reactivex.rxjava3.schedulers.Schedulers
|
||||
import org.signal.core.util.logging.Log
|
||||
import org.thoughtcrime.securesms.attachments.Attachment
|
||||
import org.thoughtcrime.securesms.attachments.AttachmentId
|
||||
import org.thoughtcrime.securesms.attachments.DatabaseAttachment
|
||||
import org.thoughtcrime.securesms.database.SignalDatabase
|
||||
import org.thoughtcrime.securesms.database.model.databaseprotos.AudioWaveFormData
|
||||
import java.io.IOException
|
||||
import java.util.concurrent.locks.ReentrantReadWriteLock
|
||||
import kotlin.concurrent.read
|
||||
import kotlin.concurrent.write
|
||||
|
||||
/**
|
||||
* Uses [AudioWaveFormGenerator] to generate audio wave forms.
|
||||
*
|
||||
* Maintains an in-memory cache of recently requested wave forms.
|
||||
*/
|
||||
@RequiresApi(23)
|
||||
object AudioWaveForms {
|
||||
|
||||
private val TAG = Log.tag(AudioWaveForms::class.java)
|
||||
|
||||
private val cache = ThreadSafeLruCache(200)
|
||||
|
||||
@AnyThread
|
||||
@JvmStatic
|
||||
fun getWaveForm(context: Context, attachment: Attachment): Single<AudioFileInfo> {
|
||||
val uri = attachment.uri
|
||||
if (uri == null) {
|
||||
Log.i(TAG, "No uri")
|
||||
return Single.error(IllegalArgumentException("No uri from attachment"))
|
||||
}
|
||||
|
||||
val cacheKey = uri.toString()
|
||||
val cachedInfo = cache.get(cacheKey)
|
||||
if (cachedInfo != null) {
|
||||
Log.i(TAG, "Loaded wave form from cache $cacheKey")
|
||||
return Single.just(cachedInfo)
|
||||
}
|
||||
|
||||
val databaseCache = Single.fromCallable {
|
||||
val audioHash = attachment.audioHash
|
||||
return@fromCallable if (audioHash != null) {
|
||||
checkDatabaseCache(cacheKey, audioHash.audioWaveForm)
|
||||
} else {
|
||||
Miss
|
||||
}
|
||||
}.subscribeOn(Schedulers.io())
|
||||
|
||||
val generateWaveForm: Single<CacheCheckResult> = if (attachment is DatabaseAttachment) {
|
||||
Single.fromCallable { generateWaveForm(context, uri, cacheKey, attachment.attachmentId) }
|
||||
} else {
|
||||
Single.fromCallable { generateWaveForm(context, uri, cacheKey) }
|
||||
}.subscribeOn(Schedulers.io())
|
||||
|
||||
return databaseCache
|
||||
.flatMap { r ->
|
||||
if (r is Miss) {
|
||||
generateWaveForm
|
||||
} else {
|
||||
Single.just(r)
|
||||
}
|
||||
}
|
||||
.map { r ->
|
||||
if (r is Success) {
|
||||
r.audioFileInfo
|
||||
} else {
|
||||
throw IOException("Unable to generate wave form")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun checkDatabaseCache(cacheKey: String, audioWaveForm: AudioWaveFormData): CacheCheckResult {
|
||||
val audioFileInfo = AudioFileInfo.fromDatabaseProtobuf(audioWaveForm)
|
||||
if (audioFileInfo.waveForm.isEmpty()) {
|
||||
Log.w(TAG, "Recovering from a wave form generation error $cacheKey")
|
||||
return Failure
|
||||
} else if (audioFileInfo.waveForm.size != AudioWaveFormGenerator.BAR_COUNT) {
|
||||
Log.w(TAG, "Wave form from database does not match bar count, regenerating $cacheKey")
|
||||
} else {
|
||||
cache.put(cacheKey, audioFileInfo)
|
||||
Log.i(TAG, "Loaded wave form from DB $cacheKey")
|
||||
return Success(audioFileInfo)
|
||||
}
|
||||
|
||||
return Miss
|
||||
}
|
||||
|
||||
private fun generateWaveForm(context: Context, uri: Uri, cacheKey: String, attachmentId: AttachmentId): CacheCheckResult {
|
||||
try {
|
||||
val startTime = System.currentTimeMillis()
|
||||
SignalDatabase.attachments.writeAudioHash(attachmentId, AudioWaveFormData.getDefaultInstance())
|
||||
|
||||
Log.i(TAG, "Starting wave form generation ($cacheKey)")
|
||||
val fileInfo: AudioFileInfo = AudioWaveFormGenerator.generateWaveForm(context, uri)
|
||||
Log.i(TAG, "Audio wave form generation time ${System.currentTimeMillis() - startTime} ms ($cacheKey)")
|
||||
|
||||
SignalDatabase.attachments.writeAudioHash(attachmentId, fileInfo.toDatabaseProtobuf())
|
||||
cache.put(cacheKey, fileInfo)
|
||||
|
||||
return Success(fileInfo)
|
||||
} catch (e: Throwable) {
|
||||
Log.w(TAG, "Failed to create audio wave form for $cacheKey", e)
|
||||
return Failure
|
||||
}
|
||||
}
|
||||
|
||||
private fun generateWaveForm(context: Context, uri: Uri, cacheKey: String): CacheCheckResult {
|
||||
try {
|
||||
Log.i(TAG, "Not in database and not cached. Generating wave form on-the-fly.")
|
||||
|
||||
val startTime = System.currentTimeMillis()
|
||||
|
||||
Log.i(TAG, "Starting wave form generation ($cacheKey)")
|
||||
val fileInfo: AudioFileInfo = AudioWaveFormGenerator.generateWaveForm(context, uri)
|
||||
Log.i(TAG, "Audio wave form generation time ${System.currentTimeMillis() - startTime} ms ($cacheKey)")
|
||||
|
||||
cache.put(cacheKey, fileInfo)
|
||||
|
||||
return Success(fileInfo)
|
||||
} catch (e: Throwable) {
|
||||
Log.w(TAG, "Failed to create audio wave form for $cacheKey", e)
|
||||
return Failure
|
||||
}
|
||||
}
|
||||
|
||||
private class ThreadSafeLruCache(maxSize: Int) {
|
||||
private val cache = LruCache<String, AudioFileInfo>(maxSize)
|
||||
private val lock = ReentrantReadWriteLock()
|
||||
|
||||
fun put(key: String, info: AudioFileInfo) {
|
||||
lock.write { cache.put(key, info) }
|
||||
}
|
||||
|
||||
fun get(key: String): AudioFileInfo? {
|
||||
return lock.read { cache.get(key) }
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class CacheCheckResult
|
||||
private class Success(val audioFileInfo: AudioFileInfo) : CacheCheckResult()
|
||||
private object Failure : CacheCheckResult()
|
||||
private object Miss : CacheCheckResult()
|
||||
}
|
||||
|
|
@ -33,7 +33,7 @@ import org.greenrobot.eventbus.Subscribe;
|
|||
import org.greenrobot.eventbus.ThreadMode;
|
||||
import org.signal.core.util.logging.Log;
|
||||
import org.thoughtcrime.securesms.R;
|
||||
import org.thoughtcrime.securesms.audio.AudioWaveForm;
|
||||
import org.thoughtcrime.securesms.audio.AudioWaveForms;
|
||||
import org.thoughtcrime.securesms.components.voice.VoiceNotePlaybackState;
|
||||
import org.thoughtcrime.securesms.database.AttachmentTable;
|
||||
import org.thoughtcrime.securesms.events.PartProgressEvent;
|
||||
|
|
@ -43,6 +43,9 @@ import org.thoughtcrime.securesms.mms.SlideClickListener;
|
|||
import java.util.Objects;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import io.reactivex.rxjava3.android.schedulers.AndroidSchedulers;
|
||||
import io.reactivex.rxjava3.disposables.Disposable;
|
||||
|
||||
public final class AudioView extends FrameLayout {
|
||||
|
||||
private static final String TAG = Log.tag(AudioView.class);
|
||||
|
|
@ -77,6 +80,8 @@ public final class AudioView extends FrameLayout {
|
|||
private AudioSlide audioSlide;
|
||||
private Callbacks callbacks;
|
||||
|
||||
private Disposable disposable = Disposable.disposed();
|
||||
|
||||
private final Observer<VoiceNotePlaybackState> playbackStateObserver = this::onPlaybackState;
|
||||
|
||||
public AudioView(Context context) {
|
||||
|
|
@ -155,6 +160,7 @@ public final class AudioView extends FrameLayout {
|
|||
protected void onDetachedFromWindow() {
|
||||
super.onDetachedFromWindow();
|
||||
EventBus.getDefault().unregister(this);
|
||||
disposable.dispose();
|
||||
}
|
||||
|
||||
public void setProgressAndPlayBackgroundTint(@ColorInt int color) {
|
||||
|
|
@ -170,6 +176,7 @@ public final class AudioView extends FrameLayout {
|
|||
final boolean showControls,
|
||||
final boolean forceHideDuration)
|
||||
{
|
||||
this.disposable.dispose();
|
||||
this.callbacks = callbacks;
|
||||
|
||||
if (duration != null) {
|
||||
|
|
@ -212,16 +219,19 @@ public final class AudioView extends FrameLayout {
|
|||
WaveFormSeekBarView waveFormView = (WaveFormSeekBarView) seekBar;
|
||||
waveFormView.setColors(waveFormPlayedBarsColor, waveFormUnplayedBarsColor, waveFormThumbTint);
|
||||
if (android.os.Build.VERSION.SDK_INT >= 23) {
|
||||
new AudioWaveForm(getContext(), audio).getWaveForm(
|
||||
data -> {
|
||||
durationMillis = data.getDuration(TimeUnit.MILLISECONDS);
|
||||
updateProgress(0, 0);
|
||||
if (!forceHideDuration && duration != null) {
|
||||
duration.setVisibility(VISIBLE);
|
||||
}
|
||||
waveFormView.setWaveData(data.getWaveForm());
|
||||
},
|
||||
() -> waveFormView.setWaveMode(false));
|
||||
disposable = AudioWaveForms.getWaveForm(getContext(), audioSlide.asAttachment())
|
||||
.observeOn(AndroidSchedulers.mainThread())
|
||||
.subscribe(
|
||||
data -> {
|
||||
durationMillis = data.getDuration(TimeUnit.MILLISECONDS);
|
||||
updateProgress(0, 0);
|
||||
if (!forceHideDuration && duration != null) {
|
||||
duration.setVisibility(VISIBLE);
|
||||
}
|
||||
waveFormView.setWaveData(data.getWaveForm());
|
||||
},
|
||||
t -> waveFormView.setWaveMode(false)
|
||||
);
|
||||
} else {
|
||||
waveFormView.setWaveMode(false);
|
||||
if (duration != null) {
|
||||
|
|
|
|||
|
|
@ -51,6 +51,7 @@ import org.thoughtcrime.securesms.crypto.ClassicDecryptingPartInputStream;
|
|||
import org.thoughtcrime.securesms.crypto.ModernDecryptingPartInputStream;
|
||||
import org.thoughtcrime.securesms.crypto.ModernEncryptingPartOutputStream;
|
||||
import org.thoughtcrime.securesms.database.model.databaseprotos.AudioWaveFormData;
|
||||
import org.thoughtcrime.securesms.jobs.GenerateAudioWaveFormJob;
|
||||
import org.thoughtcrime.securesms.mms.MediaStream;
|
||||
import org.thoughtcrime.securesms.mms.MmsException;
|
||||
import org.thoughtcrime.securesms.mms.PartAuthority;
|
||||
|
|
@ -656,6 +657,10 @@ public class AttachmentTable extends DatabaseTable {
|
|||
//noinspection ResultOfMethodCallIgnored
|
||||
transferFile.delete();
|
||||
}
|
||||
|
||||
if (placeholder != null && MediaUtil.isAudio(placeholder)) {
|
||||
GenerateAudioWaveFormJob.enqueue(placeholder.getAttachmentId());
|
||||
}
|
||||
}
|
||||
|
||||
private static @Nullable String getVisualHashStringOrNull(@Nullable Attachment attachment) {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,94 @@
|
|||
package org.thoughtcrime.securesms.jobs
|
||||
|
||||
import android.os.Build
|
||||
import androidx.annotation.RequiresApi
|
||||
import org.signal.core.util.concurrent.safeBlockingGet
|
||||
import org.signal.core.util.logging.Log
|
||||
import org.thoughtcrime.securesms.attachments.AttachmentId
|
||||
import org.thoughtcrime.securesms.attachments.DatabaseAttachment
|
||||
import org.thoughtcrime.securesms.audio.AudioWaveForms
|
||||
import org.thoughtcrime.securesms.database.SignalDatabase
|
||||
import org.thoughtcrime.securesms.dependencies.ApplicationDependencies
|
||||
import org.thoughtcrime.securesms.jobmanager.Data
|
||||
import org.thoughtcrime.securesms.jobmanager.Job
|
||||
import org.thoughtcrime.securesms.util.MediaUtil
|
||||
import kotlin.time.Duration.Companion.days
|
||||
|
||||
/**
|
||||
* Generate and save wave form for an audio attachment.
|
||||
*/
|
||||
class GenerateAudioWaveFormJob private constructor(private val attachmentId: AttachmentId, parameters: Parameters) : BaseJob(parameters) {
|
||||
|
||||
companion object {
|
||||
private val TAG = Log.tag(GenerateAudioWaveFormJob::class.java)
|
||||
|
||||
private const val KEY_PART_ROW_ID = "part_row_id"
|
||||
private const val KEY_PAR_UNIQUE_ID = "part_unique_id"
|
||||
|
||||
const val KEY = "GenerateAudioWaveFormJob"
|
||||
|
||||
@JvmStatic
|
||||
fun enqueue(attachmentId: AttachmentId) {
|
||||
if (Build.VERSION.SDK_INT < 23) {
|
||||
Log.i(TAG, "Unable to generate waveform on this version of Android")
|
||||
} else {
|
||||
ApplicationDependencies.getJobManager().add(GenerateAudioWaveFormJob(attachmentId))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private constructor(attachmentId: AttachmentId) : this(
|
||||
attachmentId,
|
||||
Parameters.Builder()
|
||||
.setQueue("GenerateAudioWaveFormJob")
|
||||
.setLifespan(1.days.inWholeMilliseconds)
|
||||
.setMaxAttempts(1)
|
||||
.build()
|
||||
)
|
||||
|
||||
override fun serialize(): Data {
|
||||
return Data.Builder()
|
||||
.putLong(KEY_PART_ROW_ID, attachmentId.rowId)
|
||||
.putLong(KEY_PAR_UNIQUE_ID, attachmentId.uniqueId)
|
||||
.build()
|
||||
}
|
||||
|
||||
override fun getFactoryKey(): String = KEY
|
||||
|
||||
@RequiresApi(23)
|
||||
override fun onRun() {
|
||||
val attachment: DatabaseAttachment? = SignalDatabase.attachments.getAttachment(attachmentId)
|
||||
|
||||
if (attachment == null) {
|
||||
Log.i(TAG, "Unable to find attachment in database.")
|
||||
return
|
||||
}
|
||||
|
||||
if (!MediaUtil.isAudio(attachment)) {
|
||||
Log.w(TAG, "Attempting to generate wave form for a non-audio attachment type: ${attachment.contentType}")
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
AudioWaveForms
|
||||
.getWaveForm(context, attachment)
|
||||
.safeBlockingGet()
|
||||
|
||||
Log.i(TAG, "Generation successful")
|
||||
} catch (e: Exception) {
|
||||
Log.i(TAG, "Generation failed", e)
|
||||
}
|
||||
}
|
||||
|
||||
override fun onShouldRetry(e: Exception): Boolean {
|
||||
return false
|
||||
}
|
||||
|
||||
override fun onFailure() = Unit
|
||||
|
||||
class Factory : Job.Factory<GenerateAudioWaveFormJob> {
|
||||
override fun create(parameters: Parameters, data: Data): GenerateAudioWaveFormJob {
|
||||
return GenerateAudioWaveFormJob(AttachmentId(data.getLong(KEY_PART_ROW_ID), data.getLong(KEY_PAR_UNIQUE_ID)), parameters)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -105,8 +105,8 @@ public final class JobManagerFactories {
|
|||
put(FontDownloaderJob.KEY, new FontDownloaderJob.Factory());
|
||||
put(ForceUpdateGroupV2Job.KEY, new ForceUpdateGroupV2Job.Factory());
|
||||
put(ForceUpdateGroupV2WorkerJob.KEY, new ForceUpdateGroupV2WorkerJob.Factory());
|
||||
put(GenerateAudioWaveFormJob.KEY, new GenerateAudioWaveFormJob.Factory());
|
||||
put(GiftSendJob.KEY, new GiftSendJob.Factory());
|
||||
put(SendPaymentsActivatedJob.KEY, new SendPaymentsActivatedJob.Factory());
|
||||
put(GroupV1MigrationJob.KEY, new GroupV1MigrationJob.Factory());
|
||||
put(GroupCallUpdateSendJob.KEY, new GroupCallUpdateSendJob.Factory());
|
||||
put(GroupCallPeekJob.KEY, new GroupCallPeekJob.Factory());
|
||||
|
|
@ -177,6 +177,7 @@ public final class JobManagerFactories {
|
|||
put(RotateProfileKeyJob.KEY, new RotateProfileKeyJob.Factory());
|
||||
put(SenderKeyDistributionSendJob.KEY, new SenderKeyDistributionSendJob.Factory());
|
||||
put(SendDeliveryReceiptJob.KEY, new SendDeliveryReceiptJob.Factory());
|
||||
put(SendPaymentsActivatedJob.KEY, new SendPaymentsActivatedJob.Factory());
|
||||
put(SendReadReceiptJob.KEY, new SendReadReceiptJob.Factory(application));
|
||||
put(SendRetryReceiptJob.KEY, new SendRetryReceiptJob.Factory());
|
||||
put(SendViewedReceiptJob.KEY, new SendViewedReceiptJob.Factory(application));
|
||||
|
|
|
|||
Ładowanie…
Reference in New Issue