MUTE sound when playing in the decoding mode of video and audio streams from a video file
I am using two MediaCodec to decode video and audio streams. My ultimate goal is to create a new video file (.mp4 or .wmv file) after adjusting or changing the sound. Right now, I am just testing how to decode video and audio from a .wmv video file during playback.
The problem I am currently facing is that the video file is played with MediaPlayer while decoding video and audio streams, but there is no sound.
Does anyone know what is causing this?
Here is what I have tried so far.
DecodingActivity.java (SurfaceView in activity_decoding.xml)
public class DecodingActivity extends AppCompatActivity implements SurfaceHolder.Callback, MediaPlayer.OnPreparedListener{
private static final String TAG = "DecodingActivity";
private static final String FILE_PATH = Environment.getExternalStorageDirectory() + "/test.mp4";
private SurfaceView mSurfaceView;
private MediaPlayer mMediaPlayer;
private SurfaceHolder mSurfaceHolder;
private VideoDecoderThread mVideoDecoder;
private AudioDecoderThread mAudioDecoder;
@Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
requestWindowFeature(Window.FEATURE_NO_TITLE);
getWindow().setFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN,
WindowManager.LayoutParams.FLAG_FULLSCREEN);
setContentView(R.layout.activity_decoding);
mSurfaceView = (SurfaceView)findViewById(R.id.surface_view);
mSurfaceHolder = mSurfaceView.getHolder();
mSurfaceHolder.addCallback(DecodingActivity.this);
mVideoDecoder = new VideoDecoderThread();
mAudioDecoder = new AudioDecoderThread();
}
@Override
public void surfaceCreated(SurfaceHolder holder) {
mMediaPlayer = new MediaPlayer();
mMediaPlayer.setDisplay(mSurfaceHolder);
try {
mMediaPlayer.setDataSource(FILE_PATH);
//mMediaPlayer.prepare(); // this causes IllegalException Error
mMediaPlayer.setAudioStreamType(AudioManager.STREAM_MUSIC);
mMediaPlayer.setOnPreparedListener(DecodingActivity.this);
} catch (IOException e) {
e.printStackTrace();
}
}
@Override
public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) {
Log.d(TAG,"mAudioDecoder: "+mAudioDecoder);
Log.d(TAG,"mVideoDecoder: "+mVideoDecoder);
if (mVideoDecoder != null && mAudioDecoder != null) {
if (mVideoDecoder.init(holder.getSurface(), FILE_PATH)) {
mVideoDecoder.start();
mAudioDecoder.startPlay(FILE_PATH);
} else {
mVideoDecoder = null;
mAudioDecoder = null;
}
}
}
@Override
public void surfaceDestroyed(SurfaceHolder holder) {
if (mVideoDecoder != null) {
mVideoDecoder.close();
}
}
@Override
public void onPrepared(MediaPlayer mp) {
mMediaPlayer.start();
}
@Override
protected void onPause() {
super.onPause();
releaseMediaPlayer();
}
@Override
protected void onStop() {
super.onStop();
mAudioDecoder.stop();
}
@Override
protected void onDestroy() {
super.onDestroy();
releaseMediaPlayer();
}
private void releaseMediaPlayer() {
if (mMediaPlayer != null) {
mMediaPlayer.release();
mMediaPlayer = null;
}
}
}
AudioDecoderThread.java
public class AudioDecoderThread {
private static final String TAG = "AudioDecoder";
private static final int TIMEOUT_US = 1000;
private MediaExtractor mExtractor;
private MediaCodec mDecoder;
private boolean eosReceived;
private int mSampleRate = 0;
/**
*
* @param
*/
public void startPlay(String path) {
eosReceived = false;
mExtractor = new MediaExtractor();
try {
mExtractor.setDataSource(path);
} catch (IOException e) {
e.printStackTrace();
}
int channel = 0;
for (int i = 0; i < mExtractor.getTrackCount(); i++) {
MediaFormat format = mExtractor.getTrackFormat(i);
String mime = format.getString(MediaFormat.KEY_MIME);
if (mime.startsWith("audio/")) {
mExtractor.selectTrack(i);
Log.d(TAG, "format : " + format);
//{aac-profile=2, mime=audio/mp4a-latm, channel-count=2, max-input-size=1572864, isDMCMMExtractor=1, durationUs=35526530, csd-0=java.nio.ByteArrayBuffer[position=0,limit=2,capacity=2], encoder-delay=2112, encoder-padding=588, sample-rate=44100}
ByteBuffer csd = format.getByteBuffer("csd-0"); //csd-0=java.nio.ByteArrayBuffer[position=0,limit=2,capacity=2]
for (int k = 0; k < csd.capacity(); ++k) {
Log.e(TAG, "csd : " + csd.array()[k]);
}
mSampleRate = format.getInteger(MediaFormat.KEY_SAMPLE_RATE);
channel = format.getInteger(MediaFormat.KEY_CHANNEL_COUNT);
Log.d(TAG, "mSampleRate : " + mSampleRate); // 44100
Log.d(TAG, "channel : " + channel); // 2
break;
}
}
MediaFormat format = makeAACCodecSpecificData(MediaCodecInfo.CodecProfileLevel.AACObjectLC, mSampleRate, channel);
Log.d(TAG, "format[MediaFormat]: " + format);
if (format == null)
return;
try {
mDecoder = MediaCodec.createDecoderByType("audio/mp4a-latm");
} catch (IOException e) {
e.printStackTrace();
}
mDecoder.configure(format, null, null, 0);
if (mDecoder == null) {
Log.e("DecodeActivity", "Can't find video info!");
return;
}
mDecoder.start();
new Thread(AACDecoderAndPlayRunnable).start();
}
/**
* The code profile, Sample rate, channel Count is used to
* produce the AAC Codec SpecificData.
* Android 4.4.2/frameworks/av/media/libstagefright/avc_utils.cpp refer
* to the portion of the code written.
*
* MPEG-4 Audio refer : http://wiki.multimedia.cx/index.php?title=MPEG-4_Audio#Audio_Specific_Config
*
* @param audioProfile is MPEG-4 Audio Object Types
* @param sampleRate
* @param channelConfig
* @return MediaFormat
*/
private MediaFormat makeAACCodecSpecificData(int audioProfile, int sampleRate, int channelConfig) {
MediaFormat format = new MediaFormat();
format.setString(MediaFormat.KEY_MIME, "audio/mp4a-latm");
format.setInteger(MediaFormat.KEY_SAMPLE_RATE, sampleRate);
format.setInteger(MediaFormat.KEY_CHANNEL_COUNT, channelConfig);
int samplingFreq[] = {
96000, 88200, 64000, 48000, 44100, 32000, 24000, 22050,
16000, 12000, 11025, 8000
};
int sampleIndex = -1;
for (int i = 0; i < samplingFreq.length; ++i) {
if (samplingFreq[i] == sampleRate) {
Log.d(TAG, "kSamplingFreq " + samplingFreq[i] + " i : " + i); // kSamplingFreq 44100 i : 4
sampleIndex = i;
}
}
if (sampleIndex == -1) {
return null;
}
ByteBuffer csd = ByteBuffer.allocate(2);
csd.put((byte) ((audioProfile << 3) | (sampleIndex >> 1)));
csd.position(1);
csd.put((byte) ((byte) ((sampleIndex << 7) & 0x80) | (channelConfig << 3)));
csd.flip();
format.setByteBuffer("csd-0", csd); // add csd-0
for (int k = 0; k < csd.capacity(); ++k) {
Log.e(TAG, "csd : " + csd.array()[k]);
}
return format;
}
Runnable AACDecoderAndPlayRunnable = new Runnable() {
@Override
public void run() {
AACDecoderAndPlay();
}
};
/**
* After decoding AAC, Play using Audio Track.
*/
public void AACDecoderAndPlay() {
ByteBuffer[] inputBuffers = mDecoder.getInputBuffers();
ByteBuffer[] outputBuffers = mDecoder.getOutputBuffers();
Log.d(TAG, "inputBuffers: "+inputBuffers);
Log.d(TAG, "outputBuffers: "+outputBuffers);
BufferInfo info = new BufferInfo();
int buffsize = AudioTrack.getMinBufferSize(mSampleRate, AudioFormat.CHANNEL_OUT_STEREO, AudioFormat.ENCODING_PCM_16BIT);
Log.d(TAG, "buffsize: "+buffsize); // 14176
AudioTrack audioTrack = new AudioTrack(AudioManager.STREAM_MUSIC, mSampleRate,
AudioFormat.CHANNEL_OUT_STEREO,
AudioFormat.ENCODING_PCM_16BIT,
buffsize,
AudioTrack.MODE_STREAM);
//Log.d(TAG, "audioTrack: "+audioTrack); //android.media.AudioTrack@4be0ac7
audioTrack.play();
Log.d(TAG, "------------------------------------------------: ");
while (!eosReceived) {
int inIndex = mDecoder.dequeueInputBuffer(TIMEOUT_US); // 버퍼 index
Log.d(TAG, "inIndex: "+inIndex);
if (inIndex > -1) {
ByteBuffer buffer = inputBuffers[inIndex];
int sampleSize = mExtractor.readSampleData(buffer, 0);
Log.d(TAG, "buffer: "+buffer);
Log.d(TAG, "sampleSize: "+sampleSize);
if (sampleSize < 0) {
// We shouldn't stop the playback at this point, just pass the EOS
// flag to mDecoder, we will get it again from the
// dequeueOutputBuffer
Log.d(TAG, "InputBuffer BUFFER_FLAG_END_OF_STREAM");
mDecoder.queueInputBuffer(inIndex, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM);
} else {
mDecoder.queueInputBuffer(inIndex, 0, sampleSize, mExtractor.getSampleTime(), 0);
mExtractor.advance();
}
int outIndex = mDecoder.dequeueOutputBuffer(info, TIMEOUT_US);
Log.d(TAG, "outIndex: "+outIndex);
switch (outIndex) { // outputBufferIndex
case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED:
Log.d(TAG, "INFO_OUTPUT_BUFFERS_CHANGED");
outputBuffers = mDecoder.getOutputBuffers();
break;
case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED:
MediaFormat format = mDecoder.getOutputFormat();
Log.d(TAG, "New format " + format);
audioTrack.setPlaybackRate(format.getInteger(MediaFormat.KEY_SAMPLE_RATE));
break;
case MediaCodec.INFO_TRY_AGAIN_LATER:
Log.d(TAG, "dequeueOutputBuffer timed out!");
break;
default:
ByteBuffer outBuffer = outputBuffers[outIndex];
Log.v(TAG, "We can't use this buffer but render it due to the API limit, " + outBuffer);
final byte[] chunk = new byte[info.size];
outBuffer.get(chunk); // Read the buffer all at once
outBuffer.clear(); // ** MUST DO!!! OTHERWISE THE NEXT TIME YOU GET THIS SAME BUFFER BAD THINGS WILL HAPPEN
audioTrack.write(chunk, info.offset, info.offset + info.size); // AudioTrack write data
mDecoder.releaseOutputBuffer(outIndex, false);
break;
}
// All decoded frames have been rendered, we can stop playing now
if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
Log.d(TAG, "OutputBuffer BUFFER_FLAG_END_OF_STREAM");
break;
}
}
}
mDecoder.stop();
mDecoder.release();
mDecoder = null;
mExtractor.release();
mExtractor = null;
audioTrack.stop();
audioTrack.release();
audioTrack = null;
}
public void stop() {
eosReceived = true;
}
}
VideoDecoderThread.java
public class VideoDecoderThread extends Thread {
private static final String VIDEO = "video/";
private static final String TAG = "VideoDecoder";
private MediaExtractor mExtractor;
private MediaCodec mDecoder;
private boolean eosReceived;
public boolean init(Surface surface, String filePath) {
eosReceived = false;
try {
mExtractor = new MediaExtractor();
mExtractor.setDataSource(filePath);
for (int i = 0; i < mExtractor.getTrackCount(); i++) {
MediaFormat format = mExtractor.getTrackFormat(i);
String mime = format.getString(MediaFormat.KEY_MIME);
Log.d(TAG, "mime : " + mime); // video/avc
if (mime.startsWith(VIDEO)) {
mExtractor.selectTrack(i);
mDecoder = MediaCodec.createDecoderByType(mime);
try {
Log.d(TAG, "format : " + format);
mDecoder.configure(format, surface, null, 0 /* Decoder */);
} catch (IllegalStateException e) {
Log.e(TAG, "codec '" + mime + "' failed configuration. " + e);
return false;
}
mDecoder.start();
break;
}
}
} catch (IOException e) {
e.printStackTrace();
}
return true;
}
@Override
public void run() {
BufferInfo info = new BufferInfo();
ByteBuffer[] inputBuffers = mDecoder.getInputBuffers();
mDecoder.getOutputBuffers();
//Log.d(TAG, "mDecoder.getInputBuffers(); : " + mDecoder.getInputBuffers());
//Log.d(TAG, "mDecoder.getOutputBuffers(); : " + mDecoder.getOutputBuffers());
boolean isInput = true;
boolean first = false;
long startWhen = 0;
while (!eosReceived) {
if (isInput) {
int inputIndex = mDecoder.dequeueInputBuffer(10000);
if (inputIndex >= 0) {
// fill inputBuffers[inputBufferIndex] with valid data
ByteBuffer inputBuffer = inputBuffers[inputIndex];
int sampleSize = mExtractor.readSampleData(inputBuffer, 0);
if (mExtractor.advance() && sampleSize > 0) {
mDecoder.queueInputBuffer(inputIndex, 0, sampleSize, mExtractor.getSampleTime(), 0);
} else {
Log.d(TAG, "InputBuffer BUFFER_FLAG_END_OF_STREAM");
mDecoder.queueInputBuffer(inputIndex, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM);
isInput = false;
}
}
}
int outIndex = mDecoder.dequeueOutputBuffer(info, 10000);
switch (outIndex) {
case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED:
Log.d(TAG, "INFO_OUTPUT_BUFFERS_CHANGED");
mDecoder.getOutputBuffers();
break;
case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED:
Log.d(TAG, "INFO_OUTPUT_FORMAT_CHANGED format : " + mDecoder.getOutputFormat());
break;
case MediaCodec.INFO_TRY_AGAIN_LATER:
// Log.d(TAG, "INFO_TRY_AGAIN_LATER");
break;
default:
if (!first) {
startWhen = System.currentTimeMillis();
first = true;
}
try {
long sleepTime = (info.presentationTimeUs / 1000) - (System.currentTimeMillis() - startWhen);
Log.d(TAG, "info.presentationTimeUs : " + (info.presentationTimeUs / 1000) + " playTime: " + (System.currentTimeMillis() - startWhen) + " sleepTime : " + sleepTime);
if (sleepTime > 0)
Thread.sleep(sleepTime);
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
mDecoder.releaseOutputBuffer(outIndex, true /* Surface init */);
break;
}
// All decoded frames have been rendered, we can stop playing now
if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
Log.d(TAG, "OutputBuffer BUFFER_FLAG_END_OF_STREAM");
break;
}
}
mDecoder.stop();
mDecoder.release();
mExtractor.release();
}
public void close() {
eosReceived = true;
}
}
+3
source to share
No one has answered this question yet
Check out similar questions: