MediaCodec解码得到Image
对于仅仅需要将视频切分为一帧一帧并保存为图片的用户来说,使用这种方法比bigflake的方法会快10倍左右,因为没有OpenGL渲染,以及转换为Bitmap的开销。而对于需要获得视频帧YUV格式数据的用户来说,这种方法能够直接得到YUV格式数据,中间没有数学运算,不会出现不必要的精度损失,而且,也是效率最高的。
package com. vision. opengl. decoder;
import android. graphics. Bitmap;
import android. graphics. BitmapFactory;
import android. graphics. ImageFormat;
import android. graphics. Rect;
import android. graphics. YuvImage;
import android. media. Image;
import android. media. MediaCodec;
import android. media. MediaCodecInfo;
import android. media. MediaExtractor;
import android. media. MediaFormat;
import android. util. Log;
import java. io. ByteArrayOutputStream;
import java. io. File;
import java. io. IOException;
import java. nio. ByteBuffer;
import java. util. concurrent. LinkedBlockingQueue;
public class VideoToFrames {
private static final String TAG = "VideoToFrames" ;
private static final boolean VERBOSE = false ;
private static final long DEFAULT_TIMEOUT_US = 10000 ;
private boolean stopDecode = false ;
private final LinkedBlockingQueue< byte[ ] > queue = new LinkedBlockingQueue< > ( 50 ) ;
public Callback2 callback;
public interface Callback2 {
void onFinishDecode ( ) ;
void onDecodeFrame ( int index, Bitmap bitmap) ;
}
public void setCallback ( Callback2 callback) {
this . callback = callback;
}
public void stopDecode ( ) {
stopDecode = true ;
}
public LinkedBlockingQueue< byte[ ] > decode ( String videoFilePath) {
new Thread ( ( ) -> videoDecode ( videoFilePath) ) . start ( ) ;
return queue;
}
private void videoDecode ( String videoFilePath) {
MediaExtractor extractor = null ;
MediaCodec decoder = null ;
try {
File videoFile = new File ( videoFilePath) ;
extractor = new MediaExtractor ( ) ;
extractor. setDataSource ( videoFile. toString ( ) ) ;
int trackIndex = selectTrack ( extractor) ;
if ( trackIndex < 0 ) {
throw new RuntimeException ( "No video track found in " + videoFilePath) ;
}
extractor. selectTrack ( trackIndex) ;
MediaFormat mediaFormat = extractor. getTrackFormat ( trackIndex) ;
String mime = mediaFormat. getString ( MediaFormat. KEY_MIME) ;
decoder = MediaCodec. createDecoderByType ( mime) ;
showSupportedColorFormat ( decoder. getCodecInfo ( ) . getCapabilitiesForType ( mime) ) ;
int decodeColorFormat = MediaCodecInfo. CodecCapabilities. COLOR_FormatYUV420Flexible;
if ( isColorFormatSupported ( decodeColorFormat, decoder. getCodecInfo ( ) . getCapabilitiesForType ( mime) ) ) {
mediaFormat. setInteger ( MediaFormat. KEY_COLOR_FORMAT, decodeColorFormat) ;
Log. i ( TAG, "set decode color format to type " + decodeColorFormat) ;
} else {
Log. i ( TAG, "unable to set decode color format, color format type " + decodeColorFormat + " not supported" ) ;
}
decodeFramesToImage ( decoder, extractor, mediaFormat) ;
decoder. stop ( ) ;
} catch ( IOException | InterruptedException e) {
e. printStackTrace ( ) ;
} finally {
if ( decoder != null ) {
decoder. stop ( ) ;
decoder. release ( ) ;
}
if ( extractor != null ) {
extractor. release ( ) ;
}
}
}
private void showSupportedColorFormat ( MediaCodecInfo. CodecCapabilities caps) {
System. out. print ( "supported color format: " ) ;
for ( int c : caps. colorFormats) {
System. out. print ( c + "\t" ) ;
}
System. out. println ( ) ;
}
private boolean isColorFormatSupported ( int colorFormat, MediaCodecInfo. CodecCapabilities caps) {
for ( int c : caps. colorFormats) {
if ( c == colorFormat) {
return true ;
}
}
return false ;
}
private void decodeFramesToImage ( MediaCodec decoder, MediaExtractor extractor, MediaFormat mediaFormat) throws InterruptedException {
MediaCodec. BufferInfo info = new MediaCodec. BufferInfo ( ) ;
boolean sawInputEOS = false ;
boolean sawOutputEOS = false ;
decoder. configure ( mediaFormat, null , null , 0 ) ;
decoder. start ( ) ;
int outputFrameCount = 0 ;
while ( ! sawOutputEOS && ! stopDecode) {
if ( ! sawInputEOS) {
int inputBufferId = decoder. dequeueInputBuffer ( DEFAULT_TIMEOUT_US) ;
if ( inputBufferId >= 0 ) {
ByteBuffer inputBuffer = decoder. getInputBuffer ( inputBufferId) ;
int sampleSize = extractor. readSampleData ( inputBuffer, 0 ) ;
if ( sampleSize < 0 ) {
decoder. queueInputBuffer ( inputBufferId, 0 , 0 , 0L , MediaCodec. BUFFER_FLAG_END_OF_STREAM) ;
sawInputEOS = true ;
} else {
long presentationTimeUs = extractor. getSampleTime ( ) ;
decoder. queueInputBuffer ( inputBufferId, 0 , sampleSize, presentationTimeUs, 0 ) ;
extractor. advance ( ) ;
}
}
}
int outputBufferId = decoder. dequeueOutputBuffer ( info, DEFAULT_TIMEOUT_US) ;
if ( outputBufferId >= 0 ) {
if ( ( info. flags & MediaCodec. BUFFER_FLAG_END_OF_STREAM) != 0 ) {
sawOutputEOS = true ;
}
boolean doRender = ( info. size != 0 ) ;
if ( doRender) {
outputFrameCount++ ;
Image image = decoder. getOutputImage ( outputBufferId) ;
compressToJpeg2 ( image) ;
image. close ( ) ;
decoder. releaseOutputBuffer ( outputBufferId, true ) ;
}
}
}
if ( callback != null ) {
callback. onFinishDecode ( ) ;
}
}
private static int selectTrack ( MediaExtractor extractor) {
int numTracks = extractor. getTrackCount ( ) ;
for ( int i = 0 ; i < numTracks; i++ ) {
MediaFormat format = extractor. getTrackFormat ( i) ;
String mime = format. getString ( MediaFormat. KEY_MIME) ;
if ( mime. startsWith ( "video/" ) ) {
if ( VERBOSE) {
Log. d ( TAG, "Extractor selected track " + i + " (" + mime + "): " + format) ;
}
return i;
}
}
return - 1 ;
}
private static boolean isImageFormatSupported ( Image image) {
int format = image. getFormat ( ) ;
switch ( format) {
case ImageFormat. YUV_420_888:
case ImageFormat. NV21:
case ImageFormat. YV12:
return true ;
}
return false ;
}
private static byte[ ] getDataFromImage ( Image image) {
if ( ! isImageFormatSupported ( image) ) {
throw new RuntimeException ( "can't convert Image to byte array, format " + image. getFormat ( ) ) ;
}
Rect crop = image. getCropRect ( ) ;
int format = image. getFormat ( ) ;
int width = crop. width ( ) ;
int height = crop. height ( ) ;
Image. Plane[ ] planes = image. getPlanes ( ) ;
byte[ ] data = new byte[ width * height * ImageFormat. getBitsPerPixel ( format) / 8 ] ;
byte[ ] rowData = new byte[ planes[ 0 ] . getRowStride ( ) ] ;
if ( VERBOSE) Log. v ( TAG, "get data from " + planes. length + " planes" ) ;
int channelOffset = 0 ;
int outputStride = 1 ;
for ( int i = 0 ; i < planes. length; i++ ) {
switch ( i) {
case 0 :
channelOffset = 0 ;
outputStride = 1 ;
break ;
case 1 :
channelOffset = width * height + 1 ;
outputStride = 2 ;
break ;
case 2 :
channelOffset = width * height;
outputStride = 2 ;
break ;
}
ByteBuffer buffer = planes[ i] . getBuffer ( ) ;
int rowStride = planes[ i] . getRowStride ( ) ;
int pixelStride = planes[ i] . getPixelStride ( ) ;
if ( VERBOSE) {
Log. v ( TAG, "pixelStride " + pixelStride) ;
Log. v ( TAG, "rowStride " + rowStride) ;
Log. v ( TAG, "width " + width) ;
Log. v ( TAG, "height " + height) ;
Log. v ( TAG, "buffer size " + buffer. remaining ( ) ) ;
}
int shift = ( i == 0 ) ? 0 : 1 ;
int w = width > > shift;
int h = height > > shift;
buffer. position ( rowStride * ( crop. top > > shift) + pixelStride * ( crop. left > > shift) ) ;
for ( int row = 0 ; row < h; row++ ) {
int length;
if ( pixelStride == 1 && outputStride == 1 ) {
length = w;
buffer. get ( data , channelOffset, length) ;
channelOffset += length;
} else {
length = ( w - 1 ) * pixelStride + 1 ;
buffer. get ( rowData, 0 , length) ;
for ( int col = 0 ; col < w; col++ ) {
data [ channelOffset] = rowData[ col * pixelStride] ;
channelOffset += outputStride;
}
}
if ( row < h - 1 ) {
buffer. position ( buffer. position ( ) + rowStride - length) ;
}
}
if ( VERBOSE) Log. v ( TAG, "Finished reading data from plane " + i) ;
}
return data ;
}
ByteArrayOutputStream outStream = new ByteArrayOutputStream ( ) ;
private Bitmap compressToJpeg2 ( Image image) throws InterruptedException {
outStream. reset ( ) ;
Rect rect = image. getCropRect ( ) ;
YuvImage yuvImage = new YuvImage ( getDataFromImage ( image) , ImageFormat. NV21, rect. width ( ) , rect. height ( ) , null ) ;
yuvImage. compressToJpeg ( rect, 100 , outStream) ;
byte[ ] bytes = outStream. toByteArray ( ) ;
queue. put ( bytes) ;
return null ;
}
}
MediaCodec通过Surface播放数据
package com. vision. editor. utils;
import java. io. IOException;
import java. nio. ByteBuffer;
import android. media. MediaCodec;
import android. media. MediaCodec. BufferInfo;
import android. media. MediaExtractor;
import android. media. MediaFormat;
import android. util. Log;
import android. view. Surface;
public class VideoDecoderThread extends Thread {
private static final String VIDEO = "video/" ;
private static final String TAG = "VideoDecoder" ;
private MediaExtractor mExtractor;
private MediaCodec mDecoder;
private boolean eosReceived;
public boolean init ( Surface surface, String filePath) {
eosReceived = false ;
try {
mExtractor = new MediaExtractor ( ) ;
mExtractor. setDataSource ( filePath) ;
Log. d ( TAG, "getTrackCount: " + mExtractor. getTrackCount ( ) ) ;
for ( int i = 0 ; i < mExtractor. getTrackCount ( ) ; i++ ) {
MediaFormat format = mExtractor. getTrackFormat ( i) ;
String mime = format. getString ( MediaFormat. KEY_MIME) ;
if ( mime. startsWith ( VIDEO) ) {
mExtractor. selectTrack ( i) ;
mDecoder = MediaCodec. createDecoderByType ( mime) ;
try {
Log. d ( TAG, "format : " + format) ;
mDecoder. configure ( format, surface, null , 0 ) ;
} catch ( IllegalStateException e) {
Log. e ( TAG, "codec '" + mime + "' failed configuration. " + e) ;
return false ;
}
mDecoder. start ( ) ;
break ;
}
}
} catch ( IOException e) {
e. printStackTrace ( ) ;
}
return true ;
}
@Override
public void run ( ) {
BufferInfo info = new BufferInfo ( ) ;
ByteBuffer[ ] inputBuffers = mDecoder. getInputBuffers ( ) ;
mDecoder. getOutputBuffers ( ) ;
boolean isInput = true ;
boolean first = false ;
long startWhen = 0 ;
while ( ! eosReceived) {
if ( isInput) {
int inputIndex = mDecoder. dequeueInputBuffer ( 10000 ) ;
if ( inputIndex >= 0 ) {
ByteBuffer inputBuffer = inputBuffers[ inputIndex] ;
int sampleSize = mExtractor. readSampleData ( inputBuffer, 0 ) ;
if ( mExtractor. advance ( ) && sampleSize > 0 ) {
mDecoder. queueInputBuffer ( inputIndex, 0 , sampleSize, mExtractor. getSampleTime ( ) , 0 ) ;
} else {
Log. d ( TAG, "InputBuffer BUFFER_FLAG_END_OF_STREAM" ) ;
mDecoder. queueInputBuffer ( inputIndex, 0 , 0 , 0 , MediaCodec. BUFFER_FLAG_END_OF_STREAM) ;
isInput = false ;
}
}
}
int outIndex = mDecoder. dequeueOutputBuffer ( info, 10000 ) ;
switch ( outIndex) {
case MediaCodec. INFO_OUTPUT_BUFFERS_CHANGED:
Log. d ( TAG, "INFO_OUTPUT_BUFFERS_CHANGED" ) ;
mDecoder. getOutputBuffers ( ) ;
break ;
case MediaCodec. INFO_OUTPUT_FORMAT_CHANGED:
Log. d ( TAG, "INFO_OUTPUT_FORMAT_CHANGED format : " + mDecoder. getOutputFormat ( ) ) ;
break ;
case MediaCodec. INFO_TRY_AGAIN_LATER:
break ;
default:
if ( ! first) {
startWhen = System. currentTimeMillis ( ) ;
first = true ;
}
try {
long sleepTime = ( info. presentationTimeUs / 1000 ) - ( System. currentTimeMillis ( ) - startWhen) ;
if ( sleepTime > 0 )
Thread. sleep ( sleepTime) ;
} catch ( InterruptedException e) {
e. printStackTrace ( ) ;
}
mDecoder. releaseOutputBuffer ( outIndex, true ) ;
break ;
}
if ( ( info. flags & MediaCodec. BUFFER_FLAG_END_OF_STREAM) != 0 ) {
Log. d ( TAG, "OutputBuffer BUFFER_FLAG_END_OF_STREAM" ) ;
break ;
}
}
mDecoder. stop ( ) ;
mDecoder. release ( ) ;
mExtractor. release ( ) ;
}
public void close ( ) {
eosReceived = true ;
}
}
参考资料