Generate mp4 video with image files using MediaCodec in Android

Use MediaCodec in Android to encode images to video.

The following code is wrote based on the examples from http://www.bigflake.com/mediacodec/, please refer it for more information.

/**
 * Generates a series of video frames, encodes them, decodes them, and tests for
 * significant divergence from the original.
 */
public class EncodeDecode
{
	private static final String TAG = "EncodeDecode";
	private static final boolean VERBOSE = false; // lots of logging
	// parameters for the encoder
	private static final String MIME_TYPE = "video/avc"; // H.264 Advanced Video
															// Coding
	private static final int FRAME_RATE = 10; // 10fps
	private static final int IFRAME_INTERVAL = 10; // 10 seconds between
													// I-frames
	// size of a frame, in pixels
	private int mWidth = -1;
	private int mHeight = -1;
	// bit rate, in bits per second
	private int mBitRate = -1;
	// largest color component delta seen (i.e. actual vs. expected)
	private int mLargestColorDelta;

	private File outputFile = null;
	private MediaCodec mEncoder;
	private MediaMuxer mMuxer;
	private int mTrackIndex;
	private boolean mMuxerStarted;
	private ArrayList<File> frames;

	public EncodeDecode(ArrayList<File> frames, File outputFile)
	{
		this.frames = frames;
		this.outputFile = outputFile;
	}

	/**
	 * Tests streaming of AVC video through the encoder and decoder. Data is
	 * encoded from a series of byte[] buffers and decoded into Surfaces. The
	 * output is checked for validity.
	 */
	public boolean encodeDecodeVideoFromBufferToSurface(int width, int height,
			int bitRate) throws Throwable
	{
		setParameters(width, height, bitRate);
		return encodeDecodeVideoFromBuffer();
	}

	/**
	 * Sets the desired frame size and bit rate.
	 */
	private void setParameters(int width, int height, int bitRate)
	{
		if ((width % 16) != 0 || (height % 16) != 0)
		{
			Log.w(TAG, "WARNING: width or height not multiple of 16");
		}
		mWidth = width;
		mHeight = height;
		mBitRate = bitRate;
	}

	/**
	 * Tests encoding and subsequently decoding video from frames generated into
	 * a buffer.
	 */
	@SuppressLint("InlinedApi")
	public boolean encodeDecodeVideoFromBuffer()
			throws Exception
	{
		mLargestColorDelta = -1;
		boolean result = true;
		try
		{
			MediaCodecInfo codecInfo = selectCodec(MIME_TYPE);
			if (codecInfo == null)
			{
				// Don't fail CTS if they don't have an AVC codec
				Log.e(TAG, "Unable to find an appropriate codec for "
						+ MIME_TYPE);
				return false;
			}
			if (VERBOSE)
				Log.d(TAG, "found codec: " + codecInfo.getName());
			int colorFormat;
			try
			{
				colorFormat = selectColorFormat(codecInfo, MIME_TYPE);
			} catch (Exception e)
			{
				colorFormat = MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420SemiPlanar;
			}
			if (VERBOSE)
				Log.d(TAG, "found colorFormat: " + colorFormat);
			// We avoid the device-specific limitations on width and height by
			// using values that
			// are multiples of 16, which all tested devices seem to be able to
			// handle.
			MediaFormat format = MediaFormat.createVideoFormat(MIME_TYPE,
					mWidth, mHeight);
			// Set some properties. Failing to specify some of these can cause
			// the MediaCodec
			// configure() call to throw an unhelpful exception.
			format.setInteger(MediaFormat.KEY_COLOR_FORMAT, colorFormat);
			format.setInteger(MediaFormat.KEY_BIT_RATE, mBitRate);
			format.setInteger(MediaFormat.KEY_FRAME_RATE, FRAME_RATE);
			format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, IFRAME_INTERVAL);
			if (VERBOSE)
				Log.d(TAG, "format: " + format);
			// Create a MediaCodec for the desired codec, then configure it as
			// an encoder with
			// our desired properties.
			mEncoder = MediaCodec.createByCodecName(codecInfo.getName());
			mEncoder.configure(format, null, null,
					MediaCodec.CONFIGURE_FLAG_ENCODE);
			mEncoder.start();
			// Create a MediaCodec for the decoder, just based on the MIME type.
			// The various
			// format details will be passed through the csd-0 meta-data later
			// on.
			String outputPath = outputFile.getAbsolutePath();
			try
			{
				mMuxer = new MediaMuxer(outputPath,
						MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
			} catch (IOException ioe)
			{
				// throw new RuntimeException("MediaMuxer creation failed",
				// ioe);
				ioe.printStackTrace();
			}
			result = doEncodeDecodeVideoFromBuffer(mEncoder, colorFormat);
		} finally
		{
			if (mEncoder != null)
			{
				mEncoder.stop();
				mEncoder.release();
			}
			if (mMuxer != null)
			{
				mMuxer.stop();
				mMuxer.release();
			}
			if (VERBOSE)
				Log.i(TAG, "Largest color delta: " + mLargestColorDelta);
		}
		return result;
	}

	/**
	 * Returns the first codec capable of encoding the specified MIME type, or
	 * null if no match was found.
	 */
	private static MediaCodecInfo selectCodec(String mimeType)
	{
		int numCodecs = MediaCodecList.getCodecCount();
		for (int i = 0; i < numCodecs; i++)
		{
			MediaCodecInfo codecInfo = MediaCodecList.getCodecInfoAt(i);
			if (!codecInfo.isEncoder())
			{
				continue;
			}
			String[] types = codecInfo.getSupportedTypes();
			for (int j = 0; j < types.length; j++)
			{
				if (types[j].equalsIgnoreCase(mimeType))
				{
					return codecInfo;
				}
			}
		}
		return null;
	}

	/**
	 * Returns a color format that is supported by the codec and by this test
	 * code. If no match is found, this throws a test failure -- the set of
	 * formats known to the test should be expanded for new platforms.
	 */
	private static int selectColorFormat(MediaCodecInfo codecInfo,
			String mimeType)
	{
		MediaCodecInfo.CodecCapabilities capabilities = codecInfo
				.getCapabilitiesForType(mimeType);
		for (int i = 0; i < capabilities.colorFormats.length; i++)
		{
			int colorFormat = capabilities.colorFormats[i];
			if (isRecognizedFormat(colorFormat))
			{
				return colorFormat;
			}
		}
		return 0; // not reached
	}

	/**
	 * Returns true if this is a color format that this test code understands
	 * (i.e. we know how to read and generate frames in this format).
	 */
	private static boolean isRecognizedFormat(int colorFormat)
	{
		switch (colorFormat)
		{
		// these are the formats we know how to handle for
			case MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Planar:
			case MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420PackedPlanar:
			case MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420SemiPlanar:
			case MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420PackedSemiPlanar:
			case MediaCodecInfo.CodecCapabilities.COLOR_TI_FormatYUV420PackedSemiPlanar:
				return true;
			default:
				return false;
		}
	}

	/**
	 * Does the actual work for encoding frames from buffers of byte[].
	 */
	@SuppressLint("InlinedApi")
	private boolean doEncodeDecodeVideoFromBuffer(MediaCodec encoder,
			int encoderColorFormat)
	{
		final int TIMEOUT_USEC = 10000;
		ByteBuffer[] encoderInputBuffers = encoder.getInputBuffers();
		MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
		int generateIndex = 0;
		// yuv format
		byte[] frameData = new byte[mWidth * mHeight * 3 / 2];
		// Loop until the output side is done.
		boolean inputDone = false;
		// If we're not done submitting frames, generate a new one and submit
		// it. By
		// doing this on every loop we're working to ensure that the encoder
		// always has
		// work to do.
		while (!inputDone)
		{
			int inputBufIndex = encoder.dequeueInputBuffer(TIMEOUT_USEC);
			if (inputBufIndex >= 0)
			{
				long ptsUsec = computePresentationTime(generateIndex);
				if (generateIndex >= frames.size())
				{
					// Send an empty frame with the end-of-stream flag set. If
					// we set EOS
					// on a frame with data, that frame data will be ignored,
					// and the
					// output will be short one frame.
					encoder.queueInputBuffer(inputBufIndex, 0, 0, ptsUsec,
							MediaCodec.BUFFER_FLAG_END_OF_STREAM);
					inputDone = true;
					drainEncoder(true, info);
				} else
				{
					try
					{
						generateFrame(generateIndex, encoderColorFormat,
								frameData);
					} catch (Exception e)
					{
						Log.i(TAG, "meet a different type of image");
						Arrays.fill(frameData, (byte) 0);
					}
					if (VERBOSE)
						Log.i(TAG, "generateIndex: " + generateIndex
								+ ", size: " + frames.size());
					ByteBuffer inputBuf = encoderInputBuffers[inputBufIndex];
					// the buffer should be sized to hold one full frame
					inputBuf.clear();
					inputBuf.put(frameData);
					encoder.queueInputBuffer(inputBufIndex, 0,
							frameData.length, ptsUsec, 0);
					drainEncoder(false, info);
				}
				generateIndex++;
			} else
			{
				// either all in use, or we timed out during initial setup
				if (VERBOSE)
					Log.i(TAG, "input buffer not available");
			}
		}
		return true;
	}

	/**
	 * use Muxer to generate mp4 file with data from encoder
	 * 
	 * @param endOfStream
	 *            if this is the last frame
	 * @param mBufferInfo
	 *            the BufferInfo of data from encoder
	 */
	private void drainEncoder(boolean endOfStream, BufferInfo mBufferInfo)
	{
		final int TIMEOUT_USEC = 10000;

		if (endOfStream)
		{
			try
			{
				mEncoder.signalEndOfInputStream();
			} catch (Exception e)
			{
			}
		}

		ByteBuffer[] encoderOutputBuffers = mEncoder.getOutputBuffers();
		while (true)
		{
			int encoderStatus = mEncoder.dequeueOutputBuffer(mBufferInfo,
					TIMEOUT_USEC);
			if (encoderStatus == MediaCodec.INFO_TRY_AGAIN_LATER)
			{
				// no output available yet
				if (!endOfStream)
				{
					break; // out of while
				} else
				{
					if (VERBOSE)
						Log.i(TAG, "no output available, spinning to await EOS");
				}
			} else if (encoderStatus == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED)
			{
				// not expected for an encoder
				encoderOutputBuffers = mEncoder.getOutputBuffers();
			} else if (encoderStatus == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED)
			{
				// should happen before receiving buffers, and should only
				// happen once
				if (mMuxerStarted)
				{
					throw new RuntimeException("format changed twice");
				}
				MediaFormat newFormat = mEncoder.getOutputFormat();
				if (VERBOSE)
					Log.i(TAG, "encoder output format changed: " + newFormat);

				// now that we have the Magic Goodies, start the muxer
				mTrackIndex = mMuxer.addTrack(newFormat);
				mMuxer.start();
				mMuxerStarted = true;
			} else if (encoderStatus < 0)
			{
				if (VERBOSE)
					Log.i(TAG,
							"unexpected result from encoder.dequeueOutputBuffer: "
									+ encoderStatus);
			} else
			{
				ByteBuffer encodedData = encoderOutputBuffers[encoderStatus];
				if (encodedData == null)
				{
					throw new RuntimeException("encoderOutputBuffer "
							+ encoderStatus + " was null");
				}

				if ((mBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0)
				{
					// The codec config data was pulled out and fed to the muxer
					// when we got
					// the INFO_OUTPUT_FORMAT_CHANGED status. Ignore it.
					if (VERBOSE)
						Log.d(TAG, "ignoring BUFFER_FLAG_CODEC_CONFIG");
					mBufferInfo.size = 0;
				}

				if (mBufferInfo.size != 0)
				{
					if (!mMuxerStarted)
					{
						throw new RuntimeException("muxer hasn't started");
					}

					// adjust the ByteBuffer values to match BufferInfo
					encodedData.position(mBufferInfo.offset);
					encodedData.limit(mBufferInfo.offset + mBufferInfo.size);

					if (VERBOSE)
						Log.d(TAG, "BufferInfo: " + mBufferInfo.offset + ","
								+ mBufferInfo.size + ","
								+ mBufferInfo.presentationTimeUs);

					try
					{
						mMuxer.writeSampleData(mTrackIndex, encodedData,
								mBufferInfo);
					} catch (Exception e)
					{
						Log.i(TAG, "Too many frames");
					}
				}

				mEncoder.releaseOutputBuffer(encoderStatus, false);

				if ((mBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0)
				{
					if (!endOfStream)
					{
						if (VERBOSE)
							Log.i(TAG, "reached end of stream unexpectedly");
					} else
					{
						if (VERBOSE)
							Log.i(TAG, "end of stream reached");
					}
					break; // out of while
				}
			}
		}
	}

	/**
	 * Generates data for frame N into the supplied buffer.
	 */
	private void generateFrame(int frameIndex, int colorFormat, byte[] frameData)
	{
		// Set to zero. In YUV this is a dull green.
		Arrays.fill(frameData, (byte) 0);

		Mat mat = Highgui.imread(frames.get(frameIndex).getAbsolutePath());

//		Mat dst = new Mat(mWidth, mHeight * 3 / 2, CvType.CV_8UC1);
		Mat dst = new Mat();
		Imgproc.cvtColor(mat, dst, Imgproc.COLOR_RGBA2YUV_I420);

		// use array instead of mat to improve the speed
		dst.get(0, 0, frameData);

		byte[] temp = frameData.clone();
		int margin = mHeight / 4;
		int location = mHeight;
		int step = 0;
		for (int i = mHeight; i < mHeight + margin; i++)
		{
			for (int j = 0; j < mWidth; j++)
			{
				byte uValue = temp[i * mWidth + j];
				byte vValue = temp[(i + margin) * mWidth + j];

				frameData[location * mWidth + step] = uValue;
				frameData[location * mWidth + step + 1] = vValue;
				step += 2;
				if (step >= mWidth)
				{
					location++;
					step = 0;
				}
			}
		}
	}

	/**
	 * Generates the presentation time for frame N, in microseconds.
	 */
	private static long computePresentationTime(int frameIndex)
	{
		long value = frameIndex;
		return 132 + value * 1000000 / FRAME_RATE;
	}
}

The code of dealing with arrays in "generateFrame()" is used for changing the format of images between YUV420p(I420) and YUV420sp (they are both YUV formats but with different arrange of U and V).

The following code show how to use this class:

/**
	 * make video file with images (hardware encoding)
	 * 
	 * @param images
	 *            the images
	 * @param location
	 *            the path
	 * @param name
	 *            the video file name
	 * @param width
	 *            the width of video
	 * @param height
	 *            the height of video
	 * @param bitRate
	 *            the bitRate of video
	 * @return the path of video file
	 */
	public String hardwareMakeVideo(ArrayList<File> images, String location,
			String name, int width, int height, int bitRate)
	{
		File directory = new File(location);
		if (!directory.exists())
		{
			directory.mkdir();
		}
		File file = new File(directory, name + ".mp4");
		try
		{
			EncodeDecode encodeDecoder = new EncodeDecode(images, file);
			encodeDecoder.encodeDecodeVideoFromBufferToSurface(width, height,
					bitRate);
		} catch (Throwable e)
		{
			e.printStackTrace();
		}
		return file.getAbsolutePath();
	}

Basically, MediaCodec is faster than Jcodec on encoding images to videos.