Android音视频开发图解

Android音视频开发学习路线，具备这些知识

摄像头相关配置参数

实时获取摄像头原始数据

NV21原始数据编码操作的线程，编码h.264格式数据

public void startEncoderThread(){
        Thread EncoderThread = new Thread(new Runnable() {

            @SuppressLint("NewApi")
            @Override
            public void run() {
                isRuning = true;
                byte[] input = null;
                long pts =  0;
                long generateIndex = 0;

                while (isRuning) {
                    if (CameraSurfaceView.YUVQueue.size() >0){
                        input = CameraSurfaceView.YUVQueue.poll();
                        byte[] yuv420sp = new byte[m_width*m_height*3/2];
                        byte[] yuv420 = new byte[m_width*m_height*3/2];
                        VideoYUVUtil.NV21ToNV12(input,yuv420sp,m_width,m_height);
//                        VideoYUVUtil.YUV420spRotate90Clockwise(yuv420sp,yuv420,m_width,m_height);
                        input = yuv420sp;
                    }
                    if (input != null) {
                        try {
                            long startMs = System.currentTimeMillis();
                            ByteBuffer[] inputBuffers = mediaCodec.getInputBuffers();
                            ByteBuffer[] outputBuffers = mediaCodec.getOutputBuffers();
                            int inputBufferIndex = mediaCodec.dequeueInputBuffer(-1);
                            if (inputBufferIndex >= 0) {
                                pts = computePresentationTime(generateIndex);
                                ByteBuffer inputBuffer = inputBuffers[inputBufferIndex];
                                inputBuffer.clear();
                                inputBuffer.put(input);
                                mediaCodec.queueInputBuffer(inputBufferIndex, 0, input.length, pts, 0);
                                generateIndex += 1;
                            }

                            MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo();
                            int outputBufferIndex = mediaCodec.dequeueOutputBuffer(bufferInfo, TIMEOUT_USEC);
                            while (outputBufferIndex >= 0) {
                                //Log.i("AvcEncoder", "Get H264 Buffer Success! flag = "+bufferInfo.flags+",pts = "+bufferInfo.presentationTimeUs+"");
                                ByteBuffer outputBuffer = outputBuffers[outputBufferIndex];
                                byte[] outData = new byte[bufferInfo.size];
                                outputBuffer.get(outData);
                                if(bufferInfo.flags == 2){
                                    configbyte = new byte[bufferInfo.size];
                                    configbyte = outData;
                                }else if(bufferInfo.flags == 1){
                                    byte[] keyframe = new byte[bufferInfo.size + configbyte.length];
                                    System.arraycopy(configbyte, 0, keyframe, 0, configbyte.length);
                                    System.arraycopy(outData, 0, keyframe, configbyte.length, outData.length);

                                    outputStream.write(keyframe, 0, keyframe.length);
                                    if(onEncoderVideoListener!=null){
                                        onEncoderVideoListener.encoderVideoData(keyframe,keyframe.length);
                                    }
                                }else{
                                    outputStream.write(outData, 0, outData.length);
                                    if(onEncoderVideoListener!=null){
                                        onEncoderVideoListener.encoderVideoData(outData,outData.length);
                                    }
                                }

                                mediaCodec.releaseOutputBuffer(outputBufferIndex, false);
                                outputBufferIndex = mediaCodec.dequeueOutputBuffer(bufferInfo, TIMEOUT_USEC);
                            }

                        } catch (Throwable t) {
                            t.printStackTrace();
                        }
                    } else {
                        try {
                            Thread.sleep(500);
                        } catch (InterruptedException e) {
                            e.printStackTrace();
                        }
                    }
                }
            }
        });
        EncoderThread.start();

    }

实时采集原始PCM音频数据的线程

原始PCM数据编码AAC音频数据

 /**
     * 编码AAC音频线程
     */
    public void startAudioEncodeAAC() {
        audioEncoderThread = new Thread() {
            @Override
            public void run() {
                try {
                 while (!audioEncoderLoop && !Thread.interrupted()) {
                    AudioData audioData = audioQueue.take();
                    byte[] buffer = audioData.audioData;
                if(audioData != null && encoder != null)
                {
                    int size = audioData.audioData.length;
                    //录音时间  size/ 采样率*声道数 * bits/8
                    recordTime += size * 1.0 / (audioSamplerate * 2 * (16 / 8));
//                    LogUtils.d("recordTime = " + recordTime);

                    int inputBufferindex = encoder.dequeueInputBuffer(0);
                    if(inputBufferindex >= 0)
                    {
                        ByteBuffer byteBuffer = encoder.getInputBuffers()[inputBufferindex];
                        byteBuffer.clear();
                        byteBuffer.put(buffer);
                        encoder.queueInputBuffer(inputBufferindex, 0, size, 0, 0);
                    }

                    int index = encoder.dequeueOutputBuffer(info, 0);
                    while(index >= 0)
                    {
                        try {
                            perpcmsize = info.size + 7;
                            outByteBuffer = new byte[perpcmsize];

                            ByteBuffer byteBuffer = encoder.getOutputBuffers()[index];
                            byteBuffer.position(info.offset);
                            byteBuffer.limit(info.offset + info.size);

                            addADtsHeader(outByteBuffer, perpcmsize, aacsamplerate);

                            byteBuffer.get(outByteBuffer, 7, info.size);
                            byteBuffer.position(info.offset);
                            if (audioEncderListener!=null){
                                audioEncderListener.encoderAudioData(outByteBuffer,perpcmsize);
                            }
                            encoder.releaseOutputBuffer(index, false);
                            index = encoder.dequeueOutputBuffer(info, 0);
                            outByteBuffer = null;
                        } catch (Exception e) {
                            e.printStackTrace();
                        }
                    }
                }}
                } catch (InterruptedException e) {
                    e.printStackTrace();
                }
            }
        };
        audioEncoderThread.start();
    }

编码好的h.264视频数据和AAC音频数据经过封包成数据包通过网络协议(rtmp)实时发送数据。

将接收到的视频h.264数据解码成原始数据并播放。

送入解码线程进行解码操作

H.264实时解码成原始数据并进行渲染画面

接收到的AAC音频数据送入解码音频的线程进行解码还原成原始PCM音频数据

/**
 * aac解码+播放
 */
public void decode(byte[] buf, int offset, int length) {
    //输入ByteBuffer
    ByteBuffer[] codecInputBuffers = mDecoder.getInputBuffers();
    //输出ByteBuffer
    ByteBuffer[] codecOutputBuffers = mDecoder.getOutputBuffers();
    //等待时间，0->不等待，-1->一直等待
    long kTimeOutUs = 0;
    try {
        //返回一个包含有效数据的input buffer的index,-1->不存在
        int inputBufIndex = mDecoder.dequeueInputBuffer(kTimeOutUs);
        if (inputBufIndex >= 0) {
            //获取当前的ByteBuffer
            ByteBuffer dstBuf = codecInputBuffers[inputBufIndex];
            //清空ByteBuffer
            dstBuf.clear();
            //填充数据
            dstBuf.put(buf, offset, length);
            //将指定index的input buffer提交给解码器
            mDecoder.queueInputBuffer(inputBufIndex, 0, length, 0, 0);
        }
        //编解码器缓冲区
        MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
        //返回一个output buffer的index，-1->不存在
        int outputBufferIndex = mDecoder.dequeueOutputBuffer(info, kTimeOutUs);
        if (outputBufferIndex < 0) {
            //记录解码失败的次数
            count++;
        }
        ByteBuffer outputBuffer;
        while (outputBufferIndex >= 0) {
            //获取解码后的ByteBuffer
            outputBuffer = codecOutputBuffers[outputBufferIndex];
            //用来保存解码后的数据
            byte[] outData = new byte[info.size];
            outputBuffer.get(outData);
            //清空缓存
            outputBuffer.clear();
            //播放解码后的数据
            mPlayer.playAudioTrack(outData, 0, info.size);
            //释放已经解码的buffer
            mDecoder.releaseOutputBuffer(outputBufferIndex, false);
            //解码未解完的数据
            outputBufferIndex = mDecoder.dequeueOutputBuffer(info, kTimeOutUs);
        }
    } catch (Exception e) {
        e.printStackTrace();
    }

将解码后的PCM送入AudioTrack进行播放，这个时候就有声音了。

 

因为音频和视频解码是分开的，所以在解码后，需要做音视频同步。通常有一些音视频同步算法。如在ffmepg中，以音频作为参考时钟，视频用于比较当前时钟和音频时钟的差值，如果快了，就要增大延迟，以便下一帧显示的晚一些；如果慢了，就要减少延迟，加快显示下一帧。

感谢各位关注。