android4.0 stagefright RTP发送264视频
任务:usb camera -> 264encoder -> rtp -> pc vlc播放
1.任务分解:
集成usb camera到开发板
编写mediarecorder测试程序,测试整个摄像头通路
分析stagefright源码,以确定需要哪些开发工作
oRTP编译与移植+测试
实现基于oRTP 发送的MediaWriter
基于oRTP发送的MediaRecorder测试例子
pc端VLC的播放
2.具体实现
2.1 集成usb camera到开发板
内核里打开UVC配置选项(这个网上很多,这里不重复了),重新编译内核并烧写到目标板子
接上usb camera,通过dmesg查看驱动信息,
ls /dev/video* 查看设备的加载情况
我手上的开发板HAL层本身已经支持了usb camera所以这部分工作量很大的工作我不需要去实现。
打开Camera.apk测试摄像头通路,一切OK
2.2 编写mediarecorder测试程序,测试整个摄像头通路
使用eclipse 建立工程
main.xml (一个url框,一个surfaceview用于显示,三个button
activity :
public class MediaRecorder_001 extends Activity implements SurfaceHolder.Callback{
MediaPlayer player;
}
编译运行,点击运行‘音频视频录像’一切都很正常,录下来的my.mp4有音频视频,vlc播放正常
2.3 分析stagefright源码,以确定需要哪些开发工作
图1
图1为mediaplayer在android层中的类结构图,可看StagefrightPlayer只是IMediaPlayer是抽象定义StagefrightPlayer是其中一种实现方式,接下来重点分析StagefrightPlayer的结构
class DataSource //数据源,可来自http,rtsp,file
class MediaExtractor //相当ffmpeg中的demuxer组件集合
class MediaSource //表示一个媒体流,可以是压缩编码过的,如264流
//也可以表示一个解码过的,如rawvideo的 yuv420图像 //流,
struct MediaWriter //用于做录像用的,如mpeg4writer(直接记录成mp4文件)
这些类都在 frameworks/base/media/libstagefright 里实现,编译出来后生成libstagefright.so AwesomePlayer.cpp 是这些类的主要调用者,基本上看AwesomePlayer.cpp就可以看出调用流程来,代码也不复杂,这里不多说。
大概流程为: DataSource(读取文件)-->MediaExtractor(解复用,解出n个MediaSource),每个mediasource代码一个tracker比如264流+AAC流,各是一个MediaSource -->Decoder(也是解成MediaSource)-->MediaWriter或者Render 这样流程结束。
从上述分析看,我只需要实现一个MediaWriter的实现类并注册到stagefright播放器里就可以了,我们暂定为class RFC3984Writer:public MediaWriter
2.4 oRTP编译与移植+测试
264流通过RTP发送需要封装成RFC3984的封装包(UDP包)这部分功能在linphone里有现成的实现,所以做这个任务的最佳方式是从linphone里移植相关的库
。我们从linphone官网下载linphone for android的源码包,从中提取出oRTP库和相关的RFC3984功能模块,到目标平台上编译通过,得到libortp.so
把ortp库中的tests目录下的rtprecv.c和rtpsend.c编译并测试一下,一切都很正常。
2.5 实现基于oRTP发送的MediaWriter
现在是时候实现RFC3984Writer类了,我选定了一个google实现的
struct ARTPWriter : public MediaWriter 直接修改成
struct RFC3984Writer:public MediaWriter
RFC3984Writer.h
//add by kali@neobv.com
#ifndef A_RFC3984WRITER_H_
#define A_RFC3984WRITER_H_
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#define LOG_TO_FILES 0
namespace android {
struct ABuffer;
struct MediaBuffer;
struct RFC3984Writer : public MediaWriter {
RFC3984Writer(const AString &uri);//IP:UDP-port
protected:
private:
uint32_t mRTPTimeBase;
uint32_t m_nPort;
Rfc3984Context *m_packer;
SessionSet *m_pSessionSet;
RtpSession *m_Session;
MSQueue m_RFC3984queue;
unsigned int m_nUser_Timestamp;
int mFd;
};
} // namespace android
#endif // A_RFC3984WRITER_H_
====================RFC3984Writer.cpp===============
//#define LOG_NDEBUG 0
#define LOG_TAG "RFC3984Writer"
#include
#include "RFC3984Writer.h"
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#define PT 96
#define PT_STR "96"
#define NAL_START_CODE_SIZE 4
namespace android {
RFC3984Writer::RFC3984Writer(int fd)
m_nPort(1234),
mRTPTimeBase(0),
m_nUser_Timestamp(0),
LOGD("RFC3984Writer +++");
LOGD("RFC3984Writer ---");
}
RFC3984Writer::RFC3984Writer(const AString &uri)
: mFlags(0),
muri(uri),
m_nPort(1234),
mRTPTimeBase(0),
{
mLooper->setName("rfc3984 writer");
}
RFC3984Writer::~RFC3984Writer() {
LOGD("~RFC3984Writer +++");
if( mFd )
{
}
LOGD("~RFC3984Writer ---");
}
status_t RFC3984Writer::addSource(const sp &source) {
}
bool RFC3984Writer::reachedEOS() {
}
status_t RFC3984Writer::start(MetaData *params) {
LOGD("RFC3984Writer::start +++");
}
ortp_init();
ortp_scheduler_init();
//rfc3984 init +++
m_packer=rfc3984_new();
rfc3984_set_mode(m_packer,1); //mode 1
rfc3984_enable_stap_a(m_packer,FALSE);
//rfc3984 init ---
printf("==> Scheduler initializedn");
rtp_profile_set_payload(&av_profile,96,&payload_type_h264); //set 264 playload type
m_Session = rtp_session_new(RTP_SESSION_SENDONLY);
m_nPort = 1234;
rtp_session_set_scheduling_mode(m_Session,1);
rtp_session_set_blocking_mode(m_Session,0);
rtp_session_set_remote_addr(m_Session,"192.168.1.3", m_nPort);//这是我运行VLC播放器的pc端IP地址
rtp_session_set_send_payload_type(m_Session,96); //96 is h.264
m_pSessionSet = session_set_new();
session_set_set(m_pSessionSet,m_Session);
session_set_select(NULL,m_pSessionSet,NULL);
ms_queue_init(&m_RFC3984queue);
LOGD("RFC3984Writer::start ---");
}
status_t RFC3984Writer::stop() {
LOGD("RFC3984Writer::stop +++");
(new AMessage(kWhatStop, mReflector->id()))->post();
rtp_session_destroy(m_Session);
session_set_destroy(m_pSessionSet);
usleep(250*1000);
ortp_exit();
LOGD("RFC3984Writer::stop ---");
}
status_t RFC3984Writer::pause() {
}
void RFC3984Writer::onMessageReceived(const sp &msg) {
LOGD("kWhatStop is event");
default:
}
void RFC3984Writer::onRead(const sp &msg) {
}
}
static void x264_buf_to_msgb(const uint8_t *buf, int bufsize, MSQueue * nalus)
{
//int i,kkk,size;
mblk_t *m;
m=allocb(bufsize+14,0);
memcpy(m->b_wptr,buf,bufsize);
m->b_wptr+=bufsize;
ms_queue_put(nalus,m);
}
void RFC3984Writer::sendAVCData(MediaBuffer *mediaBuf) {
int nallen = mediaBuf->range_length();
MSQueue nalus;
ms_queue_init(&nalus);
int i=0,sps_size,pps_size;
int sps_idx,pps_idx,frame_idx;
for(i = 0;i < nallen-4 && (pnal[4]==0x67);i++ )
{
if(pnal[i]==0x00 && pnal[i+1]==0x00 && pnal[i+2]==0x00 &&pnal[i+3]==0x01)
{
if(pnal[i+4]==0x67)
{
sps_idx = i+4;
//LOGD("found sps");
}
else if(pnal[i+4]==0x68)
{
pps_idx = i+4;
//LOGD("found sps+pps");
}
else if(pnal[i+4]==0x65 || pnal[i+4]==0x41)
{
//LOGD("found sps+pps+frame");
frame_idx = i+4;
x264_buf_to_msgb(&pnal[sps_idx],pps_idx-sps_idx-NAL_START_CODE_SIZE,&nalus);
x264_buf_to_msgb(&pnal[pps_idx],frame_idx-pps_idx-NAL_START_CODE_SIZE,&nalus);
x264_buf_to_msgb(&pnal[frame_idx],nallen-frame_idx,&nalus);
break;
}
else
{
LOGE(" %s %d say : Invaild encoded data!!",__FUNCTION__,__LINE__);
break;
}
}
}
if((pnal[0]==0x00 && pnal[1]==0x00 && pnal[2]==0x00 && pnal[3]==0x01) && (pnal[4]==0x65 || pnal[4]==0x41))
{
//if(pnal[4]==0x65)
// LOGD("found I frame");
//if(pnal[4]==0x41)
// LOGD("found P frame");
x264_buf_to_msgb(&pnal[NAL_START_CODE_SIZE],nallen-NAL_START_CODE_SIZE,&nalus);
}
else
{
LOGE(" %s %d say : Invaild encoded data!!",__FUNCTION__,__LINE__);
}
rfc3984_pack(m_packer,&nalus,&m_RFC3984queue,rtpTime);//m_nUser_Timestamp
mblk_t *im = ms_queue_get(&m_RFC3984queue);
do {
mblk_t *header;
if (im){
header = rtp_session_create_packet(m_Session, 12, NULL, 0);
rtp_set_markbit(header, mblk_get_marker_info(im));
header->b_cont = im;
rtp_session_sendm_with_ts(m_Session, header, rtpTime);//m_nUser_Timestamp
//LOGD("send rtp packages ts=%d",m_nUser_Timestamp);
}
else
{
break;
}
}while ((im = ms_queue_get(&m_RFC3984queue)) != NULL);
m_nUser_Timestamp+=3600;
}//end of sendAVCData
} // namespace android
编译进libstagefright.so,但只是有了实现类,还需要接入到MediaRecorder框架里去,并在MediaRecorder.java里也相应地实现一下
需要修改的类有:
framework/include/media/Mediarecorder.h 增加一个OUTPUT_FORMAT_NBV_RTP264 = 10,的定义
enum output_format {
};
frameworks/base/media/libmedia/mediarecorder.cpp
//这是Bp端的代码
//需要修改setOutputFormat否则google的默认代码应用>3以上的outputformat是audio的会导致错误返回
status_t MediaRecorder::setOutputFormat(int of)
{
}
framework/media/libmediaplayerservice/StagefrightRecorder.cpp
//这里是增加针对OUTPUT_FORMAT_NBV_RTP264 这个新outputformat的调用
//当客户设置了OUTPUT_FORMAT_NBV_RTP264后,我们应该去new RFC3984Writer的实例
status_t StagefrightRecorder::start() {
status = startRTPRecording();
}
//add by kali
status_t StagefrightRecorder::startNBV264Sending() {
}
framework/base/media/java/android/media/MediaRecorder.java
public static final int NBV_RTP264=10; //add by kali@neobv.com
2.6基于oRTP发送的MediaRecorder测试例子
2.7 pc端VLC测试
在pc端生成一个rtp-sdp.sdp文件,内容如下
========================================
v=0
o=- 0 0 IN IP4 127.0.0.1
s=No Name
t=0 0
c=IN IP4 127.0.0.1
m=video 1234 RTP/AVP 96
a=rtpmap:96 H264/90000
a=fmtp:96 packetization-mode=1;profile-level-id=42001E
=======================================================
打开VLC播放器,播放rtp-sdp.sdp文件,这时vlc就会打开1234这个udp端口,并按sdp文件中指定的协议来解释来自android的rtp包了,到此一切正常播放
但VLC中播放有些delay,而且有现在只实现了视频,下一步实现一个音频的方案