工作中需在Linux下完成双录功能开发,在这里记录一下用到的相关知识点做一个备份。
1、录屏基于x11的linux底层库进行开发,注意img的内存释放问题。
2、基于V4l2的linux底层库进行开发,相对而言流程比较赋值,注意可以查看摄像头对应的VID&PID设备号,同一型号的摄像头他们的ID是一致的。
3、基于alsa库进行录音设备采集,生成的PCM文件经过ffmpeg重采样就可以完成编码了。注意编码的时候最好保持采用率的一致,不然声音听起来会比较怪的。
1、x11录屏
查看博客和ffmpeg中X11grap中的录屏部分源码提取
xcb_get_image_reply_t *img;
xcb_get_image_cookie_t iq;
xcb_generic_error_t *e = NULL;
xcb_connection_t *conn;
xcb_screen_t *screen;
int m_iDesktopWidth = 1280;
int m_iDesktopHeight = 1024;
bool InitVideoInput()
{
int screen_num = 1;
conn = xcb_connect(NULL, &screen_num);
if (xcb_connection_has_error(conn))
qDebug() << "xcb_connection_has_error faild!";
qDebug() << "xcb_connection_has_error success!" << screen_num;
const xcb_setup_t * setup = xcb_get_setup(conn);
xcb_screen_iterator_t iter = xcb_setup_roots_iterator (setup);
for (int i = 0; i < screen_num; ++i)
{
xcb_screen_next (&iter);
}
screen = iter.data;
qDebug() << "Informations of screen " << screen->root << screen->width_in_pixels << screen->height_in_pixels;
m_iDesktopWidth = screen->width_in_pixels;
m_iDesktopHeight = screen->height_in_pixels;
qDebug() << "xcb success!";
}
while(1)
{
//xgrab方法
iq = xcb_get_image(conn, XCB_IMAGE_FORMAT_Z_PIXMAP, screen->root, 0, 0, m_iDesktopWidth, m_iDesktopHeight, ~0);
img = xcb_get_image_reply(conn, iq, &e);
if (e)
{
qDebug() << "xcb_get_image_reply faild!";
continue;
}
uint8_t *data;
if (!img)
{
qDebug() << "img = NULL faild!";
continue;
}
data = xcb_get_image_data(img);
int length = xcb_get_image_data_length(img);
}
2、V4l2摄像头获取
这个还需要深究V4L2库进行分析
FILE* fpfile;
char* filename;
int CMBCamera::StartCamera(int &width, int &height, std::string strCameraVPid)
{
int ret = 0;
//输出YUV文件测试
QDateTime localTime = QDateTime::currentDateTime();//获取系统时 间
QString currentTime = localTime.toString("yyyyMMddhhmmss");//格式转换
char* LocalTime = qstringToChar(currentTime); //QString 转char*
filename = new char[50];
strcpy(filename, SAVEPICTURE); //拼接字 符串
strcat(filename, LocalTime);
strcat(filename, PICTURETAIL);
fpfile = fopen(filename, "wb+"); //打开文件, 以写格式
if(fpfile == NULL)
{
qDebug()<< "create yuv file failure ";
return 0;
}
qDebug() << "StartCamera Start! width = " << width << " height = " << height << " strCameraVPid = " << strCameraVPid.c_str();
m_CurrentWidth = width;
m_CurrentHeight = height;
m_strCameraVPid = GetVPID(strCameraVPid);
bStart = true;
ret = open_device();
if(ret == 0)
{
qDebug() << "StartCamera Fail -> open_device Fail!";
return 0;
}
printSolution(fd);
ret = init_device();
if(ret == 0)
{
qDebug() << "StartCamera Fail -> init_device Fail!";
return 0;
}
start_capturing();
sleep(1);
initFrameCallBackFun();//读取数据
//适配摄像头大小
if(m_CurrentWidth != width || m_CurrentHeight != height)
{
width = m_CurrentWidth;
height = m_CurrentHeight;
}
m_pCacheThread = new Poco::Thread();
m_pCacheRa = new Poco::RunnableAdapter<CMBCamera>(*this,&CMBCamera::CacheVideoFrame);//处理数据
m_pCacheThread->start(*m_pCacheRa);
qDebug() << "StartCamera Success! m_CurrentWidth = " << m_CurrentWidth << " m_CurrentHeight = " << m_CurrentHeight;
return 1;
}
int CMBCamera::StopCamera()
{
//输出YUV文件测试
delete []filename;
filename = NULL;
fclose(fpfile);
fpfile = NULL;
bStart = false;
StopReadThread();
stop_capturing();
uninit_device();
close_device();
this->fd = -1;
free(setfps);
setfps = NULL;
//free(CaptureVideoBuffer);
sleep(1);
qDebug() << "StopCamera Success!";
return 0;
}
void CMBCamera::close_device(void)
{
g_vcV4l2Pix.clear();
if (-1 == close(fd))
qDebug() << "close";
fd = -1;
qDebug() << "close_device() ok\n";
}
void CMBCamera::uninit_device(void)
{
unsigned int i;
switch (io)
{
case IO_METHOD_READ:
free(buffers[0].start);
break;
case IO_METHOD_MMAP:
for (i = 0; i < n_buffers; ++i)
if (-1 == munmap(buffers[i].start, buffers[i].length))
qDebug() << "munmap";
break;
case IO_METHOD_USERPTR:
for (i = 0; i < n_buffers; ++i)
free(buffers[i].start);
break;
}
free(buffers);
buffers = NULL;
qDebug() << "uninit_device() ok\n";
//MBLogInfo("uninit_device() ok");
}
void CMBCamera::stop_capturing(void)
{
enum v4l2_buf_type type;
switch (io)
{
case IO_METHOD_READ:
break;
case IO_METHOD_MMAP:
case IO_METHOD_USERPTR:
type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
if (-1 == xioctl(fd, VIDIOC_STREAMOFF, &type))
qDebug() << "VIDIOC_STREAMOFF";
break;
}
qDebug() << "stop_capturing() ok\n";
//MBLogInfo("stop_capturing() ok");
}
bool CMBCamera::StopReadThread()
{
qDebug() << "StopReadThread()\n";
//CMediaEngineUtils::MBSleepThread(100);
QThread::msleep(100);
// C_CONFIG::CConfig::SleepMS(100);
//MBLogInfo("enter");
if (m_ThreadRead)
{
//MBLogInfo("Release Thread[%p].", m_ThreadRead);
m_ThreadRead->join();
delete (m_ThreadRead);
m_ThreadRead = NULL;
}
//MBLogInfo("leave");
return true;
}
void CMBCamera::initFrameCallBackFun()
{
StartReadThread();
}
bool CMBCamera::StartReadThread()
{
qDebug() << "StartReadThread()";
if (NULL == m_ThreadRead)
{
m_ThreadRead = new Poco::Thread();
qDebug() << "Create m_Thread=" << m_ThreadRead;
}
m_ThreadRead->start(*this);
//bInit = true;
qDebug() << "bReadThreadStarted leave.";
return true;
}
void CMBCamera::run()
{
ThreadWrapperThreadFunc(this);
printf("run() end!!\n");
}
void* CMBCamera::ThreadWrapperThreadFunc(void* me)
{
CMBCamera* reporter = static_cast<CMBCamera*>(me);
if (NULL != reporter)
reporter->FrameThreadFunc();
else
qDebug() << "me = " << me ;
return NULL;
}
void CMBCamera::FrameThreadFunc()
{
void* buf = NULL;
lock_dev.lock();
for (;;)
{
// printf("FrameThreadFunc() bStart:%d\n", bStart);
fd_set fds;
struct timeval tv;
int r;
if (fd <= 0 || !bStart)
{
break;
// usleep(100);
// continue;
}
FD_ZERO(&fds);
FD_SET(fd, &fds);
tv.tv_sec = 1;
tv.tv_usec = 0;
if (fd <= 0 || !bStart)
{
break;
}
r = select(fd + 1, &fds, NULL, NULL, &tv);
if (-1 == r)
{
if (EINTR == errno)
continue;
// errno_exit("=======================================select");
}
if (0 == r)
{
fprintf(stderr, "======================================select timeout\n");
exit(EXIT_FAILURE);
continue;
}
if (fd <= 0 || !bStart)
{
break;
}
read_frame();
}
qDebug() << "FrameThreadFunc() return";
//MBLogInfo("FrameThreadFunc() return");
// StopReadThread();
lock_dev.unlock();
return;
}
int CMBCamera::read_frame(void)
{
//printf("read_frame()\n");
struct v4l2_buffer buf;
unsigned int i;
switch (io)
{
case IO_METHOD_READ:
if (-1 == read(fd, buffers[0].start, buffers[0].length))
{
switch (errno)
{
case EAGAIN:
return 0;
case EIO:
default:
qDebug() << "read";
return 0;
}
}
if (fd <= 0 || !bStart)
break;
printf("--------222-----\r\n");
//videoFrameCB(buffers[0].start, buffers[0].length, NULL, NULL);
m_cacheBufMutex.lock();
m_cacheBuf.assign((const unsigned char*)buffers[0].start, buffers[0].length);
m_cacheBufMutex.unlock();
m_cacheEvent.set();
// process_image(buffers[0].start, buffers[0].length);
break;
case IO_METHOD_MMAP:
CLEAR(buf);
buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
buf.memory = V4L2_MEMORY_MMAP;
if (-1 == xioctl(fd, VIDIOC_DQBUF, &buf))//从队列中取出帧
{
switch (errno)
{
case EAGAIN:
return 0;
case EIO:
default:
char buf_[50] = { 0x0 };
sprintf(buf_, "VIDIOC_DQBUF fd:%d,isStart:%d", fd, bStart);
//errno_exit(buf_);
if (errno == 22)
return 0;
// return 0;
}
}
if (fd <= 0 || !bStart)
break;
assert(buf.index < n_buffers);
// printf("---------3333----\r\n");
//videoFrameCB(buffers[buf.index].start, buffers[buf.index].length, NULL,NULL);
m_cacheBufMutex.lock();
m_cacheBuf.assign((const unsigned char *)(buffers[buf.index].start), buffers[buf.index].length);
m_cacheBufMutex.unlock();
m_cacheEvent.set();
// process_image(buffers[buf.index].start, buffers[buf.index].length);
if (-1 == xioctl(fd, VIDIOC_QBUF, &buf)) //把帧放入队列
{
qDebug() << "VIDIOC_QBUF";
return 0;
}
break;
case IO_METHOD_USERPTR:
CLEAR(buf);
buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
buf.memory = V4L2_MEMORY_USERPTR;
if (-1 == xioctl(fd, VIDIOC_DQBUF, &buf))
{
switch (errno)
{
case EAGAIN:
return 0;
case EIO:
default:
qDebug() << "VIDIOC_DQBUF";
return 0;
}
}
for (i = 0; i < n_buffers; ++i)
if (buf.m.userptr == (unsigned long)buffers[i].start &&
buf.length == buffers[i].length)
break;
if (fd <= 0 || !bStart)
break;
assert(i < n_buffers);
printf("----------hhjjj=======\r\n");
//videoFrameCB((void*)buf.m.userptr, 0, NULL, NULL);
// process_image((void*)buf.m.userptr, 0);
if (-1 == xioctl(fd, VIDIOC_QBUF, &buf))
{
qDebug() << "VIDIOC_QBUF";
return 0;
}
break;
}
return 1;
}
int CMBCamera::open_device(void)
{
qDebug() << "CMBCamera::open_device Start!";
bool bFind = false;
int nCameraId = 0;
for (int i = 0; i < 10; i++)
{
if (isVaildCamera(i, m_strCameraVPid) == 0)
{
bFind = true;
nCameraId = i;
break;
}
}
if(!bFind)
{
qDebug() << "CMBCamera::open_device isVaildCamera Fail!";
return 0;
}
// nCameraId = SmartSeeMediaConfigManager::GetInstance()->mVideoCameraId;
char szDevName[256] = {0};
sprintf(szDevName, "/dev/video%d", nCameraId);
qDebug() << "CMBCamera::open_device" << szDevName;
struct stat st;
dev_name = szDevName;//"/dev/video0";
if (-1 == stat(szDevName, &st))
{
fprintf(stderr, "Cannot identify %s: %d, %s\n", szDevName, errno,strerror(errno));
qDebug() << "open_device() stat Fail!";
return 0;
//exit(EXIT_FAILURE);
}
if (!S_ISCHR(st.st_mode))
{
fprintf(stderr, "%s is no device\n", szDevName);
qDebug() << "open_device() S_ISCHR Fail!";
return 0;
//exit(EXIT_FAILURE);
}
fd = open(szDevName, O_RDWR | O_NONBLOCK, 0);
if (-1 == fd)
{
fprintf(stderr, "Cannot open \E2\80?%s\E2\80?: %d, %s\n", szDevName, errno,
strerror(errno));
qDebug() << "open_device() open Fail!";
return 0;
//exit(EXIT_FAILURE);
}
qDebug() << "open_device() ok";
return 1;
}
void CMBCamera::printSolution(int fd)
{ //获取摄像头所支持的分辨率
enum v4l2_buf_type type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
struct v4l2_fmtdesc fmt_1;
struct v4l2_frmsizeenum frmsize;
structFromatRecord formatRecord;
// struct v4l2_pix_format pixFormat;
// ClearVideoInParamsList();
qDebug() << "printSolution Camera Support param";
fmt_1.index = 0;
fmt_1.type = type;
while (ioctl(fd, VIDIOC_ENUM_FMT, &fmt_1) >= 0)
{
frmsize.pixel_format = fmt_1.pixelformat;
frmsize.index = 0;
while (ioctl(fd, VIDIOC_ENUM_FRAMESIZES, &frmsize) >= 0)
{
//获取支持的宽高格式
//V4L2_PIX_FMT_YUYV 1448695129
//V4L2_PIX_FMT_YYUV 1448434009
//V4L2_PIX_FMT_MJPEG 1196444237
formatRecord.width = frmsize.discrete.width;
formatRecord.height = frmsize.discrete.height;
formatRecord.pixfromat = fmt_1.pixelformat;//fmt_1.pixelformat;
//formatRecord.fps = getfps.parm.capture.timeperframe.denominator;
g_vcV4l2Pix.push_back(formatRecord);
if (frmsize.type == V4L2_FRMSIZE_TYPE_DISCRETE)
{
qDebug() << "1.V4L2_FRMSIZE_TYPE_DISCRETE line:" << __LINE__ << " : " << formatRecord.pixfromat << "- " << frmsize.discrete.width << "X" << frmsize.discrete.height;
}
else if (frmsize.type == V4L2_FRMSIZE_TYPE_STEPWISE)
{
qDebug() << "2.V4L2_FRMSIZE_TYPE_STEPWISE line:" << __LINE__ << " : " << formatRecord.pixfromat << "- " << frmsize.discrete.width << "X" << frmsize.discrete.height;
}
frmsize.index++;
}
fmt_1.index++;
}
}
int CMBCamera::xioctl(int fd, int request, void* arg)
{
int r;
do
r = ioctl(fd, request, arg);
while (-1 == r && EINTR == errno);
return r;
}
int CMBCamera::init_device()
{
struct v4l2_capability cap;
struct v4l2_cropcap cropcap;
struct v4l2_crop crop;
struct v4l2_format fmt;
unsigned int min;
int i_pixret = -1;
if (-1 == xioctl(fd, VIDIOC_QUERYCAP, &cap))
{
if (EINVAL == errno)
{
fprintf(stderr, "%s is no V4L2 device\n", dev_name);
// exit(EXIT_FAILURE);
qDebug() << "no V4L2 device!";
return 0;
}
else
{
qDebug() << "VIDIOC_QUERYCAP Success";
}
}
if (!(cap.capabilities & V4L2_CAP_VIDEO_CAPTURE))
{
fprintf(stderr, "%s is no video capture device\n", dev_name);
qDebug() << "no video capture device";
//exit(EXIT_FAILURE);
return 0;
}
switch (io)
{
case IO_METHOD_READ:
if (!(cap.capabilities & V4L2_CAP_READWRITE))
{
fprintf(stderr, "%s does not support read i/o\n", dev_name);
qDebug() << "not support read i/o";
return 0;
}
break;
case IO_METHOD_MMAP:
case IO_METHOD_USERPTR:
if (!(cap.capabilities & V4L2_CAP_STREAMING))
{
fprintf(stderr, "%s does not support streaming i/o\n", dev_name);
qDebug() << "not support streaming i/o";
return 0;
}
break;
}
CLEAR(cropcap);
cropcap.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
if (0 == xioctl(fd, VIDIOC_CROPCAP, &cropcap))
{
crop.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
crop.c = cropcap.defrect;
if (-1 == xioctl(fd, VIDIOC_S_CROP, &crop))
{
switch (errno)
{
case EINVAL:
break;
default:
break;
}
}
}
else
{
}
CLEAR(fmt);
fmt.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
// int w = WWIDTH;
// int h = HHEIGHT;
// getSolution(params, w, h);
// setCurrentVideo(m_bDisplayBG, w, h, m_bRestartEncode);
// fmt.fmt.pix.width = w;
// fmt.fmt.pix.height = h;
// //fmt.fmt.pix.pixelformat = V4L2_PIX_FMT_YUYV;
// fmt.fmt.pix.pixelformat = V4L2_PIX_FMT_MJPEG;
// m_iCameraFormat = fmt.fmt.pix.pixelformat;
//设置宽高采集样式
// fmt.fmt.pix.width = g_vcV4l2Pix[0].width;
// fmt.fmt.pix.height = g_vcV4l2Pix[0].height;
//返回合适的宽高
i_pixret = GetCurrentCamera();
if(i_pixret == 1)
{
fmt.fmt.pix.pixelformat = V4L2_PIX_FMT_MJPEG;
}
else if(i_pixret == 0)
{
fmt.fmt.pix.pixelformat = V4L2_PIX_FMT_YUYV;
}
else {
qDebug() << "GetCurrentCamera error!";
return 0;
}
fmt.fmt.pix.width = m_CurrentWidth;
fmt.fmt.pix.height = m_CurrentHeight;
//fmt.fmt.pix.pixelformat = V4L2_PIX_FMT_YUYV;
//fmt.fmt.pix.pixelformat = g_vcV4l2Pix[0].pixfromat;
//fmt.fmt.pix.pixelformat = V4L2_PIX_FMT_MJPEG;
m_iCameraFormat = fmt.fmt.pix.pixelformat;
fmt.fmt.pix.field = V4L2_FIELD_INTERLACED;
if (-1 == xioctl(fd, VIDIOC_S_FMT, &fmt))
qDebug() << "VIDIOC_S_FMT";
//设置帧率
setfps = (struct v4l2_streamparm*)calloc(1, sizeof(struct v4l2_streamparm));
if(set_camera_streamparm(fd, setfps, 30) != 0)
{
qDebug() << "set_camera_streamparm 30FPS fail!";
}
min = fmt.fmt.pix.width * 2;
if (fmt.fmt.pix.bytesperline < min)
fmt.fmt.pix.bytesperline = min;
min = fmt.fmt.pix.bytesperline * fmt.fmt.pix.height;
if (fmt.fmt.pix.sizeimage < min)
fmt.fmt.pix.sizeimage = min;
struct v4l2_streamparm getfps;
getfps.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
int ret = ioctl(fd, VIDIOC_G_PARM, &getfps);
if(ret)
{
qDebug() << "VIDIOC_G_PARM to get fps failed : " << ret;
}
else
{
qDebug() << "VIDIOC_G_PARM to get fps succeed : FPS = " << getfps.parm.capture.timeperframe.denominator;
}
switch (io)
{
case IO_METHOD_READ:
init_read(fmt.fmt.pix.sizeimage);
break;
case IO_METHOD_MMAP:
init_mmap();
break;
case IO_METHOD_USERPTR:
init_userp(fmt.fmt.pix.sizeimage);
break;
}
qDebug() << "init_device() ok\n";
return 1;
}
//根据传入的摄像头宽高 找到匹配的摄像头支持的宽高
int CMBCamera::GetCurrentCamera()
{
int iminMJPEG = (std::abs((int)g_vcV4l2Pix[0].width - m_CurrentWidth) + std::abs((int)g_vcV4l2Pix[0].height - m_CurrentHeight));
int iminYUY2 = (std::abs((int)g_vcV4l2Pix[0].width - m_CurrentWidth) + std::abs((int)g_vcV4l2Pix[0].height - m_CurrentHeight));
int temp;
int iIndexMJPEG = 0;
int iIndexYUY2 = 0;
bool bIsMJPEG = false;
bool bIsYUY2 = false;
for(int i = 0; i < g_vcV4l2Pix.size(); i++)
{
if(g_vcV4l2Pix[i].pixfromat == V4L2_PIX_FMT_MJPEG)
{
bIsMJPEG = true;
temp = (std::abs((int)g_vcV4l2Pix[i].width - m_CurrentWidth) + std::abs((int)g_vcV4l2Pix[i].height - m_CurrentHeight));
if(iminMJPEG > temp)
{
iminMJPEG = temp;
iIndexMJPEG = i;
}
}
else if(g_vcV4l2Pix[i].pixfromat == V4L2_PIX_FMT_YUYV)
{
bIsYUY2 = true;
temp = (std::abs((int)g_vcV4l2Pix[i].width - m_CurrentWidth) + std::abs((int)g_vcV4l2Pix[i].height - m_CurrentHeight));
if(iminYUY2 > temp)
{
iminYUY2 = temp;
iIndexYUY2 = i;
}
}
}
if(bIsMJPEG)
{
m_CurrentWidth = g_vcV4l2Pix[iIndexMJPEG].width;
m_CurrentHeight = g_vcV4l2Pix[iIndexMJPEG].height;
return 1;
}
else if (bIsYUY2)
{
m_CurrentWidth = g_vcV4l2Pix[iIndexYUY2].width;
m_CurrentHeight = g_vcV4l2Pix[iIndexYUY2].height;
return 0;
}
return -1;
}
void CMBCamera::start_capturing(void)
{
unsigned int i;
enum v4l2_buf_type type;
switch (io)
{
case IO_METHOD_READ:
break;
case IO_METHOD_MMAP:
for (i = 0; i < n_buffers; ++i)
{
struct v4l2_buffer buf;
CLEAR(buf);
buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
buf.memory = V4L2_MEMORY_MMAP;
buf.index = i;
if (-1 == xioctl(fd, VIDIOC_QBUF, &buf))
qDebug() << "VIDIOC_QBUF";
}
type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
if (-1 == xioctl(fd, VIDIOC_STREAMON, &type))
qDebug() << "VIDIOC_STREAMON";
break;
case IO_METHOD_USERPTR:
for (i = 0; i < n_buffers; ++i)
{
struct v4l2_buffer buf;
CLEAR(buf);
buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
buf.memory = V4L2_MEMORY_USERPTR;
buf.m.userptr = (unsigned long)buffers[i].start;
buf.length = buffers[i].length;
if (-1 == xioctl(fd, VIDIOC_QBUF, &buf))
qDebug() << "VIDIOC_QBUF";
}
type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
if (-1 == xioctl(fd, VIDIOC_STREAMON, &type))
qDebug() << "VIDIOC_STREAMON";
break;
}
qDebug() << "start_capturing() ok\n";
}
void CMBCamera::init_read(unsigned int buffer_size)
{
buffers = (buffer*)calloc(1, sizeof(*buffers));
if (!buffers)
{
fprintf(stderr, "Out of memory\n");
qDebug() << "Out of memory Fail!";
exit(EXIT_FAILURE);
}
buffers[0].length = buffer_size;
buffers[0].start = malloc(buffer_size);
if (!buffers[0].start)
{
fprintf(stderr, "Out of memory\n");
qDebug() << "Out of memory2 Fail!";
exit(EXIT_FAILURE);
}
}
void CMBCamera::init_mmap(void)
{
struct v4l2_requestbuffers req;
CLEAR(req);
req.count = 4;
req.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
req.memory = V4L2_MEMORY_MMAP;
if (-1 == xioctl(fd, VIDIOC_REQBUFS, &req))
{
if (EINVAL == errno)
{
fprintf(stderr,
"%s does not support "
"memory mapping\n",
dev_name);
qDebug() << "init_mmap xioctl Fail!";
exit(EXIT_FAILURE);
}
else
{
qDebug() << "VIDIOC_REQBUFS";
}
}
if (req.count < 2)
{
fprintf(stderr, "Insufficient buffer memory on %s\n", dev_name);
// exit(EXIT_FAILURE);
}
buffers = (buffer*)calloc(req.count, sizeof(*buffers));
if (!buffers)
{
fprintf(stderr, "Out of memory\n");
qDebug() << "init_mmap Out of memory3 Fail!";
exit(EXIT_FAILURE);
}
for (n_buffers = 0; n_buffers < req.count; ++n_buffers)
{
struct v4l2_buffer buf;
CLEAR(buf);
buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
buf.memory = V4L2_MEMORY_MMAP;
buf.index = n_buffers;
if (-1 == xioctl(fd, VIDIOC_QUERYBUF, &buf))
qDebug() << "VIDIOC_QUERYBUF";
buffers[n_buffers].length = buf.length;
buffers[n_buffers].start = mmap(NULL, buf.length, PROT_READ | PROT_WRITE,
MAP_SHARED, fd, buf.m.offset);
if (MAP_FAILED == buffers[n_buffers].start)
qDebug() << "mmap";
}
}
void CMBCamera::init_userp(unsigned int buffer_size)
{
struct v4l2_requestbuffers req;
CLEAR(req);
req.count = 4;
req.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
req.memory = V4L2_MEMORY_USERPTR;
if (-1 == xioctl(fd, VIDIOC_REQBUFS, &req))
{
if (EINVAL == errno)
{
fprintf(stderr,
"%s does not support "
"user pointer i/o\n",
dev_name);
qDebug() << "xioctl does not support user pointer i/o";
exit(EXIT_FAILURE);
}
else
{
qDebug() << "VIDIOC_REQBUFS";
}
}
buffers = (buffer*)calloc(4, sizeof(*buffers));
if (!buffers)
{
fprintf(stderr, "Out of memory\n");
qDebug() << "Out of memory";
exit(EXIT_FAILURE);
}
for (n_buffers = 0; n_buffers < 4; ++n_buffers)
{
buffers[n_buffers].length = buffer_size;
buffers[n_buffers].start = malloc(buffer_size);
if (!buffers[n_buffers].start)
{
fprintf(stderr, "Out of memory\n");
qDebug() << "Out of memory2";
exit(EXIT_FAILURE);
}
}
}
3、alsa 录音
bool IniitAudioInfo()
{
int err = -1;
snd_pcm_hw_params_t *hwParam = NULL;
snd_pcm_uframes_t frames= 32;
snd_output_t* log;
err = snd_output_stdio_attach(&log, stdout, 0);
const char *name = "plughw:2,0";
//if ((err = snd_pcm_open(&captureHandle, name, SND_PCM_STREAM_CAPTURE, 0)) < 0)
if ((err = snd_pcm_open(&captureHandle, "default", SND_PCM_STREAM_CAPTURE, 0)) < 0)
{
qDebug() << "snd_pcm_open faild!" << snd_strerror(err);
return false;
}
if ((err = snd_pcm_hw_params_malloc(&hwParam)) < 0)
{
qDebug() << "snd_pcm_hw_params_malloc faild!" << snd_strerror(err);
return false;
}
if ((err = snd_pcm_hw_params_any(captureHandle, hwParam)) < 0)
{
qDebug() << "snd_pcm_hw_params_any faild!" << snd_strerror(err);
return false;
}
if ((err = snd_pcm_hw_params_set_access(captureHandle, hwParam, SND_PCM_ACCESS_RW_INTERLEAVED)) < 0)
{
qDebug() << "snd_pcm_hw_params_set_access faild!" << snd_strerror(err);
return false;
}
//16位
if ((err = snd_pcm_hw_params_set_format(captureHandle, hwParam, SND_PCM_FORMAT_S16_LE)) < 0)
{
qDebug() << "snd_pcm_hw_params_set_format faild!" << snd_strerror(err);
return false;
}
//shuang通道
if ((err = snd_pcm_hw_params_set_channels(captureHandle, hwParam, 2)) < 0)
{
qDebug() << "snd_pcm_hw_params_set_channels faild!" << snd_strerror(err);
return false;
}
int dir = 0;;
unsigned int val = 48000;
//设置采样率,如果采样率不支持,会用硬件支持最接近的采样率
snd_pcm_hw_params_set_rate_near(captureHandle, hwParam,&val, &dir);
unsigned int buffer_time,period_time;
//获取最大的缓冲时间,buffer_time单位为us,500000us=0.5s
snd_pcm_hw_params_get_buffer_time_max(hwParam, &buffer_time, 0);
//printf("max_buffer_time:%d\n",buffer_time);
if ( buffer_time >500000)
buffer_time = 500000;
//设置缓冲时间
snd_pcm_hw_params_set_buffer_time_near(captureHandle, hwParam, &buffer_time, 0);
//设置采样周期时间,计算方法38帧/秒,48000/38=1263点/帧
period_time = 26315;
snd_pcm_hw_params_set_period_time_near(captureHandle, hwParam, &period_time, 0);
//让这些参数设置到PCM设备
snd_pcm_hw_params(captureHandle, hwParam);
//这个frames并不是指帧率,而是1263采样点数/帧
snd_pcm_hw_params_get_period_size(hwParam,&frames, &dir);
// LenTotal = frames * 4 * 2;//两倍申请
int LenTotal = 12000;//两倍申请
if(m_Audiobuffer == NULL)
{
m_Audiobuffer = (unsigned char *) malloc(LenTotal);
}
if(m_Audiobuffer == NULL)
{
qDebug() << "snd_pcm_hw_params_set_channels faild!" << snd_strerror(err);
return false;
}
qDebug() << "snd_pcm_open success!";
// int size = frames * 4;
// FILE *fp = NULL;
// int file_len = 0;
// fp = fopen("record_dump.raw", "w+");
// if(fp == NULL) {
// qDebug() << "open file fail!\n";
// exit(1);
// }
// int rc;
// int i = 1000;
// frames = 1263;
// while (i--) {
// memset(m_Audiobuffer, 0, size);
// rc = snd_pcm_readi(captureHandle, m_Audiobuffer, frames);
// // printf("snd_pcm_readi,frames:%d, m_Audiobuffer:%s\n",frames, m_Audiobuffer);
// if (rc == -EPIPE)
// {
// /* EPIPE means overrun */
// qDebug() << "overrun occurred\n";
// snd_pcm_prepare(captureHandle);
// continue ;
// } else if (rc < 0) {
// qDebug() << "error from read: " << snd_strerror(rc);
// continue ;
// } else if (rc != (int)frames) {
// qDebug() << "short read, read %d frames\n";
// continue ;
// }
// if (fp) {
// //file_len = fwrite(m_Audiobuffer, 1, frames, fp);
// file_len = fwrite(m_Audiobuffer, 1, size, fp);
// qDebug() << "fwrite:"<< file_len << " bytes \n" ; //5024字节
// } else {
// qDebug() << "%s[%d] file open fail\n" << __func__<< __LINE__;
// }
// }
// fclose(fp);
// //close(fd_f);
// snd_pcm_drain(captureHandle);
// snd_pcm_close(captureHandle);
// free(m_Audiobuffer);
return true;
}
这里注意参数的设置,并且重采样的时候采样率要保持一致 否则可能声音会不清楚
//获取数据 还需要重采样
snd_pcm_uframes_t frames= 1024;
sizeReadi = snd_pcm_readi(captureHandle, m_Audiobuffer, frames);
if(sizeReadi < 0)
{
continue;
}
memcpy(m_pinputFramebuf,m_Audiobuffer,m_inputFrameBuffsize);
m_AudioTimePts = av_rescale_q(av_gettime()-m_first_aud_time, time_base_q, m_pAudio_st->time_base);
m_inputFrame->pkt_pts = m_inputFrame->pts = m_AudioTimePts;
//重采样
AVFrame *pOutFrame = NULL;
ret = AudioConvert(m_inputFrame, AV_SAMPLE_FMT_FLTP, 2, 48000, &pOutFrame);
if(ret != 0)
{
qDebug() << "AudioConvert Fail!";
continue;
}
/
int32_t CManager::AudioConvert(
const AVFrame* pInFrame, // 输入音频帧
AVSampleFormat eOutSmplFmt, // 输出音频格式
int32_t nOutChannels, // 输出音频通道数
int32_t nOutSmplRate, // 输出音频采样率
AVFrame** ppOutFrame) // 输出视频帧
{
//struct SwrContext* pSwrCtx = nullptr;
AVFrame* pOutFrame = nullptr;
// 创建格式转换器,
int64_t nInChnlLayout = av_get_default_channel_layout(pInFrame->channels);
int64_t nOutChnlLayout = (nOutChannels == 1) ? AV_CH_LAYOUT_MONO : AV_CH_LAYOUT_STEREO;
if(m_pSwrCtx == NULL)
{
m_pSwrCtx = swr_alloc();
if (m_pSwrCtx == nullptr)
{
qDebug() << "swr_alloc Fail";
return -1;
}
swr_alloc_set_opts(m_pSwrCtx,
nOutChnlLayout, eOutSmplFmt, nOutSmplRate, nInChnlLayout,
(enum AVSampleFormat)(pInFrame->format), pInFrame->sample_rate,
0, nullptr);
swr_init(m_pSwrCtx);
}
// 计算重采样转换后的样本数量,从而分配缓冲区大小
int64_t nCvtBufSamples = av_rescale_rnd(pInFrame->nb_samples, nOutSmplRate, pInFrame->sample_rate, AV_ROUND_UP);
// 创建输出音频帧
pOutFrame = av_frame_alloc();
pOutFrame->format = eOutSmplFmt;
pOutFrame->nb_samples = (int)nCvtBufSamples;
pOutFrame->channel_layout = (uint64_t)nOutChnlLayout;
int res = av_frame_get_buffer(pOutFrame, 0); // 分配缓冲区
if (res < 0)
{
//swr_free(&m_pSwrCtx);
qDebug() << "av_frame_get_buffer Fail";
av_frame_free(&pOutFrame);
return -2;
}
// 进行重采样转换处理,返回转换后的样本数量
int nCvtedSamples = swr_convert(m_pSwrCtx,
const_cast<uint8_t**>(pOutFrame->data),
(int)nCvtBufSamples,
const_cast<const uint8_t**>(pInFrame->data),
pInFrame->nb_samples);
if (nCvtedSamples <= 0)
{
//swr_free(&m_pSwrCtx);
qDebug() << "nCvtedSamples <= 0";
av_frame_free(&pOutFrame);
return -3;
}
pOutFrame->nb_samples = nCvtedSamples;
pOutFrame->pts = pInFrame->pts; // pts等时间戳沿用
pOutFrame->pkt_pts = pInFrame->pkt_pts;
(*ppOutFrame) = pOutFrame;
//swr_free(&m_pSwrCtx); // 释放转换器
return 0;
}
4、录音和摄像头视频拼接
int nYIndex = 0;
int nUVIndex = 0;
for (int i = 0; i < m_nCameraHeigth; i++)
{
memcpy(m_pFrameYUV->data[0] + (i)*m_nWidth + (m_nWidth - m_nCameraWidth), m_pVideoYuv420Buffer+ nYIndex*m_nCameraWidth, m_nCameraWidth);
nYIndex++;
}
for (int i = 0; i < iHalfCameraHeight; i++)
{
memcpy(m_pFrameYUV->data[1] + (i)*(i_HalfWidht) + (i_HalfWidht - iHalfCameraWidht), m_pVideoYuv420Buffer + iUPos + nUVIndex*iHalfCameraWidht, iHalfCameraWidht);
memcpy(m_pFrameYUV->data[2] + (i)*(i_HalfWidht) + (i_HalfWidht - iHalfCameraWidht), m_pVideoYuv420Buffer + iVPos + nUVIndex*iHalfCameraWidht, iHalfCameraWidht);
nUVIndex++;
}