openwrt使用mjpg-streamer定制开发智能目标检测识别系统

文章目录

• • 概要
  • 整体架构流程
  • 技术细节
  • 小结

概要

在物联网产品的开发过程中，经常需要做高效图传处理，开源软件mjpg_streamer非常适合

低码率的应用场景。

其将USB摄像头采集的图像通过HTTP或RTSP的推流方式推向客户端显示，编码格式为mjpeg，

帧率可达25fps。

本文在图像采集环节对目标进行图像分类和检测识别，并将目标框选出来。

整体架构流程

数据流向

摄像头采集→ yuv422格式→ 转换rgb24 → 图像识别 → turbomjpeg压缩→ mjpeg显示，

其中图像识别采用基于haar特征的级联分类器用于目标检测。

技术细节

mjpg-streamer-r182/plugins/input_uvc/input_uvc.c

cam_thread线程任务用于从内核中将uvc摄像头数据搬运到用户层。

/******************************************************************************
Description.: this thread worker grabs a frame and copies it to the global buffer
Input Value.: unused
Return Value: unused, always NULL
******************************************************************************/
void *cam_thread(void *arg)
{

    context *pcontext = arg;
    pglobal = pcontext->pglobal;

    /* set cleanup handler to cleanup allocated ressources */
    //pthread_cleanup_push(cam_cleanup, pcontext);

    while(!pglobal->stop && !is_cam_thread_stop) {

        while(pcontext->videoIn->streamingState == STREAMING_PAUSED && !is_cam_thread_stop) {
            usleep(1); // maybe not the best way so FIXME
        }

       
	    if(is_cam_thread_stop) break;

        /* grab a frame */
        if(uvcGrab(pcontext->videoIn) < 0) {
            IPRINT("Error grabbing frames\n");
            exit(EXIT_FAILURE);
        }

        if(is_cam_thread_stop) break;

        //DBG("received frame of size: %d from plugin: %d\n", pcontext->videoIn->buf.bytesused, pcontext->id);

#if 1
        /*
         * Workaround for broken, corrupted frames:
         * Under low light conditions corrupted frames may get captured.
         * The good thing is such frames are quite small compared to the regular pictures.
         * For example a VGA (640x480) webcam picture is normally >= 8kByte large,
         * corrupted frames are smaller.
         */
        if(pcontext->videoIn->tmpbytesused < minimum_size) {
            //DBG("dropping too small frame, assuming it as broken\n");
            continue;
        }
#endif


#if 0
        // use software frame dropping on low fps
        if (pcontext->videoIn->soft_framedrop == 1) {
            unsigned long last = pglobal->in[pcontext->id].timestamp.tv_sec * 1000 +
                                (pglobal->in[pcontext->id].timestamp.tv_usec/1000); // convert to ms
            unsigned long current = pcontext->videoIn->tmptimestamp.tv_sec * 1000 +
                                    pcontext->videoIn->tmptimestamp.tv_usec/1000; // convert to ms

            // if the requested time did not esplashed skip the frame 
            if ((current - last) < pcontext->videoIn->frame_period_time) {
                //DBG("Last frame taken %d ms ago so drop it\n", (current - last));
                continue;
            }
           // DBG("Lagg: %ld\n", (current - last) - pcontext->videoIn->frame_period_time);
        }

#endif

        /* copy JPG picture to global buffer */
        pthread_mutex_lock(&pglobal->in[pcontext->id].db);


#if 0 
        #ifndef NO_LIBJPEG
        if ((pcontext->videoIn->formatIn == V4L2_PIX_FMT_YUYV) ||
            (pcontext->videoIn->formatIn == V4L2_PIX_FMT_RGB565) ||
            (pcontext->videoIn->formatIn == V4L2_PIX_FMT_RGB24)) {
            
			memcpy(pglobal->in[pcontext->id].buf, pcontext->videoIn->framebuffer, pcontext->videoIn->tmpbytesused);

        } else {
        #endif
             
		    memcpy(pglobal->in[pcontext->id].buf, pcontext->videoIn->tmpbuffer, pcontext->videoIn->tmpbytesused);

        #ifndef NO_LIBJPEG
        }
        #endif
#endif   
        

		memcpy(pglobal->in[pcontext->id].buf, pcontext->videoIn->framebuffer, pcontext->videoIn->tmpbytesused);

        pglobal->in[pcontext->id].size  =  pcontext->videoIn->tmpbytesused;
  

#if 0
        /* motion detection can be done just by comparing the picture size, but it is not very accurate!! */
        if((prev_size - global->size)*(prev_size - global->size) > 4 * 1024 * 1024) {
            DBG("motion detected (delta: %d kB)\n", (prev_size - global->size) / 1024);
        }
        prev_size = global->size;
#endif

        /* copy this frame's timestamp to user space */
        pglobal->in[pcontext->id].timestamp = pcontext->videoIn->tmptimestamp;

        /* signal fresh_frame */
        pthread_cond_broadcast(&pglobal->in[pcontext->id].db_update);
        pthread_mutex_unlock(&pglobal->in[pcontext->id].db);
    }

    DBG("leaving input thread, calling cleanup function now\n");
    
	 if(pglobal->in[pcontext->id].buf != NULL)
         free(pglobal->in[pcontext->id].buf);
   

    return NULL;
}

mjpg-streamer-r182/plugins/input_uvc/v4l2uvc.c

使用linux v4l2视频采集框架和c_detect函数实时检测目标。

int init_videoIn(struct vdIn *vd, char *device, int width,
                 int height, int fps, int format, int grabmethod, globals *pglobal, int id, v4l2_std_id vstd)
{
    if(vd == NULL || device == NULL)
        return -1;
    if(width == 0 || height == 0)
        return -1;
    if(grabmethod < 0 || grabmethod > 1)
        grabmethod = 1;     //mmap by default;
    vd->videodevice = NULL;
    vd->status = NULL;
    vd->pictName = NULL;
    vd->videodevice = (char *) calloc(1, 16 * sizeof(char));
    vd->status = (char *) calloc(1, 100 * sizeof(char));
    vd->pictName = (char *) calloc(1, 80 * sizeof(char));
    snprintf(vd->videodevice, 16, "%s", device);
    vd->toggleAvi = 0;
    vd->getPict = 0;
    vd->signalquit = 1;
    vd->width = width;
    vd->height = height;
    vd->fps = fps;
    vd->formatIn = format;
	vd->vstd = vstd;
    vd->grabmethod = grabmethod;
    vd->soft_framedrop = 0;
    if(init_v4l2(vd) < 0) {
        fprintf(stderr, " Init v4L2 failed !! exit fatal \n");
        goto error;;
    }

    // getting the name of the input source
    struct v4l2_input in_struct;
    memset(&in_struct, 0, sizeof(struct v4l2_input));
    in_struct.index = 0;
    if (xioctl(vd->fd, VIDIOC_ENUMINPUT,  &in_struct) == 0) {
        int nameLength = strlen((char*)&in_struct.name);
        pglobal->in[id].name = malloc((1+nameLength)*sizeof(char));
        sprintf(pglobal->in[id].name, "%s", in_struct.name);
        DBG("Input name: %s\n", in_struct.name);
    } else {
        DBG("VIDIOC_ENUMINPUT failed\n");
    }
    
	
    tHandle = GetInstance();

    lib_detect=dlopen("/usr/lib/libvision_ctl.so", RTLD_LAZY);
    if (lib_detect == NULL){
	    DBG("the so file is not loaded\n");
	}

    // enumerating formats
    c_detect = dlsym(lib_detect, "detect");
	
	
	
    struct v4l2_format currentFormat;
    memset(&currentFormat, 0, sizeof(struct v4l2_format));
    currentFormat.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
    if (xioctl(vd->fd, VIDIOC_G_FMT, &currentFormat) == 0) {
        DBG("Current size: %dx%d\n",
             currentFormat.fmt.pix.width,
             currentFormat.fmt.pix.height);
    }

    pglobal->in[id].in_formats = NULL;
    for(pglobal->in[id].formatCount = 0; 1; pglobal->in[id].formatCount++) {
        struct v4l2_fmtdesc fmtdesc;
        memset(&fmtdesc, 0, sizeof(struct v4l2_fmtdesc));
        fmtdesc.index = pglobal->in[id].formatCount;
        fmtdesc.type  = V4L2_BUF_TYPE_VIDEO_CAPTURE;
        if(xioctl(vd->fd, VIDIOC_ENUM_FMT, &fmtdesc) < 0) {
            break;
        }

        if (pglobal->in[id].in_formats == NULL) {
            pglobal->in[id].in_formats = (input_format*)calloc(1, sizeof(input_format));
        } else {
            pglobal->in[id].in_formats = (input_format*)realloc(pglobal->in[id].in_formats, (pglobal->in[id].formatCount + 1) * sizeof(input_format));
        }

        if (pglobal->in[id].in_formats == NULL) {
            DBG("Calloc/realloc failed: %s\n", strerror(errno));
            return -1;
        }

        memcpy(&pglobal->in[id].in_formats[pglobal->in[id].formatCount], &fmtdesc, sizeof(struct v4l2_fmtdesc));

        if(fmtdesc.pixelformat == format)
            pglobal->in[id].currentFormat = pglobal->in[id].formatCount;

        DBG("Supported format: %s\n", fmtdesc.description);
        struct v4l2_frmsizeenum fsenum;
        memset(&fsenum, 0, sizeof(struct v4l2_frmsizeenum));
        fsenum.pixel_format = fmtdesc.pixelformat;
        int j = 0;
        pglobal->in[id].in_formats[pglobal->in[id].formatCount].supportedResolutions = NULL;
        pglobal->in[id].in_formats[pglobal->in[id].formatCount].resolutionCount = 0;
        pglobal->in[id].in_formats[pglobal->in[id].formatCount].currentResolution = -1;
        while(1) {
            fsenum.index = j;
            j++;
            if(xioctl(vd->fd, VIDIOC_ENUM_FRAMESIZES, &fsenum) == 0) {
                pglobal->in[id].in_formats[pglobal->in[id].formatCount].resolutionCount++;

                if (pglobal->in[id].in_formats[pglobal->in[id].formatCount].supportedResolutions == NULL) {
                    pglobal->in[id].in_formats[pglobal->in[id].formatCount].supportedResolutions = (input_resolution*)
                            calloc(1, sizeof(input_resolution));
                } else {
                    pglobal->in[id].in_formats[pglobal->in[id].formatCount].supportedResolutions = (input_resolution*)
                            realloc(pglobal->in[id].in_formats[pglobal->in[id].formatCount].supportedResolutions, j * sizeof(input_resolution));
                }

                if (pglobal->in[id].in_formats[pglobal->in[id].formatCount].supportedResolutions == NULL) {
                    DBG("Calloc/realloc failed\n");
                    return -1;
                }

                pglobal->in[id].in_formats[pglobal->in[id].formatCount].supportedResolutions[j-1].width = fsenum.discrete.width;
                pglobal->in[id].in_formats[pglobal->in[id].formatCount].supportedResolutions[j-1].height = fsenum.discrete.height;
                if(format == fmtdesc.pixelformat) {
                    if ((fsenum.discrete.width == width) && (fsenum.discrete.height == height))
                        pglobal->in[id].in_formats[pglobal->in[id].formatCount].currentResolution = (j - 1);
                    DBG("\tSupported size with the current format: %dx%d\n", fsenum.discrete.width, fsenum.discrete.height);
                } else {
                    DBG("\tSupported size: %dx%d\n", fsenum.discrete.width, fsenum.discrete.height);
                }
            } else {
                break;
            }
        }
        DBG("current resolution: %d\n", pglobal->in[id].in_formats[pglobal->in[id].formatCount].currentResolution);
    }

    /* alloc a temp buffer to reconstruct the pict */
    vd->framesizeIn = (vd->width * vd->height << 1);

    switch(vd->formatIn) {
    case V4L2_PIX_FMT_MJPEG: // in JPG mode the frame size is varies at every frame, so we allocate a bit bigger buffer
         vd->tmpbuffer = (unsigned char *) calloc(1, (size_t) vd->framesizeIn);

        if(!vd->tmpbuffer)
            goto error;
        vd->framebuffer =
            (unsigned char *) calloc(1, (size_t) vd->width * (vd->height + 8) * 2);
        break;
    case V4L2_PIX_FMT_RGB565: // buffer allocation for non varies on frame size formats
    case V4L2_PIX_FMT_YUYV:
    case V4L2_PIX_FMT_RGB24:
        vd->framebuffer =
            (unsigned char *) calloc(1, (size_t) vd->framesizeIn);
        break;
    default:
        fprintf(stderr, " should never arrive exit fatal !!\n");
        goto error;
        break;

    }

    if(!vd->framebuffer)
        goto error;
    return 0;
error:
    free(pglobal->in[id].in_parameters);
    free(vd->videodevice);
    free(vd->status);
    free(vd->pictName);
    CLOSE_VIDEO(vd->fd);
    return -1;
}




int uvcGrab(struct vdIn *vd)
{
//#define HEADERFRAME1 0xaf
    struct v4l2_buffer buf;
    int ret;


    if(vd->streamingState == STREAMING_OFF) {
        if(video_enable(vd))
            goto err;
    }

    memset(&buf, 0, sizeof(struct v4l2_buffer));
    buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
    buf.memory = V4L2_MEMORY_MMAP;

    //读取缓存
    ret = xioctl(vd->fd, VIDIOC_DQBUF, &buf);
    if(ret < 0) {
        perror("Unable to dequeue buffer");
        goto err;
    }

 


    if(VIDEO_MODE)
    {		
		double angle;

		c_detect(tHandle, vd->mem[buf.index], buf.bytesused,&angle);

        //角度精度控制在0.1°
        if(2 == VIDEO_MODE && !(angle > -0.1f && angle < 0.1f) ){
	       Robot_Camera_Servo_CTL(angle);
		}
    }


     //只支持yuyv
     //memcpy(vd->framebuffer, vd->mem[buf.index], (size_t) buf.bytesused);
	 if(buf.bytesused > vd->framesizeIn){
         memcpy(vd->framebuffer, vd->mem[buf.index], (size_t) vd->framesizeIn);
		 vd->tmpbytesused =  vd->framesizeIn;
	 }
     else{
         memcpy(vd->framebuffer, vd->mem[buf.index], (size_t) buf.bytesused);
		 vd->tmpbytesused =  buf.bytesused;
	 }
        
     
     vd->tmptimestamp =  buf.timestamp;



	//重新放入缓存队列
    ret = xioctl(vd->fd, VIDIOC_QBUF, &buf);
    if(ret < 0) {
        perror("Unable to requeue buffer");
        goto err;
    }

    return 0;

err:
    vd->signalquit = 0;
    return -1;
}

图像识别封装层

#include "opencv2/objdetect/objdetect_c.h"
#include "opencv2/imgproc/imgproc_c.h"
#include "opencv2/core/core_c.h"

#ifndef _HANDLE_WRAPPER_H_
#define _HANDLE_WRAPPER_H_

struct tagHandle;

#ifdef __cplusplus

extern "C" {

#endif

struct tagHandle *GetInstance(void);

void ReleaseInstance(struct tagHandle **handlerInstance);

uchar* detect(struct tagHandle *hand, uchar* buffer, int size, int* length);

#ifdef __cplusplus
};
#endif
#endif



#include "HandleWrapper.h"
#include "handle.h"

using namespace std;
using namespace cv;
#ifdef __cplusplus

extern "C" {

#endif

struct tagHandle
{
	Handle handle;
};

struct tagHandle *GetInstance(void)
{
	return new struct tagHandle;
};

void ReleaseInstance(struct tagHandle **handleInstance)
{
	delete *handleInstance;
	*handleInstance = 0;
}

uchar* detect(struct tagHandle *hand, uchar* buffer, int size, int* length)
{
	return hand->handle._detect(buffer,size,length);
}

#ifdef __cplusplus
};
#endif

基于opencv的图像识别实现层，

detectMultiScale用于通过滑动不同比例和大小的窗口并合并高置信度封闭样本来填充当前帧中的检测对象。

#ifndef _HANDLE_H_
#define _HANDLE_H_
#include "opencv2/objdetect.hpp"
#include "opencv2/imgproc.hpp"
#include "opencv2/core.hpp"
#include "opencv2/highgui.hpp"
#include "jpeglib.h"
#include <iostream>
using namespace std;
using namespace cv;

class Handle
{
public:
	Handle();	
	vector<Rect> true_object;
	vector<Point> points;
	Point temp_point;
	Rect rect;
	JSAMPROW pointer[1];
        struct jpeg_compress_struct c_cinfo;
        struct jpeg_decompress_struct d_cinfo;
	struct jpeg_error_mgr jerr;
	IplImage out;
	vector<int> param;
	Mat gray, img, resize_img;
	uchar* _detect(uchar* buffer,int size, int* length);
	CascadeClassifier cascade;
};

#endif





#include "handle.h"
#include <unistd.h>

using namespace std;
using namespace cv;

Handle::Handle()
{
	cascade.load("/etc/cascade.xml");

	c_cinfo.err = jpeg_std_error(&jerr);
    d_cinfo.err = jpeg_std_error(&jerr);

}

uchar* Handle::_detect(uchar* buffer, int size, int* length)
{
	jpeg_create_decompress(&d_cinfo);
	jpeg_mem_src(&d_cinfo, buffer, size);
	jpeg_read_header(&d_cinfo, TRUE);
	jpeg_start_decompress(&d_cinfo);
	uchar* data = new uchar[320*240*3];
	while (d_cinfo.output_scanline < d_cinfo.output_height)
	{
		pointer[0] = &data[d_cinfo.output_scanline*320*3];
                jpeg_read_scanlines(&d_cinfo, pointer, 1);
	}

	jpeg_finish_decompress(&d_cinfo);
    jpeg_destroy_decompress(&d_cinfo);
	
	Mat img(240, 320, CV_8UC3, data);

	vector<uchar> buf,vec_buf;

	resize(img, resize_img, cvSize(img.cols/4,img.rows/4));
        cvtColor(resize_img, gray, COLOR_BGR2GRAY);
        cascade.detectMultiScale(gray, true_object, 1.2, 5);
        vector<Rect>::const_iterator vi;
	
	for (vi = true_object.begin(); vi != true_object.end(); vi++)       
		rectangle(img, Point(vi->x*4, vi->y*4), Point(vi->x*4 + vi->width*4, vi->y*4 + vi->height*4), Scalar(255,128,0), 3);
        true_object.clear();

	uchar* img_data = img.data;  

	unsigned long len = 0;	
	uchar* out=NULL;

	jpeg_create_compress(&c_cinfo);

	jpeg_mem_dest(&c_cinfo, &out, &len);

	c_cinfo.image_width = 320;
        c_cinfo.image_height = 240;
        c_cinfo.input_components = 3;
        c_cinfo.in_color_space = JCS_RGB;
        jpeg_set_defaults(&c_cinfo);
        jpeg_set_quality(&c_cinfo, 75, TRUE);

	jpeg_start_compress(&c_cinfo, TRUE);

	while (c_cinfo.next_scanline < c_cinfo.image_height)
        {
                pointer[0] = &img_data[c_cinfo.next_scanline * 320*3];
                jpeg_write_scanlines(&c_cinfo, pointer, 1);
        }

	jpeg_finish_compress(&c_cinfo);
        jpeg_destroy_compress(&c_cinfo);
	free(data);

	*length = len;
	return out;
}

mjpg-streamer-r182/plugins/output_http/output_http.c

send_stream函数负责基于http协议封装每一帧mjpeg,可以在此处自定义帧头帧尾实现

自定义解析格式。

/******************************************************************************
Description.: Send a complete HTTP response and a stream of JPG-frames.
Input Value.: fildescriptor fd to send the answer to
Return Value: -
******************************************************************************/
void send_stream(cfd *context_fd, int input_number)
{
    unsigned char *frame = NULL, *tmp = NULL;
    int frame_size = 0, max_frame_size = 0;
    char buffer[BUFFER_SIZE] = {0};
    struct timeval timestamp;

   
    //define head

    #define IMG_HEAD_LEN   (20)
    static unsigned char img_head_num;

    unsigned char img_head[IMG_HEAD_LEN];
    memset(img_head,0,IMG_HEAD_LEN);

    //定义序号
    //定义格式
    img_head[1]  = 0;//deafult
    //定义长宽
    img_head[2]  = 320 >> 8;
    img_head[3]  = 320 & 0xFF;

    img_head[4]  = 240 >> 8;
    img_head[5]  = 240 & 0xFF;
    //定义总字节数
    
    //定义压缩格式
    img_head[10] = 0;//deafult
    //后面保留
   
 

    while(!pglobal->stop) {

        /* wait for fresh frames */
        pthread_mutex_lock(&pglobal->in[input_number].db);
        pthread_cond_wait(&pglobal->in[input_number].db_update, &pglobal->in[input_number].db);

        /* read buffer */
        frame_size = pglobal->in[input_number].size;

        /* check if framebuffer is large enough, increase it if necessary */
        if(frame_size > max_frame_size) {
            DBG("increasing buffer size to %d\n", frame_size);

            max_frame_size = frame_size + TEN_K;
            if((tmp = realloc(frame, max_frame_size)) == NULL) {
                free(frame);
                pthread_mutex_unlock(&pglobal->in[input_number].db);
                send_error(context_fd->fd, 500, "not enough memory");
                return;
            }

            frame = tmp;
        }

        /* copy v4l2_buffer timeval to user space */
        timestamp = pglobal->in[input_number].timestamp;

        memcpy(frame, pglobal->in[input_number].buf, frame_size);

        pthread_mutex_unlock(&pglobal->in[input_number].db);

        #ifdef MANAGMENT
        update_client_timestamp(context_fd->client);
        #endif

        /*
         * print the individual mimetype and the length
         * sending the content-length fixes random stream disruption observed
         * with firefox
         
        sprintf(buffer, "Content-Type: image/jpeg\r\n" \
                "Content-Length: %d\r\n" \
                "X-Timestamp: %d.%06d\r\n" \
                "\r\n", frame_size, (int)timestamp.tv_sec, (int)timestamp.tv_usec);
        DBG("sending intemdiate header\n");
        if(write(context_fd->fd, buffer, strlen(buffer)) < 0) break;*/

     // printf("sending frame\n");
     //	printf("%d\n", pglobal->in[input_number].size);
       

        img_head[0]  = img_head_num++; 
       
        img_head[6]  = frame_size >> 24;  
        img_head[7]  = frame_size >> 16; 
        img_head[8]  = frame_size >> 8;
        img_head[9]  = frame_size & 0xFF;

	write(context_fd->fd, img_head, IMG_HEAD_LEN); 
	if(write(context_fd->fd, frame, frame_size) < 0) break;

// 	int i=0;
//	int offset=0;
/*	while (i<128)
	{
		int out = write(context_fd->fd, frame+offset, 1200);
		offset+=out;
		i+=1;
	}
*/
//	printf("the out is %d\n", out);

       /*
        DBG("sending boundary\n");
        sprintf(buffer, "\r\n--" BOUNDARY "\r\n");
        if(write(context_fd->fd, buffer, strlen(buffer)) < 0) break;*/
    }

    free(frame);
}

小结

完整源码