OV5640、FPGA、Fast角点检测、非极大值抑制、以太网实时图像传输、Zynq、Linux、SDL视频显示

本项目基于Zynq-7020硬件平台和linux系统，实现了720p视频实时Fast特征检测，可用于SLAM中的Fast特征检测及二次开发。

Zynq下linux系统移植请参考：Zynq 传统方式移植 linux。如果嫌麻烦也可以直接用本文提供的镜像，获取完整Vivadao工程请联系微信：cxw3020302521。

​Fast模块实现从摄像头图像数据中实时检测Fast角点特征（包含3x3高斯滤波和7x7非极大值抑制），并将视频流通过AXI接口写入ddr中，然后Ps端从ddr中读取视频数据并通过UDP协议将数据传输到CP端。PC端接收UDP图像数据并通过SDL库进行显示。

FPGA Ps端linux系统下图像传输代码如下，项目中采用4个帧缓存,发送时选择最近的一帧图像发送到PC上进行显示，当前帧缓存号"ctrl[3]"由Fast模块实时生成，"ctrl[0]"、"ctrl[1]"、"ctrl[2]"负责控制Fast算法的阈值参数。

#include <time.h>
#include <stdio.h>
#include <fcntl.h>
#include <unistd.h>
#include <stdlib.h>
#include <string.h>
#include <sys/time.h>
#include <sys/mman.h>
#include <arpa/inet.h>
#include <sys/socket.h>
#include <netinet/in.h>

#define width   1280
#define height  720 

#define IMAG_ADDR  0x10000000
#define IMAG_SIZE  width*height*4
uint8_t *imag_buf  = NULL;

#define CTRL_ADDR 0x11000000
uint8_t *ctrl = NULL; 

#define PORT    8888            
#define MAXLINE (width*height*4/60)

int main(void)
{
    //UDP初始化
    int sockfd;
    struct sockaddr_in servaddr;
    if((sockfd = socket(AF_INET, SOCK_DGRAM, 0)) < 0) {
        perror("Socket creation failed");
        exit(1);
    }
    bzero(&servaddr, sizeof(servaddr));
    servaddr.sin_family = AF_INET;
    servaddr.sin_port = htons(PORT);
    servaddr.sin_addr.s_addr = inet_addr("192.168.1.2");
    
    //内存映射
    int fd_imag=open("/dev/mem", O_RDWR | O_NDELAY);/* 读写权限，非阻塞 */
    if(fd_imag<0){
        printf("open mem fd_imag failed\n");      
        return -1;  
    }  
    imag_buf =(uint8_t *)mmap(NULL, IMAG_SIZE*4, PROT_READ | PROT_WRITE, MAP_SHARED, fd_imag, IMAG_ADDR);  
    ctrl     =(uint8_t *)mmap(NULL, IMAG_SIZE*1, PROT_READ | PROT_WRITE, MAP_SHARED, fd_imag, CTRL_ADDR);

    //Fast 参数设置
    ctrl[0] = 15;
    ctrl[1] =  9;
    ctrl[2] = 35;

    //数据发送
    while(1) {
        //起始时间
        struct timeval T_Begin; gettimeofday(&T_Begin, NULL);

        //帧号
        uint8_t fnc=ctrl[3];if(fnc==0){fnc=3;}else{fnc=fnc-1;}

        //帧地址
        uint8_t *mag_bufc=imag_buf+fnc*IMAG_SIZE;

        //发送
        start: sendto(sockfd, "start", sizeof("start"), MSG_CONFIRM, (const struct sockaddr*)&servaddr, sizeof(servaddr));   
        for(unsigned int i=0;i<IMAG_SIZE/MAXLINE;i++){
            ssize_t n = sendto(sockfd, mag_bufc+i*MAXLINE, MAXLINE, MSG_CONFIRM, (const struct sockaddr*)&servaddr, sizeof(servaddr));
            if(n!=MAXLINE){
                goto start;
            } 
        } 

        //结束时间
        struct timeval T_End; gettimeofday(&T_End, NULL);
        float dt = 1000.0f * (T_End.tv_sec - T_Begin.tv_sec) + ((T_End.tv_usec - T_Begin.tv_usec) / 1000.0);//计算所耗毫秒数
        printf("%.2fms\t%d\n",dt,fnc);
    }

    //资源释放
    munmap(imag_buf, MAXLINE);
    close(fd_imag);
    close(sockfd);
    return 0;
}

PC端代码如下，采用SDL图像库进行视频显示，除了简单没啥优点，每个像素32bit，格式为: {[8bit:是否Fast特征,1为极大值,2为极小值],[8bit:灰度值],[16bit:RGB565]}。

#include <SDL.h>
#include <stdio.h>
#include <fcntl.h>
#include <unistd.h>
#include <stdlib.h>
#include <string.h>
#include <sys/time.h>
#include <sys/socket.h>
#include <netinet/in.h>

#define width   1280
#define height  720 
uint8_t rx_buf  [width*height*4];
uint8_t imag_buf[width*height*4];

#define PORT    8888        
#define MAXLINE (width*height*4/60)

typedef struct 
{
    unsigned int   x;
    unsigned int   y;
    float         cx;
    float         cy;
    unsigned int  cd;
    unsigned char dw;
}FV;
FV fv[width*height];
unsigned int  fvn=0;

int main() {
    //UDP初始化
    int sockfd;
    socklen_t len;
    struct sockaddr_in servaddr, cliaddr;
    sockfd = socket(AF_INET, SOCK_DGRAM, 0);
    if(sockfd < 0) {
        perror("Socket creation failed");
        exit(1);
    }
    bzero(&servaddr, sizeof(servaddr));
    servaddr.sin_family = AF_INET;
    servaddr.sin_addr.s_addr = htonl(INADDR_ANY);
    servaddr.sin_port = htons(PORT);
    if(bind(sockfd, (struct sockaddr*)&servaddr, sizeof(servaddr)) < 0) {
        perror("Bind failed");
        exit(1);
    }

    //SDL初始化
    SDL_Init(SDL_INIT_VIDEO);
    SDL_Event event;
    char QT_flag =0;
    SDL_Window  * window   = SDL_CreateWindow("WeVideo", SDL_WINDOWPOS_CENTERED, SDL_WINDOWPOS_CENTERED, width, height, SDL_WINDOW_SHOWN);
    SDL_Surface * sfc_win  = SDL_GetWindowSurface(window);
    SDL_Surface * sfc_img  = SDL_CreateRGBSurfaceFrom(imag_buf, width, height, 32, width*4, 0x000000FF, 0x0000FF00, 0x00FF0000, 0x00000000);
    
    //图像接收及显示
    while(!QT_flag) {
        //起始时间
        struct timeval T_Begin; gettimeofday(&T_Begin, NULL);
        
        //帧头检测
        start:while(1){
            len = sizeof(cliaddr);
            ssize_t n = recvfrom(sockfd, (char*)imag_buf, MAXLINE, MSG_WAITALL, (struct sockaddr*)&cliaddr, &len);
            if(n==sizeof("start") && strcmp(imag_buf,"start")==0){
                break;
            }
        }

        //收数据
        rx:for(unsigned int i=0;i<width * height * 4/MAXLINE;i++){
            len = sizeof(cliaddr);
            ssize_t n = recvfrom(sockfd, ((char*)rx_buf)+i*MAXLINE, MAXLINE, MSG_WAITALL, (struct sockaddr*)&cliaddr, &len);
            if(n!=MAXLINE){
                if(n==sizeof("start") && strcmp(rx_buf,"start")==0){
                    goto rx;   
                }
                else{
                    goto start;
                }
            }     
        }

        //图像显示
        fvn=0;for(int i=0;i<height;i++){
            for(int j=0;j<width;j++){
                if(i>=14&&j>=14&&i<height-14&&j<width-14){
                    if(rx_buf[(i*width+j)*4+3]==1||rx_buf[(i*width+j)*4+3]==2){//Fast特征标记
                        fv[fvn  ].y=i;
                        fv[fvn++].x=j;
                        ((unsigned*)imag_buf)[(i  )*width+(j  )]=0x0000FF00;
                        ((unsigned*)imag_buf)[(i+1)*width+(j  )]=0x000007e0;
                        ((unsigned*)imag_buf)[(i-1)*width+(j  )]=0x0000FF00;
                        ((unsigned*)imag_buf)[(i  )*width+(j+1)]=0x000007e0;
                        ((unsigned*)imag_buf)[(i  )*width+(j-1)]=0x0000FF00;
                    }
                    else{
                        unsigned temp=((unsigned*)rx_buf)[i*width+j];

                        // //RGB显示
                        // imag_buf[(i*width+j)*4+3]=(temp&0x00ff0000)>>16;
                        // imag_buf[(i*width+j)*4+2]=(temp&0x0000f800)>> 8;
                        // imag_buf[(i*width+j)*4+1]=(temp&0x000007e0)>> 3;
                        // imag_buf[(i*width+j)*4+0]=(temp&0x0000001F)<< 3;

                        //灰度显示
                        imag_buf[(i*width+j)*4+3]=(temp&0x00ff0000)>>16;
                        imag_buf[(i*width+j)*4+2]=(temp&0x00ff0000)>>16;
                        imag_buf[(i*width+j)*4+1]=(temp&0x00ff0000)>>16;
                        imag_buf[(i*width+j)*4+0]=(temp&0x00ff0000)>>16;
                    }
                }
                else{
                    ((unsigned*)imag_buf)[(i  )*width+(j  )]=0x00000000;
                }
            }
        }
        SDL_BlitSurface(sfc_img,NULL,sfc_win,NULL);
        SDL_UpdateWindowSurface(window);
        
        //结束时间
        struct timeval T_End; gettimeofday(&T_End, NULL);
        float dt = 1000.0f * (T_End.tv_sec - T_Begin.tv_sec) + ((T_End.tv_usec - T_Begin.tv_usec) / 1000.0);//计算所耗毫秒数
        printf("%.2fms\t%d\n",dt,fvn);

        //窗口事件检测
        while(SDL_PollEvent(&event)){
            if(event.type==SDL_QUIT){
                QT_flag=1;
            }
        }
    }

    //资源释放
    exit(0);
    return 0;
}