基于形状数直方图匹配的手势识别

要点

以形状数直方图的方式提取手的形状轮廓,用卡方距离进行直方图的比较,从而进行手势识别,能判断手的张和握两种状态

前言

学过形状数,学过直方图比较,两者结合来为形状数直方图,进行形状识别,算是一次另类的尝试.
本方法存了张手和握拳两个种类的样本.通过分别计算目标和两个种类的卡方距离,
来判断目标更靠近哪一类样本,从而把目标归类.
课上另一组同学的方法思路是先给两种样本划线,再判断新来的目标是在线的哪一边,从而归类.

知识准备

轮廓表达

多边形表达的目标：要用尽可能少的线段，来代表边界，并保持边界的基本形状，这样就可以用较少的数据和较简洁的形式来表达和描述边界
在数字图像中，如果多边形的线段数与边界上的点数相等，则多边形可以完全准确的表达边界

链码

利用一系列具有特定长度和方向的相连的直线段来表示目标的边界，每个线段的长度固定，而方向数目取为有限，只要边界的起点用（绝对）坐标表示，其余点只用方向来代表偏移量。

差分码

用链码的1阶差分来重新构造1个序列(1个表示原链码各段之间方向变化的新序列)，相当于把链码进行旋转归一化

形状数

是一种基于链码的边界形状描述符,形状数是其自然数值最小的差分码。

实现平台

语言:C++
编译器:VS 2017
插件:OPENCV-3.3.1

实验思路

1.每张图片提取形状数直方图

2.比较两张图片的形状数直方图

3.对张和握分别求出目标和多个模板的卡方距离均值

4.比较所求的两个均值

实验过程

素材

若只拿一张张手和一张握拳的图作为模板,与目标图片则很难比对成功.因为张手和握手的程度会有差异,仅用一份模板不保证能有代表性,也不能消除偶然性.所以我们选取了多张模板图参与比对.

提取

结果

分析

成果

• 完成了基于形状数直方图,对张手和握手图的识别

优势

• 采用易于理解的形状数直方图来反映形状
• 用比较直方图的卡方距离方法进行形状比对,通俗易懂

劣势

• 图片拍摄要求背景与肤色相异并为纯色,手和相机的距离不能太大也不能太小, 图片为1440x1080的分辨率
• 相机要闪光灯,仅自然光不能很好地提取轮廓
• 对某些图提取链码时易出现断格而导致提取不成功
• 仅仅采用一种方式进行形状比对,没有结合其他方式横向考虑

源码

#include <iostream>   
#include <string>  
#include<math.h>
#include <iomanip>  
#include <sstream>
#include "stdafx.h"
#include <opencv2/imgproc/imgproc.hpp>  
#include <opencv2/core/core.hpp>       
#include <opencv2/highgui/highgui.hpp>
#include<Windows.h>
#include<algorithm>
#include<vector>
#include<fstream>

using namespace cv;
using namespace std;
std::string open[20], close[20];
int openn,closen;
void calcul(double *b, int *a, int an) {
    for (int i = 0; i < an; i++) {
            b[a[i]]++;
    }
    for (int i = 0; i < 8; i++) {
        b[i] = b[i] / an;
    }
}
double kafang(double *a ,double *b) {
    double x = 0;
    for (int i = 0; i < 8; i++) {
        double anb = a[i] + b[i];
        if (anb == 0) {
            anb = 0.000001;//避免除数是0
        }
        x += pow(a[i] - b[i], 2) /anb ;
    }
    //printf("kafang=%lf\n", x);
    return x;
}

typedef struct {
    int num;
    int nei[8];
    int xm;
    int xx;
    int ym;
    int yx;
    int pass;
}part;
void Reverse(int* a, int p, int q)
{
    for (; p < q; p++, q--)
    {
        int temp = a[q];
        a[q] = a[p];
        a[p] = temp;
    }
}

void RightShift(int* a, int n, int k)
{
    k %= n;
    Reverse(a, 0, n - k - 1);
    Reverse(a, n - k, n - 1);
    Reverse(a, 0, n - 1);
}
void guiyi(int *a, int lianmanum) {
    int b[10800], minb[10800];
    memcpy(b, a, lianmanum * sizeof(int));
    memcpy(minb, a, lianmanum * sizeof(int));

    for (int i = 0; i < lianmanum; i++) {
        RightShift(b, lianmanum, 1);//右移
        int flag = 0;
        for (int j = 0; j < lianmanum; j++) {
            if (minb[j] < b[j]) {//右移后的数大
                flag = 1; break;
            }
            if (minb[j] > b[j]) {//右移后的数小
                break;
            }

        }
        if (flag == 0) {
            memcpy(minb, b, lianmanum * sizeof(int));
        }
    }
    memcpy(a, minb, lianmanum * sizeof(int));
}
void chafen(int *b, int *a, int lianmanum) {
    for (int i = 0; i < lianmanum; i++) {
        b[i] = (a[i + 1] - a[i]) % 8;
        if (b[i] < 0)b[i] += 8;

    }
}
void gao(double zhifang[8],vector< Point > zb, const std::string hand) {
    int size1 = zb.size();
    //1440*1080
    int a = 0;
    part sq[10800];
    for (int i = 0; i < 10800; i++) {//给集找x,y值
        sq[i].xm = ((i + 1) % 120) * 12;
        sq[i].xx = sq[i].xm - 12;
        sq[i].ym = (i / 120 + 1) * 12;
        sq[i].yx = sq[i].ym - 12;
    }

    for (int i = 0; i < 10800; i++) {//给集找邻居集
        sq[i].num = 0;
        sq[i].pass = 1;
        if (i % 120 == 0) {//left
            sq[i].nei[3] = 12000;
            sq[i].nei[4] = 12000;
            sq[i].nei[5] = 12000;
            sq[i].nei[2] = i - 120;
            sq[i].nei[1] = i - 119;
            sq[i].nei[0] = i + 1;
            sq[i].nei[6] = i + 120;
            sq[i].nei[7] = i + 121;
        }
        else if (i % 120 == 119) {//right
            sq[i].nei[1] = 12000;
            sq[i].nei[0] = 12000;
            sq[i].nei[7] = 12000;
            sq[i].nei[3] = i - 121;
            sq[i].nei[2] = i - 120;
            sq[i].nei[4] = i - 1;
            sq[i].nei[5] = i + 119;
            sq[i].nei[6] = i + 120;
        }
        else {
            sq[i].nei[3] = i - 121;
            sq[i].nei[2] = i - 120;
            sq[i].nei[1] = i - 119;
            sq[i].nei[4] = i - 1;
            sq[i].nei[0] = i + 1;
            sq[i].nei[5] = i + 119;
            sq[i].nei[6] = i + 120;
            sq[i].nei[7] = i + 121;
        }

        if (0 <= i && i <= 119) {
            sq[i].nei[2] = 12000;
            sq[i].nei[1] = 12000;
            sq[i].nei[3] = 12000;
        }
        else if (10680 <= i&&i <= 10799) {
            sq[i].nei[5] = 12000;
            sq[i].nei[6] = 12000;
            sq[i].nei[7] = 12000;
        }
        for (int uu = 0; uu < 8; uu++) {
            if (sq[i].nei[uu] < 0 || sq[i].nei[uu] >10799)
                sq[i].nei[uu] = 12000;
        }
    }

    for (int i = 0; i < 10800; i++) {
        for (int y = 0; y < 8; y++) {
            //  printf("%d %d:%d\n", i, y, sq[i].nei[y]);
        }
    }



    for (int i = 0; i < size1; i++)//将点归集
        for (int j = 0; j < 10800; j++)
            if (zb[i].x >= sq[j].xx&&zb[i].x<sq[j].xm&&zb[i].y>sq[j].yx&&zb[i].y < sq[j].ym)
                sq[j].num++;

    for (int i = 0; i < 10800; i++) {
        //  printf("%d  %d\n", i, sq[i].num);
    }

    int maxsq = 0;
    int maxsqid = 0;
    for (int i = 0; i < 10800; i++) {//得到一个点最多的集
        if (sq[i].num > maxsq) {
            maxsq = sq[i].num;
            maxsqid = i;
        }
    }
    //if (hand != "0")printf("\n----MAX----%d\n", maxsqid);


    int loc[10800];
    memset(loc, 0, sizeof(loc));
    loc[0] = maxsqid;
    int lianma[10800];
    //memset(lianma, 0, sizeof(lianma));
    int b[10800];
    int c[10800];
    //memset(b, 0, sizeof(b));
    int lianmanum = 0;
    int bijiao = 0;
    int p;
    int yy = 0;
    sq[loc[0]].pass = 0;
    for (p = 0; p < 10800; p++) {//得到连码
        if (p == 10) {
            sq[loc[0]].pass = 1;
        }
        bijiao = 0;
        for (int u = 0; u < 8; u++) {
            yy = sq[loc[p]].nei[u];//取周边点id
            if (yy <= 10799) {//判断是不是有效值
                if (sq[yy].pass == 1) {//比较最大
                    if (sq[yy].num > bijiao) {
                        bijiao = sq[loc[p]].nei[u];
                        loc[p + 1] = sq[loc[p]].nei[u];
                        lianma[lianmanum] = u;
                    }
                }
            }
        }

        //printf("%d ", loc[p]);
        sq[loc[p + 1]].pass = 0;
        lianmanum++;
        if (loc[p + 1] == loc[0])
            break;
        if (loc[p + 1] == 0)
            break;
    }p++;
    //printf("\nwang ge:\n");
    /*for (int i = 0; i < p; i++) {
        printf("%d ", loc[i]);

    }*/
    guiyi(lianma, lianmanum);
    if (hand != "0")printf("\nlian ma:\n");
    for (int i = 0; i < lianmanum; i++) {
        if (hand != "0")printf("%d ", lianma[i]);
    }

    chafen(b, lianma, lianmanum);

    if (hand != "0")printf("\ncha fen ma:\n");
    for (int i = 0; i < lianmanum - 1; i++) {
        if (hand != "0")printf("%d ", b[i]);
    }

    guiyi(b, lianmanum);
    if (hand != "0")printf("\nxing zhuang shu:\n");
    for (int i = 0; i < lianmanum - 1; i++) {
        if (hand != "0")printf("%d ", b[i]);
    }
    calcul(zhifang, b, lianmanum - 1);
    fstream f;

    std::string fname = hand + ".txt";
    if (hand != "0")f.open(fname.c_str(), ios::out);
    if (hand != "0")printf("\nzhi fang:\n");
    for (int i = 0; i < 8; i++) {
        if (hand != "0")printf("%lf ", zhifang[i]);
        if (hand != "0")f << zhifang[i] << " ";
    }
    f.close();

}
double cmpfile(double zhifang[8], std::string s[10], int u) {
    double filezf[8];
    double avgkf = 0;
    for (int i = 0; i < u; i++) {
        fstream f;
        std::string handtxt = s[i] + ".txt";
        f.open(handtxt, ios::in);
        int j = 0;
        while (1) {
            double x;
            f >> x;
            if (f.peek() == EOF)break;
            filezf[j] = x;
            j++;
        }
        f.close();
        double kf = kafang(zhifang, filezf);
        avgkf += kf;
    }
    avgkf = avgkf / u;
    return avgkf;
}
void judge(const std::string hand,int muban) {//得到形状数直方图
    double zhifang[8];
    int delay = 1;

    char c;
    int frameNum = -1;          // Frame counter  
    bool bHandFlag = false;
    std::string handjpg = hand + ".jpg";
    VideoCapture captRefrnc(handjpg);
    Size refS = Size((int)captRefrnc.get(CV_CAP_PROP_FRAME_WIDTH),
        (int)captRefrnc.get(CV_CAP_PROP_FRAME_HEIGHT));
    const char* WIN_SRC = "Source";
    const char* WIN_RESULT = "Result";
    // Windows  
    namedWindow(WIN_SRC, 0);
    resizeWindow(WIN_SRC, 480, 320);
    moveWindow(WIN_SRC, 0, 0);
    namedWindow(WIN_RESULT, 0);
    resizeWindow(WIN_RESULT, 480, 320);
    moveWindow(WIN_RESULT, 481, 0);
    vector< vector<Point> > contours;   // 轮廓  
    vector< vector<Point> > filterContours; // 筛选后的轮廓  
    vector< Vec4i > hierarchy;    // 轮廓的结构信息  
    vector< Point > hull; // 凸包络的点集  
    vector< Point > gaohull;
    Mat frame;  // 输入视频帧序列  
    Mat frameHSV;   // hsv空间  
    Mat mask(frame.rows, frame.cols, CV_8UC1);  // 2值掩膜  
    Mat dst(frame); // 输出图像  

        captRefrnc >> frame;


        imshow(WIN_SRC, frame);

        // 中值滤波，去除噪声  
        medianBlur(frame, frame, 5);
        cvtColor(frame, frameHSV, CV_BGR2HSV);
        Mat dstTemp1(frame.rows, frame.cols, CV_8UC1);
        Mat dstTemp2(frame.rows, frame.cols, CV_8UC1);
        // 对HSV空间进行量化，得到2值图像，亮的部分为手的形状  
        inRange(frameHSV, Scalar(0, 30, 30), Scalar(96, 170, 256), dstTemp1);
        inRange(frameHSV, Scalar(156, 30, 30), Scalar(180, 170, 256), dstTemp2);
        bitwise_or(dstTemp1, dstTemp2, mask);
        frame.copyTo(dst, mask);
        contours.clear();
        hierarchy.clear();
        filterContours.clear();
        // 得到手的轮廓  
        findContours(mask, contours, hierarchy, CV_RETR_EXTERNAL, CV_CHAIN_APPROX_SIMPLE);
        // 去除伪轮廓  
        for (size_t i = 0; i < contours.size(); i++)
        {
            if (fabs(contourArea(Mat(contours[i]))) > 30000) //判断手进入区域的阈值  
            {
                filterContours.push_back(contours[i]);

            }
        }

        // 画轮廓  
        drawContours(dst, filterContours, -1, Scalar(255, 255, 255), 3);
        Point pi;
        for (int i = 0; i < filterContours.size(); i++) {
            for (int j = 0; j < filterContours[i].size(); j++) {
                //printf("(%d,%d) ", filterContours[i][j].x, filterContours[i][j].y);

                if (!gaohull.empty()) {
                    pi.x = (4 * pi.x + filterContours[i][j].x) / 5;
                    pi.y = (4 * pi.y + filterContours[i][j].y) / 5;
                    //printf("pi(%d,%d) ", pi.x, pi.y);
                    gaohull.push_back(pi);
                    pi.x = (3 * pi.x +2*filterContours[i][j].x) / 5;
                    pi.y = (3 * pi.y + 2*filterContours[i][j].y) / 5;
                    //printf("pi(%d,%d) ", pi.x, pi.y);
                    gaohull.push_back(pi);
                    pi.x = (2 * pi.x + 3*filterContours[i][j].x) / 5;
                    pi.y = (2 * pi.y + 3*filterContours[i][j].y) / 5;
                    //printf("pi(%d,%d) ", pi.x, pi.y);
                    gaohull.push_back(pi);
                    pi.x = (pi.x + 4 * filterContours[i][j].x) / 5;
                    pi.y = (pi.y + 4 * filterContours[i][j].y) / 5;
                    //  printf("pi(%d,%d) ",pi.x, pi.y);
                    gaohull.push_back(pi);
                }
                gaohull.push_back(filterContours[i][j]);
                pi.x = filterContours[i][j].x;
                pi.y = filterContours[i][j].y;

            }
        }
        memset(zhifang, 0, sizeof(zhifang));printf("\n%s:\n",hand.c_str());
        if(muban==1)gao(zhifang,gaohull, hand);//得到形状数直方图
        if (muban == 0) {//当前图和之前存的模板比较

            gao(zhifang, gaohull, "0");//得到形状数直方图


            double openx = cmpfile(zhifang, open, openn);
            printf("OPEN:compare with %d templates\nChi-square Distance=%lf\n",openn,openx);

            double closex = cmpfile(zhifang,close, closen);
            printf("CLOSE:compare with %d templates\nChi-square Distance=%lf\n",closen,closex);
            if (openx < closex) {
                printf("result:open hand\n");
            }
            else {
                printf("result:close hand\n");
            }





        }
        // 得到轮廓的凸包络  
        for (size_t j = 0; j < filterContours.size(); j++)
        {
            convexHull(Mat(filterContours[j]), hull, true);
            int hullcount = (int)hull.size();

            for (int i = 0; i < hullcount - 1; i++)
            {
                line(dst, hull[i + 1], hull[i], Scalar(255, 0, 255), 2, CV_AA);
            }
            line(dst, hull[hullcount - 1], hull[0], Scalar(155, 0, 255), 2, CV_AA);

        }


        imshow(WIN_RESULT, dst);dst.release();
        if(muban==0)waitKey(2000);


}
int main()
{
    //初始化模板
    openn = 8;//成对模板数
    closen =8;
    for (int i = 0; i < 20; i++) {
        stringstream s;
        string snum;
        int num = i + 1;
        s << num;
        s >> snum;
        s.clear();
        open[i] =  "open" +snum ;
        close[i] =  "close"+ snum;
    }
    int t;
    printf("Reload templates?(1:yes,0:no)\n");
    scanf("%d", &t);
    if (t == 1) {//将模板的形状数直方图存入文件
        for (int i = 0; i < openn; i++) {
            judge(open[i], 1);
        }
        for (int i = 0; i < closen; i++) {
            judge(close[i], 1);
        }
    }
    else {//测试图像跟存好的模板比较
        judge("test1", 0);
        judge("test2", 0);
        judge("test3", 0);
        judge("test4", 0);
        judge("test5", 0);
        judge("test6", 0);
    }
    system("pause");

}