基于Kinect姿态比对的太极拳动作检测方案改进

    月底啦月底啦对本月在Kinect姿态识别的问题上做一个总结。本文所有工程内容均基于Unity 5.6与Kinect V2.0

Kinect自带的PoseDetectorScript在太极拳云手动作识别上出现的问题

        （绿色模型是直接根据Kinect获取用户图像生成的骨骼数据生成，动画模型是由骨骼数据蒙皮生成）

        理论上来说，骨骼数据赋予模型能够减少用户身形给匹配度带来的误差。可是，在肘关节角度，手臂长度上因为转体侧身问题，骨骼数据蒙皮后产生了较大偏差。

为了解决这个问题，直接舍弃采用PoseDetectorScript的方案。保留其中采用关节与关节之间角度差作为匹配度判断的标准。

 

 使用Visual Gesture Builder进行姿态匹配 

此后也在网上搜了一些可用作姿态识别的方案，其中有一种是基于机器学习的方案。针对Kinect V2,具体实现可以结合下面的链接

https://blog.csdn.net/nijiayy/article/details/68926979

下面分别是我做的training和testing

       

左侧是训练标注，右侧为testing生成的矩形波判定图

矩形波的波峰表示由训练后的模型判定该姿势为真。其中蓝色短线部分是人为标注姿势标准的部分。的确在少量的训练样本下已经有了一定的效果。但是因为单关节的不匹配被人工标注为false，降低了该模型的判断标准姿势的置信度。与此同时存在需要大量样本输入与置信度较低的问题，所以开始在别的方案上作尝试。

基于Kinect骨骼数据直接比对的方案

        最后采用了基于Kinect骨骼数据直接比对的方案，在转体与侧身的效果上表现还不错。其实如果没有由Kinect自带的姿态比对Demo或许也不会绕那么大的弯子。直接比对作为最基础的方案其实是应该首先被想到的。因为之前做的简单动作比对是建立在骨骼数据蒙皮的情况上，最初是想在原方案上做些许改进就能用的，直到后来为了动画模型的匹配弯子越绕越大，最后就直接放弃了那种比对方法。

        话不多说，直接开始。

如何得到标准骨骼数据

标准动作下的从Kinect获得骨骼数据即为标准骨骼数据

在KinectDemo中，KinectAvatarDemo2 里面有个cubemanController脚本（就是通过Kinect数据生成可视化cube在场景中显示的），可在场景中生成由cube构成的骨骼模型，骨骼与骨骼之间还有连线。

另外，这里有一份简易脚本，可以在场景中生成由sphere组成的骨骼模型

https://blog.csdn.net/l1336037686/article/details/79952033   见其中的案例四

如何进行比对

将生成骨骼数据直接由场景中获取出来，一个一个骨骼节点地读。（我试想过写一个脚本将Kinect数据保存为文件，但是因为急于比对看效果，就直接从场景中读了）然后保存为Vector数组。比对与约束的脚本如下。沿用了部分PoseDetectorScript的内容。

using UnityEngine;
using System.Collections;
using System.Collections.Generic;
using System.Text;

public class TaijiPoseDetector : MonoBehaviour {

    [Tooltip("List of joints to compare.")]
    public List<KinectInterop.JointType> poseJoints = new List<KinectInterop.JointType>();

    [Tooltip("Threshold, above which we consider the pose is matched.")]
    public float matchThreshold = 0.7f;

    [Tooltip("GUI-Text to display information messages.")]
    public UnityEngine.UI.Text infoText;

    // match percent (between 0 and 1)
    private float fMatchPercent = 0f;
    // whether the pose is matched or not
    private bool bPoseMatched = false;

    private long userID = 0;
    private KinectManager kinectManager = null;

    Vector3[] bones_taiji1 = new Vector3[]
{
        new Vector3(-0.1269219f,-0.1779729f,2.144094f),
        new Vector3(-0.1397094f,0.1497776f,2.116632f),
        new Vector3(-0.1515284f,0.4661052f,2.07562f),

        new Vector3(-0.2245634f,0.6100131f,2.088759f),
        new Vector3(-0.3133955f,0.3584203f,2.260613f),
        new Vector3(-0.5098417f,0.4435955f,2.287583f),

        new Vector3(-0.6600204f,0.5048931f,2.129346f),
        new Vector3(-0.6660378f,0.5076889f,2.098843f),
        new Vector3(-0.04722613f,0.3208956f,1.975395f),

        new Vector3(-0.1542291f,0.1432333f,1.871686f),
        new Vector3(-0.3563171f,-0.05396684f,1.84365f),
        new Vector3(-0.4461585f,-0.1256309f,1.851563f),

        new Vector3(-0.1734671f,-0.1820317f,2.17852f),
        new Vector3(-0.3282039f,-0.4669569f,2.043032f),
        new Vector3(-0.3884017f,-0.900813f,2.238681f),

        new Vector3(-0.3980916f,-0.9317673f,2.167058f),
        new Vector3(-0.07615215f,-0.1676129f,2.036556f),
        new Vector3(0.09549776f,-0.4468162f,2.045715f),

        new Vector3(0.3352442f, -0.8987229f, 2.2612760f),
        new Vector3(0.3424421f, -0.9373463f, 2.1723000f),
        new Vector3(-0.1487706f, 0.3886429f, 2.0881840f),

        new Vector3(-0.6819705f, 0.5310901f, 1.9950510f),
        new Vector3(-0.6770689f, 0.4791934f, 1.9909120f),
        new Vector3(-0.5145940f, -0.1775658f, 1.8596680f),

        new Vector3(-0.4989214f, -0.1280237f, 1.8697780f)


};

    /// <summary>
    /// Gets the pose match percent.
    /// </summary>
    /// <returns>The match percent (value between 0 and 1).</returns>
    public float GetMatchPercent()
    {
        return fMatchPercent;
    }


    /// <summary>
    /// Determines whether the target pose is matched or not.
    /// </summary>
    /// <returns><c>true</c> if the target pose is matched; otherwise, <c>false</c>.</returns>
    public bool IsPoseMatched()
    {
        return bPoseMatched;
    }

    //get angle of vector
    public float GetAngle(KinectInterop.JointType poseJoint1, KinectInterop.JointType nextJoint1, KinectInterop.JointType poseJoint2, KinectInterop.JointType nextJoint2, bool isPose)
    {
        Vector3 vBone1, vBone2;

        vBone1 = isPose ? (bones_taiji1[(int)nextJoint1] - bones_taiji1[(int)poseJoint1]).normalized
            : (kinectManager.GetJointKinectPosition(userID, (int)nextJoint1) - kinectManager.GetJointKinectPosition(userID, (int)poseJoint1)).normalized;
        vBone2 = isPose ? (bones_taiji1[(int)nextJoint2] - bones_taiji1[(int)poseJoint2]).normalized
            : (kinectManager.GetJointKinectPosition(userID, (int)nextJoint2) - kinectManager.GetJointKinectPosition(userID, (int)poseJoint2)).normalized;

        return Vector3.Angle(vBone1, vBone2);
    }

    //get 2D angle of vector
    public float GetAngle_2D(KinectInterop.JointType poseJoint1, KinectInterop.JointType nextJoint1, KinectInterop.JointType poseJoint2, KinectInterop.JointType nextJoint2, bool isPose)
    {
        Vector2 vBone1, vBone2;
        //vec3 to vec2 省略 z轴坐标
        vBone1 = isPose ? (bones_taiji1[(int)nextJoint1] - bones_taiji1[(int)poseJoint1]).normalized
            : (kinectManager.GetJointKinectPosition(userID, (int)nextJoint1) - kinectManager.GetJointKinectPosition(userID, (int)poseJoint1)).normalized;
        vBone2 = isPose ? (bones_taiji1[(int)nextJoint2] - bones_taiji1[(int)poseJoint2]).normalized
            : (kinectManager.GetJointKinectPosition(userID, (int)nextJoint2) - kinectManager.GetJointKinectPosition(userID, (int)poseJoint2)).normalized;

        return Vector2.Angle(vBone1, vBone2);
    }


    void Update()
    {
        kinectManager = KinectManager.Instance;

        if (kinectManager != null && kinectManager.IsInitialized())
        {
            // get the difference
            string sDiffDetails = string.Empty;
            string sAngle = string.Empty;

            fMatchPercent = 1f - GetPoseDifference(true, ref sDiffDetails);
            bPoseMatched = (fMatchPercent >= matchThreshold);

            StringBuilder sbDetails = new StringBuilder();

            //肩关节 转身判定
            float spineshoulder_angle_p = GetAngle_2D(KinectInterop.JointType.SpineShoulder, KinectInterop.JointType.ShoulderLeft, KinectInterop.JointType.SpineShoulder, KinectInterop.JointType.ShoulderRight, true);
            float spineshoulder_angle_a = GetAngle_2D(KinectInterop.JointType.SpineShoulder, KinectInterop.JointType.ShoulderLeft, KinectInterop.JointType.SpineShoulder, KinectInterop.JointType.ShoulderRight, false);
            float spineshoulder_angle_diff = spineshoulder_angle_a - spineshoulder_angle_p;

            float spineshoulder_match = 100 - Mathf.Abs(spineshoulder_angle_diff) * 3;

            sbDetails.AppendFormat("双肩匹配度  {0:F0} {1}", spineshoulder_match,
                spineshoulder_match > 80 ? " - Matched" : " ").AppendLine();


            //user右手肘判定
            float shoulderleft_angle_p = GetAngle(KinectInterop.JointType.SpineShoulder, KinectInterop.JointType.ShoulderLeft, KinectInterop.JointType.ShoulderLeft, KinectInterop.JointType.ElbowLeft, true);
            float shoulderleft_angle_a = GetAngle(KinectInterop.JointType.SpineShoulder, KinectInterop.JointType.ShoulderLeft, KinectInterop.JointType.ShoulderLeft, KinectInterop.JointType.ElbowLeft, false);
            float shoulderleft_angle_diff = shoulderleft_angle_p - shoulderleft_angle_a;

            sbDetails.AppendFormat("左肩 - {0:F0} ", Mathf.Abs(shoulderleft_angle_diff));



            float elbowleft_angle_p = GetAngle(KinectInterop.JointType.ShoulderLeft, KinectInterop.JointType.ElbowLeft, KinectInterop.JointType.ElbowLeft, KinectInterop.JointType.WristLeft, true);
            float elbowleft_angle_a = GetAngle(KinectInterop.JointType.ShoulderLeft, KinectInterop.JointType.ElbowLeft, KinectInterop.JointType.ElbowLeft, KinectInterop.JointType.WristLeft, false);
            float elbowleft_angle_diff = elbowleft_angle_p - elbowleft_angle_a;


            sbDetails.AppendFormat("左肘 - {0:F0} ", Mathf.Abs(elbowleft_angle_diff));

            float wristleft_angle_p = GetAngle(KinectInterop.JointType.ElbowLeft, KinectInterop.JointType.WristLeft, KinectInterop.JointType.WristLeft, KinectInterop.JointType.HandLeft, true);
            float wristleft_angle_a = GetAngle(KinectInterop.JointType.ElbowLeft, KinectInterop.JointType.WristLeft, KinectInterop.JointType.WristLeft, KinectInterop.JointType.HandLeft, false);
            float wristleft_angle_diff = wristleft_angle_p - wristleft_angle_a;

            sbDetails.AppendFormat("左手腕 - {0:F0} ", Mathf.Abs(wristleft_angle_diff));

            float arm_left_match = (400 - Mathf.Abs(shoulderleft_angle_diff) * 4 - Mathf.Abs(elbowleft_angle_diff) * 4 - Mathf.Abs(wristleft_angle_diff) * 1) / 4;

            sbDetails.AppendFormat("左臂匹配度  {0:F0} {1}", arm_left_match, arm_left_match > 80 ? " - Matched" : " ").AppendLine();

            //user左手肘判定
            float shoulderright_angle_p = GetAngle(KinectInterop.JointType.SpineShoulder, KinectInterop.JointType.ShoulderRight, KinectInterop.JointType.ShoulderRight, KinectInterop.JointType.ElbowRight, true);
            float shoulderright_angle_a = GetAngle(KinectInterop.JointType.SpineShoulder, KinectInterop.JointType.ShoulderRight, KinectInterop.JointType.ShoulderRight, KinectInterop.JointType.ElbowRight, false);
            float shoulderright_angle_diff = shoulderright_angle_p - shoulderright_angle_a;
            

            sbDetails.AppendFormat("右肩 - {0:F0} ", Mathf.Abs(shoulderright_angle_diff));


            float elbowright_angle_p = GetAngle(KinectInterop.JointType.ShoulderRight, KinectInterop.JointType.ElbowRight, KinectInterop.JointType.ElbowRight, KinectInterop.JointType.WristRight, true);
            float elbowright_angle_a = GetAngle(KinectInterop.JointType.ShoulderRight, KinectInterop.JointType.ElbowRight, KinectInterop.JointType.ElbowRight, KinectInterop.JointType.WristRight, false);
            float elbowright_angle_diff = elbowright_angle_p - elbowright_angle_a;

            sbDetails.AppendFormat("右肘 - {0:F0} ", Mathf.Abs(elbowright_angle_diff));

            float arm_right_match = 100 - Mathf.Abs(shoulderright_angle_diff) - Mathf.Abs(elbowright_angle_diff);

            sbDetails.AppendFormat("右臂匹配度  {0:F0} {1}", arm_right_match, arm_right_match > 80 ? " - Matched" : " ").AppendLine();


            //蹲下判定
            float kneeleft_angle_p = GetAngle(KinectInterop.JointType.HipLeft, KinectInterop.JointType.KneeLeft, KinectInterop.JointType.KneeLeft, KinectInterop.JointType.AnkleLeft, true);
            float kneeleft_angle_a = GetAngle(KinectInterop.JointType.HipLeft, KinectInterop.JointType.KneeLeft, KinectInterop.JointType.KneeLeft, KinectInterop.JointType.AnkleLeft, false);
            float kneeleft_angle_diff = kneeleft_angle_p - kneeleft_angle_a;

            sbDetails.AppendFormat("左膝  - {0:F0} ", Mathf.Abs(kneeleft_angle_diff));


            float kneeright_angle_p = GetAngle(KinectInterop.JointType.HipRight, KinectInterop.JointType.KneeRight, KinectInterop.JointType.KneeRight, KinectInterop.JointType.AnkleRight, true);
            float kneeright_angle_a = GetAngle(KinectInterop.JointType.HipRight, KinectInterop.JointType.KneeRight, KinectInterop.JointType.KneeRight, KinectInterop.JointType.AnkleRight, false);
            float kneeright_angle_diff = kneeright_angle_p - kneeright_angle_a;

            sbDetails.AppendFormat("右膝 - {0:F0} ", Mathf.Abs(kneeright_angle_diff));

            float knee_match = 100 - Mathf.Abs(kneeleft_angle_diff) - Mathf.Abs(kneeright_angle_diff);

            sbDetails.AppendFormat("蹲动作匹配度  {0:F0} {1}", knee_match, knee_match > 80 ? " - Matched" : " ").AppendLine();



            string sPoseMessage = string.Format("Pose match: {0:F0}% {1}", fMatchPercent * 100f,
                                                (bPoseMatched ? "- Matched" : ""));
            sAngle = sbDetails.ToString();
            if (infoText != null)
            {
                infoText.text = sPoseMessage + "\n\n" + sAngle + "\n\n" + sDiffDetails;
            }
        }
        else
        {
            // no user found
            if (infoText != null)
            {
                infoText.text = "Try to match the pose on the left.";
            }
        }
    }


    // gets angle or percent difference in pose
    public float GetPoseDifference(bool bPercentDiff, ref string sDiffDetails)
    {
        float fAngleDiff = 0f;
        float fMaxDiff = 0f;
        sDiffDetails = string.Empty;

        if (!kinectManager)
        {
            return 0f;
        }

        StringBuilder sbDetails = new StringBuilder();
        sbDetails.Append("Joint differences:").AppendLine();

        userID = kinectManager.GetPrimaryUserID();//获取用户userID , 如果未获得用户返回0
        if (userID != 0)
        {
            for (int i = 0,j = 1; i < kinectManager.GetJointCount() - 1; i++)
            {
                Vector3 userBone_1 = kinectManager.GetJointKinectPosition(userID, i);
                Vector3 userBone_2 = kinectManager.GetJointKinectPosition(userID, j);
                

                Vector3 vUserBone = (userBone_2 - userBone_1).normalized;
                Vector3 vPoseBone = (bones_taiji1[j] - bones_taiji1[i]).normalized;

                j++;
                Debug.Log("vUserBone " + vUserBone.ToString("f2") + i);



                float fDiff = Vector3.Angle(vPoseBone, vUserBone);

                if (fDiff > 90f) fDiff = 90f;

                fAngleDiff += fDiff;
                fMaxDiff += 90f;  // we assume the max diff could be 90 degrees

                //sbDetails.AppendFormat("{0} - {1:F0} deg.", i, fDiff).AppendLine();

                //print("joint " + i + vec3.ToString("f7"));//打印关节点坐标
                //joints[i].transform.position = kinectManager.GetJointKinectPosition(userID, i);
            }
        }

        float fPercentDiff = 0f;
        if (bPercentDiff && fMaxDiff > 0f)
        {
            fPercentDiff = fAngleDiff / fMaxDiff;
        }

        // details info
        sbDetails.AppendLine();
        sbDetails.AppendFormat("Sum-Diff: - {0:F0} deg out of {1:F0} deg", fAngleDiff, fMaxDiff).AppendLine();
        sbDetails.AppendFormat("Percent-Diff: {0:F0}%", fPercentDiff * 100).AppendLine();
        sDiffDetails = sbDetails.ToString();

        return (bPercentDiff ? fPercentDiff : fAngleDiff);
    }
}

 针对转体、侧身问题的约束

     

        在转身问题上，转身变化较大的肩-背-肩关节在三维空间中的角度变化其实并不明显。于是将围绕spine shoulder关节（双肩之间的节点）的三点形成的两个向量做垂直于Z轴的平面的正交投影。通过这样的方案来增加因转身而产生的角度变化，提高识别转身侧身的准确性。以下为获得关节角度投影的函数。此外,还可以通过Position之间的相对距离的方法来判断,因为图中所示关键帧用角度就已经能够很好地约束了,而太极拳云手动作中也存在识别不清楚的抬手动作,所以,加入相对距离的方案应该也是可行的。

    //get 2D angle of vector
    public float GetAngle_2D(KinectInterop.JointType poseJoint1, KinectInterop.JointType nextJoint1, KinectInterop.JointType poseJoint2, KinectInterop.JointType nextJoint2, bool isPose)
    {
        Vector2 vBone1, vBone2;
        //vec3 to vec2 省略 z轴坐标
        vBone1 = isPose ? (bones_taiji1[(int)nextJoint1] - bones_taiji1[(int)poseJoint1]).normalized
            : (kinectManager.GetJointKinectPosition(userID, (int)nextJoint1) - kinectManager.GetJointKinectPosition(userID, (int)poseJoint1)).normalized;
        vBone2 = isPose ? (bones_taiji1[(int)nextJoint2] - bones_taiji1[(int)poseJoint2]).normalized
            : (kinectManager.GetJointKinectPosition(userID, (int)nextJoint2) - kinectManager.GetJointKinectPosition(userID, (int)poseJoint2)).normalized;

        return Vector2.Angle(vBone1, vBone2);
    }

针对重要关节问题的约束

太极拳云手的动作中往往存在的转体不足与手臂弯曲等问题。但是在整体匹配度的对单个关节匹配度的弱化中，使得不标准的动作都能达标。所以不仅仅需要依靠与标准骨骼数据的比对，而是通过添加关节角度来进行自约束的方法来提高识别的准确性。

    //get angle of vector
    public float GetAngle(KinectInterop.JointType poseJoint1, KinectInterop.JointType nextJoint1, KinectInterop.JointType poseJoint2, KinectInterop.JointType nextJoint2, bool isPose)
    {
        Vector3 vBone1, vBone2;

        vBone1 = isPose ? (bones_taiji1[(int)nextJoint1] - bones_taiji1[(int)poseJoint1]).normalized
            : (kinectManager.GetJointKinectPosition(userID, (int)nextJoint1) - kinectManager.GetJointKinectPosition(userID, (int)poseJoint1)).normalized;
        vBone2 = isPose ? (bones_taiji1[(int)nextJoint2] - bones_taiji1[(int)poseJoint2]).normalized
            : (kinectManager.GetJointKinectPosition(userID, (int)nextJoint2) - kinectManager.GetJointKinectPosition(userID, (int)poseJoint2)).normalized;

        return Vector3.Angle(vBone1, vBone2);
    }


//在主函数中添加约束
{
            float shoulderleft_angle_p = GetAngle(KinectInterop.JointType.SpineShoulder, KinectInterop.JointType.ShoulderLeft, KinectInterop.JointType.ShoulderLeft, KinectInterop.JointType.ElbowLeft, true);
            float shoulderleft_angle_a = GetAngle(KinectInterop.JointType.SpineShoulder, KinectInterop.JointType.ShoulderLeft, KinectInterop.JointType.ShoulderLeft, KinectInterop.JointType.ElbowLeft, false);
            float shoulderleft_angle_diff = shoulderleft_angle_p - shoulderleft_angle_a;

            sbDetails.AppendFormat("左肩 - {0:F0} ", Mathf.Abs(shoulderleft_angle_diff));

}

 

整体下来效果都还可以,手腕部分识别准确率还有提升的空间，但是其对整体姿势影响比较少。以上

 

附上Kinect25关节点图