Hololens+Unity手势交互脚本开发

Unity手部关节示意图：

脚本实现的功能：

• 定位左右手位置；
• 跟踪手部某个关节的空间位置变换；
• 判断手部姿态；
• 物体能够跟随手部移动。

具体实现过程请看以下代码：

using Microsoft.MixedReality.Toolkit;
using Microsoft.MixedReality.Toolkit.Input;
using Microsoft.MixedReality.Toolkit.Utilities;
using MRTKExtensions.Gesture;
using UnityEngine;

namespace MRTKExtensions.Gesture
{
    public static class GestureUtils
    {
        private const float PinchThreshold = 0.7f;
        private const float GrabThreshold = 0.4f;

        public static bool IsPinching(Handedness trackedHand)
        {
            return HandPoseUtils.CalculateIndexPinch(trackedHand) > PinchThreshold;
        }

        public static bool IsGrabbing(Handedness trackedHand)
        {
            return !IsPinching(trackedHand) &&
                   HandPoseUtils.MiddleFingerCurl(trackedHand) > GrabThreshold &&
                   HandPoseUtils.RingFingerCurl(trackedHand) > GrabThreshold &&
                   HandPoseUtils.PinkyFingerCurl(trackedHand) > GrabThreshold &&
                   HandPoseUtils.ThumbFingerCurl(trackedHand) > GrabThreshold;
        }
    }
}

namespace MyHandPoseControl
{
    public class HandPose: MonoBehaviour
    {
        //SerializeField 非常方便的功能可以帮助用户将下面的 成员变量 在Inspector中显示并修改
        [SerializeField]
        //TrackedHandJoint跟踪哪个手部关节，这里是食指中间关节 
        private TrackedHandJoint trackedHandJoint = TrackedHandJoint.IndexMiddleJoint;

        [SerializeField]
        private float grabDistance = 0.1f;

        [SerializeField]
        //左右手手势识别
        private Handedness trackedHand = Handedness.Both;

        [SerializeField]
        //可以拇指食指捏
        private bool trackPinch = true;

        [SerializeField]
        //可以握拳抓
        private bool trackGrab = true;

        private IMixedRealityHandJointService handJointService;

        //？的用法：用于判断对象是否为空，如果对象为空，则无论该对象调用什么皆不会抛出异常，直接返回null
        //??的用法： 可用于判断一个变量在为null时返回一个指定的值。例如，a??1 等价于 a==null?1:a
        //Link: https://blog.csdn.net/qq_40323256/article/details/86771753
        private IMixedRealityHandJointService HandJointService =>
            handJointService ??
            (handJointService = CoreServices.GetInputSystemDataProvider<IMixedRealityHandJointService>());

        private MixedRealityPose? previousLeftHandPose;
        private MixedRealityPose? previousRightHandPose;

        private void Update()
        {
            var leftHandPose = GetHandPose(Handedness.Left, previousLeftHandPose != null);
            var rightHandPose = GetHandPose(Handedness.Right, previousRightHandPose != null);
            {
                var jointTransform = HandJointService.RequestJointTransform(trackedHandJoint, trackedHand);
                if (rightHandPose != null && previousRightHandPose != null)//有右手
                {
                    //在有右手的情况下，又同时存在左手
                    if (leftHandPose != null && previousLeftHandPose != null)
                    {
                        // fight! pick the closest one
                        var isRightCloser = Vector3.Distance(rightHandPose.Value.Position, jointTransform.position) <
                                            Vector3.Distance(leftHandPose.Value.Position, jointTransform.position);

                        ProcessPoseChange(
                            isRightCloser ? previousRightHandPose : previousLeftHandPose,
                            isRightCloser ? rightHandPose : leftHandPose);
                    }
                    else//只有右手
                    {
                        ProcessPoseChange(previousRightHandPose, rightHandPose);
                    }
                }
                else if (leftHandPose != null && previousLeftHandPose != null)//有左手（因为上面讨论了，所以在这里没有考虑同时有右手的情况）
                {
                    ProcessPoseChange(previousLeftHandPose, leftHandPose);
                }
            }
            previousLeftHandPose = leftHandPose;
            previousRightHandPose = rightHandPose;
        }

        private MixedRealityPose? GetHandPose(Handedness hand, bool hasBeenGrabbed)
        {
            if ((trackedHand & hand) == hand)//确保当前手是在预设范围内
            {
                //如果检测到手且（（当前手势是捏且允许捏）||（当前手势是握拳且允许握拳））
                if (HandJointService.IsHandTracked(hand) &&
                    ((GestureUtils.IsPinching(hand) && trackPinch) ||
                     (GestureUtils.IsGrabbing(hand) && trackGrab)))
                {
                    //关节和手掌的空间变换
                    //var类似于C++的auto类型
                    var jointTransform = HandJointService.RequestJointTransform(trackedHandJoint, hand);
                    var palmTransForm = HandJointService.RequestJointTransform(TrackedHandJoint.Palm, hand);

                    if (hasBeenGrabbed ||
                       Vector3.Distance(gameObject.transform.position, jointTransform.position) <= grabDistance)//手已经抓住物体或手离物体的距离足够近
                    {
                        return new MixedRealityPose(jointTransform.position, palmTransForm.rotation);
                    }
                }
            }

            return null;
        }

        private void ProcessPoseChange(MixedRealityPose? previousPose, MixedRealityPose? currentPose)
        {
            var delta = currentPose.Value.Position - previousPose.Value.Position;
            var deltaRotation = Quaternion.FromToRotation(previousPose.Value.Forward, currentPose.Value.Forward);
            gameObject.transform.position += delta;
            gameObject.transform.rotation *= deltaRotation;
        }
    }
}

结果

绿色模型——可以捏，但不可以握拳控制；
黄色模型——不可以捏，但可以握拳控制；
蓝色模型——既可捏又可握拳控制。

Reference

https://localjoost.github.io/Basic-hand-gesture-recognition-with-MRTK-on-HoloLens-2/