VR Infinite Gesture
原来适配的SteamVR1.1.1,目前使用的steamVR的版本为1.2.3,部分API更改报错,具体如下
原来API:
void OnEnable()
{
SteamVR_Utils.Event.Listen("device_connected", OnDeviceConnected);
SteamVR_Utils.Event.Listen("TrackedDeviceRoleChanged", OnTrackedDeviceRoleChanged);
}
在新的版本SteamVR包中事件系统使用SteamVR_Events类封装,更改后:
SteamVR_Events.Action deviceConnectedAction, trackedDeviceRoleChangedAction;
void OnEnable()
{
deviceConnectedAction =SteamVR_Events.DeviceConnectedAction(OnDeviceConnected);
trackedDeviceRoleChangedAction = SteamVR_Events.SystemAction(EVREventType.VREvent_TrackedDeviceRoleChanged, OnTrackedDeviceRoleChanged);
}
private void Start()
{
deviceConnectedAction.enabled = true;
trackedDeviceRoleChangedAction.enabled = true;
}
void OnDestroy()
{
deviceConnectedAction.enabled = false;
trackedDeviceRoleChangedAction.enabled = false;
}
在UI界面的开发没有应用UI框架,先实现功能,再用UI框架迭代,所有UI面板的调用使用了MonoBehaviorSimplify中的事件广播机制。主界面通过广播机制调出手势信息面板,手势信息面板根据当前手势信息实例化出手势信息预制体(预制体制作时使用垂直排布组件完成垂直布局)
主界面:
/****************************************************
文件:GestureMainPanel.cs
作者:Paul 邮箱: 794451358@qq.com
日期:2019/9/30 9:18:6
功能:手势识别主界面
*****************************************************/
using UnityEngine;
using Edwon.VR;
using Edwon.VR.Gesture;
using PFarmeWork;
using UnityEngine.UI;
public class GestureMainPanel : MonoBehaviourSimplify
{
public Button btn_GestureInfo;
public Button btn_GestureDetect;
//使用手势识别需要用到的两个变量
private VRGestureSettings gestureSettings;
private VRGestureRig gestureRig;
private void Awake()
{
//使用手势识别需要用到的两个变量赋值
gestureSettings = Utils.GetGestureSettings();
gestureRig = VRGestureRig.GetPlayerRig(gestureSettings.playerID);
//为主页上的两个按钮添加监听事件********************
btn_GestureInfo.onClick.AddListener(()=>{
Broadcast(MEventType.ShowGestureInfoPanel);
});
btn_GestureDetect.onClick.AddListener(() =>
{
Broadcast(MEventType.ShowGestureDetect);
this.HideGo();
});
//*********************************************
//为主页面板添加监听事件
AddListener(MEventType.ShowGestureMainPanel, Show);
}
private void OnDestroy()
{
RemoveListener(MEventType.ShowGestureMainPanel, Show);
}
private void Show()
{
gestureRig.uiState = VRGestureUIState.Idle;
gameObject.SetActive(true);
}
}
手势信息面板:
/****************************************************
文件:GestureInfoPanel.cs
作者:Paul 邮箱: 794451358@qq.com
日期:2019/9/30 10:6:6
功能:手势信息面板
*****************************************************/
using UnityEngine;
using UnityEngine.UI;
using Edwon.VR;
using Edwon.VR.Gesture;
using System.Collections.Generic;
using PFarmeWork;
public class GestureInfoPanel : MonoBehaviourSimplify
{
public GameObject go_GestureItem;
//使用手势识别需要用到的两个变量
private VRGestureSettings gestureSettings;
private VRGestureRig gestureRig;
private void Awake()
{ //使用手势识别需要用到的两个变量赋值
gestureSettings = Utils.GetGestureSettings();
gestureRig = VRGestureRig.GetPlayerRig(gestureSettings.playerID);
//为手势信息面板添加监听事件
AddListener(MEventType.ShowGestureInfoPanel, Show);
Init();
}
private void OnDestroy()
{
RemoveListener(MEventType.ShowGestureInfoPanel, Show);
}
private List<Gesture>GetGestureList()
{
gestureSettings.RefreshGestureBank(true);
return gestureSettings.gestureBank;
}
private void Init()
{
foreach (var gesture in GetGestureList())
{
GameObject go = Instantiate(go_GestureItem, transform.Find("Parent"));
go.GetComponent<GestureInfoItem>().Init(gesture);
}
gameObject.SetActive(false);
}
private void Show()
{
gestureRig.uiState = VRGestureUIState.Gestures;
gameObject.SetActive(true);
}
}
手势信息项目预制体脚本:
/****************************************************
文件:GestureInfoItem.cs
作者:Paul 邮箱: 794451358@qq.com
日期:2019/9/30 10:35:53
功能:手势信息项目详细信息的赋值
*****************************************************/
using UnityEngine;
using UnityEngine.UI;
using Edwon.VR.Gesture;
using Edwon.VR;
using PFarmeWork;
public class GestureInfoItem : MonoBehaviourSimplify
{
public Text txt_GestureName;
public Text txt_GestureExampleCount;
public Button editButton;
private Gesture gesture;
public override void Start()
{
base.Start();
editButton.onClick.AddListener(() =>
{
Broadcast<string>(MEventType.ShowGestureEditorPanel, gesture.name);
});
}
public void Init(Gesture gesture)
{
this.gesture = gesture;
OnEnable();
}
private void OnEnable()
{
if (gesture==null)
{
return;
}
txt_GestureName.text = gesture.name;
txt_GestureExampleCount.text = gesture.exampleCount.ToString();
}
}
显示手势图案的预制体:首先在父物体下创建一个图片组件,用作背景图片。然后在Canvas外新建一个空的游戏物体,将其Transform归零,然后将其拖到背景图片下,使其成为其子物体(直接在背景图片下创建子物体中的transform是RectTransform)为其添加LineRenderer组件,用于绘制手势图案,将其Position属性下的Size字段置0。将背景图片重命名为GestureExampleItem,子物体重命名为Line。为GestureExampleItem添加一个Button组件用于点击(点击事件为删除手势样例)。Line的position Z设置为-40。
在父物体Parent上添加一个Grid Layout Group用于排版预制体,将Grid Layout Group Padding属性下的Left字段设置为30,Top字段设置为20。
至此,GestureExampleItem预制体制作完毕。
手势编辑面板脚本:
/****************************************************
文件:GestureEditPanel.cs
作者:Paul 邮箱: 794451358@qq.com
日期:2019/9/30 14:26:41
功能:手势编辑面板
*****************************************************/
using UnityEngine;
using UnityEngine.UI;
using Edwon.VR.Gesture;
using Edwon.VR;
using PFarmeWork;
using System.Collections.Generic;
public class GestureEditPanel : MonoBehaviourSimplify
{
public Text txt_GestureName;
public GameObject go_ExampleItem;
public Material exampleMaterial;
public Button backButton;
public Button recordButton;
private VRGestureSettings gestureSettings;
private VRGestureRig gestureRig;
private List<GestureExample> gestureExamples = new List<GestureExample>();
private List<GameObject> exampleGo = new List<GameObject>();
/// <summary>
/// 手势名字记录
/// </summary>
private string m_GustureName;
private void Awake()
{
gestureSettings = Utils.GetGestureSettings();
gestureRig = VRGestureRig.GetPlayerRig(gestureSettings.playerID);
backButton.onClick.AddListener(() =>
{
Broadcast(MEventType.ShowGestureInfoPanel);
});
recordButton.onClick.AddListener(() =>
{
RecordGesture();
});
AddListener<string>(MEventType.ShowGestureEditorPanel, Show);
AddListener(MEventType.FinishedOneRecord, OnFinishOneRecord);
}
private void OnDestroy()
{
RemoveListener<string>(MEventType.ShowGestureEditorPanel, Show);
RemoveListener(MEventType.FinishedOneRecord, OnFinishOneRecord);
}
private void Show(string gestureName)
{
m_GustureName = gestureName;
gameObject.SetActive(true);
txt_GestureName.text = gestureName;
BeginEditGesture(gestureName);
}
/// <summary>
/// 获取该手势下的所有手势样例并且进行规整
/// </summary>
/// <param name="gestureName"></param>
private void GetGestureAllExample(string gestureName)
{
gestureExamples = Utils.GetGestureExamples(gestureName, gestureSettings.currentNeuralNet);
foreach (var item in gestureExamples)
{
//对图案进行规整
if (item.raw)
{
item.data = Utils.SubDivideLine(item.data);
item.data = Utils.DownScaleLine(item.data);
}
}
}
/// <summary>
/// 生成手势样例图案
/// </summary>
private void GenerateExamplesGrid()
{
foreach (var obj in exampleGo)
{
Destroy(obj);
}
exampleGo.Clear();
for (int i = 0; i < gestureExamples.Count; i++)
{
GameObject go = Instantiate(go_ExampleItem, transform.Find("Parent"));
go.GetComponent<GestureExampleItem>().Init(gestureExamples[i].name, i);
LineRenderer line = go.GetComponentInChildren<LineRenderer>();
line.useWorldSpace = false;
line.material = exampleMaterial;
line.startColor = Color.blue;
line.endColor = Color.green;
float lineWidth = 0.01f;
line.startWidth = lineWidth - (lineWidth * 0.5f);
line.endWidth = lineWidth + (lineWidth * 0.5f);
line.positionCount = gestureExamples[i].data.Count;
//放大手势样例的位置,让其可以清晰显示
for (int j = 0; j < gestureExamples[i].data.Count; j++)
{
gestureExamples[i].data[j] = gestureExamples[i].data[j] * 90;
}
line.SetPositions(gestureExamples[i].data.ToArray());
exampleGo.Add(go);
}
}
/// <summary>
/// 开始编辑手势样例
/// </summary>
/// <param name="gestureName"></param>
private void BeginEditGesture(string gestureName)
{
gestureRig.uiState = VRGestureUIState.Editing;
gestureRig.BeginEditing(gestureName);
GetGestureAllExample(gestureName);
GenerateExamplesGrid();
}
private void RecordGesture()
{
//函数体中包含了 VRGestureUIState的改变,所以不需要再为其赋值。
gestureRig.BeginReadyToRecord(m_GustureName);
}
private void OnFinishOneRecord()
{
GetGestureAllExample(m_GustureName);
GenerateExamplesGrid();
}
public void UpdateGestureExample(string gestureName)
{
GetGestureAllExample(gestureName);
GenerateExamplesGrid();
}
}
删除手势样例
GestureExampleItem.cs
/****************************************************
文件:GestureExampleItem.cs
作者:Paul 邮箱: 794451358@qq.com
日期:2019/9/30 17:37:59
功能:手势样例删除功能
*****************************************************/
using UnityEngine;
using UnityEngine.UI;
using Edwon.VR.Gesture;
using Edwon.VR;
using PFarmeWork;
using System.Collections.Generic;
public class GestureExampleItem : MonoBehaviourSimplify
{
private VRGestureSettings gestureSettings;
private string m_GestureName;
private int m_LineCount;
private void Awake()
{
gestureSettings = Utils.GetGestureSettings();
GetComponent<Button>().onClick.AddListener(DeleteGesture);
}
public void Init(string gestureName,int lineCount)
{
m_GestureName = gestureName;
m_LineCount = lineCount;
}
private void DeleteGesture()
{
Gesture gesture = gestureSettings.FindGesture(m_GestureName);
gesture.exampleCount--;
Utils.DeleteGestureExample(gestureSettings.currentNeuralNet, m_GestureName, m_LineCount);
GetComponentInParent<GestureEditPanel>().UpdateGestureExample(m_GestureName);
}
}
绘制完成手势要显示,需要在:
void StopCapturing()
{
if (leftCapture.state == VRGestureCaptureState.Capturing || rightCapture.state == VRGestureCaptureState.Capturing)
{
//do nothing
}
else
{
//set state to READY
if (uiState == VRGestureUIState.Recording)
{
Broadcast(MEventType.FinishedOneRecord);//广播事件
uiState = VRGestureUIState.ReadyToRecord;
}
else if (uiState == VRGestureUIState.Detecting)
{
uiState = VRGestureUIState.ReadyToDetect;
}
}
}
如果进入了手势录制页面,尝试去删除一个手势时就会出现bug,每次删除的时候都是触发录制事件。为了解决这个问题,可以用UniRx的响应式属性:
private VRTK_Pointer uIPointer;
private BoolReactiveProperty m_CanRecord = new BoolReactiveProperty(false);
Awake()
{
uIPointer = GameObject.Find("RightController").GetComponent<VRTK_Pointer>();
uIPointer.DestinationMarkerExit += UIPointer_DestinationMarkerExit;
uIPointer.DestinationMarkerEnter += UIPointer_DestinationMarkerEnter;
//监听射线在UI上的Hover事件
recordButton.onClick.AddListener(() =>
{
RecordGesture();
m_CanRecord.Subscribe(canRecord =>//利用其响应式属性调用开始录制或停止录制
{
if (canRecord)
{
RecordGesture();
}
else
{
gestureRig.uiState = VRGestureUIState.Editing;
}
});
});
}
private void UIPointer_DestinationMarkerEnter(object sender, DestinationMarkerEventArgs e)
{
m_CanRecord.Value = false;
}
private void UIPointer_DestinationMarkerExit(object sender, DestinationMarkerEventArgs e)
{
m_CanRecord.Value = true;
}
手势检测页面脚本:
/****************************************************
文件:GestureDetectedPanel.cs
作者:Paul 邮箱: 794451358@qq.com
日期:2019/10/8 10:46:26
功能:手势检测面板
*****************************************************/
using UnityEngine;
using Edwon.VR.Gesture;
using Edwon.VR;
using PFarmeWork;
using UnityEngine.UI;
using System.Collections;
using System.Collections.Generic;
public class GestureDetectedPanel : MonoBehaviourSimplify
{
public Text txt_GestureName;
public Text txt_GestureConfidence;
public Button btn_Back;
private VRGestureSettings gestureSettings;
private VRGestureRig gestureRig;
private void Awake()
{
gestureSettings = Utils.GetGestureSettings();
gestureRig = VRGestureRig.GetPlayerRig(gestureSettings.playerID);
AddListener(MEventType.ShowGestureDetect, Show);
GestureRecognizer.GestureDetectedEvent += GestureRecognizer_GestureDetectedEvent;
this.HideGo();
}
private void OnDestroy()
{
RemoveListener(MEventType.ShowGestureDetect, Show);
GestureRecognizer.GestureDetectedEvent -= GestureRecognizer_GestureDetectedEvent;
}
private void GestureRecognizer_GestureDetectedEvent(string gestureName, double confidence, Handedness hand, bool isDouble = false)
{
StartCoroutine(Delay(gestureName, confidence));
}
IEnumerator Delay(string gestureName,double confidence)
{
txt_GestureName.text = gestureName;
txt_GestureConfidence.text = confidence.ToString();
yield return new WaitForSeconds(1);
txt_GestureName.text = string.Empty;
txt_GestureConfidence.text = string.Empty;
}
private void Show()
{
this.ShowGo();
BeginDetectedMode();
}
/// <summary>
/// 进入测试模式
/// </summary>
private void BeginDetectedMode()
{
gestureRig.BeginDetect();
}
}
每次添加手势后,需要神经网络重新学习一次:
private void BeginStudyGesture()
{
gestureSettings.BeginTraining(OnFinshStudy);
}
void OnFinshStudy(string netName)
{
}
把以上代码插入到手势编辑面板的返回按钮点击的事件注册中,每次录制完成返回都让神经网络学习一次。
backButton.onClick.AddListener(() =>
{
Broadcast(MEventType.ShowGestureInfoPanel);
BeginStudyGesture();
});
手势触发技能或者其他事件,可以通过手势名字和技能名字的一个字典去找到对应的事件
private void GestureRecognizer_GestureDetectedEvent(string gestureName, double confidence, Handedness hand, bool isDouble = false)
{
//在此通过字典找到技能或者事件,调用即可。
}