Unreal语音示例:讯飞在线语音听写UE4 Demo

关键字: Unreal 蓝图 语音识别 C++

---

由于项目需求，使用语音进行游戏交互，踩了两周多的坑，终于搞出来了，记录一下踩坑经历

先贴上demo：https://github.com/303snowing/UnrealXunFeiSpeech
demo的引擎版本是4.17.1，使用Visual Studio 2017 15.3

---

准备工作

1. 先到科大讯飞的语音平台注册、创建一个应用(这步必须，因为只有拥有appid才能下载对应的sdk)
2. 下载在线语音听写sdk：http://www.xfyun.cn/services/voicedictation，解压
3. 建立Unreal C++空白项目，不需要包含Start Content，示例的项目名称为UnrealXunFeiSpeech
4. 编译项目并启动项目实例
5. 建立一个继承自Actor的C++类，可见性为Public

讯飞库在 Windows_voice_1166_59940824/libs 和 Windows_voice_1166_59940824/bin 目录下,头文件在 Windows_voice_1166_59940824/include 目录下

导入讯飞库

1. 在项目根目录下建立一个XunFei文件夹
2. 将Windows_voice_1166_59940824/libs和Windows_voice_1166_59940824/include目录拷贝到XunFei文件夹中
3. 将Windows_voice_1166_59940824/bin下的msc_x64.dll文件拷贝到项目工程的Binaries/Win64目录下(该.dll文件在项目迁移或打包过程中都不可缺少)

配置讯飞库搜索路径

编辑Source/UnrealXunFeiSpeech/UnrealXunFeiSpeech.Build.cs文件，代码如下

// Fill out your copyright notice in the Description page of Project Settings.

using UnrealBuildTool;

public class UnrealXunFeiSpeech : ModuleRules
{
	public UnrealXunFeiSpeech(ReadOnlyTargetRules Target) : base(Target)
	{
		PCHUsage = PCHUsageMode.UseExplicitOrSharedPCHs;
	
		PublicDependencyModuleNames.AddRange(new string[] { "Core", "CoreUObject", "Engine", "InputCore", "Json" });

		PrivateDependencyModuleNames.AddRange(new string[] {  });

        // Uncomment if you are using Slate UI
        // PrivateDependencyModuleNames.AddRange(new string[] { "Slate", "SlateCore" });

        // Uncomment if you are using online features
        // PrivateDependencyModuleNames.Add("OnlineSubsystem");

        // To include OnlineSubsystemSteam, add it to the plugins section in your uproject file with the Enabled attribute set to true

		// 此处指定文件搜索路径
        PrivateIncludePaths.Add("UnrealXunFeiSpeech/Private");
        PublicIncludePaths.Add("UnrealXunFeiSpeech/Public");
		//引入讯飞静态库
        PublicLibraryPaths.AddRange(new string[] { "..\\XunFei\\libs" });
        PublicAdditionalLibraries.AddRange(new string[] { "msc_x64.lib" });
		//添加文件搜索路径
        PublicIncludePaths.AddRange(new string[] { "..\\XunFei\\include" });
    }
}

编写代码

1. 创建FWinRec类

FWinRec类对应讯飞官方例子的winrec.c文件,是封装的Windows录音功能，在Source/UnrealXunFeiSpeech/Public下创建WinRec.h，在Source/UnrealXunFeiSpeech/Private下创建WinRec.cpp，并在项目中添加到对应目录下

WinRec.h | WinRec.cpp
由于在其他文件中，均直接或者间接的包含WinRec.h,所以在WinRec.h中添加了自定义的Log标签。

// Fill out your copyright notice in the Description page of Project Settings.
/*
* @file
* @brief a record interface in windows
*
* it encapsluate the windows API waveInxxx;
* Common steps:
*	create_recorder,
*	open_recorder,
*	start_record,
*	stop_record,
*	close_recorder,
*	destroy_recorder
*
* @author	303snowing
* @date		2017/09/09
*/
#pragma once
//包含项目文件，可在该文件中使用Unreal的库
#include "UnrealXunFeiSpeech.h"

#include <stdlib.h>
#include <windows.h>
#include <mmsystem.h>   
#include <process.h>
#include <errno.h>
// 自定义定义静态Log 包含WinRec.h使用
DEFINE_LOG_CATEGORY_STATIC(SnowingLog, Log, All);
DEFINE_LOG_CATEGORY_STATIC(SnowingWarning, Warning, All);
DEFINE_LOG_CATEGORY_STATIC(SnowingError, Error, All);

/* error code */
enum {
	RECORD_ERR_BASE = 0,
	RECORD_ERR_GENERAL,
	RECORD_ERR_MEMFAIL,
	RECORD_ERR_INVAL,
	RECORD_ERR_NOT_READY
};

/* recorder object. */
struct recorder {
	void(*on_data_ind)(char *data, unsigned long len, void *user_para);
	void * user_cb_para;
	volatile int state;		/* internal record state */

	void * wavein_hdl;
	void * rec_thread_hdl;
	void * bufheader;
	unsigned int bufcount;
};
//每个类中声明自己的Log标签，方便调试
DECLARE_LOG_CATEGORY_EXTERN(WinRec, Warning, All);

class FWinRec
{
	//将回调函数声明为友元，以方便访问私有方法
	friend static unsigned int  __stdcall record_thread_proc(void * para); /* the recording callback thread procedure */

public:
	FWinRec() = default;
	FWinRec(FString);
	~FWinRec();

private:
	void dbg_wave_header(WAVEHDR * buf);
	int create_callback_thread(void *thread_proc_para, HANDLE *thread_hdl_out);
	void close_callback_thread(HANDLE thread);
	int open_rec_device(int dev, WAVEFORMATEX *format, HANDLE thread, HWAVEIN *wave_hdl_out);
	int prepare_rec_buffer(HWAVEIN wi, WAVEHDR ** bufheader_out, unsigned int headercount, unsigned int bufsize);
	void free_rec_buffer(HWAVEIN wi, WAVEHDR *first_header, unsigned headercount);
	void close_rec_device(HWAVEIN wi);
	int start_record_internal(HWAVEIN wi, WAVEHDR *header, unsigned int bufcount);
	int stop_record_internal(HWAVEIN wi);
	void data_proc(struct recorder *rec, MSG *msg);
	int is_stopped_internal(struct recorder *rec);
	int open_recorder_internal(struct recorder * rec, unsigned int dev, WAVEFORMATEX * fmt);
	void close_recorder_internal(struct recorder *rec);

public:
	/**
	* @fn
	* @brief	Get the default input device ID
	*
	* @return	returns WAVE_MAPPER in windows.
	*/
	int get_default_input_dev();

	/**
	* @fn
	* @brief	Get the total number of active input devices.
	* @return	the number. 0 means no active device.
	*/
	unsigned int get_input_dev_num();

	/**
	* @fn
	* @brief	Create a recorder object.
	* @return	int			- Return 0 in success, otherwise return error code.
	* @param	out_rec		- [out] recorder object holder
	* @param	on_data_ind	- [in]	callback. called when data coming.
	* @param	user_cb_para	- [in] user params for the callback.
	* @see
	*/
	int create_recorder(struct recorder ** out_rec,
		void(*on_data_ind)(char *data, unsigned long len, void *user_para),
		void* user_cb_para);

	/**
	* @fn
	* @brief	Destroy recorder object. free memory.
	* @param	rec	- [in]recorder object
	*/
	void destroy_recorder(struct recorder *rec);

	/**
	* @fn
	* @brief	open the device.
	* @return	int			- Return 0 in success, otherwise return error code.
	* @param	rec			- [in] recorder object
	* @param	dev			- [in] device id, from 0.
	* @param	fmt			- [in] record format.
	* @see
	* 	get_default_input_dev()
	*/
	int open_recorder(struct recorder * rec, unsigned int dev, WAVEFORMATEX * fmt);

	/**
	* @fn
	* @brief	close the device.
	* @param	rec			- [in] recorder object
	*/

	void close_recorder(struct recorder *rec);

	/**
	* @fn
	* @brief	start record.
	* @return	int			- Return 0 in success, otherwise return error code.
	* @param	rec			- [in] recorder object
	*/
	int start_record(struct recorder * rec);

	/**
	* @fn
	* @brief	stop record.
	* @return	int			- Return 0 in success, otherwise return error code.
	* @param	rec			- [in] recorder object
	*/
	int stop_record(struct recorder * rec);

	/**
	* @fn
	* @brief	test if the recording has been stopped.
	* @return	int			- 1: stopped. 0 : recording.
	* @param	rec			- [in] recorder object
	*/
	int is_record_stopped(struct recorder *rec);
};
//定义一个静态变量供C++代码使用，以访问全局变量
static FWinRec * winrec = new FWinRec(FString("static winrec be created !"));

2.创建FSpeechRecoginzer类

FSpeechRecoginzer类对应讯飞官方例子的speechrecoginzer.c文件，封装了语音在线听写功能，在Source/UnrealXunFeiSpeech/Public下创建SpeechRecognizer.h，在Source/UnrealXunFeiSpeech/Private下创建SpeechRecognizer.cpp，并在项目中添加到对应目录下

SpeechRecoginzer.h | SpeechRecoginzer.cpp
基于录音接口和讯飞MSC接口封装一个MIC录音识别的模块

// Fill out your copyright notice in the Description page of Project Settings.
/*
@file
@brief 基于录音接口和讯飞MSC接口封装一个MIC录音识别的模块

@author		taozhang9
@date		2016/05/27
*/
#pragma once

#include <stdlib.h>
#include <windows.h>

#include "qisr.h"
#include "msp_cmn.h"
#include "msp_errors.h"

#include "WinRec.h"
#include "SpeechActor.h"

enum sr_audsrc
{
	SR_MIC,	/* write data from mic */
	SR_USER	/* write data from user by calling API */
};

#define DEFAULT_INPUT_DEVID     (-1)


#define E_SR_NOACTIVEDEVICE		1
#define E_SR_NOMEM				2
#define E_SR_INVAL				3
#define E_SR_RECORDFAIL			4
#define E_SR_ALREADY			5


struct speech_rec_notifier {
	void(*on_result)(const char *result, char is_last);
	void(*on_speech_begin)();
	void(*on_speech_end)(int reason);	/* 0 if VAD.  others, error : see E_SR_xxx and msp_errors.h  */
};

#define END_REASON_VAD_DETECT	0	/* detected speech done  */

struct speech_rec {
	enum sr_audsrc aud_src;  /* from mic or manual  stream write */
	struct speech_rec_notifier notif;
	const char * session_id;
	int ep_stat;
	int rec_stat;
	int audio_status;
	struct recorder *recorder;
	volatile int state;
	char * session_begin_params;
};

DECLARE_LOG_CATEGORY_EXTERN(SpeechRecoginzer, Warning, All);

//声明代理
//DECLARE_DELEGATE_RetVal(FString, OnGetResult)

class FSpeechRecoginzer
{
	friend static void iat_cb(char *data, unsigned long len, void *user_para);

public:
	FSpeechRecoginzer() = default;
	FSpeechRecoginzer(FString);
	virtual ~FSpeechRecoginzer();

	//OnGetResult GettedResult;

private:
	void end_sr_on_error(struct speech_rec *sr, int errcode);
	void end_sr_on_vad(struct speech_rec *sr);
	char * skip_space(char *s);
	int update_format_from_sessionparam(const char * session_para, WAVEFORMATEX *wavefmt);
	void wait_for_rec_stop(struct recorder *rec, unsigned int timeout_ms);

public:
	/* must init before start . devid = -1, then the default device will be used.
	devid will be ignored if the aud_src is not SR_MIC */
	int sr_init(struct speech_rec * sr, const char * session_begin_params, enum sr_audsrc aud_src, int devid, struct speech_rec_notifier * notifier);
	int sr_start_listening(struct speech_rec *sr);
	int sr_stop_listening(struct speech_rec *sr);
	/* only used for the manual write way. */
	int sr_write_audio_data(struct speech_rec *sr, char *data, unsigned int len);
	/* must call uninit after you don't use it */
	void sr_uninit(struct speech_rec * sr);

};

//定义一个静态变量供C++代码使用，以访问全局变量
static  FSpeechRecoginzer * speechrecoginzer = new FSpeechRecoginzer("static soeech recoginzer be created !");

3. 创建FXunFeiSpeech类

FXunFeiSpeech类中封装了语音识别的执行方法，包含整体流程控住与事件控制，在Source/UnrealXunFeiSpeech/Public下创建XunFeiSpeech.h，在Source/UnrealXunFeiSpeech/Private下创建XunFeiSpeech.cpp，并在项目中添加到对应目录下

XunFeiSpeech.h | XunFeiSpeech.cpp
语音听写(iFly Auto Transform)技术能够实时地将语音转换成对应的文字。

#pragma once

#include <conio.h>
#include "msp_cmn.h"
#include "msp_errors.h"
#include "SpeechRecoginzer.h"

/*
* 语音听写(iFly Auto Transform)技术能够实时地将语音转换成对应的文字。
*/
#define FRAME_LEN	640
#define BUFFER_SIZE	4096
// 识别状态类型
enum {
	EVT_START = 0,
	EVT_STOP,
	EVT_QUIT,
	EVT_TOTAL
};

DECLARE_LOG_CATEGORY_EXTERN(XunFeiSpeech, Warning, All);

class FXunFeiSpeech
{

public:
	//struct speech_rec iat;

public:

	FXunFeiSpeech();
	FXunFeiSpeech(FString);
	//事件触发，控制录音的开始、结束、与程序的退出
	void SetStart();
	void SetStop();
	void SetQuit();
	//整个流程控制
	void speech_mic(const char* session_beging_params);
	//将识别结果返回
	const char* get_result() const;

};
//定义静态实例方便其他C++代码使用
static FXunFeiSpeech * xunfeispeech = new FXunFeiSpeech(FString("static xunfeispeech be created !"));

4. 创建FSpeechTask类

FSpeechTask类继承FNonAbandonableTask，用来将语音识别作为独立线程启动，避免在语音录入和识别时阻塞游戏主线程，在Source/UnrealXunFeiSpeech/Public下创建SpeechTask.h，在Source/UnrealXunFeiSpeech/Private下创建SpeechTask.cpp，并在项目中添加到对应目录下

SpeechTask.h | SpeechTask.cpp

#pragma once

#include "XunFeiSpeech.h"
#include "AsyncWork.h"

class FSpeechTask : public FNonAbandonableTask
{

	friend class FAutoDeleteAsyncTask<FSpeechTask>;

	FSpeechTask()
	{
		UE_LOG(SnowingWarning, Warning, TEXT("Speech Task be Create !"));
	}

	void DoWork();
	FORCEINLINE TStatId GetStatId() const
	{
		RETURN_QUICK_DECLARE_CYCLE_STAT(FSpeechTask, STATGROUP_ThreadPoolAsyncTasks);
	}
};

编写ASpeechActor类

ASpeechActor类为蓝图暴露操作方法，包含语音初始化、打开录音、停止录音和退出录音释放资源操作

• SpeechActor.h

// Fill out your copyright notice in the Description page of Project Settings.

#pragma once
//包含SpeechTask，在初始化的时候，启动语音功能
#include "SpeechTask.h"

#include "CoreMinimal.h"
#include "GameFramework/Actor.h"
#include "SpeechActor.generated.h"

UCLASS()
class UNREALXUNFEISPEECH_API ASpeechActor : public AActor
{
	GENERATED_BODY()
private:
	//存放语音识别结果
	FString Result;

public:	
	// Sets default values for this actor's properties
	ASpeechActor();

protected:
	// Called when the game starts or when spawned
	virtual void BeginPlay() override;

public:	
	// Called every frame
	virtual void Tick(float DeltaTime) override;

	UFUNCTION(BlueprintCallable, Category = "XunFei", meta = (DisplayName = "SpeechInit", Keywords = "Speech Recognition Initialization"))
		void SpeechInit();

	UFUNCTION(BlueprintCallable, Category = "XunFei", meta = (DisplayName = "SpeechOpen", Keywords = "Speech Recognition Open"))
		void SpeechOpen();

	UFUNCTION(BlueprintCallable, Category = "XunFei", meta = (DisplayName = "SpeechStop", Keywords = "Speech Recognition Stop"))
		void SpeechStop();

	UFUNCTION(BlueprintCallable, Category = "XunFei", meta = (DisplayName = "SpeechQuit", Keywords = "Speech Recognition Quit"))
		void SpeechQuit();

	UFUNCTION(BlueprintCallable, Category = "XunFei", meta = (DisplayName = "SpeechResult", Keywords = "Speech Recognition GetResult"))
		FString SpeechResult();
};

• SpeechActor.cpp

// Fill out your copyright notice in the Description page of Project Settings.
#pragma once

#include "SpeechActor.h"
//引入Unreal的Json库，用来解析识别结果(!!!注意在Build.cs文件的Module中加载Json)
/*
PublicDependencyModuleNames.AddRange(new string[] { "Core", "CoreUObject", "Engine", "InputCore", "Json" });
*/
#include "Serialization/JsonReader.h"
#include "Dom/JsonObject.h"
#include "Serialization/JsonSerializer.h"



// Sets default values
ASpeechActor::ASpeechActor() :
	Result{}
{
 	// Set this actor to call Tick() every frame.  You can turn this off to improve performance if you don't need it.
	PrimaryActorTick.bCanEverTick = false;

}

// Called when the game starts or when spawned
void ASpeechActor::BeginPlay()
{
	Super::BeginPlay();
}

// Called every frame
void ASpeechActor::Tick(float DeltaTime)
{
	Super::Tick(DeltaTime);

}

void ASpeechActor::SpeechInit()
{
	//创建一个SpeechTask任务实例
	FAutoDeleteAsyncTask<FSpeechTask>* SpeechTask = new FAutoDeleteAsyncTask<FSpeechTask>();

	if (SpeechTask)
	{
		//异步启动SpeechTask实例 会去单开线程异步执行SpeechTask中的DoWork方法
		SpeechTask->StartBackgroundTask();
	}
	else
	{
		UE_LOG(SnowingError, Error, TEXT("XunFei task object could not be create !"));
		return;
	}

	UE_LOG(SnowingWarning, Warning, TEXT("XunFei Task Stopped !"));
	return;
}

void ASpeechActor::SpeechOpen()
{
	xunfeispeech->SetStart();
	return;
}

void ASpeechActor::SpeechStop()
{
	xunfeispeech->SetStop();
	return;
}

void ASpeechActor::SpeechQuit()
{
	xunfeispeech->SetQuit();
	Sleep(300);//延迟等待资源释放完成
	return;
}

FString ASpeechActor::SpeechResult()
{	
	Result = FString(UTF8_TO_TCHAR(xunfeispeech->get_result()));
	//去掉讯飞生成结果中的标点符号json串
	FString LajiString("{\"sn\":2,\"ls\":true,\"bg\":0,\"ed\":0,\"ws\":[{\"bg\":0,\"cw\":[{\"sc\":0.00,\"w\":\"\"}]}]}");
	int32 LajiIndex = Result.Find(*LajiString);
	if (LajiIndex != -1)
	{
		Result.RemoveFromEnd(LajiString);
	}
	TSharedPtr<FJsonObject> JsonObject;
	TSharedRef< TJsonReader<TCHAR> > Reader = TJsonReaderFactory<TCHAR>::Create(Result);
	//解析并拼接结果 返回给调用者(蓝图)
	if (FJsonSerializer::Deserialize(Reader, JsonObject))
	{
		Result.Reset();
		TArray< TSharedPtr<FJsonValue> > TempArray = JsonObject->GetArrayField("ws");
		for (auto rs : TempArray)
		{
			Result.Append((rs->AsObject()->GetArrayField("cw"))[0]->AsObject()->GetStringField("w"));
		}
	}
	UE_LOG(SnowingError, Error, TEXT("%s"), *Result);
	
	return Result;
	
}

编写蓝图脚本示例

这里就比较随意啦，在蓝图中构建一个SpeechActor即可使用其中的方法

注意：

1. 语音初始化在关卡运行时只需要执行一次，避免重复初始化
2. 在关卡结束的时候，或者SpeechActor实例被销毁之前，需要执行SpeechQuit方法，释放语音资源，否则在本次游戏实例中无法再次初始化
3. 在SpeechResult调用之前务必延迟至少0.3秒，等待语音识别结果完整返回
4. 请务必使用Custom Event的形式调用SpeechResult，如果直接在游戏主线程中直接使用函数调用，会造成游戏卡帧

补充
如果出现识别准确率不够，或者对自己的词语不太友好，可以在官方平台的应用管理>语音听写>个性化听写页面上传热词文件，可以优化识别率。