语音识别

语音识别的应用领域非常广泛，洋文名Speech Recognition。它所要解决的问题是让计算机能够“听懂”人类的语音，将语音中包含的文字信息“提取”出来。

语音识别是前文《聊天机器人》必不可少的一个组件，本帖就使用TensorFlow做一个中文语音识别。

使用的数据集

THCHS30是Dong Wang, Xuewei Zhang, Zhiyong Zhang这几位大神发布的开放语音数据集，可用于开发中文语音识别系统。

为了感谢这几位大神，我是跪在电脑前写的本帖代码。

下载中文语音数据集（5G+）：

[python]  view plain  copy

1. $ wget http://data.cslt.org/thchs30/zip/wav.tgz  
2. $ wget http://data.cslt.org/thchs30/zip/doc.tgz  
3. $ wget http://data.cslt.org/thchs30/zip/lm.tgz  
4. # 解压  
5. $ tar xvf wav.tgz  
6. $ tar xvf doc.tgz  
7. $ tar xvf lm.tgz  

在开始之前，先好好检视一下数据集。

训练

[python]  view plain  copy

1. import tensorflow as tf  # 0.12  
2. import numpy as np  
3. import os  
4. from collections import Counter  
5. import librosa  # https://github.com/librosa/librosa  
6.    
7. # 训练样本路径  
8. wav_path = 'data/wav/train'  
9. label_file = 'data/doc/trans/train.word.txt'  
10.    
11. # 获得训练用的wav文件路径列表  
12. def get_wav_files(wav_path=wav_path):  
13.     wav_files = []  
14.     for (dirpath, dirnames, filenames) in os.walk(wav_path):  
15.         for filename in filenames:  
16.             if filename.endswith('.wav') or filename.endswith('.WAV'):  
17.                 filename_path = os.sep.join([dirpath, filename])  
18.                 if os.stat(filename_path).st_size < 240000:  # 剔除掉一些小文件  
19.                     continue  
20.                 wav_files.append(filename_path)  
21.     return wav_files  
22.    
23. wav_files = get_wav_files()  
24.    
25. # 读取wav文件对应的label  
26. def get_wav_lable(wav_files=wav_files, label_file=label_file):  
27.     labels_dict = {}  
28.     with open(label_file, 'r') as f:  
29.         for label in f:  
30.             label = label.strip('\n')  
31.             label_id = label.split(' ', 1)[0]  
32.             label_text = label.split(' ', 1)[1]  
33.             labels_dict[label_id] = label_text  
34.    
35.     labels = []  
36.     new_wav_files = []  
37.     for wav_file in wav_files:  
38.         wav_id = os.path.basename(wav_file).split('.')[0]  
39.         if wav_id in labels_dict:  
40.             labels.append(labels_dict[wav_id])  
41.             new_wav_files.append(wav_file)  
42.    
43.     return new_wav_files, labels  
44.    
45. wav_files, labels = get_wav_lable()  
46. print("样本数:", len(wav_files))  # 8911  
47. #print(wav_files[0], labels[0])  
48. # wav/train/A11/A11_0.WAV -> 绿 是 阳春 烟 景 大块 文章 的 底色 四月 的 林 峦 更是 绿 得 鲜活 秀媚 诗意 盎然  
49.    
50. # 词汇表(参看练习1和7)  
51. all_words = []  
52. for label in labels:  
53.     all_words += [word for word in label]  
54. counter = Counter(all_words)  
55. count_pairs = sorted(counter.items(), key=lambda x: -x[1])  
56.    
57. words, _ = zip(*count_pairs)  
58. words_size = len(words)  
59. print('词汇表大小:', words_size)  
60.    
61. word_num_map = dict(zip(words, range(len(words))))  
62. to_num = lambda word: word_num_map.get(word, len(words))  
63. labels_vector = [ list(map(to_num, label)) for label in labels]  
64. #print(wavs_file[0], labels_vector[0])  
65. #wav/train/A11/A11_0.WAV -> [479, 0, 7, 0, 138, 268, 0, 222, 0, 714, 0, 23, 261, 0, 28, 1191, 0, 1, 0, 442, 199, 0, 72, 38, 0, 1, 0, 463, 0, 1184, 0, 269, 7, 0, 479, 0, 70, 0, 816, 254, 0, 675, 1707, 0, 1255, 136, 0, 2020, 91]  
66. #print(words[479]) #绿  
67. label_max_len = np.max([len(label) for label in labels_vector])  
68. print('最长句子的字数:', label_max_len)  
69.    
70. wav_max_len = 0  # 673  
71. for wav in wav_files:  
72.     wav, sr = librosa.load(wav, mono=True)  
73.     mfcc = np.transpose(librosa.feature.mfcc(wav, sr), [1,0])  
74.     if len(mfcc) > wav_max_len:  
75.         wav_max_len = len(mfcc)  
76. print("最长的语音:", wav_max_len)  
77.    
78. batch_size = 16  
79. n_batch = len(wav_files) // batch_size  
80.    
81. # 获得一个batch  
82. pointer = 0  
83. def get_next_batches(batch_size):  
84.     global pointer  
85.     batches_wavs = []  
86.     batches_labels = []  
87.     for i in range(batch_size):  
88.         wav, sr = librosa.load(wav_files[pointer], mono=True)  
89.         mfcc = np.transpose(librosa.feature.mfcc(wav, sr), [1,0])  
90.         batches_wavs.append(mfcc.tolist())  
91.         batches_labels.append(labels_vector[pointer])  
92.         pointer += 1  
93.    
94.     # 补零对齐  
95.     for mfcc in batches_wavs:  
96.         while len(mfcc) < wav_max_len:  
97.             mfcc.append([0]*20)  
98.     for label in batches_labels:  
99.         while len(label) < label_max_len:  
100.             label.append(0)  
101.     return batches_wavs, batches_labels  
102.    
103. X = tf.placeholder(dtype=tf.float32, shape=[batch_size, None, 20])  
104. sequence_len = tf.reduce_sum(tf.cast(tf.not_equal(tf.reduce_sum(X, reduction_indices=2), 0.), tf.int32), reduction_indices=1)  
105. Y = tf.placeholder(dtype=tf.int32, shape=[batch_size, None])  
106.    
107. # conv1d_layer  
108. conv1d_index = 0  
109. def conv1d_layer(input_tensor, size, dim, activation, scale, bias):  
110.     global conv1d_index  
111.     with tf.variable_scope('conv1d_' + str(conv1d_index)):  
112.         W = tf.get_variable('W', (size, input_tensor.get_shape().as_list()[-1], dim), dtype=tf.float32, initializer=tf.random_uniform_initializer(minval=-scale, maxval=scale))  
113.         if bias:  
114.             b = tf.get_variable('b', [dim], dtype=tf.float32, initializer=tf.constant_initializer(0))  
115.         out = tf.nn.conv1d(input_tensor, W, stride=1, padding='SAME') + (b if bias else 0)  
116.         if not bias:  
117.             beta = tf.get_variable('beta', dim, dtype=tf.float32, initializer=tf.constant_initializer(0))  
118.             gamma = tf.get_variable('gamma', dim, dtype=tf.float32, initializer=tf.constant_initializer(1))  
119.             mean_running = tf.get_variable('mean', dim, dtype=tf.float32, initializer=tf.constant_initializer(0))  
120.             variance_running = tf.get_variable('variance', dim, dtype=tf.float32, initializer=tf.constant_initializer(1))  
121.             mean, variance = tf.nn.moments(out, axes=range(len(out.get_shape()) - 1))  
122.             def update_running_stat():  
123.                 decay = 0.99  
124.                 update_op = [mean_running.assign(mean_running * decay + mean * (1 - decay)), variance_running.assign(variance_running * decay + variance * (1 - decay))]  
125.                 with tf.control_dependencies(update_op):  
126.                     return tf.identity(mean), tf.identity(variance)  
127.                 m, v = tf.cond(tf.Variable(False, trainable=False, collections=[tf.GraphKeys.LOCAL_VARIABLES]), update_running_stat, lambda: (mean_running, variance_running))  
128.                 out = tf.nn.batch_normalization(out, m, v, beta, gamma, 1e-8)  
129.         if activation == 'tanh':  
130.             out = tf.nn.tanh(out)  
131.         if activation == 'sigmoid':  
132.             out = tf.nn.sigmoid(out)  
133.    
134.         conv1d_index += 1  
135.         return out  
136. # aconv1d_layer  
137. aconv1d_index = 0  
138. def aconv1d_layer(input_tensor, size, rate, activation, scale, bias):  
139.     global aconv1d_index  
140.     with tf.variable_scope('aconv1d_' + str(aconv1d_index)):  
141.         shape = input_tensor.get_shape().as_list()  
142.         W = tf.get_variable('W', (1, size, shape[-1], shape[-1]), dtype=tf.float32, initializer=tf.random_uniform_initializer(minval=-scale, maxval=scale))  
143.         if bias:  
144.             b = tf.get_variable('b', [shape[-1]], dtype=tf.float32, initializer=tf.constant_initializer(0))  
145.         out = tf.nn.atrous_conv2d(tf.expand_dims(input_tensor, dim=1), W, rate=rate, padding='SAME')  
146.         out = tf.squeeze(out, [1])  
147.         if not bias:  
148.             beta = tf.get_variable('beta', shape[-1], dtype=tf.float32, initializer=tf.constant_initializer(0))  
149.             gamma = tf.get_variable('gamma', shape[-1], dtype=tf.float32, initializer=tf.constant_initializer(1))  
150.             mean_running = tf.get_variable('mean', shape[-1], dtype=tf.float32, initializer=tf.constant_initializer(0))  
151.             variance_running = tf.get_variable('variance', shape[-1], dtype=tf.float32, initializer=tf.constant_initializer(1))  
152.             mean, variance = tf.nn.moments(out, axes=range(len(out.get_shape()) - 1))  
153.             def update_running_stat():  
154.                 decay = 0.99  
155.                 update_op = [mean_running.assign(mean_running * decay + mean * (1 - decay)), variance_running.assign(variance_running * decay + variance * (1 - decay))]  
156.                 with tf.control_dependencies(update_op):  
157.                     return tf.identity(mean), tf.identity(variance)  
158.                 m, v = tf.cond(tf.Variable(False, trainable=False, collections=[tf.GraphKeys.LOCAL_VARIABLES]), update_running_stat, lambda: (mean_running, variance_running))  
159.                 out = tf.nn.batch_normalization(out, m, v, beta, gamma, 1e-8)  
160.         if activation == 'tanh':  
161.             out = tf.nn.tanh(out)  
162.         if activation == 'sigmoid':  
163.             out = tf.nn.sigmoid(out)  
164.    
165.         aconv1d_index += 1  
166.         return out  
167. # 定义神经网络  
168. def speech_to_text_network(n_dim=128, n_blocks=3):  
169.     out = conv1d_layer(input_tensor=X, size=1, dim=n_dim, activation='tanh', scale=0.14, bias=False)  
170.     # skip connections  
171.     def residual_block(input_sensor, size, rate):  
172.             conv_filter = aconv1d_layer(input_sensor, size=size, rate=rate, activation='tanh', scale=0.03, bias=False)  
173.             conv_gate = aconv1d_layer(input_sensor, size=size, rate=rate,  activation='sigmoid', scale=0.03, bias=False)  
174.             out = conv_filter * conv_gate  
175.             out = conv1d_layer(out, size=1, dim=n_dim, activation='tanh', scale=0.08, bias=False)  
176.             return out + input_sensor, out  
177.     skip = 0  
178.     for _ in range(n_blocks):  
179.         for r in [1, 2, 4, 8, 16]:  
180.             out, s = residual_block(out, size=7, rate=r)  
181.             skip += s  
182.    
183.     logit = conv1d_layer(skip, size=1, dim=skip.get_shape().as_list()[-1], activation='tanh', scale=0.08, bias=False)  
184.     logit = conv1d_layer(logit, size=1, dim=words_size, activation=None, scale=0.04, bias=True)  
185.    
186.     return logit  
187.    
188. class MaxPropOptimizer(tf.train.Optimizer):  
189.     def __init__(self, learning_rate=0.001, beta2=0.999, use_locking=False, name="MaxProp"):  
190.         super(MaxPropOptimizer, self).__init__(use_locking, name)  
191.         self._lr = learning_rate  
192.         self._beta2 = beta2  
193.         self._lr_t = None  
194.         self._beta2_t = None  
195.     def _prepare(self):  
196.         self._lr_t = tf.convert_to_tensor(self._lr, name="learning_rate")  
197.         self._beta2_t = tf.convert_to_tensor(self._beta2, name="beta2")  
198.     def _create_slots(self, var_list):  
199.         for v in var_list:  
200.             self._zeros_slot(v, "m", self._name)  
201.     def _apply_dense(self, grad, var):  
202.         lr_t = tf.cast(self._lr_t, var.dtype.base_dtype)  
203.         beta2_t = tf.cast(self._beta2_t, var.dtype.base_dtype)  
204.         if var.dtype.base_dtype == tf.float16:  
205.             eps = 1e-7  
206.         else:  
207.             eps = 1e-8  
208.         m = self.get_slot(var, "m")  
209.         m_t = m.assign(tf.maximum(beta2_t * m + eps, tf.abs(grad)))  
210.         g_t = grad / m_t  
211.         var_update = tf.assign_sub(var, lr_t * g_t)  
212.         return tf.group(*[var_update, m_t])  
213.     def _apply_sparse(self, grad, var):  
214.         return self._apply_dense(grad, var)  
215.    
216. def train_speech_to_text_network():  
217.     logit = speech_to_text_network()  
218.    
219.     # CTC loss  
220.     indices = tf.where(tf.not_equal(tf.cast(Y, tf.float32), 0.))  
221.     target = tf.SparseTensor(indices=indices, values=tf.gather_nd(Y, indices) - 1, shape=tf.cast(tf.shape(Y), tf.int64))  
222.     loss = tf.nn.ctc_loss(logit, target, sequence_len, time_major=False)  
223.     # optimizer  
224.     lr = tf.Variable(0.001, dtype=tf.float32, trainable=False)  
225.     optimizer = MaxPropOptimizer(learning_rate=lr, beta2=0.99)  
226.     var_list = [t for t in tf.trainable_variables()]  
227.     gradient = optimizer.compute_gradients(loss, var_list=var_list)  
228.     optimizer_op = optimizer.apply_gradients(gradient)  
229.    
230.     with tf.Session() as sess:  
231.         sess.run(tf.global_variables_initializer())  
232.    
233.         saver = tf.train.Saver(tf.global_variables())  
234.    
235.         for epoch in range(16):  
236.             sess.run(tf.assign(lr, 0.001 * (0.97 ** epoch)))  
237.    
238.             global pointer  
239.             pointer = 0  
240.             for batch in range(n_batch):  
241.                 batches_wavs, batches_labels = get_next_batches(batch_size)  
242.                 train_loss, _ = sess.run([loss, optimizer_op], feed_dict={X: batches_wavs, Y: batches_labels})  
243.                 print(epoch, batch, train_loss)  
244.             if epoch % 5 == 0:  
245.                 saver.save(sess, 'speech.module', global_step=epoch)  
246.    
247. # 训练  
248. train_speech_to_text_network()  
249.    
250. # 语音识别  
251. # 把batch_size改为1  
252. def speech_to_text(wav_file):  
253.     wav, sr = librosa.load(wav_file, mono=True)  
254.     mfcc = np.transpose(np.expand_dims(librosa.feature.mfcc(wav, sr), axis=0), [0,2,1])  
255.    
256.     logit = speech_to_text_network()  
257.    
258.     saver = tf.train.Saver()  
259.     with tf.Session() as sess:  
260.         saver.restore(sess, tf.train.latest_checkpoint('.'))  
261.    
262.         decoded = tf.transpose(logit, perm=[1, 0, 2])  
263.         decoded, _ = tf.nn.ctc_beam_search_decoder(decoded, sequence_len, merge_repeated=False)  
264.         predict = tf.sparse_to_dense(decoded[0].indices, decoded[0].shape, decoded[0].values) + 1  
265.         output = sess.run(decoded, feed_dict={X: mfcc})          
266.         #print(output)  

后续：从麦克风获得语音输入，使用上面的模型进行识别。

相关资源：

• TensorFlow练习8: 基于RNN生成音乐
• 深度学习大牛Andrew Ng：Speech Recognition and Beyond
• https://github.com/kaldi-asr/kaldi
• http://cmusphinx.sourceforge.net
• https://pypi.python.org/pypi/SpeechRecognition

• http://blog.csdn.net/u014365862/article/details/53869701
  

Android 轻松实现语音识别的完整代码 (转)

• 博客分类： 
• Android

 

苹果的iphone 有语音识别用的是Google 的技术，做为Google 力推的Android 自然会将其核心技术往Android 系统里面植入，并结合google 的云端技术将其发扬光大。 

所以Google Voice Recognition在Android 的实现就变得极其轻松。 

语音识别，借助于云端技术可以识别用户的语音输入，包括语音控制等技术，下面我们将利用Google 提供的Api 实现这一功能。 

功能点为：通过用户语音将用户输入的语音识别出来，并打印在列表上。 

功能界面如下： 

用户通过点击speak按钮显示界面： 

用户说完话后，将提交到云端搜索： 

在云端搜索完成后，返回打印数据： 

* Copyright (C) 2008 The Android Open Source Project 
* 
* Licensed under the Apache License, Version 2.0 (the "License"); 
* you may not use this file except in compliance with the License. 
* You may obtain a copy of the License at 
* 
* http://www.apache.org/licenses/LICENSE-2.0 
* 
* Unless required by applicable law or agreed to in writing, software 
* distributed under the License is distributed on an "AS IS" BASIS, 
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 
* See the License for the specific language governing permissions and 
* limitations under the License. 
*/ 

package com.example.android.apis.app; 

import com.example.android.apis.R; 

import android.app.Activity; 
import android.content.Intent; 
import android.content.pm.PackageManager; 
import android.content.pm.ResolveInfo; 
import android.os.Bundle; 
import android.speech.RecognizerIntent; 
import android.view.View; 
import android.view.View.OnClickListener; 
import android.widget.ArrayAdapter; 
import android.widget.Button; 
import android.widget.ListView; 

import java.util.ArrayList; 
import java.util.List; 

/** 
* Sample code that invokes the speech recognition intent API. 
*/ 
public class VoiceRecognition extends Activity implements OnClickListener { 

private static final int VOICE_RECOGNITION_REQUEST_CODE = 1234; 

private ListView mList; 

/** 
* Called with the activity is first created. 
*/ 
@Override 
public void onCreate(Bundle savedInstanceState) { 
super.onCreate(savedInstanceState); 

// Inflate our UI from its XML layout description. 
setContentView(R.layout.voice_recognition); 

// Get display items for later interaction 
Button speakButton = (Button) findViewById(R.id.btn_speak); 

mList = (ListView) findViewById(R.id.list); 

// Check to see if a recognition activity is present 
PackageManager pm = getPackageManager(); 
List activities = pm.queryIntentActivities( 
new Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH), 0); 
if (activities.size() != 0) { 
speakButton.setOnClickListener(this); 
} else { 
speakButton.setEnabled(false); 
speakButton.setText("Recognizer not present"); 
} 
} 

/** 
* Handle the click on the start recognition button. 
*/ 
public void onClick(View v) { 
if (v.getId() == R.id.btn_speak) { 
startVoiceRecognitionActivity(); 
} 
} 

/** 
* Fire an intent to start the speech recognition activity. 
*/ 
private void startVoiceRecognitionActivity() { 
Intent intent = new Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH); 
intent.putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL, 
RecognizerIntent.LANGUAGE_MODEL_FREE_FORM); 
intent.putExtra(RecognizerIntent.EXTRA_PROMPT, "Speech recognition demo"); 
startActivityForResult(intent, VOICE_RECOGNITION_REQUEST_CODE); 
} 

/** 
* Handle the results from the recognition activity. 
*/ 
@Override 
protected void onActivityResult(int requestCode, int resultCode, Intent data) { 
if (requestCode == VOICE_RECOGNITION_REQUEST_CODE && resultCode == RESULT_OK) { 
// Fill the list view with the strings the recognizer thought it could have heard 
ArrayList matches = data.getStringArrayListExtra( 
RecognizerIntent.EXTRA_RESULTS); 
mList.setAdapter(new ArrayAdapter(this, android.R.layout.simple_list_item_1, 
matches)); 
} 

super.onActivityResult(requestCode, resultCode, data); 
} 
}

Share the post "TensorFlow练习15: 中文语音识别"