食物声音识别笔记总结

引言

赛题以语音识别为背景，要求选手使用提供的语音数据训练模型并完成语音分类的任务。数据集来自Eating Sound Collection，数据集中包含20种不同食物的咀嚼声音，赛题任务是给这些声音数据建模，准确分类。作为零基础入门语音识别的新人赛，本次任务不涉及复杂的声音模型、语言模型，希望大家通过两种baseline的学习能体验到语音识别的乐趣。

数据介绍

• train文件夹：完整的训练集；
• train_sample文件夹：部分训练集；
• test文件夹：测试集；

赛题包含的类别：

aloe
ice-cream
ribs
chocolate
cabbage
candied_fruits
soup
jelly
grapes
pizza
gummies
salmon
wings
burger
pickles
carrots
fries
chips
noodles
drinks

评估标准

赛题使用准确率（Accuracy）衡量选手结果与真实标签的差异性。

数据处理

本次需要处理的数据格式为：

名称	大小	Link
test_a.zip	1.02GB	http://tianchi-competition.oss-cn-hangzhou.aliyuncs.com/531887/test_a.zip
train.zip	3.53GB	http://tianchi-competition.oss-cn-hangzhou.aliyuncs.com/531887/train.zip
train_sample.zip	515.64MB	http://tianchi-competition.oss-cn-hangzhou.aliyuncs.com/531887/train_sample.zip

解压后分析train_sample是一系列的wav文件，如果想要提取查看，可以选用如下代码：

import scipy.io.wavfile as wav
from python_speech_features import mfcc
fs, audio = wav.read("abc.wav")
feature_mfcc = mfcc(audio, samplerate=fs)
print(feature_mfcc)
print(feature_mfcc.shape)

参考：https://blog.csdn.net/zuck888/article/details/80676730

数据分析

未完待续

baseline

# 基本库
import pandas as pd
import numpy as np

from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report
from sklearn.model_selection import GridSearchCV

from sklearn.preprocessing import MinMaxScaler

# 搭建分类模型所需要的库
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Conv2D, Flatten, Dense, MaxPool2D, Dropout
from tensorflow.keras.utils import to_categorical 

from sklearn.ensemble import RandomForestClassifier
from sklearn.svm import SVC

# 其他库
import os
import librosa
import librosa.display
import glob 

feature = []
label = []
# 建立类别标签，不同类别对应不同的数字。
label_dict = {'aloe': 0, 'burger': 1, 'cabbage': 2,'candied_fruits':3, 'carrots': 4, 'chips':5,
                  'chocolate': 6, 'drinks': 7, 'fries': 8, 'grapes': 9, 'gummies': 10, 'ice-cream':11,
                  'jelly': 12, 'noodles': 13, 'pickles': 14, 'pizza': 15, 'ribs': 16, 'salmon':17,
                  'soup': 18, 'wings': 19}
label_dict_inv = {v:k for k,v in label_dict.items()}

from tqdm import tqdm
def extract_features(parent_dir, sub_dirs, max_file=10, file_ext="*.wav"):
    c = 0
    label, feature = [], []
    for sub_dir in sub_dirs:
        for fn in tqdm(glob.glob(os.path.join(parent_dir, sub_dir, file_ext))[:max_file]): # 遍历数据集的所有文件
            
           # segment_log_specgrams, segment_labels = [], []
            #sound_clip,sr = librosa.load(fn)
            #print(fn)
            label_name = fn.split('/')[-2]
            label.extend([label_dict[label_name]])
            X, sample_rate = librosa.load(fn,res_type='kaiser_fast')
            mels = np.mean(librosa.feature.melspectrogram(y=X,sr=sample_rate).T,axis=0) # 计算梅尔频谱(mel spectrogram),并把它作为特征
            feature.extend([mels])
            
    return [feature, label]

# 自己更改目录
parent_dir = './train_sample/'
save_dir = "./"
folds = sub_dirs = np.array(['aloe','burger','cabbage','candied_fruits',
                             'carrots','chips','chocolate','drinks','fries',
                            'grapes','gummies','ice-cream','jelly','noodles','pickles',
                            'pizza','ribs','salmon','soup','wings'])

# 获取特征feature以及类别的label
temp = extract_features(parent_dir,sub_dirs,max_file=100)

temp = np.array(temp)
data = temp.transpose()


# 获取特征
X = np.vstack(data[:, 0])

# 获取标签
Y = np.array(data[:, 1])
print('X的特征尺寸是：',X.shape)
print('Y的特征尺寸是：',Y.shape)

# 在Keras库中：to_categorical就是将类别向量转换为二进制（只有0和1）的矩阵类型表示
Y = to_categorical(Y)

X_train, X_test, Y_train, Y_test = train_test_split(X, Y, random_state = 1, stratify=Y)

X_train = X_train.reshape(-1, 16, 8, 1)
X_test = X_test.reshape(-1, 16, 8, 1)

model = Sequential()

# 输入的大小
input_dim = (16, 8, 1)

model.add(Conv2D(64, (3, 3), padding = "same", activation = "tanh", input_shape = input_dim))# 卷积层
model.add(MaxPool2D(pool_size=(2, 2)))# 最大池化
model.add(Conv2D(128, (3, 3), padding = "same", activation = "tanh")) #卷积层
model.add(MaxPool2D(pool_size=(2, 2))) # 最大池化层
model.add(Dropout(0.1))
model.add(Flatten()) # 展开
model.add(Dense(1024, activation = "tanh"))
model.add(Dense(20, activation = "softmax")) # 输出层：20个units输出20个类的概率

# 编译模型，设置损失函数，优化方法以及评价标准
model.compile(optimizer = 'adam', loss = 'categorical_crossentropy', metrics = ['accuracy'])

# 训练模型
model.fit(X_train, Y_train, epochs = 20, batch_size = 15, validation_data = (X_test, Y_test))

def extract_features(test_dir, file_ext="*.wav"):
    feature = []
    for fn in tqdm(glob.glob(os.path.join(test_dir, file_ext))[:]): # 遍历数据集的所有文件
        X, sample_rate = librosa.load(fn,res_type='kaiser_fast')
        mels = np.mean(librosa.feature.melspectrogram(y=X,sr=sample_rate).T,axis=0) # 计算梅尔频谱(mel spectrogram),并把它作为特征
        feature.extend([mels])
    return feature

X_test = extract_features('./test_a/')

X_test = np.vstack(X_test)
predictions = model.predict(X_test.reshape(-1, 16, 8, 1))

preds = np.argmax(predictions, axis = 1)
preds = [label_dict_inv[x] for x in preds]

path = glob.glob('./test_a/*.wav')
result = pd.DataFrame({'name':path, 'label': preds})

result['name'] = result['name'].apply(lambda x: x.split('/')[-1])
result.to_csv('submit.csv',index=None)