构建视频分类模型（python）

"""
@version: python3.6
@author: 'Johnson'
@contact: 593956670@qq.com
@file: get_img.py
@time: 2019/9/18 11:01
"""
import cv2
import math
import matplotlib.pyplot as plt
import pandas as pd
from keras.preprocessing import image #预处理图像库
import numpy as np
from keras.utils import np_utils
from skimage.transform import resize
from sklearn.model_selection import train_test_split
from glob import glob
from  tqdm import tqdm

#导入训练集txt文件，里面有视频名字列表
f = open("trainlist01.txt","r")
temp = f.read()
videos = temp.split("\n")

#创建含有视频名字列表的dataframe
train = pd.DataFrame()
train["video_name"] = videos
train = train[-1]


# 导入测试集txt文件，里面有视频名字列表
f = open("testlist01.txt", "r")
temp = f.read()
videos = temp.split('\n')

# 创建含有视频名字列表的dataframe
test = pd.DataFrame()
test['video_name'] = videos
test = test[:-1]

#接下来，我们将添加每个视频的标签（用于训练和测试集）

#为训练数据集创建标签
train_video_tag = []
for i in range(train.shape[0]):
    train_video_tag.append(train['video_name'][i].split("/")[0])

train['tag'] = train_video_tag

#为测试数据集创建标签
test_video_tag = []
for i in range(test.shape[0]):
    test_video_tag.append(test['video_name'][i].split("/")[0])

test["tag"] = test_video_tag

#接下来，我们将从训练集视频中提取帧，这些视频将拥有训练模型，我将所有帧存放在train_1的文件中

#存储训练集视频的帧
for i in tqdm(range(train.shape[0])):
    count = 0
    videoFile = train["video_name"][i]
    cap = cv2.VideoCapture('UCF/'+videoFile.split(' ')[0].split('/')[1])   # 从给定路径获取视频
    frameRate = cap.get(5) #帧率
    x = 1
    while(cap.isOpened()):
        frameId = cap.get(1) #当前帧编号
        ret,frame = cap.read()
        if (ret!=True):
            break
        if (frameId%math.floor(frameRate)==0):
            #存储在trian_1文件夹
            filename ='train_1/' + videoFile.split('/')[1].split(' ')[0] +"_frame%d.jpg" % count;count+=1
            cv2.imwrite(filename,frame)
    cap.release()

#提取帧以后，我们将在.csv文件中保存这些帧的名称以及对应的标签
# 获取所有图像的名字
images = glob("train_1/*.jpg")
train_image = []
train_class = []
for i in tqdm(range(len(images))):
    # 创建图像名
    train_image.append(images[i].split('/')[1])
    # 创建图像类标
    train_class.append(images[i].split('/')[1].split('_')[1])

# 存储在dataframe里
train_data = pd.DataFrame()
train_data['image'] = train_image
train_data['class'] = train_class

# 转换dataframe为csv文件
train_data.to_csv('UCF/train_new.csv',header=True, index=False)


#读取所有视频帧
import keras
from keras.models import Sequential
from keras.applications.vgg16 import VGG16
from keras.layers import Dense, InputLayer, Dropout, Flatten
from keras.layers import Conv2D, MaxPooling2D, GlobalMaxPooling2D
from keras.preprocessing import image
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from tqdm import tqdm
from sklearn.model_selection import train_test_split

train = pd.read_csv('UCF/train_new.csv')
train.head()


#我们将读取先前提取的帧，然后将这些帧存储为numpy数组
#创建空列表
train_image = []

# 循环读取和保存帧
for i in tqdm(range(train.shape[0])):
    # 载入图片
    img = image.load_img('train_1/'+train['image'][i], target_size=(224,224,3))
    # 转换为array
    img = image.img_to_array(img)
    # 标准化像素值
    img = img/255
    # 保存到train_image列表
    train_image.append(img)

# 转换为numpy数组
X = np.array(train_image)


#创建验证集
# 分离数据集
y = train['class']

# 创建训练与测试集
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42, test_size=0.2, stratify = y)

#one-hot处理
y_train = pd.get_dummies(y_train)
y_test = pd.get_dummies(y_test)


#定义视频分类模型的结构

#创建预训练的VGG16基本模型
base_model = VGG16(weights='imagenet', include_top=False)


# 从训练集的帧中提取特征
X_train = base_model.predict(X_train)
X_train.shape


# 从验证集的帧中提取特征
X_test = base_model.predict(X_test)
X_test.shape

# 验证集中有14,769个图像，这些图像的形状也变为(7,7,512)。我们现在将使用完全连接的网络来微调模型。这个完全连接的网络以单一维度输入。因此，我们将图像重塑为一个维度
X_train = X_train.reshape(59075, 7*7*512)
X_test = X_test.reshape(14769, 7*7*512)


# 始终建议对像素值进行归一化，即将像素值保持在0和1之间。这有助于模型更快地收敛。
# 标准化像素值
max = X_train.max()
X_train = X_train/max
X_test = X_test/max


#定义结构
model = Sequential()
model.add(Dense(1024, activation='relu', input_shape=(25088,)))
model.add(Dropout(0.5))
model.add(Dense(512, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(256, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(128, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(101, activation='softmax'))

###训练视频分类模型
# 保存权重函数
from keras.callbacks import ModelCheckpoint
mcp_save = ModelCheckpoint('weight.hdf5', save_best_only=True, monitor='val_loss', mode='min')


# 编译模型
model.compile(loss='categorical_crossentropy',optimizer='Adam',metrics=['accuracy'])


# 训练模型
model.fit(X_train, y_train, epochs=200, validation_data=(X_test, y_test), callbacks=[mcp_save], batch_size=128)



#######评估视频分类模型
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten
from keras.layers import Conv2D, MaxPooling2D
from keras.preprocessing import image
import numpy as np
import pandas as pd
from tqdm import tqdm
from keras.applications.vgg16 import VGG16
import cv2
import math
import os
from glob import glob
from scipy import stats as s


# 接下来，我们将定义模型结构，它与我们在训练模型时的模型结构类似：
base_model = VGG16(weights='imagenet', include_top=False)

model = Sequential()
model.add(Dense(1024, activation='relu', input_shape=(25088,)))
model.add(Dropout(0.5))
model.add(Dense(512, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(256, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(128, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(101, activation='softmax'))

#载入权重
model.load_weights("weights.hdf5")
# 编译模型
model.compile(loss='categorical_crossentropy',optimizer='Adam',metrics=['accuracy'])

# 创建测试数据

# 获取测试列表
f = open("testlist01.txt", "r")
temp = f.read()
videos = temp.split('\n')

# 创建dataframe
test = pd.DataFrame()
test['video_name'] = videos
test = test[:-1]
test_videos = test['video_name']
test.head()

# 创建标签
train = pd.read_csv('UCF/train_new.csv')
y = train['class']
y = pd.get_dummies(y)


### 测试视频的预测
'''让我总结一下在查看代码之前我们将在此步骤中执行的操作。以下步骤将帮助你了解预测部分：

首先，我们将创建两个空列表,一个用于存储预测标签，另一个用于存储实际标签
然后，我们将从测试集中获取每个视频，提取该视频的帧并将其存储在一个文件夹中(在当前目录中创建一个名为temp的文件夹来存储帧)。我们将在每次迭代时从此文件夹中删除所有其他文件
接下来，我们将读取temp文件夹中的所有帧，使用预先训练的模型提取这些帧的特征，进行预测得到标签后将其附加到第一个列表中
我们将在第二个列表中为每个视频添加实际标签
让我们编写这些步骤并生成预测：'''
# 创建两个列表来存储预测的和实际的标签
predict = []
actual = []

# for循环从每个测试视频中提取帧
for i in tqdm(range(test_videos.shape[0])):
    count = 0
    videoFile = test_videos[i]
    cap = cv2.VideoCapture('UCF/'+videoFile.split(' ')[0].split('/')[1])   # 从给定路径获取视频
    frameRate = cap.get(5) #帧率
    x=1
    # 从临时文件夹中删除所有其他文件
    files = glob('temp/*')
    for f in files:
        os.remove(f)
    while(cap.isOpened()):
        frameId = cap.get(1) #当前帧编号
        ret, frame = cap.read()
        if (ret != True):
            break
        if (frameId % math.floor(frameRate) == 0):
            # 将此特定视频的帧存储在temp文件夹中
            filename ='temp/' + "_frame%d.jpg" % count;count+=1
            cv2.imwrite(filename, frame)
    cap.release()

    # 从临时文件夹中读取所有帧
    images = glob("temp/*.jpg")

    prediction_images = []
    for i in range(len(images)):
        img = image.load_img(images[i], target_size=(224,224,3))
        img = image.img_to_array(img)
        img = img/255
        prediction_images.append(img)

    # 将测试视频的所有帧转换为numpy数组
    prediction_images = np.array(prediction_images)
    # extracting features using pre-trained model
    prediction_images = base_model.predict(prediction_images)
    # 使用预训练模型提取特征
    prediction_images = prediction_images.reshape(prediction_images.shape[0], 7*7*512)
    # 转换一维数组中的特征
    prediction = model.predict_classes(prediction_images)
    # 在预测列表中添加预测模式，将标签分配给视频
    predict.append(y.columns.values[s.mode(prediction)[0][0]])
    # 添加上视频的真实标签
    actual.append(videoFile.split('/')[1].split('_')[1])



# 评估模型
from sklearn.metrics import accuracy_score
accuracy_score(predict, actual)*100