keras实现swin_transformer

import os
import cv2
import dlib
import numpy as np
import math
import keras
from keras.models import Sequential, Model
from keras.layers import Dense, Dropout, Input, Embedding, Flatten, GlobalAveragePooling1D
from keras.utils import np_utils
from sklearn.model_selection import train_test_split
from keras.layers import Layer
from keras import backend as K

# 自定义位置编码层
class Position_Embedding(Layer):
    def __init__(self, size=None, mode='sum', **kwargs):
        self.size = size  # 位置编码的维度
        self.mode = mode  # 'sum' 或 'concat' 模式
        super(Position_Embedding, self).__init__(**kwargs)

    def call(self, x):
        # 如果未指定size或模式为'sum'，从输入推断size
        if (self.size is None) or (self.mode == 'sum'):
            self.size = int(x.shape[-1])
        batch_size, seq_len = K.shape(x)[0], K.shape(x)[1]
        position_j = 1. / K.pow(10000., 2 * K.arange(self.size / 2, dtype='float32') / self.size)
        position_j = K.expand_dims(position_j, 0)
        position_i = K.cumsum(K.ones_like(x[:, :, 0]), 1) - 1  # 生成位置索引
        position_i = K.expand_dims(position_i, 2)
        position_ij = K.dot(position_i, position_j)
        position_ij = K.concatenate([K.cos(position_ij), K.sin(position_ij)], 2)
        if self.mode == 'sum':
            return position_ij + x
        elif self.mode == 'concat':
            return K.concatenate([position_ij, x], 2)

    def compute_output_shape(self, input_shape):
        if self.mode == 'sum':
            return input_shape
        elif self.mode == 'concat':
            return (input_shape[0], input_shape[1], input_shape[2] + self.size)


# 自定义多头自注意力层
class Attention(Layer):
    def __init__(self, nb_head, size_per_head, **kwargs):
        self.nb_head = nb_head  # 注意力头数
        self.size_per_head = size_per_head  # 每个头的维度
        self.output_dim = nb_head * size_per_head  # 输出维度
        super(Attention, self).__init__(**kwargs)

    def build(self, input_shape):
        # 初始化Q、K、V转换的权重
        self.WQ = self.add_weight(name='WQ', shape=(input_shape[0][-1], self.output_dim),
                                  initializer='glorot_uniform', trainable=True)
        self.WK = self.add_weight(name='WK', shape=(input_shape[1][-1], self.output_dim),
                                  initializer='glorot_uniform', trainable=True)
        self.WV = self.add_weight(name='WV', shape=(input_shape[2][-1], self.output_dim),
                                  initializer='glorot_uniform', trainable=True)
        super(Attention, self).build(input_shape)

    def Mask(self, inputs, seq_len, mode='mul'):
        if seq_len is None:
            return inputs
        else:
            mask = K.one_hot(seq_len[:, 0], K.shape(inputs)[1])
            mask = 1 - K.cumsum(mask, 1)
            for _ in range(len(inputs.shape) - 2):
                mask = K.expand_dims(mask, 2)
            if mode == 'mul':
                return inputs * mask
            if mode == 'add':
                return inputs - (1 - mask) * 1e12

    def call(self, x):
        if len(x) == 3:
            Q_seq, K_seq, V_seq = x
            Q_len, V_len = None, None
        elif len(x) == 5:
            Q_seq, K_seq, V_seq, Q_len, V_len = x

        Q_seq = K.dot(Q_seq, self.WQ)
        Q_seq = K.reshape(Q_seq, (-1, K.shape(Q_seq)[1], self.nb_head, self.size_per_head))
        Q_seq = K.permute_dimensions(Q_seq, (0, 2, 1, 3))
        K_seq = K.dot(K_seq, self.WK)
        K_seq = K.reshape(K_seq, (-1, K.shape(K_seq)[1], self.nb_head, self.size_per_head))
        K_seq = K.permute_dimensions(K_seq, (0, 2, 1, 3))
        V_seq = K.dot(V_seq, self.WV)
        V_seq = K.reshape(V_seq, (-1, K.shape(V_seq)[1], self.nb_head, self.size_per_head))
        V_seq = K.permute_dimensions(V_seq, (0, 2, 1, 3))

        A = K.batch_dot(Q_seq, K_seq, axes=[3, 3]) / self.size_per_head**0.5
        A = K.permute_dimensions(A, (0, 3, 2, 1))
        A = self.Mask(A, V_len, 'add')
        A = K.permute_dimensions(A, (0, 3, 2, 1))
        A = K.softmax(A)

        O_seq = K.batch_dot(A, V_seq, axes=[3, 2])
        O_seq = K.permute_dimensions(O_seq, (0, 2, 1, 3))
        O_seq = K.reshape(O_seq, (-1, K.shape(O_seq)[1], self.output_dim))
        O_seq = self.Mask(O_seq, Q_len, 'mul')
        return O_seq

    def compute_output_shape(self, input_shape):
        return (input_shape[0][0], input_shape[0][1], self.output_dim)


# 自定义回调函数，用于跟踪训练过程中的损失和准确率
class LossHistory(keras.callbacks.Callback):
    def on_train_begin(self, logs={}):
        self.losses = {'batch': [], 'epoch': []}
        self.accuracy = {'batch': [], 'epoch': []}
        self.val_loss = {'batch': [], 'epoch': []}
        self.val_acc = {'batch': [], 'epoch': []}

    def on_batch_end(self, batch, logs={}):
        self.losses['batch'].append(logs.get('loss'))
        self.accuracy['batch'].append(logs.get('accuracy'))
        self.val_loss['batch'].append(logs.get('val_loss'))
        self.val_acc['batch'].append(logs.get('val_accuracy'))

    def on_epoch_end(self, batch, logs={}):
        self.losses['epoch'].append(logs.get('loss'))
        self.accuracy['epoch'].append(logs.get('accuracy'))
        self.val_loss['epoch'].append(logs.get('val_loss'))
        self.val_acc['epoch'].append(logs.get('val_accuracy'))

    def loss_plot(self, loss_type):
        iters = range(len(self.losses[loss_type]))
        plt.figure()
        plt.plot(iters, self.accuracy[loss_type], 'r', label='训练准确率')
        plt.plot(iters, self.losses[loss_type], 'g', label='训练损失')
        if loss_type == 'epoch':
            plt.plot(iters, self.val_acc[loss_type], 'b', label='验证准确率')
            plt.plot(iters, self.val_loss[loss_type], 'k', label='验证损失')
        plt.grid(True)
        plt.xlabel(loss_type)
        plt.ylabel('准确率-损失')
        plt.legend(loc="upper right")
        plt.show()


# 使用dlib检测人脸和获取关键点的代码略

# 加载数据和预处理
t = 0
k = 0
x2 = []
y2 = []
time = 1
path_traindata = "MMEW_Final/Micro_Expression"
path1 = os.listdir(path_traindata)
for subject in path1:
    if subject == 'others':
        continue
    nowpath1 = path_traindata + '/' + subject
    path2 = os.listdir(nowpath1)
    for emotion in path2:
        nowpath2 = nowpath1 + '/' + emotion
        path3 = os.listdir(nowpath2)
        for sentence in path3:
            filepath = nowpath2 + '/' + sentence
            # 获取文件名中的标签（情绪类型）
            Q = (filepath[28] + filepath[29])
            
            imge = cv2.imread(filepath)  # 读取图像
            face = detector(imge, 0)  # 检测人脸
            list = []
            landmarks = []
            if(len(face) == 0):
                continue
            if(len(face) == 1):
                for k, d in enumerate(face):
                    # 获取关键点
                    for p in predictor(imge, d).parts():
                        landmarks.append(p)
                    # 计算人脸特征距离
                    list.append(math.sqrt((landmarks[17].x - landmarks[21].x) ** 2 + (landmarks[17].y - landmarks[21].y) ** 2))
                    list.append(math.sqrt((landmarks[22].x - landmarks[26].x) ** 2 + (landmarks[22].y - landmarks[26].y) ** 2))
                    list.append(math.sqrt((landmarks[36].x - landmarks[39].x) ** 2 + (landmarks[36].y - landmarks[39].y) ** 2))
                    list.append(math.sqrt((landmarks[42].x - landmarks[45].x) ** 2 + (landmarks[42].y - landmarks[45].y) ** 2))
                    list.append(math.sqrt((landmarks[31].x - landmarks[35].x) ** 2 + (landmarks[31].y - landmarks[35].y) ** 2))
                    list.append(math.sqrt((landmarks[48].x - landmarks[54].x) ** 2 + (landmarks[48].y - landmarks[54].y) ** 2))
                    list.append(math.sqrt((landmarks[1].x - landmarks[15].x) ** 2 + (landmarks[1].y - landmarks[15].y) ** 2))
                    list.append(math.sqrt((landmarks[2].x - landmarks[14].x) ** 2 + (landmarks[2].y - landmarks[14].y) ** 2))
                    list.append(math.sqrt((landmarks[3].x - landmarks[13].x) ** 2 + (landmarks[3].y - landmarks[13].y) ** 2))
                    list.append(math.sqrt((landmarks[4].x - landmarks[12].x) ** 2 + (landmarks[4].y - landmarks[12].y) ** 2))
                    list.append(math.sqrt((landmarks[5].x - landmarks[11].x) ** 2 + (landmarks[5].y - landmarks[11].y) ** 2))
                    list.append(math.sqrt((landmarks[6].x - landmarks[10].x) ** 2 + (landmarks[6].y - landmarks[10].y) ** 2))
                    list.append(math.sqrt((landmarks[7].x - landmarks[9].x) ** 2 + (landmarks[7].y - landmarks[9].y) ** 2))
                    x2.append(list)  # 加入特征数据
                    if 'N' in Q:
                        y2.append(0)  # 加入标签
                    if 'D' in Q:
                        y2.append(1)  # 加入标签
# 加载数据和预处理的代码继续
# 请注意，以下代码片段是上一个代码片段的延续

# 加载数据和预处理（续）
X_train, X_test, y_train, y_test = train_test_split(x2, y2, test_size=0.2, random_state=1)  # 划分训练集和测试集

# 将数据转换为numpy数组，并进行归一化处理
X_train = np.array(X_train) / 80
X_test = np.array(X_test) / 80
y_train = np.array(y_train)
y_test = np.array(y_test)

# 定义模型结构
model = Sequential()
model.add(Embedding(10000, 256, input_length=13))
model.add(Position_Embedding())
model.add(Dropout(0.2))
model.add(Attention(8, 16))
model.add(GlobalAveragePooling1D())
model.add(Dense(2, activation='softmax'))

# 编译模型
model.compile(loss='sparse_categorical_crossentropy', optimizer='adam', metrics=['accuracy'])

# 定义回调函数并训练模型
history = LossHistory()
model.fit(X_train, y_train, validation_data=(X_test, y_test), epochs=20, batch_size=128, callbacks=[history])

# 绘制训练过程中的损失和准确率曲线
history.loss_plot('epoch')