同样的数据用神经网络训练和验证的差距很大

  下面代码主要是利用pytorch对轴承数据的预处理并搭建神经网络进行训练和测试，在测试中发现，输出结果与实际不符，然后将测试用的数据换成了和训练用数据相同的数据，但最后的训练和测试的结果却完全不同，下面为全部的代码

import torch
from scipy.io import loadmat
import numpy as np
import os
from sklearn import preprocessing  # 0-1编码
from sklearn.model_selection import StratifiedShuffleSplit  # 随机划分，保证每一类比例相同
from tensorboardX import SummaryWriter
from torch import nn
from torch.nn import Conv1d, ReLU,MaxPool1d,Linear,Flatten



# def prepro(d_path, length=2048, number=1000, normal=True, rate=[0.5, 0.25, 0.25], enc=True, enc_step=28):
#     """对数据进行预处理,返回train_X, train_Y, valid_X, valid_Y, test_X, test_Y样本.
#
#     :param d_path: 源数据地址
#     :param length: 信号长度，默认2个信号周期，864
#     :param number: 每种信号个数,总共10类,默认每个类别1000个数据
#     :param normal: 是否标准化.True,Fales.默认True
#     :param rate: 训练集/验证集/测试集比例.默认[0.5,0.25,0.25],相加要等于1
#     :param enc: 训练集、验证集是否采用数据增强.Bool,默认True
#     :param enc_step: 增强数据集采样顺延间隔
#     :return: Train_X, Train_Y, Valid_X, Valid_Y, Test_X, Test_Y
#
#     ```
#     import preprocess.preprocess_nonoise as pre
#
#     train_X, train_Y, valid_X, valid_Y, test_X, test_Y = pre.prepro(d_path=path,
#                                                                     length=864,
#                                                                     number=1000,
#                                                                     normal=False,
#                                                                     rate=[0.5, 0.25, 0.25],
#                                                                     enc=True,
#                                                                     enc_step=28)
#     ```
#     """
    # 获得该文件夹下所有.mat文件名
d_path='D:\\chengxu\\torch_learn\\pytorch2\\轴承诊断\\12K采样频率下的驱动端轴承故障数据（1797rmin）'#文件路径
filenames = os.listdir(d_path)

def capture(original_path):
    """读取mat文件，返回字典

    :param original_path: 读取路径
    :return: 数据字典
    """
    files = {}
    for i in filenames:#遍历文件夹下的各个mat文件
        # 文件路径

        file_path = os.path.join(d_path, i)
        file = loadmat(file_path)
        file_keys = file.keys()#单个mat文件中又有多个子mat文件，得到各个子mat文件的名字

        for key in file_keys:#遍历子mat文件
            if 'DE' in key:
                files[i] = file[key].ravel()#取出子mat文件中名字有DE的
    return files
files=capture('D:\\chengxu\\torch_learn\\pytorch2\\轴承诊断\\12K采样频率下的驱动端轴承故障数据（1797rmin）')
print(len(files['97.mat']))
data = capture(original_path=d_path)#得到的是字典，名字分别是各个mat文件的名字


rate = [0.5, 0.25, 0.25]#训练集，验证集，测试集各占的比例
number=1000#每种类型的数据的个数，在本例中共六个类型，每个类型去1000组数据（后续再进行更改，增大数据量）
enc=True#是否进行数据增强
length=2048#每组数据的长度
enc_step=28#增强数据集采样顺延间隔
def slice_enc(data, slice_rate=rate[1] + rate[2]):
    """将数据切分为前面多少比例，后面多少比例.

    :param data: 单挑数据
    :param slice_rate: 验证集以及测试集所占的比例
    :return: 切分好的数据
    """
    keys = data.keys()#得到各个mat文件的名字
    Train_Samples = {}#先建立空的训练集字典
    Test_Samples = {}#先建立空的测试集字典
    for i in keys:#遍历mat文件
        slice_data = data[i]
        all_lenght = len(slice_data)#单个mat文件包含数据的长度
        end_index = int(all_lenght * (1 - slice_rate))
        samp_train = int(number * (1 - slice_rate))  #训练集需要的数据个数
        Train_sample = []
        Test_Sample = []
        if enc:
            enc_time = length // enc_step
            samp_step = 0  # 用来计数Train采样次数
            for j in range(samp_train):
                random_start = np.random.randint(low=0, high=(end_index - 2 * length))
                label = 0
                for h in range(enc_time):
                    samp_step += 1
                    random_start += enc_step
                    sample = slice_data[random_start: random_start + length]#得到对应长度的数据
                    Train_sample.append(sample)#将得到的数据添加到建立的训练集中
                    if samp_step == samp_train:
                        label = 1
                        break
                if label:
                    break
        else:
            for j in range(samp_train):
                random_start = np.random.randint(low=0, high=(end_index - length))
                sample = slice_data[random_start:random_start + length]
                Train_sample.append(sample)

        # 抓取测试数据
        for h in range(number - samp_train):
            random_start = np.random.randint(low=end_index, high=(all_lenght - length))#测试集的数据从整个数据集划分的后半部分获取
            sample = slice_data[random_start:random_start + length]
            Test_Sample.append(sample)
        Train_Samples[i] = Train_sample
        Test_Samples[i] = Test_Sample
    return Train_Samples, Test_Samples
train,test=slice_enc(data,slice_rate=rate[1] + rate[2])
print(len(train['97.mat']))


# 仅抽样完成，打标签
def add_labels(train_test):
    X = []
    Y = []
    label = 0
    for i in filenames:
        x = train_test[i]
        X += x
        lenx = len(x)
        print(lenx)
        Y += [label] * lenx
        print(Y)
        label += 1
    return X, Y
    # 为训练集制作标签，返回X，Y
Train_X, Train_Y = add_labels(train)
# 为测试集制作标签，返回X，Y
Test_X, Test_Y = add_labels(test)
Train_X=np.array(Train_X)
Test_X=np.array(Test_X)
Train_X,Test_X=Train_X[:,:,np.newaxis],Test_X[:,:,np.newaxis]
print(Train_X)
# Train_X=Train_X.shape[1:]
# Test_X=Test_X.shape[1:]
print(len(Train_X))
print(len(Test_X))
# print(Train_X)
def one_hot(Train_Y, Test_Y):
    Train_Y = np.array(Train_Y).reshape([-1, 1])
    Test_Y = np.array(Test_Y).reshape([-1, 1])
    Encoder = preprocessing.OneHotEncoder()
    Encoder.fit(Train_Y)
    Train_Y = Encoder.transform(Train_Y).toarray()
    Test_Y = Encoder.transform(Test_Y).toarray()
    Train_Y = np.asarray(Train_Y, dtype=np.int32)
    Test_Y = np.asarray(Test_Y, dtype=np.int32)
    return Train_Y, Test_Y
Train_Y, Test_Y = one_hot(Train_Y, Test_Y)
print(Train_Y[1000].argmax(0))
print(len(Train_Y))
print(Train_X[1].shape)
class juanji(nn.Module):
    def __init__(self):
        super().__init__()
        self.model=nn.Sequential(
            Conv1d(1,16,64,3,padding=24),
            ReLU(),
            MaxPool1d(2,1),
            Conv1d(16,32,3,1,padding=1),
            ReLU(),
            MaxPool1d(2,2),
            Conv1d(32,64,3,2,padding=1),
            ReLU(),
            MaxPool1d(2,2),
            Conv1d(64,64,3,1,padding=1),
            ReLU(),
            MaxPool1d(2,2),
            Conv1d(64,64,3,1),
            ReLU(),
            MaxPool1d(2,2),
            Flatten(),
            Linear(1280,100),
            Linear(100,6)

        )
    def forward(self,x):
        x=self.model(x)
        return x

device=torch.device('cuda')
train_on_gpu = torch.cuda.is_available()
if not train_on_gpu:
    print('CUDA is not available.  Training on CPU ...')
else:
    print('CUDA is available!  Training on GPU ...')

device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

my_model=juanji()
my_model=my_model.to(device)

loss_fn=nn.CrossEntropyLoss()
loss_fn=loss_fn.cuda()
Train_Y=torch.tensor(Train_Y)
Test_Y=torch.tensor(Test_Y)
Test_Y=Test_Y.to(device,dtype=torch.float)
Train_Y=Train_Y.to(device,dtype=torch.float)
#优化器
learning_rate=0.01
optim=torch.optim.SGD(my_model.parameters(),lr=learning_rate)

#设置训练网络的一些参数
total_train_step=0    #记录训练的次数
total_test_step=0     #记录测试的次数
epoch=5              #训练的轮数

#添加tensorboard
writer=SummaryWriter("D:\\chengxu\\torch_learn\\logs")


for i in range(epoch):
    print(f"---------------第{i+1}轮训练开始------------")
    #训练步骤开始
    # my_model.train()
    train_loss=0
    train_accuracy=0
    total=0
    for a in range(3000):
        input=Train_X[a]
        input=torch.tensor(input)
        input = input.type(torch.cuda.FloatTensor)
        input=input.reshape(-1,1,2048)
        input=input.cuda()

        # print(output)
        # print(Train_Y[a])
        # print(input)
        #优化器优化模型
        optim.zero_grad()
        output = my_model(input)
        # print(output)
        output = output.flatten()
        output = output.to(device)
        loss = loss_fn(output, Train_Y[a])
        loss.backward()
        optim.step()
        total_train_step+=1
        train_loss+=loss
        output = torch.unsqueeze(output, dim=0)
        train_accuracy = (output.argmax(1) == Train_Y[a].argmax(0))
        total+=train_accuracy
        if total_train_step%100==0:
         print(f"训练次数：{total_train_step},loss:{train_loss}")
         writer.add_scalar("train_loss",loss.item(),total_train_step)
    print(f"整体训练集上的正确率是：{total / 3000}")
    torch.save(my_model.state_dict(),'my_model.pth')
    #测试步骤开始
    # my_model.eval()
    total_test_loss=0
    total_accuacy=0
    my_model.load_state_dict(torch.load('D:\\chengxu\\torch_learn\\pytorch2\\轴承诊断\\my_model.pth'))
    with torch.no_grad():
        for a in range(3000):
            input = Train_X[a]
            Train_Y[a]=Train_Y[a].to(device)
            input = torch.tensor(input)

            input = input.type(torch.cuda.FloatTensor)
            input = input.reshape(-1, 1, 2048)

            input = input.cuda()
            my_model.load_state_dict(torch.load('D:\\chengxu\\torch_learn\\pytorch2\\轴承诊断\\my_model.pth'))
            output = my_model(input)
            output = output.flatten()
            output=output.to(device)
            loss=loss_fn(output,Train_Y[a])
            total_test_loss+=loss.item()
            # output=output.reshape(1,6)


            output=torch.unsqueeze(output,dim=0)


            # print(output)
            #
            # print(output.argmax(1))
            # print(Train_Y[a].argmax(0))
            accuracy=(output.argmax(1)==Train_Y[a].argmax(0))
            total_accuacy+=accuracy
    print(f"整体测试集上的loss:{total_test_loss}")
    print(f"整体测试集上的正确率是：{total_accuacy/3000}")
    writer.add_scalar("test_loss",total_test_loss,total_test_step)
    writer.add_scalar("test_accuracy",total_accuacy/3000,total_test_step)
    total_test_step+=1



writer.close()

用相同的数据进行训练和测试的部分（本人怀疑代码有问题的部分，从完整的代码中截取）如下

for i in range(epoch):
    print(f"---------------第{i+1}轮训练开始------------")
    #训练步骤开始
    # my_model.train()
    train_loss=0
    train_accuracy=0
    total=0
    for a in range(3000):
        input=Train_X[a]
        input=torch.tensor(input)
        input = input.type(torch.cuda.FloatTensor)
        input=input.reshape(-1,1,2048)
        input=input.cuda()

        # print(output)
        # print(Train_Y[a])
        # print(input)
        #优化器优化模型
        optim.zero_grad()
        output = my_model(input)
        # print(output)
        output = output.flatten()
        output = output.to(device)
        loss = loss_fn(output, Train_Y[a])
        loss.backward()
        optim.step()
        total_train_step+=1
        train_loss+=loss
        output = torch.unsqueeze(output, dim=0)
        train_accuracy = (output.argmax(1) == Train_Y[a].argmax(0))
        total+=train_accuracy
        if total_train_step%100==0:
         print(f"训练次数：{total_train_step},loss:{train_loss}")
         writer.add_scalar("train_loss",loss.item(),total_train_step)
    print(f"整体训练集上的正确率是：{total / 3000}")
    torch.save(my_model.state_dict(),'my_model.pth')
    #测试步骤开始
    # my_model.eval()
    total_test_loss=0
    total_accuacy=0
    my_model.load_state_dict(torch.load('D:\\chengxu\\torch_learn\\pytorch2\\轴承诊断\\my_model.pth'))
    with torch.no_grad():
        for a in range(3000):
            input = Train_X[a]
            Train_Y[a]=Train_Y[a].to(device)
            input = torch.tensor(input)

            input = input.type(torch.cuda.FloatTensor)
            input = input.reshape(-1, 1, 2048)

            input = input.cuda()
            my_model.load_state_dict(torch.load('D:\\chengxu\\torch_learn\\pytorch2\\轴承诊断\\my_model.pth'))
            output = my_model(input)
            output = output.flatten()
            output=output.to(device)
            loss=loss_fn(output,Train_Y[a])
            total_test_loss+=loss.item()
            # output=output.reshape(1,6)


            output=torch.unsqueeze(output,dim=0)


            # print(output)
            #
            # print(output.argmax(1))
            # print(Train_Y[a].argmax(0))
            accuracy=(output.argmax(1)==Train_Y[a].argmax(0))
            total_accuacy+=accuracy
    print(f"整体测试集上的loss:{total_test_loss}")
    print(f"整体测试集上的正确率是：{total_accuacy/3000}")
    writer.add_scalar("test_loss",total_test_loss,total_test_step)
    writer.add_scalar("test_accuracy",total_accuacy/3000,total_test_step)
    total_test_step+=1



writer.close()

如上图所示，训练集和测试集的数据相同，但是测试集的输出结果的最大索引一直是5，最终导致测试集上的准确率一直是0.167，不知道问题出现在了哪里，烦请大家帮忙看一下，非常感谢！