模型训练-保存训练数据

无妄无望

已于 2024-04-13 00:02:08 修改

阅读量745

点赞数 6

文章标签：人工智能 chatgpt

于 2024-04-11 20:25:15 首次发布

本文链接：https://blog.csdn.net/PLANTTHESON/article/details/137651696

版权

1.目的

找到一个可运行的代码，可以每个epoch打印训练数据，但是不会保存。因为在改进模型需要这些训练数据进行对比，所以需要将每个epoch的训练数据保存下来，写到一个文件中。

2.解决方案1

直接问ChatGPT，提示词如下：

做一个深度学习的图像分类任务，现在需要将每个epoch的训练数据存到一个文件，用python来实现

import torch
import torch.nn as nn
import torch.optim as optim
from torch.utils.data import DataLoader
from torchvision import datasets, transforms
import json

# 定义一个简单的网络
class SimpleCNN(nn.Module):
    def __init__(self):
        super(SimpleCNN, self).__init__()
        self.conv1 = nn.Conv2d(1, 32, kernel_size=3, stride=1, padding=1)
        self.relu = nn.ReLU()
        self.fc1 = nn.Linear(32 * 28 * 28, 10)

    def forward(self, x):
        x = self.conv1(x)
        x = self.relu(x)
        x = x.view(x.size(0), -1)
        x = self.fc1(x)
        return x

# 设置超参数
batch_size = 64
num_epochs = 10
learning_rate = 0.01

# 创建数据集和数据加载器
transform = transforms.Compose([transforms.ToTensor()])
train_dataset = datasets.MNIST(root='data/', train=True, transform=transform, download=True)
train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)

# 创建网络、损失函数和优化器
model = SimpleCNN()
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(model.parameters(), lr=learning_rate)

# 文件名
output_file = "training_data.json"

# 开始训练
training_data = []

for epoch in range(num_epochs):
    epoch_loss = 0.0
    correct = 0
    total = 0

    # 训练一个 epoch
    for inputs, labels in train_loader:
        optimizer.zero_grad()
        outputs = model(inputs)
        loss = criterion(outputs, labels)
        loss.backward()
        optimizer.step()

        epoch_loss += loss.item() * inputs.size(0)
        _, predicted = outputs.max(1)
        correct += predicted.eq(labels).sum().item()
        total += labels.size(0)

    # 计算平均损失和准确度
    avg_loss = epoch_loss / total
    accuracy = correct / total

    # 将 epoch 的数据添加到列表中
    epoch_data = {
        "epoch": epoch + 1,
        "avg_loss": avg_loss,
        "accuracy": accuracy
    }
    training_data.append(epoch_data)

    # 打印 epoch 的训练结果
    print(f"Epoch {epoch + 1}: Avg Loss: {avg_loss:.4f}, Accuracy: {accuracy:.4f}")

# 将所有 epoch 的训练数据保存到文件中
with open(output_file, 'w') as f:
    json.dump(training_data, f)

print(f"Training data saved to {output_file}")

由于缺少数据集，也懒得去找一个数据集，就将自己py文件中训练部分摘出来给ChatGPT，让它在训练方法中实现这个功能。

def train():
    data_format(os.path.join(config.root_path, './data/train.txt'),
    os.path.join(config.root_path, './data/data'), os.path.join(config.root_path, './data/train.json'))
    data = read_from_file(config.train_data_path, config.data_dir, config.only)
    train_data, val_data = train_val_split(data)
    train_loader = processor(train_data, config.train_params)
    val_loader = processor(val_data, config.val_params)

    best_acc = 0
    epoch = config.epoch
    for e in range(epoch):
        print('-' * 20 + ' ' + 'Epoch ' + str(e+1) + ' ' + '-' * 20)
        # 训练模型
        tloss, tloss_list = trainer.train(train_loader)
        print('Train Loss: {}'.format(tloss))
        # writer.add_scalar('Training/loss', tloss, e)
        # 验证模型
        vloss, vacc = trainer.valid(val_loader)
        print('Valid Loss: {}'.format(vloss))
        print('Valid Acc: {}'.format(vacc))
        # writer.add_scalar('Validation/loss', vloss, e)
        # writer.add_scalar('Validacc/acc', vacc, e)
        # 保存训练数据
        training_data = {
            "epoch": e + 1,
            "train_loss": tloss,
            "valid_loss": vloss,
            "valid_acc": vacc
        }
        with open('training_data.json', 'a') as f:
            json.dump(training_data, f)
            f.write('\n')
        print("数据保存完成")
        # 保存最佳模型
        if vacc > best_acc:
            best_acc = vacc
            save_model(config.output_path, config.fuse_model_type, model)
            print('Update best model!')


     
    print('-' * 20 + ' ' + 'Training Finished' + ' ' + '-' * 20)
    print('Best Validation Accuracy: {}'.format(best_acc))

在我的代码中具体加入的是下列几行代码

# 保存训练数据
training_data = {
    "epoch": e + 1,
    "train_loss": tloss,
    "valid_loss": vloss,
    "valid_acc": vacc
}
with open('training_data.json', 'a') as f:
    json.dump(training_data, f)
    f.write('\n')
print("数据保存完成")

代码意思如下：

with open('training_data.json', 'a') as f:: 打开名为 'training_data.json' 的文件，以追加模式 'a'，并将其赋给变量 f。如果文件不存在，将会创建一个新文件。
json.dump(training_data, f): 将变量 training_data 中的数据以 JSON 格式写入到文件 f 中。这个操作会将 training_data 中的内容转换成 JSON 格式，并写入到文件中。
f.write('\n'): 写入一个换行符 \n 到文件 f 中，确保每次写入 JSON 数据后都有一个新的空行，使得每个 JSON 对象都独占一行，便于后续处理。

这段代码的作用是将变量 training_data 中的数据以 JSON 格式写入到文件 'training_data.json' 中，并确保每次写入后都有一个换行符分隔。

结果

可以在每个epoch训练完成后，将训练损失，验证损失和验证准确率保存在training_data.json文件中。

3.解决方案2

这是另外一个代码，使用的是tensoflow来构建网络，代码如下

model = Sequential()

model.add(Conv2D(32, kernel_size=(3, 3), activation='relu', input_shape=(48,48,1)))
model.add(Conv2D(64, kernel_size=(3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2),strides=(2,2)))
model.add(Dropout(0.25))

model.add(Conv2D(128, kernel_size=(3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2),strides=(2,2)))
model.add(Conv2D(128, kernel_size=(3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2),strides=(2,2)))
model.add(Dropout(0.25))

model.add(Flatten())
model.add(Dense(1024, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(7, activation='softmax'))



# If you want to train the same model or try other models, go for this
if mode == "train":
    # 使用旧版本的Adam优化器


    model.compile(loss='categorical_crossentropy',    optimizer = tf.keras.optimizers.legacy.Adam(learning_rate=0.001, decay=1e-6),metrics=['accuracy'])



    model_info = model.fit_generator(
            train_generator,
            steps_per_epoch=num_train // batch_size,
            epochs=num_epoch,
            validation_data=validation_generator,
            validation_steps=num_val // batch_size,

    )
    train_loss = model_info.history['loss']
    train_accuracy = model_info.history['accuracy']
    val_loss = model_info.history['val_loss']
    val_accuracy = model_info.history['val_accuracy']

    plot_model_history(model_info)
    model.save_weights('model.h5')

询问ChatGPT

import tensorflow as tf

class SaveTrainingDataCallback(tf.keras.callbacks.Callback):
    def __init__(self, file_path):
        super(SaveTrainingDataCallback, self).__init__()
        self.file_path = file_path

    def on_epoch_end(self, epoch, logs=None):
        with open(self.file_path, 'a') as file:
            file.write(f'Epoch {epoch} - Loss: {logs["loss"]}, Accuracy: {logs["accuracy"]}\n')
            # 您可以根据需要保存其他训练数据，如验证集的损失和准确率等

# 创建模型
model = tf.keras.Sequential([
    tf.keras.layers.Conv2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1)),
    tf.keras.layers.MaxPooling2D((2, 2)),
    tf.keras.layers.Flatten(),
    tf.keras.layers.Dense(10, activation='softmax')
])

# 编译模型
model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])

# 创建Callback来保存训练数据
save_data_callback = SaveTrainingDataCallback(file_path='training_data.txt')

# 训练模型并在每个epoch结束时保存训练数据
model.fit(x_train, y_train, epochs=10, callbacks=[save_data_callback])

要保存每个epoch的训练数据并将其存放到一个文件中，您可以使用Callback回调函数来实现。Callback提供了在训练过程中执行特定操作的灵活性，包括在每个epoch结束时保存训练数据。

然后改写到自己代码为

model = Sequential()

model.add(Conv2D(32, kernel_size=(3, 3), activation='relu', input_shape=(48,48,1)))
model.add(Conv2D(64, kernel_size=(3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2),strides=(2,2)))
model.add(Dropout(0.25))

model.add(Conv2D(128, kernel_size=(3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2),strides=(2,2)))
model.add(Conv2D(128, kernel_size=(3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2),strides=(2,2)))
model.add(Dropout(0.25))

model.add(Flatten())
model.add(Dense(1024, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(7, activation='softmax'))

import tensorflow as tf

class SaveTrainingDataCallback(tf.keras.callbacks.Callback):
    def __init__(self, file_path):
        super(SaveTrainingDataCallback, self).__init__()
        self.file_path = file_path

    def on_epoch_end(self, num_epoch, logs=None):
        with open(self.file_path, 'a') as file:
            file.write(f'Epoch {num_epoch} - Loss: {logs["loss"]}, Accuracy: {logs["accuracy"]}, Val Loss: {logs["val_loss"]}, Val Accuracy: {logs["val_accuracy"]}\n')



# If you want to train the same model or try other models, go for this
if mode == "train":
    # 使用旧版本的Adam优化器


    model.compile(loss='categorical_crossentropy',    optimizer = tf.keras.optimizers.legacy.Adam(learning_rate=0.001, decay=1e-6),metrics=['accuracy'])

    # 创建Callback来保存训练数据
    save_data_callback = SaveTrainingDataCallback(file_path='training_data.txt')

    model_info = model.fit_generator(
            train_generator,
            steps_per_epoch=num_train // batch_size,
            epochs=num_epoch,
            validation_data=validation_generator,
            validation_steps=num_val // batch_size,
            callbacks=[save_data_callback]
    )
    train_loss = model_info.history['loss']
    train_accuracy = model_info.history['accuracy']
    val_loss = model_info.history['val_loss']
    val_accuracy = model_info.history['val_accuracy']

    plot_model_history(model_info)
    model.save_weights('model.h5')

添加的代码为

import tensorflow as tf

class SaveTrainingDataCallback(tf.keras.callbacks.Callback):
    def __init__(self, file_path):
        super(SaveTrainingDataCallback, self).__init__()
        self.file_path = file_path

    def on_epoch_end(self, num_epoch, logs=None):
        with open(self.file_path, 'a') as file:
            file.write(f'Epoch {num_epoch} - Loss: {logs["loss"]}, Accuracy: {logs["accuracy"]}, Val Loss: {logs["val_loss"]}, Val Accuracy: {logs["val_accuracy"]}\n')
#在这个示例中，我们定义了一个自定义的回调函数SaveTrainingDataCallback
，在每个epoch结束时将训练和验证数据写入到指定的文件中。然后我们创建了一个回调函数实例，并将其传递给模型的fit方法中，以在模型训练时保存训练数据。

# 创建Callback来保存训练数据
save_data_callback = SaveTrainingDataCallback(file_path='training_data.txt')