PyTorch 深度学习实践-加载数据集

视频指路
参考博客笔记
参考笔记二

介绍

在深度学习时用min-batch来平衡训练时间和性能上的需求，之后训练周期要写成两层嵌套循环。epoch：所有训练样本进行完一次前向和反向传播，batch-size：训练的时候的样本数量，Iteration：总训练数据/训练样本数

集成了dataset之后编写的类就可以用dataset的功能了

train_loader = DataLoader(dataset=dataset, batch_size=32,shuffle=True, num_workers=2)

训练的代码要用main包装起来

解释：数据集，批数据大小，是否打乱，构成batch读取数据的时候是不是要用多线程，几个并行的线程

1、需要mini_batch 就需要import DataSet和DataLoader

2、继承DataSet的类需要重写init，getitem,len魔法函数。分别是为了加载数据集，获取数据索引，获取数据总量。

3、DataLoader对数据集先打乱(shuffle)，然后划分成mini_batch。

4、len函数的返回值 除以 batch_size 的结果就是每一轮epoch中需要迭代的次数。

5、inputs, labels = data中的inputs的shape是[32,8],labels 的shape是[32,1]。表明成功使用mini-batch=32

# class DiabetesDataset(Dataset):
#     def __init__(self, filepath):
#         xy = np.loadtxt(filepath, delimiter=',', dtype=np.float32)
#         self.len = xy.shape[0]  # shape(多少行，多少列)
#         self.x_data = torch.from_numpy(xy[:, :-1])
#         self.y_data = torch.from_numpy(xy[:, [-1]])
#
#     def __getitem__(self, index):
#         return self.x_data[index], self.y_data[index]
#
#     def __len__(self):
#         return self.len

课程代码

import torch
import numpy as np
from torch.utils.data import Dataset
from torch.utils.data import DataLoader
from sklearn.model_selection import train_test_split

# 读取原始数据，并划分训练集和测试集
raw_data = np.loadtxt('diabetes.csv', delimiter=',', dtype=np.float32)
X = raw_data[:, :-1]
y = raw_data[:, [-1]]
Xtrain, Xtest, Ytrain, Ytest = train_test_split(X, y, test_size=0.3)
Xtest = torch.from_numpy(Xtest)
Ytest = torch.from_numpy(Ytest)


# 将训练数据集进行批量处理
# prepare dataset

class DiabetesDataset(Dataset):
    def __init__(self, data, label):
        self.len = data.shape[0]  # shape(多少行，多少列)
        self.x_data = torch.from_numpy(data)
        self.y_data = torch.from_numpy(label)

    def __getitem__(self, index):
        return self.x_data[index], self.y_data[index]

    def __len__(self):
        return self.len


train_dataset = DiabetesDataset(Xtrain, Ytrain)
train_loader = DataLoader(dataset=train_dataset, batch_size=32, shuffle=True, num_workers=0)  # num_workers 多线程


# design model using class


class Model(torch.nn.Module):
    def __init__(self):
        super(Model, self).__init__()
        self.linear1 = torch.nn.Linear(8, 6)
        self.linear2 = torch.nn.Linear(6, 4)
        self.linear3 = torch.nn.Linear(4, 2)
        self.linear4 = torch.nn.Linear(2, 1)
        self.sigmoid = torch.nn.Sigmoid()

    def forward(self, x):
        x = self.sigmoid(self.linear1(x))
        x = self.sigmoid(self.linear2(x))
        x = self.sigmoid(self.linear3(x))
        x = self.sigmoid(self.linear4(x))
        return x


model = Model()

# construct loss and optimizer
criterion = torch.nn.BCELoss(reduction='mean')
optimizer = torch.optim.SGD(model.parameters(), lr=0.01)


# training cycle forward, backward, update

def train(epoch):
    train_loss = 0.0
    count = 0
    for i, data in enumerate(train_loader, 0):
        inputs, labels = data
        y_pred = model(inputs)

        loss = criterion(y_pred, labels)

        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
        train_loss += loss.item()
        count = i

    if epoch % 200 == 199:
        print("train loss:", train_loss / count, end=',')


def test():
    with torch.no_grad():
        y_pred = model(Xtest)
        y_pred_label = torch.where(y_pred >= 0.5, torch.tensor([1.0]), torch.tensor([0.0]))
        acc = torch.eq(y_pred_label, Ytest).sum().item() / Ytest.size(0)
        print("test acc:", acc)


if __name__ == '__main__':
    for epoch in range(5000):
        train(epoch)
        if epoch % 200 == 199:
            test()

作业实现

数据集下载地址，找到train.csv和test.csv，重点是这里的数据，由于数据集中包含非数值数据和缺失值，需要对数据进行清理和转换，这里是问gpt实现的

import torch
import numpy as np
import pandas as pd
from torch.utils.data import Dataset, DataLoader
from sklearn.preprocessing import StandardScaler

# 数据预处理
def preprocess_data(df):
    # 处理缺失值
    df['Age'].fillna(df['Age'].median(), inplace=True)
    df['Fare'].fillna(df['Fare'].median(), inplace=True)
    df['Embarked'].fillna(df['Embarked'].mode()[0], inplace=True)

    # 将分类数据转换为数值数据
    df['Sex'] = df['Sex'].map({'male': 0, 'female': 1})
    df['Embarked'] = df['Embarked'].map({'C': 0, 'Q': 1, 'S': 2})

    # 选择特征
    features = ['Pclass', 'Sex', 'Age', 'SibSp', 'Parch', 'Fare', 'Embarked']
    df = df[features]

    return df

# 读取训练数据并预处理
train_df = pd.read_csv('./dataset/train.csv')
train_df = preprocess_data(train_df)

# 提取特征和标签
x_train = train_df.values
y_train = pd.read_csv('./dataset/train.csv')['Survived'].values

# 标准化特征
scaler = StandardScaler()
x_train = scaler.fit_transform(x_train)

# 将测试数据加载并预处理
test_df = pd.read_csv('./dataset/test.csv')
passenger_ids = test_df['PassengerId'].values
test_df = preprocess_data(test_df)
Xtest = scaler.transform(test_df.values)
Xtest = torch.from_numpy(Xtest).float()

# 创建自定义数据集
class TitanicDataset(Dataset):
    def __init__(self, data, labels):
        self.data = torch.from_numpy(data).float()
        self.labels = torch.from_numpy(labels).float()

    def __len__(self):
        return len(self.data)

    def __getitem__(self, idx):
        return self.data[idx], self.labels[idx]

# 创建数据加载器
train_dataset = TitanicDataset(x_train, y_train)
train_loader = DataLoader(dataset=train_dataset, batch_size=32, shuffle=True)

# 定义模型
class Model(torch.nn.Module):
    def __init__(self):
        super().__init__()
        self.linear1 = torch.nn.Linear(7, 32)
        self.linear2 = torch.nn.Linear(32, 16)
        self.linear3 = torch.nn.Linear(16, 1)
        self.activate = torch.nn.ReLU()

    def forward(self, x):
        x = self.activate(self.linear1(x))
        x = self.activate(self.linear2(x))
        x = torch.sigmoid(self.linear3(x))
        return x

# 实例化模型
model = Model()

# 定义损失函数和优化器
criterion = torch.nn.BCELoss()
optimizer = torch.optim.SGD(model.parameters(), lr=0.01)

# 训练模型
def train(epoch):
    model.train()
    train_loss = 0.0
    for inputs, labels in train_loader:
        optimizer.zero_grad()
        outputs = model(inputs)
        labels = labels.view(-1, 1)  # 调整标签形状以匹配输出
        loss = criterion(outputs, labels)
        loss.backward()
        optimizer.step()
        train_loss += loss.item()

    if epoch % 200 == 199:
        print(f"Epoch {epoch + 1}, Train Loss: {train_loss / len(train_loader)}")

# 训练循环
if __name__ == '__main__':
    for epoch in range(5000):
        train(epoch)

# 预测测试数据
with torch.no_grad():
    model.eval()
    y_pred = model(Xtest)
    y_pred_label = (y_pred >= 0.5).float().numpy().astype(int)
    print(y_pred_label)


# 将预测结果保存为CSV文件
output = pd.DataFrame({'PassengerId': passenger_ids, 'Survived': y_pred_label.flatten()})
output.to_csv('submission.csv', index=False)

得分：0.72966

让gpt改进了一下网络部分：

import torch
import numpy as np
import pandas as pd
from torch.utils.data import Dataset, DataLoader
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split


# 数据预处理
def preprocess_data(df):
    df['Age'].fillna(df['Age'].median(), inplace=True)
    df['Fare'].fillna(df['Fare'].median(), inplace=True)
    df['Embarked'].fillna(df['Embarked'].mode()[0], inplace=True)
    df['Sex'] = df['Sex'].map({'male': 0, 'female': 1})
    df['Embarked'] = df['Embarked'].map({'C': 0, 'Q': 1, 'S': 2})
    features = ['Pclass', 'Sex', 'Age', 'SibSp', 'Parch', 'Fare', 'Embarked']
    df = df[features]
    return df

# 读取训练数据并预处理
train_df = pd.read_csv('./dataset/train.csv')
train_df = preprocess_data(train_df)
x_train = train_df.values
y_train = pd.read_csv('./dataset/train.csv')['Survived'].values

# 标准化特征
scaler = StandardScaler()
x_train = scaler.fit_transform(x_train)

# 将测试数据加载并预处理
test_df = pd.read_csv('./dataset/test.csv')
passenger_ids = test_df['PassengerId'].values
test_df = preprocess_data(test_df)
Xtest = scaler.transform(test_df.values)
Xtest = torch.from_numpy(Xtest).float()

# 创建自定义数据集
class TitanicDataset(Dataset):
    def __init__(self, data, labels):
        self.data = torch.from_numpy(data).float()
        self.labels = torch.from_numpy(labels).float()

    def __len__(self):
        return len(self.data)

    def __getitem__(self, idx):
        return self.data[idx], self.labels[idx]

# 创建数据加载器
train_dataset = TitanicDataset(x_train, y_train)
train_loader = DataLoader(dataset=train_dataset, batch_size=32, shuffle=True)

# 定义改进的模型
class Model(torch.nn.Module):
    def __init__(self):
        super().__init__()
        self.linear1 = torch.nn.Linear(7, 64)
        self.linear2 = torch.nn.Linear(64, 32)
        self.linear3 = torch.nn.Linear(32, 16)
        self.linear4 = torch.nn.Linear(16, 1)
        self.activate = torch.nn.ReLU()
        self.dropout = torch.nn.Dropout(p=0.5)

    def forward(self, x):
        x = self.activate(self.linear1(x))
        x = self.dropout(x)
        x = self.activate(self.linear2(x))
        x = self.dropout(x)
        x = self.activate(self.linear3(x))
        x = torch.sigmoid(self.linear4(x))
        return x

# 实例化模型
model = Model()

# 定义损失函数和优化器
criterion = torch.nn.BCELoss()
optimizer = torch.optim.Adam(model.parameters(), lr=0.001)

# 训练模型
def train(epoch):
    model.train()
    train_loss = 0.0
    for inputs, labels in train_loader:
        optimizer.zero_grad()
        outputs = model(inputs)
        labels = labels.view(-1, 1)
        loss = criterion(outputs, labels)
        loss.backward()
        optimizer.step()
        train_loss += loss.item()

    if epoch % 200 == 199:
        print(f"Epoch {epoch + 1}, Train Loss: {train_loss / len(train_loader)}")

# 训练循环
if __name__ == '__main__':
    for epoch in range(5000):
        train(epoch)

# 预测测试数据
with torch.no_grad():
    model.eval()
    y_pred = model(Xtest)
    y_pred_label = (y_pred >= 0.5).float().numpy().astype(int)

# 将预测结果保存为CSV文件
output = pd.DataFrame({'PassengerId': passenger_ids, 'Survived': y_pred_label.flatten()})
output.to_csv('submission4.csv', index=False)

得分0.75358好吧看上去没有优化多少