深度学习——多GPU训练代码实现

1.数据并行性。

一台机器有K个GPU，给定训练模型，每个GPU参数值是相同且同步，每个GPU独立地维护一组完整地模型参数。k=2时数据并行地训练模型，利用两个GPU上的数据，并行计算小批量随机梯度下降。

 K个GPU并行训练过程：

①任何一次训练迭代中，给定的随机小批量样本分成K个部分，均分分给GPU

②每个GPU根据分配它的小批量子集，计算模型参数的损失和梯度

③将K个GPU中的局部梯度聚合，获得当前小批量的随机梯度

④聚合的梯度再重新分发给每个GPU

⑤每个GPU使用这个梯度，进行参数更新。

【代码说明】

1.搭建LeNet网络

# 初始化模型参数
scale = 0.01
W1 = torch.randn(size=(20, 1, 3, 3)) * scale
b1 = torch.zeros(20)
W2 = torch.randn(size=(50, 20, 5, 5)) * scale
b2 = torch.zeros(50)
W3 = torch.randn(size=(800, 128)) * scale
b3 = torch.zeros(128)
W4 = torch.randn(size=(128, 10)) * scale
b4 = torch.zeros(10)
params = [W1, b1, W2, b2, W3, b3, W4, b4]


# 定义模型
def lenet(X, params):
    h1_conv = F.conv2d(input=X, weight=params[0], bias=params[1])
    h1_activation = F.relu(h1_conv)
    h1 = F.avg_pool2d(input=h1_activation, kernel_size=(2, 2), stride=(2, 2))
    h2_conv = F.conv2d(input=h1, weight=params[2], bias=params[3])
    h2_activation = F.relu(h2_conv)
    h2 = F.avg_pool2d(input=h2_activation, kernel_size=(2, 2), stride=(2, 2))
    h2 = h2.reshape(h2.shape[0], -1)
    h3_linear = torch.mm(h2, params[4]) + params[5]
    h3 = F.relu(h3_linear)
    y_hat = torch.mm(h3, params[6]) + params[7]
    return y_hat


# 交叉熵损失函数
loss = nn.CrossEntropyLoss(reduction='none')

2.数据同步：①分发参数给GPU并附加梯度

# 向多设备分发参数
def get_params(params, device):
    new_params = [p.clone().to(device) for p in params]  # new_params获取了全部的权重和偏置，并且复制到设备device
    for p in new_params:
        p.requires_grad_()  # 是参数可以获得梯度
    return new_params

②对多个设备的参数求和

# allreduce 函数将所有的向量相加，结果广播给所有GPU
def allreduce(data):
    for i in range(1, len(data)):
        data[0][:] += data[i].to(data[0].device)  # data有值，有设备。data[0]就是全部的值相加
    for i in range(1, len(data)):
        data[i] = data[0].to(data[i].device)  # 把相加后的data[0]复制到其他GPU去，并更新data[i]

3.数据分发：将小批量的数据均匀的分布给多个GPU，同时拆分数据和标签split_batch函数

# 将X和y拆分到多个设备上
def split_batch(X, y, devices):
    """将X和y拆分到多个设备上"""
    assert X.shape[0] == y.shape[0]
    return (nn.parallel.scatter(X, devices),
            nn.parallel.scatter(y, devices))

4.训练

①小批量上实现多GPU训练

def train_batch(X, y, device_params, devices, lr):
    X_shards, y_shards = split_batch(X, y, devices)  # X_shards是在每个GPU均分的小批量子集。
    # 在每个GPU上分别计算损失
    ls = [loss(lenet(X_shard, device_W), y_shard).sum()  # 求各个GPU的损失函数
          for X_shard, y_shard, device_W in zip(  # 拿出各个GPU分发的小批量子集在上面求损失函数
            X_shards, y_shards, device_params)]
    for l in ls:  # 反向传播在每个GPU上分别执行
        l.backward()
    # 将每个GPU的所有梯度相加，并将其广播到所有GPU
    with torch.no_grad():
        for i in range(len(device_params[0])):  # device_params[0]是参数
            allreduce(  # 求梯度的和
                [device_params[c][i].grad for c in range(len(devices))])  # 把c每个GPU的W的梯度求出来放到device_params
    # 在每个GPU上分别更新模型参数
    for param in device_params:
        d2l.sgd(param, lr, X.shape[0])  # 全尺寸的小批量进行调优

②训练函数

def train(num_gpus, batch_size, lr):
    train_iter, test_iter = d2l.load_data_fashion_mnist(batch_size)  # 训练集，测试集
    devices = [d2l.try_gpu(i) for i in range(num_gpus)]  # 获取全部GPU
    # 将模型参数复制到num_gpus个GPU
    device_params = [get_params(params, d) for d in devices]  # device_params[0]是参数 device_params[1]设备
    num_epochs = 10
    animator = d2l.Animator('epoch', 'test acc', xlim=[1, num_epochs])
    timer = d2l.Timer()
    for epoch in range(num_epochs):
        timer.start()
        for X, y in train_iter:
            # 为单个小批量执行多GPU训练
            train_batch(X, y, device_params, devices, lr)
            torch.cuda.synchronize()
        timer.stop()
        # 在GPU0上评估模型
        animator.add(epoch + 1, (d2l.evaluate_accuracy_gpu(
            lambda x: lenet(x, device_params[0]), test_iter, devices[0]),))
    print(f'测试精度：{animator.Y[0][-1]:.2f}，{timer.avg():.1f}秒/轮，'
          f'在{str(devices)}')


train(num_gpus=1, batch_size=256, lr=0.2)

 

测试精度：0.84，2.4秒/轮，在[device(type='cuda', index=0)]

【简洁实现】

1.搭建简单网络 resnet18

import torch
from torch import nn
from d2l import torch as d2l

def resnet18(num_classes, in_channels=1):
    """稍加修改的ResNet-18模型"""
    def resnet_block(in_channels, out_channels, num_residuals,
                     first_block=False):
        blk = []
        for i in range(num_residuals):
            if i == 0 and not first_block:
                blk.append(d2l.Residual(in_channels, out_channels,
                                        use_1x1conv=True, strides=2))
            else:
                blk.append(d2l.Residual(out_channels, out_channels))
        return nn.Sequential(*blk)

    # 该模型使用了更小的卷积核、步长和填充，而且删除了最大汇聚层
    net = nn.Sequential(
        nn.Conv2d(in_channels, 64, kernel_size=3, stride=1, padding=1),
        nn.BatchNorm2d(64),
        nn.ReLU())
    net.add_module("resnet_block1", resnet_block(
        64, 64, 2, first_block=True))
    net.add_module("resnet_block2", resnet_block(64, 128, 2))
    net.add_module("resnet_block3", resnet_block(128, 256, 2))
    net.add_module("resnet_block4", resnet_block(256, 512, 2))
    net.add_module("global_avg_pool", nn.AdaptiveAvgPool2d((1,1)))
    net.add_module("fc", nn.Sequential(nn.Flatten(),
                                       nn.Linear(512, num_classes)))
    return net

2.网络初始化

net = resnet18(10)
# 获取GPU列表
devices = d2l.try_all_gpus()
# 我们将在训练代码实现中初始化网络

3.训练

net = nn.DataParallel(net, device_ids=devices)

①需要在每个GPU上初始化网络参数w,b

②将小批量数据集均分到各个GPU上

③在每个GPU上计算损失函数以及梯度

④聚合梯度，并相应更新参数

def train(net, num_gpus, batch_size, lr):
    train_iter, test_iter = d2l.load_data_fashion_mnist(batch_size)  # 训练集，测试集
    devices = [d2l.try_gpu(i) for i in range(num_gpus)]  # 获取gpu真实的的个数

    def init_weights(m):
        if type(m) in [nn.Linear, nn.Conv2d]:
            nn.init.normal_(m.weight, std=0.01)  # 获取w参数

    net.apply(init_weights)  # 每个模型的结构更新参数
    # 在多个GPU上设置模型
    net = nn.DataParallel(net, device_ids=devices)  # DataParallel在每个GPU上都部署模型，参数，数据x平均分到各个GPU
    trainer = torch.optim.SGD(net.parameters(), lr)  # 定义优化器
    loss = nn.CrossEntropyLoss()  # 定义损失函数
    timer, num_epochs = d2l.Timer(), 10
    animator = d2l.Animator('epoch', 'test acc', xlim=[1, num_epochs])
    for epoch in range(num_epochs):
        net.train()  # 训练模式
        timer.start()
        for X, y in train_iter:
            trainer.zero_grad()
            X, y = X.to(devices[0]), y.to(devices[0])
            l = loss(net(X), y)
            l.backward()
            trainer.step()
        timer.stop()
        animator.add(epoch + 1, (d2l.evaluate_accuracy_gpu(net, test_iter),))
    print(f'测试精度：{animator.Y[0][-1]:.2f}，{timer.avg():.1f}秒/轮，'
          f'在{str(devices)}')


train(net, num_gpus=1, batch_size=256, lr=0.1)