ResNet笔记

1. ResNet网络

1.1 整体网络结构

论文：《Deep Residual Learning for Image Recognition》

网络中的亮点：

• 超深的网络结构。（突破1000层）
• 提出 Residual 模块。
• 使用 Batch Normalization 加速训练。（丢弃dropout）

对于网络模型不断加深，会出现的问题：①梯度消失和梯度爆炸现象②退化现象

解决方法 ：①对数据进行标准化处理、还有权重初始化，或者是BN初始化处理，可以解决梯度消失和梯度爆炸。

②Residual 模块解决退化现象。

1.2 Batch Normalization

BN:Batch Normalization

• 计算一个Batch数据的feature map然后在进行标准化（batch越大越接近整个数据集的分布，效果越好）
• BN层放在卷积层和激活层之间，且无需设置bias
• 训练时需要设置True，测试时设置False

1.3 residual 结构

residual 结构

主分支和短路连接（Shortcut）结果相加

 残差模块：一条路不变（恒等映射）；另一条路负责拟合相对于原始网络的残差，去纠正原始网络的偏差，而不是让整体网络去拟合全部的底层映射，这样网络只需要纠正偏差。

 虚线表示的是用1x1的卷积核进行了维度处理，用于深层的神经网络。

 1.4 分组卷积

 将一个图像的channel拆分成不同的group卷积，这样所需要的参数更少（3通道以上使用）

2. 网络构建 

2.1 构建残差网络

# 18或34层的残差结构。
class BasicBlock(nn.Module):
    expansion = 1  # 残差结构所使用卷积核个数的一个变化，1表示是之前的一倍，没有变化。

    def __init__(self, in_channel, out_channel, stride=1, downsample=None, **kwargs):
        super(BasicBlock, self).__init__()
        self.conv1 = nn.Conv2d(in_channels=in_channel, out_channels=out_channel,
                               kernel_size=3, stride=stride, padding=1, bias=False) # BatchNorm2d不需要偏置。
        self.bn1 = nn.BatchNorm2d(out_channel)
        self.relu = nn.ReLU()
        self.conv2 = nn.Conv2d(in_channels=out_channel, out_channels=out_channel,
                               kernel_size=3, stride=1, padding=1, bias=False)
        self.bn2 = nn.BatchNorm2d(out_channel)
        self.downsample = downsample   # downsample为是虚线还是实线的残差结构。

    def forward(self, x):
        identity = x   # 残差结构的分支
        if self.downsample is not None: # 是否是虚线的残差结构
            identity = self.downsample(x)

        out = self.conv1(x)
        out = self.bn1(out)
        out = self.relu(out)

        out = self.conv2(out)
        out = self.bn2(out)

        out += identity
        out = self.relu(out)

        return out

# 50、101、152层的残差结构。
class Bottleneck(nn.Module):
    """
    注意：原论文中，在虚线残差结构的主分支上，第一个1x1卷积层的步距是2，第二个3x3卷积层步距是1。
    但在pytorch官方实现过程中是第一个1x1卷积层的步距是1，第二个3x3卷积层步距是2，
    这么做的好处是能够在top1上提升大概0.5%的准确率。
    可参考Resnet v1.5 https://ngc.nvidia.com/catalog/model-scripts/nvidia:resnet_50_v1_5_for_pytorch
    """
    expansion = 4   # 卷积核个数变为原来的4倍

    def __init__(self, in_channel, out_channel, stride=1, downsample=None,
                 groups=1, width_per_group=64):
        super(Bottleneck, self).__init__()

        width = int(out_channel * (width_per_group / 64.)) * groups

        self.conv1 = nn.Conv2d(in_channels=in_channel, out_channels=width,
                               kernel_size=1, stride=1, bias=False)  # squeeze channels
        self.bn1 = nn.BatchNorm2d(width)
        # -----------------------------------------
        self.conv2 = nn.Conv2d(in_channels=width, out_channels=width, groups=groups,
                               kernel_size=3, stride=stride, bias=False, padding=1)
        self.bn2 = nn.BatchNorm2d(width)
        # -----------------------------------------
        self.conv3 = nn.Conv2d(in_channels=width, out_channels=out_channel*self.expansion, # 这里卷积核个数变为原来的4倍。
                               kernel_size=1, stride=1, bias=False)  # unsqueeze channels
        self.bn3 = nn.BatchNorm2d(out_channel*self.expansion)
        self.relu = nn.ReLU(inplace=True) # inplace为True不影响结果，可以节省内存。
        self.downsample = downsample

    def forward(self, x):
        identity = x
        if self.downsample is not None:
            identity = self.downsample(x)

        out = self.conv1(x)
        out = self.bn1(out)
        out = self.relu(out)

        out = self.conv2(out)
        out = self.bn2(out)
        out = self.relu(out)

        out = self.conv3(out)
        out = self.bn3(out)

        out += identity
        out = self.relu(out)

        return out

2.2 创造各个卷积层

    # 创建每种卷积层。
    # channel为主分支卷积核的个数，block_num为卷积层的层数。
    def _make_layer(self, block, channel, block_num, stride=1):
        downsample = None
        # 分支是否是虚线。
        if stride != 1 or self.in_channel != channel * block.expansion:
            downsample = nn.Sequential(
                nn.Conv2d(self.in_channel, channel * block.expansion, kernel_size=1, stride=stride, bias=False),
                nn.BatchNorm2d(channel * block.expansion))

        layers = [] # 新建一个列表，存放每一个卷积层。
        # 第一层的分支为虚线，先添加进去。
        layers.append(block(self.in_channel,
                            channel,
                            downsample=downsample,
                            stride=stride,
                            groups=self.groups,
                            width_per_group=self.width_per_group))
        self.in_channel = channel * block.expansion # 更新输入通道数

        # 遍历剩下的层，逐个添加到列表中。
        for _ in range(1, block_num): # 从1开始，因为第一层已经添加了，0表示第一层。
            layers.append(block(self.in_channel,
                                channel,
                                groups=self.groups,
                                width_per_group=self.width_per_group))

        return nn.Sequential(*layers)

    def forward(self, x):
        # conv1
        x = self.conv1(x)
        x = self.bn1(x)
        x = self.relu(x)
        x = self.maxpool(x)

        x = self.layer1(x) # conv2_x
        x = self.layer2(x) # conv3_x
        x = self.layer3(x) # conv4_x
        x = self.layer4(x) # conv5_x

        if self.include_top:
            x = self.avgpool(x) #池化
            x = torch.flatten(x, 1) # 展平处理
            x = self.fc(x) #全联接层

        return x
    def forward(self, x):
        # conv1
        x = self.conv1(x)
        x = self.bn1(x)
        x = self.relu(x)
        x = self.maxpool(x)

        x = self.layer1(x) # conv2_x
        x = self.layer2(x) # conv3_x
        x = self.layer3(x) # conv4_x
        x = self.layer4(x) # conv5_x

        if self.include_top:
            x = self.avgpool(x) #池化
            x = torch.flatten(x, 1) # 展平处理
            x = self.fc(x) #全联接层

        return x

2.3 定义网络模型

# 定义ResNet模型。
class ResNet(nn.Module):

    def __init__(self,
                 block, # 用的那种残差结构
                 blocks_num, # 每层的残差结构的个数
                 num_classes=1000, # 类别数目
                 include_top=True, # 表示在ResNet的基础上构建更复杂的模型，默认为true
                 groups=1,
                 width_per_group=64):
        super(ResNet, self).__init__()
        self.include_top = include_top
        self.in_channel = 64   # 输入的通道数为64

        self.groups = groups
        self.width_per_group = width_per_group

        # 第一种卷积层 conv1。
        # RGD图片输入的通道数为3，输出为64（卷积核的组数）。
        self.conv1 = nn.Conv2d(3, self.in_channel, kernel_size=7, stride=2,
                               padding=3, bias=False)
        self.bn1 = nn.BatchNorm2d(self.in_channel)
        self.relu = nn.ReLU(inplace=True)
        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
        self.layer1 = self._make_layer(block, 64, blocks_num[0]) # 对应conv2_x
        self.layer2 = self._make_layer(block, 128, blocks_num[1], stride=2) # 对应conv3_x
        self.layer3 = self._make_layer(block, 256, blocks_num[2], stride=2) # 对应conv4_x
        self.layer4 = self._make_layer(block, 512, blocks_num[3], stride=2) # 对应conv5_x
        if self.include_top:
            # 自适应采样，不管输入尺寸是多少，输出为(1, 1)。
            self.avgpool = nn.AdaptiveAvgPool2d((1, 1))  # output size = (1, 1)
            self.fc = nn.Linear(512 * block.expansion, num_classes) # 全连接层

        # 初始化权重。
        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')

3. Train

• 加载预训练模型的方法
         model_weight_path = "resnet34.pth" # 预训练模型的权重
         net.load_state_dict(torch.load(model_weight_path, map_location='cpu'))
• 对batch进行归一化
         transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])])
• 冻结网络，只训练最后一层Linear

  for param in net.parameters():
      param.requires_grad = False
  in_channel = net.fc.in_features # 全连接层输入的深度。
  net.fc = nn.Linear(in_channel, 5)

import json
import torch
import torch.nn as nn
import torch.optim as optim
from torch.utils.data import DataLoader
from torchvision import transforms, datasets
from model import resnet34

def main():
    device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
    print(device)

    data_transform = {
        "train": transforms.Compose([transforms.RandomResizedCrop(224),
                                     transforms.RandomHorizontalFlip(),
                                     transforms.ToTensor(),
                                     transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])]),
        "val": transforms.Compose([transforms.Resize(256),
                                   transforms.CenterCrop(224), # 中心裁剪
                                   transforms.ToTensor(),
                                   transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])])}

    train_path = "F:/HUWEI/pytorch_learning/GoogLeNet/zhang/train"
    val_path = "F:/HUWEI/pytorch_learning/GoogLeNet/zhang/val"
    train_data = datasets.ImageFolder(root=train_path,transform=data_transform["train"])
    val_data = datasets.ImageFolder(root=val_path,transform = data_transform["val"])
    train_dataset = DataLoader(train_data,batch_size=32,shuffle=True)
    val_dataset = DataLoader(val_data, batch_size=32, shuffle=True)
    val_num = len(val_data)

    # {'daisy':0, 'dandelion':1, 'roses':2, 'sunflower':3, 'tulips':4}
    flower_list = train_data.class_to_idx
    cla_dict = dict((val, key) for key, val in flower_list.items())
    # write dict into json file
    json_str = json.dumps(cla_dict, indent=4)
    with open('class_indices.json', 'w') as json_file:
        json_file.write(json_str)


    # 初始化模型。
    net = resnet34()
    # 使用迁移学习的方式进行训练。
    # load pretrain weights
    # download url: https://download.pytorch.org/models/resnet34-333f7ec4.pth
    model_weight_path = "resnet34-333f7ec4.pth" # 预训练模型的权重
    net.load_state_dict(torch.load(model_weight_path, map_location='cpu')) # 加载预训练模型
    for param in net.parameters():
        param.requires_grad = False

    # change fc layer structure
    in_channel = net.fc.in_features # 全连接层输入的深度。
    net.fc = nn.Linear(in_channel, 5)
    net.to(device)

    # define loss function
    loss_function = nn.CrossEntropyLoss()

    # construct an optimizer
    params = [p for p in net.parameters() if p.requires_grad]
    optimizer = optim.Adam(params, lr=0.0001)

    epochs = 3
    best_acc = 0.0
    save_path = './resNet34.pth'
    train_steps = len(train_dataset)
    for epoch in range(epochs):
        # train
        net.train()# 控制网络中BatchNorm2d的状态。
        running_loss = 0.0
        for step, data in enumerate(train_dataset):
            images, labels = data
            optimizer.zero_grad()
            logits = net(images.to(device))
            loss = loss_function(logits, labels.to(device))
            loss.backward()
            optimizer.step()

            # print statistics
            running_loss += loss.item()


        # validate
        net.eval() # 控制网络中BatchNorm2d的状态。
        acc = 0.0  # accumulate accurate number / epoch
        with torch.no_grad():

            for val_data in val_dataset:
                val_images, val_labels = val_data
                outputs = net(val_images.to(device))
                # loss = loss_function(outputs, test_labels)
                predict_y = torch.max(outputs, dim=1)[1]
                acc += torch.eq(predict_y, val_labels.to(device)).sum().item()


        val_accurate = acc / val_num
        print('[epoch %d] train_loss: %.3f  val_accuracy: %.3f' %
              (epoch + 1, running_loss / train_steps, val_accurate))

        if val_accurate > best_acc:
            best_acc = val_accurate
            torch.save(net.state_dict(), save_path)

    print('Finished Training')


if __name__ == '__main__':
    main()

4. predict

import json
import torch
from PIL import Image
from torchvision import transforms
from model import resnet34

data_transform = transforms.Compose([
            transforms.Resize((224,244)),
            transforms.ToTensor(),
            transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))
    ])
img_path = "./th.jpg"
img = Image.open(img_path)
img = data_transform(img)
#扩充维度，否则图片是三维的
img = torch.unsqueeze(img,dim = 0)
try:
    json_file = open("class_indices.json","r")
    class_indict = json.load(json_file)
except Exception as e:
    print(e)
    exit(-1)
#初始化网络
model =  resnet34(num_classes=5)
model_path = "resNet34.pth"
model.load_state_dict(torch.load(model_path))
model.eval()
with torch.no_grad():
    output = torch.squeeze(model(img))          #压缩掉第一个维度
    predict = torch.softmax(output,dim=0)       #得到概率
    predict_cla = torch.argmax(predict).numpy()

print(class_indict[str(predict_cla)],predict[predict_cla].item())

需对batch归一化