点云处理：基于Paddle2.0实现PointNet++对点云进行分类处理②

纲要

    一、简介

    二、数据处理

    三、PointNet++(SSG)网络搭建

    四、训练、测试

一、简介

    在上一节点云处理：基于Paddle2.0实现PointNet++对点云进行分类处理①中，我们实现了PointNet++中比较重要的几个基础部分的搭建，包括Sampling层，Grouping层等的搭建，那么这一节我们将一起来实现PointNet++(SSG)整体网络搭建，并进行训练和测试。

二、数据处理

!unzip data/data70460/shapenet_part_seg_hdf5_data.zip
!mv hdf5_data dataset

import os
import numpy as np
import random
import h5py
import paddle
import paddle.nn as nn
import paddle.nn.functional as F
from util.model import PointNetSetAbstraction #引入模块

train_list = ['ply_data_train0.h5', 'ply_data_train1.h5', 'ply_data_train2.h5', 'ply_data_train3.h5', 'ply_data_train4.h5', 'ply_data_train5.h5']
test_list = ['ply_data_test0.h5', 'ply_data_test1.h5']
val_list = ['ply_data_val0.h5']

def pointDataLoader(mode='train'):
    path = './dataset/'
    BATCHSIZE = 64
    MAX_POINT = 1024

    datas = []
    labels = []
    if mode == 'train':
        for file_list in train_list:
            f = h5py.File(os.path.join(path, file_list), 'r') #读取h5文件
            
            #for key in f.keys():
                #print("aaaaaaaaaaaaaaaaaaaaaaaa")
                #print(f[key], key, f[key].name)
            #break
           
            datas.extend(f['data'][:, :MAX_POINT, :]) #（2048，1024，3）的列表
            labels.extend(f['label']) #（2048，1）的列表
            f.close()
    elif mode == 'test':
        for file_list in test_list:
            f = h5py.File(os.path.join(path, file_list), 'r')
            datas.extend(f['data'][:, :MAX_POINT, :])
            labels.extend(f['label'])
            f.close()
    else:
        for file_list in val_list:
            f = h5py.File(os.path.join(path, file_list), 'r')
            datas.extend(f['data'][:, :MAX_POINT, :])
            labels.extend(f['label'])
            f.close()

    datas = np.array(datas)
    labels = np.array(labels)
    print('==========load over==========')

    index_list = list(range(len(datas)))

    def pointDataGenerator():
        if mode == 'train':
            random.shuffle(index_list)
        datas_list = []
        labels_list = []
        for i in index_list:
            datas_list.append(datas[i].T.astype('float32')) 
            labels_list.append(labels[i].astype('int64'))
            if len(datas_list) == BATCHSIZE: #64个样本为一批
                yield np.array(datas_list), np.array(labels_list)#（64，3，1024）（64，1）
                datas_list = []
                labels_list = []
        if len(datas_list) > 0:
            yield np.array(datas_list), np.array(labels_list)

    return pointDataGenerator

补充注释：

1.

 for key in f.keys():

   print("aaaaaaaaaaaaaaaaaaaaaaaa")

   print(f[key], key, f[key].name)

aaaaaaaaaaaaaaaaaaaaaaaa
<HDF5 dataset "data": shape (2048, 2048, 3), type "<f4"> data /data
<HDF5 dataset "label": shape (2048, 1), type "|u1"> label /label
<HDF5 dataset "pid": shape (2048, 2048), type "|u1"> pid /pid

2.np.array(datas).shape   np.array(labels).shape

训练时：

(2048, 1024, 3)
(2048, 1)
(4096, 1024, 3)
(4096, 1)
(6144, 1024, 3)
(6144, 1)
(8192, 1024, 3)
(8192, 1)
(10240, 1024, 3)
(10240, 1)
(12137, 1024, 3)
(12137, 1)

测试时：

预测时：

(1870, 1024, 3)
(1870, 1)

三、PointNet++(SSG)网络搭建

PointNet++(SSG)网络搭建过程：

    1、首先将多个SetAbstraction层堆叠起来对点云进行特征提取，SetAbstraction层作用可以类比于图像处理中的卷积层，能够提取点云中局部的全局特征。（Figure1中的Part1）

    2、然后最终的SetAbstraction层调用sample_and_group_all函数，作用可以类比于图像处理中的全局池化，将点云中的特征聚拢。（Figure1中的Part2）

    3、最后经过一个mlp搭建分类最后一部分网络。（Figure1中的Part3）

class PointNet2(nn.Layer):
    def __init__(self, num_class=16, normal_channel=False):#是否使用其他特征信息
        super(PointNet2, self).__init__()
        in_channel = 6 if normal_channel else 3
        self.normal_channel = normal_channel
        self.sa1 = PointNetSetAbstraction(npoint=512, radius=0.2, nsample=32, in_channel=in_channel, mlp=[64, 64, 128], group_all=False)
        #Conv (3,64,1) (64,64,1) (64,128,1)
        self.sa2 = PointNetSetAbstraction(npoint=128, radius=0.4, nsample=64, in_channel=128 + 3, mlp=[128, 128, 256], group_all=False)
        #Conv (128,128,1) (128,128,1) (128,256,1)
        self.sa3 = PointNetSetAbstraction(npoint=None, radius=None, nsample=None, in_channel=256 + 3, mlp=[256, 512, 1024], group_all=True)
        #Conv (256,256,1) (256,512,1) (512,1024,1)
        self.fc1 = nn.Linear(1024, 512)
        self.bn1 = nn.BatchNorm1D(512)
        self.drop1 = nn.Dropout(0.4)
        self.fc2 = nn.Linear(512, 256)
        self.bn2 = nn.BatchNorm1D(256)
        self.drop2 = nn.Dropout(0.4)
        self.fc3 = nn.Linear(256, num_class)
    
    def forward(self, xyz): #(64,3,1024)
        B, _, _ = xyz.shape  #B=64
        if self.normal_channel:
            norm = xyz[:, 3:, :]
            xyz = xyz[:, :3, :]
        else:
            norm = None
        l1_xyz, l1_points = self.sa1(xyz, norm) #(64,3,512) (64,128,512)
        l2_xyz, l2_points = self.sa2(l1_xyz, l1_points) #(64,3,128) (64,256,128)
        l3_xyz, l3_points = self.sa3(l2_xyz, l2_points) #(64,3,1) (64,1024,1)
        x = l3_points.reshape([B, 1024]) #(64,1024)
        x = self.drop1(F.relu(self.bn1(self.fc1(x))))#(64,512)
        x = self.drop2(F.relu(self.bn2(self.fc2(x))))#(64,256)
        x = self.fc3(x) #(64,16)
        x = F.log_softmax(x, -1)#（64，16）

        return x

if __name__ == '__main__':
    PointNet2 = PointNet2()
    paddle.summary(PointNet2, (64, 3, 1024))

四、训练与测试

def train():
    train_loader = pointDataLoader(mode='train')
    val_loader = pointDataLoader(mode='val')

    model = PointNet2()
    model.train()
    optim = paddle.optimizer.Adam(parameters=model.parameters(), weight_decay=0.001)

    epoch_num = 2
    for epoch in range(epoch_num):
        # train
        print("===================================train===========================================")
        for batch_id, data in enumerate(train_loader()):
            inputs = paddle.to_tensor(data[0]) #（64，3，1024） 
            labels = paddle.to_tensor(data[1])#（64，1）

            predicts = model(inputs)#（64，16）
            loss = F.nll_loss(predicts, labels)
            acc = paddle.metric.accuracy(predicts, labels)        

            if batch_id % 10 == 0: 
                print("train: epoch: {}, batch_id: {}, loss is: {}, accuracy is: {}".format(epoch, batch_id, loss.numpy(), acc.numpy()))
            
            loss.backward()
            optim.step()
            optim.clear_grad()

        if epoch % 2 == 0:
            paddle.save(model.state_dict(), './model/PointNet2.pdparams')
            paddle.save(optim.state_dict(), './model/PointNet2.pdopt')
        
        # validation
        print("===================================val===========================================")
        model.eval()
        accuracies = []
        losses = []
        for batch_id, data in enumerate(val_loader()):
            inputs = paddle.to_tensor(data[0])
            labels = paddle.to_tensor(data[1])

            predicts = model(inputs)

            loss = F.nll_loss(predicts, labels)
            acc = paddle.metric.accuracy(predicts, labels)    
            
            losses.append(loss.numpy())
            accuracies.append(acc.numpy())

        avg_acc, avg_loss = np.mean(accuracies), np.mean(losses)
        print("validation: loss is: {}, accuracy is: {}".format(avg_loss, avg_acc))
        model.train()

if __name__ == '__main__':
    train()

二、PointNet++(MSG)网络简介

    PointNet++(MSG)网络是在PointNet++(SSG)网络基础上加入了多尺度特征提取策略，其中的MSG，即Multi-scale Grouping(多尺度)构建网络，是PointNet++网络基础版升级版。通过在同一PointNetSetAbstractionMsg层中，采取不同的尺度(Grouding中球形领域半径R的不同)对点云特征进行提取并concat起来，实现了多尺度特征提取，使得网络性能更好。

    PointNet++(MSG)网络实现了在点云同一个FeatureMap中对多尺度的特征提取，采用不同尺度意味着不同大小的感受野，这个可以类比于GoogleNet的Inception结构。

PointNetSetAbstractionMsg层

    PointNetSetAbstractionMsg层中多尺度特征提取依靠radius_list，radius_list输入的是一个list，里面对应的是球形领域中的不同半径，用于对不同的半径做Grouding，经过特征提取后，这样就能得到多尺度的特征，最终将不同半径下的点云特征（即不同尺度的特征）保存在new_points_list中，再最后将这些不同尺度的特征拼接到一起。

    PointNetSetAbstractionMsg层

Input:
    xyz: input points position data, [B, C, N]
    points: input points data, [B, D, N]
Return:
    new_xyz: sampled points position data, [B, C, S]
    new_points_concat: sample points feature data, [B, D', S]

class PointNetSetAbstractionMsg(nn.Layer):
    def __init__(self, npoint, radius_list, nsample_list, in_channel, mlp_list):
        super(PointNetSetAbstractionMsg, self).__init__()
        self.npoint = npoint
        self.radius_list = radius_list
        self.nsample_list = nsample_list
        self.conv_blocks = []
        self.bn_blocks = []
        for i in range(len(mlp_list)):
            convs = []
            bns = []
            last_channel = in_channel + 3
            for out_channel in mlp_list[i]:
                convs.append(nn.Conv2D(last_channel, out_channel, 1))
                bns.append(nn.BatchNorm2D(out_channel))
                last_channel = out_channel
            self.conv_blocks.append(convs)
            self.bn_blocks.append(bns)

    def forward(self, xyz, points):
        xyz = xyz.transpose([0, 2, 1]) #(B,N,3)
        if points is not None:
            points = points.transpose([0, 2, 1]) #(B,N,D)

        B, N, C = xyz.shape
        S = self.npoint
        new_xyz = index_points(xyz, farthest_point_sample(xyz, S))#(B,S,3)
        new_points_list = []
        for i, radius in enumerate(self.radius_list):
            K = self.nsample_list[i]
            group_idx = query_ball_point(radius, K, xyz, new_xyz)
            grouped_xyz = index_points(xyz, group_idx) #(64,S,K,C)
            grouped_xyz -= new_xyz.reshape([B, S, 1, C])
            if points is not None:
                grouped_points = index_points(points, group_idx) #(64,S,K,D)
                grouped_points = paddle.concat([grouped_points, grouped_xyz], axis=-1)
                #(64,S,K,C+D)
            else:
                grouped_points = grouped_xyz#(64,S,K,C)

            grouped_points = grouped_points.transpose([0, 3, 2, 1])
            for j in range(len(self.conv_blocks[i])):
                conv = self.conv_blocks[i][j]
                bn = self.bn_blocks[i][j]
                grouped_points =  F.relu(bn(conv(grouped_points)))
            new_points = paddle.max(grouped_points, 2) # [B, D', S]
            new_points_list.append(new_points)

        new_xyz = new_xyz.transpose([0, 2, 1])
        new_points_concat = paddle.concat(new_points_list, axis=1)
        return new_xyz, new_points_concat

三、PointNet++(MSG)网络搭建

class PointNet2(nn.Layer):
    def __init__(self, num_class=16, normal_channel=False):
        super(PointNet2, self).__init__()
        in_channel = 3 if normal_channel else 0
        self.normal_channel = normal_channel
        self.sa1 = PointNetSetAbstractionMsg(512, [0.1, 0.2, 0.4], [16, 32, 128], in_channel, [[32, 32, 64], [64, 64, 128], [64, 96, 128]])
                     #r=0.1 K=16 mlp=[32,32,64]  -->(64,64,512)
        #(64,3,1024) #r=0.2 K=32 mlp=[64,64,128] -->(64,128,512)     (64,320,512)
                     #r=0.4 K=32 mlp=[64,96,128] -->(64,128,512)
        #l1_xyz:(64,3,512) l1_points:(64,320,512)
        self.sa2 = PointNetSetAbstractionMsg(128, [0.2, 0.4, 0.8], [32, 64, 128], 320, [[64, 64, 128], [128, 128, 256], [128, 128, 256]])            
                    #r=0.2 K=32  mlp=[64,64,128]   -->(64,128,128)
                    #r=0.4 K=64  mlp=[128,128,256] -->(64,256,128)    (64,640,128)
                    #r=0.8 K=128 mlp=[128,128,256] -->(64,256,128)
        #l2_xyz:(64,3,128) l2_points:(64,640,128)      
        self.sa3 = PointNetSetAbstraction(None, None, None, 640 + 3, [256, 512, 1024], True)
                    #mlp=[256,512,1024]  -->(64,1024,1)
        #l3_xyz:(64,3,1) l3_points:(64,1024,1)
        self.fc1 = nn.Linear(1024, 512)
        self.bn1 = nn.BatchNorm1D(512)
        self.drop1 = nn.Dropout(0.4)
        self.fc2 = nn.Linear(512, 256)
        self.bn2 = nn.BatchNorm1D(256)
        self.drop2 = nn.Dropout(0.5)
        self.fc3 = nn.Linear(256, num_class)

    def forward(self, xyz): #(64,3,1024)
        B, _, _ = xyz.shape
        if self.normal_channel:#判断是否有其他特征信息
            norm = xyz[:, 3:, :]
            xyz = xyz[:, :3, :]
        else:
            norm = None
        l1_xyz, l1_points = self.sa1(xyz, norm)#多尺度集合抽象层
        l2_xyz, l2_points = self.sa2(l1_xyz, l1_points)#多尺度集合抽象层
        l3_xyz, l3_points = self.sa3(l2_xyz, l2_points)#单尺度集合抽象层
        x = l3_points.reshape([B, 1024]) #(64,1024)
        x = self.drop1(F.relu(self.bn1(self.fc1(x)))) #(64,512)
        x = self.drop2(F.relu(self.bn2(self.fc2(x)))) #(64,256)
        x = self.fc3(x) #(64,16)
        x = F.log_softmax(x, -1) #(64,16)

        return x