Pointnet++ 代码阅读笔记

Pointnet++ 代码阅读笔记

代码地址:https://github.com/yanx27/Pointnet_Pointnet2_pytorch

PointNet++ 论文笔记可以参考这篇博客。

一、网络框架

可以由论文的图中看到， 主要分为三个部分:

1. PointNetSetAbstraction
   这部分源码用了 farthest point sampling, query ball 算法。
2. PointNetFeaturePropagation
   这部分包括 sample and grouping 算法。
3. Classifier
   主要是卷积层。根据分支不同，代码有所不同。

先来看一下 Segmentation 的模型框架:

class get_model(nn.Module):
    def __init__(self, num_classes, normal_channel=False):
        super(get_model, self).__init__()
        if normal_channel:
            additional_channel = 3
        else:
            additional_channel = 0

        self.normal_channel = normal_channel
        #    PointNetAbstractionMsg(npoint, radius_list, nsample_list, in_channel, mlp_list):
        self.sa1 = PointNetAbstractionMsg(512, [0.1, 0.2, 0.4], [32, 64, 128], 3+additional_channel, [[32, 32, 64], [64, 64, 128], [64, 96, 128]])
        self.sa2 = PointNetSetAbstractionMsg(128, [0.4,0.8], [64, 128], 128+128+64, [[128, 128, 256], [128, 196, 256]])
        #    PointNetSetAbstraction(npoint, radius, nsample, in_channel, mlp, group_all)
        self.sa3 = PointNetSetAbstraction(npoint=None, radius=None, nsample=None, in_channel=512 + 3, mlp=[256, 512, 1024], group_all=True)
        #    PointNetFeaturePropagation(in_channel, mlp)
        self.fp1 = PointNetFeaturePropagation(in_channel=150+additional_channel, mlp=[128, 128])
        self.fp2 = PointNetFeaturePropagation(in_channel=576, mlp=[256, 128])
        self.fp3 = PointNetFeaturePropagation(in_channel=1536, mlp=[256, 256])

        self.conv1 = nn.Conv1d(128, 128, 1)
        self.bn1 = nn.BatchNorm1d(128)
        self.drop1 = nn.Dropout(0.5)
        self.conv2 = nn.Conv1d(128, num_classes, 1)

    def forward(self, xyz, cls_label):
        # Set Abstraction layers
        B, N, C = xyz.shape
        if self.normal_channel: #是否使用normal特征
            l0_points = xyz # 所有点
            l0_xyz = xyz[:, :3, :] # 点的xyz
        else:
            l0_points = xyz
            l0_xyz = xyz
        l1_xyz, l1_points = self.sa1(l0_xyz, l0_points) # 第一个PointNetSetAbstractionMSG
        l2_xyz, l2_points = self.sa2(l1_xyz, l1_points) # 第二个PointNetSetAbstractionMSG
        l3_xyz, l3_points = self.sa3(l2_xyz, l2_points) # PointNetSetAbstraction
        # Feature Propagation layers
        l2_points = self.fp3(l2_xyz, l3_xyz, l2_points, l3_points)
        l1_poitns = self.fp2(l1_xyz, l2_xyz, l1_points, l2_points)
        cls_label_one_hot = cls_label.view(B, 16, 1).repeat(1, 1, N)
        l0_points = self.fp1(l0_xyz, l1_xyz, torch.cat([cls_label_one_hot,l0_xyz,l0_points],1), l1_points)
        feat = F.relu(self.bn1(self.conv1(l0_points)))
        x = self.drop1(feat)
        x = self.conv2(x)
        x = F.log_softmax(x, dim=1)
        x = x.permute(0, 2, 1)
        return x, l3_points

二、Set Abstraction Multi-scale grouping

数据进入网络的第一个阶段—SetAbstractionLayers。源码使用了MSG，并没有提供MRG。但是作者论文提到MRG比MSG计算效率更好。而在这一个阶段，源码使用了farthest point sampling, query ball 算法。 经过那两个算法之后，简单的PointNet会进行特征提取。 先来看以下这个Module的代码:

class PointNetSetAbstractionMsg(nn.Module):
    def __init__(self, npoint, radius_list, nsample_list, in_channel, mlp_list):
        super(PointNetSetAbstractionMsg, self).__init__()
        self.npoint = npoint
        self.radius_list = radius_list
        self.nsample_list = nsample_list
        self.conv_blocks = nn.ModuleList()
        self.bn_blocks = nn.ModuleList()
        for i in range(len(mlp_list)):
            convs = nn.ModuleList()
            bns = nn.ModuleList()
            last_channel = in_channel + 3
            for out_channel in mlp_list[i]:
                convs.append(nn.Conv2d(last_channel, out_channel, 1))
                bns.append(nn.BatchNorm2d(out_channel))
                last_channel = out_channel
            self.conv_blocks.append(convs)
            self.bn_blocks.append(bns)

    def forward(self, xyz, points):
        """
        Input: 
            xyz: input points position data, [B, C, N];  x,y,z 
            points: input points data, [B, D, N]; x,y,z,normal    
        Return:
            new_xyz: sampled points position data, [B, C, S]
            new_points_concat: sample points feature data, [B, D', S]
        """
        xyz = xyz.permute(0, 2, 1)
        if points is not None:
            points = points.permute(0, 2, 1)
        
        B, N, C = xyz.shape
        S = self.npoint
        new_xyz = index_points(xyz, farthest_point_sample(xyz, S)) # sampling layer
        new_points_list = []
        for i, radius in enumerate(self.radius_list): # query_ball radius, 不同的半径，形成了MSG
            K = self.nsample_list[i]
            group_idx = query_ball_point(radius, K, xyz, new_xyz) # query_ball 算法
            grouped_xyz = index_points(xyz, group_idx)     # grouped_xyz:[B, npoint, nsample, C]
            grouped_xyz -= new_xyz.view(B, S, 1, C)     # grouped_xyz减去采样点即中心值
            if points is not None:    # 如果每个点上面有新的特征的维度，则用新的特征与旧的特征拼接，否则直接返回旧的特征
                grouped_points = index_points(points, group_idx)
                grouped_points = torch.cat([grouped_points, grouped_xyz], dim=-1) 
            else:
                grouped_points = grouped_xyz
            
            grouped_points = grouped_points.permute(0, 3, 2, 1) # [B, D, K, S]
            # K is the number of points in the neighborhood of centroid points

            # 进行简单的PointNet特征提取。
            for j in range(len(self.conv_blocks[i])):
                conv = self.conv_blocks[i][j]
                bn = self.bn_blocks[i][j]
                grouped_points = F.relu(bn(conv(grouped_points)))

            new_points = torch.max(grouped_points, 2)[0]
            new_points_list.append(new_points)

        new_xyz = new_xyz.permute(0, 2, 1)
        new_points_concat = torch.cat(new_points_list, dim=1)
        return new_xyz, new_points_concat

可以看到，数据点一进来就进行了farthest_point_sample函数进行sampling。这一层是sampling layer具体如下:

1.Farthest Point Sample(FPS)算法

Sampling layer, iterative farthest point sampling(FPS)to choose a subset of points { x 1 , x 2 , . . . , x m } \{x_1, x_2, ..., x_m\} {x1​,x2​,...,xm​}。具体算法可参考[1]。

def farthest_point_sample(xyz, npoint):
    """
    Input:
        xyz: pointcloud data. [B, N, 3]
        npoint: number of samples
    
    Return:
        centroids: sampled pointcloud index, [B, npoint]
    """
    device = xyz.device
    B, N, C = xyz.shape
    centroids = torch.zeros(B, npoint, dtype=torch.long).to(device)# 用来存每个样本的npoint个采样点的索引位置。
    distance = torch.ones(B, N).to(device) * 1e10 # 存每一个样本中所有点到某一个点的距离。
    farthest = torch.randint(0, N, (B,), dtype=torch.long).to(device) #randint(low=0, high, size, out=None, dtype=None)  随机选取一个点作为最远距离点
    batch_indices = torch.arange(B, dtype=torch.long).to(device) 
    for i in range(npoint): # Interative
        centroids[:, i] = farthest #更新第i个最远点
        centroid = xyz[batch_indices, farthest, :].view(B, 1, 3) # 取第i个样本中最远的点
        dist = torch.sum((xyz - centorid) ** 2 , -1) # 计算点集中的所有点到这个最远点的欧式距离
        mask = dist < distance  
        distance[mask] = dist[mask]# 更新distances，记录样本中每个点距离所有已出现的采样点的最小距离
        farthest = torch.max(distance, -1)[1]# 从更新后的distances矩阵中找出距离最远的点，作为最远点用于下一轮迭代
    
    return centroids #返回最远点索引

采样完之后，就会到Query ball 算法。这一层是grouping layer。具体如下:

2. Query Ball算法

def square_distance(src, dst):
    """
    Calculate Euclid distance between each two points.

    src^T * dst = xn * xm + yn *ym + zn * zm
    sum(src^2, dim=-1) = xn*xn + yn*yn + zn*zn;
    sum(dst^2, dim=-1) = xm*xm + ym*ym + zm*zm;

    dist = (xn - xm)^2 + (yn - ym)^2 + (zn - zm)^2
         = sum(src^2, dim=-1) + sum(dst^2, dim=-1)-2src^T*dst
    
    
    Input:
        src: source points, [B, N, C]
        dst: target points, [B, M, C]
    Output:
        dist: per-point square distance, [B, N, M]
    """
    B, N, _ = src.shape
    _, M, _ = dst.shape
    dist = -2 * torch.matmul(src, dst.permute(0, 2, 1)) 
    dist += torch.sum(src ** 2, -1).view(B, N, 1)
    dist += torch.sum(dst ** 2, -1).view(B, 1, M)
    return dist
    


def query_ball_point(radius, nsample, xyz, new_xyz):
    """
    Input:
        radius: local region radius
        nsample: max sample number in local
        xyz: all points, [B, N, 3]
        new_xyz: query points, [B, S, 3]
    Return:
        group_idx: grouped points index, [B, S, nsample]
    """
    device = xyz.device
    B, N, C = xyz.shape
    _, S, _ = new_xyz.shape
    group_idx = torch.arange(N, dtype=torch.long).to(device).view(1, 1, N).repeat([B, S, 1])
    sqrdists = square_distance(new_xyz, xyz) # sqrdists: [B, S, N] 记录中心点与所有点之间的欧几里德距离
    group_idx[sqrdists > radius ** 2] = N # 找到所有距离大于radius^2的，其group_idx直接置为N；其余的保留原来的值
    group_idx = group.idx.sort(dim=-1)[0][:, :, :nsample]  # 做升序排列，前面大于radius^2的都是N，会是最大值，所以会直接在剩下的点中取出前nsample个点
    # 考虑到有可能前nsample个点中也有被赋值为N的点（即球形区域内不足nsample个点），这种点需要舍弃，直接用第一个点来代替即可
    group_first = group_idx[:, :, 0].view(B, S, 1).repeat([1, 1, nsample])# group_first: [B, S, k]， 实际就是把group_idx中的第一个点的值复制为了[B, S, K]的维度，便利于后面的替换
    mask = group_idx == N     # 找到group_idx中值等于N的点
    group_idx[mask] = group_first[mask]    # 将这些点的值替换为第一个点的值

    return group_idx 

三、PointNetSetAbstraction

这一个Module主要是通过sample_and_group的操作形成局部的group，然后对局部的group中的每一个点做MLP操作，最后进行局部的最大池化，得到局部的全局特征。具体代码如下:

class PointNetSetAbstraction(nn.Module):
    def __init__(self, npoint, radius, nsample, in_channel, mlp, group_all):
        super(PointNetSetAbstraction, self).__init__()
        self.npoint = npoint
        self.radius = radius
        self.nsample = nsample
        self.mlp_convs = nn.ModuleList()
        self.mlp_bns = nn.ModuleList()
        last_channel = in_channel
        for out_channel in mlp:
            self.mlp_convs.append(nn.Conv2d(last_channel, out_channel, 1))
            self.mlp_bns.append(nn.BatchNorm2d(out_channel))
            last_channel = out_channel
        self.group_all = group_all

    def forward(self, xyz, points):
        """
        Input:
            xyz: input points position data, [B, N, C]
            points: input points data, [B, D, N]
        Return:
            new_xyz: sampled points position data, [B, C, S]
            new_points_concat: sample points feature data, [B, D', S]
        """
        xyz = xyz.permute(0, 2, 1)
        if points is not None:
            points = points.permute(0, 2, 1)
        
        if self.group_all:
            new_xyz, new_points = sample_and_group_all(xyz, points)
        else:
            new_xyz, new_points = sample_and_group(self.npoint, self.radius, self.nsample, xyz, points)
        
        # new_xyz: sampled points position data, [B, npoint, C]
        # new_points: sampled points data, [B, npoint, nsample, C+D]
        new_points = new_points.permute(0, 3, 2, 1) # [B, C+D, nsample, npoint]
        for i, conv in enumerate(self.mlp_convs):
            bn = self.mlp_bns[i]
            new_points = F.relu(bn(conv(new_points)))
        
        new_points = torch.max(new_points, 2)[0]
        new_xyz = new_xyz.permute(0, 2, 1)
        return new_xyz, new_points

有两个sample and group。 一个是sample and group all， 另一个是sample and group，其区别在于sample_and_group_all直接将所有点作为一个group.

1. Sample and group all

sample_and_group_all直接将所有点作为一个group，即增加一个长度为1的维度而已，也存在拼接新的特征的过程。具体如下:

def sample_and_group_all(xyz, points):
    """
    Input:
        xyz: input points position data, [B, N, 3]
        points: input points data, [B, N, D]
    Return:
        new_xyz: sampled points position data, [B, 1, 3]
        new_points: sampled points data, [B, 1, N, 3+D]
    """
    device = xyz.device
    B, N, C = xyz.shape
    new_xyz = torch.zeros(B, 1, C).to(device)
    grouped_xyz = xyz.view(B, 1, N, C)
    if points is not None:
        new_points = torch.cat([grouped_xyz, points.view(B, 1, N, -1)], dim=-1)
    else:
        new_points = grouped_xyz
    return new_xyz, new_points

2. Sample and group

sample_and_group的实现步骤如下：

• 先用farthest_point_sample函数实现最远点采样FPS得到采样点的索引，再通过index_points将这些点的从原始点中挑出来，作为new_xyz
• 利用query_ball_point和index_points将原始点云通过new_xyz 作为中心分为npoint个球形区域其中每个区域有nsample个采样点
• 每个区域的点减去区域的中心值
• 如果每个点上面有新的特征的维度，则用新的特征与旧的特征拼接，否则直接返回旧的特征

def sample_and_group(npoint, radius, nsample, xyz, points, returnfps=False):
    """
    Input:
        npoint:
        radius:
        nsample:
        xyz: input points position data, [B, N, 3]
        points: input points data, [B, N, D]
    
    Return:
        new_xyz: sampled points position data, [B, npoint, nsample, 3]
        new_points: sampled points data, [B, npoint, nsample, 3+D]
    """
    B, N, C = xyz.shape
    S = npoint
    fps_idx = farthest_point_sample(xyz, npoint) # [B, npoint, C]
    new_xyz = index_points(xyz, fps_idx)
    idx = query_ball_point(radius, nsample, xyz, new_xyz)
    grouped_xyz = index_points(xyz, idx) # [B, npoint, nsample, C]
    grouped_xyz_norm = grouped_xyz - new_xyz.view(B, S, 1, C)

    if points is not None:
        grouped_points = index_points(points, idx)
        new_points = torch.cat([grouped_xyz_norm, grouped_points], dim=-1) # [B, npoint, nsample, C+D]
    else:
        new_points = grouped_xyz_norm
    if returnfps:
        return new_xyz, new_points, grouped_xyz, fps_idx
    else:
        return new_xyz, new_points

四、 PointNetFeaturePropagation

这个Module是用于segmentation分支的。论文中的公式如下:

$$f^{(j)}(x)=\frac{{\sum }_{i=1}^k{w}_i(x)f_i^{(j)}}{{\sum }_{i=1}^k{w}_i(x)}$$
其中，$w_i(x)=\frac{1}{d(x,x_i{)}^p},j=1,...,C$, $p=2,k=3$

class PointNetFeaturePropagation(nn.Module):
    def __init__(self, in_channel, mlp):
        super(PointNetFeaturePropagation, self).__init__()
        self.mlp_convs = nn.ModuleList()
        self.mlp_bns = nn.ModuleList()
        last_channel = in_channel
        for out_channel in mlp:
            self.mlp_convs.append(nn.Conv1d(last_channel, out_channel, 1))
            self.mlp_bns.append(nn.BatchNorm1d(out_channel))
            last_channel = out_channel

    def forward(self, xyz1, xyz2, points1, points2):
        """
        Input:
            xyz1: input points position data, [B, C, N]
            xyz2: sampled input points position data, [B, C, S]
            points1: input points data, [B, D, N]
            points2: input points data, [B, D, S]
        Return:
            new_points: upsampled points data, [B, D', N]
        """
        xyz1 = xyz1.permute(0, 2, 1)
        xyz2 = xyz2.permute(0, 2, 1)

        points2 = points2.permute(0, 2, 1)
        B, N, C = xyz1.shape
        _, S, _ = xyz2.shape

        if S == 1:
            interpolated_points = points2.repeat(1, N, 1)
        else:
            dists = square_distance(xyz1, xyz2) # d(x,x_i)^p
            dists, idx = dists.sort(dim=-1)
            dists, idx = dists[:, :, :3], idx[:, :, :3]  # [B, N, 3]

            dist_recip = 1.0 / (dists + 1e-8)
            norm = torch.sum(dist_recip, dim=2, keepdim=True)
            weight = dist_recip / norm
            interpolated_points = torch.sum(index_points(points2, idx) * weight.view(B, N, 3, 1), dim=2)

        if points1 is not None:
            points1 = points1.permute(0, 2, 1)
            new_points = torch.cat([points1, interpolated_points], dim=-1)
        else:
            new_points = interpolated_points

        new_points = new_points.permute(0, 2, 1)
        for i, conv in enumerate(self.mlp_convs):
            bn = self.mlp_bns[i]
            new_points = F.relu(bn(conv(new_points)))
        return new_points

五、Classification Branch

分类网络分支前面都是一样，只有后面稍有区别:

class get_model(nn.Module):
    def __init__(self,num_class,normal_channel=True):
        super(get_model, self).__init__()
        in_channel = 3 if normal_channel else 0
        self.normal_channel = normal_channel
        self.sa1 = PointNetSetAbstractionMsg(512, [0.1, 0.2, 0.4], [16, 32, 128], in_channel,[[32, 32, 64], [64, 64, 128], [64, 96, 128]])
        self.sa2 = PointNetSetAbstractionMsg(128, [0.2, 0.4, 0.8], [32, 64, 128], 320,[[64, 64, 128], [128, 128, 256], [128, 128, 256]])
        self.sa3 = PointNetSetAbstraction(None, None, None, 640 + 3, [256, 512, 1024], True)
        self.fc1 = nn.Linear(1024, 512)
        self.bn1 = nn.BatchNorm1d(512)
        self.drop1 = nn.Dropout(0.4)
        self.fc2 = nn.Linear(512, 256)
        self.bn2 = nn.BatchNorm1d(256)
        self.drop2 = nn.Dropout(0.5)
        self.fc3 = nn.Linear(256, num_class)

    def forward(self, xyz):
        B, _, _ = xyz.shape
        if self.normal_channel:
            norm = xyz[:, 3:, :]
            xyz = xyz[:, :3, :]
        else:
            norm = None
        l1_xyz, l1_points = self.sa1(xyz, norm)
        l2_xyz, l2_points = self.sa2(l1_xyz, l1_points)
        l3_xyz, l3_points = self.sa3(l2_xyz, l2_points)
        x = l3_points.view(B, 1024)
        x = self.drop1(F.relu(self.bn1(self.fc1(x))))
        x = self.drop2(F.relu(self.bn2(self.fc2(x))))
        x = self.fc3(x)
        x = F.log_softmax(x, -1)


        return x,l3_points

六、Loss function

分割网络和分类网络的loss都是一样的，是log+sorfmax + NLLLoss，也就是CrossEntropyLoss。

class get_loss(nn.Module):
    def __init__(self):
        super(get_loss, self).__init__()

    def forward(self, pred, target, trans_feat):
        total_loss = F.nll_loss(pred, target)

        return total_loss

Reference

1. https://blog.csdn.net/weixin_39373480/article/details/88934146