Surface splatting (2D Gaussian splatting)代码分析

源码地址

colab.research.google.com/drive/1qoclD7HJ3-o0O1R8cvV3PxLhoDCMsH8W

核心代码

surface_splatting

def surface_splatting(means3D, scales, quats, colors, opacities, intrins, viewmat, projmat):
    # Rasterization setup
    projmat = torch.zeros(4,4).cuda()
    projmat[:3,:3] = intrins
    projmat[-1,-2] = 1.0
    projmat = projmat.T
    # T 是论文中的 M
    T, colors, opacities, center, depth, radii = setup(means3D, scales, quats, opacities, colors, viewmat, projmat)

    # Rasterization
    # 1. Generate pixels
    W, H = (intrins[0,-1] * 2).long(), (intrins[1,-1] * 2).long()
    W, H = W.item(), H.item()
    pix = torch.stack(torch.meshgrid(torch.arange(W),
        torch.arange(H), indexing='xy'), dim=-1).to('cuda')

    # 2. Compute ray splat intersection # Eq.9 and Eq.10
    x = pix.reshape(-1,1,2)[..., :1]
    y = pix.reshape(-1,1,2)[..., 1:]
    k = -T[None][..., 0] + x * T[None][..., 3] # 这个是 h_u, 因为公式8 h_u = ⊤ · h_x = ⊤ · (−1, 0, 0, 𝑥) = -T_0 + x · T_3
    l = -T[None][..., 1] + y * T[None][..., 3] # 这个是 h_v
    # 因为论文中 h_u 和 (𝑢, 𝑣, 1, 1) 点成为0， h_v 和 (𝑢, 𝑣, 1, 1) 点成为0
    # 所以 h_u 和 h_v  与 (𝑢, 𝑣, 1, 1) 垂直
    # h_u 和 h_v 的 叉乘是 (𝑢, 𝑣, 1, 1)
    points = torch.cross(k, l, dim=-1) # 叉乘
    s = points[..., :2] / points[..., -1:]

    # 3. add low pass filter # Eq. 11
    # when a point (2D Gaussian) viewed from a far distance or from a slended angle
    # the 2D Gaussian will falls between pixels and no fragment is used to rasterize the Gaussian
    # so we should add a low pass filter to handle such aliasing.
    dist3d = (s * s).sum(dim=-1)
    filtersze = np.sqrt(2) / 2
    dist2d = (1/filtersze)**2 * (torch.cat([x,y], dim=-1) - center[None,:,:2]).norm(dim=-1)**2
    # min of dist2 is equal to max of Gaussian exp(-0.5 * dist2)
    dist2 = torch.min(dist3d, dist2d)
    # dist2 = dist3d
    depth_acc = (homogeneous(s) * T[None,..., -1]).sum(dim=-1)

    # 4. accumulate 2D gaussians through alpha blending # Eq.12
    image, depthmap = alpha_blending_with_gaussians(dist2, colors, opacities, depth_acc, H, W)
    return image, depthmap, center, radii, dist2