3D目标检测SE-SSD 模型工程化复现

3D目标检测SE-SSD 模型工程化复现

3d目标检测se-ssd模型代码偏比赛，记录工程化中的一些问题

1、环境适配

pytorch： 1.7
cuda：10.2
cudnn：8.0
显卡：RTX3090
[se-ssd代码](https://github.com/Vegeta2020/SE-SSD)
[版本适配问题汇总](https://blog.csdn.net/weixin_44316541/article/details/124268113)

2、数据准备

数据集下载
大神链接
数据集准备：
DET3D环境安装
kitti数据集准备
ps（非常重要）：

det3d安装好，转化完kitti数据以后，需要删除，通过se-ssd算法环境安装教程重装

模型训练

源代码在cuda block运算会出现越界问题

cd SE-SSD-master/det3d/core/iou3d/src/

修改iou3d_kernel.cu文件，305-323行

__global__ void boxes_iou_bev_kernel(const int num_a, const float *boxes_a, const int num_b, const float *boxes_b, float *ans_iou){
    const long long int a_idx = blockIdx.y * THREADS_PER_BLOCK + threadIdx.y;
    const long long int b_idx = blockIdx.x * THREADS_PER_BLOCK + threadIdx.x;

    if (a_idx >= num_a || b_idx >= num_b){
        return;
    }
    const float * cur_box_a = boxes_a + a_33hangidx * 5;
    const float * cur_box_b = boxes_b + b_idx * 5;
    float cur_iou_bev = iou_bev(cur_box_a, cur_box_b);
    long long int idx = a_idx * num_b + b_idx;
    if (idx < 0)
    {
        printf("idx :%lld \n", idx);
    }
    ans_iou[idx] = cur_iou_bev;
}

模型推理

摄像机内参校准
大神链接
图像可视化

            detections = test(model, calib, img_size, save_dir=None, distributed=distributed)   ## 同源代码 result_dict, detections = test(data_loader, model, save_dir=None, distributed=distributed)
            out_img = show_image_with_boxes(img, detections[0], calib, show3d=False)
            if out_cap is None:
                out_cap_h, out_cap_w = out_img.shape[:2]
                fourcc = cv2.VideoWriter_fourcc(*'MJPG')
                out_cap = cv2.VideoWriter(
                    os.path.join('/home/video', '{}.avi'.format('se_ssd')),
                    fourcc, 5, (out_cap_w, out_cap_h))
            out_cap.write(out_img)
        if out_cap:
            out_cap.release()
        cv2.destroyAllWindows()

# -- coding: utf-8 --
# @Time : 2022/4/12 14:36
# @Author : lz

import numpy as np
import cv2

def roty(t):
    # Rotation about the y-axis.
    c = np.cos(t)
    s = np.sin(t)
    return np.array([[c, 0, s],
                     [0, 1, 0],
                     [-s, 0, c]])

def project_to_image(pts_3d, P):
    ''' Project 3d points to image plane.

    Usage: pts_2d = projectToImage(pts_3d, P)
      input: pts_3d: nx3 matrix
             P:      3x4 projection matrix
      output: pts_2d: nx2 matrix

      P(3x4) dot pts_3d_extended(4xn) = projected_pts_2d(3xn)
      => normalize projected_pts_2d(2xn)

      <=> pts_3d_extended(nx4) dot P'(4x3) = projected_pts_2d(nx3)
          => normalize projected_pts_2d(nx2)
    '''
    n = pts_3d.shape[0]
    pts_3d_extend = np.hstack((pts_3d, np.ones((n, 1))))
    # print(('pts_3d_extend shape: ', pts_3d_extend.shape))
    pts_2d = np.dot(pts_3d_extend, np.transpose(P))  # nx3
    pts_2d[:, 0] /= pts_2d[:, 2]
    pts_2d[:, 1] /= pts_2d[:, 2]
    return pts_2d[:, 0:2]

def compute_box_3d(obj, P, i):
    ''' Takes an object and a projection matrix (P) and projects the 3d
        bounding box into the image plane.
        Returns:
            corners_2d: (8,2) array in left image coord.
            corners_3d: (8,3) array in in rect camera coord.
    '''
    # compute rotational matrix around yaw axis
    R = roty(obj['rotation_y'][i])

    # 3d bounding box dimensions
    # l = obj.l
    # w = obj.w
    # h = obj.h
    l = obj['dimensions'][i][0]
    w = obj['dimensions'][i][2]
    h = obj['dimensions'][i][1]

    # 3d bounding box corners
    x_corners = [l / 2, l / 2, -l / 2, -l / 2, l / 2, l / 2, -l / 2, -l / 2]
    y_corners = [0, 0, 0, 0, -h, -h, -h, -h]
    z_corners = [w / 2, -w / 2, -w / 2, w / 2, w / 2, -w / 2, -w / 2, w / 2]

    # rotate and translate 3d bounding box
    corners_3d = np.dot(R, np.vstack([x_corners, y_corners, z_corners]))
    # print corners_3d.shape
    corners_3d[0, :] = corners_3d[0, :] + obj['location'][i][0]
    corners_3d[1, :] = corners_3d[1, :] + obj['location'][i][1]
    corners_3d[2, :] = corners_3d[2, :] + obj['location'][i][2]
    # print 'cornsers_3d: ', corners_3d
    # only draw 3d bounding box for objs in front of the camera
    if np.any(corners_3d[2, :] < 0.1):
        corners_2d = None
        return corners_2d, np.transpose(corners_3d)

    # project the 3d bounding box into the image plane
    corners_2d = project_to_image(np.transpose(corners_3d), P)
    # print 'corners_2d: ', corners_2d
    return corners_2d, np.transpose(corners_3d)

def draw_projected_box3d(image, qs, color=(255, 0, 255), thickness=2):
    ''' Draw 3d bounding box in image
        qs: (8,3) array of vertices for the 3d box in following order:
            1 -------- 0
           /|         /|
          2 -------- 3 .
          | |        | |
          . 5 -------- 4
          |/         |/
          6 -------- 7
    '''
    qs = qs.astype(np.int32)
    for k in range(0, 4):
        # Ref: http://docs.enthought.com/mayavi/mayavi/auto/mlab_helper_functions.html
        i, j = k, (k + 1) % 4
        # use LINE_AA for opencv3
        cv2.line(image, (qs[i, 0], qs[i, 1]), (qs[j, 0], qs[j, 1]), color, thickness)

        i, j = k + 4, (k + 1) % 4 + 4
        cv2.line(image, (qs[i, 0], qs[i, 1]), (qs[j, 0], qs[j, 1]), color, thickness)

        i, j = k, k + 4
        cv2.line(image, (qs[i, 0], qs[i, 1]), (qs[j, 0], qs[j, 1]), color, thickness)
    return image


def show_image_with_boxes(img, objects, calib, show3d=False):
    # print('detections:{')
    # print('            name:{}'.format(detections[0]['name']))
    # print('            truncated:{}'.format(detections[0]['truncated']))
    # print('            occluded:{}'.format(detections[0]['occluded']))
    # print('            alpha:{}'.format(detections[0]['alpha']))
    # print('            bbox:{}'.format(detections[0]['bbox']))
    # print('            dimensions:{}'.format(detections[0]['dimensions']))
    # print('            location:{}'.format(detections[0]['location']))
    # print('            rotation_y:{}'.format(detections[0]['rotation_y']))
    # print('            score:{}'.format(detections[0]['score']))
    # print('            metadata:{}'.format(detections[0]['metadata']))
    # print('}')
    img2 = np.copy(img)
    for i in range(len(objects['name'])):
        box3d_pts_2d, box3d_pts_3d = compute_box_3d(objects, calib['P2'], i)
        if box3d_pts_2d is not None:
            img2 = draw_projected_box3d(img2, box3d_pts_2d, color=[125, 15, 15])
    if show3d:
        cv2.imshow("img", img2)
    return img2

result

附-雷达点云可视化

# -*- coding: utf-8 -*-
# @Time    : 2022/4/1 10:51
# @Author  : lz

import os
import numpy as np
import struct
import open3d

def read_bin_velodyne(path):
    pc_list = []
    # path = 'proprecess/t1.npy'
    with open(path, 'rb') as f:
        content = f.read()
        pc_iter = struct.iter_unpack('ffff', content)
        # pc_iter = np.load(path).reshape(-1, 4)
        # print('points:{}'.format(pc_iter[0]))
        idx_cnt = 0
        for idx, point in enumerate(pc_iter):
            idx_cnt = idx if idx > idx_cnt else idx_cnt
            if idx < 10:
                print('points:{}'.format(point))
            pc_list.append([point[0], point[1], point[2]])
        print('idx_cnt:{}'.format(idx_cnt))
    return np.asarray(pc_list, dtype=np.float32)

def main():
    # root_dir = 'data(1)'
    root_dir = 'val_data'  # .bin文件目录
    filename = os.listdir(root_dir)
    file_number = len(filename)

    pcd = open3d.open3d.geometry.PointCloud()

    for i in range(file_number):
        path = os.path.join(root_dir, filename[i])
        print(path)
        if path.endswith('bin'):
        # if path.endswith('npy'):
            example = read_bin_velodyne(path)
        # From numpy to Open3D
        pcd.points = open3d.open3d.utility.Vector3dVector(example)
        open3d.open3d.visualization.draw_geometries([pcd])

if __name__=="__main__":
    main()