利用D455相机制作linemod数据集用于densefusion训练

         由于D455相机的单位是m，公开数据集的单位是mm，所以在用D455相机制作数据集的时候需要将mm转化为m。

参考文档：
https://blog.csdn.net/qq_41977396/article/details/135569808，制作自己的Linemod数据集（ObjectDatasetTools） | 马浩飞丨博客。

目前已经完成objectdatastools安装。

图像读取

使用Realsense相机录制一段物体的视频，对于旧模型使用record.py，对librealsense SDK 2.0使用recordf2.py。在终端中输入以下代码，

python record2.py LINEMOD/F（F为自己取的文件夹名称），python3为ubuntu自带的python

python3 record2.py LINEMOD/F

计算第一帧的变换，以制定的间隔（可在config/registrationParameters修改间隔），将变换（4x4矩阵）保存为numpy数组。计算结果保存在LINEMOD/OBJECTNAME/transforms.npy

python3 compute_gt_poses.py LINEMOD/F

下面代码会原始的registeredScene.ply将保存在指定的目录下（LINEMOD/F/registeredScene.ply）。registerScene.ply是整个场景的点云，包括桌面、标记纸，物体等等相机中的对象。

python3 register_scene.py LINEMOD/F

以上代码需要手动处理点云，具体方法放在在评论区

也可以使用以下代码生成但是效果不好

python3 register_segmented.py LINEMOD/F

制作models_info.yml

当完成了物体mesh网格文件的生成后，使用以下程序创建图像掩码和标签

python3 create_label_files.py LINEMOD/F

观察create_label_files.py是否和以下代码一样，如果无法生成min_xyz和size_xyz请修改成以下代码。

"""
create_label_files.py
---------------

This script produces:

1. Reorient the processed registered_scene mesh in a mesh with an AABB centered at the
   origin and the same dimensions as the OBB, saved under the name foldername.ply
2. Create label files with class labels and projections of 3D BBs in the format
   singleshotpose requires, saved under labels
3. Create pixel-wise masks, saved under mask
4. Save the homogeneous transform of object in regards to the foldername.ply in each 
   frame
"""

import numpy as np
from pykdtree.kdtree import KDTree
import trimesh
import cv2
import glob
import os
import sys
from tqdm import trange
from scipy.optimize import minimize
from config.registrationParameters import *
import json

def get_camera_intrinsic(folder):
    with open(folder+'intrinsics.json', 'r') as f:
        camera_intrinsics = json.load(f)


    K = np.zeros((3, 3), dtype='float64')
    K[0, 0], K[0, 2] = float(camera_intrinsics['fx']), float(camera_intrinsics['ppx'])
    K[1, 1], K[1, 2] = float(camera_intrinsics['fy']), float(camera_intrinsics['ppy'])

    K[2, 2] = 1.
    return (camera_intrinsics, K)

def compute_projection(points_3D,internal_calibration):
    points_3D = points_3D.T
    projections_2d = np.zeros((2, points_3D.shape[1]), dtype='float32')
    camera_projection = (internal_calibration).dot(points_3D)
    projections_2d[0, :] = camera_projection[0, :]/camera_projection[2, :]
    projections_2d[1, :] = camera_projection[1, :]/camera_projection[2, :]
    return projections_2d


def print_usage():
    
    print("Usage: create_label_files.py <path>")
    print("path: all or name of the folder")
    print("e.g., create_label_files.py all, create_label_files.py LINEMOD/Cheezit")
    
    
if __name__ == "__main__":
  
    try:
        if sys.argv[1] == "all":
            folders = glob.glob("LINEMOD/*/")
        elif sys.argv[1]+"/" in glob.glob("LINEMOD/*/"):
            folders = [sys.argv[1]+"/"]
        else:
            print_usage()
            exit()
    except:
        print_usage()
        exit()

    
    for classlabel,folder in enumerate(folders):
        # print(folder[8:-1], "is assigned class label:", classlabel)
        print("%s is assigned class label %d." % (folder[8:-1],classlabel))
        camera_intrinsics, K = get_camera_intrinsic(folder)
        path_label = folder + "labels"
        if not os.path.exists(path_label):
            os.makedirs(path_label)

        path_mask = folder + "mask"
        if not os.path.exists(path_mask):
            os.makedirs(path_mask)

        path_transforms = folder + "transforms"
        if not os.path.exists(path_transforms):
            os.makedirs(path_transforms)



        transforms_file = folder + 'transforms.npy'
        try:
            transforms = np.load(transforms_file)
        except:
            print("transforms not computed, run compute_gt_poses.py first")
            continue
        
        mesh = trimesh.load(folder + "registeredScene.ply")

        Tform = mesh.apply_obb()
        
        mesh.export(file_obj = folder + folder[8:-1] +".ply")

        points = mesh.bounding_box.vertices
        center = mesh.centroid
        min_x = np.min(points[:,0])
        min_y = np.min(points[:,1])
        min_z = np.min(points[:,2])
        max_x = np.max(points[:,0])
        max_y = np.max(points[:,1])
        max_z = np.max(points[:,2])
        points = np.array([[min_x, min_y, min_z], [min_x, min_y, max_z], [min_x, max_y, min_z],
                           [min_x, max_y, max_z], [max_x, min_y, min_z], [max_x, min_y, max_z],
                           [max_x, max_y, min_z], [max_x, max_y, max_z]])
        print(folder + "\tmin_x=" + str(min_x) + "\tmin_y=" + str(min_y) + "\tmin_z=" + str(min_z) + "\tsize_x=" + str((max_x-min_x)) + "\tsize_y=" + str((max_y-min_y)) + "\tsize_z=" + str((max_z-min_z)))

        points_original = np.concatenate((np.array([[center[0],center[1],center[2]]]), points))
        points_original = trimesh.transformations.transform_points(points_original,
                                                                   np.linalg.inv(Tform))
                    
        projections = [[],[]]
        
        for i in trange(len(transforms)):
            mesh_copy = mesh.copy()
            img = cv2.imread(folder+"JPEGImages/" + str(i*LABEL_INTERVAL) + ".jpg")
            transform = np.linalg.inv(transforms[i])
            transformed = trimesh.transformations.transform_points(points_original, transform)

            
            corners = compute_projection(transformed,K)
            corners = corners.T
            corners[:,0] = corners[:,0]/int(camera_intrinsics['width'])
            corners[:,1] = corners[:,1]/int(camera_intrinsics['height'])

            T = np.dot(transform, np.linalg.inv(Tform))
            mesh_copy.apply_transform(T)
            filename = path_transforms + "/"+ str(i*LABEL_INTERVAL)+".npy"
            np.save(filename, T)
            
            sample_points = mesh_copy.sample(10000)
            masks = compute_projection(sample_points,K)
            masks = masks.T


            min_x = np.min(masks[:,0])
            min_y = np.min(masks[:,1])
            max_x = np.max(masks[:,0])
            max_y = np.max(masks[:,1])
            # print(folder + "\tmin_x=" + str(min_x) + "\tmin_y=" + str(min_y) + "\tsize_x=" + str((max_x-min_x)) + "\tsize_y=" + str((max_y-min_y)))


            image_mask = np.zeros(img.shape[:2],dtype = np.uint8)
            for pixel in masks:
                cv2.circle(image_mask,(int(pixel[0]),int(pixel[1])), 5, 255, -1)
   
            thresh = cv2.threshold(image_mask, 30, 255, cv2.THRESH_BINARY)[1]
    
            # _, contours, _ = cv2.findContours(thresh.copy(), cv2.RETR_EXTERNAL,
            #                                   cv2.CHAIN_APPROX_SIMPLE)
            contours, _ = cv2.findContours(thresh.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
            cnt = max(contours, key=cv2.contourArea)
    
            image_mask = np.zeros(img.shape[:2],dtype = np.uint8)
            cv2.drawContours(image_mask, [cnt], -1, 255, -1)

            mask_path = path_mask+"/"+ str(i*LABEL_INTERVAL)+".png"
            cv2.imwrite(mask_path, image_mask)
                
            
            
            file = open(path_label+"/"+ str(i*LABEL_INTERVAL)+".txt","w")
            message = str(classlabel)[:8] + " "
            file.write(message)
            for pixel in corners:
                for digit in pixel:
                    message = str(digit)[:8]  + " "
                    file.write(message)
            message = str((max_x-min_x)/float(camera_intrinsics['width']))[:8]  + " "
            file.write(message) 
            message = str((max_y-min_y)/float(camera_intrinsics['height']))[:8]
            file.write(message)
            file.close()

这一步骤会生成一个名为F.ply的文件，用meshlab打开此文件，另存为mesh并取消勾选binary，保存的文件就是数据集的模型文件。其AABB以原点为圆心，并与OBB的尺寸相同，同时在mask文件夹下会生成图像的掩码，transforms文件夹下会保存新mesh的变换矩阵，labels文件夹内保存标签文件。

同时将打印出的min_xyz和size_xyz复制到models_info.yml文件中。

python3 getmeshscale.py

将物体直径复制到models_info.yml文件中。

models_info.yml中的文件需要扩大1000倍，即改成mm量级。

利用以下代码在根目录创建annotations.csv文件，包含所有图片的物体类别的标签和边界框信息。

python3 get_BBs.py

数据集后处理

为将数据集制作成标准的数据集需要进行后处理。

在LINEMOD/F⽂件夹下依次创建如下python脚本并运行

1.gt_info.py

import os
import numpy as np
import matplotlib.image


f = open("gt.txt", "w")

count = len(os.listdir("./transforms"))
for k in range(count):
    print('正在读取第'+str(k)+"张\n")
    data_load = np.load("transforms" + "/" + str(k) + ".npy")
    cam_r = []
    for i in range(3):
        for j in range(3):
            cam_r.append(data_load[i][j])      #cam_R_m2c

    cam_t = [data_load[0][3] * 1000,data_load[1][3] * 1000,data_load[2][3] * 1000]   #cam_t_m2c由于D455相机的单位是mm所以需要将位置信息扩大1000倍

    im = matplotlib.image.imread('mask/' + str(k) +'.png')
    r = []
    c = []
    ls1 = [0]
    ls2 = [0]

    for i in range(480):
        for j in range(1, 640):
            if im[i][j - 1] == 0 and im[i][j] == 1:
                r.append(i)
                c.append(j)
                break
    for i in range(480):
        for j in range(1, 640):
            if im[i][j - 1] == 1 and im[i][j] == 0:
                ls1[0] = i
                ls2[0] = j
    r.append(ls1[0])
    c.append(ls2[0])
    #需符合densefusion数据的读取
    # rmin = min(r)
    # rmax = max(r)
    # cmin = min(c)
    # cmax = max(c)
    rmin = min(c)
    rmax = min(r)
    cmin = max(c)-min(c)
    cmax = max(r)-min(r)
    r.clear()
    c.clear()
    bb=[]
    bb.append(rmin)
    bb.append(rmax)
    bb.append(cmin)
    bb.append(cmax)
    print(cam_t)
    print(cam_r)
    print(bb)
    f.write("{}:\n".format(k))
    f.write("- cam_R_m2c: [{}, {}, {}, {}, {}, {}, {}, {}, {}]\n".format(cam_r[0],cam_r[1],cam_r[2],cam_r[3],cam_r[4],cam_r[5],cam_r[6],cam_r[7],cam_r[8]))
    f.write("  cam_t_m2c: [{}, {}, {}]\n".format(cam_t[0],cam_t[1],cam_t[2]))
    f.write("  obj_bb: [{}, {}, {}, {}]\n".format(bb[0],bb[1],bb[2],bb[3]))
    f.write("  obj_id: 1\n")
    cam_r.clear()
    bb.clear()
    cam_t.clear()
f.close()

os.rename("./gt.txt","./gt.yml")

运行后得到gt.yml文件

2.rename.py

import os
import cv2


root_rgb = "./JPEGImages/"
root_mask = "./mask/"
root_depth = "./depth/"

ls_rgb = os.listdir(root_rgb)
ls_mask = os.listdir(root_mask)
ls_depth = os.listdir(root_depth)

os.mkdir("rgb")

for file in ls_rgb:
    os.rename(root_rgb + file,"./rgb/" +"0" * int(4 - len(file[:-4])) + file[:-4] + ".jpg")
for file in ls_mask:
    os.rename(root_mask + file,root_mask +"0" * int(4 - len(file[:-4])) + file[:-4] + ".png")
for file in ls_depth:
    os.rename(root_depth + file,root_depth +"0" * int(4 - len(file[:-4])) + file[:-4] + ".png")

os.rmdir(root_rgb)

rgb = "./rgb"
ls__rgb = os.listdir(rgb)
i = 0
for file in ls__rgb:
    print("正在进行图片" + str(i) + "的转码")
    img = cv2.imread("./rgb/" + file)
    cv2.imwrite("./rgb/" + file[:-3] + "png",img)
    os.remove("./rgb/"+ file[:-3] + "jpg")
    i += 1

运行后会将jpg换成png格式，图片名字也会改变

3.info.py

记得将fx,0.,cx,0.,fy,cy等参数按照intrinsics.json换成自己的相机参数，同时将range(0,600)换成自己图片的张数。

import yaml
import os

count = len(os.listdir("./rgb"))
for i in range(0,600):
    list = [fx,0.,cx,0.,fy,cy,0.,0.,1.]
    d={
        i:{
            "cam_K": list,
            "depth_scale": 0.001,
        }
    }
    f=open("info.yml","a",encoding="utf-8")
    yaml.dump(d,f)
    f.close()

4.models_info.yaml

制作方法如制作models_info.yml标题所示。

5.plym2mm

（将ply文件中的坐标单位由m转为mm），使用meshlab打开F.ply文件，删除无效点后保存为F_utf8.ply，勾选normal和color，取消勾选binary encoding。在ObjectDatasetTools-master文件夹下创建以下代码。

# encoding: utf-8

import os
import sys
import glob
import shutil
import cv2

# ply由m单位转为mm单位
def m2mm(object_path):
    ply_name = object_path.split("/")[1] + "_utf8.ply"
    ply_path = os.path.join(object_path, ply_name)
    vertex = 0
    row = 0
    with open(ply_path, "r") as plyfile,open("%s_mm.ply" % ply_path, "w") as plyfile_new:
        for line in plyfile:
            row += 1
            newline = line
            if row == 4:
                vertex = line.split(" ")[2]
            if row > 17 and row <= int(vertex) + 17:
                line_list = line.split(" ")[:10]
                line_list[0] = str(float(line_list[0]) * 1000)
                line_list[1] = str(float(line_list[1]) * 1000)
                line_list[2] = str(float(line_list[2]) * 1000)
                newline = " ".join(line_list) + " \n"
            plyfile_new.write(newline)
    print("Created %s!", ply_path)
            
        


def print_usage():
    print("Usage: rename.py <path>")
    print("path: all or name of the folder")
    print("e.g., rename.py all, rename.py LINEMOD/test")

if __name__ == "__main__":
  
    try:
        if sys.argv[1] == "all":
            folders = glob.glob("LINEMOD/*/")
        elif sys.argv[1]+"/" in glob.glob("LINEMOD/*/"):
            folders = [sys.argv[1]+"/"]
        else:
            print_usage()
            exit()
    except:
        print_usage()
        exit()

    for path in folders:
        # print(path)
        m2mm(path)

然后运行下面的指令，将点云文件的单位由m转换为mm：运行后会生成F_utf8.ply_mm.ply文件。

python3 plym2mm.py all

6.re-format.py

这一步主要是生成训练需要的文件目录结构，第21行os.rename(“./registeredScene.ply”, “./obj_01.ply”)

##记得改成自己那个ply的文件名将registeredScene.ply改成F_utf8.ply_mm.ply

import os
import shutil

os.mkdir("./data")
os.mkdir("./data/01")
os.mkdir("./models")
os.mkdir("./segnet_results")
os.mkdir("./segnet_results/01_label")

msk = os.listdir("./mask")
for file in msk:
    shutil.copy("./mask/" + file,"./segnet_results/01_label/")


shutil.move("./rgb", "./data/01/")
shutil.move("./mask", "./data/01/")
shutil.move("./depth", "./data/01/")
shutil.move("./gt.yml", "./data/01/")
shutil.move("./info.yml", "./data/01/")

os.rename("./registeredScene.ply", "./obj_01.ply")   ##记得改成自己那个ply的文件名
shutil.move("./obj_01.ply", "./models")
shutil.move("./models_info.yml", "./models")
shutil.rmtree("./labels")
shutil.rmtree("./transforms")
'''os.remove("intrinsics.json")
os.remove("registeredScene.ply")
os.remove("transforms.npy")

root = os.listdir("./")
for file in root:
    if file[-2:] == "py":
        if file == "compute_model_info.py":
            continue
        os.remove(file)'''

os.mkdir("./Linemod_preprocessed")
shutil.move("data", "./Linemod_preprocessed/")
shutil.move("models", "./Linemod_preprocessed/")
shutil.move("segnet_results", "./Linemod_preprocessed/")

7.train_test_txt.py

import os

files = len(os.listdir("./Linemod_preprocessed/data/01/depth/"))

_train = open("./Linemod_preprocessed/data/01/train.txt","w")
_test = open("./Linemod_preprocessed/data/01/test.txt","w")
for i in range(files):
    num = (4-len(str(i))) * '0' + str(i)
    if i % 5 == 4:
        _test.write(num + "\n")
    else:
        _train.write(num + "\n")
_train.close()
_test.close()

以上就已经完成数据集制作