制作自己的ILSVRC2015 VID数据集的一些脚本以及流程

前言：刚开始做视频目标检测的时候需要把我们实验室的医疗数据制作成ILSVRC2015 VID这样格式的数据，但是在网上没怎么找到攻略，故把自己的一些经验记录下来。

1、json转xml：由labelme这个软件标注出来的数据集一般都是json格式的，需要把json标注格式转化成xml格式。

import os
 
from typing import List, Any
 
 
 
import numpy as np
 
import codecs
 
import json
 
from glob import glob
 
import cv2
 
import shutil
 
from sklearn.model_selection import train_test_split
 
# 1.标签路径
 
path = "/media/work/data/lx/mega_data/tempdatas/train"              #原始labelme标注数据路径
saved_path = "/media/work/data/lx/mega_data/val_train/"                #保存路径
 
 
 
    # 3.获取待处理文件

for tt in os.listdir(path):
    labelme_path='/media/work/data/lx/mega_data/tempdatas/train'+'/'+tt+'/'
    cont=os.listdir(labelme_path)
    if len(cont)<20:
        continue
    print(labelme_path)
    files = glob(labelme_path + "*.json")
    #print(files)
    files = [i.replace("\\","/").split("/")[-1].split(".json")[0] for i in files]
    
    print(files)
    
    # 4.读取标注信息并写入 xml
    
    for json_file_ in files:
    
        json_filename = labelme_path + json_file_ + ".json"
    
        json_file = json.load(open(json_filename, "r", encoding="utf-8"))
    
        height, width, channels = cv2.imread(labelme_path + json_file_ + ".JPEG").shape

        temp=saved_path+ "/"+ "Annotations/"+"VID/"+"train/"+tt
        if not os.path.exists(temp):
           os.makedirs(temp)
    
        with codecs.open(temp+"/"+ json_file_ + ".xml", "w", "utf-8") as xml:
    
            xml.write('<annotation>\n')
    
            xml.write('\t<folder>' + 'WH_data' + '</folder>\n')
    
            xml.write('\t<filename>' + json_file_ + ".JPEG" + '</filename>\n')
    
            xml.write('\t<source>\n')
    
            xml.write('\t\t<database>WH Data</database>\n')
    
            xml.write('\t\t<annotation>WH</annotation>\n')
    
            xml.write('\t\t<image>flickr</image>\n')
    
            xml.write('\t\t<flickrid>NULL</flickrid>\n')
    
            xml.write('\t</source>\n')
    
            xml.write('\t<owner>\n')
    
            xml.write('\t\t<flickrid>NULL</flickrid>\n')
    
            xml.write('\t\t<name>WH</name>\n')
    
            xml.write('\t</owner>\n')
    
            xml.write('\t<size>\n')
    
            xml.write('\t\t<width>' + str(width) + '</width>\n')
    
            xml.write('\t\t<height>' + str(height) + '</height>\n')
    
            xml.write('\t\t<depth>' + str(channels) + '</depth>\n')
    
            xml.write('\t</size>\n')
    
            xml.write('\t\t<segmented>0</segmented>\n')
    
            for multi in json_file["shapes"]:
    
                points = np.array(multi["points"])
    
                labelName=multi["label"]
    
                xmin = min(points[:, 0])
    
                xmax = max(points[:, 0])
    
                ymin = min(points[:, 1])
    
                ymax = max(points[:, 1])
    
                label = multi["label"]
    
                if xmax <= xmin:
    
                    pass
    
                elif ymax <= ymin:
    
                    pass
    
                else:
    
                    xml.write('\t<object>\n')
    
                    xml.write('\t\t<name>' + labelName.lower()+ '</name>\n')
    
                    xml.write('\t\t<pose>Unspecified</pose>\n')
    
                    xml.write('\t\t<truncated>1</truncated>\n')
    
                    xml.write('\t\t<difficult>0</difficult>\n')
    
                    xml.write('\t\t<bndbox>\n')
    
                    xml.write('\t\t\t<xmin>' + str(int(xmin)) + '</xmin>\n')
    
                    xml.write('\t\t\t<ymin>' + str(int(ymin)) + '</ymin>\n')
    
                    xml.write('\t\t\t<xmax>' + str(int(xmax)) + '</xmax>\n')
    
                    xml.write('\t\t\t<ymax>' + str(int(ymax)) + '</ymax>\n')
    
                    xml.write('\t\t</bndbox>\n')
    
                    xml.write('\t</object>\n')
    
                    print(json_filename, xmin, ymin, xmax, ymax, label)
    
            xml.write('</annotation>')
    
    # 5.复制图片到 VOC2007/JPEGImages/下
    
    image_files = glob(labelme_path + "*.JPEG")
    
    print("copy image files to VOC007/JPEGImages/")

    temp2=saved_path+"/Data/VID/train/"+tt
    if not os.path.exists(temp2):
           os.makedirs(temp2)
    
    for image in image_files:
    
        shutil.copy(image, temp2)
    
    # 6.split files for txt
    
    txtsavepath = saved_path + "ImageSets/Main/"
    
    ftrainval = open(txtsavepath + '/trainval.txt', 'w')
    
    ftest = open(txtsavepath + '/test.txt', 'w')
    
    ftrain = open(txtsavepath + '/train.txt', 'w')
    
    fval = open(txtsavepath + '/val.txt', 'w')
    
    total_files = glob("./VOC2007/Annotations/*.xml")
    
    total_files = [i.replace("\\","/").split("/")[-1].split(".xml")[0] for i in total_files]
    
    trainval_files=[]
    
    test_files=[]
    
    if isUseTest:
    
        trainval_files, test_files = train_test_split(total_files, test_size=0.15, random_state=55)
    
    else:
    
        trainval_files=total_files
    
    for file in trainval_files:
    
        ftrainval.write(file+"\n")
    
    # split
    
    train_files, val_files = train_test_split(trainval_files, test_size=0.15, random_state=55)
    
    # train
    
    for file in train_files:
    
        ftrain.write(file + "\n")
    
    # val
    
    for file in val_files:
    
        fval.write(file +"\n")
    
    for file in test_files:
    
        print(file)
    
        ftest.write(file + "\n")
    
    ftrainval.close()
    
    ftrain.close()
    
    fval.close()
    
    ftest.close()

2、当组织好了Annotations和Data的每一个视频一一对应之后，对每一个视频帧文件夹进行重命名排序（我跑的那个模型可能需要，其他的可能不太清楚）

# -*- coding:utf8 -*-
 
import os
 
class BatchRename():
    '''
    批量重命名文件夹中的图片文件
    '''
    def __init__(self):
        self.path = '/home/liutianen/datasets/ILSVRC2015/Data/VID/train'
        
   
           
            
 
    def rename(self):
        filelist = os.listdir(self.path)
        total_num = len(filelist)
        for tt in filelist:
            i = 0
            print(tt)
            ttt=os.listdir('/home/liutianen/datasets/ILSVRC2015/Data/VID/train'+'/'+tt)
            ttt.sort()
            #ttt.sort(key=lambda x:str(x.split('.')[1]))
            print(ttt)
            for item in ttt:
                print(item)
                if item.endswith('.jpg'):
                    temp=' '
                    print(1111111111111)
                    if i<10:
                        temp='00000'+str(i)
                    if i<100 and i>=10:
                        temp='0000'+str(i)
                    if i<1000 and i>=100:
                        temp='000'+str(i)
                    if i>=1000 and i<10000:
                        temp='00'+str(i)
                    if i>=10000:
                        temp=str(i)
                    src = os.path.join(os.path.abspath(self.path), tt+'/'+item)
                    dst = os.path.join(os.path.abspath(self.path), tt+'/'+temp+ '.JPEG')
                    temp=' '
                    print(src)
                    print(dst)
                    
                    try:
                        os.rename(src, dst)
                        i=i+1
                        print('converting %s to %s ...' % (src, dst))
                    except:
                        continue

        print ('total %d to rename & converted %d jpgs' % (total_num, i))
 
if __name__ == '__main__':
    demo = BatchRename()
    demo.rename()

3、制作VID_val_frames.txt等ImageSets文档，大体上每个txt制作差不多，一些细节上的内容可能要更改。

import os
 
from typing import List, Any
 
 
 
import numpy as np
 
import codecs
 
import json
 
from glob import glob
 
import cv2
 
import shutil
 
 
# 1.标签路径
 
path = "/home/liutianen/datasets/ILSVRC2015/Data/VID/val"              #原始labelme标注数据路径
saved_path = "/home/liutianen/datasets/ILSVRC2015"                #保存路径
 
 
 
    # 3.获取待处理文件
file = open('/home/liutianen/datasets/ILSVRC2015/ImageSets/VID_val_frames.txt','w')
dd=os.listdir(path)
dd.sort()
j=1
for tt in dd:
    labelme_path='/home/liutianen/datasets/ILSVRC2015/Data/VID/val'+'/'+tt+'/'
    print(labelme_path)    

    
    image_files = glob(labelme_path + "*.jpg")
    

    temp2=saved_path+"/" +"Data/VID/val/"+tt
    lenth=len(os.listdir(temp2))
    xx="val/"+tt
    ww=os.listdir(temp2)
    ww.sort()
    for aa in ww:
       temp3=xx+"/"+ aa.split('.')[0]+" "+str(j)+"\n"
       file.write(temp3)
       j=j+1