BDD100K数据集制作的流程(1)

第一步：json=>csv

利用json_to_csv.ipynb把路径"G:/Data/bdd100k_info/bdd100k/info/100k/val"和"G:/Data/bdd100k_info/bdd100k/info/100k/train"下面的json数据，全部转为csv数据。得到的文件是存储在路径G:\Data\bdd100k_info\bdd100k\info\100k\train_csv\each_train_csv和G:\Data\bdd100k_info\bdd100k\info\100k\val_csv下面。
数据转换的程序如下：

# data_dir = "G:/Data/bdd100k_info/bdd100k/info/100k/val"
data_dir = "G:/Data/bdd100k_info/bdd100k/info/100k/train"
data_path = os.listdir(data_dir)
data_path.sort()

# m = 4501
# json_path = os.path.join(data_dir,data_path[h])#将路径与文件名结合起来就是每个文件的完整路径
# json_content=json.load(open(json_path))
# print(len(json_content.keys()))
# print(data_path[h])

m = 0
h = 0
k = 0
DataResult = pd.DataFrame(columns=['imgid', 'rideid', 'latitude', 'longitude', 'startTime', 'endTime', 'speed'])
for num in range(0,len(data_path)):
    
    if m >20:
        break
    json_path = os.path.join(data_dir,data_path[m])#将路径与文件名结合起来就是每个文件的完整路径

#     json_path = os.path.join(data_dir,json_name)#将路径与文件名结合起来就是每个文件的完整路径
    json_content=json.load(open(json_path))   
    print("The number of variable m is %d" % (m))
    m = m + 1

    # Delete damaged files of json, the number of the key of json_content is 10
    if len(json_content.keys())==10:      
        #     print(json_content['gps'])
        #     print(json.dumps(json_content['gps'], sort_keys=True, indent=4))    
        # 重复从imgid，重复存rideID,一个json里面只有一个起始和结束时间（所以得到的时间都是相同的），因为bdd100k\info\100k\val是40秒视频里面的1帧
        for location in json_content['gps']:
            NewData = pd.DataFrame({'imgid':[data_path[m]],
                                    'rideid':[json_content['rideID']],
                                    'latitude':[location['latitude'] ],
                                    'longitude':[location['longitude'] ],
                                    'startTime': [timeStamp( json_content['startTime'] )],
                                    'endTime': [timeStamp( json_content['endTime'] )],
                                    'speed': [location['speed']]
                                    })    
            DataResult = DataResult.append(NewData, ignore_index=True)   
        
        print("The number of variable h is %d" % (h))
        print("The number of variable k is %d" % (k))
        k = k + 1
        h = h + 1
        if k==10:
#             DataResult.to_csv("G:/Data/bdd100k_info/bdd100k/info/100k/val_csv/%d.csv"%(h), index=None)
            DataResult=[]
            DataResult = pd.DataFrame(columns=['imgid', 'rideid', 'latitude', 'longitude', 'startTime', 'endTime', 'speed'])
            k = 1
            

第二步:csv=>csv

把文件下each_train_csv和each_val_csv下面的数据合并为train_info.csv或val_info.csv文件(用all_csv_to_train_val_info.ipynb)，程序如下：

import json
import numpy as np
import os
from pathlib import Path
import json
import pandas as pd
import time
from datetime import datetime
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D

#filename可以直接从盘符开始，标明每一级的文件夹直到csv文件，header=None表示头部为空
k = 1
DataResult = pd.DataFrame(columns=['imgid', 'rideid', 'latitude', 'longitude', 'startTime', 'endTime', 'speed'])

for info in os.listdir('G:/Data/bdd100k_info/bdd100k/info/100k/train_csv/each_train_csv/'):
#     if k > 5:
#         break
    print(k)
    domain = os.path.abspath(r'G:/Data/bdd100k_info/bdd100k/info/100k/train_csv/each_train_csv/') #获取文件夹的路径
#     domain = os.path.abspath(r'G:/Data/bdd100k_info/bdd100k/info/100k/val_csv/each_val_csv/') #获取文件夹的路径
    info = os.path.join(domain, info) #将路径与文件名结合起来就是每个文件的完整路径  
    NewData = pd.read_csv(info)
    
    DataResult = DataResult.append(NewData, ignore_index=True)
    k = k + 1
print(DataResult.head(2))
# result.to_csv("G:/Data/bdd100k_info/bdd100k/info/100k/train_csv/train_info.csv", index=None)  
# result.to_csv("G:/Data/bdd100k_info/bdd100k/info/100k/train_csv/val_info.csv", index=None)   

第三步：统计每张图片里面类别出现的频次

统计每张图片里面类别出现的频次得到train_class_accumulate.csv和val_class_accumulate.csv(class_accumulate.ipynb)
程序如下：

import json
import numpy as np
import os
from pathlib import Path
import json
import pandas as pd
import time
from datetime import datetime
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D

### 获得每张图片类别数量和对应的GPS数据

def get_train_val_class_number():
    # train_data_dir = "bdd100k_labels_images_train.json"
#     train_data_dir = "bdd100k_labels_images_val.json"

    train_data=json.load(open(train_data_dir)) 
    className = {
        1: 'bus',
        2: 'traffic light',
        3: 'traffic sign',
        4: 'person',
        5: 'bike',
        6: 'truck',
        7: 'motor',
        8: 'car',
        9: 'train',
        10: 'rider',
    }   
    classNameDF = pd.DataFrame({
                            '1':[ className[1] ],
                            '2':[ className[2] ],
                            '3':[ className[3] ],
                            '4':[ className[4] ],
                            '5':[ className[5] ],
                            '6':[ className[6] ],
                            '7':[ className[7] ],
                            '8':[ className[8] ],
                            '9':[ className[9] ],
                            '10':[ className[10] ]
                            })


    # 初始化类别的个数全为0
    image_name = {}
    class_accumulate = np.zeros((len(train_data), classNameDF.shape[1]))
    for i in range(0,len(train_data)):
        each_class_accumulate = np.zeros((1, classNameDF.shape[1]))
        print(i)
    #     if i >10:
    #         break
        each_image_data = train_data[i]
        image_name[i] = each_image_data['name']
        each_image_labels = each_image_data['labels']
        for j in range(0,len(each_image_labels)):
            # 统计每个类别的个数
            each_class = each_image_labels[j]['category']
            each_class_index = classNameDF == each_class
            each_class_counter = np.array(each_class_index)+0
            each_class_accumulate = each_class_counter + each_class_accumulate
    #     print(each_image_data['name'])
    #     print(each_class_accumulate)
        class_accumulate[i] = each_class_accumulate    
    
    class_accumulate_df = pd.DataFrame(class_accumulate)

    image_name_df = pd.DataFrame([image_name]).T
    image_name_df.rename(columns={ image_name_df.columns[0]: "img_id" }, inplace=True)
    # print(image_name_df)
    result = pd.concat([image_name_df, class_accumulate_df], axis=1)
    print(result.head(5))
#     result.to_csv("G:/Data/bdd100k_info/bdd100k/info/100k/train_csv/train_class_accumulate.csv", index=None)    
#     result.to_csv("G:/Data/bdd100k_info/bdd100k/info/100k/val_csv/val_class_accumulate.csv", index=None)    
    
get_train_val_class_number() 把类别统计数据存储为train_class_accumulate.csv和val_class_accumulate.csv

第四步： 筛选出不同区域的图片

#filename可以直接从盘符开始，标明每一级的文件夹直到csv文件，header=None表示头部为空
k = 1
DataResult = pd.DataFrame(columns=['imgid', 'rideid', 'latitude', 'longitude', 'startTime', 'endTime', 'speed'])

for info in os.listdir('G:/Data/bdd100k_info/bdd100k/info/100k/train_csv/each_train_csv/'):
#     if k > 5:
#         break
    print(k)
    domain = os.path.abspath(r'G:/Data/bdd100k_info/bdd100k/info/100k/train_csv/each_train_csv/') #获取文件夹的路径
#     domain = os.path.abspath(r'G:/Data/bdd100k_info/bdd100k/info/100k/val_csv/each_val_csv/') #获取文件夹的路径
    info = os.path.join(domain, info) #将路径与文件名结合起来就是每个文件的完整路径  
    NewData = pd.read_csv(info)
    
    DataResult = DataResult.append(NewData, ignore_index=True)
    k = k + 1
print(DataResult.head(2))

SubDataResult = DataResult # train_info.csv和val_info.csv文件数据也可以

### 筛选出不同区域的图片
state_boundaries_path = "state_boundaries.json"
state_minmax_latlon=json.load(open(state_boundaries_path)) 

new_york_min_lat = state_minmax_latlon['NY']['min_lat']
new_york_max_lat = state_minmax_latlon['NY']['max_lat']
new_york_min_lng = state_minmax_latlon['NY']['min_lng']
new_york_max_lng = state_minmax_latlon['NY']['max_lng']

NewYorkData = SubDataResult[(SubDataResult['latitude'] >= new_york_min_lat) & (SubDataResult['latitude'] <= new_york_max_lat) & (SubDataResult['longitude'] >= new_york_min_lng) & (SubDataResult['longitude'] <= new_york_max_lng)]



print(NewYorkData.head(3))
NewYorkData.to_csv("NewYorkData.csv", index=None)

第五步：把全部数据转换为VOC2007的数据格式

程序的细节解释，可以参考json2xml.py和json_extract.py
5.1: json_extract.py把json提取出来,然后保存为outputval.json(train和val分别制作)

#coding=utf-8
import json
import io

className = {
    1: 'bus',
    2: 'traffic light',
    3: 'traffic sign',
    4: 'person',
    5: 'bike',
    6: 'truck',
    7: 'motor',
    8: 'car',
    9: 'train',
    10: 'rider',
}

classNum = [1, 2, 3, 4, 5, 6, 7, 9, 10]

inputfile = []
inner = {}

# PATH_TO_JSON = 'G:/Data/bdd100k_labels_release/bdd100k/labels/bdd100k_labels_images_train.json'
PATH_TO_JSON = 'G:/Data/bdd100k_labels_release/bdd100k/labels/bdd100k_labels_images_val.json'

with open(PATH_TO_JSON, "r+") as f:
    allData = json.load(f)
    print("read ready")

def writeNum(Num):
#     with open("train_outputval.json", "a+", encoding='utf-8') as f:
    with open("val_outputval.json", "a+", encoding='utf-8') as f:
        f.write(str(Num))

inputfile = []
inner = {}

    k = 1
    for data in allData:
#         print(data.keys())
        print(k)
#         if k >1:
#             break
            
        k = k + 1
        for item in data['labels']:
#             print(item.keys())
            if 'box2d' in item:
                inner = {
                    "filename": str(data["name"]).zfill(6),
                    "name": item['category'],
                    "box2d": item['box2d'],
                    "id": item['id'],
                    "truncated": item['attributes']['truncated'],
                    "occluded": item['attributes']['occluded']
                }
#                 print(item['attributes'])
                inputfile.append(inner)

    inputfile = json.dumps(inputfile,indent=4)
    writeNum(inputfile)# 把数据存为inputfile,调用自定义函数,输出数据为outputval

5.2 json2xml.py把outputval.json转化为.xml数据(train和val分别制作)

### 把json标签转为.xml标签
#https://blog.csdn.net/qq_17278169/article/details/82189776 

# https://github.com/YaoLing13/DL_Tools 程序主要来自于这个网站


# 把json文件转化为xml,便于后面的训练使用，同时把图片复制到对应的文件夹里面 
#coding=utf-8
import xml.etree.ElementTree as ET
import json
import os
from skimage import io, data
import shutil

x1 = []
y1 = []
x2 = []
y2 = []
classname = []
truncate = []
occlude = []
# 读取json里的内容,allData表示整个.json的内容，data表示遍历的当前元素
# with open("train_outputval.json", "r+") as f:
with open("val_outputval.json", "r+") as f:
    allData = json.load(f)
    # data = allData["annotations"]
    print("read ready")


#####################此步骤将每一个图片的全部信息读为img_all列表的一个元素
one_img = []
img_all = []
j = 0
for i in range(1, len(allData)):
#     if j > 1:
#         break            
    if allData[i]['filename']==allData[i-1]['filename']: # train_outputval.json文件,一个label+一个filename,所以是多个label有一个filename
        one_img.append(allData[i-1]) # 最终得到的是一个图片里面存在多个label的坐标信息
    else:
        one_img.append(allData[i-1])#处理图片中最后一个bbox
        j += 1
        img_all.append(one_img)
        one_img = []
# print(one_img[9999]['filename'])
img_all.append(one_img)

# 美化xml文件  prettyXml(annotation, '\t', '\n') # \t 表示空格  \n表示换行
def prettyXml(element, indent, newline, level = 0): # element为传进来的Elment类，参数indent用于缩进，newline用于换行
#     print(element.text)
    if element:  # 判断element是否有子元素
        if element.text == None or element.text.isspace(): # 如果element的text没有内容
            element.text = newline + indent * (level + 1) 
        else:
            element.text = newline + indent * (level + 1) + element.text.strip() + newline + indent * (level + 1)
    #else:  # 此处两行如果把注释去掉，Element的text也会另起一行
        #element.text = newline + indent * (level + 1) + element.text.strip() + newline + indent * level
    temp = list(element) # 将elemnt转成list
#     print(temp)
    for subelement in temp:
        if temp.index(subelement) < (len(temp) - 1): # 如果不是list的最后一个元素，说明下一个行是同级别元素的起始，缩进应一致
            subelement.tail = newline + indent * (level + 1)
        else:  # 如果是list的最后一个元素， 说明下一行是母元素的结束，缩进应该少一个
            subelement.tail = newline + indent * level
        prettyXml(subelement, indent, newline, level = level + 1) # 对子元素进行递归操作


x1 = []
y1 = []
x2 = []
y2 = []
classname = []
truncate = []
occlude = []
# 开始对每一张图片生成xml文件
k = 1
for single in img_all:
    print(k)
#     if k > 1:
#         break
    k += 1
    data = single[0]  # 获取图像名字
    img_name = data['filename']  # 获取图像名字

    # 把原文件里面的图片复制到目标文件夹里面
#     source_file_path = 'G:/Data/bdd100k_images/bdd100k/images/100k/train/'
    source_file_path = 'G:/Data/bdd100k_images/bdd100k/images/100k/val/'
    source_img_path = os.path.join(source_file_path, img_name)  #得到图像绝对路径

#     target_file_path = 'G:/Data/bdd100k_images/bdd100k/images/100k/SubTrain/JPEGImages/'
    target_file_path = 'G:/Data/bdd100k_images/bdd100k/images/100k/SubVal/JPEGImages/'
    target_img_path = os.path.join(target_file_path, img_name)  #得到图像绝对路径
    shutil.copyfile(source_img_path,target_img_path)
    
    #get the coordinates of the bbox, name, truncated, occluded（object的子节点们，需要循环）
#     img = io.imread(source_img_path)
#     h = img.shape[0]
#     w = img.shape[1]
#     d = img.shape[2]
#     print(h)
#     print(w)
#     print(d)

    h = 720
    w = 1280
    d = 3    
    
    for element in single:
        if 'box2d' in element:
            x1.append(element['box2d']['x1'])
            x2.append(element['box2d']['x2'])
            y1.append(element['box2d']['y1'])
            y2.append(element['box2d']['y2'])
            classname.append(element['name'])
            if element['truncated']==True:
                truncate.append('1')
            else:
                truncate.append('0')
            if element['occluded']==True:
                occlude.append('1')
            else:
                occlude.append('0')


    annotation = ET.Element("annotation")
    #创建annotation的子节点folder，并添加数据
    folder = ET.SubElement(annotation,"folder")
    folder.text = 'BDD 100K'
    #创建annotation的子节点filename，并添加属性
    filename = ET.SubElement(annotation,"filename")
    filename.text = img_name
    #创建annotation的子节点source，并添加属性
    source = ET.SubElement(annotation,"source")
    #
    size = ET.SubElement(annotation, "size")
    #
    segmented = ET.SubElement(annotation, "segmented")
    segmented.text = '0'


    #object
    for i in range(0, len(x1)):
        object = ET.SubElement(annotation, "object")
        #object的六个子节点
        name = ET.SubElement(object, 'name')
        name.text = classname[i]
        #
        pose = ET.SubElement(object, 'pose')
        pose.text = "Unspecified"
        #
        truncated = ET.SubElement(object, 'truncated')
        truncated.text = truncate[i]
        #
        occluded = ET.SubElement(object, 'occluded')
        occluded.text = occlude[i]
        #
        bndbox = ET.SubElement(object, 'bndbox')

        #
        difficult = ET.SubElement(object, 'difficult')
        difficult.text = "0"
        #

        #子节点bndbox下的四个子节点
        xmin = ET.SubElement(bndbox, 'xmin')
        xmin.text = str(x1[i])
        #
        ymin = ET.SubElement(bndbox, 'ymin')
        ymin.text = str(y1[i])
        #
        xmax = ET.SubElement(bndbox, 'xmax')
        xmax.text = str(x2[i])
        #
        ymax = ET.SubElement(bndbox, 'ymax')
        ymax.text = str(y2[i])
    #####################################

    #source的三个子节点
    database = ET.SubElement(source,"database")
    database.text = "bdd_100k_database"
    #
    annotation_1 = ET.SubElement(source,"annotation")
    annotation_1.text = "bdd_100k_database"
    #
    image = ET.SubElement(source,"image")
    image.text = "flickr"
    #

    #size的三个子节点
    width = ET.SubElement(size,"width")
    width.text = str(w)
    #
    height = ET.SubElement(size,"height")
    height.text = str(h)
    #
    depth = ET.SubElement(size,"depth")
    depth.text = str(d)
    #



    prettyXml(annotation, '\t', '\n')            #执行美化方法
    tree = ET.ElementTree(annotation) # 全部合并起来
    # data['filename'][:-4]把0000f77c-6257be58.jpg的'.jpg'后四个字符串去掉
#     xml_name = 'G:/Data/bdd100k_images/bdd100k/images/100k/SubTrain/Annotations/'+data['filename'][:-4] + '.xml'
    xml_name = 'G:/Data/bdd100k_images/bdd100k/images/100k/SubVal/Annotations/'+data['filename'][:-4] + '.xml'
#     print(data['filename'][:-4])
    tree.write(xml_name)
    print(data['filename'], 'finished')

    x1 = []
    y1 = []
    x2 = []
    y2 = []
    classname = []
    truncate = []
    occlude = []

第六步：每个车辆ID的数据存储为txt

利用create_data_lists.py把存储的txt转为.json数据(TEST_images.json, TEST_objects.json, TRAIN_images.json和TRAIN_objects.json)

TRAIN_images.json存储的是图片的路径;TRAIN_objects.json存储的是每个框的坐标和对应的标签