2020-09-12

xml数据集转json数据集记录

针对不同深度学习网络需要使用不同的数据集格式，现记录将voc数据集转化为coco数据集
虽然github上有相当多的开源代码，但是本人voc数据集格式存在问题，无法直接转换成功。
参考github地址voc2coco

voc与coco的数据格式

voc的格式为
voc
–Annotations（xml文件）
–ImageSets/Main(图片目录)/test.txt; train.txt; val.txt; train.txt
–JPEGImages(图片数据)
–labels（txt坐标文件）
coco
–Annotations（json文件）
–train（训练集图片）
–test（训练集图片）
…

由于voc数据集中的数据集图片是放在一个文件夹中，因此需要根据ImageSets/Main(图片目录)/test.txt; train.txt; val.txt; train.txt将图片集划分，代码如下：
感谢https://blog.csdn.net/Jacob_sea/article/details/107522775?utm_medium=distribute.pc_relevant.none-task-blog-BlogCommendFromMachineLearnPai2-2.nonecase&depth_1-utm_source=distribute.pc_relevant.none-task-blog-BlogCommendFromMachineLearnPai2-2.nonecase

import os
import shutil

f = open("/home/b307/MobileNetV3-SSD/VOCdevkit/VOC2007/ImageSets/Main/addxml/trainval.txt")             #打开写好文件名的文本文档，每一行是一个文件名
lines = f.readlines()

frompath = "/home/b307/MobileNetV3-SSD/VOCdevkit/VOC2007/JPEGImages/"            #需要复制的图片文件夹
topath ="/home/b307/MobileNetV3-SSD/VOCdevkit/VOC2007/cocodata/trainval/"                #目标文件夹
for line in lines:
    file_path = frompath + line.split("\n")[0] + ".jpg"
    #print(file_path)
    shutil.copy(file_path, topath)
f.close

分好之后需要将xml转换json，voc2coco.py
代码如下：

# pip install lxml

import sys
import os
import json
import xml.etree.ElementTree as ET


START_BOUNDING_BOX_ID = 1
PRE_DEFINE_CATEGORIES = {}
# If necessary, pre-define category and its id
#  PRE_DEFINE_CATEGORIES = {"aeroplane": 1, "bicycle": 2, "bird": 3, "boat": 4,
                         #  "bottle":5, "bus": 6, "car": 7, "cat": 8, "chair": 9,
                         #  "cow": 10, "diningtable": 11, "dog": 12, "horse": 13,
                         #  "motorbike": 14, "person": 15, "pottedplant": 16,
                         #  "sheep": 17, "sofa": 18, "train": 19, "tvmonitor": 20}


def get(root, name):
    vars = root.findall(name)
    return vars


def get_and_check(root, name, length):
    vars = root.findall(name)
    if len(vars) == 0:
        raise NotImplementedError('Can not find %s in %s.'%(name, root.tag))
    if length > 0 and len(vars) != length:
        raise NotImplementedError('The size of %s is supposed to be %d, but is %d.'%(name, length, len(vars)))
    if length == 1:
        vars = vars[0]
    return vars


def get_filename_as_int(filename):
    try:
        filename = os.path.splitext(filename)[0]
        return str(filename)
    except:
        raise NotImplementedError('Filename %s is supposed to be an integer.'%(filename))


def convert(xml_list, xml_dir, json_file):
    list_fp = open(xml_list, 'r')
    json_dict = {"images":[], "type": "instances", "annotations": [],
                 "categories": []}
    categories = PRE_DEFINE_CATEGORIES
    bnd_id = START_BOUNDING_BOX_ID
    for line in list_fp:
        line = line.strip()
        print("Processing %s"%(line))
        xml_f = os.path.join(xml_dir, line)
        tree = ET.parse(xml_f)
        root = tree.getroot()
        path = get(root, 'path')
        if len(path) == 1:
            filename = os.path.basename(path[0].text)
        elif len(path) == 0:
            filename = get_and_check(root, 'filename', 1).text
        else:
            raise NotImplementedError('%d paths found in %s'%(len(path), line))
        ## The filename must be a number
        image_id = get_filename_as_int(filename)
        size = get_and_check(root, 'size', 1)
        width = int(get_and_check(size, 'width', 1).text)
        height = int(get_and_check(size, 'height', 1).text)
        image = {'file_name': filename, 'height': height, 'width': width,
                 'id':image_id}
        json_dict['images'].append(image)
        ## Cruuently we do not support segmentation
        #  segmented = get_and_check(root, 'segmented', 1).text
        #  assert segmented == '0'
        for obj in get(root, 'object'):
            category = get_and_check(obj, 'name', 1).text
            if category not in categories:
                new_id = len(categories)
                categories[category] = new_id
            category_id = categories[category]
            bndbox = get_and_check(obj, 'bndbox', 1)
            xmin = int(get_and_check(bndbox, 'xmin', 1).text) - 1
            ymin = int(get_and_check(bndbox, 'ymin', 1).text) - 1
            xmax = int(get_and_check(bndbox, 'xmax', 1).text)
            ymax = int(get_and_check(bndbox, 'ymax', 1).text)
            assert(xmax > xmin)
            assert(ymax > ymin)
            o_width = abs(xmax - xmin)
            o_height = abs(ymax - ymin)
            ann = {'area': o_width*o_height, 'iscrowd': 0, 'image_id':
                   image_id, 'bbox':[xmin, ymin, o_width, o_height],
                   'category_id': category_id, 'id': bnd_id, 'ignore': 0,
                   'segmentation': []}
            json_dict['annotations'].append(ann)
            bnd_id = bnd_id + 1

    for cate, cid in categories.items():
        cat = {'supercategory': 'none', 'id': cid, 'name': cate}
        json_dict['categories'].append(cat)
    json_fp = open(json_file + '.jpg', 'w')
    json_str = json.dumps(json_dict)
    json_fp.write(json_str)
    json_fp.close()
    list_fp.close()


if __name__ == '__main__':
    if len(sys.argv) <= 1:
        print('3 auguments are need.')
        print('Usage: %s XML_LIST.txt XML_DIR OUTPU_JSON.json'%(sys.argv[0]))
        exit(1)

    convert(sys.argv[1], sys.argv[2], sys.argv[3])

调用格式为

python voc2coco.py path/test.txt path/Annotations path/test.json

但是直接运行会出现错误，因为该代码根据txt文件来查询xml文件，但本人txt文件中数据不带.xml后缀，因此需要添加
打开终端cd 到txt文件夹中，代码如下：

sed -i 's/$/.xml/g' *.txt 

重新运行voc2coco.py, 生成json文件。

PS：这玩意儿训练报错，不知道是不是数据集的问题，后续检查。