数据集处理代码合集
1.coco转voc格式
from pycocotools.coco import COCO
import os
import shutil
from tqdm import tqdm
import skimage.io as io
import matplotlib.pyplot as plt
import cv2
from PIL import Image, ImageDraw
# the path you want to save your results for coco to voc
savepath = "cocodata/xml/"
img_dir = savepath + "images/"
anno_dir = savepath + "Annotations/"
datasets_list = ["train2017", "val2017", "test2017"]
classes_names = [
"person",
"bicycle",
"car",
"motorcycle",
"airplane",
"bus",
"train",
"truck",
"boat",
"traffic light",
"fire hydrant",
"stop sign",
"parking meter",
"bench",
"bird",
"cat",
"dog",
"horse",
"sheep",
"cow",
"elephant",
"bear",
"zebra",
"giraffe",
"backpack",
"umbrella",
"handbag",
"tie",
"suitcase",
"frisbee",
"skis",
"snowboard",
"sports ball",
"kite",
"baseball bat",
"baseball glove",
"skateboard",
"surfboard",
"tennis racket",
"bottle",
"wine glass",
"cup",
"fork",
"knife",
"spoon",
"bowl",
"banana",
"apple",
"sandwich",
"orange",
"broccoli",
"carrot",
"hot dog",
"pizza",
"donut",
"cake",
"chair",
"couch",
"potted plant",
"bed",
"dining table",
"toilet",
"tv",
"laptop",
"mouse",
"remote",
"keyboard",
"cell phone",
"microwave",
"oven",
"toaster",
"sink",
"refrigerator",
"book",
"clock",
"vase",
"scissors",
"teddy bear",
"hair drier",
"toothbrush",
]
# Store annotations and train2014/val2014/... in this folder
dataDir = "E:\code\object_detection\yolov3-other-master\YOLOV3-master\cocodata/"
headstr = """\
<annotation>
<folder>VOC</folder>
<filename>%s</filename>
<source>
<database>My Database</database>
<annotation>COCO</annotation>
<image>flickr</image>
<flickrid>NULL</flickrid>
</source>
<owner>
<flickrid>NULL</flickrid>
<name>company</name>
</owner>
<size>
<width>%d</width>
<height>%d</height>
<depth>%d</depth>
</size>
<segmented>0</segmented>
"""
objstr = """\
<object>
<name>%s</name>
<pose>Unspecified</pose>
<truncated>0</truncated>
<difficult>0</difficult>
<bndbox>
<xmin>%d</xmin>
<ymin>%d</ymin>
<xmax>%d</xmax>
<ymax>%d</ymax>
</bndbox>
</object>
"""
tailstr = """\
</annotation>
"""
# if the dir is not exists,make it,else delete it
def mkr(path):
if os.path.exists(path):
shutil.rmtree(path)
os.mkdir(path)
else:
os.mkdir(path)
mkr(img_dir)
mkr(anno_dir)
def id2name(coco):
classes = dict()
for cls in coco.dataset["categories"]:
classes[cls["id"]] = cls["name"]
return classes
def write_xml(anno_path, head, objs, tail):
f = open(anno_path, "w")
f.write(head)
for obj in objs:
f.write(objstr % (obj[0], obj[1], obj[2], obj[3], obj[4]))
f.write(tail)
def save_annotations_and_imgs(coco, dataset, filename, objs):
# eg:COCO_train2014_000000196610.jpg-->COCO_train2014_000000196610.xml
anno_path = anno_dir + filename[:-3] + "xml"
img_path = dataDir + dataset + "/" + filename
# print(img_path)
dst_imgpath = img_dir + filename
img = cv2.imread(img_path)
if img.shape[2] == 1:
# print(filename + " not a RGB image")
return
# shutil.copy(img_path, dst_imgpath)
head = headstr % (filename, img.shape[1], img.shape[0], img.shape[2])
tail = tailstr
write_xml(anno_path, head, objs, tail)
def showimg(coco, dataset, img, classes, cls_id, show=True):
global dataDir
I = Image.open("%s/%s/%s" % (dataDir, dataset, img["file_name"]))
# 通过id，得到注释的信息
annIds = coco.getAnnIds(imgIds=img["id"], catIds=cls_id, iscrowd=None)
# print(annIds)
anns = coco.loadAnns(annIds)
# print(anns)
# coco.showAnns(anns)
objs = []
for ann in anns:
class_name = classes[ann["category_id"]]
if class_name in classes_names:
# print(class_name)
if "bbox" in ann:
bbox = ann["bbox"]
xmin = int(bbox[0])
ymin = int(bbox[1])
if xmin <= 0:
xmin += 1
if ymin <= 0:
ymin += 1
xmax = int(bbox[2] + bbox[0])
ymax = int(bbox[3] + bbox[1])
obj = [class_name, xmin, ymin, xmax, ymax]
objs.append(obj)
draw = ImageDraw.Draw(I)
draw.rectangle([xmin, ymin, xmax, ymax])
if show:
plt.figure()
plt.axis("off")
plt.imshow(I)
plt.show()
return objs
if __name__ == "__main__":
i = 0
for dataset in datasets_list:
a = datasets_list
# ./COCO/annotations/instances_train2014.json
annFile = "{}/annotations/instances_{}.json".format(dataDir, dataset)
# COCO API for initializing annotated data
coco = COCO(annFile)
"""
COCO finished:
loading annotations into memory...
Done (t=0.81s)
creating index...
index created!
end
"""
# show all classes in coco
classes = id2name(coco)
# print(classes)
# [1, 2, 3, 4, 6, 8]
classes_ids = coco.getCatIds(catNms=classes_names)
# print(classes_ids)
for cls in classes_names:
i += 1
print(i)
b = classes_names
# Get ID number of this class
cls_id = coco.getCatIds(catNms=[cls])
img_ids = coco.getImgIds(catIds=cls_id)
# print(cls, len(img_ids))
# imgIds=img_ids[0:10]
for imgId in tqdm(img_ids):
c = img_ids
img = coco.loadImgs(imgId)[0]
filename = img["file_name"]
# print(filename)
objs = showimg(
coco, dataset, img, classes, classes_ids, show=False
)
# print(objs)
save_annotations_and_imgs(coco, dataset, filename, objs)
2.xml文件转Main文件中的txt文件
# -*- coding: utf-8 -*-
# @Author : argus
# @File : make_train_val_test_set.py
# @Software: PyCharm
import os
import random
def _main():
xmlfilepath = r"E:\dataset\Marine_Litter\dataset\material_version\append\annotations/"
total_xml = os.listdir(xmlfilepath)
num = len(total_xml)
list = range(num)
ftrainval = open(
r"E:\dataset\Marine_Litter\dataset\material_version\append/test.txt",
"w",
)
for i in list:
name = total_xml[i][:-4] + "\n"
ftrainval.write(name)
ftrainval.close()
if __name__ == "__main__":
_main()
3.yolo_to_txt
import xml.etree.ElementTree as ET
from os import getcwd
sets=[('2007', 'train'), ('2007', 'val'), ('2007', 'test')]
wd = getcwd()
#classes = ["aeroplane", "bicycle", "bird", "boat", "bottle", "bus", "car", "cat", "chair", "cow", "diningtable", "dog", "horse", "motorbike", "person", "pottedplant", "sheep", "sofa", "train", "tvmonitor"]
classes =['bio','rov','plastic','timestamp','trash_metal']
def convert_annotation(year, image_id, list_file):
in_file = open(r'E:\dataset\VOC2007\Annotations/%s.xml'%(image_id))
tree=ET.parse(in_file)
root = tree.getroot()
# list_file.write('%s/VOC%s/JPEGImages/%s.jpg'%(wd, year, image_id))
list_file.write(r'E:\dataset\VOC2007\JPEGImages/%s.jpg'%(image_id))
for obj in root.iter('object'):
difficult = obj.find('difficult').text
cls = obj.find('name').text
if cls not in classes or int(difficult)==1:
continue
cls_id = classes.index(cls)
xmlbox = obj.find('bndbox')
b = (int(xmlbox.find('xmin').text), int(xmlbox.find('ymin').text), int(xmlbox.find('xmax').text), int(xmlbox.find('ymax').text))
list_file.write(" " + ",".join([str(a) for a in b]) + ',' + str(cls_id))
list_file.write('\n')
for year, image_set in sets:
image_ids = open(r'E:\dataset\VOC2007/ImageSets/Main/%s.txt'%(image_set)).read().strip().split()
list_file = open('%s_%s.txt'%(year, image_set), 'w')
for image_id in image_ids:
convert_annotation(year, image_id, list_file)
list_file.close()
生成三个文件-2007_train.txt、2007_test.txt、2007_val.txt，内容如下
4.数据集比例划分：
# coding:utf-8
import os
import random
import argparse
parser = argparse.ArgumentParser()
#xml文件的地址，根据自己的数据进行修改 xml一般存放在Annotations下
parser.add_argument('--xml_path', default=r'D:\new\22\yolov5-master\data\VOC2007\Annotations', type=str, help='input xml label path')
#数据集的划分，地址选择自己数据下的ImageSets/Main
parser.add_argument('--txt_path', default=r'D:\new\22\yolov5-master\data\VOC2007\ImageSets\Main', type=str, help='output txt label path')
opt = parser.parse_args()
trainval_percent = 1.0
train_percent = 0.9
xmlfilepath = opt.xml_path
txtsavepath = opt.txt_path
total_xml = os.listdir(xmlfilepath)
if not os.path.exists(txtsavepath):
os.makedirs(txtsavepath)
num = len(total_xml)
list_index = range(num)
tv = int(num * trainval_percent)
tr = int(tv * train_percent)
trainval = random.sample(list_index, tv)
train = random.sample(trainval, tr)
file_trainval = open(txtsavepath + '/trainval.txt', 'w')
file_test = open(txtsavepath + '/test.txt', 'w')
file_train = open(txtsavepath + '/train.txt', 'w')
file_val = open(txtsavepath + '/val.txt', 'w')
for i in list_index:
name = total_xml[i][:-4] + '\n'
if i in trainval:
file_trainval.write(name)
if i in train:
file_train.write(name)
else:
file_val.write(name)
else:
file_test.write(name)
file_trainval.close()
file_train.close()
file_val.close()
file_test.close()
5.生成每个类别的label txt文件
# -*- coding: utf-8 -*-
import xml.etree.ElementTree as ET
import os
from os import getcwd
sets = ['train', 'val', 'test']
# # 'bio','rov','plastic','timestamp', 'unknown', 'wood',
# 'rubber', 'cloth', 'fishing', 'paper')
classes = [ 'trash_etc', 'animal_eel', 'trash_fabric', 'rov', 'animal_starfish', 'trash_metal', 'trash_plastic', 'animal_fish',
'trash_wood', 'plant', 'trash_fishing_gear', 'trash_paper', 'trash_rubber', 'animal_crab', 'animal_etc'] # 改成自己的类别
# abs_path = os.getcwd()
# print(abs_path)
abs_path ='E:\dataset\Marine_Litter\dataset\material_version'
def convert(size, box):
dw = 1. / (size[0])
dh = 1. / (size[1])
x = (box[0] + box[1]) / 2.0 - 1
y = (box[2] + box[3]) / 2.0 - 1
w = box[1] - box[0]
h = box[3] - box[2]
x = x * dw
w = w * dw
y = y * dh
h = h * dh
return x, y, w, h
def convert_annotation(image_id):
# in_file = open('/home/trainingai/zyang/yolov5/paper_data/Annotations/%s.xml' % (image_id), encoding='UTF-8')
# out_file = open('/home/trainingai/zyang/yolov5/paper_data/labels/%s.txt' % (image_id), 'w')
in_file = open(r'E:\dataset\Marine_Litter\dataset\material_version/Annotations/%s.xml' % (image_id), encoding='UTF-8')
out_file = open(r'E:\dataset\Marine_Litter\dataset\material_version/labels/%s.txt' % (image_id), 'w')
tree = ET.parse(in_file)
root = tree.getroot()
size = root.find('size')
w = int(size.find('width').text)
h = int(size.find('height').text)
for obj in root.iter('object'):
# difficult = obj.find('difficult').text
# difficult = obj.find('difficult').text
cls = obj.find('name').text
# if cls not in classes or int(difficult) == 1:
# continue
cls_id = classes.index(cls)
xmlbox = obj.find('bndbox')
b = (float(xmlbox.find('xmin').text), float(xmlbox.find('xmax').text), float(xmlbox.find('ymin').text),
float(xmlbox.find('ymax').text))
b1, b2, b3, b4 = b
# 标注越界修正
if b2 > w:
b2 = w
if b4 > h:
b4 = h
b = (b1, b2, b3, b4)
bb = convert((w, h), b)
out_file.write(str(cls_id) + " " + " ".join([str(a) for a in bb]) + '\n')
wd = getcwd()
for image_set in sets:
if not os.path.exists(r'E:\dataset\Marine_Litter\dataset\material_version/labels/'):
os.makedirs(r'E:\dataset\Marine_Litter\dataset\material_version/labels/')
image_ids = open(r'E:\dataset\Marine_Litter\dataset\material_version/ImageSets/Main/%s.txt' % (image_set)).read().strip().split()
list_file = open('E:\dataset\Marine_Litter\dataset\material_version/%s.txt' % (image_set), 'w')
for image_id in image_ids:
list_file.write(abs_path + '/images/%s.jpg\n' % (image_id))
convert_annotation(image_id)
list_file.close()
6.按train、val、test划分label txt文件
# -*- coding: utf-8 -*-
import xml.etree.ElementTree as ET
import os
from os import getcwd
sets = ['val']
# # 'bio','rov','plastic','timestamp', 'unknown', 'wood',
# 'rubber', 'cloth', 'fishing', 'paper')
classes = [ 'carboard','glass','plastic'] # 改成自己的类别
abs_path = os.getcwd()
print(abs_path)
def convert(size, box):
dw = 1. / (size[0])
dh = 1. / (size[1])
x = (box[0] + box[1]) / 2.0 - 1
y = (box[2] + box[3]) / 2.0 - 1
w = box[1] - box[0]
h = box[3] - box[2]
x = x * dw
w = w * dw
y = y * dh
h = h * dh
return x, y, w, h
def convert_annotation(image_id):
# in_file = open('/home/trainingai/zyang/yolov5/paper_data/Annotations/%s.xml' % (image_id), encoding='UTF-8')
# out_file = open('/home/trainingai/zyang/yolov5/paper_data/labels/%s.txt' % (image_id), 'w')
in_file = open(r'D:/new/22/mobile_yolo/data/datasets/VOC2007/Annotations/%s.xml' % (image_id), encoding='UTF-8')
out_file = open(r'D:/new/22/mobile_yolo/data/datasets/VOC2007_dataset/labels/%s.txt' % (image_id), 'w')
tree = ET.parse(in_file)
root = tree.getroot()
size = root.find('size')
w = int(size.find('width').text)
h = int(size.find('height').text)
for obj in root.iter('object'):
# difficult = obj.find('difficult').text
# difficult = obj.find('difficult').text
cls = obj.find('name').text
# if cls not in classes or int(difficult) == 1:
# continue
cls_id = classes.index(cls)
xmlbox = obj.find('bndbox')
b = (float(xmlbox.find('xmin').text), float(xmlbox.find('xmax').text), float(xmlbox.find('ymin').text),
float(xmlbox.find('ymax').text))
b1, b2, b3, b4 = b
# 标注越界修正
if b2 > w:
b2 = w
if b4 > h:
b4 = h
b = (b1, b2, b3, b4)
bb = convert((w, h), b)
out_file.write(str(cls_id) + " " + " ".join([str(a) for a in bb]) + '\n')
wd = getcwd()
for image_set in sets:
if not os.path.exists(r'D:/new/22/mobile_yolo/data/datasets/VOC2007_dataset/labels/'):
os.makedirs(r'D:/new/22/mobile_yolo/data/datasets/VOC2007_dataset/labels/')
image_ids = open(r'D:/new/22/mobile_yolo/data/datasets/VOC2007/ImageSets/Main/%s.txt' % (image_set)).read().strip().split()
list_file = open(r'D:/new/22/mobile_yolo/data/2007_%s.txt' % (image_set), 'w')
for image_id in image_ids:
list_file.write(abs_path + r'JPEGImages/%s.jpg\n' % (image_id))
convert_annotation(image_id)
list_file.close()
7.xml转coco的json
import xml.etree.ElementTree as ET
import os
import json
coco = dict()
coco['images'] = []
coco['type'] = 'instances'
coco['annotations'] = []
coco['categories'] = []
category_set = dict()
image_set = set()
category_item_id = -1
image_id = 20180000000
annotation_id = 0
def addCatItem(name):
global category_item_id
category_item = dict()
category_item['supercategory'] = 'none'
category_item_id += 1
category_item['id'] = category_item_id
category_item['name'] = name
coco['categories'].append(category_item)
category_set[name] = category_item_id
return category_item_id
def addImgItem(file_name, size):
global image_id
if file_name is None:
raise Exception('Could not find filename tag in xml file.')
if size['width'] is None:
raise Exception('Could not find width tag in xml file.')
if size['height'] is None:
raise Exception('Could not find height tag in xml file.')
image_id += 1
image_item = dict()
image_item['id'] = image_id
image_item['file_name'] = file_name
image_item['width'] = size['width']
image_item['height'] = size['height']
coco['images'].append(image_item)
image_set.add(file_name)
return image_id
def addAnnoItem(object_name, image_id, category_id, bbox):
global annotation_id
annotation_item = dict()
annotation_item['segmentation'] = []
seg = []
# bbox[] is x,y,w,h
# left_top
seg.append(bbox[0])
seg.append(bbox[1])
# left_bottom
seg.append(bbox[0])
seg.append(bbox[1] + bbox[3])
# right_bottom
seg.append(bbox[0] + bbox[2])
seg.append(bbox[1] + bbox[3])
# right_top
seg.append(bbox[0] + bbox[2])
seg.append(bbox[1])
annotation_item['segmentation'].append(seg)
annotation_item['area'] = bbox[2] * bbox[3]
annotation_item['iscrowd'] = 0
annotation_item['ignore'] = 0
annotation_item['image_id'] = image_id
annotation_item['bbox'] = bbox
annotation_item['category_id'] = category_id
annotation_id += 1
annotation_item['id'] = annotation_id
coco['annotations'].append(annotation_item)
def _read_image_ids(image_sets_file):
ids = []
with open(image_sets_file) as f:
for line in f:
ids.append(line.rstrip())
return ids
"""通过txt文件生成"""
# split ='train' 'va' 'trainval' 'test'
def parseXmlFiles_by_txt(data_dir, json_save_path, split='val'):
print("hello")
labelfile = split + ".txt"
image_sets_file = data_dir + "/ImageSets/Main/" + labelfile
ids = _read_image_ids(image_sets_file)
for _id in ids:
xml_file = data_dir + f"/Annotations/{_id}.xml"
bndbox = dict()
size = dict()
current_image_id = None
current_category_id = None
file_name = None
size['width'] = None
size['height'] = None
size['depth'] = None
tree = ET.parse(xml_file)
root = tree.getroot()
if root.tag != 'annotation':
raise Exception('pascal voc xml root element should be annotation, rather than {}'.format(root.tag))
# elem is <folder>, <filename>, <size>, <object>
for elem in root:
current_parent = elem.tag
current_sub = None
object_name = None
if elem.tag == 'folder':
continue
if elem.tag == 'filename':
file_name = elem.text
if file_name in category_set:
raise Exception('file_name duplicated')
# add img item only after parse <size> tag
elif current_image_id is None and file_name is not None and size['width'] is not None:
if file_name not in image_set:
current_image_id = addImgItem(file_name, size)
print('add image with {} and {}'.format(file_name, size))
else:
raise Exception('duplicated image: {}'.format(file_name))
# subelem is <width>, <height>, <depth>, <name>, <bndbox>
for subelem in elem:
bndbox['xmin'] = None
bndbox['xmax'] = None
bndbox['ymin'] = None
bndbox['ymax'] = None
current_sub = subelem.tag
if current_parent == 'object' and subelem.tag == 'name':
object_name = subelem.text
if object_name not in category_set:
current_category_id = addCatItem(object_name)
else:
current_category_id = category_set[object_name]
elif current_parent == 'size':
if size[subelem.tag] is not None:
raise Exception('xml structure broken at size tag.')
size[subelem.tag] = int(subelem.text)
# option is <xmin>, <ymin>, <xmax>, <ymax>, when subelem is <bndbox>
for option in subelem:
if current_sub == 'bndbox':
if bndbox[option.tag] is not None:
raise Exception('xml structure corrupted at bndbox tag.')
bndbox[option.tag] = int(option.text)
# only after parse the <object> tag
if bndbox['xmin'] is not None:
if object_name is None:
raise Exception('xml structure broken at bndbox tag')
if current_image_id is None:
raise Exception('xml structure broken at bndbox tag')
if current_category_id is None:
raise Exception('xml structure broken at bndbox tag')
bbox = []
# x
bbox.append(bndbox['xmin'])
# y
bbox.append(bndbox['ymin'])
# w
bbox.append(bndbox['xmax'] - bndbox['xmin'])
# h
bbox.append(bndbox['ymax'] - bndbox['ymin'])
print('add annotation with {},{},{},{}'.format(object_name, current_image_id, current_category_id,
bbox))
addAnnoItem(object_name, current_image_id, current_category_id, bbox)
json.dump(coco, open(json_save_path, 'w'))
"""直接从xml文件夹中生成"""
def parseXmlFiles(xml_path, json_save_path):
for f in os.listdir(xml_path):
if not f.endswith('.xml'):
continue
bndbox = dict()
size = dict()
current_image_id = None
current_category_id = None
file_name = None
size['width'] = None
size['height'] = None
size['depth'] = None
xml_file = os.path.join(xml_path, f)
print(xml_file)
tree = ET.parse(xml_file)
root = tree.getroot()
if root.tag != 'annotation':
raise Exception('pascal voc xml root element should be annotation, rather than {}'.format(root.tag))
# elem is <folder>, <filename>, <size>, <object>
for elem in root:
current_parent = elem.tag
current_sub = None
object_name = None
if elem.tag == 'folder':
continue
if elem.tag == 'filename':
file_name = elem.text
if file_name in category_set:
raise Exception('file_name duplicated')
# add img item only after parse <size> tag
elif current_image_id is None and file_name is not None and size['width'] is not None:
if file_name not in image_set:
current_image_id = addImgItem(file_name, size)
print('add image with {} and {}'.format(file_name, size))
else:
raise Exception('duplicated image: {}'.format(file_name))
# subelem is <width>, <height>, <depth>, <name>, <bndbox>
for subelem in elem:
bndbox['xmin'] = None
bndbox['xmax'] = None
bndbox['ymin'] = None
bndbox['ymax'] = None
current_sub = subelem.tag
if current_parent == 'object' and subelem.tag == 'name':
object_name = subelem.text
if object_name not in category_set:
current_category_id = addCatItem(object_name)
else:
current_category_id = category_set[object_name]
elif current_parent == 'size':
if size[subelem.tag] is not None:
raise Exception('xml structure broken at size tag.')
size[subelem.tag] = int(subelem.text)
# option is <xmin>, <ymin>, <xmax>, <ymax>, when subelem is <bndbox>
for option in subelem:
if current_sub == 'bndbox':
if bndbox[option.tag] is not None:
raise Exception('xml structure corrupted at bndbox tag.')
bndbox[option.tag] = int(option.text)
# only after parse the <object> tag
if bndbox['xmin'] is not None:
if object_name is None:
raise Exception('xml structure broken at bndbox tag')
if current_image_id is None:
raise Exception('xml structure broken at bndbox tag')
if current_category_id is None:
raise Exception('xml structure broken at bndbox tag')
bbox = []
# x
bbox.append(bndbox['xmin'])
# y
bbox.append(bndbox['ymin'])
# w
bbox.append(bndbox['xmax'] - bndbox['xmin'])
# h
bbox.append(bndbox['ymax'] - bndbox['ymin'])
print('add annotation with {},{},{},{}'.format(object_name, current_image_id, current_category_id,
bbox))
addAnnoItem(object_name, current_image_id, current_category_id, bbox)
json.dump(coco, open(json_save_path, 'w'))
if __name__ == '__main__':
# 通过txt文件生成
voc_data_dir=r"D:\new\22\pytorch-faster-rcnn-main\VOC2007"
json_save_path=r"D:\new\22\pytorch-faster-rcnn-main/COCODevKit/annotations/instances_val2017.json"
parseXmlFiles_by_txt(voc_data_dir,json_save_path)
# # 通过文件夹生成
# ann_path = r"D:\new\22\pytorch-faster-rcnn-main\VOC2007\Annotations"
# json_save_path = r"D:\new\22\pytorch-faster-rcnn-main\VOC2007/test.json"
# parseXmlFiles(ann_path, json_save_path)
8.XML转CSV格式
import os
import glob
import pandas as pd
import xml.etree.ElementTree as ET
def xml_to_csv(path):
xml_list = []
for xml_file in glob.glob(path + '/*.xml'):
print(xml_file)
tree = ET.parse(xml_file)
root = tree.getroot()
for member in root.findall('object'):
try:
value = (root.find('filename').text,
int(root.find('size')[0].text),
int(root.find('size')[1].text),
member[0].text,
int(member[4][0].text),
int(member[4][1].text),
int(member[4][2].text),
int(member[4][3].text)
)
except ValueError:
value = (root.find('filename').text,
int(root.find('size')[0].text),
int(root.find('size')[1].text),
member[0].text,
int(member[4][1][0].text),
int(member[4][1][1].text),
int(member[4][1][2].text),
int(member[4][1][3].text)
)
xml_list.append(value)
column_name = ['filename','width','height','class','xmin','ymin','xmax','ymax']
xml_df = pd.DataFrame(xml_list,columns=column_name)
return xml_df
def main():
image_path = os.path.join(os.getcwd(),'E://pytorch/Annotations')
xml_df = xml_to_csv(image_path)
xml_df.to_csv('E://pytorch/labes.csv',index=None) #带路径 'E://'
print('finish')
main()
9. 按train、val、trainval划分
"""
1.要修改各文件夹路径
２．类别标签按自己从０修改，此处二类为０－１
３．此代码路径是ubuntu16.04系统绝对路径
"""
import os
from pathlib import Path
from shutil import copyfile
from PIL import Image, ImageDraw
from xml.dom.minidom import parse
import numpy as np
FILE_ROOT = f"E:\dataset\Marine_Litter\dataset"+"/"
IMAGE_SET_ROOT = FILE_ROOT + f"VOC2007/ImageSets/Main" # 图片区分文件的路径
IMAGE_PATH = FILE_ROOT + f"VOC2007/JPEGImages" # 图片的位置
ANNOTATIONS_PATH = FILE_ROOT + f"VOC2007/Annotations" # 数据集标签文件的位置
LABELS_ROOT = FILE_ROOT + f"VOC2007/Labels" # 进行归一化之后的标签位置
DEST_IMAGES_PATH = f"VOC2007/custom_data/images" # 区分训练集、测试集、验证集的图片目标路径
DEST_LABELS_PATH = f"VOC2007/datasets/custom_data/labels" # 区分训练集、测试集、验证集的标签文件目标路径
def cord_converter(size, box):
"""
将标注的 xml 文件标注转换为 darknet 形的坐标
:param size: 图片的尺寸： [w,h]
:param box: anchor box 的坐标 [左上角x,左上角y,右下角x,右下角y,]
:return: 转换后的 [x,y,w,h]
"""
x1 = int(box[0])
y1 = int(box[1])
x2 = int(box[2])
y2 = int(box[3])
dw = np.float32(1. / int(size[0]))
dh = np.float32(1. / int(size[1]))
w = x2 - x1
h = y2 - y1
x = x1 + (w / 2)
y = y1 + (h / 2)
x = x * dw
w = w * dw
y = y * dh
h = h * dh
return [x, y, w, h]
def save_file(img_jpg_file_name, size, img_box):
save_file_name = LABELS_ROOT + '/' + img_jpg_file_name + '.txt'
print(save_file_name)
file_path = open(save_file_name, "a+")
for box in img_box:
if box[0] == 'person':
cls_num = 0
else:
cls_num = 1#两个类别
new_box = cord_converter(size, box[1:])
file_path.write(f"{cls_num} {new_box[0]} {new_box[1]} {new_box[2]} {new_box[3]}\n")
file_path.flush()
file_path.close()
# def test_dataset_box_feature(file_name, point_array):
# """
# 使用样本数据测试数据集的建议框
# :param image_name: 图片文件名
# :param point_array: 全部的点 [建议框sx1,sy1,sx2,sy2]
# :return: None
# """
# im = Image.open(rf"{IMAGE_PATH}\{file_name}")
# imDraw = ImageDraw.Draw(im)
# for box in point_array:
# x1 = box[1]
# y1 = box[2]
# x2 = box[3]
# y2 = box[4]
# imDraw.rectangle((x1, y1, x2, y2), outline='red')
#
# im.show()
def get_xml_data(file_path, img_xml_file):
img_path = file_path + '/' + img_xml_file + '.xml'
print(img_path)
dom = parse(img_path)
root = dom.documentElement
img_name = root.getElementsByTagName("filename")[0].childNodes[0].data
img_size = root.getElementsByTagName("size")[0]
objects = root.getElementsByTagName("object")
img_w = img_size.getElementsByTagName("width")[0].childNodes[0].data
img_h = img_size.getElementsByTagName("height")[0].childNodes[0].data
img_c = img_size.getElementsByTagName("depth")[0].childNodes[0].data
# print("img_name:", img_name)
# print("image_info:(w,h,c)", img_w, img_h, img_c)
img_box = []
for box in objects:
cls_name = box.getElementsByTagName("name")[0].childNodes[0].data
x1 = int(box.getElementsByTagName("xmin")[0].childNodes[0].data)
y1 = int(box.getElementsByTagName("ymin")[0].childNodes[0].data)
x2 = int(box.getElementsByTagName("xmax")[0].childNodes[0].data)
y2 = int(box.getElementsByTagName("ymax")[0].childNodes[0].data)
# print("box:(c,xmin,ymin,xmax,ymax)", cls_name, x1, y1, x2, y2)
img_jpg_file_name = img_xml_file + '.jpg'
img_box.append([cls_name, x1, y1, x2, y2])
# print(img_box)
# test_dataset_box_feature(img_jpg_file_name, img_box)
save_file(img_xml_file, [img_w, img_h], img_box)
def copy_data(img_set_source, img_labels_root, imgs_source, type):
file_name = img_set_source + '/' + type + ".txt"
file = open(file_name)
# 判断文件夹是否存在，不存在则创建
root_file = Path(FILE_ROOT + DEST_IMAGES_PATH + '/' + type)
if not root_file.exists():
print(f"Path {root_file} is not exit")
os.makedirs(root_file)
root_file = Path(FILE_ROOT + DEST_LABELS_PATH + '/' + type)
if not root_file.exists():
print(f"Path {root_file} is not exit")
os.makedirs(root_file)
# 遍历文件夹
for line in file.readlines():
print(line)
img_name = line.strip('\n')
img_sor_file = imgs_source + '/' + img_name + '.jpg'
label_sor_file = img_labels_root + '/' + img_name + '.txt'
# print(img_sor_file)
# print(label_sor_file)
# im = Image.open(rf"{img_sor_file}")
# im.show()
# 复制图片
DICT_DIR = FILE_ROOT + DEST_IMAGES_PATH + '/' + type
img_dict_file = DICT_DIR + '/' + img_name + '.jpg'
copyfile(img_sor_file, img_dict_file)
# 复制 label
DICT_DIR = FILE_ROOT + DEST_LABELS_PATH + '/' + type
img_dict_file = DICT_DIR + '/' + img_name + '.txt'
copyfile(label_sor_file, img_dict_file)
if __name__ == '__main__':
# 生成标签
root = ANNOTATIONS_PATH
files = os.listdir(root)
for file in files:
print("file name: ", file)
file_xml = file.split(".")
get_xml_data(root, file_xml[0])
# 将文件进行 train 和 val 的区分
img_set_root = IMAGE_SET_ROOT
imgs_root = IMAGE_PATH
img_labels_root = LABELS_ROOT
copy_data(img_set_root, img_labels_root, imgs_root, "train")
copy_data(img_set_root, img_labels_root, imgs_root, "val")
copy_data(img_set_root, img_labels_root, imgs_root, "test")
10.找到文件夹下图片对应的xml
#coding=utf-8
import os
#import os.path
import shutil #Python文件复制相应模块
label_dir=r'/home/.../Annotations' #所有xml文件所在文件夹
annotion_dir='/home/.../picturexml' #粘贴对应图片名称的xml文件到指定文件夹
path = '/home/.../picture' #图片文件夹
path_list = os.listdir(path)# os.listdir(file)会历遍文件夹内的文件并返回一个列表
#print(path_list)
path_name=[] # 定义一个空列表,不需要path_list中的后缀名
# 利用循环历遍path_list列表并且利用split去掉后缀名
for i in path_list:
path_name.append(i.split(".")[0])
#print(path_name)
# 排序一下
path_name.sort()
for file_name in path_name:
# "a"表示以不覆盖的形式写入到文件中,当前文件夹如果没有"save.txt"会自动创建
with open("save.txt","a") as f:
f.write(file_name + "\n")
#print(file_name)
f.close()
f = open("save.txt","r") #设置文件对象
lines= f.readlines()
#print (lines)
s=[]
for line in lines:
line = line.strip()
print (line)
tempxmlname='%s.xml'%line
print(tempxmlname)
xmlname=os.path.join(label_dir,tempxmlname)
print (xmlname)
os.listdir(label_dir)
shutil.copy(xmlname,annotion_dir)
11. 划分分类数据集（train、test）
import os
import random
import shutil
import csv
import numpy as np
def CopyFile(imageDir,test_rate,save_test_dir,save_train_dir):#三个参数，第一个为每个类别的所有图像在计算机中的位置
#第二个为copy的图片数目所占总的比例，最后一个为移动的图片保存的位置，
image_number = len(imageDir) #图片总数目
test_number = int(image_number * test_rate)#要移动的图片数目
print("要移动到%s目录下的图片数目为:%d"%(save_test_dir,test_number))
# print("要移动到%s目录下的图片数目为:%d"%(save_test_dir,test_number))
test_samples = random.sample(imageDir, test_number)#随机截取列表imageDir中数目为test_number的元素
# copy图像到目标文件夹
if not os.path.exists(save_test_dir):
os.makedirs(save_test_dir)
print("save_test_dir has been created successfully!")
else:
print("save_test_dir already exited!")
if not os.path.exists(save_train_dir):
os.makedirs(save_train_dir)
print("save_train_dir has been created successfully!")
else:
print("save_train_dir already exited!")
for i,j in enumerate(test_samples):
shutil.copy(test_samples[i], save_test_dir+test_samples[i].split("/")[-1])
print("tets移动完成！")
for train_imgs in imageDir:
if train_imgs not in test_samples:
shutil.copy(train_imgs, save_train_dir+train_imgs.split("/")[-1])
print("train移动完成")
#只需给定file_path、test_rate即可完成整个任务
#原始路径+分割比例
################################
file_path=r"E:\dataset\trashnet\dataset-resized\dataset-resized"
test_rate = 0.2
################################
file_dirs=os.listdir(file_path)
origion_paths=[]
save_test_dirs=[]
save_train_dirs=[]
for path in file_dirs:
origion_paths.append(file_path+"/"+path+"/")
save_train_dirs.append("./train/"+path+"/")
save_test_dirs.append("./test/"+path+"/")
for i,origion_path in enumerate(origion_paths):
image_list = os.listdir(origion_path) #获得原始路径下的所有图片的name（默认路径下都是图片）
image_Dir=[]
for x,y in enumerate(image_list):
image_Dir.append (os.path.join(origion_path, y))
print("%s目录下共有%d张图片！"%(origion_path,len(image_Dir)))
CopyFile(image_Dir,test_rate,save_test_dirs[i],save_train_dirs[i])
print("all datas has been moved successfully!")
12万+
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
