深度学习数据处理

一 数据格式的转换脚本

   1 将DOTA(HBB)数据集格式转换为yolo数据

"""
#2019.4.4: 将DOTA(HBB)数据集格式转换成YOLO数据集格式
欠缺将标签保存的Txt文档保存
"""
import cv2
import os
from collections import defaultdict

imagepath="/data/maq/DataSet/pytorch_yolov3/guigang/images"
labeltxt_DOTA="/data/maq/DataSet/pytorch_yolov3/guigang/labels_dota_hbb4"

labeltxt_YOLO="/data/maq/DataSet/pytorch_yolov3/guigang/labels"


def readsingletxt_all_dotalabel(imagepath,labelspath,outputfile):

    img = cv2.imread(imagepath)
    size = img.shape
    f = open(labelspath, 'r')
    lines = f.readlines()
    file_label = open(outputfile, "a")
    for line in lines:
        line = line.split(" ")
        #取出DOTA中的标签参数
        x1=min(float(line[0]),float(line[2]),float(line[4]),float(line[6]))
        x3 = max(float(line[0]), float(line[2]), float(line[4]), float(line[6]))
        y1 = min(float(line[1]), float(line[3]), float(line[5]), float(line[7]))
        y3 = max(float(line[1]), float(line[3]), float(line[5]), float(line[7]))
        x,y,w,h=dota_convert_yolo(size, x1, y1, x3, y3)
        catrgory_label= int(line[8])-1
        #catrgory_val=line[9].split("\n")[0]
        #将类别标签保存成字典，并将数据按照YOLO标签保存到txt
        string=str(catrgory_label)+" "+str(x)+" "+str(y)+" "+str(w)+" "+str(h)+"\n"
        file_label.write(string)
    file_label.close()

#dota数据转换成yolo格式
def dota_convert_yolo(size,x1,y1,x3,y3):
    height = size[0]  # 图片的高
    dh = 1 / height
    width = size[1]  # 图片的宽
    dw = 1 / width
    # 还原YOLO中拿去的min_row,min_col,max_row,max_col
    min_row = y1
    min_rol = x1
    max_row = y3
    max_rol = x3
    # 转换成YOLO中可以用的坐标
    x = min_rol + (max_rol - min_rol) / 2
    y = min_row + (max_row - min_row) / 2
    w = max_rol - min_rol
    h = max_row - min_row
    # 将坐标进行归一化
    x = x * dw
    y = y * dh
    w = w * dw
    h = h * dh
    return (x,y,w,h)

def main():
    tempfilename = os.listdir(imagepath)
    for file in tempfilename:  # 遍历文件夹
        (filename, extension) = os.path.splitext(file)
        imagepath_tif = os.path.join(imagepath + "/" + file)
        labeltxt_dota = os.path.join(labeltxt_DOTA + "/" + filename + '.txt')  # 获取对应的DOTA数据.txt标签文件
        labeltxt_yolo = os.path.join(labeltxt_YOLO + "/" + filename + '.txt')  # 输出DOTA标签转变为yolo标签的文件
        if os.path.exists(labeltxt_yolo):  # 判断文件是否存在，存在就越过
            # continue
            os.remove(labeltxt_yolo)
        readsingletxt_all_dotalabel(imagepath_tif, labeltxt_dota, labeltxt_yolo)

if __name__=='__main__':
    main()
    print("finished!!")

  2  将VOC数据集转换成yolo数据

 

"""
#2019.4.4 将PASCAL VOC数据集转换成yolo数据集格式
"""
import xml.etree.ElementTree as ET
import os


pascal_Ann_path="C:\\Users\\lijian\\Desktop\\PASCAL_VOC\\Annotations"
yolo_Output_txt="C:\\Users\\lijian\\Desktop\\PASCAL_VOC\\VOC_TO_YOLO"

#对voc中的坐标进行转换
def convert(size, box):
    dw = 1./(size[0])
    dh = 1./(size[1])
    x = (box[0] + box[1])/2.0
    y = (box[2] + box[3])/2.0
    w = box[1] - box[0]
    h = box[3] - box[2]
    x = x*dw
    w = w*dw
    y = y*dh
    h = h*dh
    return (x,y,w,h)

classes={'person':1} #检测的内容类别

#读出文件夹中XML文件，并寻找相应的尺寸
def convert_annotation(input_vocxml_file,output_yolo_txt):
    in_file = open(input_vocxml_file)
    out_file = open(output_yolo_txt, 'a')
    tree=ET.parse(in_file)
    root = tree.getroot()
    size = root.find('size')

    w = int(size.find('width').text)
    h = int(size.find('height').text)

    for obj in root.iter('object'):
        cls = obj.find('name').text
        if cls not in classes :
            continue
        cls_id = classes[cls]
        xmlbox = obj.find('bndbox')
        b = (float(xmlbox.find('xmin').text), float(xmlbox.find('xmax').text), float(xmlbox.find('ymin').text), float(xmlbox.find('ymax').text))
        bb = convert((w,h), b)
        out_file.write(str(cls_id) + " " + " ".join([str(a) for a in bb]) + '\n')



def main():
    tempfilename = os.listdir(pascal_Ann_path)
    for file in tempfilename: #遍历文件夹
        (filename,extension)=os.path.splitext(file)
        pascal_ann_path=os.path.join(pascal_Ann_path+"\\"+file) #存放.XML的文件件
        yolo_output_txt=os.path.join(yolo_Output_txt+"\\"+filename+'.txt')

        if os.path.exists(yolo_output_txt):
            os.remove(yolo_output_txt)
        convert_annotation(pascal_ann_path,yolo_output_txt)

if __name__ == '__main__':
    main()

3 YOLO数据转DOTA(HBB)

"""
# 2019.4.3 : 将YOLO检测数据集格式转换成DOTA数据集格式(HBB)
"""
import cv2
import os

imagepath="/data/maq/DataSet/xue_oilwell_2019619/images"
labelspath="/data/maq/DataSet/xue_oilwell_2019619/labels"

output="/data/maq/DataSet/xue_oilwell_2019619/labels_dota"

def readsingletxt_all_yololabel(imagepath,labelspath,outputfile):

    img = cv2.imread(imagepath)
    size = img.shape
    # height = size[0]  # 图片的高
    # width = size[1]  # 图片的宽
    # print("####labels:", labelspath)
    # os._exit()
    f = open(labelspath, 'r')
    lines = f.readlines()
    for line in lines:
        line = line.split(" ")
        catrgory_val = int(line[0])
        if ((line[1]=="NA") + (line[2]=="NA") |(line[3]=="NA")|(line[3]=="NA"))>0:
            print("####labelspath:", labelspath)
            continue
        min_row, min_col, max_row, max_col=dota_convert_yolo(size,line[1],line[2],line[3],line[4])
        #构建矩形框在原图中的位置
        x1, y1 = min_col, min_row
        x2, y2 = max_col, min_row
        x3, y3 = max_col, max_row
        x4, y4 = min_col, max_row
        string = str(x1) + " " + str(y1) + " " + str(x2) + " " + str(y2) + " " + str(x3) + " " + str(y3) + " " + str(
            x4) + " " + str(y4)
        DOTAlabel_writetxt(string,catrgory_val,outputfile)

#yolo数据转换成dota格式
def dota_convert_yolo(size,x,y,w,h):
    height = size[0]  # 图片的高
    width = size[1]  # 图片的宽
    # 取出yolo中的标签参数
    x = float(x) * width
    y = float(y) * height
    w = float(w) * width
    h = float(h) * height
    # 还原矩形框最小/大的行/列数值
    min_row = round(y - h / 2)
    max_row = round(y + h / 2)
    min_col = round(x - w / 2)
    max_col = round(x + w / 2)
    return min_row,min_col,max_row,max_col

catrgory={"oilwell":0}
#将DOTA数据加上相应标签写入文件
def DOTAlabel_writetxt(string,catrgory_val,outputfile):
    for catr, val in catrgory.items():
        if val == catrgory_val:
            stringall = string + " " + catr + " " + str(val) + "\n"
            file_label = open(outputfile, "a")
            file_label.write(stringall)

def main():
    tempfilename = os.listdir(imagepath)
    for file in tempfilename:  # 遍历文件夹
        (filename, extension) = os.path.splitext(file)
        imagepath_tif = os.path.join(imagepath + "/" + file)
        labelspath_txt = os.path.join(labelspath + "/" + filename + '.txt')  # 获取对应的.txt标签文件
        output_file = os.path.join(output + "/" + filename + '.txt')  # 输出yolo标签转变为DOTA标签的文件
        if os.path.exists(output_file):  # 判断文件是否存在，存在就越过
            # continue
            os.remove(output_file)
        readsingletxt_all_yololabel(imagepath_tif, labelspath_txt, output_file)

if __name__ == '__main__':
    main()
    print("finished!!!")

4 YOLO转VOC数据格式

"""
2019.4.8: 将YOLO数据集格式转换成VOC数据集格式
"""

import os
from xml.dom.minidom import Document
import cv2

input_Yolo_txt="C:\\Users\\lijian\\Desktop\\lables1"
input_Yolo_image="C:\\Users\\lijian\\Desktop\\image_labels1"
output_Voc_xml="C:\\Users\\lijian\\Desktop\\voc_xml"


def get_YoloInfo(txt_dir):
    f = open(txt_dir, 'r')
    line_info = f.readlines()

    L = len(line_info)
    label_classes = []
    for line in range(L):
        label_classes.append('x')
        line_info[line] = line_info[line].strip()
        each_data = line_info[line].split(' ')
        classes = int(each_data[0])
        x_center = float(each_data[1])
        y_center = float(each_data[2])
        width_nor = float(each_data[3])
        height_nor = float(each_data[4])

        label_classes[line] = [classes, x_center, y_center, width_nor, height_nor]
    return label_classes

catrgory={"house":1}   #原始字典
new_dict = {v : k for k, v in catrgory.items()}   #将原始字典的key与val相互换

#将yolo的原始数据进行转换
def Yolo_voc_data(size,yoloinfo):
    ##修改label类别
    num = len(yoloinfo)
    # 修改图片数据
    height = size[0]  # 图片的高
    width = size[1]  # 图片的宽
    voc_data = []
    for each in range(num):
        voc_data.append('x')

        classes = new_dict[yoloinfo[each][0]]
        x_center = yoloinfo[each][1] * width
        y_center = yoloinfo[each][2] * height
        width_nor = yoloinfo[each][3] * width
        height_nor = yoloinfo[each][4] * height

        voc_xmin = str(int(x_center - (width_nor / 2)))
        voc_ymin = str(int(y_center - (height_nor / 2)))
        voc_xmax = str(int(x_center + (width_nor / 2)))
        voc_ymax = str(int(y_center + (height_nor / 2)))

        voc_data[each] = [classes, voc_xmin, voc_ymin, voc_xmax, voc_ymax]
    return voc_data

# list3存储VOC格式的数据信息
def list_save_voc(filename_txt,file_name,path_txt,info_voc,size,output_voc_xml):
    folder_txt = filename_txt
    file_name = file_name
    path_txt = path_txt
    #print(folder_txt)
    width_str = str(size[1])
    height_str =str(size[0])
    depth_str = str(size[2])
    info_voc = info_voc

    doc = Document()
    annotation = doc.createElement('annotation')

    doc.appendChild(annotation)
    # folder
    folder = doc.createElement('folder')
    folder_txt = doc.createTextNode(folder_txt)
    folder.appendChild(folder_txt)
    annotation.appendChild(folder)

    # filename
    filename = doc.createElement('filename')
    filename_txt = doc.createTextNode(file_name)
    filename.appendChild(filename_txt)
    annotation.appendChild(filename)

    # path
    path = doc.createElement('path')
    path_txt = doc.createTextNode(path_txt)
    path.appendChild(path_txt)
    annotation.appendChild(path)

    # source
    source = doc.createElement('source')
    annotation.appendChild(source)

    # database
    database = doc.createElement('database')
    database_txt = doc.createTextNode('Unknown')
    database.appendChild(database_txt)
    source.appendChild(database)

    # size
    size = doc.createElement('size')
    annotation.appendChild(size)

    # width
    width = doc.createElement('width')
    width_txt = doc.createTextNode(width_str)
    width.appendChild(width_txt)
    size.appendChild(width)

    # height
    height = doc.createElement('height')
    height_txt = doc.createTextNode(height_str)
    height.appendChild(height_txt)
    size.appendChild(height)

    # depth
    depth = doc.createElement('depth')
    depth_txt = doc.createTextNode(depth_str)
    depth.appendChild(depth_txt)
    size.appendChild(depth)

    # segmented
    segmented = doc.createElement('segmented')
    segmented_txt = doc.createTextNode('0')
    segmented.appendChild(segmented_txt)
    annotation.appendChild(folder)
    object1 = []
    for i in range(len(info_voc)):
        object1.append('name' + str(i))
        object1[i] = doc.createElement('object')
        annotation.appendChild(object1[i])

        # name
        name = doc.createElement('name')
        name_txt = doc.createTextNode(info_voc[i][0])
        name.appendChild(name_txt)
        object1[i].appendChild(name)

        # pose
        pose = doc.createElement('pose')
        pose_txt = doc.createTextNode('Unspecified')
        pose.appendChild(pose_txt)
        object1[i].appendChild(pose)

        # truncated
        truncated = doc.createElement('truncated')
        truncated_txt = doc.createTextNode('0')
        truncated.appendChild(truncated_txt)
        object1[i].appendChild(truncated)

        # difficult
        difficult = doc.createElement('difficult')
        difficult_txt = doc.createTextNode('0')
        difficult.appendChild(difficult_txt)
        object1[i].appendChild(difficult)

        # bndbox
        bndbox = doc.createElement('bndbox')
        object1[i].appendChild(bndbox)

        # xmin
        xmin = doc.createElement('xmin')
        xmin_txt = doc.createTextNode(info_voc[i][1])
        xmin.appendChild(xmin_txt)
        bndbox.appendChild(xmin)

        # ymin
        ymin = doc.createElement('ymin')
        ymin_txt = doc.createTextNode(info_voc[i][2])
        ymin.appendChild(ymin_txt)
        bndbox.appendChild(ymin)

        # xmax
        xmax = doc.createElement('xmax')
        xmax_txt = doc.createTextNode(info_voc[i][3])
        xmax.appendChild(xmax_txt)
        bndbox.appendChild(xmax)

        # ymax
        ymax = doc.createElement('ymax')
        ymax_txt = doc.createTextNode(info_voc[i][4])
        ymax.appendChild(ymax_txt)
        bndbox.appendChild(ymax)
    dirrr = output_voc_xml
    #print(dirrr)
    f = open(dirrr, 'w')
    doc.writexml(f, indent='\t', newl='\n', addindent='\t', encoding='utf-8')
    f.close()


def main():
    tmp_yolo_txt = os.listdir(input_Yolo_txt)
    for file in tmp_yolo_txt:
        (filename, extension) = os.path.splitext(file)
        input_yolo_txt = os.path.join(input_Yolo_txt + "\\" + file)
        input_yolo_image = os.path.join(input_Yolo_image + "\\" + filename + ".tif")
        output_voc_xml = os.path.join(output_Voc_xml + "\\" + filename+".xml")

        if os.path.exists(output_voc_xml):
            os.remove(output_voc_xml)

        img = cv2.imread(input_yolo_image)
        size = img.shape
        info_1 = get_YoloInfo(input_yolo_txt)
        filname_txt = input_yolo_txt.split("\\")[-2]
        file_name = input_yolo_txt.split('\\')[-1]
        path_txt = input_yolo_txt

        info_voc = Yolo_voc_data(size, info_1)
        list_save_voc(filname_txt, file_name, path_txt, info_voc, size, output_voc_xml)

if __name__=="__main__":
    main()

5 DOTA(OBB)转DOTA(HBB)

"""
2019.6.5 将dota的OBB(4点数据)数据转换成dota的HBB数据(4点)
"""
import os

#将dota的OBB数据转换成dota的HBB数据
def OBB_conver_HBB(txtfile,final_savetxt_path):

    file=open(txtfile,'r')
    if os.path.exists(final_savetxt_path):
        os.remove(final_savetxt_path)
    out_file = open(final_savetxt_path, 'a')

    for line in file:
        splitline=line.split(" ")
        xmin=min(float(splitline[0]),float(splitline[2]),float(splitline[4]),float(splitline[6]))
        xmax = max(float(splitline[0]), float(splitline[2]), float(splitline[4]), float(splitline[6]))
        ymin = min(float(splitline[1]), float(splitline[3]), float(splitline[5]), float(splitline[7]))
        ymax = max(float(splitline[1]), float(splitline[3]), float(splitline[5]), float(splitline[7]))

        x1,y1=xmin,ymin
        x2,y2=xmax,ymin
        x3,y3=xmax,ymax
        x4,y4=xmin,ymax

        string=str(x1)+" "+str(y1)+" "+str(x2)+" "+str(y2)+" "+str(x3)+" "+str(y3)+" "+str(x4)+" "+str(y4)+" "+splitline[8]
        # print("###string:",string)
        # os._exit()
        out_file.write(string)


if __name__=="__main__":
    inputfile='/data/maq/DataSet/pytorch_yolov3/guigang/labels_dota_obb4'
    outputfile='/data/maq/DataSet/pytorch_yolov3/guigang/labels_dota_hbb4'

    tempfilename=os.listdir(inputfile)
    for file in tempfilename:
        txtfile=os.path.join(inputfile+"/"+file)
        final_savetxt_path=os.path.join(outputfile+"/"+file)
        OBB_conver_HBB(txtfile,final_savetxt_path)
    print("finished!!!")