YOLOv3训练自己数据集中所用到的代码及命令总结

一. 将数据集整理为VOC2007标准格式所需要的一些代码：

1. 将所有图片重命名为000000.jpg的格式：

import os
path = "/home/Leequens/VOC/VOC2007/JPEG/test/YDXJ0013"
path1 = "/home/Leequens/VOC/VOC2007/JPEG/test/test6" # 生成到新路径中
filelist = os.listdir(path) #该文件夹下所有的文件（包括文件夹）
for file in filelist:   #遍历所有文件
    Olddir=os.path.join(path,file)   #原来的文件路径
    if os.path.isdir(Olddir):   #如果是文件夹则跳过
        continue
    filename=os.path.splitext(file)[0]   #文件名
    filetype=os.path.splitext(file)[1]   #文件扩展名
    Newdir=os.path.join(path1,str(int(filename)+7240).zfill(6)+filetype)  #用字符串函数zfill 以0补全所需位数
    os.rename(Olddir,Newdir)#重命名

2. 根据真值表生成每张图片所对应的xml文件：

# ! /usr/bin/python
# -*- coding:UTF-8 -*-
import os, sys
import glob
from PIL import Image

# VEDAI 图像存储位置
src_img_dir = os.path.abspath('.')+'/G0024173'
# VEDAI 图像的 ground truth 的 txt 文件存放位置
src_xml_dir = os.path.abspath('.')+'/xml/G0024173'


# 遍历目录读取图片
img_Lists = []
def get_img_list(dir_path):
    if os.path.isdir(dir_path):
        for x in os.listdir(dir_path):
            get_img_list(os.path.join(dir_path, x))
    elif os.path.isfile(dir_path) and dir_path.split('.')[-1] == 'jpg':
        img_Lists.append(dir_path)

get_img_list(src_img_dir)
img_Lists.sort(key=lambda x:x[-10:])
# for i in img_Lists:
#     print(i)

# 创建xml文件，存入图片信息
for img_item in img_Lists:
    im = Image.open(img_item)  #打开图片 为了记录图片的长宽数据
    img = os.path.split(img_item)[1].split('.')[0]
    width, height = im.size

    # write in xml file
    # os.mknod(src_xml_dir + '/' + img + '.xml')
    xml_file = open((src_xml_dir + '/' + str(img) + '.xml'), 'w')
    xml_file.write('<annotation>\n')
    xml_file.write('    <folder>VOC2007</folder>\n')
    xml_file.write('    <filename>' + str(img) + '.jpg' + '</filename>\n')
    xml_file.write('    <size>\n')
    xml_file.write('        <width>' + str(width) + '</width>\n')
    xml_file.write('        <height>' + str(height) + '</height>\n')
    xml_file.write('        <depth>3</depth>\n')
    xml_file.write('    </size>\n')
    xml_file.close()

    

# 读取全部信息
txt_file = open('G0024173_chage.txt')

for line in txt_file.readlines():
    gt = line.splitlines()
    # print(gt)
#     gt = txt_file.readline().splitlines()
#     # gt = open(src_txt_dir + '/gt_' + img + '.txt').read().splitlines()

    # write the region of image on xml file
    for img_each_label in gt:
        spt = img_each_label.split(' ')  # 这里如果txt里面是以逗号‘，’隔开的，那么就改为spt = img_each_label.split(',')。

        # 判断是否需要写入xml
        if spt[6] == '0': #因为上次比赛给的真值表第六列是loss，所以需要判断
            # print (gt)

            # 打开相应xml文件，下面具体写入哪一列根据真值表来修改
            # print(spt[5].zfill(6))
            xml_file = open((src_xml_dir + '/' + spt[5].zfill(6) + '.xml'), 'a') 
            xml_file.write('    <object>\n')
            xml_file.write('        <name>' + str(spt[9]) + '</name>\n')
            xml_file.write('        <pose>Unspecified</pose>\n')
            xml_file.write('        <truncated>0</truncated>\n')
            xml_file.write('        <difficult>0</difficult>\n')
            xml_file.write('        <bndbox>\n')
            xml_file.write('            <xmin>' + str(spt[1]) + '</xmin>\n')
            xml_file.write('            <ymin>' + str(spt[2]) + '</ymin>\n')
            xml_file.write('            <xmax>' + str(spt[3]) + '</xmax>\n')
            xml_file.write('            <ymax>' + str(spt[4]) + '</ymax>\n')
            xml_file.write('        </bndbox>\n')
            xml_file.write('    </object>\n')
            xml_file.close()

# 补上结尾，数值自己修改
for i in range(1800,3600): 
    xml_file = open((src_xml_dir + '/' + str(i).zfill(6) + '.xml'), 'a')
    xml_file.write('</annotation>')
    xml_file.close()

二.  一些关于YOLOv3的代码修改：

1. 开始训练命令

  先是合并txt文档

cat 2007_train.txt 2007_val.txt > train.txt

  训练之前要更改voc.data、yolov3-voc.cfg、voc.name、coco.name文件

./darknet detector train cfg/voc.data cfg/yolov3-voc.cfg darknet53.conv.74

2. 使YOLOv3批量测试图片并保存到文件夹中。

(1) 可以用下面代码替换detector.c文件（example文件夹下）的void test_detector函数。记得有三处要修改路径。

void test_detector(char *datacfg, char *cfgfile, char *weightfile, char *filename, float thresh, float hier_thresh, char *outfile, int fullscreen)
{
    list *options = read_data_cfg(datacfg);
    char *name_list = option_find_str(options, "names", "data/names.list");
    char **names = get_labels(name_list);
 
    image **alphabet = load_alphabet();
    network *net = load_network(cfgfile, weightfile, 0);
    set_batch_network(net, 1);
    srand(2222222);
    double time;
    char buff[256];
    char *input = buff;
    float nms=.45;
    int i=0;
    while(1){
        if(filename){
            strncpy(input, filename, 256);
            image im = load_image_color(input,0,0);
            image sized = letterbox_image(im, net->w, net->h);
        //image sized = resize_image(im, net->w, net->h);
        //image sized2 = resize_max(im, net->w);
        //image sized = crop_image(sized2, -((net->w - sized2.w)/2), -((net->h - sized2.h)/2), net->w, net->h);
        //resize_network(net, sized.w, sized.h);
            layer l = net->layers[net->n-1];
 
 
            float *X = sized.data;
            time=what_time_is_it_now();
            network_predict(net, X);
            printf("%s: Predicted in %f seconds.\n", input, what_time_is_it_now()-time);
            int nboxes = 0;
            detection *dets = get_network_boxes(net, im.w, im.h, thresh, hier_thresh, 0, 1, &nboxes);
            //printf("%d\n", nboxes);
            //if (nms) do_nms_obj(boxes, probs, l.w*l.h*l.n, l.classes, nms);
            if (nms) do_nms_sort(dets, nboxes, l.classes, nms);
                draw_detections(im, dets, nboxes, thresh, names, alphabet, l.classes);
                free_detections(dets, nboxes);
            if(outfile)
             {
                save_image(im, outfile);
             }
            else{
                save_image(im, "predictions");
#ifdef OPENCV
                cvNamedWindow("predictions", CV_WINDOW_NORMAL); 
                if(fullscreen){
                cvSetWindowProperty("predictions", CV_WND_PROP_FULLSCREEN, CV_WINDOW_FULLSCREEN);
                }
                show_image(im, "predictions");
                cvWaitKey(0);
                cvDestroyAllWindows();
#endif
            }
            free_image(im);
            free_image(sized);
            if (filename) break;
         } 
        else {
            printf("Enter Image Path: ");
            fflush(stdout);
            input = fgets(input, 256, stdin);
            if(!input) return;
            strtok(input, "\n");
   
            list *plist = get_paths(input);
            char **paths = (char **)list_to_array(plist);
             printf("Start Testing!\n");
            int m = plist->size;
            if(access("/home/Leequens/darknet/data/out",0)==-1)//"/home/FENGsl/darknet/data"修改成自己的路径
            {
              if (mkdir("/home/Leequens/darknet/data/out",0777))//"/home/FENGsl/darknet/data"修改成自己的路径
               {
                 printf("creat file bag failed!!!");
               }
            }
            for(i = 0; i < m; ++i){
             char *path = paths[i];
             image im = load_image_color(path,0,0);
             image sized = letterbox_image(im, net->w, net->h);
        //image sized = resize_image(im, net->w, net->h);
        //image sized2 = resize_max(im, net->w);
        //image sized = crop_image(sized2, -((net->w - sized2.w)/2), -((net->h - sized2.h)/2), net->w, net->h);
        //resize_network(net, sized.w, sized.h);
        layer l = net->layers[net->n-1];
 
 
        float *X = sized.data;
        time=what_time_is_it_now();
        network_predict(net, X);
        printf("Try Very Hard:");
        printf("%s: Predicted in %f seconds.\n", path, what_time_is_it_now()-time);
        int nboxes = 0;
        detection *dets = get_network_boxes(net, im.w, im.h, thresh, hier_thresh, 0, 1, &nboxes);
        //printf("%d\n", nboxes);
        //if (nms) do_nms_obj(boxes, probs, l.w*l.h*l.n, l.classes, nms);
        if (nms) do_nms_sort(dets, nboxes, l.classes, nms);
        draw_detections(im, dets, nboxes, thresh, names, alphabet, l.classes);
        free_detections(dets, nboxes);
        if(outfile){
            save_image(im, outfile);
        }
        else{
             
             char b[2048];
            sprintf(b,"/home/Leequens/darknet/data/out/%s",GetFilename(path));//"/home/FENGsl/darknet/data"修改成自己的路径
            
            save_image(im, b);
            printf("save %s successfully!\n",GetFilename(path));
#ifdef OPENCV
            cvNamedWindow("predictions", CV_WINDOW_NORMAL); 
            if(fullscreen){
                cvSetWindowProperty("predictions", CV_WND_PROP_FULLSCREEN, CV_WINDOW_FULLSCREEN);
            }
            show_image(im, "predictions");
            cvWaitKey(0);
            cvDestroyAllWindows();
#endif
        }
 
        free_image(im);
        free_image(sized);
        if (filename) break;
        }
      }
    }
}

 (2) 然后在前面添加*GetFilename(char *p)函数

#include"darknet.h"
#include<sys/stat.h>
#include<stdio.h>
#include<time.h>
#include<unistd.h>
#include<sys/types.h>

static int coco_ids[] = {1,2,3,4,5,6,7,8,9,10,11,13,14,15,16,17,18,19,20,21,22,23,24,25,27,28,31,32,33,34,35,36,37,38,39,40,41,42,43,44,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,67,70,72,73,74,75,76,77,78,79,80,81,82,84,85,86,87,88,89,90};
 
char *GetFilename(char *p)
{ 
    static char name[20]={""};
    char *q = strrchr(p,'/') + 1;
    strncpy(name,q,6);
    return name;
}

(3) 重新在darknet下make

(4) 执行下面命令开始测试 (把权重名字修改)

./darknet detector test cfg/voc.data yolov3-voc.cfg weights/yolov3-voc.backup -thresh 0.1

  YOLOv3默认阈值为0.25，-thresh 0是设置阈值，设为0的话可以显示出所有检测结果。

  运行完直接会让你输入 Enter Image Path:

 这里直接输入voc生成好的test文档的绝对路径就可以，里面包含了所有测试集的绝对路径。

3.使YOLOv3输出测试结果保存在txt文档中

./darknet detector valid cfg/voc.data yolov3-voc.cfg weights/yolov3-voc.backup -out detect_result.txt -thresh 0.1

4. 此次目标大赛txt文档格式为 图片id 类id 置信度 4个坐标，需要修改刚刚生成的txt

cat 123.txt | awk '{$7=$6;$6=$5;$5=$4;$4=$3;$3=$2;$2=1; print $0;}' > 123_chage.txt

5. 将生成的几个txt合并到一起

cat 1.txt 2.txt 3.txt > result.txt

6. 对合并到一起的txt按第一列（图片ID）的值进行排序，目的是使同一图的信息罗列在一起

sort -t $'\t' -k1 -n result.txt > 1result.txt # k1指的是第一列

7. 可以通过修改detector.c文件使得完成3-6步骤

  先修改validate_detector中的第二个if语句，直接复制替换

    if(0==strcmp(type, "coco")){
        if(!outfile) outfile = "coco_results";
        snprintf(buff, 1024, "%s/%s.json", prefix, outfile);
        fp = fopen(buff, "w");
        fprintf(fp, "[\n");
        coco = 1;
    } else if(0==strcmp(type, "imagenet")){
        if(!outfile) outfile = "imagenet-detection";
        snprintf(buff, 1024, "%s/%s.txt", prefix, outfile);
        fp = fopen(buff, "w");
        imagenet = 1;
        classes = 200;
    } else {
        if(!outfile) outfile = "comp4_det_test_";
        snprintf(buff, 1024, "%s/%s.txt", prefix, outfile);
        fp = fopen(buff, "w"); 
    }

  再修改validate_detector函数中的语句 print_detector_detections(fps, id, dets, nboxes, classes, w, h); 

  将fps修改为fp：print_detector_detections(fp, id, dets, nboxes, classes, w, h);

  再用下面代码替换 print_detector_detections 函数

void print_detector_detections(FILE *fp, char *id, detection *dets, int total, int classes, int w, int h)
{
    int i, j;
    int nums[4]={1,2,3,4}; #自己class有几类就相应的改掉，这里的1234对应的顺序和voc.name中一样
    for(i = 0; i < total; ++i){
        float xmin = dets[i].bbox.x - dets[i].bbox.w/2. + 1;
        float xmax = dets[i].bbox.x + dets[i].bbox.w/2. + 1;
        float ymin = dets[i].bbox.y - dets[i].bbox.h/2. + 1;
        float ymax = dets[i].bbox.y + dets[i].bbox.h/2. + 1;

        if (xmin < 1) xmin = 1;
        if (ymin < 1) ymin = 1;
        if (xmax > w) xmax = w;
        if (ymax > h) ymax = h;

        for(j = 0; j < classes; ++j){
            if (dets[i].prob[j]) fprintf(fp, "%s %d %f %f %f %f %f\n", id, nums[j], dets[i].prob[j],
                    xmin, ymin, xmax, ymax);
        }
    }
}

  然后重新make，再执行生成txt文档的语句，就可以把所有class的检测结果输出在一个txt文档中。

  PS：make之前记得删除原有的darknet可执行文件

三、处理离线比赛给的xml节点的代码

1. 将xml中的三类object分别替换为1 2 3

import os
import sys
try:
    import xml.etree.cElementTree as ET
except ImportError:
    import xml.etree.ElementTree as ET
path = "/media/leequens/File/YOLO/test/xml/123" #文件夹路径
filelist = os.listdir(path) #读取该文件夹下的所有文件
for file in filelist: #循环每个文件
    filename = os.path.splitext(file)[0]  # 文件名
    filetype = os.path.splitext(file)[1]  # 文件扩展名
    tree = ET.parse('/media/leequens/File/YOLO/test/xml/123/'+str(int(filename)).zfill(6)+'.xml')
    for elem in tree.iter(tag='name'):  # 遍历树中的name节点
        if elem.text == '"scallop"':
            elem.text = '1'
        if elem.text == '"seaurchin"':
            elem.text = '2'
        if elem.text == '"seacucumber"':
            elem.text = '3'
    tree.write('/media/leequens/File/YOLO/test/xml/111/'+str(int(filename)).zfill(6)+'.xml') #修改完毕后写入另一个文件夹中

2. 将比赛官方给的xml中的frame改为自己重命名后的图片名

from xml.etree import ElementTree
import os
path = "/media/leequens/File/YOLO/test/xml/123"
filelist = os.listdir(path)
for file in filelist:
    filename = os.path.splitext(file)[0]  # 文件名
    filetype = os.path.splitext(file)[1]  # 文件扩展名
    xmldoc = ElementTree.parse('/media/leequens/File/YOLO/test/xml/123/'+str(int(filename)).zfill(6)+'.xml')
    node= xmldoc.find('./frame')  # 遍历树中的movie节点
    node.text=str(int(filename)).zfill(6) # 如果需要图片后缀 +'.jpg'
    xmldoc.write('/media/leequens/File/YOLO/test/xml/111/'+str(int(filename)).zfill(6)+'.xml')

3. 修改xml中节点frame为filename

from xml.etree import ElementTree
import os, sys
import glob
from PIL import Image

path = "/media/leequens/File/YOLO/test/xml/11"
filelist = os.listdir(path)
for file in filelist:
    filename = os.path.splitext(file)[0]  # 文件名
    filetype = os.path.splitext(file)[1]  # 文件扩展名
    xmldoc = ElementTree.parse('/media/leequens/File/YOLO/test/xml/rebox/'+str(int(filename)).zfill(6)+'.xml')
    root = xmldoc.getroot()
    for child in root:
        if child.tag == 'frame':
            temp_node = 'filename' # 不懂为啥需要一个临时的变量，直接赋值就不对
            child.tag = temp_node
            break	
    xmldoc.write('/media/leequens/File/YOLO/test/xml/'+str(int(filename)).zfill(6)+'.xml')

4. 给xml增加图片长宽等信息

from xml.dom import minidom, Node
from xml.etree import ElementTree

import os, sys
import glob
from PIL import Image

path = "/media/leequens/File/YOLO/test/xml/xmlwithjpg/1920-1080"
filelist = os.listdir(path)
for file in filelist:
    filename = os.path.splitext(file)[0]  # 文件名
    filetype = os.path.splitext(file)[1]  # 文件扩展名
    xmldoc = ElementTree.parse('/media/leequens/File/YOLO/test/xml/xmlwithjpg/1920-1080/'+str(int(filename)).zfill(6)+'.xml')
    root = xmldoc.getroot()
    folder=ElementTree.Element('folder')
    root.insert(0,folder)
    folder.text='VOC2007'
    size = ElementTree.Element("size")
    root.insert(2,size)
    width =ElementTree.Element('width')
    width.text = '1920'
    size.insert(0,width)
    height =ElementTree.Element('height')
    height.text = '1080'
    size.insert(1,height)
    depth =ElementTree.Element('depth ')
    depth.text = '3'
    size.insert(2,depth )
    xmldoc.write('/media/leequens/File/YOLO/test/xml/perfectxml/'+str(int(filename)).zfill(6)+'.xml')

  数值自己要改，可以用下面代码查看所有图片长宽

# ! /usr/bin/python
# -*- coding:UTF-8 -*-
import os, sys
import glob
from PIL import Image

# VEDAI 图像存储位置
src_img_dir = os.path.abspath('.')+'/2018image'


# 遍历目录读取图片
img_Lists = []
def get_img_list(dir_path):
    if os.path.isdir(dir_path):
        for x in os.listdir(dir_path):
            get_img_list(os.path.join(dir_path, x))
    elif os.path.isfile(dir_path) and dir_path.split('.')[-1] == 'jpg':
        img_Lists.append(dir_path)

get_img_list(src_img_dir)
img_Lists.sort(key=lambda x:x[-10:])

for img_item in img_Lists:
    filename = os.path.splitext(img_item)[0]
    im = Image.open(img_item)  #打开图片 为了记录图片的长宽数据
    img = os.path.split(img_item)[1].split('.')[0]
    width, height = im.size
    print(filename,width,height)

5. 给object加入difficult等节点

from xml.dom import minidom, Node
from xml.etree import ElementTree

import os, sys
import glob
from PIL import Image

path = "/media/leequens/File/YOLO/test/xml/111"
filelist = os.listdir(path)
for file in filelist:
    filename = os.path.splitext(file)[0]  # 文件名
    filetype = os.path.splitext(file)[1]  # 文件扩展名
    xmldoc = ElementTree.parse('/media/leequens/File/YOLO/test/xml/111/'+str(int(filename)).zfill(6)+'.xml')
    root = xmldoc.getroot()
    pose=ElementTree.Element('pose')
    pose.text='Unspecified'
    pose.tail='\n\t'
    truncated = ElementTree.Element('truncated')
    truncated.text='0'
    truncated.tail ='\n\t'
    difficult = ElementTree.Element('difficult')
    difficult.text = '0'
    difficult.tail ='\n\t'
    for node in xmldoc.iter(tag='object'):
    #node = root.find('./object')
    #objec=ElementTree.Element("object")
        node.insert(1,pose)
        node.insert(2,truncated)
        node.insert(3,difficult)
    xmldoc.write('/media/leequens/File/YOLO/test/xml/'+str(int(filename)).zfill(6)+'.xml')

6. 因为测试集是随机分配的，计算mAP时需要提取出测试集所对应的xml文件，测试集的名字都被存储到了test.txt文档里，所以需要编写一个程序把这些xml全部提取出来

import shutil
txt_file= open("111.txt","r",encoding="utf-8",errors="ignore")
while True:
    line = txt_file.readline() #表示一次读取一行
    line=line.strip('\n') # 这里要删除换行符，不然找不到对应文件，头疼
    if not line: #读到数据最后跳出，结束循环。数据的最后也就是读不到数据了，mystr为空的时候
        break
    oldname=u"/media/leequens/File/YOLO/test/xml/111/"+line+".xml" #旧目录
    newname=u"/media/leequens/File/YOLO/test/xml/"+line+".xml" # 新目录
    shutil.copyfile(oldname, newname)

./darknet detector demo cfg/voc.data yolov3-voc.cfg yolov3-voc_20000.weights "http://192.168.254.1:8090/?action=stream"