1.环境配置
2.数据集获取
3.训练集获取
4.训练
5.调用测试训练结果
6.代码讲解
本文是第三篇,获取tfboard训练集。
前面我们拿到了所有图片对应的标注信息的xml文件,现在我们需要先把这些xml文件整合到一个csv里面,然后把他们转为tfrecord文件
整合为csv文件需要执行以下代码(xml_to_csv.py):
import os import glob import pandas as pd import xml.etree.ElementTree as ET DIR_NAME = 'out_xml' def xml_to_csv(path): xml_list = [] for xml_file in glob.glob(path + '/*.xml'):#glob.glob会返回制定路径下所有符合格式的文件列表,我们这里对列表遍历 tree = ET.parse(xml_file)#创建解析树 root = tree.getroot()#得到解析树的根元素 for member in root.findall('object'):#对xml里面的object遍历 value = (root.find('filename').text, int(root.find('size')[0].text), int(root.find('size')[1].text), member[0].text, int(member[4][0].text), int(member[4][1].text), int(member[4][2].text), int(member[4][3].text) ) xml_list.append(value)#拿到数据 column_name = ['filename', 'width', 'height', 'class', 'xmin', 'ymin', 'xmax', 'ymax']#列名 xml_df = pd.DataFrame(xml_list, columns=column_name)#生成csv return xml_df def main(): image_path = os.path.join(os.getcwd(), DIR_NAME)#将DIR_NAME里的xml文件全部读入 xml_df = xml_to_csv(image_path)#得到csv xml_df.to_csv('raccoon_labels.csv', index=None)#写入文件 print('执行完毕') main()
我们将之前的xml文件分为两批,一批做训练集,一批做测试集,然后分别用训练集和测试集所在的目录替换DIR_NAME生成train.csv和test.csv文件。
下一步就是生成tfrecord格式的文件,之所以要生成tfrecord格式的文件,是因为他是二进制的,操作效率很高,在运算方面比较快。
按照说明执行即可,代码我都写好了注释:
""" 用法: # 在 tensorflow/models目录下,打开命令行 # 生成训练集的tfrecord文件,执行以下命令: python generate_tfrecord.py --csv_input=train_labels.csv --output_path=train.record # 生成测试集的tfrecord文件,执行以下命令: python generate_tfrecord.py --csv_input=test_labels.csv --output_path=test.record #记得把里面文件名改为对应的文件名, """ from __future__ import division from __future__ import print_function from __future__ import absolute_import import os import io import pandas as pd import tensorflow as tf from PIL import Image from object_detection.utils import dataset_util from collections import namedtuple, OrderedDict flags = tf.app.flags flags.DEFINE_string('csv_input', '', 'Path to the CSV input') flags.DEFINE_string('output_path', '', 'Path to output TFRecord') FLAGS = flags.FLAGS # labelmap,一个labelname对应一个value def class_text_to_int(row_label): if row_label == 'red': return 1 elif row_label == 'blue': return 2 else: None def split(df, group): data = namedtuple('data', ['filename', 'object'])#命名元组,元素分别为filename和object gb = df.groupby(group)#得到group划分后的list return [data(filename, gb.get_group(x)) for filename, x in zip(gb.groups.keys(), gb.groups)]#zip生成一一对应关系,对每个group对应的块生成data格式的元组,并最终组合成list返回 def create_tf_example(group, path): with tf.gfile.GFile(os.path.join(path, '{}'.format(group.filename)), 'rb') as fid:#以二进制读的方式打开file encoded_jpg = fid.read()#读入filename对应的图片数据 encoded_jpg_io = io.BytesIO(encoded_jpg)#转化为二进制数据 image = Image.open(encoded_jpg_io) width, height = image.size filename = group.filename.encode('utf8')#utf-8编码存储 image_format = b'jpg' xmins = [] xmaxs = [] ymins = [] ymaxs = [] classes_text = [] classes = [] for index, row in group.object.iterrows():#对每个Object操作 xmins.append(row['xmin'] / width) xmaxs.append(row['xmax'] / width) ymins.append(row['ymin'] / height) ymaxs.append(row['ymax'] / height) classes_text.append(row['class'].encode('utf8')) classes.append(class_text_to_int(row['class'])) tf_example = tf.train.Example(features=tf.train.Features(feature={ 'image/height': dataset_util.int64_feature(height), 'image/width': dataset_util.int64_feature(width), 'image/filename': dataset_util.bytes_feature(filename), 'image/source_id': dataset_util.bytes_feature(filename), 'image/encoded': dataset_util.bytes_feature(encoded_jpg), 'image/format': dataset_util.bytes_feature(image_format), 'image/object/bbox/xmin': dataset_util.float_list_feature(xmins), 'image/object/bbox/xmax': dataset_util.float_list_feature(xmaxs), 'image/object/bbox/ymin': dataset_util.float_list_feature(ymins), 'image/object/bbox/ymax': dataset_util.float_list_feature(ymaxs), 'image/object/class/text': dataset_util.bytes_list_feature(classes_text), 'image/object/class/label': dataset_util.int64_list_feature(classes), }))#生成example协议块 return tf_example def main(): writer = tf.python_io.TFRecordWriter(FLAGS.output_path)#创建tfrecord存储器 path = os.path.join(os.getcwd(), 'images')#配置文件所在的路径 examples = pd.read_csv(FLAGS.csv_input)#输入的csv文件所在路径读入 grouped = split(examples, 'filename')#根据filename进行划分,得到list for group in grouped: tf_example = create_tf_example(group, path) writer.write(tf_example.SerializeToString())#压缩example中的map为二进制并写入tfrecord writer.close() output_path = os.path.join(os.getcwd(), FLAGS.output_path) print('Successfully created the TFRecords: {}'.format(output_path)) main()
然后我们配置下labelmap
touch car_label_map.pbtxt
gedit car_label_map.pbtxt
输入以下内容:
item { id: 1 name: 'red' } item { id: 2 name: 'blue' }
基本上完成了训练集和测试集的生成操作。