tensorflow制作自己的VOC2018数据集

 

下载图片

从这里的标签制作文件夹下拿到，jpg图片，xml标签数据，l
由于先前的数据（人脸识别）文件名字中含有空格，在进行xml解析式用的是str().split(‘ ’）切分所以那批数据僵硬了。在VOC2007截取了50张图片进行这个测试，以便在对超大型数据进行制作record文件会有点帮助。从jpg-->csv-->record的路有点长，所以改为jpg-->record，可能会快一点吧。

首先模仿VOC2007创建几个文件夹

--my_data
      |--Annotations ——xml标注文件
      |--ImageSets ——类别标签
          |--Main  ———train.txt,等文件
      |--JPEGImages ——jpg图像文件
          

1.把图片copy到JPEGImages 目录
2.把xml文件copy到Annotations目录
3.把 lale_map.pbtxt 放到当前目录
如下：

 

my_data目录

 

creat_name.py如下，这是生成train.txt，val.txt文件的程序。
修改一下目录和训练集比例（=0.5两个数据集是一样大），就生成可以train.txt,val.txt,test.txt文件。

#!/usr/bin/env python
import os
import random
import os
trainval_percent = 1 # trainval数据集占所有数据的比例
train_percent = 0.5 # train数据集占trainval数据的比例
xmlfilepath = 'Annotations'
txtsavepath = 'ImageSets\Main'
total_xml = os.listdir(xmlfilepath)
num=len(total_xml)
list=range(num)
tv=int(num*trainval_percent)
tr=int(tv*train_percent)
trainval= random.sample(list,tv)
train=random.sample(trainval,tr)
ftrainval = open('ImageSets/Main/trainval.txt', 'w')
ftest = open('ImageSets/Main/test.txt', 'w')
ftrain = open('ImageSets/Main/train.txt', 'w')
fval = open('ImageSets/Main/val.txt', 'w')
for i in list:
    name=total_xml[i][:-4]+'\n'
    if i in trainval:
        ftrainval.write(name)
        if i in train:
            ftrain.write(name)
        else:
            fval.write(name)
    else:
        ftest.write(name)
ftrainval.close()
ftrain.close()
fval.close()
ftest .close()

制作record文件

这一步需要借助object_detetion的API
从object_detection\dataset_tools下把create_pascal_tf_record.py文件复制到最外层目录文件夹下命名为creat_my_data_tf_record.py。如下：

 

creat_my_data_tf_record.py

 

我的数据在这里：

 

my_data

修改下代码，去掉了Year，变成my_data:

#creat_my_data_tf_record.py
import hashlib
import io
import logging
import os

from lxml import etree
import PIL.Image
import tensorflow as tf

from tensorflow.models.research.object_detection.utils import dataset_util
from tensorflow.models.research.object_detection.utils import label_map_util
#from tensorflow.models.research.object_detection.utils import label_map_util
#
#from tensorflow.models.research.object_detection.utils import visualization_utils as vis_util


flags = tf.app.flags
flags.DEFINE_string('data_dir', '', 'Root directory to raw PASCAL VOC dataset.')
flags.DEFINE_string('set', 'train', 'Convert training set, validation set or '
                    'merged set.')
flags.DEFINE_string('annotations_dir', 'Annotations',
                    '(Relative) path to annotations directory.')
flags.DEFINE_string('output_path', '', 'Path to output TFRecord')
flags.DEFINE_string('label_map_path', 'VOCdevkit/pascal_label_map.pbtxt',
                    'Path to label map proto')
flags.DEFINE_boolean('ignore_difficult_instances', False, 'Whether to ignore '
                     'difficult instances')
FLAGS = flags.FLAGS

SETS = ['train', 'val', 'trainval', 'test']


def dict_to_tf_example(data,
                       dataset_directory,
                       label_map_dict,
                       ignore_difficult_instances=False,
                       image_subdirectory='JPEGImages'):
  """Convert XML derived dict to tf.Example proto.

  Notice that this function normalizes the bounding box coordinates provided
  by the raw data.

  Args:
    data: dict holding PASCAL XML fields for a single image (obtained by
      running dataset_util.recursive_parse_xml_to_dict)
    dataset_directory: Path to root directory holding PASCAL dataset
    label_map_dict: A map from string label names to integers ids.
    ignore_difficult_instances: Whether to skip difficult instances in the
      dataset  (default: False).
    image_subdirectory: String specifying subdirectory within the
      PASCAL dataset directory holding the actual image data.

  Returns:
    example: The converted tf.Example.

  Raises:
    ValueError: if the image pointed to by data['filename'] is not a valid JPEG
  """
  img_path = os.path.join(data['folder'], image_subdirectory, data['filename'])
  full_path = os.path.join(dataset_directory, img_path)
  with tf.gfile.GFile(full_path, 'rb') as fid:
    encoded_jpg = fid.read()
  encoded_jpg_io = io.BytesIO(encoded_jpg)
  image = PIL.Image.open(encoded_jpg_io)
  if image.format != 'JPEG':
    raise ValueError('Image format not JPEG')
  key = hashlib.sha256(encoded_jpg).hexdigest()

  width = int(data['size']['width'])
  height = int(data['size']['height'])

  xmin = []
  ymin = []
  xmax = []
  ymax = []
  classes = []
  classes_text = []
  truncated = []
  poses = []
  difficult_obj = []
  for obj in data['object']:
    difficult = bool(int(obj['difficult']))
    if ignore_difficult_instances and difficult:
      continue

    difficult_obj.append(int(difficult))

    xmin.append(float(obj['bndbox']['xmin']) / width)
    ymin.append(float(obj['bndbox']['ymin']) / height)
    xmax.append(float(obj['bndbox']['xmax']) / width)
    ymax.append(float(obj['bndbox']['ymax']) / height)
    classes_text.append(obj['name'].encode('utf8'))
    classes.append(label_map_dict[obj['name']])
    truncated.append(int(obj['truncated']))
    poses.append(obj['pose'].encode('utf8'))

  example = tf.train.Example(features=tf.train.Features(feature={
      'image/height': dataset_util.int64_feature(height),
      'image/width': dataset_util.int64_feature(width),
      'image/filename': dataset_util.bytes_feature(
          data['filename'].encode('utf8')),
      'image/source_id': dataset_util.bytes_feature(
          data['filename'].encode('utf8')),
      'image/key/sha256': dataset_util.bytes_feature(key.encode('utf8')),
      'image/encoded': dataset_util.bytes_feature(encoded_jpg),
      'image/format': dataset_util.bytes_feature('jpeg'.encode('utf8')),
      'image/object/bbox/xmin': dataset_util.float_list_feature(xmin),
      'image/object/bbox/xmax': dataset_util.float_list_feature(xmax),
      'image/object/bbox/ymin': dataset_util.float_list_feature(ymin),
      'image/object/bbox/ymax': dataset_util.float_list_feature(ymax),
      'image/object/class/text': dataset_util.bytes_list_feature(classes_text),
      'image/object/class/label': dataset_util.int64_list_feature(classes),
      'image/object/difficult': dataset_util.int64_list_feature(difficult_obj),
      'image/object/truncated': dataset_util.int64_list_feature(truncated),
      'image/object/view': dataset_util.bytes_list_feature(poses),
  }))
  return example


def main(_):
  if FLAGS.set not in SETS:
    raise ValueError('set must be in : {}'.format(SETS))

  data_dir = FLAGS.data_dir
  datasets = ['my_data']

  writer = tf.python_io.TFRecordWriter(FLAGS.output_path)

  label_map_dict = label_map_util.get_label_map_dict(FLAGS.label_map_path)

  for dataset in datasets:
    logging.info('Reading from PASCAL %s dataset.', dataset)
    examples_path = os.path.join(data_dir, dataset, 'ImageSets', 'Main/' + FLAGS.set + '.txt')
    annotations_dir = os.path.join(data_dir, dataset, FLAGS.annotations_dir)
    examples_list = dataset_util.read_examples_list(examples_path)
    for idx, example in enumerate(examples_list):
      if idx % 100 == 0:
        logging.info('On image %d of %d', idx, len(examples_list))
      path = os.path.join(annotations_dir, example + '.xml')
      with tf.gfile.GFile(path, 'r') as fid:
        xml_str = fid.read()
      xml = etree.fromstring(xml_str)
      data = dataset_util.recursive_parse_xml_to_dict(xml)['annotation']

      tf_example = dict_to_tf_example(data, FLAGS.data_dir, label_map_dict,
                                      FLAGS.ignore_difficult_instances)
      writer.write(tf_example.SerializeToString())

  writer.close()


if __name__ == '__main__':
  tf.app.run()

运行create_my_data_tf_record.py

python create_my_data_tf_record.py --data_dir=VOCdevkit/  --set=train --output_path=VOCdevkit/my_train.record  --label_map_path ./VOCdevkit/data/label_map.pbtxt

python create_my_data_tf_record.py --data_dir=VOCdevkit/  --set=val --output_path=VOCdevkit/my_val.record  --label_map_path ./VOCdevkit/data/label_map.pbtxt

如下图：

 

得到record