由于有相关的项目要移植到android手机上,我尝试了tensorflow object_detection ssd_mobilenet,效果还行,现在把步骤记录下来,一方面可以作为自己的总结,一方面可以给网友提供以下参考:主要有四个方面:
一.图像数据的标定;
二.把标定的数据转化为tfrecord;
三.tensorflow object_detection ssd_mobilenet api进行训练;
四.移植到android平台上。
一.图像数据的标定:
可以利用labelimg工具,https://github.com/tzutalin/labelImg,很简单,这里就不展开讨论了
二.把标定的数据转化为tfrecord:
2.1把图像分割成train,valid还有test三部分:
#-*-coding:utf-8-*-
import os
import random
import time
import shutil
xmlfilepath = './xml'
saveBasePath = "./annotations"
trainval_percent = 0.9
train_percent = 0.85
total_xml = os.listdir(xmlfilepath)
num = len(total_xml)
list = range(num)
tv = int(num * trainval_percent)
tr = int(tv * train_percent)
trainval = random.sample(list, tv)
train = random.sample(trainval, tr)
print("train and val size", tv)
print("train size", tr)
# print(total_xml[1])
start = time.time()
# print(trainval)
# print(train)
test_num = 0
val_num = 0
train_num = 0
# for directory in ['train','test',"val"]:
# xml_path = os.path.join(os.getcwd(), 'annotations/{}'.format(directory))
# if(not os.path.exists(xml_path)):
# os.mkdir(xml_path)
# # shutil.copyfile(filePath, newfile)
# print(xml_path)
for i in list:
name = total_xml[i]
# print(i)
if i in trainval: # train and val set
# ftrainval.write(name)
if i in train:
# ftrain.write(name)
# print("train")
# print(name)
# print("train: "+name+" "+str(train_num))
directory = "train"
train_num += 1
xml_path = os.path.join(os.getcwd(), 'annotations/{}'.format(directory))
if (not os.path.exists(xml_path)):
os.mkdir(xml_path)
filePath = os.path.join(xmlfilepath, name)
newfile = os.path.join(saveBasePath, os.path.join(directory, name))
shutil.copyfile(filePath, newfile)
else:
# fval.write(name)
# print("val")
# print("val: "+name+" "+str(val_num))
directory = "validation"
xml_path = os.path.join(os.getcwd(), 'annotations/{}'.format(directory))
if (not os.path.exists(xml_path)):
os.mkdir(xml_path)
val_num += 1
filePath = os.path.join(xmlfilepath, name)
newfile = os.path.join(saveBasePath, os.path.join(directory, name))
shutil.copyfile(filePath, newfile)
# print(name)
else: # test set
# ftest.write(name)
# print("test")
# print("test: "+name+" "+str(test_num))
directory = "test"
xml_path = os.path.join(os.getcwd(), 'annotations/{}'.format(directory))
if (not os.path.exists(xml_path)):
os.mkdir(xml_path)
test_num += 1
filePath = os.path.join(xmlfilepath, name)
newfile = os.path.join(saveBasePath, os.path.join(directory, name))
shutil.copyfile(filePath, newfile)
# print(name)
# End time
end = time.time()
seconds = end - start
print("train total : " + str(train_num))
print("validation total : " + str(val_num))
print("test total : " + str(test_num))
total_num = train_num + val_num + test_num
print("total number : " + str(total_num))
print("Time taken : {0} seconds".format(seconds))
把上面的代码命名为train_test_split.py,直接命令行跑,annotations文件夹为产生三个文件夹train,valid,test。
2.2把xml文件转化为csv文件:
import os
import glob
import pandas as pd
import xml.etree.ElementTree as ET
def xml_to_csv(path):
xml_list = []
for xml_file in glob.glob(path + '/*.xml'):
tree = ET.parse(xml_file)
root = tree.getroot()
# print(root)
print(root.find('filename').text)
for member in root.findall('object'):
value = (root.find('filename').text,
int(root.find('size')[0].text), # width
int(root.find('size')[1].text), # height
member[0].text,
int(member[4][0].text),
int(float(member[4][1].text)),
int(member[4][2].text),
int(member[4][3].text)
)
xml_list.append(value)
column_name = ['filename', 'width', 'height', 'class', 'xmin', 'ymin', 'xmax', 'ymax']
xml_df = pd.DataFrame(xml_list, columns=column_name)
return xml_df
def main():
for directory in ['train', 'test', 'validation']:
xml_path = os.path.join(os.getcwd(), 'annotations/{}'.format(directory))
# image_path = os.path.join(os.getcwd(), 'merged_xml')
xml_df = xml_to_csv(xml_path)
# xml_df.to_csv('hks.csv', index=None)
xml_df.to_csv('data/hks_{}_labels.csv'.format(directory), index=None)
print('Successfully converted xml to csv.')
main()
上述代码命名为xml_to_csv.py。利用2.1生成的annotaitions,通过运行上述代码,将在data文件夹下产生三个.csv文件,分别为hks_train_labels.csv,hks_test_labels.csv还有hks_validation_labels.csv,具体格式如下:
2.3将.csv文件转换成TFRecord文件:
#-*-coding:utf-8-*-
from __future__ import division
from __future__ import print_function
from __future__ import absolute_import
import os
import io
import pandas as pd
import tensorflow as tf
print(tf.__version__)
from PIL import Image
from utils import dataset_util
from collections import namedtuple, OrderedDict
# flags = tf.app.flags
# flags.DEFINE_string('csv_input', '', 'Path to the CSV input')
# flags.DEFINE_string('output_path', '', 'Path to output TFRecord')
# FLAGS = flags.FLAGS
# TO-DO replace this with label map
def class_text_to_int(row_label, filename):
if row_label == 'cup':
return 1
elif row_label == 'tea':
return 2
else:
print("------------------nonetype:", filename)
None
def split(df, group):
data = namedtuple('data', ['filename', 'object'])
gb = df.groupby(group)
return [data(filename, gb.get_group(x)) for filename, x in zip(gb.groups.keys(), gb.groups)]
def create_tf_example(group, path):
with tf.gfile.GFile(os.path.join(path, '{}'.format(group.filename)), 'rb') as fid:
encoded_jpg = fid.read()
encoded_jpg_io = io.BytesIO(encoded_jpg)
image = Image.open(encoded_jpg_io)
width, height = image.size
filename = group.filename.encode('utf8')
image_format = b'png'
xmins = []
xmaxs = []
ymins = []
ymaxs = []
classes_text = []
classes = []
for index, row in group.object.iterrows():
xmins.append(row['xmin'] / width)
xmaxs.append(row['xmax'] / width)
ymins.append(row['ymin'] / height)
ymaxs.append(row['ymax'] / height)
classes_text.append(row['class'].encode('utf8'))
classes.append(class_text_to_int(row['class'], group.filename))
tf_example = tf.train.Example(features=tf.train.Features(feature={
'image/height': dataset_util.int64_feature(height),
'image/width': dataset_util.int64_feature(width),
'image/filename': dataset_util.bytes_feature(filename),
'image/source_id': dataset_util.bytes_feature(filename),
'image/encoded': dataset_util.bytes_feature(encoded_jpg),
'image/format': dataset_util.bytes_feature(image_format),
'image/object/bbox/xmin': dataset_util.float_list_feature(xmins),
'image/object/bbox/xmax': dataset_util.float_list_feature(xmaxs),
'image/object/bbox/ymin': dataset_util.float_list_feature(ymins),
'image/object/bbox/ymax': dataset_util.float_list_feature(ymaxs),
'image/object/class/text': dataset_util.bytes_list_feature(classes_text),
'image/object/class/label': dataset_util.int64_list_feature(classes),
}))
return tf_example
def main():
writer = tf.python_io.TFRecordWriter("data/hks_test.tfrecord")
path = os.path.join(os.getcwd(), 'images/test')//如果是train文件,修改image/train
examples = pd.read_csv('/home/simple/下载/models-master/research/object_detection/ssd_moblile_data/data/hks_test_labels.csv')
grouped = split(examples, 'filename')
num = 0
for group in grouped:
num += 1
tf_example = create_tf_example(group, path)
writer.write(tf_example.SerializeToString())
if (num % 100 == 0): # 每完成100个转换,打印一次
print(num)
writer.close()
output_path = os.path.join(os.getcwd(), 'data/train.tfrecord')
print('Successfully created the TFRecords: {}'.format(output_path))
if __name__ == '__main__':
main()
其中generate_tfrecord.py是上述代码的命名,输入的.csv文件的路径为'/home/simple/下载/models-master/research/object_detection/ssd_moblile_data/data/hks_test_labels.csv',图片的存储路径为:path=图像存放的位置(image文件有两个子文件夹train还有test),输出的test文件 test.record.同理,train.record也是同样的操作,只要把hks_test_labels.csv换成hks_train_labels.csv.,最后在data文件夹里面生成train.tfrecord和test.tfrecord.
三.利用 tensorflow object_detection ssd_mobilenet api进行训练
3.1安装tensorflow object_detection ssd_mobilenet api:
git clone https://github.com/tensorflow/tensorflow.git
(1)下载Google Object Detection API
在github上下载项目(https://github.com/tensorflow/models)到本地,解压。
(2)Protobuf编译:
先安转protoc,不要采用apt-get install,从https://github.com/protocolbuffers/protobuf/releases/,下载,我下载了最新版的protoc-3.8.0-linux-x86_64.zip,解压把bin里面的protoc直接copy到 /usr/bin/,以及include直接copy到/usr/local/include/
Tensorflow Object Detection API 使用Protobufs 来配置模型(model)和训练变量(parameters)。切换到目录/models/research/,在终端输入:
cd ~/model-master/models/research/
protoc object_detection/protos/*.proto --python_out=.
(3)添加环境变量 PYTHONPATH
cd ~/tensorflow/models/research/
export PYTHONPATH=$PYTHONPATH:pwd:pwd/slim
(4)测试API是否安装成功:
在models/research/下运行命令行,得到OK则成功。
python3 object_detection/builders/model_builder_test.py
3.2训练自己的数据:
对于检测模型,可以考虑选择几种最常用的模型:
ssd_mobilenet_v1_coco
ssd_mobilenet_v2_coco
faster_rcnn_resnet50_coco
faster_rcnn_resnet101_coco
以ssd_mobilenet_v1_coco为例,按照如下步骤进行配置:
(1)在 object_detection 下建立文件夹 pretrained_model,存放预训练模型
将下载的文件解压到该目录下(ssd_mobilenet_v1_coco_2018_01_28)
(2)在 object_detection 下建立文件夹 training,存放训练数据
(3)修改config文件
将 /samples/configs/ssd_mobilenet_v1_coco.config 拷贝到training目录,几个可能需要修改的地方:
- num_classes: 90 # 类别数量
- batch_size: 24 # 根据机器性能调整
- #fine_tune_checkpoint: "PATH_TO_BE_CONFIGURED/model.ckpt" # 预训练模型位置,可以修改为:
fine_tune_checkpoint: "training_model/ssd_mobilenet_v1_coco_2018_01_28/model.ckpt"
- num_steps: 200000 # 需要训练的step
- train_input_reader & eval_input_reader
input_path: "data/train.record" # 以及 test.record
label_map_path: "data/person_label_map.pbtxt" # train eval共用label_map
- num_examples: 8000 # eval样本数量,根据实际修改
(4)开始训练自己的数据:
python3 object_detection/model_main.py --pipeline_config_path=/home/simple/models-master/research/object_detection/training/ssd_mobilenet_v1_coco.config --model_dir=training --num_train_steps=60000 --num_eval_steps=20 --alsologtostderr
如果命令窗没打印出messgae,那么在model_main.py 加上
tf.logging.set_verbosity(tf.logging.INFO)
(5)查看tensorboard:
训练完成后,可以通过 tensorboard 查看训练过程曲线,在 object_detection 目录执行: tensorboard --logdir='training'
(6)测试结果:
a.训练完成后,首先将训练模型导出,这里可以采用自带的 export_inference_graph.py,根据上面的训练曲线,选择loss小的check point,通常可以选择最新的,在 research目录执行:
python3 object_detection/export_inference_graph.py --input_type image_tensor --pipeline_config_path training/ssd_mobilenet_v1_coco.config --trained_checkpoint_prefix training/model.ckpt-xxxxx --output_directory person_inference_graph
如果出现ValueError: anchor_strides must be a list with the same length as self._box_specs的错误,以下方法可以解决:
Update! I went into the source code --> tensorflow/models/research/object_detection/anchor_generators/multiple_grid_anchor_generator.py
and changed line 100 from:
self._anchor_strides = anchor_strides
to
self._anchor_strides = list(anchor_strides)
I solved this problem by just repeatedly going into multiple_grid_anchor_generator.py and printing the box_specs and anchor_strides until I figured out that the length of the list wasn't the issue, it was actually that anchor_strides was being stored as a zip object rather than a list. :)
通过 tensorflow 加载, freeze完成后,可以通过原生的tensorflow加载,参考代码:
import numpy as np
import tensorflow as tf
import cv2 as cv
# Read the graph.
with tf.gfile.FastGFile('frozen_inference_graph.pb', 'rb') as f:
graph_def = tf.GraphDef()
graph_def.ParseFromString(f.read())
with tf.Session() as sess:
# Restore session
sess.graph.as_default()
tf.import_graph_def(graph_def, name='')
# Read and preprocess an image.
img = cv.imread('example.jpg')
inp = cv.resize(img, (300, 300))
inp = inp[:, :, [2, 1, 0]] # BGR2RGB
# Run the model
out = sess.run([sess.graph.get_tensor_by_name('num_detections:0'),
sess.graph.get_tensor_by_name('detection_scores:0'),
sess.graph.get_tensor_by_name('detection_boxes:0'),
sess.graph.get_tensor_by_name('detection_classes:0')],
feed_dict={'image_tensor:0': inp.reshape(1, inp.shape[0], inp.shape[1], 3)})
# Visualize detected bounding boxes.
num_detections = int(out[0][0])
for i in range(num_detections):
classId = int(out[3][0][i])
score = float(out[1][0][i])
bbox = [float(v) for v in out[2][0][i]]
if score > 0.3:
x = bbox[1] * img.shape[1]
y = bbox[0] * img.shape[0]
right = bbox[3] * img.shape[1]
bottom = bbox[2] * img.shape[0]
cv.rectangle(img, (int(x), int(y)), (int(right), int(bottom)), (255, 255, 0), thickness=1)
cv.imshow('TensorFlow MobileNet-SSD', img)
cv.waitKey()
b.基于opencv的inference
建议使用opencv版本4.0.0,因为从这个版本开始,opencv开始支持tensorflow的 Faster RCNN 和 Mask RCNN 模型。
步骤:
1)下载opencv源码,需要用到 samples/dnn 下的4个python脚本:
tf_text_graph_common.py tf_text_graph_ssd.py tf_text_graph_faster_rcnn.py tf_text_graph_mask_rcnn.py
2)转换pb,生成 pbtxt
python tf_text_graph_ssd.py --input /path/to/model.pb --config /path/to/example.config --output /path/to/graph.pbtxt
3)cv2 加载模型
参考代码:
import cv2 as cv
cvNet = cv.dnn.readNetFromTensorflow('frozen_inference_graph.pb', 'graph.pbtxt')
img = cv.imread('example.jpg')
cvNet.setInput(cv.dnn.blobFromImage(img, size=(300, 300), swapRB=True, crop=False))
cvOut = cvNet.forward()
for detection in cvOut[0,0,:,:]:
score = float(detection[2])
if score > 0.3:
left = detection[3] * img.shape[1]
top = detection[4] * img.shape[0]
right = detection[5] * img.shape[1]
bottom = detection[6] * img.shape[0]
cv.rectangle(img, (int(left), int(top)), (int(right), int(bottom)), (255, 255, 0), thickness=1)
cv.imshow('img', img)
cv.waitKey()
四.移植到android平台上:
4.1生成tflite的pd模型:
object_detection/export_tflite_ssd_graph.py \
--pipeline_config_path=$CONFIG_FILE \
--trained_checkpoint_prefix=$CHECKPOINT_PATH \
--output_directory=$OUTPUT_DIR \
--add_postprocessing_op=true
注:#(CONFIG_FILE:模型训练完成后的pipeline.config文件位置)
#(CHECKPOINT_PATH:模型训练完成后生成的.ckpt文件位置,以实际名为准)
#(OUTPUT_DTR:根据自己实际目录用于存放导出的文件)
举个例子,执行object_detection目录下的export_tflite_ssd_graph.py:
cd /home/simple/models-master/research
python3 object_detection/export_tflite_ssd_graph.py --pipeline_config_path=/home/simple/models-master/models/pipeline.config --trained_checkpoint_prefix=/home/simple/models-master/research/training/model.ckpt-49254 --output_directory=/home/simple/models-master/research/data --add_postprocessing_op=true
运行后将在output_directory目录生成tflite_graph.pb 和tflite_graph.pbtxt两个文件。
4.2生成的detect.tflite便可移植至安卓客户端:
有两种方法,一种是直接通过toco工具,一种是通过bazel工具:
a.toco工具
toco --graph_def_file=tflite_graph.pb --output_file=optimized_graph.tflite --input_format=TENSORFLOW_GRAPHDEF --output_format=TFLITE --input_shape=1,300,300,3 --input_arrays=normalized_input_image_tensor --output_arrays='TFLite_Detection_PostProcess','TFLite_Detection_PostProcess:1','TFLite_Detection_PostProcess:2','TFLite_Detection_PostProcess:3' --inference_type=FLOAT --mean_values=128 --std_dev_values=128 --change_concat_input_ranges=false --allow_custom_ops
或者
toco --graph_def_file=tflite_graph.pb --output_file=optimized_graph.lite --input_format=TENSORFLOW_GRAPHDEF --output_format=TFLITE --input_shape=1,300,300,3 --input_arrays=normalized_input_image_tensor --output_arrays='TFLite_Detection_PostProcess','TFLite_Detection_PostProcess:1','TFLite_Detection_PostProcess:2','TFLite_Detection_PostProcess:3' --inference_type=QUANTIZED_UINT8 --mean_values=128 --std_dev_values=128 --change_concat_input_ranges=false --allow_custom_ops --default_ranges_min=-3 --default_ranges_max=6
b.bazel工具
下载tensorflow工程代码
git clone https://github.com/tensorflow/tensorflow.git
cd tensorflow/
bazel build tensorflow/contrib/lite/toco:toco
利用bazel生成tflite文件:
bazel-bin/tensorflow/contrib/lite/toco/toco --input_format=TENSORFLOW_GRAPHDEF --output_format=TFLITE --input_file=face/tflite_graph.pb --output_file=detect.tflite --input_shapes=1,320,320,3 --input_arrays=normalized_input_image_tensor --output_arrays='TFLite_Detection_PostProcess','TFLite_Detection_PostProcess:1','TFLite_Detection_PostProcess:2','TFLite_Detection_PostProcess:3' --inference_type=QUANTIZED_UINT8 --mean_values=128.0 --std_values=128.0 --change_concat_input_ranges=false --allow_custom_ops
其中input_shapes=要注意写对了,还有--inference_type是量化的还是Float的,都要注意。
4.3在Android上测试:
下面的目录中tensorflow lite的例子,可以替换原来的detect.tflite
文件,修改对应的coco_labels_list.txt
文件,建议改成不一样的名称,修改代码,不然运行的时候,detect.tflite
和coco_labels_list.txt
会重新下载,又被覆盖掉了。
https://github.com/tensorflow/examples
根据自己的模型,修改org/tensorflow/lite/examples/detection/DetectorActivity.java一下,参数:
private static final int TF_OD_API_INPUT_SIZE = 320;(特别注意,要修改为你的ssd一致大小,不然会报错)
private static final boolean TF_OD_API_IS_QUANTIZED = true;(是不是量化模型)
private static final String TF_OD_API_MODEL_FILE = "mydetect.tflite";(自己模型的名称,放在assets文件里面)
private static final String TF_OD_API_LABELS_FILE = "file:///android_asset/mylabelmap.txt";(自己模型的类别名称,放在assets文件里面,注意第一行要加上???,再加上类别名,类别名,要跟piple.config一致)
private static final DetectorMode MODE = DetectorMode.TF_OD_API;//不需要修改
// Minimum detection confidence to track a detection.
private static final float MINIMUM_CONFIDENCE_TF_OD_API = 0.1f;//置信度,大了漏捡,小了误捡
private static final boolean MAINTAIN_ASPECT = false;//不需要修改
private static final Size DESIRED_PREVIEW_SIZE = new Size(640, 480);//不需要修改
private static final boolean SAVE_PREVIEW_BITMAP = false;//不需要修改
private static final float TEXT_SIZE_DIP = 10;//不需要修改
还有一处需要修改 TFLiteObjectDetectionAPIModel.jave,
private static final int NUM_DETECTIONS = 10;//根据ssd模型设置
如果用到ssd_mobilenetv2的模型,要修改一下代码,因为官方的,是ssd_mobilenetv1的,具体在 TFLiteObjectDetectionAPIModel.jave,如下:
public class TFLiteObjectDetectionAPIModel implements Classifier {
// in label file and class labels start from 1 to number_of_classes+1,
// while outputClasses correspond to class index from 0 to number_of_classes
int labelOffset = 1;
- recognitions.add(
- new Recognition(
- "" + i,
- labels.get((int) outputClasses[0][i] + labelOffset),
- outputScores[0][i],
- detection));
+ final int classLabel = (int) outputClasses[0][i] + labelOffset;
+ if (inRange(classLabel, labels.size(), 0) && inRange(outputScores[0][i], 1, 0)) {
+ recognitions.add(
+ new Recognition(
+ "" + i,
+ labels.get(classLabel),
+ outputScores[0][i],
+ detection));
+ }
}
Trace.endSection(); // "recognizeImage"
return recognitions;
}
+ private boolean inRange(float number, float max, float min) {
+ return number < max && number >= min;
+ }
+