tensorflow object_detection ssd_mobilenet 以及移植到android系统上教程

         由于有相关的项目要移植到android手机上，我尝试了tensorflow object_detection ssd_mobilenet,效果还行，现在把步骤记录下来，一方面可以作为自己的总结，一方面可以给网友提供以下参考：主要有四个方面：

一.图像数据的标定;

二.把标定的数据转化为tfrecord;

三.tensorflow object_detection ssd_mobilenet api进行训练;

四.移植到android平台上。

 

一.图像数据的标定：

            可以利用labelimg工具，https://github.com/tzutalin/labelImg，很简单，这里就不展开讨论了

二.把标定的数据转化为tfrecord：

   2.1把图像分割成train，valid还有test三部分：

    

#-*-coding:utf-8-*-
import os
import random
import time
import shutil

xmlfilepath = './xml'
saveBasePath = "./annotations"

trainval_percent = 0.9
train_percent = 0.85
total_xml = os.listdir(xmlfilepath)
num = len(total_xml)
list = range(num)
tv = int(num * trainval_percent)
tr = int(tv * train_percent)
trainval = random.sample(list, tv)
train = random.sample(trainval, tr)
print("train and val size", tv)
print("train size", tr)
# print(total_xml[1])
start = time.time()
# print(trainval)
# print(train)
test_num = 0
val_num = 0
train_num = 0
# for directory in ['train','test',"val"]:
#         xml_path = os.path.join(os.getcwd(), 'annotations/{}'.format(directory))
#         if(not os.path.exists(xml_path)):
#             os.mkdir(xml_path)
#         # shutil.copyfile(filePath, newfile)
#         print(xml_path)
for i in list:
    name = total_xml[i]
    # print(i)
    if i in trainval:  # train and val set
        # ftrainval.write(name)
        if i in train:
            # ftrain.write(name)
            # print("train")
            # print(name)
            # print("train: "+name+" "+str(train_num))
            directory = "train"
            train_num += 1
            xml_path = os.path.join(os.getcwd(), 'annotations/{}'.format(directory))
            if (not os.path.exists(xml_path)):
                os.mkdir(xml_path)
            filePath = os.path.join(xmlfilepath, name)
            newfile = os.path.join(saveBasePath, os.path.join(directory, name))
            shutil.copyfile(filePath, newfile)

        else:
            # fval.write(name)
            # print("val")
            # print("val: "+name+" "+str(val_num))
            directory = "validation"
            xml_path = os.path.join(os.getcwd(), 'annotations/{}'.format(directory))
            if (not os.path.exists(xml_path)):
                os.mkdir(xml_path)
            val_num += 1
            filePath = os.path.join(xmlfilepath, name)
            newfile = os.path.join(saveBasePath, os.path.join(directory, name))
            shutil.copyfile(filePath, newfile)
            # print(name)
    else:  # test set
        # ftest.write(name)
        # print("test")
        # print("test: "+name+" "+str(test_num))
        directory = "test"
        xml_path = os.path.join(os.getcwd(), 'annotations/{}'.format(directory))
        if (not os.path.exists(xml_path)):
            os.mkdir(xml_path)
        test_num += 1
        filePath = os.path.join(xmlfilepath, name)
        newfile = os.path.join(saveBasePath, os.path.join(directory, name))
        shutil.copyfile(filePath, newfile)
        # print(name)

# End time
end = time.time()
seconds = end - start
print("train total : " + str(train_num))
print("validation total : " + str(val_num))
print("test total : " + str(test_num))
total_num = train_num + val_num + test_num
print("total number : " + str(total_num))
print("Time taken : {0} seconds".format(seconds))

把上面的代码命名为train_test_split.py，直接命令行跑，annotations文件夹为产生三个文件夹train，valid，test。

   2.2把xml文件转化为csv文件：

import os
import glob
import pandas as pd
import xml.etree.ElementTree as ET


def xml_to_csv(path):
    xml_list = []
    for xml_file in glob.glob(path + '/*.xml'):
        tree = ET.parse(xml_file)
        root = tree.getroot()
        # print(root)
        print(root.find('filename').text)
        for member in root.findall('object'):
            value = (root.find('filename').text,
                     int(root.find('size')[0].text),  # width
                     int(root.find('size')[1].text),  # height
                     member[0].text,
                     int(member[4][0].text),
                     int(float(member[4][1].text)),
                     int(member[4][2].text),
                     int(member[4][3].text)
                     )
            xml_list.append(value)
    column_name = ['filename', 'width', 'height', 'class', 'xmin', 'ymin', 'xmax', 'ymax']
    xml_df = pd.DataFrame(xml_list, columns=column_name)
    return xml_df


def main():
    for directory in ['train', 'test', 'validation']:
        xml_path = os.path.join(os.getcwd(), 'annotations/{}'.format(directory))
        # image_path = os.path.join(os.getcwd(), 'merged_xml')
        xml_df = xml_to_csv(xml_path)
        # xml_df.to_csv('hks.csv', index=None)
        xml_df.to_csv('data/hks_{}_labels.csv'.format(directory), index=None)
        print('Successfully converted xml to csv.')


main()

上述代码命名为xml_to_csv.py。利用2.1生成的annotaitions，通过运行上述代码，将在data文件夹下产生三个.csv文件，分别为hks_train_labels.csv,hks_test_labels.csv还有hks_validation_labels.csv，具体格式如下：

2.3将.csv文件转换成TFRecord文件:

#-*-coding:utf-8-*-
from __future__ import division
from __future__ import print_function
from __future__ import absolute_import

import os
import io
import pandas as pd
import tensorflow as tf
print(tf.__version__)

from PIL import Image
from utils import dataset_util
from collections import namedtuple, OrderedDict

# flags = tf.app.flags
# flags.DEFINE_string('csv_input', '', 'Path to the CSV input')
# flags.DEFINE_string('output_path', '', 'Path to output TFRecord')
# FLAGS = flags.FLAGS


# TO-DO replace this with label map
def class_text_to_int(row_label, filename):
    if row_label == 'cup':
        return 1
    elif row_label == 'tea':
        return 2
    else:
        print("------------------nonetype:", filename)
        None


def split(df, group):
    data = namedtuple('data', ['filename', 'object'])
    gb = df.groupby(group)
    return [data(filename, gb.get_group(x)) for filename, x in zip(gb.groups.keys(), gb.groups)]


def create_tf_example(group, path):
    with tf.gfile.GFile(os.path.join(path, '{}'.format(group.filename)), 'rb') as fid:
        encoded_jpg = fid.read()
    encoded_jpg_io = io.BytesIO(encoded_jpg)
    image = Image.open(encoded_jpg_io)
    width, height = image.size

    filename = group.filename.encode('utf8')
    image_format = b'png'
    xmins = []
    xmaxs = []
    ymins = []
    ymaxs = []
    classes_text = []
    classes = []

    for index, row in group.object.iterrows():
        xmins.append(row['xmin'] / width)
        xmaxs.append(row['xmax'] / width)
        ymins.append(row['ymin'] / height)
        ymaxs.append(row['ymax'] / height)
        classes_text.append(row['class'].encode('utf8'))
        classes.append(class_text_to_int(row['class'], group.filename))

    tf_example = tf.train.Example(features=tf.train.Features(feature={
        'image/height': dataset_util.int64_feature(height),
        'image/width': dataset_util.int64_feature(width),
        'image/filename': dataset_util.bytes_feature(filename),
        'image/source_id': dataset_util.bytes_feature(filename),
        'image/encoded': dataset_util.bytes_feature(encoded_jpg),
        'image/format': dataset_util.bytes_feature(image_format),
        'image/object/bbox/xmin': dataset_util.float_list_feature(xmins),
        'image/object/bbox/xmax': dataset_util.float_list_feature(xmaxs),
        'image/object/bbox/ymin': dataset_util.float_list_feature(ymins),
        'image/object/bbox/ymax': dataset_util.float_list_feature(ymaxs),
        'image/object/class/text': dataset_util.bytes_list_feature(classes_text),
        'image/object/class/label': dataset_util.int64_list_feature(classes),
    }))
    return tf_example


def main():
    writer = tf.python_io.TFRecordWriter("data/hks_test.tfrecord")
    path = os.path.join(os.getcwd(), 'images/test')//如果是train文件，修改image/train
    examples = pd.read_csv('/home/simple/下载/models-master/research/object_detection/ssd_moblile_data/data/hks_test_labels.csv')
    grouped = split(examples, 'filename')
    num = 0
    for group in grouped:
        num += 1
        tf_example = create_tf_example(group, path)
        writer.write(tf_example.SerializeToString())
        if (num % 100 == 0):  # 每完成100个转换，打印一次
            print(num)

    writer.close()
    output_path = os.path.join(os.getcwd(), 'data/train.tfrecord')
    print('Successfully created the TFRecords: {}'.format(output_path))


if __name__ == '__main__':
    main()

其中generate_tfrecord.py是上述代码的命名，输入的.csv文件的路径为'/home/simple/下载/models-master/research/object_detection/ssd_moblile_data/data/hks_test_labels.csv'，图片的存储路径为：path=图像存放的位置（image文件有两个子文件夹train还有test），输出的test文件 test.record.同理，train.record也是同样的操作，只要把hks_test_labels.csv换成hks_train_labels.csv.，最后在data文件夹里面生成train.tfrecord和test.tfrecord.

三.利用 tensorflow object_detection ssd_mobilenet api进行训练

3.1安装tensorflow object_detection ssd_mobilenet api：

git clone https://github.com/tensorflow/tensorflow.git

（1）下载Google Object Detection API

在github上下载项目(https://github.com/tensorflow/models)到本地，解压。

（2）Protobuf编译：

先安转protoc，不要采用apt-get install，从https://github.com/protocolbuffers/protobuf/releases/,下载，我下载了最新版的protoc-3.8.0-linux-x86_64.zip，解压把bin里面的protoc直接copy到  /usr/bin/，以及include直接copy到/usr/local/include/

Tensorflow Object Detection API 使用Protobufs 来配置模型（model）和训练变量（parameters）。切换到目录/models/research/，在终端输入：

cd ~/model-master/models/research/

protoc object_detection/protos/*.proto --python_out=.

 (3)添加环境变量 PYTHONPATH
cd ~/tensorflow/models/research/
export PYTHONPATH=$PYTHONPATH:pwd:pwd/slim

（4）测试API是否安装成功:

在models/research/下运行命令行，得到OK则成功。

python3 object_detection/builders/model_builder_test.py

3.2训练自己的数据：

https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/detection_model_zoo.md

对于检测模型，可以考虑选择几种最常用的模型：
         ssd_mobilenet_v1_coco
         ssd_mobilenet_v2_coco
         faster_rcnn_resnet50_coco
         faster_rcnn_resnet101_coco

    以ssd_mobilenet_v1_coco为例，按照如下步骤进行配置：

   （1）在 object_detection 下建立文件夹 pretrained_model，存放预训练模型
         将下载的文件解压到该目录下（ssd_mobilenet_v1_coco_2018_01_28）

   （2）在 object_detection 下建立文件夹 training，存放训练数据

   （3）修改config文件
         将 /samples/configs/ssd_mobilenet_v1_coco.config 拷贝到training目录，几个可能需要修改的地方：
            - num_classes: 90   # 类别数量
            - batch_size: 24   # 根据机器性能调整
            -  #fine_tune_checkpoint: "PATH_TO_BE_CONFIGURED/model.ckpt"   # 预训练模型位置，可以修改为：
               fine_tune_checkpoint: "training_model/ssd_mobilenet_v1_coco_2018_01_28/model.ckpt"
            - num_steps: 200000   # 需要训练的step
            - train_input_reader & eval_input_reader
               input_path: "data/train.record"   # 以及 test.record
               label_map_path: "data/person_label_map.pbtxt"   # train eval共用label_map
            - num_examples: 8000   # eval样本数量，根据实际修改

    （4）开始训练自己的数据：

python3 object_detection/model_main.py     --pipeline_config_path=/home/simple/models-master/research/object_detection/training/ssd_mobilenet_v1_coco.config     --model_dir=training     --num_train_steps=60000     --num_eval_steps=20     --alsologtostderr

如果命令窗没打印出messgae，那么在model_main.py 加上

tf.logging.set_verbosity(tf.logging.INFO)

      (5)查看tensorboard：

          训练完成后，可以通过 tensorboard 查看训练过程曲线，在 object_detection 目录执行： tensorboard --logdir='training' 

   （6）测试结果：

         a.训练完成后，首先将训练模型导出，这里可以采用自带的 export_inference_graph.py，根据上面的训练曲线，选择loss小的check point，通常可以选择最新的，在 research目录执行：

python3 object_detection/export_inference_graph.py --input_type image_tensor --pipeline_config_path training/ssd_mobilenet_v1_coco.config  --trained_checkpoint_prefix training/model.ckpt-xxxxx  --output_directory person_inference_graph

如果出现ValueError: anchor_strides must be a list with the same length as self._box_specs的错误，以下方法可以解决：

Update! I went into the source code --> tensorflow/models/research/object_detection/anchor_generators/multiple_grid_anchor_generator.py and changed line 100 from:

self._anchor_strides = anchor_strides

to

self._anchor_strides = list(anchor_strides)

I solved this problem by just repeatedly going into multiple_grid_anchor_generator.py and printing the box_specs and anchor_strides until I figured out that the length of the list wasn't the issue, it was actually that anchor_strides was being stored as a zip object rather than a list. :)

    通过 tensorflow 加载, freeze完成后，可以通过原生的tensorflow加载，参考代码：

import numpy as np
import tensorflow as tf
import cv2 as cv
 
# Read the graph.
with tf.gfile.FastGFile('frozen_inference_graph.pb', 'rb') as f:
    graph_def = tf.GraphDef()
    graph_def.ParseFromString(f.read())
 
with tf.Session() as sess:
    # Restore session
    sess.graph.as_default()
    tf.import_graph_def(graph_def, name='')
 
    # Read and preprocess an image.
    img = cv.imread('example.jpg')
    inp = cv.resize(img, (300, 300))
    inp = inp[:, :, [2, 1, 0]]  # BGR2RGB
 
    # Run the model
    out = sess.run([sess.graph.get_tensor_by_name('num_detections:0'),
                    sess.graph.get_tensor_by_name('detection_scores:0'),
                    sess.graph.get_tensor_by_name('detection_boxes:0'),
                    sess.graph.get_tensor_by_name('detection_classes:0')],
                   feed_dict={'image_tensor:0': inp.reshape(1, inp.shape[0], inp.shape[1], 3)})
 
    # Visualize detected bounding boxes.
    num_detections = int(out[0][0])
    for i in range(num_detections):
        classId = int(out[3][0][i])
        score = float(out[1][0][i])
        bbox = [float(v) for v in out[2][0][i]]
        if score > 0.3:
            x = bbox[1] * img.shape[1]
            y = bbox[0] * img.shape[0]
            right = bbox[3] * img.shape[1]
            bottom = bbox[2] * img.shape[0]
            cv.rectangle(img, (int(x), int(y)), (int(right), int(bottom)), (255, 255, 0), thickness=1)
 
cv.imshow('TensorFlow MobileNet-SSD', img)
cv.waitKey()

  b.基于opencv的inference

    建议使用opencv版本4.0.0，因为从这个版本开始，opencv开始支持tensorflow的 Faster RCNN 和 Mask RCNN 模型。

    步骤：

    1）下载opencv源码，需要用到 samples/dnn 下的4个python脚本：
          tf_text_graph_common.py  tf_text_graph_ssd.py  tf_text_graph_faster_rcnn.py  tf_text_graph_mask_rcnn.py

    2）转换pb，生成 pbtxt
         python tf_text_graph_ssd.py --input /path/to/model.pb --config /path/to/example.config --output /path/to/graph.pbtxt

    3）cv2 加载模型
         参考代码：

 import cv2 as cv
 
cvNet = cv.dnn.readNetFromTensorflow('frozen_inference_graph.pb', 'graph.pbtxt')
 
img = cv.imread('example.jpg')
cvNet.setInput(cv.dnn.blobFromImage(img, size=(300, 300), swapRB=True, crop=False))
cvOut = cvNet.forward()
 
for detection in cvOut[0,0,:,:]:
    score = float(detection[2])
    if score > 0.3:
        left = detection[3] * img.shape[1]
        top = detection[4] * img.shape[0]
        right = detection[5] * img.shape[1]
        bottom = detection[6] * img.shape[0]
        cv.rectangle(img, (int(left), int(top)), (int(right), int(bottom)), (255, 255, 0), thickness=1)
 
cv.imshow('img', img)
cv.waitKey()

四.移植到android平台上：

4.1生成tflite的pd模型：

 

object_detection/export_tflite_ssd_graph.py \
 
--pipeline_config_path=$CONFIG_FILE \
 
--trained_checkpoint_prefix=$CHECKPOINT_PATH \
 
--output_directory=$OUTPUT_DIR \
 
--add_postprocessing_op=true

注：#(CONFIG_FILE：模型训练完成后的pipeline.config文件位置)

#(CHECKPOINT_PATH：模型训练完成后生成的.ckpt文件位置，以实际名为准)

#(OUTPUT_DTR：根据自己实际目录用于存放导出的文件)

举个例子，执行object_detection目录下的export_tflite_ssd_graph.py：

cd /home/simple/models-master/research

python3 object_detection/export_tflite_ssd_graph.py --pipeline_config_path=/home/simple/models-master/models/pipeline.config --trained_checkpoint_prefix=/home/simple/models-master/research/training/model.ckpt-49254 --output_directory=/home/simple/models-master/research/data --add_postprocessing_op=true

运行后将在output_directory目录生成tflite_graph.pb 和tflite_graph.pbtxt两个文件。
4.2生成的detect.tflite便可移植至安卓客户端:

有两种方法，一种是直接通过toco工具，一种是通过bazel工具：

a.toco工具

toco   --graph_def_file=tflite_graph.pb   --output_file=optimized_graph.tflite   --input_format=TENSORFLOW_GRAPHDEF   --output_format=TFLITE   --input_shape=1,300,300,3  --input_arrays=normalized_input_image_tensor --output_arrays='TFLite_Detection_PostProcess','TFLite_Detection_PostProcess:1','TFLite_Detection_PostProcess:2','TFLite_Detection_PostProcess:3'  --inference_type=FLOAT --mean_values=128 --std_dev_values=128 --change_concat_input_ranges=false --allow_custom_ops

或者

 toco   --graph_def_file=tflite_graph.pb   --output_file=optimized_graph.lite   --input_format=TENSORFLOW_GRAPHDEF   --output_format=TFLITE   --input_shape=1,300,300,3  --input_arrays=normalized_input_image_tensor --output_arrays='TFLite_Detection_PostProcess','TFLite_Detection_PostProcess:1','TFLite_Detection_PostProcess:2','TFLite_Detection_PostProcess:3'  --inference_type=QUANTIZED_UINT8 --mean_values=128 --std_dev_values=128 --change_concat_input_ranges=false --allow_custom_ops --default_ranges_min=-3 --default_ranges_max=6

b.bazel工具

下载tensorflow工程代码

git clone https://github.com/tensorflow/tensorflow.git

cd tensorflow/   
bazel build tensorflow/contrib/lite/toco:toco

利用bazel生成tflite文件：

bazel-bin/tensorflow/contrib/lite/toco/toco --input_format=TENSORFLOW_GRAPHDEF --output_format=TFLITE --input_file=face/tflite_graph.pb --output_file=detect.tflite --input_shapes=1,320,320,3 --input_arrays=normalized_input_image_tensor --output_arrays='TFLite_Detection_PostProcess','TFLite_Detection_PostProcess:1','TFLite_Detection_PostProcess:2','TFLite_Detection_PostProcess:3' --inference_type=QUANTIZED_UINT8 --mean_values=128.0 --std_values=128.0 --change_concat_input_ranges=false  --allow_custom_ops

 

其中input_shapes=要注意写对了，还有--inference_type是量化的还是Float的，都要注意。

4.3在Android上测试:

下面的目录中tensorflow lite的例子，可以替换原来的detect.tflite文件，修改对应的coco_labels_list.txt文件，建议改成不一样的名称，修改代码，不然运行的时候，detect.tflite和coco_labels_list.txt会重新下载，又被覆盖掉了。

https://github.com/tensorflow/examples

根据自己的模型，修改org/tensorflow/lite/examples/detection/DetectorActivity.java一下，参数：

private static final int TF_OD_API_INPUT_SIZE = 320;（特别注意，要修改为你的ssd一致大小，不然会报错）
private static final boolean TF_OD_API_IS_QUANTIZED = true;（是不是量化模型）
private static final String TF_OD_API_MODEL_FILE = "mydetect.tflite";（自己模型的名称，放在assets文件里面）
private static final String TF_OD_API_LABELS_FILE = "file:///android_asset/mylabelmap.txt";（自己模型的类别名称，放在assets文件里面，注意第一行要加上？？？，再加上类别名，类别名，要跟piple.config一致）

private static final DetectorMode MODE = DetectorMode.TF_OD_API;//不需要修改
// Minimum detection confidence to track a detection.
private static final float MINIMUM_CONFIDENCE_TF_OD_API = 0.1f;//置信度，大了漏捡，小了误捡
private static final boolean MAINTAIN_ASPECT = false;//不需要修改
private static final Size DESIRED_PREVIEW_SIZE = new Size(640, 480);//不需要修改
private static final boolean SAVE_PREVIEW_BITMAP = false;//不需要修改
private static final float TEXT_SIZE_DIP = 10;//不需要修改

还有一处需要修改 TFLiteObjectDetectionAPIModel.jave，

private static final int NUM_DETECTIONS = 10;//根据ssd模型设置

 

如果用到ssd_mobilenetv2的模型，要修改一下代码，因为官方的，是ssd_mobilenetv1的，具体在 TFLiteObjectDetectionAPIModel.jave,如下：

public class TFLiteObjectDetectionAPIModel implements Classifier {
       // in label file and class labels start from 1 to number_of_classes+1,
       // while outputClasses correspond to class index from 0 to number_of_classes
       int labelOffset = 1;
-      recognitions.add(
-          new Recognition(
-              "" + i,
-              labels.get((int) outputClasses[0][i] + labelOffset),
-              outputScores[0][i],
-              detection));
+        final int classLabel = (int) outputClasses[0][i] + labelOffset;
+        if (inRange(classLabel, labels.size(), 0) && inRange(outputScores[0][i], 1, 0)) {
+            recognitions.add(
+                    new Recognition(
+                            "" + i,
+                            labels.get(classLabel),
+                            outputScores[0][i],
+                            detection));
+        }
     }
     Trace.endSection(); // "recognizeImage"
     return recognitions;
   }
 
+  private boolean inRange(float number, float max, float min) {
+    return number < max && number >= min;
+  }
+