2021-06-17

tensorflow serving 使用，本文主要参考的是官网的一个分类例子，参考链接：

1. https://tensorflow.google.cn/tfx/tutorials/serving/rest_simple?hl=zh_cn
2. https://github.com/tensorflow/serving
3. https://github.com/tensorflow/serving/blob/master/tensorflow_serving/g3doc/docker.md

1、生成自己SaveModel格式的模型

以fashion mnist为例，训练代码如下：

import tensorflow as tf
from tensorflow import keras

# Helper libraries
import numpy as np
import matplotlib.pyplot as plt
import os
import subprocess

print('TensorFlow version: {}'.format(tf.__version__))

fashion_mnist = keras.datasets.fashion_mnist
(train_images, train_labels), (test_images, test_labels) = fashion_mnist.load_data()

# scale the values to 0.0 to 1.0
train_images = train_images / 255.0
test_images = test_images / 255.0

# reshape for feeding into the model
train_images = train_images.reshape(train_images.shape[0], 28, 28, 1)
test_images = test_images.reshape(test_images.shape[0], 28, 28, 1)

class_names = ['T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat',
               'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot']

print('\ntrain_images.shape: {}, of {}'.format(train_images.shape, train_images.dtype))
print('test_images.shape: {}, of {}'.format(test_images.shape, test_images.dtype))
model = keras.Sequential([
  keras.layers.Conv2D(input_shape=(28,28,1), filters=8, kernel_size=3, 
                      strides=2, activation='relu', name='conv1'),
  keras.layers.Flatten(),
  keras.layers.Dense(10, name='dense')
])
model.summary()

testing = False
epochs = 5

model.compile(optimizer='adam', 
              loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
              metrics=[keras.metrics.SparseCategoricalAccuracy()])

history = model.fit(train_images, train_labels, epochs=epochs)

test_loss, test_acc = model.evaluate(test_images, test_labels)
print('\nTest accuracy: {}'.format(test_acc))

MODEL_DIR = 'models/fashion_mnist'
version = 1
export_path = os.path.join(MODEL_DIR, str(version))
print('export_path = {}\n'.format(export_path))
if not os.path.exists(export_path):
    os.makedirs(export_path)


tf.keras.models.save_model(
    model,
    export_path,
    overwrite=True,
    include_optimizer=True,
    save_format=None,
    signatures=None,
    options=None
)

print('\nSaved model:')
print(os.listdir(export_path))

模型将保存在：

检查模型：

saved_model_cli show --dir /home/mzy/tfserver/models/fashion_mnist/1 --all

输出为：

(tf2) mzy@aiot-gpu-host1:~$ saved_model_cli show --dir /home/mzy/tfserver/models/fashion_mnist/1 --all
2021-06-17 12:58:30.858876: I tensorflow/stream_executor/platform/default/dso_loader.cc:53] Successfully opened dynamic library libcudart.so.1     
1.0
MetaGraphDef with tag-set: 'serve' contains the following SignatureDefs:

signature_def['__saved_model_init_op']:
  The given SavedModel SignatureDef contains the following input(s):
  The given SavedModel SignatureDef contains the following output(s):
    outputs['__saved_model_init_op'] tensor_info:
        dtype: DT_INVALID
        shape: unknown_rank
        name: NoOp
  Method name is:

signature_def['serving_default']:
  The given SavedModel SignatureDef contains the following input(s):
    inputs['conv1_input'] tensor_info:
        dtype: DT_FLOAT
        shape: (-1, 28, 28, 1)
        name: serving_default_conv1_input:0
  The given SavedModel SignatureDef contains the following output(s):
    outputs['dense'] tensor_info:
        dtype: DT_FLOAT
        shape: (-1, 10)
        name: StatefulPartitionedCall:0
  Method name is: tensorflow/serving/predict

Defined Functions:
  Function Name: '__call__'
    Option #1
      Callable with:
        Argument #1
          conv1_input: TensorSpec(shape=(None, 28, 28, 1), dtype=tf.float32, name='conv1_input')
        Argument #2
          DType: bool
          Value: False
        Argument #3
          DType: NoneType
          Value: None
    Option #2
      Callable with:
        Argument #1
          inputs: TensorSpec(shape=(None, 28, 28, 1), dtype=tf.float32, name='inputs')
        Argument #2
          DType: bool
          Value: False
        Argument #3
          DType: NoneType
          Value: None
    Option #3
      Callable with:
        Argument #1
          inputs: TensorSpec(shape=(None, 28, 28, 1), dtype=tf.float32, name='inputs')
        Argument #2
          DType: bool
          Value: True
        Argument #3
          DType: NoneType
          Value: None
    Option #4
      Callable with:
        Argument #1
          conv1_input: TensorSpec(shape=(None, 28, 28, 1), dtype=tf.float32, name='conv1_input')
        Argument #2
          DType: bool
          Value: True
        Argument #3
          DType: NoneType
          Value: None

  Function Name: '_default_save_signature'
    Option #1
      Callable with:
        Argument #1
          conv1_input: TensorSpec(shape=(None, 28, 28, 1), dtype=tf.float32, name='conv1_input')

  Function Name: 'call_and_return_all_conditional_losses'
    Option #1
      Callable with:
        Argument #1
          inputs: TensorSpec(shape=(None, 28, 28, 1), dtype=tf.float32, name='inputs')
        Argument #2
          DType: bool
          Value: True
        Argument #3
          DType: NoneType
          Value: None
    Option #2
      Callable with:
        Argument #1
          inputs: TensorSpec(shape=(None, 28, 28, 1), dtype=tf.float32, name='inputs')
        Argument #2
          DType: bool
          Value: False
        Argument #3
          DType: NoneType
          Value: None
    Option #3
      Callable with:
        Argument #1
          conv1_input: TensorSpec(shape=(None, 28, 28, 1), dtype=tf.float32, name='conv1_input')
        Argument #2
          DType: bool
          Value: True
        Argument #3
          DType: NoneType
          Value: None
    Option #4
      Callable with:
        Argument #1
          conv1_input: TensorSpec(shape=(None, 28, 28, 1), dtype=tf.float32, name='conv1_input')
        Argument #2
          DType: bool
          Value: False
        Argument #3
          DType: NoneType
          Value: None

模

2、安装tensorflow serving

官方推荐使用docker安装：

#cpu版
docker pull tensorflow/serving
#gpu版
docker pull tensorflow/serving:latest-gpu

3、运行tensorflow serving

#cpu
docker run -p 8501:8501 -p 8500:8500  --mount type=bind,source=/home/mzy/tfserver/models/fashion_mnist,target=/models/fashion_mnist -e MODEL_NAME=fashion_mnist -t tensorflow/serving:latest
#gpu
 docker run --gpus all -p 8501:8501 -p 8500:8500  --mount type=bind,source=/home/mzy/tfserver/models/fashion_mnist,target=/models/fashion_mnist -e MODEL_NAME=fashion_mnist -t tensorflow/serving:latest-gpu

对于容器内部执行命令，可以是：

tensorflow_model_server \
  --rest_api_port=8501 \
  --model_name=fashion_model \
  --model_base_path=/home/mzy/tfserver/models/fashion_mnist

4、进行推理

使用图片进行推理，端口8501是http的接口，8500是grpc的接口.http推理如下：

import requests

# Helper libraries
import numpy as np
import matplotlib.pyplot as plt
from PIL import Image
import os
import json
import subprocess
import tensorflow as tf 




fashion_mnist = tf.keras.datasets.fashion_mnist
(train_images, train_labels), (test_images, test_labels) = fashion_mnist.load_data()

# scale the values to 0.0 to 1.0
train_images = train_images / 255.0
test_images = test_images / 255.0

# reshape for feeding into the model
train_images = train_images.reshape(train_images.shape[0], 28, 28, 1)
test_images = test_images.reshape(test_images.shape[0], 28, 28, 1)


def show(idx, title):
  plt.figure()
  plt.imshow(test_images[idx].reshape(28,28))
  plt.axis('off')
  plt.title('\n\n{}'.format(title), fontdict={'size': 16})

def save(idx, title):
  plt.figure()
  plt.imshow(test_images[idx].reshape(28,28))
  plt.axis('off')
  plt.title('\n\n{}'.format(title), fontdict={'size': 16})
  plt.savefig(str(idx)+'.png',dpi=100,)



print('\ntrain_images.shape: {}, of {}'.format(train_images.shape, train_images.dtype))
print('test_images.shape: {}, of {}'.format(test_images.shape, test_images.dtype))

class_names = ['T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat',
               'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot']

# print('\ntrain_images.shape: {}, of {}'.format(train_images.shape, train_images.dtype))
print('test_images.shape: {}, of {}'.format(test_images.shape, test_images.dtype))


data = json.dumps({"signature_name": "serving_default", "instances": test_images[0:3].tolist()})
print('Data: {} ... {}'.format(data[:50], data[len(data)-52:]))
print("Labels: {}".format(test_labels[:3]))

headers = {"content-type": "application/json"}
json_response = requests.post('http://localhost:8501/v1/models/fashion_mnist:predict', data=data, headers=headers)
print("json result:{}".format(json.loads(json_response.text)))
predictions = json.loads(json_response.text)['predictions']

print("predictions{}".format(np.argmax(predictions,axis=1)))

# save(0, 'The model thought this was a {} (class {}), and it was actually a {} (class {})'.format(
#   class_names[np.argmax(predictions[0])], np.argmax(predictions[0]), class_names[test_labels[0]], test_labels[0]))
for id in range(3):
    save(id, 'Predicted{} (class {}), Actually a {} (class {})'.format(
    class_names[np.argmax(predictions[id])], np.argmax(predictions[id]), class_names[test_labels[id]], test_labels[id]))