tensorflow serving+docker部署mnist模型以及gRPC调用

        在上一篇博客中，我在自己的服务器上通过docker安装了tensorflow serving的GPU镜像环境，然后通过tensorflow serving源码中的简单的数值相加的模型进行了一个测试，docker tensoflow serving环境安装成功。接下来，可以使用tensorflow serving来部署mnist模型以及调用。

第一步：训练mnist并保存模型

使用keras框架，搭建卷积网络训练（这里为了不需要到处找，给出我自己找的源码）

# This was directly taken from Keras
'''Trains a simple convnet on the MNIST dataset.
Gets to 99.25% test accuracy after 12 epochs
(there is still a lot of margin for parameter tuning).
16 seconds per epoch on a GRID K520 GPU.
'''

from __future__ import print_function
import keras
from keras.datasets import mnist
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten
from keras.layers import Conv2D, MaxPooling2D
from keras import backend as K

batch_size = 128
num_classes = 10
epochs = 12

# input image dimensions
img_rows, img_cols = 28, 28

# the data, shuffled and split between train and test sets
(x_train, y_train), (x_test, y_test) = mnist.load_data()

if K.image_data_format() == 'channels_first':
    x_train = x_train.reshape(x_train.shape[0], 1, img_rows, img_cols)
    x_test = x_test.reshape(x_test.shape[0], 1, img_rows, img_cols)
    input_shape = (1, img_rows, img_cols)
else:
    x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, 1)
    x_test = x_test.reshape(x_test.shape[0], img_rows, img_cols, 1)
    input_shape = (img_rows, img_cols, 1)

x_train = x_train.astype('float32')
x_test = x_test.astype('float32')
x_train /= 255
x_test /= 255
print('x_train shape:', x_train.shape)
print(x_train.shape[0], 'train samples')
print(x_test.shape[0], 'test samples')

# convert class vectors to binary class matrices
y_train = keras.utils.to_categorical(y_train, num_classes)
y_test = keras.utils.to_categorical(y_test, num_classes)

model = Sequential()
model.add(Conv2D(32, kernel_size=(3, 3),
                 activation='relu',
                 input_shape=input_shape))
model.add(Conv2D(64, (3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(128, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(num_classes, activation='softmax'))

model.compile(loss=keras.losses.categorical_crossentropy,
              optimizer=keras.optimizers.Adadelta(),
              metrics=['accuracy'])

model.fit(x_train, y_train,
          batch_size=batch_size,
          epochs=epochs,
          verbose=1,
          validation_data=(x_test, y_test))
score = model.evaluate(x_test, y_test, verbose=0)

model.save('mnist.hdf5')
print('Test loss:', score[0])
print('Test accuracy:', score[1])

 最后会得到一个keras的mnist.hdf5模型文件

第二步：导出模型（生成符合Server要求的模型文件，主要是用builder加一下signature_def的key和method_name）（同样给出别人的源码，其他模型的导出是一样的）

#!/usr/bin/python
# (c)-2017 Amiya Patanaik amiyain@gmail.com
# Licensed under GPL v3

from keras import backend as K
from keras.models import load_model
from tensorflow.python.saved_model import builder as saved_model_builder
from tensorflow.python.saved_model import tag_constants, signature_constants
from tensorflow.python.saved_model.signature_def_utils_impl import build_signature_def, predict_signature_def
import shutil, os, argparse

# reset learning phase
K.set_learning_phase(0)

# capture commandline arguments
parser = argparse.ArgumentParser(description='Export Keras models to Tensorflow Serving format.')
parser.add_argument('-p','--path',default='mnist.hdf5', help='path to Keras model file', required=True)
parser.add_argument('-n','--name',default='mnist_cnn', help='model name', required=True)
parser.add_argument('-v','--version',default='1', help='version number', required=True)

args = vars(parser.parse_args())

export_version = args['version']
model_name = args['name']
model_path = args['path']

# loading models
model = load_model(model_path)

export_model_path = "models/" + model_name  + "/" + export_version 

# export models

builder = saved_model_builder.SavedModelBuilder(export_model_path)


# if there are multiple inputs/outputs
# signature_model = predict_signature_def(inputs={"input1": model.input[0], "input2": model.input[1]}, outputs={"output": model.output})

# please note that what you call the input and output is important
# the same keys must be used in the client as well
signature = predict_signature_def(inputs={'input': model.input},
                                  outputs={'output': model.output})

with K.get_session() as sess:
    builder.add_meta_graph_and_variables(sess=sess,
                                         tags=[tag_constants.SERVING],
                                         signature_def_map={'predict': signature})
    builder.save()

命令行： python ***.py，运行之后，会在当前目录下有一个 mnist_cnn文件夹，文件夹中内容如下：

第三步：启动服务，加载模型

参考上篇博客，就是将模型地址与名称改变下即可

docker run --runtime=nvidia -p 8501:8501 \

--mount type=bind,\

source=/home/path/to/mnist_cnn,\

target=/models/mnist \

-e MODEL_NAME=mnist, -t tensorflow/serving:1.12.0-gpu &

注：红字处为你服务器中docker环境下的tensorflow serving镜像名。

第四步：客户端申请服务

在调用服务之前可以使用saved_model_cli命令查看模型输入输出：

saved_model_cli show --dir /home/path/to/mnist_cnn/1/ --all

结果如下：

可以看到模型的输入是28*28*1的图像，输出是softmax

然后可以通过gRPC调用服务

#!/usr/bin/python
# This is directly adapted from:
# https://github.com/tobegit3hub/tensorflow_template_application/tree/master/python_predict_client

import numpy
#from keras.datasets import mnist
from grpc.beta import implementations
import tensorflow as tf

from tensorflow_serving.apis import predict_pb2
from tensorflow_serving.apis import prediction_service_pb2

from keras.datasets import mnist

from PIL import Image
'''

from tensorflow_serving.apis import predict_pb2
from tensorflow_serving.apis import prediction_service_pb2
'''

tf.app.flags.DEFINE_string("host", "172.16.102.35", "gRPC server host")
tf.app.flags.DEFINE_integer("port", 8500, "gRPC server port")
tf.app.flags.DEFINE_string("model_name", "mnist", "TensorFlow model name")
tf.app.flags.DEFINE_integer("model_version", 1, "TensorFlow model version")
tf.app.flags.DEFINE_float("request_timeout", 20.0, "Timeout of gRPC request")

#tf.app.flags.DEFINE_string("image", 'test.jpg', "Timeout of gRPC request")
FLAGS = tf.app.flags.FLAGS


(x_train, y_train), (x_test, y_test) = mnist.load_data()

print(numpy.shape(x_train))
idx = 2324
img = x_train[idx,:,:]
label = y_train[idx]
img = numpy.asarray(numpy.resize(img, (1, 28, 28, 1)))
print(label)

'''
img = Image.open(FLAGS.image)
img = img.resize([224,224])
img = numpy.asarray(img)
img = numpy.resize(img, (1, 224, 224, 3))
print(img.shape)
'''
def main():
  
    host = FLAGS.host
    port = FLAGS.port
    model_name = FLAGS.model_name
    model_version = FLAGS.model_version
    request_timeout = FLAGS.request_timeout
    
    # Generate inference data
    keys = numpy.asarray([1, 2, 3])
    keys_tensor_proto = tf.contrib.util.make_tensor_proto(keys, dtype=tf.int32)
    features_tensor_proto = tf.contrib.util.make_tensor_proto(img,dtype=tf.float32,shape=img.shape)
    
    # Create gRPC client and request
    channel = implementations.insecure_channel(host, port)
    stub = prediction_service_pb2.beta_create_PredictionService_stub(channel)
    request = predict_pb2.PredictRequest()
    request.model_spec.name = model_name
    if model_version > 0:
        request.model_spec.version.value = model_version
        request.inputs['input'].CopyFrom(features_tensor_proto)
        request.model_spec.signature_name = 'predict'
        #print(request)
        #request.inputs['features'].CopyFrom(features_tensor_proto)
        
        # Send request
        
        result = stub.Predict(request, request_timeout)
        #result = result.result()
        
        response = numpy.array(result.outputs['output'].float_val)
        prediction = numpy.argmax(response)
        
        print(result)
        print(prediction)


if __name__ == '__main__':
  main()

结果如下：

 

 

如果该博客对你有用，点个赞（*—*）