YOLO v4在视频监控云中应用实践

作者 伍增田 wu Tommy (tommywu@meshare.cn)

为了更好地服务用户, 提升用户的体验; 摄像头发生告警时, 需要识别人/ 车/ 宠物/ 快递包裹, 将相关视频和图像收集到数据中心的视频云平台, 集中进行对象检测识别的
使用ffmpeg对视频文件h264/h265解析出jpg图片, 通过http接口送给AI GPU服务器进行对象检测.
2台AI GPU服务器, centos7 , gpu硬件 Tesla P40:, 服务30万摄像头.

环境构建过程

环境配置
显卡驱动安装
前提准备

1 安装环境依赖

yum install kernel-devel gcc -y
检查内核版本和源码版本，保证一致

ls /boot | grep vmlinu

rpm -aq | grep kernel-devel
屏蔽系统自带的nouveau

查看命令：
lsmod | grep nouveau

修改dist-blacklist.conf文件：
vim /lib/modprobe.d/dist-blacklist.conf

将nvidiafb注释掉:
#blacklist nvidiafb

然后添加以下语句：
blacklist nouveau
options nouveau modeset=0

重建initramfs image步骤

mv /boot/initramfs-$(uname-r).img/boot/initramfs-$(uname -r).img.bak

dracut /boot/initramfs-$(uname -r).img $(uname -r)
修改运行级别为文本模式

systemctl set-default multi-user.target
重新启动

reboot
2 本地安装

在NVIDIA官网下载Tesla P40驱动

网址：https://www.nvidia.cn/Download/index.aspx?lang=cn

安装过程

chmod +x NVIDIA-Linux-x86_64-410.129-diagnostic.run

./NVIDIA-Linux-x86_64-410.129-diagnostic.run
安装成功

nvidia-smi

3 tensorflow-gpu + anaconda + python3.6 + CUDA + CUDNN 部署
安装anaconda

使用清华的源下载安装：https://mirror.tuna.tsinghua.edu.cn/help/anaconda/
Anaconda3-4.4.0-Linux-x86_64.sh

配置python3.6的环境(因为tensorflow-gpu目前支持的最高版本是python3.6)

使用 conda create -n py36 python=3.6

激活环境

source activate py36

cuda安装

去官网https://developer.nvidia.com/cuda-toolkit-archive寻找对应的版本下载

下载完成后进入对应目录

sh cuda_10.0.130_410.48_linux
配置环境变量

编辑~/.bashrc文件

vim ~/.bashrc
#末尾添加以下内容
export PATH=/usr/local/cuda/binKaTeX parse error: Expected '}', got 'EOF' at end of input: {PATH:+:{PATH}}
export LD_LIBRARY_PATH=/usr/local/cuda/lib64KaTeX parse error: Expected '}', got 'EOF' at end of input: …LIBRARY_PATH:+:{LD_LIBRARY_PATH}}
编辑/etc/profile

vim /etc/profile
#末尾添加以下内容
export PATH=$PATH:/usr/local/cuda/binexportL{D}_{L}IBRAR{Y}_{P}ATH=$LD_LIBRARY_PATH:/usr/local/cuda/lib64
export LIBRARY_PATH=$LIBRARY_PATH:/usr/local/cuda/lib64
使配置文件生效

source /etc/profile
安装CUDNN驱动

去官网https://developer.nvidia.com/rdp/cudnn-archive下载对应版本

安装

cp cudnn-10.0-linux-x64-v7.6.5.32.tgz
tar -zxvf cudnn-10.0-linux-x64-v7.6.5.32.tgz
cd cuda
cp include/* /usr/local/cuda
cp lib64/* /usr/local/cuda/lib64
查看cudnn版本

cat /usr/local/cuda/cudnn.h | grep CUDNN_MAJOR -A 2
e245da29f03106928416fe0624683bf4

安装TensorFlow

pip install tensorflow-gpu

代码运行
​编译darknet动态库.so，并放入darknet-server中
Makefile :

GPU=1
CUDNN=1
CUDNN_HALF=0
OPENCV=0
AVX=0
OPENMP=0
LIBSO=1
ZED_CAMERA=0 # ZED SDK 3.0 and above
ZED_CAMERA_v2_8=0 # ZED SDK 2.X

cd /data/work/morning/darknet
make clean
make all
cp libdarknet.so /data/work/morning/darknet-server/darknet-server
激活python3.6环境

source activate py36
进入脚本路径

cd /xxx/xxx/darknet-server/scripts
修改启动线程数：

vim start_multi_server.sh
while(( $int<=24 ))

将24修改为启动的线程数。后续可以将其改为配置文件的参数。

修改GPU核数

vim …/darknet-server/httpds_flask.py
GPU_NUM = 3
#将3修改为需要使用的GPU数量。后续可以将其改为配置文件的参数。
启动目标检测服务

cd /xxx/xxx/darknet-server/darknet-server
sh …/scripts/start_multi_server.sh

对darknet.py的改进:

​在这里插入代码片

netMain = None
metaMain = None
altNames = None

def detect_image(im, thresh=.5, hier_thresh=.5, nms=.45, debug= False):
    import cv2
    # custom_image_bgr = cv2.imread(im) # use: detect(,,imagePath,)
    net = netMain
    meta = metaMain
    custom_image = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)
    #custom_image = cv2.resize(custom_image,(320, 320), interpolation = cv2.INTER_NEAREST)

    #import scipy.misc
    #custom_image = scipy.misc.imread(image)
    im, arr = array_to_image(custom_image)     # you should comment line below: free_image(im)
    num = c_int(0)
    if debug: print("Assigned num")
    pnum = pointer(num)
    if debug: print("Assigned pnum")
    predict_image(net, im)
    letter_box = 0
    #predict_image_letterbox(net, im)
    #letter_box = 1
    if debug: print("did prediction")
    #dets = get_network_boxes(net, custom_image_bgr.shape[1], custom_image_bgr.shape[0], thresh, hier_thresh, None, 0, pnum, letter_box) # OpenCV
    dets = get_network_boxes(net, im.w, im.h, thresh, hier_thresh, None, 0, pnum, letter_box)
    if debug: print("Got dets")
    num = pnum[0]
    if debug: print("got zeroth index of pnum")
    if nms:
        do_nms_sort(dets, num, meta.classes, nms)
    if debug: print("did sort")
    res = []
    if debug: print("about to range")
    for j in range(num):
        if debug: print("Ranging on "+str(j)+" of "+str(num))
        if debug: print("Classes: "+str(meta), meta.classes, meta.names)
        for i in range(meta.classes):
            if debug: print("Class-ranging on "+str(i)+" of "+str(meta.classes)+"= "+str(dets[j].prob[i]))
            if dets[j].prob[i] > 0:
                b = dets[j].bbox
                if altNames is None:
                    nameTag = meta.names[i]
                else:
                    nameTag = altNames[i]
                if debug:
                    print("Got bbox", b)
                    print(nameTag)
                    print(dets[j].prob[i])
                    print((b.x, b.y, b.w, b.h))
                res.append((nameTag, dets[j].prob[i], (b.x, b.y, b.w, b.h)))
    if debug: print("did range")
    res = sorted(res, key=lambda x: -x[1])
    if debug: print("did sort")
    free_detections(dets, num)
    if debug: print("freed detections")
    return res

def InitDarknet(configPath = "./cfg/yolov4.cfg", weightPath = "./data/yolov4.weights", metaPath= "./cfg/coco.data"):
    global metaMain, netMain, altNames #pylint: disable=W0603
    if not os.path.exists(configPath):
        raise ValueError("Invalid config path `"+os.path.abspath(configPath)+"`")
    if not os.path.exists(weightPath):
        raise ValueError("Invalid weight path `"+os.path.abspath(weightPath)+"`")
    if not os.path.exists(metaPath):
        raise ValueError("Invalid data file path `"+os.path.abspath(metaPath)+"`")
    if netMain is None:
        netMain = load_net_custom(configPath.encode("ascii"), weightPath.encode("ascii"), 0, 1)  # batch size = 1
    if metaMain is None:
        metaMain = load_meta(metaPath.encode("ascii"))
    if altNames is None:
        # In Python 3, the metafile default access craps out on Windows (but not Linux)
        # Read the names file and create a list to feed to detect
        try:
            with open(metaPath) as metaFH:
                metaContents = metaFH.read()
                import re
                match = re.search("names *= *(.*)$", metaContents, re.IGNORECASE | re.MULTILINE)
                if match:
                    result = match.group(1)
                else:
                    result = None
                try:
                    if os.path.exists(result):
                        with open(result) as namesFH:
                            namesList = namesFH.read().strip().split("\n")
                            altNames = [x.strip() for x in namesList]
                except TypeError:
                    pass
        except Exception:
            pass
            

基于flask 的http server处理流程, 接收jpg图片进行预测, 返回json结果

from flask import Flask
from flask import request
import darknet
import cv2
import numpy as np
import json
from PIL import Image
import time
import sys
import logging
from libs.multiprocess_log_handler import MultiprocessHandler

GPU_NUM = 3
server_port = 0


def setup_log(log_name, appFlask):
    # logger = logging.getLogger()
    logger = appFlask.logger
    # 定义日志输出格式
    formattler = '%(asctime)s - thread-%(thread)-8d - %(levelname)s - %(filename)s - %(funcName)s - %(lineno)s - %(message)s'
    fmt = logging.Formatter(formattler)

    stream_handler = logging.StreamHandler(sys.stdout)
    stream_handler.setLevel(logging.DEBUG)
    stream_handler.setFormatter(fmt)

    file_handler = MultiprocessHandler(filename = log_name, backupCount = 30)
    file_handler.setLevel(logging.DEBUG)
    file_handler.setFormatter(fmt)

    logger.addHandler(stream_handler)
    logger.addHandler(file_handler)
    logger.setLevel(logging.DEBUG)
    return logger



app = Flask(__name__)

#logfile = sys.path[0] + '/obj-det.log'
#print(logfile)
#logger = setup_log(logfile, app)

class NpEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, np.integer):
            return int(obj)
        elif isinstance(obj, np.floating):
            return float(obj)
        elif isinstance(obj, np.ndarray):
            return obj.tolist()
        else:
            return super(NpEncoder, self).default(obj)

@app.route("/img/<imgname>",methods=['POST',])
def detect_img(imgname):
    frame = request.get_data() 
    resp = {}

    try:
        image = cv2.imdecode(np.fromstring(frame, dtype=np.uint8), cv2.IMREAD_COLOR)
    except:
        #logger.error('open image error')
        resp['code'] = -1
        resp['message'] = 'open image error'
        resp['result'] = []
    else:
        try:
            detections = darknet.detect_image(image, thresh=0.3, debug=False)
        except:
            #logger.error('detect error')
            resp['code'] = -1
            resp['message'] = 'detect error'
            resp['result'] = []
        else:
            if len(detections) :
                resp['code'] = 0
                resp['message'] = 'ok'
                result = []
                for detection in detections:
                    result.append(json.dumps({'model':detection[0], 'score':detection[1]}, cls=NpEncoder))
                resp['result'] = result
            else :
                resp['code'] = -1
                resp['message'] = 'no object recognized'
                resp['result'] = []

        data = json.dumps(resp)
        #logger.info('img:'+imgname+' port:'+str(server_port)+' '+ str(resp)+'\n')

        return data,200,{"ContentType":"application/json"}

def start_server(argv):
    global server_port
    server_port = int(argv[1])
    gpu_id = server_port%GPU_NUM
    darknet.set_gpu(gpu_id)
    darknet.InitDarknet()
    app.run(host='0.0.0.0',port =server_port,threaded=False)

if __name__ == "__main__":
    start_server(sys.argv)

数据集
在MS COCO基础上增加了3000包裹图片，自己标注的人，车，包裹，使用labelImg工具标注
训练数据集
nohup ./darknet detector train /data/package-with-coco/pack-608.data /data/package-with-coco/cfg/yolov4-608.cfg /data/package-with-coco/cfg/yolov4.conv.137 -dont_show -gpus 1 -mjpeg_port 8608 -map -chart > /data/package-with-coco/train-60838.log &

./darknet detector test /data/package-with-coco/pack.data /data/package-with-coco/cfg/yolov4.cfg /data/package-with-coco/backup/yolov4_2000.weights /data/package-with-coco/dataset/test/gettyimages-1475438368-612x612.jpg -dont_show -thresh 0.1

./darknet detector map /data/package-with-coco/pack.data /data/package-with-coco/cfg/yolov4.cfg /data/package-with-coco/backup/yolov4_2000.weights

[net]
subdivisions=8
width=608
height=608