6、hilens-only 手势识别控制 竞速小车

hilens-only 手势识别控制竞速小车

1、网络配置：

Address : 192.168.5.xxx
Netmask : 24

2、hilens 手势识别：

Ros 端代码：下载区。 文件名为：hilens_gesture.zip

3、hilens控制台代码：

# -*- coding: utf-8 -*-
# !/usr/bin/python3
# python demo —— 手势识别

import hilens
import cv2
import numpy as np

import socket
# socket
socket_server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

socket_server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)

socket_server.bind(('192.168.5.3', 1234)) # 用于接受端绑定本地的地址；同时固定IP端口号（1023-65535）；
socket_server.listen(128)

client_socket, client_address = socket_server.accept()
send_data='null'


net_size = 224
score_thres = 0.6
labels = ["Background", "Great", "OK", "Other", "Rock", "Yeah"]


# softmax
def softmax(x):
    x  = x-np.max(x,axis=0)
    ex = np.exp(x)
    return ex/np.sum(ex,axis=0)


# 画矩形框的四个角
def draw_square(img_data, left_top, right_bot, line_len, color):
    cv2.line(img_data, (left_top[0], left_top[1]),   (left_top[0]+line_len, left_top[1]),   color, 2)
    cv2.line(img_data, (left_top[0], left_top[1]),   (left_top[0], left_top[1]+line_len),   color, 2)
    cv2.line(img_data, (left_top[0], right_bot[1]),  (left_top[0]+line_len, right_bot[1]),  color, 2)
    cv2.line(img_data, (left_top[0], right_bot[1]),  (left_top[0], right_bot[1]-line_len),  color, 2)
    cv2.line(img_data, (right_bot[0], left_top[1]),  (right_bot[0]-line_len, left_top[1]),  color, 2)
    cv2.line(img_data, (right_bot[0], left_top[1]),  (right_bot[0], left_top[1]+line_len),  color, 2)
    cv2.line(img_data, (right_bot[0], right_bot[1]), (right_bot[0]-line_len, right_bot[1]), color, 2)
    cv2.line(img_data, (right_bot[0], right_bot[1]), (right_bot[0], right_bot[1]-line_len), color, 2)
    
# 矩形框四周模糊化
def blur(img_data, left_top, right_bot, k):
    img_data[:left_top[1] - 5, :, :]  = cv2.blur(img_data[:left_top[1] - 5, :, :],  ksize = (k, k))
    img_data[:, :left_top[0] - 5, :]  = cv2.blur(img_data[:, :left_top[0] - 5, :],  ksize = (k, k))
    img_data[right_bot[1] + 5:, :, :] = cv2.blur(img_data[right_bot[1] + 5:, :, :], ksize = (k, k))
    img_data[:, right_bot[0] + 5:, :] = cv2.blur(img_data[:, right_bot[0] + 5:, :], ksize = (k, k))
    
def gesture_main():
    # 1. 模型初始化
    model   = hilens.Model(hilens.get_model_dir() + "gesture-only.om")
    display = hilens.Display(hilens.HDMI)
    camera  = hilens.VideoCapture()
    
    hilens.set_log_level(hilens.INFO)
    hilens.info("Hand pose demo init successful!")    
	
    # 2. 手势识别
    while True:
        # 2.1 读取摄像头数据
        input_nv21 = camera.read() 
        input_rgb  = cv2.cvtColor(input_nv21, cv2.COLOR_YUV2RGB_NV21)
        
        img_center = [640, 360] # 画面(1280x720)中心点
        radius     = 240
        left_top   = [img_center[0]-radius, img_center[1]-radius]
        right_bot  = [img_center[0]+radius, img_center[1]+radius]

        line_len    = 40
        color       = (255,255,255)
        kernel_size = 20
        draw_square(input_rgb, left_top, right_bot, line_len, color)        
        blur(input_rgb, left_top, right_bot, kernel_size)

        # 2.2 截取出一个正方形区域作为手势识别输入
        gesture_area = input_rgb[img_center[1]-radius:img_center[1]+radius, 
                                 img_center[0]-radius:img_center[0]+radius, 
                                 :]
        
        # 2.3 数据预处理
        input_resized = cv2.resize(gesture_area, (net_size, net_size))

        # 2.3 模型推理
        outputs = model.infer([input_resized.flatten()])
        predict = softmax(outputs[0])
        max_inx = np.argmax(predict)

        # 2.4 结果展示 
        if max_inx <= 5 and predict[max_inx] > score_thres:
            font_scale = 2
            thickness  = 2
            font       = cv2.FONT_HERSHEY_SIMPLEX
            
            if labels[max_inx] not in ["Background", "Other"]:
                cv2.putText(input_rgb, labels[max_inx], (600, 100), font, font_scale, (255, 0, 0), thickness)

        output_nv21 = hilens.cvt_color(input_rgb, hilens.RGB2YUV_NV21)
        display.show(output_nv21) 
        
        # socket part
        text = '%s' % labels[max_inx]
        send_data = text
        #send_data = data  # 通过darw_boxes得到data；在utils.py中；
        print("send_data:{}".format(send_data))  # format 格式化函数
        client_socket.send(send_data.encode('utf-8'))

def run():
    ret = hilens.init("gesture") # 校验值在console创建技能时需要填写
    if ret != 0:
        hilens.error("Failed to initialize HiLens")
        return
        
    gesture_main()
    hilens.terminate()

if __name__ == '__main__':
    run()

4、操作：

roslaunch carebot_bringup carebot_bringup.launch 

rosrun hilens hilens.py 

rosrun move_gesture move_gesture_node