realsense相机 + tensorflow2.3 + pyinstaller打包
简介
目标: 用realsense相机实现目标测距。
资料:
- https://github.com/IntelRealSense/librealsense/tree/master/wrappers/tensorflow (用tensorflow写的几个realsense相机工程项目,本程序就是基于其中的person height文件修改而成的。)
- https://www.freesion.com/article/3585644384/ (realsense相机获取内参)
工具:
- anaconda创建虚拟环境
- pycharm作为编译工具
- python3.6
- tensorflow2.3
- opencv (pip install opencv-python)
- numpy (pip install numpy)
- pyrealsense2(pip install pyrealsense2)
- pyinstaller(pip install pyinstaller)
测距代码
#-*- mode: python -*-
import pyrealsense2 as rs
import numpy as np
import cv2
import tensorflow as tf
import os
W = 1280
H = 720 # 分辨率低点会更流畅一些
# Configure depth and color streams
pipeline = rs.pipeline()
config = rs.config()
config.enable_stream(rs.stream.depth, W, H, rs.format.z16, 30)
config.enable_stream(rs.stream.color, W, H, rs.format.bgr8, 30)
print("[INFO] start streaming...")
pipeline.start(config)
aligned_stream = rs.align(rs.stream.color) # 创建对齐对象(深度对齐颜色)
point_cloud = rs.pointcloud()
print("[INFO] loading model...")
PATH_TO_CKPT = "./frozen_inference_graph.pb"
# download model from: https://github.com/opencv/opencv/wiki/TensorFlow-Object-Detection-API#run-network-in-opencv
# Load the Tensorflow model into memory.
detection_graph = tf.Graph()
with detection_graph.as_default():
od_graph_def = tf.compat.v1.GraphDef()
with tf.compat.v1.gfile.GFile(PATH_TO_CKPT, 'rb') as fid:
serialized_graph = fid.read()
od_graph_def.ParseFromString(serialized_graph)
tf.compat.v1.import_graph_def(od_graph_def, name='')
sess = tf.compat.v1.Session(graph=detection_graph)
# Input tensor is the image
image_tensor = detection_graph.get_tensor_by_name('image_tensor:0')
# Output tensors are the detection boxes, scores, and classes
# Each box represents a part of the image where a particular object was detected
detection_boxes = detection_graph.get_tensor_by_name('detection_boxes:0')
# Each score represents level of confidence for each of the objects.
# The score is shown on the result image, together with the class label.
detection_scores = detection_graph.get_tensor_by_name('detection_scores:0')
detection_classes = detection_graph.get_tensor_by_name('detection_classes:0')
# Number of objects detected
num_detections = detection_graph.get_tensor_by_name('num_detections:0')
# code source of tensorflow model loading: https://www.geeksforgeeks.org/ml-training-image-classifier-using-tensorflow-object-detection-api/
while True:
frames = pipeline.wait_for_frames()
frames = aligned_stream.process(frames) # 获取对齐帧集
depth_frame = frames.get_depth_frame() # 获取对齐后的深度帧
color_frame = frames.get_color_frame() # 获取对齐后的彩色帧
# ---- 获取内参(也有参数用不着,为了简洁可以删去) ----- #
# 相机内参矩阵为3*3的矩阵:M = [fx 0 ppx ; 0 fy ppy ; 0 0 1]
depth_profile = depth_frame.get_profile()
# print(dep
# th_profile) # <pyrealsense2.video_stream_profile: 1(0) 1280x720 @ 30fps 1>
# print(depth_profile.fps()) # 30
# print(depth_profile.stream_index()) # 0
# print(depth_profile.stream_name()) # depth
color_profile = color_frame.get_profile() # 获取颜色帧内参
cvsprofile = rs.video_stream_profile(color_profile)
dvsprofile = rs.video_stream_profile(depth_profile)
color_intrin = cvsprofile.get_intrinsics()
# print(color_intrin)
# [ 1280x720 p[634.195 358.757] f[636.191 635.712]
# Inverse Brown Conrady [-0.0534637 0.0635668 -0.000268144 0.00125052 -0.0200019] ]
color_intrin_part = [color_intrin.ppx, color_intrin.ppy, color_intrin.fx, color_intrin.fy]
# print(color_intrin_part)
# [634.1949462890625, 358.75689697265625, 636.1914672851562, 635.7119750976562] ppx ppy fx fy
# 提取ppx,ppy,fx,fy
ppx = color_intrin_part[0]
ppy = color_intrin_part[1]
fx = color_intrin_part[2]
fy = color_intrin_part[3]
depth_intrin = dvsprofile.get_intrinsics()
# print('depth_intrin:',depth_intrin)
# [ 1280x720 p[634.195 358.757] f[636.191 635.712]
# Inverse Brown Conrady [-0.0534637 0.0635668 -0.000268144 0.00125052 -0.0200019] ]
extrin = depth_profile.get_extrinsics_to(color_profile)
# print(extrin)
# rotation: [1, 0, 0, 0, 1, 0, 0, 0, 1]
# translation: [0, 0, 0]
if not depth_frame or not color_frame: # 等待同一帧的彩色跟深度影像才继续执行后续图像处理,两者缺一不可
continue
# ------------------ #
points = point_cloud.calculate(depth_frame)
verts = np.asanyarray(points.get_vertices()).view(np.float32).reshape(-1, W, 3) # xyz
# Convert images to numpy arrays
color_image = np.asanyarray(color_frame.get_data())
scaled_size = (int(W), int(H))
# expand image dimensions to have shape: [1, None, None, 3]
# i.e. a single-column array, where each item in the column has the pixel RGB value
image_expanded = np.expand_dims(color_image, axis=0)
# Perform the actual detection by running the model with the image as input
(boxes, scores, classes, num) = sess.run([detection_boxes, detection_scores, detection_classes, num_detections],
feed_dict={image_tensor: image_expanded})
boxes = np.squeeze(boxes)
classes = np.squeeze(classes).astype(np.int32)
scores = np.squeeze(scores)
print("[INFO] drawing bounding box on detected objects...")
print("[INFO] each detected object has a unique color")
for idx in range(int(num)):
class_ = classes[idx]
score = scores[idx]
box = boxes[idx]
print(" [DEBUG] class : ", class_, "idx : ", idx, "num : ", num)
if score > 0.8 and class_ == 1: # 1 for human,检测其他东东自行更改数字
left = box[1] * W
top = box[0] * H
right = box[3] * W
bottom = box[2] * H
width = right - left
height = bottom - top
bbox = (int(left), int(top), int(width), int(height))
p1 = (int(bbox[0]), int(bbox[1])) # 左上角x y 坐标(原点在左上)
p2 = (int(bbox[0] + bbox[2]), int(bbox[1] + bbox[3])) # 右下角坐标
print('p1和p2分别为:', p1, p2)
# draw box
cv2.rectangle(color_image, p1, p2, (255, 0, 0), 2, 1)
# 测试中心点距离(测试用)
# text_depth = depth_frame.get_distance(640, 360) # 1280 720
# text_depth = float("{:.3f}".format(text_depth))
# text_depth_txt = str(text_depth)+'m'
# color_image = cv2.circle(color_image, (640, 360), 3, (0, 255, 255), -1)
# color_image = cv2.putText(color_image, text_depth_txt, (50, 50), cv2.FONT_HERSHEY_PLAIN, 2, (0, 0, 255), 1,
# cv2.LINE_AA)
# for i, box in enumerate(boxes):
# target_xy_pixel = [int(round((p1[0] + p2[0]) / 2)), int(round((p1[1] + p2[1]) / 2))]
target_xy_pixel = [int(round((p2[0] - p1[0]) / 2)+p1[0]), int(round((p2[1] - p1[1]) / 2)+p1[1])]
target_depth = depth_frame.get_distance(target_xy_pixel[0], target_xy_pixel[1])
target_xy_true = [(target_xy_pixel[0] - ppx) * target_depth / fx,
(target_xy_pixel[1] - ppy) * target_depth / fy]
print('识别出目标:{} 中心点像素坐标:({}, {}) 实际坐标(mm):({:.3f},{:.3f}) 深度(m):{:.3f}'.format(classes[0],
target_xy_pixel[0],
target_xy_pixel[1],
target_xy_true[0] * 1000,
-target_xy_true[1] * 1000,
target_depth ))
target_depth = float("{:.3f}".format(target_depth))
depth_txt = str(target_depth) + 'm'
font = cv2.FONT_HERSHEY_SIMPLEX
bottomLeftCornerOfText = (p1[0], p1[1] + 20)
fontScale = 1
fontColor = (255, 255, 255)
lineType = 2
cv2.putText(color_image, depth_txt,
bottomLeftCornerOfText,
font,
fontScale,
fontColor,
lineType)
# Show images
cv2.namedWindow('RealSense', cv2.WINDOW_AUTOSIZE)
cv2.imshow('RealSense', color_image)
# cv2.waitKey(1)
if cv2.waitKey(1) & 0xFF == ord('q'): # 按q键退出程序
break
# Stop streaming
pipeline.stop()
cv2.destroyAllWindows()
- 程序还是有很多不足的地方,比如说距离显示不稳定,距离太近出现重复框框的问题~~ 还需改进
- 中英文注释夹杂,也懒得改了
pyinstaller打包成可执行文件
- 在终端输入pyinstaller -F detect.py
(tensorflow=2.3) D:\***\intel realsense\detect>pyinstaller -F detect.py
(关于pyinstaller打包其他资料可以自己上网查,不多说。)
- 运行之后会生成build、dist文件夹和detect.spec文件。dist文件夹下会有一个detect.exe文件。
228310 INFO: Building EXE from EXE-00.toc completed successfully.
- 此时直接运行dist文件下的detect.exe文件会报错!
(tensorflow=2.3) D:\***\intel realsense\detect\dist>detect.exe
2021-01-18 18:13:01.932947: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library cudart64_101.dll
[INFO] start streaming...
[INFO] loading model...
Traceback (most recent call last):
File "detect.py", line 33, in <module>
File "tensorflow\python\lib\io\file_io.py", line 116, in read
File "tensorflow\python\lib\io\file_io.py", line 79, in _preread_check
UnicodeDecodeError: 'utf-8' codec can't decode byte 0xd5 in position 77: invalid continuation byte
[19404] Failed to execute script detect
- 解决办法:
将模型文件放入dist文件夹下。将dist文件拷贝到他人电脑后可直接使用,进行测距。
之前也使用tensorflow-gpu 2.2.0 运行过,pyinstaller -F detect.py ----> detect.exe 会出现如下报错:
tensorflow.python.framework.errors_impl.NotFoundError: C:\Users\**\AppData\Local\Temp\_MEI57402\tensorflow\lite\experimental\microfrontend\python\ops\_au
dio_microfrontend_op.so not found
[14232] Failed to execute script detect
- 解决办法:
在spec文件中, datas后面添加[(os.path.join(os.path.dirname(importlib.import_module(‘tensorflow’).file),
“lite/experimental/microfrontend/python/ops/_audio_microfrontend_op.so”),
“tensorflow/lite/experimental/microfrontend/python/ops/”)], 然后终端输入pyinstaller detect.spec即可。如下:
datas=[(os.path.join(os.path.dirname(importlib.import_module('tensorflow').__file__),
"lite/experimental/microfrontend/python/ops/_audio_microfrontend_op.so"),
"tensorflow/lite/experimental/microfrontend/python/ops/")],
(tensorflow-gpu) D:\**\intel realsense\detect-bag>pyinstaller detect.spec