python语言的双（多）相机图像采集
一、前言
第一篇帖子中讲解了基于python语言的AzureKinect相机图像采集代码，主要是使用了官方SDK以及pyk4a库，详情可查看下方链接：
单Azure Kinect相机图像采集（python）
本文将继续深入，讲解如何通过改写pyk4a库，实现基于python语言的，双Azure Kinect图像实时采集，并且可实时显示两台相机的视频流
二、开发原理
1、pyk4a简介
pyk4a是一个开源项目，它为Azure Kinect DK提供了一个简单易用的Python接口。它构建在libk4a之上，这是Microsoft提供的官方C库。pyk4a允许用户轻松地访问摄像头数据并控制摄像头的各种功能。
安装
安装pyk4a可以通过pip来完成：
pip install pyk4a
示例代码
下面是一个简单的示例代码，展示如何使用pyk4a来捕获和显示RGB和深度图像：
from pyk4a import PyK4A
import cv2
# 创建PyK4A实例
k4a = PyK4A()
# 开始捕获
k4a.start()
# 打开RGB和深度图像窗口
cv2.namedWindow('Color Image', cv2.WINDOW_NORMAL)
cv2.namedWindow('Depth Image', cv2.WINDOW_NORMAL)
try:
while True:
# 获取捕获帧
capture = k4a.get_capture()
# 提取RGB和深度图像
color_image = capture.color
depth_image = capture.transformed_depth
# 显示图像
cv2.imshow('Color Image', color_image)
cv2.imshow('Depth Image', depth_image)
# 按下'q'键退出循环
if cv2.waitKey(1) == ord('q'):
break
finally:
# 清理资源
k4a.stop()
cv2.destroyAllWindows()
注意事项
在使用pyk4a之前，请确保你的系统已经正确安装了Azure Kinect SDK。
2、pyk4a库的改写
原始的pyk4a不支持两台相机同时获取视频流并进行实时拍照，因此针对原有库中内容进行改写，使其支持双相机操作。已经改好已通过本账号上传
pyk4a（双相机版）下载链接
原始pyk4a库的文件结构如下图：
使用方法
修改后的pyk4a库使用时无需通过pip命令安装，直接放置于双相机图像采集代码的同一路径下，然后在采图代码中通过import命令引用相关类，即可。
改写内容
需要改写的文件如下图：
1、init.py 文件的修改
在init文件中，主要修改的是引用部分
from .pyk4a1 import ColorControlCapabilities, PyK4A1
from .pyk4a1 import ColorControlCapabilities, PyK4A
其余部分大家可以对照官方的代码看看
from .calibration import Calibration, CalibrationType
from .capture import PyK4ACapture
from .capture import PyK4A1Capture
from .config import (
FPS,
ColorControlCommand,
ColorControlMode,
ColorResolution,
Config,
DepthMode,
ImageFormat,
WiredSyncMode,
)
from .errors import K4AException, K4ATimeoutException
from .module import k4a_module
from .playback import PyK4APlayback, SeekOrigin
from .pyk4a1 import ColorControlCapabilities, PyK4A1
from .pyk4a1 import ColorControlCapabilities, PyK4A
from .record import PyK4ARecord
from .record import PyK4A1Record
from .transformation import (
color_image_to_depth_camera,
depth_image_to_color_camera,
depth_image_to_color_camera_custom,
depth_image_to_point_cloud,
)
def connected_device_count() -> int:
return k4a_module.device_get_installed_count()
__all__ = (
"Calibration",
"CalibrationType",
"FPS",
"ColorControlCommand",
"ColorControlMode",
"ImageFormat",
"ColorResolution",
"Config",
"DepthMode",
"WiredSyncMode",
"K4AException",
"K4ATimeoutException",
"PyK4A",
"PyK4ACapture",
"PyK4A1Capture",
"PyK4APlayback",
"SeekOrigin",
"ColorControlCapabilities",
"color_image_to_depth_camera",
"depth_image_to_point_cloud",
"depth_image_to_color_camera",
"depth_image_to_color_camera_custom",
"PyK4ARecord",
"PyK4A1Record",
"connected_device_count",
)
2、capture.py文件的修改
capture文件修改是多相机采图的核心，主要是将PyK4ACapture类复制，重新生成一个新的PyK4A1Capture类，类似地通过这种方式可以实现三相机、四相机的图像采集（电脑CPU够用的话）
from typing import Optional
import numpy as np
from .calibration import Calibration
from .config import ImageFormat
from .errors import K4AException
from .module import k4a_module
from .transformation import (
color_image_to_depth_camera,
depth_image_to_color_camera,
depth_image_to_color_camera_custom,
depth_image_to_point_cloud,
)
class PyK4A1Capture:
def __init__(
self, calibration: Calibration, capture_handle: object, color_format: ImageFormat, thread_safe: bool = True
):
self._calibration: Calibration = calibration
self._capture_handle: object = capture_handle # built-in PyCapsule
self.thread_safe = thread_safe
self._color_format = color_format
self._color: Optional[np.ndarray] = None
self._color_timestamp_usec: int = 0
self._color_system_timestamp_nsec: int = 0
self._color_exposure_usec: Optional[int] = None
self._color_white_balance: Optional[int] = None
self._depth: Optional[np.ndarray] = None
self._depth_timestamp_usec: int = 0
self._depth_system_timestamp_nsec: int = 0
self._ir: Optional[np.ndarray] = None
self._ir_timestamp_usec: int = 0
self._ir_system_timestamp_nsec: int = 0
self._depth_point_cloud: Optional[np.ndarray] = None
self._transformed_depth: Optional[np.ndarray] = None
self._transformed_depth_point_cloud: Optional[np.ndarray] = None
self._transformed_color: Optional[np.ndarray] = None
self._transformed_ir: Optional[np.ndarray] = None
@property
def color(self) -> Optional[np.ndarray]:
if self._color is None:
(
self._color,
self._color_timestamp_usec,
self._color_system_timestamp_nsec,
) = k4a_module.capture_get_color_image(self._capture_handle, self.thread_safe)
return self._color
@property
def color_timestamp_usec(self) -> int:
"""Device timestamp for color image. Not equal host machine timestamp!"""
if self._color is None:
self.color
return self._color_timestamp_usec
@property
def color_system_timestamp_nsec(self) -> int:
"""System timestamp for color image in nanoseconds. Corresponds to Python's time.perf_counter_ns()."""
if self._color is None:
self.color
return self._color_system_timestamp_nsec
@property
def color_exposure_usec(self) -> int:
if self._color_exposure_usec is None:
value = k4a_module.color_image_get_exposure_usec(self._capture_handle)
if value == 0:
raise K4AException("Cannot read exposure from color image")
self._color_exposure_usec = value
return self._color_exposure_usec
@property
def color_white_balance(self) -> int:
if self._color_white_balance is None:
value = k4a_module.color_image_get_white_balance(self._capture_handle)
if value == 0:
raise K4AException("Cannot read white balance from color image")
self._color_white_balance = value
return self._color_white_balance
@property
def depth(self) -> Optional[np.ndarray]:
if self._depth is None:
(
self._depth,
self._depth_timestamp_usec,
self._depth_system_timestamp_nsec,
) = k4a_module.capture_get_depth_image(self._capture_handle, self.thread_safe)
return self._depth
@property
def depth_timestamp_usec(self) -> int:
"""Device timestamp for depth image. Not equal host machine timestamp!. Like as equal IR image timestamp"""
if self._depth is None:
self.depth
return self._depth_timestamp_usec
@property
def depth_system_timestamp_nsec(self) -> int:
"""System timestamp for depth image in nanoseconds. Corresponds to Python's time.perf_counter_ns()."""
if self._depth is None:
self.depth
return self._depth_system_timestamp_nsec
@property
def ir(self) -> Optional[np.ndarray]:
"""Device timestamp for IR image. Not equal host machine timestamp!. Like as equal depth image timestamp"""
if self._ir is None:
self._ir, self._ir_timestamp_usec, self._ir_system_timestamp_nsec = k4a_module.capture_get_ir_image(
self._capture_handle, self.thread_safe
)
return self._ir
@property
def ir_timestamp_usec(self) -> int:
if self._ir is None:
self.ir
return self._ir_timestamp_usec
@property
def ir_system_timestamp_nsec(self) -> int:
"""System timestamp for IR image in nanoseconds. Corresponds to Python's time.perf_counter_ns()."""
if self._ir is None:
self.ir
return self._ir_system_timestamp_nsec
@property
def transformed_depth(self) -> Optional[np.ndarray]:
if self._transformed_depth is None and self.depth is not None:
self._transformed_depth = depth_image_to_color_camera(self._depth, self._calibration, self.thread_safe)
return self._transformed_depth
@property
def depth_point_cloud(self) -> Optional[np.ndarray]:
if self._depth_point_cloud is None and self.depth is not None:
self._depth_point_cloud = depth_image_to_point_cloud(
self._depth,
self._calibration,
self.thread_safe,
calibration_type_depth=True,
)
return self._depth_point_cloud
@property
def transformed_depth_point_cloud(self) -> Optional[np.ndarray]:
if self._transformed_depth_point_cloud is None and self.transformed_depth is not None:
self._transformed_depth_point_cloud = depth_image_to_point_cloud(
self.transformed_depth,
self._calibration,
self.thread_safe,
calibration_type_depth=False,
)
return self._transformed_depth_point_cloud
@property
def transformed_color(self) -> Optional[np.ndarray]:
if self._transformed_color is None and self.depth is not None and self.color is not None:
if self._color_format != ImageFormat.COLOR_BGRA32:
raise RuntimeError(
"color color_image must be of color_format K4A_IMAGE_FORMAT_COLOR_BGRA32 for "
"transformation_color_image_to_depth_camera"
)
self._transformed_color = color_image_to_depth_camera(
self.color, self.depth, self._calibration, self.thread_safe
)
return self._transformed_color
@property
def transformed_ir(self) -> Optional[np.ndarray]:
if self._transformed_ir is None and self.ir is not None and self.depth is not None:
result = depth_image_to_color_camera_custom(self.depth, self.ir, self._calibration, self.thread_safe)
if result is None:
return None
else:
self._transformed_ir, self._transformed_depth = result
return self._transformed_ir
class PyK4ACapture:
def __init__(
self, calibration: Calibration, capture_handle: object, color_format: ImageFormat, thread_safe: bool = True
):
self._calibration: Calibration = calibration
self._capture_handle: object = capture_handle # built-in PyCapsule
self.thread_safe = thread_safe
self._color_format = color_format
self._color: Optional[np.ndarray] = None
self._color_timestamp_usec: int = 0
self._color_system_timestamp_nsec: int = 0
self._color_exposure_usec: Optional[int] = None
self._color_white_balance: Optional[int] = None
self._depth: Optional[np.ndarray] = None
self._depth_timestamp_usec: int = 0
self._depth_system_timestamp_nsec: int = 0
self._ir: Optional[np.ndarray] = None
self._ir_timestamp_usec: int = 0
self._ir_system_timestamp_nsec: int = 0
self._depth_point_cloud: Optional[np.ndarray] = None
self._transformed_depth: Optional[np.ndarray] = None
self._transformed_depth_point_cloud: Optional[np.ndarray] = None
self._transformed_color: Optional[np.ndarray] = None
self._transformed_ir: Optional[np.ndarray] = None
@property
def color(self) -> Optional[np.ndarray]:
if self._color is None:
(
self._color,
self._color_timestamp_usec,
self._color_system_timestamp_nsec,
) = k4a_module.capture_get_color_image(self._capture_handle, self.thread_safe)
return self._color
@property
def color_timestamp_usec(self) -> int:
"""Device timestamp for color image. Not equal host machine timestamp!"""
if self._color is None:
self.color
return self._color_timestamp_usec
@property
def color_system_timestamp_nsec(self) -> int:
"""System timestamp for color image in nanoseconds. Corresponds to Python's time.perf_counter_ns()."""
if self._color is None:
self.color
return self._color_system_timestamp_nsec
@property
def color_exposure_usec(self) -> int:
if self._color_exposure_usec is None:
value = k4a_module.color_image_get_exposure_usec(self._capture_handle)
if value == 0:
raise K4AException("Cannot read exposure from color image")
self._color_exposure_usec = value
return self._color_exposure_usec
@property
def color_white_balance(self) -> int:
if self._color_white_balance is None:
value = k4a_module.color_image_get_white_balance(self._capture_handle)
if value == 0:
raise K4AException("Cannot read white balance from color image")
self._color_white_balance = value
return self._color_white_balance
@property
def depth(self) -> Optional[np.ndarray]:
if self._depth is None:
(
self._depth,
self._depth_timestamp_usec,
self._depth_system_timestamp_nsec,
) = k4a_module.capture_get_depth_image(self._capture_handle, self.thread_safe)
return self._depth
@property
def depth_timestamp_usec(self) -> int:
"""Device timestamp for depth image. Not equal host machine timestamp!. Like as equal IR image timestamp"""
if self._depth is None:
self.depth
return self._depth_timestamp_usec
@property
def depth_system_timestamp_nsec(self) -> int:
"""System timestamp for depth image in nanoseconds. Corresponds to Python's time.perf_counter_ns()."""
if self._depth is None:
self.depth
return self._depth_system_timestamp_nsec
@property
def ir(self) -> Optional[np.ndarray]:
"""Device timestamp for IR image. Not equal host machine timestamp!. Like as equal depth image timestamp"""
if self._ir is None:
self._ir, self._ir_timestamp_usec, self._ir_system_timestamp_nsec = k4a_module.capture_get_ir_image(
self._capture_handle, self.thread_safe
)
return self._ir
@property
def ir_timestamp_usec(self) -> int:
if self._ir is None:
self.ir
return self._ir_timestamp_usec
@property
def ir_system_timestamp_nsec(self) -> int:
"""System timestamp for IR image in nanoseconds. Corresponds to Python's time.perf_counter_ns()."""
if self._ir is None:
self.ir
return self._ir_system_timestamp_nsec
@property
def transformed_depth(self) -> Optional[np.ndarray]:
if self._transformed_depth is None and self.depth is not None:
self._transformed_depth = depth_image_to_color_camera(self._depth, self._calibration, self.thread_safe)
return self._transformed_depth
@property
def depth_point_cloud(self) -> Optional[np.ndarray]:
if self._depth_point_cloud is None and self.depth is not None:
self._depth_point_cloud = depth_image_to_point_cloud(
self._depth,
self._calibration,
self.thread_safe,
calibration_type_depth=True,
)
return self._depth_point_cloud
@property
def transformed_depth_point_cloud(self) -> Optional[np.ndarray]:
if self._transformed_depth_point_cloud is None and self.transformed_depth is not None:
self._transformed_depth_point_cloud = depth_image_to_point_cloud(
self.transformed_depth,
self._calibration,
self.thread_safe,
calibration_type_depth=False,
)
return self._transformed_depth_point_cloud
@property
def transformed_color(self) -> Optional[np.ndarray]:
if self._transformed_color is None and self.depth is not None and self.color is not None:
if self._color_format != ImageFormat.COLOR_BGRA32:
raise RuntimeError(
"color color_image must be of color_format K4A_IMAGE_FORMAT_COLOR_BGRA32 for "
"transformation_color_image_to_depth_camera"
)
self._transformed_color = color_image_to_depth_camera(
self.color, self.depth, self._calibration, self.thread_safe
)
return self._transformed_color
@property
def transformed_ir(self) -> Optional[np.ndarray]:
if self._transformed_ir is None and self.ir is not None and self.depth is not None:
result = depth_image_to_color_camera_custom(self.depth, self.ir, self._calibration, self.thread_safe)
if result is None:
return None
else:
self._transformed_ir, self._transformed_depth = result
return self._transformed_ir
3、playback.py文件的修改
该文件主要是将代码第17行的
from .pyk4a import ImuSample
修改为
from .pyk4a1 import ImuSample
4、pyk4a.py文件的修改
首先要将pyk4a文件重新命名为pyk4a1.py，防止与自己环境中安装的官方pyk4a包产生冲突
import sys
from typing import Any, Optional, Tuple
from .calibration import Calibration
from .capture import PyK4ACapture
from .config import ColorControlCommand, ColorControlMode, Config
from .errors import K4AException, _verify_error
from .module import k4a_module
if sys.version_info < (3, 8):
from typing_extensions import TypedDict
else:
from typing import TypedDict
class PyK4A1:
TIMEOUT_WAIT_INFINITE = -1
def __init__(self, config: Optional[Config] = None, device_id: int = 0,thread_safe: bool = True):
self._device_id = 0
self._config: Config = config if (config is not None) else Config()
self.thread_safe = thread_safe
self._device_handle: Optional[object] = None
# self._device_handle1: Optional[object] = None
self._calibration: Optional[Calibration] = None
self.is_running = False
def start(self):
"""
Open device if device not opened, then start cameras and IMU
All-in-one function
:return:
"""
if not self.opened:
self.open()
self._start_cameras()
self._start_imu()
self.is_running = True
def stop(self):
"""
Stop cameras, IMU, ... and close device
:return:
"""
self._stop_imu()
self._stop_cameras()
self._device_close()
self.is_running = False
def __del__(self):
if self.is_running:
self.stop()
elif self.opened:
self.close()
@property
def opened(self) -> bool:
return self._device_handle is not None
def open(self):
"""
Open device
You must open device before querying any information
"""
# if self.opened:
# raise K4AException("Device already opened")
self._device_open()
def close(self):
self._validate_is_opened()
self._device_close()
def save_calibration_json(self, path: Any):
with open(path, "w") as f:
f.write(self.calibration_raw)
def load_calibration_json(self, path: Any):
with open(path, "r") as f:
calibration = f.read()
self.calibration_raw = calibration
def _device_open(self):
res, handle = k4a_module.device_open(self._device_id, self.thread_safe)
self._device_handle = handle
_verify_error(res)
def _device_close(self):
res = k4a_module.device_close(self._device_handle, self.thread_safe)
_verify_error(res)
self._device_handle = None
def _start_cameras(self):
res = k4a_module.device_start_cameras(self._device_handle, self.thread_safe, *self._config.unpack())
_verify_error(res)
def _start_imu(self):
res = k4a_module.device_start_imu(self._device_handle, self.thread_safe)
_verify_error(res)
def _stop_cameras(self):
res = k4a_module.device_stop_cameras(self._device_handle, self.thread_safe)
_verify_error(res)
def _stop_imu(self):
res = k4a_module.device_stop_imu(self._device_handle, self.thread_safe)
_verify_error(res)
def get_capture1(
self,
timeout=TIMEOUT_WAIT_INFINITE,
) -> "PyK4ACapture":
"""
Fetch a capture from the device and return a PyK4ACapture object. Images are
lazily fetched.
Arguments:
:param timeout: Timeout in ms. Default is infinite.
Returns:
:return capture containing requested images and infos if they are available
in the current capture. There are no guarantees that the returned
object will contain all the requested images.
If using any ImageFormat other than ImageFormat.COLOR_BGRA32, the color color_image must be
decoded. See example/color_formats.py
"""
self._validate_is_opened()
res, capture_capsule = k4a_module.device_get_capture(self._device_handle, self.thread_safe, timeout)
_verify_error(res)
capture = PyK4ACapture(
calibration=self.calibration,
capture_handle=capture_capsule,
color_format=self._config.color_format,
thread_safe=self.thread_safe,
)
return capture
def get_imu_sample(self, timeout: int = TIMEOUT_WAIT_INFINITE) -> Optional["ImuSample"]:
self._validate_is_opened()
res, imu_sample = k4a_module.device_get_imu_sample(self._device_handle, self.thread_safe, timeout)
_verify_error(res)
return imu_sample
@property
def serial(self) -> str:
self._validate_is_opened()
ret = k4a_module.device_get_serialnum(self._device_handle, self.thread_safe)
if ret == "":
raise K4AException("Cannot read serial")
return ret
@property
def calibration_raw(self) -> str:
self._validate_is_opened()
raw = k4a_module.device_get_raw_calibration(self._device_handle, self.thread_safe)
return raw
@calibration_raw.setter
def calibration_raw(self, value: str):
self._validate_is_opened()
self._calibration = Calibration.from_raw(
value, self._config.depth_mode, self._config.color_resolution, self.thread_safe
)
@property
def sync_jack_status(self) -> Tuple[bool, bool]:
self._validate_is_opened()
res, jack_in, jack_out = k4a_module.device_get_sync_jack(self._device_handle, self.thread_safe)
_verify_error(res)
return jack_in == 1, jack_out == 1
def _get_color_control(self, cmd: ColorControlCommand) -> Tuple[int, ColorControlMode]:
self._validate_is_opened()
res, mode, value = k4a_module.device_get_color_control(self._device_handle, self.thread_safe, cmd)
_verify_error(res)
return value, ColorControlMode(mode)
def _set_color_control(self, cmd: ColorControlCommand, value: int, mode=ColorControlMode.MANUAL):
self._validate_is_opened()
res = k4a_module.device_set_color_control(self._device_handle, self.thread_safe, cmd, mode, value)
_verify_error(res)
@property
def brightness(self) -> int:
return self._get_color_control(ColorControlCommand.BRIGHTNESS)[0]
@brightness.setter
def brightness(self, value: int):
self._set_color_control(ColorControlCommand.BRIGHTNESS, value)
@property
def contrast(self) -> int:
return self._get_color_control(ColorControlCommand.CONTRAST)[0]
@contrast.setter
def contrast(self, value: int):
self._set_color_control(ColorControlCommand.CONTRAST, value)
@property
def saturation(self) -> int:
return self._get_color_control(ColorControlCommand.SATURATION)[0]
@saturation.setter
def saturation(self, value: int):
self._set_color_control(ColorControlCommand.SATURATION, value)
@property
def sharpness(self) -> int:
return self._get_color_control(ColorControlCommand.SHARPNESS)[0]
@sharpness.setter
def sharpness(self, value: int):
self._set_color_control(ColorControlCommand.SHARPNESS, value)
@property
def backlight_compensation(self) -> int:
return self._get_color_control(ColorControlCommand.BACKLIGHT_COMPENSATION)[0]
@backlight_compensation.setter
def backlight_compensation(self, value: int):
self._set_color_control(ColorControlCommand.BACKLIGHT_COMPENSATION, value)
@property
def gain(self) -> int:
return self._get_color_control(ColorControlCommand.GAIN)[0]
@gain.setter
def gain(self, value: int):
self._set_color_control(ColorControlCommand.GAIN, value)
@property
def powerline_frequency(self) -> int:
return self._get_color_control(ColorControlCommand.POWERLINE_FREQUENCY)[0]
@powerline_frequency.setter
def powerline_frequency(self, value: int):
self._set_color_control(ColorControlCommand.POWERLINE_FREQUENCY, value)
@property
def exposure(self) -> int:
# sets mode to manual
return self._get_color_control(ColorControlCommand.EXPOSURE_TIME_ABSOLUTE)[0]
@exposure.setter
def exposure(self, value: int):
self._set_color_control(ColorControlCommand.EXPOSURE_TIME_ABSOLUTE, value)
@property
def exposure_mode_auto(self) -> bool:
return self._get_color_control(ColorControlCommand.EXPOSURE_TIME_ABSOLUTE)[1] == ColorControlMode.AUTO
@exposure_mode_auto.setter
def exposure_mode_auto(self, mode_auto: bool, value: int = 2500):
mode = ColorControlMode.AUTO if mode_auto else ColorControlMode.MANUAL
self._set_color_control(ColorControlCommand.EXPOSURE_TIME_ABSOLUTE, value=value, mode=mode)
@property
def whitebalance(self) -> int:
# sets mode to manual
return self._get_color_control(ColorControlCommand.WHITEBALANCE)[0]
@whitebalance.setter
def whitebalance(self, value: int):
self._set_color_control(ColorControlCommand.WHITEBALANCE, value)
@property
def whitebalance_mode_auto(self) -> bool:
return self._get_color_control(ColorControlCommand.WHITEBALANCE)[1] == ColorControlMode.AUTO
@whitebalance_mode_auto.setter
def whitebalance_mode_auto(self, mode_auto: bool, value: int = 2500):
mode = ColorControlMode.AUTO if mode_auto else ColorControlMode.MANUAL
self._set_color_control(ColorControlCommand.WHITEBALANCE, value=value, mode=mode)
def _get_color_control_capabilities(self, cmd: ColorControlCommand) -> Optional["ColorControlCapabilities"]:
self._validate_is_opened()
res, capabilities = k4a_module.device_get_color_control_capabilities(self._device_handle, self.thread_safe, cmd)
_verify_error(res)
return capabilities
def reset_color_control_to_default(self):
for cmd in ColorControlCommand:
capability = self._get_color_control_capabilities(cmd)
self._set_color_control(cmd, capability["default_value"], capability["default_mode"])
@property
def calibration(self) -> Calibration:
self._validate_is_opened()
if not self._calibration:
res, calibration_handle = k4a_module.device_get_calibration(
self._device_handle, self.thread_safe, self._config.depth_mode, self._config.color_resolution
)
_verify_error(res)
self._calibration = Calibration(
handle=calibration_handle,
depth_mode=self._config.depth_mode,
color_resolution=self._config.color_resolution,
thread_safe=self.thread_safe,
)
return self._calibration
def _validate_is_opened(self):
if not self.opened:
raise K4AException("Device is not opened")
class ImuSample(TypedDict):
temperature: float
acc_sample: Tuple[float, float, float]
acc_timestamp: int
gyro_sample: Tuple[float, float, float]
gyro_timestamp: int
class ColorControlCapabilities(TypedDict):
color_control_command: ColorControlCommand
supports_auto: bool
min_value: int
max_value: int
step_value: int
default_value: int
default_mode: ColorControlMode
class PyK4A:
TIMEOUT_WAIT_INFINITE = -1
def __init__(self, config: Optional[Config] = None, device_id: int = 1,thread_safe: bool = True):
self._device_id = 1
self._config: Config = config if (config is not None) else Config()
self.thread_safe = thread_safe
self.thread_safe1 = thread_safe
self._device_handle: Optional[object] = None
self._calibration: Optional[Calibration] = None
self.is_running = False
def start(self):
"""
Open device if device not opened, then start cameras and IMU
All-in-one function
:return:
"""
if not self.opened:
self.open()
self._start_cameras()
self._start_imu()
self.is_running = True
def stop(self):
"""
Stop cameras, IMU, ... and close device
:return:
"""
self._stop_imu()
self._stop_cameras()
self._device_close()
self.is_running = False
def __del__(self):
if self.is_running:
self.stop()
elif self.opened:
self.close()
@property
def opened(self) -> bool:
return self._device_handle is not None
def open(self):
"""
Open device
You must open device before querying any information
"""
# if self.opened:
# raise K4AException("Device already opened")
self._device_open()
def close(self):
self._validate_is_opened()
self._device_close()
def save_calibration_json(self, path: Any):
with open(path, "w") as f:
f.write(self.calibration_raw)
def load_calibration_json(self, path: Any):
with open(path, "r") as f:
calibration = f.read()
self.calibration_raw = calibration
def _device_open(self):
res, handle = k4a_module.device_open(self._device_id, self.thread_safe)
self._device_handle = handle
_verify_error(res)
def _device_close(self):
res = k4a_module.device_close(self._device_handle, self.thread_safe)
_verify_error(res)
self._device_handle = None
def _start_cameras(self):
res = k4a_module.device_start_cameras(self._device_handle, self.thread_safe, *self._config.unpack())
_verify_error(res)
def _start_imu(self):
res = k4a_module.device_start_imu(self._device_handle, self.thread_safe)
_verify_error(res)
def _stop_cameras(self):
res = k4a_module.device_stop_cameras(self._device_handle, self.thread_safe)
_verify_error(res)
def _stop_imu(self):
res = k4a_module.device_stop_imu(self._device_handle, self.thread_safe)
_verify_error(res)
def get_capture(
self,
timeout=TIMEOUT_WAIT_INFINITE,
) -> "PyK4A1Capture":
"""
Fetch a capture from the device and return a PyK4ACapture object. Images are
lazily fetched.
Arguments:
:param timeout: Timeout in ms. Default is infinite.
Returns:
:return capture containing requested images and infos if they are available
in the current capture. There are no guarantees that the returned
object will contain all the requested images.
If using any ImageFormat other than ImageFormat.COLOR_BGRA32, the color color_image must be
decoded. See example/color_formats.py
"""
self._validate_is_opened()
res, capture_capsule = k4a_module.device_get_capture(self._device_handle, self.thread_safe1, timeout)
_verify_error(res)
capture = PyK4ACapture(
calibration=self.calibration,
capture_handle=capture_capsule,
color_format=self._config.color_format,
thread_safe=self.thread_safe,
)
return capture
def get_imu_sample(self, timeout: int = TIMEOUT_WAIT_INFINITE) -> Optional["ImuSample"]:
self._validate_is_opened()
res, imu_sample = k4a_module.device_get_imu_sample(self._device_handle, self.thread_safe, timeout)
_verify_error(res)
return imu_sample
@property
def serial(self) -> str:
self._validate_is_opened()
ret = k4a_module.device_get_serialnum(self._device_handle, self.thread_safe)
if ret == "":
raise K4AException("Cannot read serial")
return ret
@property
def calibration_raw(self) -> str:
self._validate_is_opened()
raw = k4a_module.device_get_raw_calibration(self._device_handle, self.thread_safe)
return raw
@calibration_raw.setter
def calibration_raw(self, value: str):
self._validate_is_opened()
self._calibration = Calibration.from_raw(
value, self._config.depth_mode, self._config.color_resolution, self.thread_safe
)
@property
def sync_jack_status(self) -> Tuple[bool, bool]:
self._validate_is_opened()
res, jack_in, jack_out = k4a_module.device_get_sync_jack(self._device_handle, self.thread_safe)
_verify_error(res)
return jack_in == 1, jack_out == 1
def _get_color_control(self, cmd: ColorControlCommand) -> Tuple[int, ColorControlMode]:
self._validate_is_opened()
res, mode, value = k4a_module.device_get_color_control(self._device_handle, self.thread_safe, cmd)
_verify_error(res)
return value, ColorControlMode(mode)
def _set_color_control(self, cmd: ColorControlCommand, value: int, mode=ColorControlMode.MANUAL):
self._validate_is_opened()
res = k4a_module.device_set_color_control(self._device_handle, self.thread_safe, cmd, mode, value)
_verify_error(res)
@property
def brightness(self) -> int:
return self._get_color_control(ColorControlCommand.BRIGHTNESS)[0]
@brightness.setter
def brightness(self, value: int):
self._set_color_control(ColorControlCommand.BRIGHTNESS, value)
@property
def contrast(self) -> int:
return self._get_color_control(ColorControlCommand.CONTRAST)[0]
@contrast.setter
def contrast(self, value: int):
self._set_color_control(ColorControlCommand.CONTRAST, value)
@property
def saturation(self) -> int:
return self._get_color_control(ColorControlCommand.SATURATION)[0]
@saturation.setter
def saturation(self, value: int):
self._set_color_control(ColorControlCommand.SATURATION, value)
@property
def sharpness(self) -> int:
return self._get_color_control(ColorControlCommand.SHARPNESS)[0]
@sharpness.setter
def sharpness(self, value: int):
self._set_color_control(ColorControlCommand.SHARPNESS, value)
@property
def backlight_compensation(self) -> int:
return self._get_color_control(ColorControlCommand.BACKLIGHT_COMPENSATION)[0]
@backlight_compensation.setter
def backlight_compensation(self, value: int):
self._set_color_control(ColorControlCommand.BACKLIGHT_COMPENSATION, value)
@property
def gain(self) -> int:
return self._get_color_control(ColorControlCommand.GAIN)[0]
@gain.setter
def gain(self, value: int):
self._set_color_control(ColorControlCommand.GAIN, value)
@property
def powerline_frequency(self) -> int:
return self._get_color_control(ColorControlCommand.POWERLINE_FREQUENCY)[0]
@powerline_frequency.setter
def powerline_frequency(self, value: int):
self._set_color_control(ColorControlCommand.POWERLINE_FREQUENCY, value)
@property
def exposure(self) -> int:
# sets mode to manual
return self._get_color_control(ColorControlCommand.EXPOSURE_TIME_ABSOLUTE)[0]
@exposure.setter
def exposure(self, value: int):
self._set_color_control(ColorControlCommand.EXPOSURE_TIME_ABSOLUTE, value)
@property
def exposure_mode_auto(self) -> bool:
return self._get_color_control(ColorControlCommand.EXPOSURE_TIME_ABSOLUTE)[1] == ColorControlMode.AUTO
@exposure_mode_auto.setter
def exposure_mode_auto(self, mode_auto: bool, value: int = 2500):
mode = ColorControlMode.AUTO if mode_auto else ColorControlMode.MANUAL
self._set_color_control(ColorControlCommand.EXPOSURE_TIME_ABSOLUTE, value=value, mode=mode)
@property
def whitebalance(self) -> int:
# sets mode to manual
return self._get_color_control(ColorControlCommand.WHITEBALANCE)[0]
@whitebalance.setter
def whitebalance(self, value: int):
self._set_color_control(ColorControlCommand.WHITEBALANCE, value)
@property
def whitebalance_mode_auto(self) -> bool:
return self._get_color_control(ColorControlCommand.WHITEBALANCE)[1] == ColorControlMode.AUTO
@whitebalance_mode_auto.setter
def whitebalance_mode_auto(self, mode_auto: bool, value: int = 2500):
mode = ColorControlMode.AUTO if mode_auto else ColorControlMode.MANUAL
self._set_color_control(ColorControlCommand.WHITEBALANCE, value=value, mode=mode)
def _get_color_control_capabilities(self, cmd: ColorControlCommand) -> Optional["ColorControlCapabilities"]:
self._validate_is_opened()
res, capabilities = k4a_module.device_get_color_control_capabilities(self._device_handle, self.thread_safe, cmd)
_verify_error(res)
return capabilities
def reset_color_control_to_default(self):
for cmd in ColorControlCommand:
capability = self._get_color_control_capabilities(cmd)
self._set_color_control(cmd, capability["default_value"], capability["default_mode"])
@property
def calibration(self) -> Calibration:
self._validate_is_opened()
if not self._calibration:
res, calibration_handle = k4a_module.device_get_calibration(
self._device_handle, self.thread_safe, self._config.depth_mode, self._config.color_resolution
)
_verify_error(res)
self._calibration = Calibration(
handle=calibration_handle,
depth_mode=self._config.depth_mode,
color_resolution=self._config.color_resolution,
thread_safe=self.thread_safe,
)
return self._calibration
def _validate_is_opened(self):
if not self.opened:
raise K4AException("Device is not opened")
class ImuSample(TypedDict):
temperature: float
acc_sample: Tuple[float, float, float]
acc_timestamp: int
gyro_sample: Tuple[float, float, float]
gyro_timestamp: int
class ColorControlCapabilities(TypedDict):
color_control_command: ColorControlCommand
supports_auto: bool
min_value: int
max_value: int
step_value: int
default_value: int
default_mode: ColorControlMode
5、record.py文件的修改
from pathlib import Path
from typing import Optional, Union
from .capture import PyK4ACapture
from .capture import PyK4A1Capture
from .config import Config
from .errors import K4AException
from .module import k4a_module
from .pyk4a1 import PyK4A1
from .pyk4a1 import PyK4A
from .results import Result
class PyK4A1Record:
def __init__(
self, path: Union[str, Path], config: Config, device: Optional[PyK4A1] = None, thread_safe: bool = True
):
self._path: Path = Path(path)
self.thread_safe = thread_safe
self._device: Optional[PyK4A1] = device
self._config: Config = config
self._handle: Optional[object] = None
self._header_written: bool = False
self._captures_count: int = 0
def __del__(self):
if self.created:
self.close()
def create(self) -> None:
"""Create record file"""
if self.created:
raise K4AException(f"Record already created {self._path}")
device_handle = self._device._device_handle if self._device else None
result, handle = k4a_module.record_create(
device_handle, str(self._path), self.thread_safe, *self._config.unpack()
)
if result != Result.Success:
raise K4AException(f"Cannot create record {self._path}")
self._handle = handle
def close(self):
"""Close record"""
self._validate_is_created()
k4a_module.record_close(self._handle, self.thread_safe)
self._handle = None
def write_header(self):
"""Write MKV header"""
self._validate_is_created()
if self.header_written:
raise K4AException(f"Header already written {self._path}")
result: Result = k4a_module.record_write_header(self._handle, self.thread_safe)
if result != Result.Success:
raise K4AException(f"Cannot write record header {self._path}")
self._header_written = True
def write_capture(self, capture: PyK4A1Capture):
"""Write capture to file (send to queue)"""
self._validate_is_created()
if not self.header_written:
self.write_header()
result: Result = k4a_module.record_write_capture(self._handle, capture._capture_handle, self.thread_safe)
if result != Result.Success:
raise K4AException(f"Cannot write capture {self._path}")
self._captures_count += 1
def flush(self):
"""Flush queue"""
self._validate_is_created()
result: Result = k4a_module.record_flush(self._handle, self.thread_safe)
if result != Result.Success:
raise K4AException(f"Cannot flush data {self._path}")
@property
def created(self) -> bool:
return self._handle is not None
@property
def header_written(self) -> bool:
return self._header_written
@property
def captures_count(self) -> int:
return self._captures_count
@property
def path(self) -> Path:
return self._path
def _validate_is_created(self):
if not self.created:
raise K4AException("Record not created.")
class PyK4ARecord:
def __init__(
self, path: Union[str, Path], config: Config, device: Optional[PyK4A] = None, thread_safe: bool = True
):
self._path: Path = Path(path)
self.thread_safe = thread_safe
self._device: Optional[PyK4A] = device
self._config: Config = config
self._handle: Optional[object] = None
self._header_written: bool = False
self._captures_count: int = 0
def __del__(self):
if self.created:
self.close()
def create(self) -> None:
"""Create record file"""
if self.created:
raise K4AException(f"Record already created {self._path}")
device_handle = self._device._device_handle if self._device else None
result, handle = k4a_module.record_create(
device_handle, str(self._path), self.thread_safe, *self._config.unpack()
)
if result != Result.Success:
raise K4AException(f"Cannot create record {self._path}")
self._handle = handle
def close(self):
"""Close record"""
self._validate_is_created()
k4a_module.record_close(self._handle, self.thread_safe)
self._handle = None
def write_header(self):
"""Write MKV header"""
self._validate_is_created()
if self.header_written:
raise K4AException(f"Header already written {self._path}")
result: Result = k4a_module.record_write_header(self._handle, self.thread_safe)
if result != Result.Success:
raise K4AException(f"Cannot write record header {self._path}")
self._header_written = True
def write_capture(self, capture: PyK4ACapture):
"""Write capture to file (send to queue)"""
self._validate_is_created()
if not self.header_written:
self.write_header()
result: Result = k4a_module.record_write_capture(self._handle, capture._capture_handle, self.thread_safe)
if result != Result.Success:
raise K4AException(f"Cannot write capture {self._path}")
self._captures_count += 1
def flush(self):
"""Flush queue"""
self._validate_is_created()
result: Result = k4a_module.record_flush(self._handle, self.thread_safe)
if result != Result.Success:
raise K4AException(f"Cannot flush data {self._path}")
@property
def created(self) -> bool:
return self._handle is not None
@property
def header_written(self) -> bool:
return self._header_written
@property
def captures_count(self) -> int:
return self._captures_count
@property
def path(self) -> Path:
return self._path
def _validate_is_created(self):
if not self.created:
raise K4AException("Record not created.")
二、双相机图像采集代码
1）.文件结构
项目文件结构如下，将修改后的pyk4a库与图像采集代码放在同一路径下
2）.图像采集代码源码
代码引用情况如下：
import cv2
import pyk4a
import os
import numpy as np
from pyk4a import Config, PyK4A1
图像采集之前，需在代码中对两相机进行设置，如下：
k4a1 = PyK4A(
Config(
color_resolution=pyk4a.ColorResolution.RES_3072P,
depth_mode=pyk4a.DepthMode.WFOV_UNBINNED,
camera_fps =pyk4a.FPS.FPS_15
)
)
k4a1.start()
capture = k4a1.get_capture()
k4a2 = PyK4A1(
Config(
color_resolution=pyk4a.ColorResolution.RES_3072P,
depth_mode=pyk4a.DepthMode.WFOV_UNBINNED,
camera_fps =pyk4a.FPS.FPS_15
)
)
k4a2.start()
capture2 = k4a2.get_capture1()
生成capture1、capture2两个类，用于捕获两个相机的不同视频流。
图像采集代码源码如下：
import cv2
import pyk4a
import os
import numpy as np
from pyk4a import Config, PyK4A1
def main():
k4a1 = PyK4A(
Config(
color_resolution=pyk4a.ColorResolution.RES_3072P,
depth_mode=pyk4a.DepthMode.WFOV_UNBINNED,
camera_fps =pyk4a.FPS.FPS_15
)
)
k4a1.start()
capture = k4a1.get_capture()
k4a2 = PyK4A1(
Config(
color_resolution=pyk4a.ColorResolution.RES_3072P,
depth_mode=pyk4a.DepthMode.WFOV_UNBINNED,
camera_fps =pyk4a.FPS.FPS_15
)
)
k4a2.start()
capture2 = k4a2.get_capture1()
count = 0 # 累加计数
while True:
Kinect_folder_path = r"C:\Users\22898\Desktop\1"
capture = k4a1.get_capture()
capture2 = k4a2.get_capture1()
cv2.namedWindow('Kinect-color1', cv2.WINDOW_AUTOSIZE)
cv2.imshow('Kinect-color1', cv2.resize(capture.color, ( 1024, 728)))
cv2.namedWindow('Kinect-color2', cv2.WINDOW_AUTOSIZE)
cv2.imshow('Kinect-color2', cv2.resize(capture2.color, ( 1024, 728)))
key = cv2.waitKey(1)
if key == ord('5'):
count += 1
#-----------Kinect图像保存---------------#
Kinect_rgb_filename = f"{Kinect_folder_path}/1/rgb-{count}.png"
cv2.imwrite(Kinect_rgb_filename , capture.color)
print(f"基于Kinect1的 rgb-{count}.png 已保存！")
Kinect_depth_filename = f"{Kinect_folder_path}/1/depth-{count}.png"
cv2.imwrite(Kinect_depth_filename , capture.depth)
print(f"基于Kinect1的 depth-{count}.png 已保存！")
Kinect_ir_filename = f"{Kinect_folder_path}/1/ir-{count}.png"
cv2.imwrite(Kinect_ir_filename , capture.ir)
print(f"基于Kinect1的 ir-{count}.png 已保存！")
Kinect_morp_filename = f"{Kinect_folder_path}/1/morp-{count}.png"
cv2.imwrite(Kinect_morp_filename , capture.transformed_depth)
print(f"基于Kinect1的 morp-{count}.png 已保存！")
#-----------Kinect图像保存---------------#
Kinect_rgb_filename = f"{Kinect_folder_path}/2/rgb-{count}.png"
cv2.imwrite(Kinect_rgb_filename , capture2.color)
print(f"基于Kinect2的 rgb-{count}.png 已保存！")
Kinect_depth_filename = f"{Kinect_folder_path}/2/depth-{count}.png"
cv2.imwrite(Kinect_depth_filename , capture2.depth)
print(f"基于Kinect2的 depth-{count}.png 已保存！")
Kinect_ir_filename = f"{Kinect_folder_path}/2/ir-{count}.png"
cv2.imwrite(Kinect_ir_filename , capture2.ir)
print(f"基于Kinect2的 ir-{count}.png 已保存！")
Kinect_morp_filename = f"{Kinect_folder_path}/2/morp-{count}.png"
cv2.imwrite(Kinect_morp_filename , capture2.transformed_depth)
print(f"基于Kinect2的 morp-{count}.png 已保存！")
if key == ord('1'):
k4a1.stop()
k4a2.stop()
break
if __name__ == "__main__":
main()
代码运行效果如下：
按“1”退出、按“5”采图
总结
本文讲述了一种通过修改pyk4a库来实现多相机图像采集的方法，并在最后提供了图像采集代码。但有一些改进空间、以及一些提示与大家分享
1、在进行图像采集时，两台相机占用CPU较高，经测，一般商务笔记本是难以带动的。个人猜测可以通过多线程、修改视频流采样率等方式，降低双相机图像采集的算力要求，如果真有朋友在这方面进行研究，可以一起尝试一下。
2、在进行图像采集时，代码运行流畅度与图像数据写入的速度有很大关系，若代码中的图像存储路径为机械硬盘空间，则图像存储速率会收到较影响，有采图实时性要求的朋友可以注意一下这一点。
288
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
