【AirSim+Python】AirSim在UE4中显示第一人称图像窗口

跟着b站上【皮卡丘上大学】up主学AirSim，up主在b站发了代码了，但是那个settings设置代码我从b站直接复制下来一直报错，找了一下午bug（报错忘截图了，代号好像是[101]），差点又回到解放前了。。。实在找不到了（可能是b站复制下来有奇奇怪怪的字符，看不到），我自己又照着up主的内容手敲了一遍，然后就能用了。下面是这个up主在b站上分享的代码，应该复制下来能直接用了吧。。。（pygame实现方法的代码我没跑）

1.settings.json配置

{
  "SettingsVersion": 1.2,
  "SeeDocsAt": "https://github.com/Microsoft/AirSim/blob/master/docs/settings.md",
  "SimMode": "Multirotor",
  "ViewMode": "FlyWithMe",
"SubWindows":[{
"WindowID":0,
"CameraName":"front_center_custom",
"ImageType":0,
"Visible":true,
"ImageSize":[480,270],
"CameraPosition":[0.0,0.0,-2.5],
"CameraRotation":[0.0,0.0,0.0]
},
{
"WindowID":1,
"CameraName":"front_center_custom",
"ImageType":3,
"Visible":true,
"ImageSize":[480,270],
"CameraPosition":[0.0,0.0,-2.5],
"CameraRotation":[0.0,0.0,0.0]
},
{
"WindowID":2,
"CameraName":"front_center_custom",
"ImageType":5,
"Visible":true,
"ImageSize":[480,270],
"CameraPosition":[0.0,0.0,-2.5],
"CameraRotation":[0.0,0.0,0.0]
}],
"Vehicles":{
"Drone":{
"VehicleType":"SimpleFlight",
"DisplayName":"My First Drone",
"AutoCreate":true
}
}
}

2.pygame实现图像窗口

import sys
import time
import airsim
import pygame
import CV2
import numpy as np

# >------>>>  pygame settings   <<<------< #
pygame.init()

screen = pygame.display.set_mode((800, 144))
pygame.display.set_caption('screen')
screen.fill((0, 0, 0))

# >------>>>  AirSim settings   <<<------< #
# 这里改为你要控制的无人机名称(settings文件里面设置的)
vehicle_name = "Drone"
AirSim_client = airsim.MultirotorClient()
AirSim_client.confirmConnection()
AirSim_client.enableApiControl(True, vehicle_name=vehicle_name)
AirSim_client.armDisarm(True, vehicle_name=vehicle_name)
AirSim_client.takeoffAsync(vehicle_name=vehicle_name).join()
image_types = {
   "scene": airsim.ImageType.Scene,
   "depth": airsim.ImageType.DepthVis,
   "seg": airsim.ImageType.Segmentation,
   "normals": airsim.ImageType.SurfaceNormals,
   "segmentation": airsim.ImageType.Segmentation,
   "disparity": airsim.ImageType.DisparityNormalized,
   "Infrared": airsim.ImageType.Infrared
}

# 基础的控制速度(m/s)
vehicle_velocity = 2.0
# 设置临时加速比例
speedup_ratio = 10.0
# 用来设置临时加速
speedup_flag = False

# 基础的偏航速率
vehicle_yaw_rate = 5.0

while True:
   yaw_rate = 0.0
   velocity_x = 0.0
   velocity_y = 0.0
   velocity_z = 0.0

   time.sleep(0.02)

   for event in pygame.event.get():
       if event.type == pygame.QUIT:
           sys.exit()

   scan_wrapper = pygame.key.get_pressed()

   # 按下空格键加速10倍
   if scan_wrapper[pygame.K_SPACE]:
       scale_ratio = speedup_ratio
   else:
       scale_ratio = speedup_ratio / speedup_ratio

   # 根据 'A' 和 'D' 按键来设置偏航速率变量
   if scan_wrapper[pygame.K_a] or scan_wrapper[pygame.K_d]:
       yaw_rate = (scan_wrapper[pygame.K_d] - scan_wrapper[pygame.K_a]) * scale_ratio * vehicle_yaw_rate

   # 根据 'UP' 和 'DOWN' 按键来设置pitch轴速度变量(NED坐标系，x为机头向前)
   if scan_wrapper[pygame.K_UP] or scan_wrapper[pygame.K_DOWN]:
       velocity_x = (scan_wrapper[pygame.K_UP] - scan_wrapper[pygame.K_DOWN]) * scale_ratio

   # 根据 'LEFT' 和 'RIGHT' 按键来设置roll轴速度变量(NED坐标系，y为正右方)
   if scan_wrapper[pygame.K_LEFT] or scan_wrapper[pygame.K_RIGHT]:
       velocity_y = -(scan_wrapper[pygame.K_LEFT] - scan_wrapper[pygame.K_RIGHT]) * scale_ratio

   # 根据 'W' 和 'S' 按键来设置z轴速度变量(NED坐标系，z轴向上为负)
   if scan_wrapper[pygame.K_w] or scan_wrapper[pygame.K_s]:
       velocity_z = -(scan_wrapper[pygame.K_w] - scan_wrapper[pygame.K_s]) * scale_ratio

   # print(f": Expectation gesture: {velocity_x}, {velocity_y}, {velocity_z}, {yaw_rate}")

   # 设置速度控制以及设置偏航控制
   AirSim_client.moveByVelocityBodyFrameAsync(vx=velocity_x, vy=velocity_y, vz=velocity_z, duration=1,
                                              yaw_mode=airsim.YawMode(True, yaw_or_rate=yaw_rate), vehicle_name=vehicle_name)

   temp_image1 = AirSim_client.simGetImage('0', image_types["scene"], vehicle_name=vehicle_name)
   image1 = CV2.imdecode(airsim.string_to_uint8_array(temp_image1), CV2.IMREAD_COLOR)
   CV2.imwrite('screen/visual1.png', image1)
   # 利用pygame库加载保存的第一视角图像，
   screen_image1 = pygame.image.load("screen/visual1.png")
   # 图像坐标系，左上角为(0, 0)，在此放置图片
   screen.blit(screen_image1, (0, 0))
   pygame.display.flip()
   pygame.display.update()

   temp_image2 = AirSim_client.simGetImage('0', image_types["Infrared"], vehicle_name=vehicle_name)
   image2 = CV2.imdecode(airsim.string_to_uint8_array(temp_image2), CV2.IMREAD_COLOR)
   CV2.imwrite('screen/visual2.png', image2)
   # 利用pygame库加载保存的第一视角图像，
   screen_image2 = pygame.image.load("screen/visual2.png")
   # 图像坐标系，左上角为(0, 0)，在此放置图片
   screen.blit(screen_image2, (272, 0))
   pygame.display.flip()
   pygame.display.update()

   temp_image3 = AirSim_client.simGetImage('0', image_types["segmentation"], vehicle_name=vehicle_name)
   image3 = CV2.imdecode(airsim.string_to_uint8_array(temp_image3), CV2.IMREAD_COLOR)
   CV2.imwrite('screen/visual3.png', image3)
   # 利用pygame库加载保存的第一视角图像，
   screen_image3 = pygame.image.load("screen/visual3.png")
   # 图像坐标系，左上角为(0, 0)，在此放置图片
   screen.blit(screen_image3, (544, 0))
   pygame.display.flip()
   pygame.display.update()

   if scan_wrapper[pygame.K_ESCAPE]:
       pygame.quit()
       sys.exit()