在Carla上应用深度强化学习实现自动驾驶（二）

以下代码参考自：（如有侵权，请联系我立即删除）使用 Carla 和 Python 的自动驾驶汽车第 3 部分 —— 强化学习环境_carla跑强化学习-CSDN博客

本篇文章是小编在pycharm上自己手敲代码学习自动驾驶的第二篇文章，主要讲述如何构造强化学习环境。

1、导入carla（其中的路径根据自己的实际情况修改）

import glob
import os
import sys
 
try:
    sys.path.append(glob.glob(
        r'D:\postgraduate\code\CARLA_0.9.14\WindowsNoEditor\PythonAPI\carla\dist\carla-0.9.14-py3.7-win-amd64.egg')[0])
except IndexError:
    pass
 
import carla

2、接下来，我们将创建环境的类，我们将其称为CarEnv 。首先定义一些常量：

SHOW_CAM表示我们是否想要使用opencv显示rgb摄像头获取到的图像。为了调试的目的，查看它可能会很有用，但您不一定希望一直显示它，因为执行所有这些操作可能会耗费大量资源；

STEER_AMT表示转向的大小。现在是一个大转弯。之后我们可能会发现，控制的力度要小一些，或许可以做一些累积的事情来代替……等。现在，全速前进吧!

class CarEnv:
    SHOW_CAM = SHOW_PREVIEW
    STEER_AMT = 1.0

    im_width = IMG_WIDTH
    im_height = IMG_HEIGHT
    actor_list = []

    front_camera = None
    collision_hist = []

3、定义CarEnv类的init方法：主要用来连接Carla服务器，获取世界，访问蓝图，然后选择汽车模型。

    def __init__(self):
        self.client = carla.Client('localhost', 2000)
        self.client.set_timeout(2.0)

        self.world = self.client.get_world()

        blueprint_library = self.world.get_blueprint_library()

        # Now let's filter all the blueprints of type 'vehicle' and choose one at random.
        # print(blueprint_library.filter('vehicle'))
        self.model_3 = blueprint_library.filter('model3')[0]

4、定义CarEnv类的reset()方法，该方法的作用是在每一回合开始前重置环境。

该方法的主要内容如下：

（1）将碰撞列表和演员列表重设为空列表

（2）在Carla地图上随机选择一个生成点用于生成我们的汽车，并将其添加到演员列表中

（3）在车辆的前方添加rgb摄像头传感器，设置图像的相应属性，并将该传感器添加到演员列表中，另外还要设置该传感器的回调函数，用于处理摄像头获取到的图像数据，该回调函数是process_img函数

（4）控制汽车的油门和刹车均为0，即使汽车保持静止状态

（5）注意这一步操作：等待4秒之后再给汽车添加碰撞传感器，这个4秒是给在地图上生成汽车留下足够的时间，因为在Carla上生成的汽车是从天上落下来的，这有可能会被碰撞传感器误判成碰撞，所以需要先等待4秒等汽车安全落地之后再给车辆添加碰撞传感器。碰撞传感器的位置与前面的rgb摄像头传感器的位置相同，将该传感器也添加到演员列表中，并设置它的回调函数，用于记录和处理碰撞事件，该回调函数是collision_data函数

（6）然后等待，直到我们的rgb摄像头能够正常获取到前方的图像数据，即front_camera不为空

（7）记录下每一回合开始的时间，然后确保油门和刹车没有被使用，最后返回rgb摄像头观察到的第一张图像数据

    def reset(self):
        self.collision_hist = []
        self.actor_list = []

        self.transform = random.choice(self.world.get_map().get_spawn_points())
        self.vehicle = self.world.spawn_actor(self.model_3, self.transform)
        self.actor_list.append(self.vehicle)

        self.rgb_cam = self.world.get_blueprint_library().find('sensor.camera.rgb')

        self.rgb_cam.set_attribute('image_size_x', f'{self.im_width}')
        self.rgb_cam.set_attribute('image_size_y', f'{self.im_height}')
        self.rgb_cam.set_attribute('fov', '110')


        transform = carla.Transform(carla.Location(x=2.5, z=0.7))

        self.sensor = self.world.spawn_actor(self.rgb_cam, transform, attach_to=self.vehicle)

        self.actor_list.append(self.sensor)
        self.sensor.listen(lambda data: self.process_img(data))

        self.vehicle.apply_control(carla.VehicleControl(throttle=0.0, brake=0.0))

        time.sleep(4) # sleep to get things started and to not detect a collision when the car spawns/falls from sky.

        colsensor = self.world.get_blueprint_library().find('sensor.other.collision')
        self.colsensor = self.world.spawn_actor(colsensor, transform, attach_to=self.vehicle)
        self.actor_list.append(self.colsensor)
        self.colsensor.listen(lambda event: self.collision_data(event))

        while self.front_camera is None:
            time.sleep(0.01)

        self.episode_start = time.time()

        self.vehicle.apply_control(carla.VehicleControl(brake=0.0, throttle=0.0))

        return self.front_camera

    def collision_data(self, event):
        self.collision_hist.append(event)

    def process_img(self, image):
        i = np.array(image.raw_data)
        #np.save("iout.npy", i)
        i2 = i.reshape((self.im_height, self.im_width, 4))
        i3 = i2[:, :, :3]
        if self.SHOW_CAM:
            cv2.imshow("",i3)
            cv2.waitKey(1)
        self.front_camera = i3

5、 定义CarEnv类的step()方法，这个方法采取一个行动，然后返回观察，奖励，完成，任何额外的信息。

该方法的主要内容如下：

（1）设置汽车可以采取的离散动作的种类，有直行、左转和右转

（2）获取车辆的速度，并将其转换成km/h

（3）下面是奖励函数的定义：如果发生了碰撞，立即停止该回合，奖励值为-200,；如果速度小于50，继续该回合，奖励值为-1；其他行为继续该回合，奖励值为1

（4）如果车辆在该回合的运行时间超过了预计的时间，那么立即停止该回合

    def step(self, action):
        '''
        For now let's just pass steer left, center, right?
        0, 1, 2
        '''
        if action == 0:
            self.vehicle.apply_control(carla.VehicleControl(throttle=1.0, steer=0))
        if action == 1:
            self.vehicle.apply_control(carla.VehicleControl(throttle=1.0, steer=-1*self.STEER_AMT))
        if action == 2:
            self.vehicle.apply_control(carla.VehicleControl(throttle=1.0, steer=1*self.STEER_AMT))

        v = self.vehicle.get_velocity()
        kmh = int(3.6 * math.sqrt(v.x**2 + v.y**2 + v.z**2))

        if len(self.collision_hist) != 0:
            done = True
            reward = -200
        elif kmh < 50:
            done = False
            reward = -1
        else:
            done = False
            reward = 1

        if self.episode_start + SECONDS_PER_EPISODE < time.time():
            done = True

        return self.front_camera, reward, done, None

好的，本篇文章到这里就结束了，以下是CarEnv类的完整代码：

import glob
import os
import sys
import random
import time
import numpy as np
import cv2
import math
try:
    sys.path.append(glob.glob(
        r'D:\postgraduate\code\CARLA_0.9.14\WindowsNoEditor\PythonAPI\carla\dist\carla-0.9.14-py3.7-win-amd64.egg')[0])
except IndexError:
    pass
import carla


SHOW_PREVIEW = False
IMG_WIDTH = 640
IMG_HEIGHT = 480
SECONDS_PER_EPISODE = 10

class CarEnv:
    SHOW_CAM = SHOW_PREVIEW
    STEER_AMT = 1.0

    im_width = IMG_WIDTH
    im_height = IMG_HEIGHT
    actor_list = []

    front_camera = None
    collision_hist = []

    def __init__(self):
        self.client = carla.Client('localhost', 2000)
        self.client.set_timeout(2.0)

        self.world = self.client.get_world()

        blueprint_library = self.world.get_blueprint_library()

        # Now let's filter all the blueprints of type 'vehicle' and choose one at random.
        # print(blueprint_library.filter('vehicle'))
        self.model_3 = blueprint_library.filter('model3')[0]

    def reset(self):
        self.collision_hist = []
        self.actor_list = []

        self.transform = random.choice(self.world.get_map().get_spawn_points())
        self.vehicle = self.world.spawn_actor(self.model_3, self.transform)
        self.actor_list.append(self.vehicle)

        self.rgb_cam = self.world.get_blueprint_library().find('sensor.camera.rgb')

        self.rgb_cam.set_attribute('image_size_x', f'{self.im_width}')
        self.rgb_cam.set_attribute('image_size_y', f'{self.im_height}')
        self.rgb_cam.set_attribute('fov', '110')


        transform = carla.Transform(carla.Location(x=2.5, z=0.7))

        self.sensor = self.world.spawn_actor(self.rgb_cam, transform, attach_to=self.vehicle)

        self.actor_list.append(self.sensor)
        self.sensor.listen(lambda data: self.process_img(data))

        self.vehicle.apply_control(carla.VehicleControl(throttle=0.0, brake=0.0))

        time.sleep(4) # sleep to get things started and to not detect a collision when the car spawns/falls from sky.

        colsensor = self.world.get_blueprint_library().find('sensor.other.collision')
        self.colsensor = self.world.spawn_actor(colsensor, transform, attach_to=self.vehicle)
        self.actor_list.append(self.colsensor)
        self.colsensor.listen(lambda event: self.collision_data(event))

        while self.front_camera is None:
            time.sleep(0.01)

        self.episode_start = time.time()

        self.vehicle.apply_control(carla.VehicleControl(brake=0.0, throttle=0.0))

        return self.front_camera

    def collision_data(self, event):
        self.collision_hist.append(event)

    def process_img(self, image):
        i = np.array(image.raw_data)
        #np.save("iout.npy", i)
        i2 = i.reshape((self.im_height, self.im_width, 4))
        i3 = i2[:, :, :3]
        if self.SHOW_CAM:
            cv2.imshow("",i3)
            cv2.waitKey(1)
        self.front_camera = i3

    def step(self, action):
        '''
        For now let's just pass steer left, center, right?
        0, 1, 2
        '''
        if action == 0:
            self.vehicle.apply_control(carla.VehicleControl(throttle=1.0, steer=0))
        if action == 1:
            self.vehicle.apply_control(carla.VehicleControl(throttle=1.0, steer=-1*self.STEER_AMT))
        if action == 2:
            self.vehicle.apply_control(carla.VehicleControl(throttle=1.0, steer=1*self.STEER_AMT))

        v = self.vehicle.get_velocity()
        kmh = int(3.6 * math.sqrt(v.x**2 + v.y**2 + v.z**2))

        if len(self.collision_hist) != 0:
            done = True
            reward = -200
        elif kmh < 50:
            done = False
            reward = -1
        else:
            done = False
            reward = 1

        if self.episode_start + SECONDS_PER_EPISODE < time.time():
            done = True

        return self.front_camera, reward, done, None