强化学习库gym之mountain-car源码解读

一、导入相关需要的包

import math
import numpy as np
import gym
from gym import spaces
from gym.utils import seeding

 

二、定义MountainCarEnv类，并且继承gym的env环境，在类中分别定义方法

1、初始参数方法 

def __init__(self, goal_velocity = 0):
        self.min_position = -1.2 # 最小位置点
        self.max_position = 0.6 # 最大位置点
        self.max_speed = 0.07 # 小车的最大速度
        self.goal_position = 0.5 # 小车需要达到的目标位置
        self.goal_velocity = goal_velocity # 小车的目标速度
        
        self.force=0.001 # 小车受力（应该是摩擦力）
        self.gravity=0.0025 # 小车受到的重力

        self.low = np.array([self.min_position, -self.max_speed], dtype=np.float32) #小车的最小位置点和最小速度
        self.high = np.array([self.max_position, self.max_speed], dtype=np.float32) # 小车的最大位置点和最大速度

        self.viewer = None # 是否选择可视化，这个暂且默认为None

        self.action_space = spaces.Discrete(3) # 小车的动作空间，三个动作空间，加速、减速、不变
        self.observation_space = spaces.Box(self.low, self.high, dtype=np.float32) # 小车的状态空间，这个是继承了gym环境里面sapces的Box环境，只需要设置最大值和最小值，Box就可以给出状态空间，我看了Box环境，返回时二维的数组，应该是速度和位置的两个变量

        self.seed() # 随机种子设置

 2、定义初始化随机种子的方法

def seed(self, seed=None):
        self.np_random, seed = seeding.np_random(seed)
        return [seed]

 3、step方法。此方法在这个环境中很关键，主要是agent和环境interaction的方法

def step(self, action):
        assert self.action_space.contains(action), "%r (%s) invalid" % (action, type(action)) #声明动作空间已经包含输入的action参数

        position, velocity = self.state #将状态赋值给位置和速度
        velocity += (action-1)*self.force + math.cos(3*position)*(-self.gravity) # 计算速度的变化
        velocity = np.clip(velocity, -self.max_speed, self.max_speed) # 计算速度的变化
        position += velocity # 计算位置的变化
        position = np.clip(position, self.min_position, self.max_position) # 计算位置的变化
        if (position==self.min_position and velocity<0): velocity = 0 #如果位置和速度小于设定范围，更改速度

        done = bool(position >= self.goal_position and velocity >= self.goal_velocity) # 当速度和位置达到目标的速度并且也达到目标位置，就赋值done为真True
        reward = -1.0

        self.state = (position, velocity) #此时将速度和位置赋值给状态
        return np.array(self.state), reward, done, {} #返回状态、奖励、和done

4、reset函数

def reset(self):
        self.state = np.array([self.np_random.uniform(low=-0.6, high=-0.4), 0]) #初始化时随机给定state，在low和high之间给随机赋值
        return np.array(self.state) # 返回state

5、关于图像显示的函数

图像显示的函数和强化学习关系不是很大，就不细讲了。

def _height(self, xs):
        return np.sin(3 * xs)*.45+.55

    def render(self, mode='human'):
        screen_width = 600
        screen_height = 400

        world_width = self.max_position - self.min_position
        scale = screen_width/world_width
        carwidth=40
        carheight=20


        if self.viewer is None:
            from gym.envs.classic_control import rendering
            self.viewer = rendering.Viewer(screen_width, screen_height)
            xs = np.linspace(self.min_position, self.max_position, 100)
            ys = self._height(xs)
            xys = list(zip((xs-self.min_position)*scale, ys*scale))

            self.track = rendering.make_polyline(xys)
            self.track.set_linewidth(4)
            self.viewer.add_geom(self.track)

            clearance = 10

            l,r,t,b = -carwidth/2, carwidth/2, carheight, 0
            car = rendering.FilledPolygon([(l,b), (l,t), (r,t), (r,b)])
            car.add_attr(rendering.Transform(translation=(0, clearance)))
            self.cartrans = rendering.Transform()
            car.add_attr(self.cartrans)
            self.viewer.add_geom(car)
            frontwheel = rendering.make_circle(carheight/2.5)
            frontwheel.set_color(.5, .5, .5)
            frontwheel.add_attr(rendering.Transform(translation=(carwidth/4,clearance)))
            frontwheel.add_attr(self.cartrans)
            self.viewer.add_geom(frontwheel)
            backwheel = rendering.make_circle(carheight/2.5)
            backwheel.add_attr(rendering.Transform(translation=(-carwidth/4,clearance)))
            backwheel.add_attr(self.cartrans)
            backwheel.set_color(.5, .5, .5)
            self.viewer.add_geom(backwheel)
            flagx = (self.goal_position-self.min_position)*scale
            flagy1 = self._height(self.goal_position)*scale
            flagy2 = flagy1 + 50
            flagpole = rendering.Line((flagx, flagy1), (flagx, flagy2))
            self.viewer.add_geom(flagpole)
            flag = rendering.FilledPolygon([(flagx, flagy2), (flagx, flagy2-10), (flagx+25, flagy2-5)])
            flag.set_color(.8,.8,0)
            self.viewer.add_geom(flag)

        pos = self.state[0]
        self.cartrans.set_translation((pos-self.min_position)*scale, self._height(pos)*scale)
        self.cartrans.set_rotation(math.cos(3 * pos))

        return self.viewer.render(return_rgb_array = mode=='rgb_array')
    
    def get_keys_to_action(self):
        return {():1,(276,):0,(275,):2,(275,276):1} #control with left and right arrow keys 
    
    def close(self):
        if self.viewer:
            self.viewer.close()
            self.viewer = None

三、完整的gym中的mountainCarEnv代码

"""
http://incompleteideas.net/sutton/MountainCar/MountainCar1.cp
permalink: https://perma.cc/6Z2N-PFWC
"""

import math
import numpy as np
import gym
from gym import spaces
from gym.utils import seeding

class MountainCarEnv(gym.Env):
    metadata = {
        'render.modes': ['human', 'rgb_array'],
        'video.frames_per_second': 30
    }

    def __init__(self, goal_velocity = 0):
        self.min_position = -1.2
        self.max_position = 0.6
        self.max_speed = 0.07
        self.goal_position = 0.5
        self.goal_velocity = goal_velocity
        
        self.force=0.001
        self.gravity=0.0025

        self.low = np.array([self.min_position, -self.max_speed], dtype=np.float32)
        self.high = np.array([self.max_position, self.max_speed], dtype=np.float32)

        self.viewer = None

        self.action_space = spaces.Discrete(3)
        self.observation_space = spaces.Box(self.low, self.high, dtype=np.float32)

        self.seed()

    def seed(self, seed=None):
        self.np_random, seed = seeding.np_random(seed)
        return [seed]

    def step(self, action):
        assert self.action_space.contains(action), "%r (%s) invalid" % (action, type(action))

        position, velocity = self.state
        velocity += (action-1)*self.force + math.cos(3*position)*(-self.gravity)
        velocity = np.clip(velocity, -self.max_speed, self.max_speed)
        position += velocity
        position = np.clip(position, self.min_position, self.max_position)
        if (position==self.min_position and velocity<0): velocity = 0

        done = bool(position >= self.goal_position and velocity >= self.goal_velocity)
        reward = -1.0

        self.state = (position, velocity)
        return np.array(self.state), reward, done, {}

    def reset(self):
        self.state = np.array([self.np_random.uniform(low=-0.6, high=-0.4), 0])
        return np.array(self.state)

    def _height(self, xs):
        return np.sin(3 * xs)*.45+.55

    def render(self, mode='human'):
        screen_width = 600
        screen_height = 400

        world_width = self.max_position - self.min_position
        scale = screen_width/world_width
        carwidth=40
        carheight=20


        if self.viewer is None:
            from gym.envs.classic_control import rendering
            self.viewer = rendering.Viewer(screen_width, screen_height)
            xs = np.linspace(self.min_position, self.max_position, 100)
            ys = self._height(xs)
            xys = list(zip((xs-self.min_position)*scale, ys*scale))

            self.track = rendering.make_polyline(xys)
            self.track.set_linewidth(4)
            self.viewer.add_geom(self.track)

            clearance = 10

            l,r,t,b = -carwidth/2, carwidth/2, carheight, 0
            car = rendering.FilledPolygon([(l,b), (l,t), (r,t), (r,b)])
            car.add_attr(rendering.Transform(translation=(0, clearance)))
            self.cartrans = rendering.Transform()
            car.add_attr(self.cartrans)
            self.viewer.add_geom(car)
            frontwheel = rendering.make_circle(carheight/2.5)
            frontwheel.set_color(.5, .5, .5)
            frontwheel.add_attr(rendering.Transform(translation=(carwidth/4,clearance)))
            frontwheel.add_attr(self.cartrans)
            self.viewer.add_geom(frontwheel)
            backwheel = rendering.make_circle(carheight/2.5)
            backwheel.add_attr(rendering.Transform(translation=(-carwidth/4,clearance)))
            backwheel.add_attr(self.cartrans)
            backwheel.set_color(.5, .5, .5)
            self.viewer.add_geom(backwheel)
            flagx = (self.goal_position-self.min_position)*scale
            flagy1 = self._height(self.goal_position)*scale
            flagy2 = flagy1 + 50
            flagpole = rendering.Line((flagx, flagy1), (flagx, flagy2))
            self.viewer.add_geom(flagpole)
            flag = rendering.FilledPolygon([(flagx, flagy2), (flagx, flagy2-10), (flagx+25, flagy2-5)])
            flag.set_color(.8,.8,0)
            self.viewer.add_geom(flag)

        pos = self.state[0]
        self.cartrans.set_translation((pos-self.min_position)*scale, self._height(pos)*scale)
        self.cartrans.set_rotation(math.cos(3 * pos))

        return self.viewer.render(return_rgb_array = mode=='rgb_array')
    
    def get_keys_to_action(self):
        return {():1,(276,):0,(275,):2,(275,276):1} #control with left and right arrow keys 
    
    def close(self):
        if self.viewer:
            self.viewer.close()
            self.viewer = None

注：这个是我的个人见解，文中如有不对之处和写的不好的地方请多多包涵，谢谢您！

参考文献：

1. https://gym.openai.com/