强化学习cartpole demo 源码阅读理解

cartpole.py文件下
类成员变量的定义

    def __init__(self):
        self.gravity = 9.8
        self.masscart = 1.0 #车子的质量
        self.masspole = 0.1 #杆的质量
        self.total_mass = (self.masspole + self.masscart) #总质量
        self.length = 0.5 # actually half the pole's length实际杆长的一半
        self.polemass_length = (self.masspole * self.length)
        self.force_mag = 10.0 // 我理解这里是系统所受的力
        self.tau = 0.02  # seconds between state updates // 两帧之间更新的时间
        self.kinematics_integrator = 'euler'

        # Angle at which to fail the episode
        self.theta_threshold_radians = 12 * 2 * math.pi / 360 //角度的阈值
        self.x_threshold = 2.4

        # Angle limit set to 2 * theta_threshold_radians so failing observation is still within bounds
        high = np.array([
            self.x_threshold * 2,
            np.finfo(np.float32).max,
            self.theta_threshold_radians * 2,
            np.finfo(np.float32).max])

        self.action_space = spaces.Discrete(2)#动作空间，目前就两个动作
        self.observation_space = spaces.Box(-high, high, dtype=np.float32)#观测矩阵

        self.seed()
        self.viewer = None
        self.state = None

        self.steps_beyond_done = None

step函数：

    def step(self, action):
        assert self.action_space.contains(action), "%r (%s) invalid"%(action, type(action))
        state = self.state
        x, x_dot, theta, theta_dot = state #theta是角度，theta_dot是转速
        force = self.force_mag if action==1 else -self.force_mag #个人理解这代表系统水平方向上的受力
        costheta = math.cos(theta)
        sintheta = math.sin(theta)
        temp = (force + self.polemass_length * theta_dot * theta_dot * sintheta) / self.total_mass ##车摆的动力学方程式，即加速度与动作之间的关系。
        thetaacc = (self.gravity * sintheta - costheta* temp) / (self.length * (4.0/3.0 - self.masspole * costheta * costheta / self.total_mass)) #摆的角加速度
        xacc  = temp - self.polemass_length * thetaacc * costheta / self.total_mass #小车的平移加速
        if self.kinematics_integrator == 'euler':
            x  = x + self.tau * x_dot
            x_dot = x_dot + self.tau * xacc
            theta = theta + self.tau * theta_dot
            theta_dot = theta_dot + self.tau * thetaacc
        else: # semi-implicit euler
            x_dot = x_dot + self.tau * xacc
            x  = x + self.tau * x_dot
            theta_dot = theta_dot + self.tau * thetaacc
            theta = theta + self.tau * theta_dot
        self.state = (x,x_dot,theta,theta_dot)
        done =  x < -self.x_threshold \
                or x > self.x_threshold \
                or theta < -self.theta_threshold_radians \
                or theta > self.theta_threshold_radians
        done = bool(done)

        if not done:
            reward = 1.0
        elif self.steps_beyond_done is None:
            # Pole just fell!
            self.steps_beyond_done = 0
            reward = 1.0
        else:
            if self.steps_beyond_done == 0:
                logger.warn("You are calling 'step()' even though this environment has already returned done = True. You should always call 'reset()' once you receive 'done = True' -- any further steps are undefined behavior.")
            self.steps_beyond_done += 1
            reward = 0.0

        return np.array(self.state), reward, done, {}