【demo】DDQN玩gym之立木杆

import gym
from contents.OpenAI_gym.my_rl_brain import Double_DQN_Agent

my_env = gym.make("CartPole-v0")

my_env = my_env.unwrapped


print("observation:",my_env.observation_space)
print("observation_max",my_env.observation_space.high)
print("observation_mim",my_env.observation_space.low)
print("actions:",my_env.action_space)

my_agent = Double_DQN_Agent(dim_observation=my_env.observation_space.shape[0],
                            dim_action=my_env.action_space.n,
                            reward_decay=0.95,
                            learning_rate=0.05,
                            bacth_size=32,
                            memory_size=800,
                            epsilon=0.7,
                            output_graph=True)

total_steps = 0

for i in range(1000):

    i = my_agent.episode

    observation = my_env.reset()

    epi_reward = 0

    while True:

        my_env.render()

        action = my_agent.choose_action(observation)

        observation_, reward, done, info  = my_env.step(action)

        # 构造reward
        x, x_dot, theta, theta_dot = observation_
        r1 = (my_env.x_threshold - abs(x)) / my_env.x_threshold - 0.8
        r2 = (my_env.theta_threshold_radians - abs(theta)) / my_env.theta_threshold_radians - 0.5
        reward = 0.6*r1 + 0.4*r2

        epi_reward = epi_reward + reward

        my_agent.store_trainsition(observation,action,reward,observation_)

        total_steps = total_steps + 1

        if total_steps > 1000:
            my_agent.learning()

        if done:
            break

        observation = observation_

    my_agent.episode = my_agent.episode + 1

    print("当前第%s轮episode，总reward为%s，当前轮的贪婪值：%s" % (i, epi_reward, my_agent.epsilon))

my_agent.plot_cost()

import tensorflow as tf
import numpy as np
import pandas as pd
import random


class Double_DQN_Agent:

    def __init__(self,dim_observation,dim_action,reward_decay,learning_rate,bacth_size,memory_size,epsilon,output_graph):
        self.dim_observation = dim_observation
        self.dim_action = dim_action
        self.gamma = reward_decay
        self.memory_size = memory_size
        self.learning_rate = learning_rate
        self.batch_size = bacth_size
        self.episode = 0
        self.epsilon = epsilon

        self.memory = np.zeros((memory_size,dim_observation*2+2))

        print("dim_observation",dim_observation)
        print("dim_action",dim_action)

        self.trainsition_count = 0

        num_steps_sum  = 0

        self.target_params = tf.get_collection("target_network_para")
        self.evaluate_params = tf.get_collection("evaluate_network_para")

        self.sess = tf.Session()

        if output_graph:
            tf.summary.FileWriter("logs/", self.sess.graph)

        self.sess.run(tf.global_variables_initializer())

        self.cost_list = []

        self.episode_count = 0

        self.epsilon_base = epsilon

        self.build_network()

        self.sess.run(tf.global_variables_initializer())

        self.step_counter = 0




    def replace_target(self):
        # 把target_work 参数 赋值给 evaluate_network
        for target,evaluate in zip(self.target_params,self.evaluate_params):
            tf.assign(target,evaluate)


    def build_network(self):

        #--------------------------------------- 定义 evaluate_network------------------------------------------------

        # 定义输入
        self.state = tf.placeholder(tf.float32,[None,self.dim_observation],name="input_state")
        self.q_target = tf.placeholder(tf.float32,[None,self.dim_action],name = "input_q_target")

        # 定义网络结构和参数集合

        with tf.variable_scope("evaluate_network"):
            c_names,n_l1,w_initializer,b_initializer = ["evaluate_network_para",tf.GraphKeys.GLOBAL_VARIABLES],\
                                                       10,\
                                                       tf.random_normal_initializer(0.0,0.3),\
                                                       tf.constant_initializer(0.2)

            with tf.variable_scope("l1"):
                w1 = tf.get_variable("w1", [self.dim_observation,n_l1],initializer=w_initializer,collections=c_names)
               #w1 = tf.get_variable('w1', [self.n_features, n_l1], initializer=w_initializer, collections=c_names)
                b1 = tf.get_variable("b1",[1,n_l1],initializer=b_initializer,collections=c_names)
                l1 = tf.nn.relu(tf.matmul(self.state,w1) + b1)

            with tf.variable_scope("l2"):
                w2 = tf.get_variable("w2",[n_l1,self.dim_action],initializer=w_initializer,collections=c_names)
                b2 = tf.get_variable("b2",[1,self.dim_action],initializer=b_initializer,collections=c_names )
                self.q_eval = tf.matmul(l1,w2) + b2

        with tf.variable_scope("loss"):
            self.loss = tf.reduce_mean(tf.squared_difference(self.q_eval,self.q_target))

        with tf.variable_scope("train"):
            self.train_op = tf.train.RMSPropOptimizer(self.learning_rate).minimize(self.loss)

        #--------------------------------------- 定义 target_network------------------------------------------------

        self.state_next = tf.placeholder(tf.float32,[None,self.dim_observation],name = "input_next_state")
        with tf.variable_scope("target_network"):
            c_names,n_l1,w_initializer,b_initializer = ["target_network_para",tf.GraphKeys.GLOBAL_VARIABLES],10,tf.random_normal_initializer(0.,0.3),tf.constant_initializer(0.2)

            with tf.variable_scope("l1"):
                w1 = tf.get_variable("w1", [self.dim_observation, n_l1], initializer=w_initializer,
                                     collections=c_names)
                b1 = tf.get_variable("b1", [1, n_l1], initializer=b_initializer, collections=c_names)
                l1 = tf.nn.relu(tf.matmul(self.state_next,w1) + b1)

            with tf.variable_scope("l2"):
                w2 = tf.get_variable("w1", [n_l1, self.dim_action], initializer=w_initializer,
                                     collections=c_names)
                b2 = tf.get_variable("b2", [1, self.dim_action], initializer=b_initializer, collections=c_names)
                self.q_next = tf.matmul(l1,w2) + b2


    def store_trainsition(self,s,a,r,s_):
        item = np.hstack((s,[a,r],s_))
        index = self.trainsition_count%self.memory_size
        self.memory[index,:] = item
        self.trainsition_count = self.trainsition_count + 1

    def choose_action(self,observation):
        random_value = random.random()
        observation = observation[np.newaxis,:]
        if random_value < self.epsilon: # 举例： 90%的概率选择 Qmax的action
            action_values = self.sess.run(self.q_eval,feed_dict={self.state:observation})
       #    action_values = self.sess.run(self.q_eval,feed_dict={self.s    :observation})

            action = np.argmax(action_values)
        else:
            action = np.random.randint(0,self.dim_action)
        return action


    def learning(self):


        if self.episode>90:
            self.epsilon = 1-(1-self.epsilon_base)*(400/(400+self.episode))
        else:
            self.epsilon = self.epsilon_base

        #print("当前transition数量：%s"%self.trainsition_count)

        if self.trainsition_count<self.batch_size:
            print("经验不足，继续探索")


        if self.step_counter%50 ==0:
            self.replace_target()
            #print("参数覆盖！")


        if self.batch_size<=self.trainsition_count and self.trainsition_count<=self.memory_size:
            #print("self.trainsition_count",self.trainsition_count)
            sample_index = np.random.choice(self.trainsition_count, size=self.batch_size)
        else:
            #print("self.memory_size",self.memory_size)
            sample_index = np.random.choice(self.memory_size, size=self.batch_size)

        batch = self.memory[sample_index, : ]

        #  bacth中每一行的格式为： s,a,r,s_
        #  学习过程：根据batch，拆成 s a r s_
        #          (1)  s 结合 实时更新的evaluate_network 生成 q_evaluate
        #         （2）  根据r ,以及，s_结合迟滞更新的target_network得到max的q_next，得到q_target = r + gamma*q_next

        s = batch[:,:self.dim_observation]
        r = batch[:,self.dim_observation+1]
        s_ = batch[:,-self.dim_observation:]
        a = batch[:,self.dim_observation].astype(int)


        q_next,q_evaluate = self.sess.run([self.q_next,self.q_eval],feed_dict={self.state_next:s_,self.state:s})

        # 接下来计算q_target，用q_target来计算self.loss ,对q_target = r + gamma*q_next进行改写
        q_target = q_evaluate.copy()

        q_target[:,a] = r+self.gamma*np.max(q_next,axis = 1)

        _,cost = self.sess.run([self.train_op,self.loss],feed_dict={self.q_target:q_target,self.state:s})  # 只针对evaluate_network训练


        #print("第 %s 轮的cost：%s ,当前epsilon大小为%s"%(self.episode,cost,self.epsilon))



        self.cost_list.append(cost)


    def plot_cost(self):
        import matplotlib.pyplot as plt
        plt.plot(np.arange(len(self.cost_list)), self.cost_list)
        plt.ylabel('Cost')
        plt.xlabel('training steps')
        plt.show()

• 明确一点，DQN的input是连续的observation。action的数量是固定的，离散的，已经embedding到网络结构当中了。也就是说DQN处理的是：state连续，action离散且固定的一类强化学习问题。
• Double-DQN当中有两个网络：evaluate-network 和 target-network。
  • loss也就是error来自于：q_evaluate-（r+gamma*q_next）
  • 得到的loss第一时间更新“用来生成q_evaluate的evaluate-network”，几个时间步后通过参数覆盖的方式迟滞更新target-network
• DQN是不存在收敛的，只不过震动幅度大大缩小【直觉】
• DQN是一种“值函数近似方法”。值函数近似方法指的是：对于一个指定的state，可以由函数计算得到该状态的state-value以及action-state-value。
  • 问题来了：当同一个state对应的action非常多的时候，网络非常难以训练，再极端一点，动作是连续的无限的，这种方法彻底失效了。此时需要policy-gradient方法。
  • 基于value策略的问题的思想都是：从众多行为价值中选择一个 最大价值的行为，这是一个确定性策略。但是很多时候最优策略是随机策略，于是基于value的策略失效了。
  • 基于value的强化学习可以解决很多问题，但是当“action连续/observation部分可见/随机策略比最优策略更合适”等问题出现时，value-base就不太行了。【于是，才引入了policy-based方法】