完整代码:https://github.com/zle1992/Reinforcement_Learning_Game
Policy Gradient 可以直接预测出动作,也可以预测连续动作,但是无法单步更新。
QLearning 先预测出Q值,根据Q值选动作,无法预测连续动作、或者动作种类多的情况,但是可以单步更新。
一句话概括 Actor Critic 方法:
结合了 Policy Gradient (Actor) 和 Function Approximation (Critic) 的方法.
Actor
基于概率选行为 Critic
基于 Actor
的行为评判行为的得分,
Actor
根据 Critic
的评分修改选行为的概率.
Actor Critic 方法的优势: 可以进行单步更新, 比传统的 Policy Gradient 要快.
Actor Critic 方法的劣势: 取决于 Critic 的价值判断, 但是 Critic 难收敛, 再加上 Actor 的更新, 就更难收敛. 为了解决收敛问题, Google Deepmind 提出了 Actor Critic
升级版 Deep Deterministic Policy Gradient
. 后者融合了 DQN 的优势, 解决了收敛难的问题.
Actor Critic 方法与Policy Gradinet的区别:
Policy Gradinet 中的梯度下降 :
grad[logPi(s,a) * v_t]
其中v_t是真实的reward ,通过记录每个epoch的每一个state,action,reward得到。
而Actor中的v_t 是td_error 由Critic估计得到,不一定准确哦。
Actor Critic 方法与DQN的区别:
DQN 评价网络与动作网络其实是一个网络,只是采用了TD的方法,用滞后的网络去评价当前的动作。
Actor-Critic 就是在求解策略的同时用值函数进行辅助,用估计的值函数替代采样的reward,提高样本利用率。
Q-learning 是一种基于值函数估计的强化学习方法,Policy Gradient是一种策略搜索强化学习方法
Critic估计td_error跟DQN一样,用到了贝尔曼方程,
贝尔曼方程 :
Critic利用的是V函数的贝尔曼方程,来得到TD_error,
gradient = grad[r + gamma * V(s_) - V(s)]
Q-learning 利用的是Q函数的贝尔曼方程,来更新Q函数。
q_target = r + gamma * maxq(s_next)
q_eval = maxq(s)
Actor网络的输入(st,at,TDerror)
Actor 网络与policy gradient 差不多,多分类网络,在算loss时候,policy gradient需要乘一个权重Vt,而Vt是根据回报R 累计计算的。
在Actor中,在算loss时候,loss的权重是TDerror
TDerror是Critic网络计算出来的。
Critic网络的输入(st,vt+1,r),输出TDerror
V_eval = network(st)
# TD_error = (r+gamma*V_next) - V_eval
学习的时候输入:(st, r, st+1)
vt+1 = network(st+1)
Critic网络(st,vt+1,r)
ACNetwork.py
1 import os 2 import numpy as np 3 import tensorflow as tf 4 from abc import ABCMeta, abstractmethod 5 np.random.seed(1) 6 tf.set_random_seed(1) 7 8 import logging # 寮曞叆logging妯″潡 9 logging.basicConfig(level=logging.DEBUG, 10 format='%(asctime)s - %(filename)s[line:%(lineno)d] - %(levelname)s: %(message)s') # logging.basicConfig鍑芥暟瀵规棩蹇楃殑杈撳嚭鏍煎紡鍙婃柟寮忓仛鐩稿叧閰嶇疆 11 # 鐢变簬鏃ュ織鍩烘湰閰嶇疆涓骇鍒缃负DEBUG锛屾墍浠ヤ竴涓嬫墦鍗颁俊鎭皢浼氬叏閮ㄦ樉绀哄湪鎺у埗鍙颁笂 12 13 tfconfig = tf.ConfigProto() 14 tfconfig.gpu_options.allow_growth = True 15 session = tf.Session(config=tfconfig) 16 17 18 class ACNetwork(object): 19 __metaclass__ = ABCMeta 20 """docstring for ACNetwork""" 21 def __init__(self, 22 n_actions, 23 n_features, 24 learning_rate, 25 memory_size, 26 reward_decay, 27 output_graph, 28 log_dir, 29 model_dir, 30 ): 31 super(ACNetwork, self).__init__() 32 33 self.n_actions = n_actions 34 self.n_features = n_features 35 self.learning_rate=learning_rate 36 self.gamma=reward_decay 37 self.memory_size =memory_size 38 self.output_graph=output_graph 39 self.lr =learning_rate 40 41 self.log_dir = log_dir 42 43 self.model_dir = model_dir 44 # total learning step 45 self.learn_step_counter = 0 46 47 48 self.s = tf.placeholder(tf.float32,[None]+self.n_features,name='s') 49 self.s_next = tf.placeholder(tf.float32,[None]+self.n_features,name='s_next') 50 51 self.r = tf.placeholder(tf.float32,[None,],name='r') 52 self.a = tf.placeholder(tf.int32,[None,],name='a') 53 54 55 56 57 58 with tf.variable_scope('Critic'): 59 60 self.v = self._build_c_net(self.s, scope='v', trainable=True) 61 self.v_ = self._build_c_net(self.s_next, scope='v_next', trainable=False) 62 63 self.td_error =self.r + self.gamma * self.v_ - self.v 64 self.loss_critic = tf.square(self.td_error) 65 with tf.variable_scope('train'): 66 self.train_op_critic = tf.train.AdamOptimizer(self.lr).minimize(self.loss_critic) 67 68 69 70 with tf.variable_scope('Actor'): 71 self.acts_prob = self._build_a_net(self.s, scope='actor_net', trainable=True) 72 # this is negative log of chosen action 73 log_prob = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=self.acts_prob, labels=self.a) 74 75 self.loss_actor = tf.reduce_mean(log_prob*self.td_error) 76 with tf.variable_scope('train'): 77 self.train_op_actor = tf.train.AdamOptimizer(self.lr).minimize(-self.loss_actor) 78 79 80 self.sess = tf.Session() 81 if self.output_graph: 82 tf.summary.FileWriter(self.log_dir,self.sess.graph) 83 84 self.sess.run(tf.global_variables_initializer()) 85 86 self.cost_his =[0] 87 88 89 self.saver = tf.train.Saver() 90 91 if not os.path.exists(self.model_dir): 92 os.mkdir(self.model_dir) 93 94 checkpoint = tf.train.get_checkpoint_state(self.model_dir) 95 if checkpoint and checkpoint.model_checkpoint_path: 96 self.saver.restore(self.sess, checkpoint.model_checkpoint_path) 97 print ("Loading Successfully") 98 self.learn_step_counter = int(checkpoint.model_checkpoint_path.split('-')[-1]) + 1 99 100 101 @abstractmethod 102 def _build_a_net(self,x,scope,trainable): 103 104 raise NotImplementedError 105 def _build_c_net(self,x,scope,trainable): 106 107 raise NotImplementedError 108 def learn(self,data): 109 110 111 112 113 batch_memory_s = data['s'] 114 batch_memory_a = data['a'] 115 batch_memory_r = data['r'] 116 batch_memory_s_ = data['s_'] 117 118 119 120 _, cost = self.sess.run( 121 [self.train_op_critic, self.loss_critic], 122 feed_dict={ 123 self.s: batch_memory_s, 124 self.a: batch_memory_a, 125 self.r: batch_memory_r, 126 self.s_next: batch_memory_s_, 127 128 }) 129 130 _, cost = self.sess.run( 131 [self.train_op_actor, self.loss_actor], 132 feed_dict={ 133 self.s: batch_memory_s, 134 self.a: batch_memory_a, 135 self.r: batch_memory_r, 136 self.s_next: batch_memory_s_, 137 138 }) 139 140 141 self.cost_his.append(cost) 142 143 self.learn_step_counter += 1 144 # save network every 100000 iteration 145 if self.learn_step_counter % 10000 == 0: 146 self.saver.save(self.sess,self.model_dir,global_step=self.learn_step_counter) 147 148 149 150 def choose_action(self,s): 151 s = s[np.newaxis,:] 152 153 probs = self.sess.run(self.acts_prob,feed_dict={self.s:s}) 154 return np.random.choice(np.arange(probs.shape[1]), p=probs.ravel())
game.py
1 import sys 2 import gym 3 import numpy as np 4 import tensorflow as tf 5 sys.path.append('./') 6 sys.path.append('model') 7 8 from util import Memory ,StateProcessor 9 from ACNetwork import ACNetwork 10 np.random.seed(1) 11 tf.set_random_seed(1) 12 13 import logging # 引入logging模块 14 logging.basicConfig(level=logging.DEBUG, 15 format='%(asctime)s - %(filename)s[line:%(lineno)d] - %(levelname)s: %(message)s') # logging.basicConfig函数对日志的输出格式及方式做相关配置 16 # 由于日志基本配置中级别设置为DEBUG,所以一下打印信息将会全部显示在控制台上 17 import os 18 os.environ["CUDA_VISIBLE_DEVICES"] = "1" 19 tfconfig = tf.ConfigProto() 20 tfconfig.gpu_options.allow_growth = True 21 session = tf.Session(config=tfconfig) 22 23 24 25 class ACNetwork4CartPole(ACNetwork): 26 """docstring for ClassName""" 27 def __init__(self, **kwargs): 28 super(ACNetwork4CartPole, self).__init__(**kwargs) 29 30 def _build_a_net(self,x,scope,trainable): 31 w_initializer, b_initializer = tf.random_normal_initializer(0., 0.3), tf.constant_initializer(0.1) 32 33 with tf.variable_scope(scope): 34 e1 = tf.layers.dense(inputs=x, 35 units=32, 36 bias_initializer = b_initializer, 37 kernel_initializer=w_initializer, 38 activation = tf.nn.relu, 39 trainable=trainable) 40 q = tf.layers.dense(inputs=e1, 41 units=self.n_actions, 42 bias_initializer = b_initializer, 43 kernel_initializer=w_initializer, 44 activation = tf.nn.softmax, 45 trainable=trainable) 46 47 return q 48 49 def _build_c_net(self,x,scope,trainable): 50 w_initializer, b_initializer = tf.random_normal_initializer(0., 0.3), tf.constant_initializer(0.1) 51 52 with tf.variable_scope(scope): 53 e1 = tf.layers.dense(inputs=x, 54 units=32, 55 bias_initializer = b_initializer, 56 kernel_initializer=w_initializer, 57 activation = tf.nn.relu, 58 trainable=trainable) 59 q = tf.layers.dense(inputs=e1, 60 units=1, 61 bias_initializer = b_initializer, 62 kernel_initializer=w_initializer, 63 activation =None, 64 trainable=trainable) 65 66 return q 67 68 69 70 batch_size = 32 71 72 memory_size =100 73 #env = gym.make('Breakout-v0') #离散 74 env = gym.make('CartPole-v0') #离散 75 76 77 n_features= list(env.observation_space.shape) 78 n_actions= env.action_space.n 79 env = env.unwrapped 80 81 def run(): 82 83 RL = ACNetwork4CartPole( 84 n_actions=n_actions, 85 n_features=n_features, 86 learning_rate=0.01, 87 reward_decay=0.9, 88 89 memory_size=memory_size, 90 91 output_graph=True, 92 log_dir = 'log/ACNetwork4CartPole/', 93 94 model_dir = 'model_dir/ACNetwork4CartPole/' 95 ) 96 97 memory = Memory(n_actions,n_features,memory_size=memory_size) 98 99 100 step = 0 101 ep_r = 0 102 for episode in range(2000): 103 # initial observation 104 observation = env.reset() 105 106 while True: 107 108 109 # RL choose action based on observation 110 action = RL.choose_action(observation) 111 # logging.debug('action') 112 # print(action) 113 # RL take action and get_collectiot next observation and reward 114 observation_, reward, done, info=env.step(action) # take a random action 115 116 # the smaller theta and closer to center the better 117 x, x_dot, theta, theta_dot = observation_ 118 r1 = (env.x_threshold - abs(x))/env.x_threshold - 0.8 119 r2 = (env.theta_threshold_radians - abs(theta))/env.theta_threshold_radians - 0.5 120 reward = r1 + r2 121 122 123 124 125 memory.store_transition(observation, action, reward, observation_) 126 127 128 if (step > 200) and (step % 1 == 0): 129 130 data = memory.sample(batch_size) 131 RL.learn(data) 132 #print('step:%d----reward:%f---action:%d'%(step,reward,action)) 133 # swap observation 134 observation = observation_ 135 ep_r += reward 136 # break while loop when end of this episode 137 if(episode>700): 138 env.render() # render on the screen 139 if done: 140 print('step: ',step, 141 'episode: ', episode, 142 'ep_r: ', round(ep_r, 2), 143 'loss: ',RL.cost_his[-1] 144 ) 145 ep_r = 0 146 147 break 148 step += 1 149 150 # end of game 151 print('game over') 152 env.destroy() 153 154 def main(): 155 156 run() 157 158 159 160 if __name__ == '__main__': 161 main() 162 #run2()