DQN解决MountainCar
安装依赖
!pip uninstall -y parl # 说明:AIStudio预装的parl版本太老,容易跟其他库产生兼容性冲突,建议先卸载
!pip uninstall -y pandas scikit-learn # 提示:在AIStudio中卸载这两个库再import parl可避免warning提示,不卸载也不影响parl的使用
!pip install gym
!pip install paddlepaddle==1.6.3
!pip install parl==1.3.1
# 说明:安装日志中出现两条红色的关于 paddlehub 和 visualdl 的 ERROR 与parl无关,可以忽略,不影响使用
导入依赖
import parl
from parl import layers
import paddle.fluid as fluid
import copy
import numpy as np
import os
import gym
from parl.utils import logger
LEARN_FREQ = 5 # 训练频率,不需要每一个step都learn,攒一些新增经验后再learn,提高效率
MEMORY_SIZE = 20000 # replay memory的大小,越大越占用内存
MEMORY_WARMUP_SIZE = 200 # replay_memory 里需要预存一些经验数据,再从里面sample一个batch的经验让agent去learn
BATCH_SIZE = 32 # 每次给agent learn的数据数量,从replay memory随机里sample一批数据出来
GAMMA = 0.99 # reward 的衰减因子,一般取 0.9 到 0.999 不等
LEARNING_RATE = 0.01 # 学习率
搭建Model、Algorithm、Agent架构
Agent把产生的数据传给algorithm,algorithm根据model的模型结构计算出Loss,使用SGD或者其他优化器不断的优化,PARL架构可以很方便的应用在各类深度强化学习问题中。
class Model(parl.Model):
def __init__(self, act_dim):
hid1_size = 128
hid2_size = 128
# 3层全连接网络
self.fc1 = layers.fc(size=hid1_size, act='relu')
self.fc2 = layers.fc(size=hid2_size, act='relu')
self.fc3 = layers.fc(size=act_dim, act=None)
def value(self, obs):
# 定义网络
# 输入state,输出所有action对应的Q,[Q(s,a1), Q(s,a2), Q(s,a3)...]
h1 = self.fc1(obs)
h2 = self.fc2(h1)
Q = self.fc3(h2)
return Q
from parl.algorithms import DQN # 直接从parl库中导入DQN算法,无需自己重写算法
class Agent(parl.Agent):
def __init__(self,
algorithm,
obs_dim,
act_dim,
e_greed=0.1,
e_greed_decrement=0):
assert isinstance(obs_dim, int)
assert isinstance(act_dim, int)
self.obs_dim = obs_dim
self.act_dim = act_dim
super(Agent, self).__init__(algorithm)
self.global_step = 0
self.update_target_steps = 200 # 每隔200个training steps再把model的参数复制到target_model中
self.e_greed = e_greed # 有一定概率随机选取动作,探索
self.e_greed_decrement = e_greed_decrement # 随着训练逐步收敛,探索的程度慢慢降低
def build_program(self):
self.pred_program = fluid.Program()
self.learn_program = fluid.Program()
with fluid.program_guard(self.pred_program): # 搭建计算图用于 预测动作,定义输入输出变量
obs = layers.data(
name='obs', shape=[self.obs_dim], dtype='float32')
self.value = self.alg.predict(obs)
with fluid.program_guard(self.learn_program): # 搭建计算图用于 更新Q网络,定义输入输出变量
obs = layers.data(
name='obs', shape=[self.obs_dim], dtype='float32')
action = layers.data(name='act', shape=[1], dtype='int32')
reward = layers.data(name='reward', shape=[], dtype='float32')
next_obs = layers.data(
name='next_obs', shape=[self.obs_dim], dtype='float32')
terminal = layers.data(name='terminal', shape=[], dtype='bool')
self.cost = self.alg.learn(obs, action, reward, next_obs, terminal)
def sample(self, obs):
sample = np.random.rand() # 产生0~1之间的小数
if sample < self.e_greed:
act = np.random.randint(self.act_dim) # 探索:每个动作都有概率被选择
else:
act = self.predict(obs) # 选择最优动作
self.e_greed = max(
0.01, self.e_greed - self.e_greed_decrement) # 随着训练逐步收敛,探索的程度慢慢降低
return act
def predict(self, obs): # 选择最优动作
obs = np.expand_dims(obs, axis=0)
pred_Q = self.fluid_executor.run(
self.pred_program,
feed={'obs': obs.astype('float32')},
fetch_list=[self.value])[0]
pred_Q = np.squeeze(pred_Q, axis=0)
act = np.argmax(pred_Q) # 选择Q最大的下标,即对应的动作
return act
def learn(self, obs, act, reward, next_obs, terminal):
# 每隔200个training steps同步一次model和target_model的参数
if self.global_step % self.update_target_steps == 0:
self.alg.sync_target()
self.global_step += 1
act = np.expand_dims(act, -1)
feed = {
'obs': obs.astype('float32'),
'act': act.astype('int32'),
'reward': reward,
'next_obs': next_obs.astype('float32'),
'terminal': terminal
}
cost = self.fluid_executor.run(
self.learn_program, feed=feed, fetch_list=[self.cost])[0] # 训练一次网络
return cost
**ReplayMemory经验池:**用于存储多条经验,实现 经验回放。
# replay_memory.py
import random
import collections
import numpy as np
class ReplayMemory(object):
def __init__(self, max_size):
self.buffer = collections.deque(maxlen=max_size)
# 增加一条经验到经验池中
def append(self, exp):
self.buffer.append(exp)
# 从经验池中选取N条经验出来
def sample(self, batch_size):
mini_batch = random.sample(self.buffer, batch_size)
obs_batch, action_batch, reward_batch, next_obs_batch, done_batch = [], [], [], [], []
for experience in mini_batch:
s, a, r, s_p, done = experience
obs_batch.append(s)
action_batch.append(a)
reward_batch.append(r)
next_obs_batch.append(s_p)
done_batch.append(done)
return np.array(obs_batch).astype('float32'), \
np.array(action_batch).astype('float32'), np.array(reward_batch).astype('float32'),\
np.array(next_obs_batch).astype('float32'), np.array(done_batch).astype('float32')
def __len__(self):
return len(self.buffer)
训练和测试
# 训练一个episode
def run_episode(env, agent, rpm):
total_reward = 0
obs = env.reset()
step = 0
while True:
step += 1
action = agent.sample(obs) # 采样动作,所有动作都有概率被尝试到
next_obs, reward, done, _ = env.step(action)
rpm.append((obs, action, reward, next_obs, done))
# train model
if (len(rpm) > MEMORY_WARMUP_SIZE) and (step % LEARN_FREQ == 0):
(batch_obs, batch_action, batch_reward, batch_next_obs,
batch_done) = rpm.sample(BATCH_SIZE)
train_loss = agent.learn(batch_obs, batch_action, batch_reward,
batch_next_obs,
batch_done) # s,a,r,s',done
total_reward += reward
obs = next_obs
if done:
break
return total_reward
# 评估 agent, 跑 5 个episode,总reward求平均
def evaluate(env, agent, render=False):
eval_reward = []
for i in range(5):
obs = env.reset()
episode_reward = 0
while True:
action = agent.predict(obs) # 预测动作,只选最优动作
obs, reward, done, _ = env.step(action)
episode_reward += reward
if render:
env.render()
if done:
break
eval_reward.append(episode_reward)
return np.mean(eval_reward)
创建环境和Agent,创建经验池,启动训练,保存模型
# 创建环境
env = gym.make('MountainCar-v0')
action_dim = env.action_space.n # MountainCar-v0: 3
obs_shape = env.observation_space.shape # MountainCar-v0: (2,)
# 创建经验池
rpm = ReplayMemory(MEMORY_SIZE) # DQN的经验回放池
# 根据parl框架构建agent
######################################################################
######################################################################
#
# 4. 请参考课堂Demo,嵌套Model, DQN, Agent构建 agent
#
######################################################################
######################################################################
model = Model(act_dim=action_dim)
algorithm = DQN(model, act_dim=action_dim, gamma=GAMMA, lr=LEARNING_RATE)
agent = Agent(
algorithm,
obs_dim=obs_shape[0],
act_dim=action_dim,
e_greed=0.1, # 有一定概率随机选取动作,探索
e_greed_decrement=1e-6) # 随着训练逐步收敛,探索的程度慢慢降低
# 加载模型
# save_path = './dqn_model.ckpt'
# agent.restore(save_path)
# 先往经验池里存一些数据,避免最开始训练的时候样本丰富度不够
while len(rpm) < MEMORY_WARMUP_SIZE:
run_episode(env, agent, rpm)
max_episode = 2000
# 开始训练
episode = 0
while episode < max_episode: # 训练max_episode个回合,test部分不计算入episode数量
# train part
for i in range(0, 50):
total_reward = run_episode(env, agent, rpm)
episode += 1
# test part
eval_reward = evaluate(env, agent, render=False) # render=True 查看显示效果
logger.info('episode:{} e_greed:{} test_reward:{}'.format(
episode, agent.e_greed, eval_reward))
# 训练结束,保存模型
save_path = './dqn_model.ckpt'
agent.save(save_path)
最终结果:
[06-18 22:16:08 MainThread @machine_info.py:88] Cannot find available GPU devices, using CPU now.
[06-18 22:16:08 MainThread @machine_info.py:88] Cannot find available GPU devices, using CPU now.
[06-18 22:16:08 MainThread @machine_info.py:88] Cannot find available GPU devices, using CPU now.
[06-18 22:16:46 MainThread @<ipython-input-12-0c89431cde62>:51] episode:50 e_greed:0.09023499999999024 test_reward:-200.0
[06-18 22:17:25 MainThread @<ipython-input-12-0c89431cde62>:51] episode:100 e_greed:0.08063599999998064 test_reward:-200.0
[06-18 22:18:05 MainThread @<ipython-input-12-0c89431cde62>:51] episode:150 e_greed:0.0708979999999709 test_reward:-200.0
[06-18 22:18:45 MainThread @<ipython-input-12-0c89431cde62>:51] episode:200 e_greed:0.061383999999961386 test_reward:-200.0
[06-18 22:19:25 MainThread @<ipython-input-12-0c89431cde62>:51] episode:250 e_greed:0.05172599999995173 test_reward:-200.0
[06-18 22:20:06 MainThread @<ipython-input-12-0c89431cde62>:51] episode:300 e_greed:0.04198799999994199 test_reward:-200.0
[06-18 22:20:45 MainThread @<ipython-input-12-0c89431cde62>:51] episode:350 e_greed:0.03274799999993275 test_reward:-200.0
[06-18 22:21:23 MainThread @<ipython-input-12-0c89431cde62>:51] episode:400 e_greed:0.02389099999992389 test_reward:-149.2
[06-18 22:22:01 MainThread @<ipython-input-12-0c89431cde62>:51] episode:450 e_greed:0.015057999999916041 test_reward:-200.0
[06-18 22:22:41 MainThread @<ipython-input-12-0c89431cde62>:51] episode:500 e_greed:0.01 test_reward:-200.0
[06-18 22:23:20 MainThread @<ipython-input-12-0c89431cde62>:51] episode:550 e_greed:0.01 test_reward:-200.0
[06-18 22:23:56 MainThread @<ipython-input-12-0c89431cde62>:51] episode:600 e_greed:0.01 test_reward:-200.0
[06-18 22:24:32 MainThread @<ipython-input-12-0c89431cde62>:51] episode:650 e_greed:0.01 test_reward:-168.0
[06-18 22:25:07 MainThread @<ipython-input-12-0c89431cde62>:51] episode:700 e_greed:0.01 test_reward:-200.0
[06-18 22:25:48 MainThread @<ipython-input-12-0c89431cde62>:51] episode:750 e_greed:0.01 test_reward:-200.0
[06-18 22:26:28 MainThread @<ipython-input-12-0c89431cde62>:51] episode:800 e_greed:0.01 test_reward:-104.2
[06-18 22:27:02 MainThread @<ipython-input-12-0c89431cde62>:51] episode:850 e_greed:0.01 test_reward:-141.2
[06-18 22:27:39 MainThread @<ipython-input-12-0c89431cde62>:51] episode:900 e_greed:0.01 test_reward:-112.4
[06-18 22:28:18 MainThread @<ipython-input-12-0c89431cde62>:51] episode:950 e_greed:0.01 test_reward:-200.0
[06-18 22:28:59 MainThread @<ipython-input-12-0c89431cde62>:51] episode:1000 e_greed:0.01 test_reward:-200.0
[06-18 22:29:34 MainThread @<ipython-input-12-0c89431cde62>:51] episode:1050 e_greed:0.01 test_reward:-126.0
[06-18 22:30:13 MainThread @<ipython-input-12-0c89431cde62>:51] episode:1100 e_greed:0.01 test_reward:-163.8
[06-18 22:30:49 MainThread @<ipython-input-12-0c89431cde62>:51] episode:1150 e_greed:0.01 test_reward:-140.2
[06-18 22:31:26 MainThread @<ipython-input-12-0c89431cde62>:51] episode:1200 e_greed:0.01 test_reward:-200.0
[06-18 22:31:59 MainThread @<ipython-input-12-0c89431cde62>:51] episode:1250 e_greed:0.01 test_reward:-138.4
[06-18 22:32:31 MainThread @<ipython-input-12-0c89431cde62>:51] episode:1300 e_greed:0.01 test_reward:-117.4
[06-18 22:33:03 MainThread @<ipython-input-12-0c89431cde62>:51] episode:1350 e_greed:0.01 test_reward:-151.4
[06-18 22:33:33 MainThread @<ipython-input-12-0c89431cde62>:51] episode:1400 e_greed:0.01 test_reward:-139.4
[06-18 22:34:09 MainThread @<ipython-input-12-0c89431cde62>:51] episode:1450 e_greed:0.01 test_reward:-200.0
[06-18 22:34:43 MainThread @<ipython-input-12-0c89431cde62>:51] episode:1500 e_greed:0.01 test_reward:-180.2
[06-18 22:35:16 MainThread @<ipython-input-12-0c89431cde62>:51] episode:1550 e_greed:0.01 test_reward:-169.0
[06-18 22:35:50 MainThread @<ipython-input-12-0c89431cde62>:51] episode:1600 e_greed:0.01 test_reward:-163.6
[06-18 22:36:25 MainThread @<ipython-input-12-0c89431cde62>:51] episode:1650 e_greed:0.01 test_reward:-139.4
[06-18 22:36:57 MainThread @<ipython-input-12-0c89431cde62>:51] episode:1700 e_greed:0.01 test_reward:-170.6
[06-18 22:37:33 MainThread @<ipython-input-12-0c89431cde62>:51] episode:1750 e_greed:0.01 test_reward:-143.8
[06-18 22:38:08 MainThread @<ipython-input-12-0c89431cde62>:51] episode:1800 e_greed:0.01 test_reward:-172.0
[06-18 22:38:41 MainThread @<ipython-input-12-0c89431cde62>:51] episode:1850 e_greed:0.01 test_reward:-115.4
[06-18 22:39:12 MainThread @<ipython-input-12-0c89431cde62>:51] episode:1900 e_greed:0.01 test_reward:-149.4
[06-18 22:39:42 MainThread @<ipython-input-12-0c89431cde62>:51] episode:1950 e_greed:0.01 test_reward:-133.6
[06-18 22:40:14 MainThread @<ipython-input-12-0c89431cde62>:51] episode:2000 e_greed:0.01 test_reward:-112.4
PG解决Pong
安装依赖
!pip uninstall -y parl # 说明:AIStudio预装的parl版本太老,容易跟其他库产生兼容性冲突,建议先卸载
!pip uninstall -y pandas scikit-learn # 提示:在AIStudio中卸载这两个库再import parl可避免warning提示,不卸载也不影响parl的使用
!pip install gym
!pip install atari-py # 玩Gym的Atari游戏必装依赖,本次作业使用了Atari的Pong(乒乓球)环境
!pip install paddlepaddle==1.6.3
!pip install parl==1.3.1
# 说明:安装日志中出现两条红色的关于 paddlehub 和 visualdl 的 ERROR 与parl无关,可以忽略,不影响使用
# 检查依赖包版本是否正确
!pip list | grep paddlepaddle
!pip list | grep parl
导入依赖
import os
import gym
import numpy as np
import paddle.fluid as fluid
import parl
from parl import layers
from parl.utils import logger
设置超参数
LEARNING_RATE = 0.005
搭建Model、Algorithm、Agent架构
Agent把产生的数据传给algorithm,algorithm根据model的模型结构计算出Loss,使用SGD或者其他优化器不断的优化,PARL这种架构可以很方便的应用在各类深度强化学习问题中。
class Model(parl.Model):
def __init__(self, act_dim):
act_dim = act_dim
hid1_size = act_dim * 10
self.fc1 = layers.fc(size=hid1_size, act='tanh')
self.fc2 = layers.fc(size=act_dim, act='softmax')
def forward(self, obs): # 可直接用 model = Model(5); model(obs)调用
out = self.fc1(obs)
out = self.fc2(out)
return out
from parl.algorithms import PolicyGradient # 直接从parl库中导入PolicyGradient算法,无需重复写算法
class Agent(parl.Agent):
def __init__(self, algorithm, obs_dim, act_dim):
self.obs_dim = obs_dim
self.act_dim = act_dim
super(Agent, self).__init__(algorithm)
def build_program(self):
self.pred_program = fluid.Program()
self.learn_program = fluid.Program()
with fluid.program_guard(self.pred_program): # 搭建计算图用于 预测动作,定义输入输出变量
obs = layers.data(
name='obs', shape=[self.obs_dim], dtype='float32')
self.act_prob = self.alg.predict(obs)
with fluid.program_guard(
self.learn_program): # 搭建计算图用于 更新policy网络,定义输入输出变量
obs = layers.data(
name='obs', shape=[self.obs_dim], dtype='float32')
act = layers.data(name='act', shape=[1], dtype='int64')
reward = layers.data(name='reward', shape=[], dtype='float32')
self.cost = self.alg.learn(obs, act, reward)
def sample(self, obs):
obs = np.expand_dims(obs, axis=0) # 增加一维维度
act_prob = self.fluid_executor.run(
self.pred_program,
feed={'obs': obs.astype('float32')},
fetch_list=[self.act_prob])[0]
act_prob = np.squeeze(act_prob, axis=0) # 减少一维维度
act = np.random.choice(range(self.act_dim), p=act_prob) # 根据动作概率选取动作
return act
def predict(self, obs):
obs = np.expand_dims(obs, axis=0)
act_prob = self.fluid_executor.run(
self.pred_program,
feed={'obs': obs.astype('float32')},
fetch_list=[self.act_prob])[0]
act_prob = np.squeeze(act_prob, axis=0)
act = np.argmax(act_prob) # 根据动作概率选择概率最高的动作
return act
def learn(self, obs, act, reward):
act = np.expand_dims(act, axis=-1)
feed = {
'obs': obs.astype('float32'),
'act': act.astype('int64'),
'reward': reward.astype('float32')
}
cost = self.fluid_executor.run(
self.learn_program, feed=feed, fetch_list=[self.cost])[0]
return cost
训练和测试
def run_episode(env, agent):
obs_list, action_list, reward_list = [], [], []
obs = env.reset()
while True:
obs = preprocess(obs) # from shape (210, 160, 3) to (100800,)
obs_list.append(obs)
action = agent.sample(obs) # 采样动作
action_list.append(action)
obs, reward, done, info = env.step(action)
reward_list.append(reward)
if done:
break
return obs_list, action_list, reward_list
# 评估 agent, 跑 5 个episode,求平均
def evaluate(env, agent, render=False):
eval_reward = []
for i in range(5):
obs = env.reset()
episode_reward = 0
while True:
obs = preprocess(obs) # from shape (210, 160, 3) to (100800,)
action = agent.predict(obs) # 选取最优动作
obs, reward, isOver, _ = env.step(action)
episode_reward += reward
if render:
env.render()
if isOver:
break
eval_reward.append(episode_reward)
return np.mean(eval_reward)
创建环境和Agent,启动训练,保存模型
# Pong 图片预处理
def preprocess(image):
""" 预处理 210x160x3 uint8 frame into 6400 (80x80) 1维 float vector """
image = image[35:195] # 裁剪
image = image[::2,::2,0] # 下采样,缩放2倍
image[image == 144] = 0 # 擦除背景 (background type 1)
image[image == 109] = 0 # 擦除背景 (background type 2)
image[image != 0] = 1 # 转为灰度图,除了黑色外其他都是白色
return image.astype(np.float).ravel()
# 根据一个episode的每个step的reward列表,计算每一个Step的Gt
def calc_reward_to_go(reward_list, gamma=0.99):
"""calculate discounted reward"""
reward_arr = np.array(reward_list)
for i in range(len(reward_arr) - 2, -1, -1):
# G_t = r_t + γ·r_t+1 + ... = r_t + γ·G_t+1
reward_arr[i] += gamma * reward_arr[i + 1]
# normalize episode rewards
reward_arr -= np.mean(reward_arr)
reward_arr /= np.std(reward_arr)
return reward_arr
# 创建环境
env = gym.make('Pong-v0')
obs_dim = 80 * 80
act_dim = env.action_space.n
logger.info('obs_dim {}, act_dim {}'.format(obs_dim, act_dim))
# 根据parl框架构建agent
model = Model(act_dim=act_dim)
alg = PolicyGradient(model, lr=LEARNING_RATE)
agent = Agent(alg, obs_dim=obs_dim, act_dim=act_dim)
# 加载模型
# if os.path.exists('./model.ckpt'):
# agent.restore('./model.ckpt')
for i in range(1000):
obs_list, action_list, reward_list = run_episode(env, agent)
# if i % 10 == 0:
# logger.info("Train Episode {}, Reward Sum {}.".format(i,
# sum(reward_list)))
batch_obs = np.array(obs_list)
batch_action = np.array(action_list)
batch_reward = calc_reward_to_go(reward_list)
agent.learn(batch_obs, batch_action, batch_reward)
if (i + 1) % 100 == 0:
total_reward = evaluate(env, agent, render=False)
logger.info('Episode {}, Test reward: {}'.format(i + 1,
total_reward))
# save the parameters to ./model.ckpt
agent.save('./model.ckpt')
最终结果:
[06-21 09:11:30 MainThread @<ipython-input-12-5c9781b79443>:29] obs_dim 6400, act_dim 6
[06-21 09:11:30 MainThread @machine_info.py:88] Cannot find available GPU devices, using CPU now.
[06-21 09:11:30 MainThread @machine_info.py:88] Cannot find available GPU devices, using CPU now.
[06-21 09:19:24 MainThread @<ipython-input-12-5c9781b79443>:63] Episode 100, Test reward: -15.6
[06-21 09:30:30 MainThread @<ipython-input-12-5c9781b79443>:63] Episode 200, Test reward: -14.2
[06-21 09:43:31 MainThread @<ipython-input-12-5c9781b79443>:63] Episode 300, Test reward: -12.4
[06-21 09:59:16 MainThread @<ipython-input-12-5c9781b79443>:63] Episode 400, Test reward: -8.6
[06-21 10:16:45 MainThread @<ipython-input-12-5c9781b79443>:63] Episode 500, Test reward: -4.2
[06-21 10:35:27 MainThread @<ipython-input-12-5c9781b79443>:63] Episode 600, Test reward: -8.6
[06-21 10:53:53 MainThread @<ipython-input-12-5c9781b79443>:63] Episode 700, Test reward: 1.2
[06-21 11:13:45 MainThread @<ipython-input-12-5c9781b79443>:63] Episode 800, Test reward: 4.2
[06-21 11:32:22 MainThread @<ipython-input-12-5c9781b79443>:63] Episode 900, Test reward: 6.0
[06-21 11:50:41 MainThread @<ipython-input-12-5c9781b79443>:63] Episode 1000, Test reward: 7.8