【莫烦python 强化学习】代码 笔记 深度强化学习DQN 倒立摆 Pytorch实现

视频教程：B站【莫烦Python】PyTorch 神经网络 P27

算法原理部分看不懂可参考B站视频 强化学习中的数学原理 或其他教学视频

所需环境：

1、anaconda3 + pytorch：安装和配置可以看B站小土堆 PyTorch深度学习快速入门教程的前几个视频

2、安装gym 可参考:如何在win10环境下配置强化学习gym库（使用vscode）_gym支持windows吗-CSDN博客

可视化需安装pygame：在pycharm命令行安装 pip install pygame

代码+注释：

import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np
import gym

# Hyper Parameters
BATCH_SIZE = 32
LR = 0.01                   # learning rate
EPSILON = 0.9               # greedy policy
GAMMA = 0.9                 # reward discount
TARGET_REPLACE_ITER = 100   # target update frequency Q_target（Q现实）网络更新频率
MEMORY_CAPACITY = 2000  # 记忆库大小
env = gym.make('CartPole-v1', render_mode="human")  # 立倒立摆游戏，render环境渲染
env = env.unwrapped
N_ACTIONS = env.action_space.n  # 杆子动作个数（2）动作空间，{0,1} 0-左移动， 1-右移动
N_STATES = env.observation_space.shape[0]  # 杆子状态个数（4）
ENV_A_SHAPE = 0 if isinstance(env.action_space.sample(), int) else env.action_space.sample().shape
# to confirm the shape
# 证明随机抽取的动作（0或1）为 int 型


# 神经网络
class Net(nn.Module):
    def __init__(self, ):
        super(Net, self).__init__()
        # 神经网络输入状态，输出每个动作的价值
        self.fc1 = nn.Linear(N_STATES, 50)  # 第一层 输入状态N_STATES，输出50个神经元参数到下一层
        self.fc1.weight.data.normal_(0, 0.1)   # initialization 初始化权重
        self.out = nn.Linear(50, N_ACTIONS)  # 输出层 输入为上一个self.fcl的50个参数，输出为动作数
        self.out.weight.data.normal_(0, 0.1)   # initialization

    def forward(self, x):
        x = self.fc1(x)
        x = F.relu(x) # 激励函数
        actions_value = self.out(x)
        return actions_value


class DQN(object):
    def __init__(self):
        # 建立两个神经网络 main network和 target network
        # target network 定期更新，将 main network的参数赋过来
        self.eval_net, self.target_net = Net(), Net()

        self.learn_step_counter = 0                                     # for target updating 学习到多少步
        self.memory_counter = 0                                         # 记忆库计数
        self.memory = np.zeros((MEMORY_CAPACITY, N_STATES * 2 + 2))     # 初始化记忆库
        # MEMORY_CAPACITY行，列：两个s+a+r
        self.optimizer = torch.optim.Adam(self.eval_net.parameters(), lr=LR)  # 优化器
        self.loss_func = nn.MSELoss()  # 损失函数

    def choose_action(self, x):
        x = torch.unsqueeze(torch.FloatTensor(x), 0)
        # input only one sample
        if np.random.uniform() < EPSILON:   # greedy
            actions_value = self.eval_net.forward(x)
            action = torch.max(actions_value, 1)[1].data.numpy()
            # 选取最大价值，并转化为numpy ndarray形式
            action = action[0] if ENV_A_SHAPE == 0 else action.reshape(ENV_A_SHAPE)  # return the argmax index
        else:   # random 随机选择一个动作
            action = np.random.randint(0, N_ACTIONS)
            action = action if ENV_A_SHAPE == 0 else action.reshape(ENV_A_SHAPE)
        return action

    # 记忆库
    def store_transition(self, s, a, r, s_):
        transition = np.hstack((s, [a, r], s_))  # 将数据打包
        # replace the old memory with new memory
        # 超过记忆上限，重新开始索引
        index = self.memory_counter % MEMORY_CAPACITY
        self.memory[index, :] = transition
        self.memory_counter += 1

    def learn(self):
        # 学习存储到的记忆
        # target parameter update 检测是否更新 target network
        if self.learn_step_counter % TARGET_REPLACE_ITER == 0:
            self.target_net.load_state_dict(self.eval_net.state_dict())
        self.learn_step_counter += 1

        # main network 需时刻更新
        # sample batch transitions 提取批记忆，进行批训练
        sample_index = np.random.choice(MEMORY_CAPACITY, BATCH_SIZE)
        b_memory = self.memory[sample_index, :]
        # 将数据分开
        b_s = torch.FloatTensor(b_memory[:, :N_STATES])
        b_a = torch.LongTensor(b_memory[:, N_STATES:N_STATES+1].astype(int))
        b_r = torch.FloatTensor(b_memory[:, N_STATES+1:N_STATES+2])
        b_s_ = torch.FloatTensor(b_memory[:, -N_STATES:])

        # q_eval w.r.t the action in experience
        q_eval = self.eval_net(b_s).gather(1, b_a)  # shape (batch, 1)
        # eval_net(b_s)所有动作的价值 选取当初施加的动作的价值
        q_next = self.target_net(b_s_).detach()    
        # detach 不希望 target network 更新，不需要反向传播
        q_target = b_r + GAMMA * q_next.max(1)[0].view(BATCH_SIZE, 1)   # shape (batch, 1)
        # 选择 q_next 最大的 action value
        loss = self.loss_func(q_eval, q_target)  # （预测值，真实值)

        self.optimizer.zero_grad()
        loss.backward()
        self.optimizer.step()

dqn = DQN()

print('\nCollecting experience...')
for i_episode in range(400):
    # 环境反馈
    s = env.reset()[0]
    ep_r = 0
    while True:
        a = dqn.choose_action(s)
        # take action
        s_, r, done, info, _ = env.step(a)
        # 使用默认reward比较难学，使用修改的reward function

        # modify the reward  小车位置，速度，杆的角度，杆尖速度
        # x_threshold是小车中心到达显示器边缘的限度，为2.4
        # theta_threshold_radians是杆的极限角度，为12°
        # 杆子立起来，车越靠中间，reward越大
        x, x_dot, theta, theta_dot = s_
        r1 = (env.x_threshold - abs(x)) / env.x_threshold - 0.8
        r2 = (env.theta_threshold_radians - abs(theta)) / env.theta_threshold_radians - 0.5
        r = r1 + r2

        # 存储
        dqn.store_transition(s, a, r, s_)

        ep_r += r
        if dqn.memory_counter > MEMORY_CAPACITY:
            dqn.learn()
            if done:
                print('Ep: ', i_episode,
                      '| Ep_r: ', round(ep_r, 2))

        if done:
            # 回合结束
            break
        s = s_

运行结果：倒立摆逐渐立住，在中间