演员评论家算法

Actor-Critic

    Actor（玩家）：为了玩转这个游戏得到尽量高的reward，需要一个策略：输入state，输出action，即上面的第2步。（可以用神经网络来近似这个函数。剩下的任务就是如何训练神经网络，得更高的reward。这个网络就被称为actor）

   Critic（评委）：因为actor是基于策略policy的所以需要critic来计算出对应actor的value来反馈给actor，告诉他表现得好不好。所以就要使用到之前的Q值。（当然这个Q-function所以也可以用神经网络来近似。这个网络被称为critic。)

这种思想有点类似GAN网络中的生成器和判别器，两者相互监督和牵制，最后达到较好的效果。如果之前的DQN，Policy Gradient梯度上升公式及蒙特卡洛Reinforce算法都看懂了的话，这里还是很好理解的。
与Monte Carlo Policy Gradient不同，Actor Critic放弃利用回报来评估真实价值函数，而直接使用Critic算法，利用函数逼近法(Function Approximation Methods)即神经网络，利用逼近策略梯度法而非真实策略梯度。
当代理在环境中执行操作和移动时，它将观察到的环境状态映射到两个可能的输出：

推荐动作：动作空间中每个动作的概率值。代理中负责此输出的部分称为actor（演员）。

未来预期回报：它预期在未来获得的所有回报的总和。负责此输出的代理部分是critic（评论家）。

演员和评论家学习执行他们的任务，这样演员推荐的动作就能获得最大的回报。

import gym, os
from itertools import count
import paddle
import paddle.nn as nn
import paddle.optimizer as optim
import paddle.nn.functional as F
from paddle.distribution import Categorical

#实施网络
device = paddle.get_device()
env = gym.make("CartPole-v0") 

state_size = env.observation_space.shape[0]
action_size = env.action_space.n
lr = 0.001

class Actor(nn.Layer):
    def __init__(self, state_size, action_size):
        super(Actor, self).__init__()
        self.state_size = state_size
        self.action_size = action_size
        self.linear1 = nn.Linear(self.state_size, 128)
        self.linear2 = nn.Linear(128, 256)
        self.linear3 = nn.Linear(256, self.action_size)

    def forward(self, state):
        output = F.relu(self.linear1(state))
        output = F.relu(self.linear2(output))
        output = self.linear3(output)
        distribution = Categorical(F.softmax(output, axis=-1))
        return distribution


class Critic(nn.Layer):
    def __init__(self, state_size, action_size):
        super(Critic, self).__init__()
        self.state_size = state_size
        self.action_size = action_size
        self.linear1 = nn.Linear(self.state_size, 128)
        self.linear2 = nn.Linear(128, 256)
        self.linear3 = nn.Linear(256, 1)

    def forward(self, state):
        output = F.relu(self.linear1(state))
        output = F.relu(self.linear2(output))
        value = self.linear3(output)
        return value

#训练模型
def compute_returns(next_value, rewards, masks, gamma=0.99):
    R = next_value
    returns = []
    for step in reversed(range(len(rewards))):
        R = rewards[step] + gamma * R * masks[step]
        returns.insert(0, R)
    return returns


def trainIters(actor, critic, n_iters):
    optimizerA = optim.Adam(lr, parameters=actor.parameters())
    optimizerC = optim.Adam(lr, parameters=critic.parameters())
    for iter in range(n_iters):
        state = env.reset()
        log_probs = []
        values = []
        rewards = []
        masks = []
        entropy = 0
        env.reset()

        for i in count():
         
            state = paddle.to_tensor(state,dtype="float32",place=device)
            dist, value = actor(state), critic(state)

            action = dist.sample([1])
            next_state, reward, done, _ = env.step(action.cpu().squeeze(0).numpy()) 

            log_prob = dist.log_prob(action);
           

            log_probs.append(log_prob)
            values.append(value)
            rewards.append(paddle.to_tensor([reward], dtype="float32", place=device))
            masks.append(paddle.to_tensor([1-done], dtype="float32", place=device))

            state = next_state

            if done:
                if iter % 10 == 0:
                    print('Iteration: {}, Score: {}'.format(iter, i))
                break


        next_state = paddle.to_tensor(next_state, dtype="float32", place=device)
        next_value = critic(next_state)
        returns = compute_returns(next_value, rewards, masks)

        log_probs = paddle.concat(log_probs)
        returns = paddle.concat(returns).detach()
        values = paddle.concat(values)

        advantage = returns - values

        actor_loss = -(log_probs * advantage.detach()).mean()
        critic_loss = advantage.pow(2).mean()

        optimizerA.clear_grad()
        optimizerC.clear_grad()
        actor_loss.backward()
        critic_loss.backward()
        optimizerA.step()
        optimizerC.step()
    paddle.save(actor.state_dict(), 'model/actor.pdparams')
    paddle.save(critic.state_dict(), 'model/critic.pdparams')
    env.close()



if __name__ == '__main__':
    if os.path.exists('model/actor.pdparams'):
        actor = Actor(state_size, action_size)
        model_state_dict  = paddle.load('model/actor.pdparams')
        actor.set_state_dict(model_state_dict )
        print('Actor Model loaded')
    else:
        actor = Actor(state_size, action_size)
    if os.path.exists('model/critic.pdparams'):
        critic = Critic(state_size, action_size)
        model_state_dict  = paddle.load('model/critic.pdparams')
        critic.set_state_dict(model_state_dict )
        print('Critic Model loaded')
    else:
        critic = Critic(state_size, action_size)
    trainIters(actor, critic, n_iters=201)

Actor Critic优点：
可以进行单步更新, 相较于传统的PG回合更新要快.
Actor Critic缺点：
Actor的行为取决于 Critic 的Value，但是因为 Critic本身就很难收敛和actor一起更新的话就更难收敛了。