DDQN与DQN算法用tensorflow2.0实现

深度强化学习Double Deep Q Learning算法和Deep Q Learning用tensorflow2.0实现

DQN算法实现

首先搭建网络结构，是一个很简单的三个全连接层。

from keras import layers, models


class Q_Network:
    def __init__(self, observation_n, action_n):
        self.observation_n = observation_n
        self.action_n = action_n
        self._build_model()

    def _build_model(self):
        # ------------------ build eval network -------------------------
        self.eval_model = models.Sequential(name="eval_network")
        self.eval_model.add(layers.Dense(64, activation="relu", input_shape=(None, self.observation_n)))
        self.eval_model.add(layers.Dense(64, activation="relu"))
        self.eval_model.add(layers.Dense(self.action_n))

        # print(self.eval_model.summary())

        # ------------------ build target network ---------------------
        self.target_model = models.Sequential(name="target_network")
        self.target_model.add(layers.Dense(64, activation="relu", input_shape=(None, self.observation_n)))
        self.target_model.add(layers.Dense(64, activation="relu"))
        self.target_model.add(layers.Dense(self.action_n))

        # print(self.target_model.summary())

DQN算法实现

import tensorflow as tf
import numpy as np
from Network import Q_Network


class Deep_Q_Network:
    def __init__(
            self,
            n_actions,
            n_features,
            learning_rate=0.01,
            reward_decay=0.9,
            e_greedy=0.9,
            replace_target_iter=300,
            memory_size=500,
            batch_size=32,
            e_greedy_increment=None
    ):
        """

        :param n_actions: 动作种类个数
        :param n_features: 观察向量的维度
        :param learning_rate: 学习率
        :param reward_decay: 奖励衰减系数
        :param e_greedy: 贪心策略的epsilon
        :param replace_target_iter: 多少步替换依次权重
        :param memory_size: 内存表的大小
        :param batch_size: 神经网络训练的批次
        :param e_greedy_increment: 贪心选择策略中的epsilon的衰减
        """
        self.memory_counter = 0
        self.n_actions = n_actions
        self.n_features = n_features
        self.lr = learning_rate
        self.gamma = reward_decay
        self.epsilon_max = e_greedy
        self.replace_target_iter = replace_target_iter
        self.memory_size = memory_size
        self.batch_size = batch_size
        self.epsilon_increment = e_greedy_increment
        self.epsilon = 0 if e_greedy_increment is not None else self.epsilon_max

        # total learning step
        self.learn_step_counter = 0

        # initialize zero memory [s, a, r, s_]
        self.memory = np.zeros((self.memory_size, n_features * 2 + 3))

        self.q_target = Q_Network(observation_n=n_features, action_n=self.n_actions).target_model
        self.q_eval = Q_Network(observation_n=n_features, action_n=self.n_actions).eval_model
        self.optimizer = tf.keras.optimizers.Adam(lr=self.lr)
        self.loss = tf.losses.MeanSquaredError()

    def choose_action(self, state, eps=0.1):
        state = tf.Variable(state, dtype=tf.float32)
        if len(tf.shape(state)) == 1:
            state = tf.expand_dims(state, axis=0)
        if np.random.uniform() > eps:
            action_value = self.q_eval.predict(state)
            return np.argmax(action_value)
        else:
            return np.random.choice(np.arange(self.n_actions))

    def learn(self):
        if self.learn_step_counter % self.replace_target_iter == 0:
            self.soft_update(1)

        # sample batch memory from all memory
        if self.memory_counter > self.memory_size:
            sample_index = np.random.choice(self.memory_size, size=self.batch_size)
        else:
            sample_index = np.random.choice(self.memory_counter, size=self.batch_size)

        experience = self.memory[sample_index, :]
        states = np.array([e[:self.n_features] for e in experience]).astype("float32")
        actions = np.array([e[self.n_features] for e in experience]).astype("int32")
        rewards = np.array([e[self.n_features + 1] for e in experience]).astype("float32")
        next_states = np.array([e[-self.n_features:] for e in experience]).astype("float32")
        dones = np.array([e[self.n_features+2] for e in experience])

        q_target_values = self.q_target.predict(next_states)
        q_target_values = tf.reduce_max(q_target_values, axis=-1, keepdims=True)
        q_target_values = rewards + self.gamma * (1 - dones) * tf.squeeze(q_target_values)
        with tf.GradientTape() as tape:
            q_values = self.q_eval(states, training=True)
            enum_actions = list(enumerate(actions))
            q_values = tf.gather_nd(params=q_values, indices=enum_actions)
            loss = self.loss(q_values, q_target_values)

        grads = tape.gradient(loss, self.q_eval.trainable_variables)
        self.optimizer.apply_gradients(zip(grads, self.q_eval.trainable_variables))
        print("梯度更新")
        self.memory_counter += 1

    def store_transition(self, s, a, r, done, s_):
        transition = np.hstack((s, [a, r, done], s_))
        # replace the old memory with new memory
        index = self.memory_counter % self.memory_size
        self.memory[index, :] = transition
        self.memory_counter += 1

    def soft_update(self, tau):
        for target_param, local_param in zip(self.q_target.weights, self.q_eval.weights):
            tf.compat.v1.assign(target_param, tau * local_param + (1. - tau) * target_param)

主程序

import gym
from DQN import Deep_Q_Network


def main():
    step = 0
    for episode in range(300):
        # initial observation
        observation = env.reset()

        while True:
            # fresh env
            env.render()

            # RL choose action based on observation
            action = RL.choose_action(observation)

            # RL take action and get next observation and reward
            observation_, reward, done, info = env.step(action)

            RL.store_transition(observation, action, reward, done, observation_)

            if (step > 200) and (step % 5 == 0):
                RL.learn()
                RL.soft_update(1)

            # swap observation
            observation = observation_

            # break while loop when end of this episode
            if done:
                break
            step += 1
        print("episode %d" % (episode + 1))
    # end of game
    print('game over')
    env.close()


def transformer_state(state):
    """将 position, velocity 通过线性转换映射到 [0, 40] 范围内"""
    pos, v = state
    pos_low, v_low = env.observation_space.low
    pos_high, v_high = env.observation_space.high
    pos = 40 * (pos - pos_low) / (pos_high - pos_low)
    v = 40 * (v - v_low) / (v_high - v_low)
    return int(pos), int(v)


if __name__ == "__main__":
    env = gym.make("CartPole-v1")
    RL = Deep_Q_Network(n_actions=env.action_space.n, n_features=env.observation_space.shape[0])
    main()
    

DDQN算法

网络结构部分与DQN相同，
DDQN算法部分

import tensorflow as tf
import numpy as np
from Network import Q_Network


class Double_Deep_Q_Network:
    def __init__(
            self,
            n_actions,
            n_features,
            learning_rate=0.01,
            reward_decay=0.9,
            e_greedy=0.9,
            replace_target_iter=300,
            memory_size=500,
            batch_size=32,
            e_greedy_increment=None
    ):
        """

        :param n_actions: 动作种类个数
        :param n_features: 观察向量的维度
        :param learning_rate: 学习率
        :param reward_decay: 奖励衰减系数
        :param e_greedy: 贪心策略的epsilon
        :param replace_target_iter: 多少步替换依次权重
        :param memory_size: 内存表的大小
        :param batch_size: 神经网络训练的批次
        :param e_greedy_increment: 贪心选择策略中的epsilon的衰减
        """
        self.memory_counter = 0
        self.n_actions = n_actions
        self.n_features = n_features
        self.lr = learning_rate
        self.gamma = reward_decay
        self.epsilon_max = e_greedy
        self.replace_target_iter = replace_target_iter
        self.memory_size = memory_size
        self.batch_size = batch_size
        self.epsilon_increment = e_greedy_increment
        self.epsilon = 0 if e_greedy_increment is not None else self.epsilon_max

        # total learning step
        self.learn_step_counter = 0

        # initialize zero memory [s, a, r, s_]
        self.memory = np.zeros((self.memory_size, n_features * 2 + 3))

        self.q_target = Q_Network(observation_n=n_features, action_n=self.n_actions).target_model
        self.q_eval = Q_Network(observation_n=n_features, action_n=self.n_actions).eval_model
        self.optimizer = tf.keras.optimizers.Adam(lr=self.lr)
        self.loss = tf.losses.MeanSquaredError()

    def choose_action(self, state, eps=0.1):
        state = tf.Variable(state, dtype=tf.float32)
        if len(tf.shape(state)) == 1:
            state = tf.expand_dims(state, axis=0)
        if np.random.uniform() > eps:
            action_value = self.q_eval.predict(state)
            return np.argmax(action_value)
        else:
            return np.random.choice(np.arange(self.n_actions))

    def learn(self):
        if self.learn_step_counter % self.replace_target_iter == 0:
            self.soft_update(1)

        # sample batch memory from all memory
        if self.memory_counter > self.memory_size:
            sample_index = np.random.choice(self.memory_size, size=self.batch_size)
        else:
            sample_index = np.random.choice(self.memory_counter, size=self.batch_size)

        experience = self.memory[sample_index, :]
        states = np.array([e[:self.n_features] for e in experience]).astype("float32")
        actions = np.array([e[self.n_features] for e in experience]).astype("int32")
        rewards = np.array([e[self.n_features + 1] for e in experience]).astype("float32")
        next_states = np.array([e[-self.n_features:] for e in experience]).astype("float32")
        dones = np.array([e[self.n_features+2] for e in experience])

        q_target_values = self.q_target.predict(next_states)
        # 选择动作  就这里的q_target与DQN有些许的不一样
        q_eval_values = self.q_eval.predict(next_states)
        # print(q_eval_values)
        max_actions = tf.argmax(q_eval_values, axis=1)
        # print(max_actions)
        # q_target_values = tf.reduce_max(q_target_values, axis=-1, keep_dims=True)
        enum_max_actions = list(enumerate(max_actions))
        q_target_values = tf.gather_nd(params=q_target_values, indices=enum_max_actions)
        q_target_values = rewards + self.gamma * (1 - dones) * tf.squeeze(q_target_values)

主程序部分也与DQN一致

import gym
from DDQN import Double_Deep_Q_Network


def main():
    step = 0
    for episode in range(300):
        # initial observation
        observation = env.reset()

        while True:
            # fresh env
            env.render()

            # RL choose action based on observation
            action = RL.choose_action(observation)

            # RL take action and get next observation and reward
            observation_, reward, done, info = env.step(action)

            RL.store_transition(observation, action, reward, done, observation_)

            if (step > 200) and (step % 5 == 0):
                RL.learn()
                RL.soft_update(1)

            # swap observation
            observation = observation_

            # break while loop when end of this episode
            if done:
                break
            step += 1
        print("episode %d" % (episode + 1))
    # end of game
    print('game over')
    env.close()


def transformer_state(state):
    """将 position, velocity 通过线性转换映射到 [0, 40] 范围内"""
    pos, v = state
    pos_low, v_low = env.observation_space.low
    pos_high, v_high = env.observation_space.high
    pos = 40 * (pos - pos_low) / (pos_high - pos_low)
    v = 40 * (v - v_low) / (v_high - v_low)
    return int(pos), int(v)


if __name__ == "__main__":
    env = gym.make("CartPole-v1")
    RL = Double_Deep_Q_Network(n_actions=env.action_space.n, n_features=env.observation_space.shape[0])
    main()