什么是机器学习
Twin Delayed DDPG (TD3
) 是一种用于解决连续动作空间的强化学习问题的算法,是 Deep Deterministic Policy Gradient (DDPG
) 的改进版本。TD3
引入了一些技巧,例如双Q网络(Twin Q-networks
)和延迟更新,以提高算法的性能和稳定性。
以下是一个使用 Python 和 TensorFlow/Keras
实现简单的 TD3
的示例。在这个例子中,我们将使用 OpenAI Gym 的 Pendulum
环境。
import numpy as np
import tensorflow as tf
from tensorflow.keras.models import Model
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
import gym
# 定义TD3 Agent
class TD3Agent:
def __init__(self, state_size, action_size):
self.state_size = state_size
self.action_size = action_size
self.gamma = 0.99 # 折扣因子
self.tau = 0.005 # 软更新参数
self.actor_lr = 0.001
self.critic_lr = 0.001
self.policy_noise = 0.2
self.policy_noise_clip = 0.5
self.exploration_noise = 0.1
self.buffer_size = 1000000
self.batch_size = 100
self.buffer = []
# 构建演员(Actor)网络和目标演员网络
self.actor = self.build_actor()
self.target_actor = self.build_actor()
self.target_actor.set_weights(self.actor.get_weights())
# 构建两个评论家(Critic)网络和目标评论家网络
self.critic_1 = self.build_critic()
self.target_critic_1 = self.build_critic()
self.target_critic_1.set_weights(self.critic_1.get_weights())
self.critic_2 = self.build_critic()
self.target_critic_2 = self.build_critic()
self.target_critic_2.set_weights(self.critic_2.get_weights())
def build_actor(self):
state_input = Input(shape=(self.state_size,))
dense1 = Dense(400, activation='relu')(state_input)
dense2 = Dense(300, activation='relu')(dense1)
output = Dense(self.action_size, activation='tanh')(dense2)
model = Model(inputs=state_input, outputs=output)
model.compile(loss='mse', optimizer=Adam(lr=self.actor_lr))
return model
def build_critic(self):
state_input = Input(shape=(self.state_size,))
action_input = Input(shape=(self.action_size,))
concat = tf.keras.layers.concatenate([state_input, action_input])
dense1 = Dense(400, activation='relu')(concat)
dense2 = Dense(300, activation='relu')(dense1)
output = Dense(1, activation='linear')(dense2)
model = Model(inputs=[state_input, action_input], outputs=output)
model.compile(loss='mse', optimizer=Adam(lr=self.critic_lr))
return model
def get_action(self, state):
state = np.reshape(state, [1, self.state_size])
action = self.actor.predict(state)[0]
action = np.clip(action + np.random.normal(0, self.exploration_noise, self.action_size), -1, 1)
return action
def train(self):
if len(self.buffer) < self.batch_size:
return
batch = np.random.choice(self.buffer, self.batch_size, replace=False)
states, actions, rewards, next_states, dones = zip(*batch)
states = np.vstack(states)
actions = np.vstack(actions)
rewards = np.vstack(rewards)
next_states = np.vstack(next_states)
dones = np.vstack(dones)
next_actions = self.target_actor.predict(next_states) + np.clip(np.random.normal(0, self.policy_noise, self.action_size), -self.policy_noise_clip, self.policy_noise_clip)
next_actions = np.clip(next_actions, -1, 1)
target_q_values = np.minimum(
self.target_critic_1.predict([next_states, next_actions]),
self.target_critic_2.predict([next_states, next_actions])
)
target_values = rewards + self.gamma * (1 - dones) * target_q_values
self.critic_1.train_on_batch([states, actions], target_values)
self.critic_2.train_on_batch([states, actions], target_values)
actor_gradients = np.reshape(self.critic_1.gradient(states + self.actor.predict(states)), [-1, self.action_size])
actor_gradients = actor_gradients / self.batch_size
self.actor.train_on_batch(states, actor_gradients)
self.soft_update_target_networks()
def soft_update_target_networks(self):
actor_weights = np.array(self.actor.get_weights())
target_actor_weights = np.array(self.target_actor.get_weights())
self.target_actor.set_weights(self.tau * actor_weights + (1 - self.tau) * target_actor_weights)
critic_1_weights = np.array(self.critic_1.get_weights())
target_critic_1_weights = np.array(self.target_critic_1.get_weights())
self.target_critic_1.set_weights(self.tau * critic_1_weights + (1 - self.tau) * target_critic_1_weights)
critic_2_weights = np.array(self.critic_2.get_weights())
target_critic_2_weights = np.array(self.target_critic_2.get_weights())
self.target_critic_2.set_weights(self.tau * critic_2_weights + (1 - self.tau) * target_critic_2_weights)
def store_experience(self, state, action, reward, next_state, done):
self.buffer.append((state, action, reward, next_state, done))
if len(self.buffer) > self.buffer_size:
self.buffer.pop(0)
# 初始化环境和Agent
env = gym.make('Pendulum-v0')
state_size = env.observation_space.shape[0]
action_size = env.action_space.shape[0]
agent = TD3Agent(state_size, action_size)
# 训练TD3 Agent
num_episodes = 500
for episode in range(num_episodes):
state = env.reset()
total_reward = 0
for time in range(500): # 限制每个episode的步数,防止无限循环
# env.render() # 如果想可视化训练过程,可以取消注释此行
action = agent.get_action(state)
next_state, reward, done, _ = env.step(action)
total_reward += reward
agent.store_experience(state, action, reward, next_state, done)
agent.train()
state = next_state
if done:
print("Episode: {}, Total Reward: {}".format(episode + 1, total_reward))
break
# 关闭环境
env.close()
在这个例子中,我们定义了一个简单的TD3 Agent,包括演员(Actor)和两个评论家(Critic)神经网络。在训练过程中,我们使用了两个评论家网络和一些技巧来提高稳定性,并进行了软更新。请注意,TD3算法的实现可能因问题的复杂性而有所不同,可能需要更多的技术和调整,如归一化奖励、使用更复杂的神经网络结构等。