本案例源自ML-Agents官方的示例,Github地址:https://github.com/Unity-Technologies/ml-agents,本文是详细的配套讲解。
本文基于我前面发的两篇文章,需要对ML-Agents有一定的了解,详情请见:Unity强化学习之ML-Agents的使用、ML-Agents命令及配置大全。
我前面的相关文章有:
环境说明
环境中存在有多个智能体,他们的任务是收集尽可能多的绿色食物球,并避免碰到红色球:碰到绿色球奖励+1,碰到红色球-1,此外,智能体之间可以通过发射射线冻结其他智能体,以让自己吃到更多的食物球,达到增加自己分数的目的。
观察空间:使用了网格传感器(Grid Sensor),这个传感器具体说明参考ML-Agents案例之推箱子游戏 的多人模式。
在本案例中传感器直接挂载在智能体本身,z轴(前后方向)网格数为40,x轴(左右方向)网格数为40,y轴(上下方向)网格数为1,检测的标签有,食物,坏食物,其他智能体,被冻结的智能体,墙壁,加上什么都没观测到,维度为40 * 40 * 6 = 960个观察维度。
动作空间:3个连续动作输出,对应于前后运动,左右运动和旋转。1个离散输出,对应于是否发生激光。
代码分析
食物生成
控制食物生成的脚本,挂载在空物体上,这个脚本不会自己运作,需要在智能体挂载的脚本上调用。
using UnityEngine;
using Unity.MLAgentsExamples;
public class FoodCollectorArea : Area
{
public GameObject food;
public GameObject badFood;
public int numFood;
public int numBadFood;
public bool respawnFood;
public float range;
// 生成食物
void CreateFood(int num, GameObject type)
{
for (int i = 0; i < num; i++)
{
GameObject f = Instantiate(type, new Vector3(Random.Range(-range, range), 1f,
Random.Range(-range, range)) + transform.position,
Quaternion.Euler(new Vector3(0f, Random.Range(0f, 360f), 90f)));
f.GetComponent<FoodLogic>().respawn = respawnFood;
f.GetComponent<FoodLogic>().myArea = this;
}
}
// 重置区域,随机化智能体们的位置,并生成两种食物
public void ResetFoodArea(GameObject[] agents)
{
foreach (GameObject agent in agents)
{
if (agent.transform.parent == gameObject.transform)
{
agent.transform.position = new Vector3(Random.Range(-range, range), 2f,
Random.Range(-range, range))
+ transform.position;
agent.transform.rotation = Quaternion.Euler(new Vector3(0f, Random.Range(0, 360)));
}
}
CreateFood(numFood, food);
CreateFood(numBadFood, badFood);
}
public override void ResetArea()
{
}
}
食物被吃掉的处理
挂载在食物上的脚本,同样需要调用才能起作用:
using UnityEngine;
public class FoodLogic : MonoBehaviour
{
public bool respawn;
public FoodCollectorArea myArea;
// 被吃掉后的两种选择,一个是重新随机位置,一个是直接销毁
public void OnEaten()
{
if (respawn)
{
transform.position = new Vector3(Random.Range(-myArea.range, myArea.range),
3f,
Random.Range(-myArea.range, myArea.range)) + myArea.transform.position;
}
else
{
Destroy(gameObject);
}
}
}
智能体主文件FoodCollectorAgent.cs:
初始化:
public override void Initialize()
{
m_AgentRb = GetComponent<Rigidbody>();
m_MyArea = area.GetComponent<FoodCollectorArea>();
m_FoodCollecterSettings = FindObjectOfType<FoodCollectorSettings>();
// 从配置文件中获取参数
m_ResetParams = Academy.Instance.EnvironmentParameters;
// 设置参数
SetResetParameters();
}
// 激光长度
public void SetLaserLengths()
{
m_LaserLength = m_ResetParams.GetWithDefault("laser_length", 1.0f);
}
// 设置智能体体积大小
public void SetAgentScale()
{
float agentScale = m_ResetParams.GetWithDefault("agent_scale", 1.0f);
gameObject.transform.localScale = new Vector3(agentScale, agentScale, agentScale);
}
public void SetResetParameters()
{
SetLaserLengths();
SetAgentScale();
}
观察输入(可以通过设置useVectorObs和useVectorFrozenFlag)来配置是否输入,在案例中没有选择开启这两项输入:
public override void CollectObservations(VectorSensor sensor)
{
if (useVectorObs)
{
var localVelocity = transform.InverseTransformDirection(m_AgentRb.velocity);
// 输入水平方向的两个速度
sensor.AddObservation(localVelocity.x);
sensor.AddObservation(localVelocity.z);
// 输入是否冻结和是否发射
sensor.AddObservation(m_Frozen);
sensor.AddObservation(m_Shoot);
}
else if (useVectorFrozenFlag)
{
// 输入是否冻结和是否发射
sensor.AddObservation(m_Frozen);
}
}
动作输出:
// 执行输出的主函数,里面的内容都封装到MoveAgent了
public override void OnActionReceived(ActionBuffers actionBuffers)
{
MoveAgent(actionBuffers);
}
public void MoveAgent(ActionBuffers actionBuffers)
{
m_Shoot = false;
// 超过冻结时间,解冻
if (Time.time > m_FrozenTime + 4f && m_Frozen)
{
Unfreeze();
}
// 超过被毒时间,解除毒素;超过满意时间,变为正常状态
if (Time.time > m_EffectTime + 0.5f)
{
if (m_Poisoned)
{
Unpoison();
}
if (m_Satiated)
{
Unsatiate();
}
}
var dirToGo = Vector3.zero;
var rotateDir = Vector3.zero;
var continuousActions = actionBuffers.ContinuousActions;
var discreteActions = actionBuffers.DiscreteActions;
// 没有冻结的时候才能执行动作
if (!m_Frozen)
{
// 获取三个连续输出
var forward = Mathf.Clamp(continuousActions[0], -1f, 1f);
var right = Mathf.Clamp(continuousActions[1], -1f, 1f);
var rotate = Mathf.Clamp(continuousActions[2], -1f, 1f);
dirToGo = transform.forward * forward;
dirToGo += transform.right * right;
rotateDir = -transform.up * rotate;
// 获取一个离散输出
var shootCommand = discreteActions[0] > 0;
if (shootCommand)
{
// 射击时减速
m_Shoot = true;
dirToGo *= 0.5f;
m_AgentRb.velocity *= 0.75f;
}
// 执行移动
m_AgentRb.AddForce(dirToGo * moveSpeed, ForceMode.VelocityChange);
transform.Rotate(rotateDir, Time.fixedDeltaTime * turnSpeed);
}
// 超过一定速度需要减速
if (m_AgentRb.velocity.sqrMagnitude > 25f) // slow it down
{
m_AgentRb.velocity *= 0.95f;
}
// 执行射击逻辑
if (m_Shoot)
{
var myTransform = transform;
myLaser.transform.localScale = new Vector3(1f, 1f, m_LaserLength);
var rayDir = 25.0f * myTransform.forward;
// 绘制射线,参数为起始位置,长度向量,颜色,持续时间,能否被遮挡
Debug.DrawRay(myTransform.position, rayDir, Color.red, 0f, true);
RaycastHit hit;
// 发出球状射线,参数为起始位置,球半径,距离,碰撞到的物体,距离
if (Physics.SphereCast(transform.position, 2f, rayDir, out hit, 25f))
{
// 射线碰到其他智能体,其他智能体会被冻结
if (hit.collider.gameObject.CompareTag("agent"))
{
hit.collider.gameObject.GetComponent<FoodCollectorAgent>().Freeze();
}
}
}
else
{
myLaser.transform.localScale = new Vector3(0f, 0f, 0f);
}
}
// 被冻结的逻辑,修改标签,计算时间,替换材质
void Freeze()
{
gameObject.tag = "frozenAgent";
m_Frozen = true;
m_FrozenTime = Time.time;
gameObject.GetComponentInChildren<Renderer>().material = frozenMaterial;
}
// 解冻的逻辑
void Unfreeze()
{
m_Frozen = false;
gameObject.tag = "agent";
gameObject.GetComponentInChildren<Renderer>().material = normalMaterial;
}
// 吃到坏食物的状态
void Poison()
{
m_Poisoned = true;
m_EffectTime = Time.time;
gameObject.GetComponentInChildren<Renderer>().material = badMaterial;
}
// 解毒
void Unpoison()
{
m_Poisoned = false;
gameObject.GetComponentInChildren<Renderer>().material = normalMaterial;
}
// 吃到好食物的状态
void Satiate()
{
m_Satiated = true;
m_EffectTime = Time.time;
gameObject.GetComponentInChildren<Renderer>().material = goodMaterial;
}
// 好状态消失
void Unsatiate()
{
m_Satiated = false;
gameObject.GetComponentInChildren<Renderer>().material = normalMaterial;
}
和食物碰撞的逻辑:
void OnCollisionEnter(Collision collision)
{
// 吃到好食物,奖励一分,进入满意状态
if (collision.gameObject.CompareTag("food"))
{
Satiate();
collision.gameObject.GetComponent<FoodLogic>().OnEaten();
AddReward(1f);
if (contribute)
{
m_FoodCollecterSettings.totalScore += 1;
}
}
// 吃到坏食物,扣除1分,进入中毒状态
if (collision.gameObject.CompareTag("badFood"))
{
Poison();
collision.gameObject.GetComponent<FoodLogic>().OnEaten();
AddReward(-1f);
if (contribute)
{
m_FoodCollecterSettings.totalScore -= 1;
}
}
}
一轮游戏开始执行的逻辑:
public override void OnEpisodeBegin()
{
// 消除所有异常状态
Unfreeze();
Unpoison();
Unsatiate();
m_Shoot = false;
// 速度位置都归零
m_AgentRb.velocity = Vector3.zero;
myLaser.transform.localScale = new Vector3(0f, 0f, 0f);
transform.position = new Vector3(Random.Range(-m_MyArea.range, m_MyArea.range),
2f, Random.Range(-m_MyArea.range, m_MyArea.range))
+ area.transform.position;
transform.rotation = Quaternion.Euler(new Vector3(0f, Random.Range(0, 360)));
// 重新设置参数,包括激光长短,智能体大小
SetResetParameters();
}
可以通过以下代码自己控制智能体的输出:
public override void Heuristic(in ActionBuffers actionsOut)
{
var continuousActionsOut = actionsOut.ContinuousActions;
if (Input.GetKey(KeyCode.D))
{
continuousActionsOut[2] = 1;
}
if (Input.GetKey(KeyCode.W))
{
continuousActionsOut[0] = 1;
}
if (Input.GetKey(KeyCode.A))
{
continuousActionsOut[2] = -1;
}
if (Input.GetKey(KeyCode.S))
{
continuousActionsOut[0] = -1;
}
var discreteActionsOut = actionsOut.DiscreteActions;
discreteActionsOut[0] = Input.GetKey(KeyCode.Space) ? 1 : 0;
}
配置文件
配置文件是普通的PPO和SAC配置:
behaviors:
GridFoodCollector:
trainer_type: ppo
hyperparameters:
batch_size: 1024
buffer_size: 10240
learning_rate: 0.0003
beta: 0.005
epsilon: 0.2
lambd: 0.95
num_epoch: 3
learning_rate_schedule: linear
network_settings:
normalize: false
hidden_units: 256
num_layers: 1
vis_encode_type: simple
reward_signals:
extrinsic:
gamma: 0.99
strength: 1.0
keep_checkpoints: 5
max_steps: 2000000
time_horizon: 64
summary_freq: 10000
behaviors:
GridFoodCollector:
trainer_type: sac
hyperparameters:
learning_rate: 0.0003
learning_rate_schedule: constant
batch_size: 256
buffer_size: 2048
buffer_init_steps: 0
tau: 0.005
steps_per_update: 10.0
save_replay_buffer: false
init_entcoef: 0.05
reward_signal_steps_per_update: 10.0
network_settings:
normalize: false
hidden_units: 256
num_layers: 1
vis_encode_type: simple
reward_signals:
extrinsic:
gamma: 0.99
strength: 1.0
keep_checkpoints: 5
max_steps: 2000000
time_horizon: 64
summary_freq: 60000
threaded: false
效果演示
后记
本案例虽然场景中存在多个智能体,但它们彼此竞争,所以是单智能体案例。相比于前面的案例,新颖之处在于拥有攻击手段,能够发射射线干扰对手,如何利用攻击手段也称为了智能体进化的课题。