Q-Learning完成迷宫任务

Q-Learning完成迷宫任务

 

项目地址：Q-Learning完成迷宫任务 - CAWCAW的意思是鸦叫声，Guaik的Logo是一只长相奇特的乌鸦。https://caw.guaik.io/d/23-q-learning

# 导入使用的包
import numpy as np
import matplotlib.pyplot as plt
from matplotlib import animation
from IPython.display import HTML
import matplotlib.cm as cm
%matplotlib inline

# 迷宫的初始位置

# 声明图的大小以及图的变量名
fig = plt.figure(figsize=(6, 6))
ax = plt.gca()

# 画出红色的墙壁
plt.plot([1, 1], [4, 3], color='red', linewidth=2)
plt.plot([1, 2], [3, 3], color='red', linewidth=2)
plt.plot([2, 2], [3, 2], color='red', linewidth=2)
plt.plot([1, 2], [2, 2], color='red', linewidth=2)
plt.plot([1, 1], [2, 1], color='red', linewidth=2)

plt.plot([1, 2], [5, 5], color='red', linewidth=2)
plt.plot([2, 2], [5, 4], color='red', linewidth=2)
plt.plot([2, 3], [4, 4], color='red', linewidth=2)
plt.plot([3, 3], [4, 3], color='red', linewidth=2)
plt.plot([3, 4], [3, 3], color='red', linewidth=2)

plt.plot([2, 2], [1, 0], color='red', linewidth=2)
plt.plot([2, 3], [1, 1], color='red', linewidth=2)
plt.plot([3, 3], [1, 2], color='red', linewidth=2)
plt.plot([3, 4], [2, 2], color='red', linewidth=2)
plt.plot([4, 4], [2, 1], color='red', linewidth=2)

plt.plot([3, 3], [6, 5], color='red', linewidth=2)

plt.plot([4, 4], [6, 4], color='red', linewidth=2)

plt.plot([5, 5], [5, 4], color='red', linewidth=2)
plt.plot([5, 6], [5, 5], color='red', linewidth=2)

plt.plot([5, 6], [3, 3], color='red', linewidth=2)
plt.plot([5, 6], [2, 2], color='red', linewidth=2)
plt.plot([5, 6], [1, 1], color='red', linewidth=2)

# 画出表示状态的文字S0-S35
s = 0
for i in range(0, 6):
    for j in range(0, 6):
        plt.text(0.5 + j, 5.5 - i, 'S'+str(s), size=14, ha='center')
        s += 1

plt.text(0.5, 5.3, 'START', ha='center')
plt.text(5.5, 0.3, 'GOAL', ha='center')

# 设置画图的范围
ax.set_xlim(0, 6)
ax.set_ylim(0, 6)
plt.tick_params(axis='both', which='both', bottom='off', top='off',
                labelbottom='off', right='off', left='off', labelleft='off')

# 当前位置S0用绿色圆圈画出
line, = ax.plot([0.5], [5.5], marker='o', color='g', markersize=40)

# 设定参数θ的初始值theta_0，用于确定初始方案

# 行为状态0-34，列为上右下左的移动方向
theta_0 = np.array([
    [np.nan, 1, 1, np.nan],  # S0
    [np.nan, 1, np.nan, 1],  # S1
    [np.nan, np.nan, 1, 1],  # S2
    [np.nan, np.nan, 1, np.nan],       # S3
    [np.nan, 1, 1, np.nan],  # S4
    [np.nan, np.nan, np.nan, 1],    # S5

    [1, 1, 1, np.nan],  # S6
    [np.nan, np.nan, 1, 1],  # S7
    [1, 1, np.nan, np.nan],  # S8
    [1, np.nan, 1, 1],       # S9
    [1, np.nan, 1, np.nan],  # S10
    [np.nan, np.nan, 1, np.nan],    # S11

    [1, np.nan, 1, np.nan],  # S12
    [1, 1, np.nan, np.nan],  # S13
    [np.nan, np.nan, 1, 1],  # S14
    [1, 1, np.nan, np.nan],       # S15
    [1, 1, 1, 1],  # S16
    [1, np.nan, np.nan, 1],    # S17

    [1, 1, 1, np.nan],  # S18
    [np.nan, np.nan, np.nan, 1],  # S19
    [1, 1, 1, np.nan],  # S20
    [np.nan, 1, np.nan, 1],       # S21
    [1, 1, 1, 1],  # S22
    [np.nan, np.nan, np.nan, 1],    # S23

    [1, np.nan, 1, np.nan],  # S24
    [np.nan, 1, 1, np.nan],  # S25
    [1, np.nan, np.nan, 1],  # S26
    [np.nan, np.nan, 1, np.nan],       # S27
    [1, 1, 1, np.nan],  # S28
    [np.nan, np.nan, np.nan, 1],    # S29

    [1, 1, np.nan, np.nan],  # S30
    [1, np.nan, np.nan, 1],  # S31
    [np.nan, 1, np.nan, np.nan],  # S32
    [1, 1, np.nan, 1],       # S33
    [1, 1, np.nan, 1],  # S34
    # [1, np.nan, np.nan, np.nan],    # S35
])

# 设置初始的动作价值函数

[a, b] = theta_0.shape
Q = np.random.rand(a, b) * theta_0 * 0.1 # 乘theta_0是为了使墙壁方向的值为nan

# 将策略参数theta_0转化为随机策略

def simple_convert_into_pi_from_theta(theta):
    ''' 简单计算比率 '''
    [m, n] = theta.shape
    pi = np.zeros((m, n))
    for i in range(0, m):
        pi[i, :] = theta[i, :] / np.nansum(theta[i, :])
    pi = np.nan_to_num(pi)
    return pi

# 求取随机系统策略pi_0
pi_0 = simple_convert_into_pi_from_theta(theta_0)

# 实现贪婪算法
def get_action(s, Q, epsilon, pi_0):
    direction = ['up', 'right', 'down', 'left']

    # 确定行动
    if np.random.rand() < epsilon:
        # 以概率随机行动
        next_direction = np.random.choice(direction, p=pi_0[s, :])
    else:
        # 采用q的最大值对应的动作
        next_direction = direction[np.nanargmax(Q[s, :])]

    # 为动作加上索引
    if next_direction == 'up':
        action = 0
    elif next_direction == 'right':
        action = 1
    elif next_direction == 'down':
        action = 2
    elif next_direction == 'left':
        action = 3
    return action

def get_s_next(s, a, Q, epsilon, pi_0):
    direction = ['up', 'right', 'down', 'left']
    next_direction = direction[a]

    # 通过动作确定下一个状态
    if next_direction == 'up':
        s_next = s - 6
    elif next_direction == 'right':
        s_next = s + 1
    elif next_direction == 'down':
        s_next = s + 6
    elif next_direction == 'left':
        s_next = s - 1

    return s_next

# 基于Q学习的更新动作价值函数Q
def Q_learning(s, a, r, s_next, Q, eta, gamma):
    if s_next == 35:
        Q[s, a] = Q[s, a] + eta * (r - Q[s, a])
    else:
        Q[s, a] = Q[s, a] + eta * (r + gamma * np.nanmax(Q[s_next, :]) - Q[s, a])

    return Q

# 定义基于Q-Learning求解迷宫的函数，输出状态、动作的历史记录以及更新后的Q
def goal_maze_ret_s_a_Q(Q, epsilon, eta, gamma, pi):
    s = 0
    a = a_next = get_action(s, Q, epsilon, pi) # 初始动作
    s_a_history = [[0, np.nan]] # 记录智能体移动序列

    while (1):
        a = a_next # 更新动作

        # 将动作放在现在的状态下
        s_a_history[-1][1] = a  

        # 获取有效的下一个状态
        s_next = get_s_next(s, a, Q, epsilon, pi)
        # 代入下一个状态，动作未知时为nan
        s_a_history.append([s_next, np.nan])

        if s_next == 35:
            r = 1
            a_next = np.nan
        else:
            r = 0
            a_next = get_action(s_next, Q, epsilon, pi)
        
        Q = Q_learning(s, a, r, s_next, Q, eta, gamma)

        if s_next == 35:
            break
        else:
            s = s_next
    return [s_a_history, Q]

eta = 0.1       # 学习率
gamma = 0.9     # 时间折扣率
epsilon = 0.5   # 贪婪算法初始值
v = np.nanmax(Q, axis=1) # 求每个状态价值的最大值
is_continue = True
episode = 1

V = [] # 存放每回合的状态价值
V.append(np.nanmax(Q, axis=1))

while is_continue:
    print('回合数：'+str(episode))

    # 贪婪算法的值逐渐减少
    epsilon = epsilon / 2

    # 通过Sarsa求解迷宫问题，求取移动历史和更新后的Q值
    [s_a_history, Q] = goal_maze_ret_s_a_Q(Q, epsilon, eta, gamma, pi_0)

    new_v = np.nanmax(Q, axis=1)
    print(np.sum(np.abs(new_v - v)))
    v = new_v
    V.append(v)

    print('求解迷宫问题所需步数 '+str(len(s_a_history) - 1))

    # 重复100回合
    episode = episode + 1
    if episode > 100:
        break
print(Q)

# 制作走迷宫动画
def init():
    line.set_data([], [])
    return (line,)

def animate(i):
    ''' 每一帧画面内容 '''
    state = s_a_history[i][0]
    x = (state % 6) + 0.5
    y = 5.5 - int(state / 6)
    line.set_data(x, y)
    return (line,)

anim = animation.FuncAnimation(fig, animate, init_func=init, frames=len(s_a_history), interval=100, repeat=False)
HTML(anim.to_jshtml())

# 制作Q-Learning行动价值训练动画
# def init():
#     line.set_data([], [])
#     return (line,)

# def animate(i):
#     for n in range(0, 6):
#         for j in range(0, 6):
#             if n == 5 and j == 5:
#                 line, = ax.plot([0.5 + j], [5.5 - n], marker="s", color=cm.jet(1.0), markersize=40)
#             else:
#                 line, = ax.plot([0.5 + j], [5.5 - n], marker="s", color=cm.jet(V[i][n*6+j]), markersize=40)
#     return (line,)

# anim = animation.FuncAnimation(fig, animate, init_func=init, frames=len(V), interval=100, repeat=False)
# HTML(anim.to_jshtml())