详解《treasure_on_right.py》

"""
A simple example for Reinforcement Learning using table lookup Q-learning method.
An agent "o" is on the left of a 1 dimensional world, the treasure is on the rightmost location.
Run this program and to see how the agent will improve its strategy of finding the treasure.

View more on my tutorial page: https://morvanzhou.github.io/tutorials/
"""

import numpy as np
import pandas as pd
import time

# 随机数种子
np.random.seed(2)  # reproducible

# 一维世界的长度，也是状态的个数
N_STATES = 6   # the length of the 1 dimensional world
# 动作的集合
ACTIONS = ['left', 'right']     # available actions
# 选择利用而不是探索的概率，90%的概率选择最优的动作，10%的概率随机选择动作
EPSILON = 0.9   # greedy police
# 学习率
ALPHA = 0.1     # learning rate
# 折扣因子
GAMMA = 0.9    # discount factor
# 最大迭代次数
MAX_EPISODES = 13   # maximum episodes
# 显示停留时间
FRESH_TIME = 0.3    # fresh time for one move


def build_q_table(n_states, actions):
    # 建立Q表格

    # 建立states与actions的表格
    # DataFrame是由多种类型的列构成的二维标签数据结构.
    # 包含index(行标签)和columns(列标签), 彼此独立, 互不影响
    # np.zeros返回给定维度的全零数组
    table = pd.DataFrame(
        np.zeros((n_states, len(actions))),     # q_table initial values
        columns=actions,    # actions' name
    )
    # print(table)    # show table
    return table


def choose_action(state, q_table):
    # 选择动作

    # This is how to choose an action
    # iloc[]函数，属于pandas库，全称为index location，即对数据进行位置索引，从而在数据表中提取出相应的数据
    # 在Q表格中，寻找状态state对应的那一行
    state_actions = q_table.iloc[state, :]
    # np.random.uniform(low=0.0, high=1.0, size=None)，从一个均匀分布[low,high)中随机采样
    # (state_actions == 0).all()，Pandas Series.all()函数返回是否所有元素都为True
    # 即，当该随机选择动作和此行state对所有的action都是0时
    if (np.random.uniform() > EPSILON) or ((state_actions == 0).all()):  # act non-greedy or state-action have no value
        # np.random.choice从数组中随机抽取元素
        # 随机从动作空间中选择动作
        action_name = np.random.choice(ACTIONS)
    else:   # act greedy
        # idxmax()方法返回轴上最大值第一次出现的索引。
        # 选择reward最大的action的名字
        action_name = state_actions.idxmax()    # replace argmax to idxmax as argmax means a different function in newer version of pandas
    return action_name


def get_env_feedback(S, A):
    # 得到环境的反馈

    # This is how agent will interact with the environment

    # 这是智能体如何与环境进行交互的过程
    if A == 'right':    # move right
        # N_STATES为6，当S的索引到达4的时候，形成"----oT"形状，游戏结束了，得到奖励reward=1
        # 否则游戏继续，状态state+1，得不到奖励
        if S == N_STATES - 2:   # terminate
            S_ = 'terminal'
            R = 1
        else:
            S_ = S + 1      # 向右移动，s+1，因为state序列，实际上就是"o"在一维图案上的索引值
            R = 0
    else:   # move left
        # 向左移动没有奖励reward，如果状态states到达最左端，则保持不变，否则状态state-1
        R = 0
        if S == 0:
            S_ = S  # reach the wall
        else:
            S_ = S - 1
    return S_, R


def update_env(S, episode, step_counter):
    # 更新环境

    # This is how environment be updated
    env_list = ['-']*(N_STATES-1) + ['T']   # '---------T' our environment
    if S == 'terminal':
        interaction = 'Episode %s: total_steps = %s' % (episode+1, step_counter)
        print('\r{}'.format(interaction), end='')
        time.sleep(2)
        print('\r                                ', end='')
    else:
        env_list[S] = 'o'
        interaction = ''.join(env_list)
        print('\r{}'.format(interaction), end='')
        time.sleep(FRESH_TIME)


def rl():
    # 强化学习

    # main part of RL loop

    # 初始化Q表格
    q_table = build_q_table(N_STATES, ACTIONS)
    # 循环
    for episode in range(MAX_EPISODES):
        # 记步
        step_counter = 0
        # 状态编号
        S = 0
        # 是否为终止状态
        is_terminated = False

        # 更新环境
        update_env(S, episode, step_counter)

        # 循环直到回合结束为止
        while not is_terminated:

            # 根据当前的状态state，和Q表格的对应关系，选择动作action
            A = choose_action(S, q_table)

            # 将当前的状态和动作放在环境的规则中，得到奖励和下一个新的状态
            S_, R = get_env_feedback(S, A)  # take action & get next state and reward

            # loc函数索引数据
            # 返回表格行S，列A所对应的值
            q_predict = q_table.loc[S, A]

            # 如果执行完选择之后的下一个状态不是终止状态
            if S_ != 'terminal':
                # 当前得到的Q目标 = 得到的奖励 + 加上折扣因子 * 下一个状态的最大reward值
                q_target = R + GAMMA * q_table.iloc[S_, :].max()   # next state is not terminal
            else:
                # 当前得到的Q目标 = 得到的奖励，因为这是一个回合的终止，没有下一个状态了
                q_target = R     # next state is terminal
                is_terminated = True    # terminate this episode

            # 在Q表格中，状态S下采取动作A的奖励R为 学习率ALPHA*(目标值q_target -预测值q_predict)
            q_table.loc[S, A] += ALPHA * (q_target - q_predict)  # update
            S = S_  # move to next state

            # 更新环境
            update_env(S, episode, step_counter+1)
            # 记步
            step_counter += 1
    return q_table


if __name__ == "__main__":
    q_table = rl()
    print('\r\nQ-table:\n')
    print(q_table)