tabular Q-learning 二维空间搜索小例子

莫烦python的强化学习教程中tabular Q-learning小例子是一个一维的空间，原文链接https://morvanzhou.github.io/tutorials/machine-learning/reinforcement-learning/2-1-general-rl/，本文将其改进为二维空间，代码如下.

# -*- coding: utf-8 -*-
"""
Created on Sun Feb 10 17:03:11 2019

@author: zhpwhy
"""

import numpy as np
import pandas as pd
import time

np.random.seed(1)  # 随机数种子


N_STATES = 5    # 2维世界的宽度
M_STATES = 3    # 2维世界的高度
ACTIONS = ['left', 'right','up','down']     # 探索者的可用动作
EPSILON = 0.9   # 贪婪度 greedy
ALPHA = 0.1     # 学习率
GAMMA = 0.9    # 奖励递减值
MAX_EPISODES = 33    # 最大回合数
FRESH_TIME = 0.3    # 移动间隔时间


def build_q_table(n_states, m_states, actions):
    table = pd.DataFrame(
        np.zeros((n_states*m_states, len(actions))),     # q_table 全 0 初始
        columns=actions,    # columns 对应的是行为名称
    )
    # print(table)    # show table
    return table

# 在某个 state 地点, 选择行为
def choose_action(state, q_table):
    # This is how to choose an action
    state_actions = q_table.iloc[state, :]# 选出这个 state 的所有 action 值
    if (np.random.uniform() > EPSILON) or ((state_actions == 0).all()):  # 非贪婪 or 或者这个 state 还没有探索过
        action_name = np.random.choice(ACTIONS)
    else:   # act greedy
        action_name = state_actions.idxmax()    # 贪婪模式
    return action_name


def get_env_feedback(S, A):
    # This is how agent will interact with the environment
    n_now_states=int(S / N_STATES) #现在目标所在行位置
    m_now_states=int(S % N_STATES) #现在目标所在列位置
    print(n_now_states,m_now_states)
    if A == 'right':    # move right
        if m_now_states == N_STATES - 2 and n_now_states== M_STATES-1:   # terminate
            S_ = 'terminal'
            R = 1
        elif (S+1) % N_STATES ==0 :
            S_ = S
            R = 0
        else:
            S_ = S + 1
            R = 0
    elif A == 'left':    # move left
        R = 0
        if S % N_STATES ==0 :
            S_ = S
            R = 0
        else:
            S_ = S - 1
            R = 0
    elif A == 'down':    # move down
        if  n_now_states == M_STATES - 2 and m_now_states== N_STATES-1:   # terminate
            S_ = 'terminal'
            R = 1
        elif S >= N_STATES*(M_STATES-1) :
            S_ = S
            R = 0
        else:
            S_ = S + N_STATES
            R = 0
    elif A == 'up':    # move up
        R = 0
        if S <= N_STATES :
            S_ = S
            R = 0
        else:
            S_ = S - N_STATES
            R = 0
    return S_, R


def update_env(S, episode, step_counter):
    # This is how environment be updated
    env_list = ['-']*(N_STATES*M_STATES-1) + ['T']   # '---------T' our environment
    if S == 'terminal':
        interaction = 'Episode %s: total_steps = %s' % (episode+1, step_counter)
        print('\r{}'.format(interaction))
        time.sleep(2)
        print('\r                                ')
    else:
        env_list[S] = 'o'
        print('\r{}'.format('******************'))
        for i in range(M_STATES):
            interaction = ''.join(env_list[N_STATES*i:N_STATES*(i+1)])    
            print('\r{}'.format(interaction))
        time.sleep(FRESH_TIME)


def rl():
    # main part of RL loop
    q_table = build_q_table(N_STATES, M_STATES, ACTIONS) 
    for episode in range(MAX_EPISODES):
        step_counter = 0
        S = 0
        is_terminated = False
        update_env(S, episode, step_counter)
        while not is_terminated:

            A = choose_action(S, q_table)
            S_, R = get_env_feedback(S, A)  # take action & get next state and reward
            q_predict = q_table.loc[S, A]
            if S_ != 'terminal':
                q_target = R + GAMMA * q_table.iloc[S_, :].max()   # next state is not terminal
            else:
                q_target = R     # next state is terminal
                is_terminated = True    # terminate this episode

            q_table.loc[S, A] += ALPHA * (q_target - q_predict)  # update
            S = S_  # move to next state

            update_env(S, episode, step_counter+1)
            step_counter += 1
    return q_table


if __name__ == "__main__":
    q_table = rl()
    print('\r\nQ-table:\n')
    print(q_table)