强化学习实例1：简单最短路径学习

让程序学习到从O到T最短的路径是一直往右走

                                                                   O----T

import numpy as np
import pandas as pd
import time

np.random.seed(2)

N_STATES = 6  # 假设只有5步远
ACTIONS = ['left', 'right']
EPSILON = 0.9
ALPHA = 0.1
GAMMA = 0.9
MAX_EPISODES = 13
FRESH_TIME = 0.3

# 构建Q表格
def build_q_table(n_states, actions):
	table = pd.DataFrame(
		np.zeros((n_states, len(actions))),
		columns=actions,
	)
	return table
# 选择行为
def choose_action(state, q_table):
	state_actions = q_table.iloc[state, :]
	if (((state_actions==0).all())
		or (np.random.uniform() > EPSILON)  # ? 为什么加
		):
		# 两个状态都为零或一个随机概率
		action_name = np.random.choice(ACTIONS)
	else:
		action_name = state_actions.idxmax()
	return action_name
# 获取环境反馈
def get_env_feedback(S, A):
	# agent与环境的交互
	if A == 'right':
		if S == N_STATES - 2:
			S_ = 'terminal'
			R = 1
		else:
			S_ = S + 1
			R = 0
	else:
		R = 0
		if S == 0:
			S_ = S
		else:
			S_ = S - 1
	return S_, R 

# 更新环境
def update_env(S, episode, step_counter):
	env_list = ['-']*(N_STATES-1) + ['T'] # '-------T' 为环境
	if S == 'terminal':
		interaction = 'Episode %s: total_steps = %s'%(episode+1, step_counter)
		print('\r{}'.format(interaction), end='')
		time.sleep(2)
		print('\r                  ', end='')
	else:
		env_list[S] = 'o'
		interaction = ''.join(env_list)
		print('\r{}'.format(interaction), end='')
		time.sleep(FRESH_TIME)

def rl():
	q_table = build_q_table(N_STATES, ACTIONS)
	for episode in range(MAX_EPISODES):
		step_counter = 0
		S = 0
		is_terminated = False
		update_env(S, episode, step_counter)
		while not is_terminated:  # 知道到达终点
			A = choose_action(S, q_table)
			S_, R = get_env_feedback(S, A) # 一步状态和回报

			q_predict = q_table.loc[S, A]  # 当前状态行为得分
			if S_ != 'terminal':
				# 根据下一步的行为得分最高的计算回报
				# 即如果下一步预测判断更准确，当前状态取得更高分
				q_target = R + GAMMA * q_table.iloc[S_, :].max()  
			else:
				q_target = R
				is_terminated = True
			# 计算当前状态下做出行为A的更新率
			q_table.loc[S, A] += ALPHA * (q_target - q_predict) # 更新
			S = S_ # 移动到下一个状态

			update_env(S, episode, step_counter+1)
			step_counter += 1
	return q_table

if __name__ == '__main__':
	q_table = rl()
	print('\r\nQ-table:\n')
	print(q_table)

结果：

Q-table:

       left     right
0  0.000000  0.004320
1  0.000000  0.025005
2  0.000030  0.111241
3  0.000000  0.368750
4  0.027621  0.745813
5  0.000000  0.000000