该代码为可以正常运行的代码
import numpy as np
import pandas as pd
import time
np.random.seed(2) #reproducible
N_STATES = 6 #the length of the 1 dimensional world
ACTIONS = ['left','right'] #available actions
EPSILON = 0.9 #greedy police 决策上的策略
ALPHA = 0.1 #learning rate
LAMBDA = 0.9 # discount factor 对于未来奖励的衰减值
MAX_EPISODES =13 #maxinum episodes 13回合
#FRESH_TIME = 0.3 #fresh time for one move 间隔时间
FRESH_TIME = 0.001
def build_q_table(n_states,actions):
table = pd.DataFrame(
np.zeros((n_states,len(actions))), #q_table initial values
columns= actions, #action's name
)
print(table)
return table
#print(build_q_table(N_STATE,ACTIONs))
def choose_action(state,q_table):
#This is how to choose an action
state_actions = q_table.iloc[state,:]
if (np.random.uniform() > EPSILON) or (state_actions.all()==0): #act non-greedy or state-action high
action_name = np.random.choice(ACTIONS)
else: #act greedy
#action_name = state_actions.argmax()
action_name = state_actions.idxmax()
return action_name
def get_env_feedback(S,A):
#This is how agent will interact with the environment
if A == 'right': #move right
if S == N_STATES - 2:#terminate
S_ = 'terminal'
R = 1
else:
S_ = S + 1
R = 0
else:
R = 0
if S == 0:
S_ = S #reach the wall
else:
S_ = S - 1
return S_, R
# environment
def update_env(S,episode,step_counter):
#This is how environment be updated
env_list = ['_']*(N_STATES-1) + ['T'] #'____________T' our environment
if S == 'terminal':
interaction = 'Episode %s: total_steps = %s' % (episode+1,step_counter)
print('\r{}'.format(interaction),end='')
time.sleep(2)
print('\r ',end='')
else:
env_list[S] = 'o'
interaction = ''.join(env_list)
print('\r{}'.format(interaction),end='')
time.sleep(FRESH_TIME)
def rl():
# main part of RL loop
q_table = build_q_table(N_STATES,ACTIONS)
for episode in range(MAX_EPISODES):
step_counter = 0
S = 0
is_terminated = False
update_env(S,episode,step_counter)
while not is_terminated:
A = choose_action(S,q_table)
S_,R = get_env_feedback(S,A)
q_predict = q_table.loc[S, A]
if S_ != 'terminal':
q_target = R + LAMBDA * q_table.iloc[S_, :].max()
else:
q_target = R # next state is terminal
is_terminated = True #terminate this episode
q_table.loc[S, A] += ALPHA * (q_target - q_predict) # update
S = S_ #move to next state
update_env(S,episode,step_counter+1)
step_counter +=1
return q_table
if __name__ == "__main__":
q_table = rl()
print('\r\nQ-table:\n')
print(q_table)
运行程序时遇到的问题:
当直接按照视频中的代码运行时,q_predict = q_table.ix[S, A],该行代码报错: ‘DataFrame’ object has no attribute ‘ix’ 。
于是,到网上查到是因为新库对名称变化,将ix改为iloc,变为q_predict = q_table.iloc[S, A]。
但是,运行后又开始报错ValueError: Location based indexing can only have [integer, integer slice (START point is INCLUDED, END point is EXCLUDED), listlike of integers, boolean array] types。
到网上查找原因:说使用iloc方法的时候,两个关键变量都要是下标位置而不是列的名称,iloc[下标位置,下标位置]
。然后将其修改为
q_predict = q_table.ix[S,ACTIONS.index(A)]
第一轮可以运行,后面又开始报错,因为后面会出现A为1的情况,于是,又去网上搜,发现:
`loc:通过行标签索引数据
iloc:通过行号索引行数据
ix:通过行标签或行号索引数据(基于loc和iloc的混合)`
于是试图采用try,except的方法来跳过这个问题,即修改为
try:
q_predict = q_table.iloc[S, A]
except:
q_predict = q_table.loc[S, A]
之后,代码确实能够运行,并且不报错,但是速度奇慢,当把EPSILON ,也就是策略调成0.1的时候才能看到最终结果,就去翻评论,然后再看代码,表示代码逻辑上是没有问题的,可能是try,except导致的速度变慢,后来经过多次查找,终于找到一位大佬修改的pytorch版本的代码,比对之后,发现
def choose_action(state,q_table):
#This is how to choose an action
state_actions = q_table.iloc[state,:]
if (np.random.uniform() > EPSILON) or (state_actions.all()==0): #act non-greedy or state-action high
action_name = np.random.choice(ACTIONS)
else: #act greedy
#action_name = state_actions.argmax()
action_name = state_actions.idxmax()
return action_name
之前该函数中的最后一条语句 action_name = state_actions.argmax()
返回的是最大数的索引.argmax有一个参数axis,默认是0,表示第几维的最大值。因本实验中行索引为数字,列索引为字符串,所以会出现之后单独使用q_table.loc或q_table.iloc报错的情况。于是将其改为action_name = state_actions.idxmax()
,后面凡是用到.ix的地方改成.loc就解决该问题了。