策略迭代法,可以进一步优化为最大化价值v
# 价值迭代法
class ValueIteration(object):
def value_iteration(self, agent, max_iter=-1):
iteration = 0
while True:
iteration += 1
new_value_pi = np.zeros_like(agent.value_pi)
for i in range(1, agent.s_len):
value_sas = []
for j in range(0, agent.a_len):
value_sa = np.dot(agent.p[j,i,:],
agent.r+agent.gamma*agent.value_pi)
value_sas.append(value_sa)
new_value_pi[i] = max(value_sas)
diff = np.sqrt(np.sum(np.power(agent.value_pi - new_value_pi, 2)))
if diff < 1e-6:
break
else:
agent.value_pi = new_value_pi
if iteration == max_iter:
break
print("Iter {} rounds converge".format(iteration))
for i in range(1, agent.s_len):
for j in range(0, agent.a_len):
agent.value_q[i,j] = np.dot(agent.p[j,i,:],
agent.r+agent.gamma*agent.value_pi)
max_act = np.argmax(agent.value_q[i,:])
agent.pi[i] = max_act
def value_iteration_demo():
np.random.seed(0)
env = SnakeEnv(10, [3,6])
agent = TableAgent(env)
vi_algo = ValueIteration()
vi_algo.value_iteration(agent)
print('return_pi={}'.format(eval_game(env,agent)))
print(agent.pi)
value_iteration_demo()