强化学习实例7：价值迭代法（value iteration）

策略迭代法，可以进一步优化为最大化价值v

# 价值迭代法
class ValueIteration(object):
    def value_iteration(self, agent, max_iter=-1):
        iteration = 0
        while True:
            iteration += 1
            new_value_pi = np.zeros_like(agent.value_pi)
            for i in range(1, agent.s_len):
                value_sas = []
                for j in range(0, agent.a_len):
                    value_sa = np.dot(agent.p[j,i,:],
                                     agent.r+agent.gamma*agent.value_pi)
                    value_sas.append(value_sa)
                new_value_pi[i] = max(value_sas)
            diff = np.sqrt(np.sum(np.power(agent.value_pi - new_value_pi, 2)))
            if diff < 1e-6:
                break
            else:
                agent.value_pi = new_value_pi
            if iteration == max_iter:
                break
        print("Iter {} rounds converge".format(iteration))
        for i in range(1, agent.s_len):
            for j in range(0, agent.a_len):
                agent.value_q[i,j] = np.dot(agent.p[j,i,:],
                                           agent.r+agent.gamma*agent.value_pi)
            max_act = np.argmax(agent.value_q[i,:])
            agent.pi[i] = max_act

def value_iteration_demo():
    np.random.seed(0)
    env = SnakeEnv(10, [3,6])
    agent = TableAgent(env)
    vi_algo = ValueIteration()
    vi_algo.value_iteration(agent)
    print('return_pi={}'.format(eval_game(env,agent)))
    print(agent.pi)

value_iteration_demo()