强化学习-蒙特卡洛方法估计一个策略在马尔可夫决策过程中的状态价值

""" 蒙特卡洛采样法估计一个策略在马尔可夫决策过程中的状态价值 :定义一个采样函数，采样函数需要遵从状态转移矩阵和相应的策略"""
def join(str1,str2):
    return str1 +"-"+ str2
def sample(MDP,pi,timestep_max,number):
    """ 采样函数,策略pi,限制最长的时间步timestep_max,总共采样序列数number """
    S,A,P,R,gamma=MDP
    episodes=[]
    for _ in range(number):
        episode=[]
        timestep=0
        s=S[np.random.randint(0,len(S)-1)]#随机选择一个除s5终止状态以外的状态作为起点
        while s!="s5" and timestep<=timestep_max:
            timestep+=1
            rand,temp=np.random.rand(),0
            #在状态s下根据策略选择动作
            for a_opt in A:
                temp+=pi.get(join(s,a_opt),0)
                if temp > rand:
                    a=a_opt
                    r=R.get(join(s,a),0)
                    break
            rand,temp=np.random.rand(),0
            #根据状态转移矩阵得到下一个状态s_next
            for s_opt in S:
                temp+=P.get(join(join(s,a),s_opt),0)
                if temp>rand:
                    s_next=s_opt
                    break
            episode.append((s,a,r,s_next)) #把元组(s,a,r,s_next)放入序列中
            s=s_next#s_next变成当前状态，开始接下来的循环
        episodes.append(episode)
    return episodes
def MC(episodes,V,N,gamma):
    for episode in episodes:
        G=0
        for i in range(len(episode)-1,-1,-1):#一个序列从后往前计算
            (s,a,r,s_next)=episode[i]
            G=r+gamma*G
            N[s]=N[s]+1
            V[s]=V[s]+1/N[s]*(G-V[s])

S=["s1","s2","s3","s4","s5","s6"]
A=["保持s1","前往s1","前往s2","前往s3","前往s4","前往s5","概率前往"]#动作集合
P={"s1-保持s1-s1":1.0,"s1-前往s2-s2":1.0,"s2-前往s1-s1":1.0,"s2-前往s3-s3":1.0,"s3-前往s4-s4":1.0,"s3-前往s5-s5":1.0,"s4-前往s5-s5":1.0,"s4-概率前往-s2":0.2,"s4-概率前往-s3":0.4,"s4-概率前往-s4":0.4}#状态转移函数
R={"s1-保持s1":-1,"s1-前往s2":0,"s2-前往s1":-1,"s2-前往s3":-2,"s3-前往s4":-2,"s3-前往s5":0,"s4-前往s5":10,"s4-概率前往":1}#奖励函数
gamma=0.5#折扣因子
MDP=(S,A,P,R,gamma)
#策略一,随机策略
pi_1={"s1-保持s1":0.5,"s1-前往s2":0.5,"s2-前往s1":0.5,"s2-前往s3":0.5,"s3-前往s4":0.5,"s3-前往s5":0.5,"s4-前往s5":0.5,"s4-概率前往":0.5}
#策略二
pi_2={"s1-保持s1":0.6,"s1-前往s2":0.4,"s2-前往s1":0.3,"s2-前往s3":0.7,"s3-前往s4":0.5,"s3-前往s5":0.5,"s4-前往s5":0.1,"s4-概率前往":0.9}
np.random.seed()
episodes=sample(MDP=MDP,pi=pi_1,timestep_max=20,number=5)
print("第一条序列\n",episodes[0])
print("第二条序列\n",episodes[1])
print("第五条序列\n",episodes[4])
#采样1000次
episodes=sample(MDP=MDP,pi=pi_1,timestep_max=20,number=1000)
V={"s1":0,"s2":0,"s3":0,"s4":0,"s5":0}
N={"s1":0,"s2":0,"s3":0,"s4":0,"s5":0}
MC(episodes,V,N,gamma)
print("使用蒙特卡洛采样法计算MDP的状态价值为:\n",V)
""" 小结 """
#1.用蒙特卡洛方法估计得到的状态价值和用MRP解析解得到的状态价值很接近，得益于大量的采样数据。