蒙特卡洛采样法近似估计马尔可夫决策过程MDP中不同策略下的占用度量

""" 合法的占用度量是指存在一个策略使得智能体和MDP交互产生的状态动作对被访问的概率 """
""" 近似估计一个策略的占用度量 ，即设置一个较大的采样轨迹长度的最大值，然后采样很多次，用状态动作对出现的频率估计实际概率"""
def join(str1,str2):
    return str1 +"-"+ str2
def sample(MDP,pi,timestep_max,number):
    """ 采样函数,策略pi,限制最长的时间步timestep_max,总共采样序列数number """
    S,A,P,R,gamma=MDP
    episodes=[]
    for _ in range(number):
        episode=[]
        timestep=0
        s=S[np.random.randint(0,len(S)-1)]#随机选择一个除s5终止状态以外的状态作为起点
        while s!="s5" and timestep<=timestep_max:
            timestep+=1
            rand,temp=np.random.rand(),0
            #在状态s下根据策略选择动作
            for a_opt in A:
                temp+=pi.get(join(s,a_opt),0)
                if temp > rand:
                    a=a_opt
                    r=R.get(join(s,a),0)
                    break
            rand,temp=np.random.rand(),0
            #根据状态转移矩阵得到下一个状态s_next
            for s_opt in S:
                temp+=P.get(join(join(s,a),s_opt),0)
                if temp>rand:
                    s_next=s_opt
                    break
            episode.append((s,a,r,s_next)) #把元组(s,a,r,s_next)放入序列中
            s=s_next#s_next变成当前状态，开始接下来的循环
        episodes.append(episode)
    return episodes
def occupancy_measure(episodes,s,a,timestep_max,gamma):
    rho=0
    total_times=np.zeros(timestep_max)#记录每个时间步各被经历过的次数
    occupied_times=np.zeros(timestep_max)#记录(s_t,a_t)=(s,a)的次数
    for episode in episodes:
        for i in range(len(episode)):
            (s_opt,a_opt,r,s_next)=episode[i]
            total_times[i]+=1
            if s==s_opt and a==a_opt:
                occupied_times[i]+=1
    for i in reversed(range(timestep_max)):
        if total_times[i]:
            rho+=gamma**i *occupied_times[i]/total_times[i]
    return (1-gamma)*rho
S=["s1","s2","s3","s4","s5","s6"]
A=["保持s1","前往s1","前往s2","前往s3","前往s4","前往s5","概率前往"]#动作集合
P={"s1-保持s1-s1":1.0,"s1-前往s2-s2":1.0,"s2-前往s1-s1":1.0,"s2-前往s3-s3":1.0,"s3-前往s4-s4":1.0,"s3-前往s5-s5":1.0,"s4-前往s5-s5":1.0,"s4-概率前往-s2":0.2,"s4-概率前往-s3":0.4,"s4-概率前往-s4":0.4}#状态转移函数
R={"s1-保持s1":-1,"s1-前往s2":0,"s2-前往s1":-1,"s2-前往s3":-2,"s3-前往s4":-2,"s3-前往s5":0,"s4-前往s5":10,"s4-概率前往":1}#奖励函数
gamma=0.5#折扣因子
MDP=(S,A,P,R,gamma)
#策略一,随机策略
pi_1={"s1-保持s1":0.5,"s1-前往s2":0.5,"s2-前往s1":0.5,"s2-前往s3":0.5,"s3-前往s4":0.5,"s3-前往s5":0.5,"s4-前往s5":0.5,"s4-概率前往":0.5}
#策略二
pi_2={"s1-保持s1":0.6,"s1-前往s2":0.4,"s2-前往s1":0.3,"s2-前往s3":0.7,"s3-前往s4":0.5,"s3-前往s5":0.5,"s4-前往s5":0.1,"s4-概率前往":0.9}
np.random.seed()
episodes_1=sample(MDP=MDP,pi=pi_1,timestep_max=1000,number=1000)
episodes_2=sample(MDP=MDP,pi=pi_2,timestep_max=1000,number=1000)
rho_1=occupancy_measure(episodes=episodes_1,s="s4",a="概率前往",timestep_max=1000,gamma=0.5)
rho_2=occupancy_measure(episodes=episodes_2,s="s4",a="概率前往",timestep_max=1000,gamma=0.5)
print("使用蒙特卡洛采样法计算的占用度量为:\n",rho_1,rho_2)