""" 合法的占用度量是指存在一个策略使得智能体和MDP交互产生的状态动作对被访问的概率 """
""" 近似估计一个策略的占用度量 ,即设置一个较大的采样轨迹长度的最大值,然后采样很多次,用状态动作对出现的频率估计实际概率"""
def join(str1,str2):
return str1 +"-"+ str2
def sample(MDP,pi,timestep_max,number):
""" 采样函数,策略pi,限制最长的时间步timestep_max,总共采样序列数number """
S,A,P,R,gamma=MDP
episodes=[]
for _ in range(number):
episode=[]
timestep=0
s=S[np.random.randint(0,len(S)-1)]#随机选择一个除s5终止状态以外的状态作为起点
while s!="s5" and timestep<=timestep_max:
timestep+=1
rand,temp=np.random.rand(),0
#在状态s下根据策略选择动作
for a_opt in A:
temp+=pi.get(join(s,a_opt),0)
if temp > rand:
a=a_opt
r=R.get(join(s,a),0)
break
rand,temp=np.random.rand(),0
#根据状态转移矩阵得到下一个状态s_next
for s_opt in S:
temp+=P.get(join(join(s,a),s_opt),0)
if temp>rand:
s_next=s_opt
break
episode.append((s,a,r,s_next)) #把元组(s,a,r,s_next)放入序列中
s=s_next#s_next变成当前状态,开始接下来的循环
episodes.append(episode)
return episodes
def occupancy_measure(episodes,s,a,timestep_max,gamma):
rho=0
total_times=np.zeros(timestep_max)#记录每个时间步各被经历过的次数
occupied_times=np.zeros(timestep_max)#记录(s_t,a_t)=(s,a)的次数
for episode in episodes:
for i in range(len(episode)):
(s_opt,a_opt,r,s_next)=episode[i]
total_times[i]+=1
if s==s_opt and a==a_opt:
occupied_times[i]+=1
for i in reversed(range(timestep_max)):
if total_times[i]:
rho+=gamma**i *occupied_times[i]/total_times[i]
return (1-gamma)*rho
S=["s1","s2","s3","s4","s5","s6"]
A=["保持s1","前往s1","前往s2","前往s3","前往s4","前往s5","概率前往"]#动作集合
P={"s1-保持s1-s1":1.0,"s1-前往s2-s2":1.0,"s2-前往s1-s1":1.0,"s2-前往s3-s3":1.0,"s3-前往s4-s4":1.0,"s3-前往s5-s5":1.0,"s4-前往s5-s5":1.0,"s4-概率前往-s2":0.2,"s4-概率前往-s3":0.4,"s4-概率前往-s4":0.4}#状态转移函数
R={"s1-保持s1":-1,"s1-前往s2":0,"s2-前往s1":-1,"s2-前往s3":-2,"s3-前往s4":-2,"s3-前往s5":0,"s4-前往s5":10,"s4-概率前往":1}#奖励函数
gamma=0.5#折扣因子
MDP=(S,A,P,R,gamma)
#策略一,随机策略
pi_1={"s1-保持s1":0.5,"s1-前往s2":0.5,"s2-前往s1":0.5,"s2-前往s3":0.5,"s3-前往s4":0.5,"s3-前往s5":0.5,"s4-前往s5":0.5,"s4-概率前往":0.5}
#策略二
pi_2={"s1-保持s1":0.6,"s1-前往s2":0.4,"s2-前往s1":0.3,"s2-前往s3":0.7,"s3-前往s4":0.5,"s3-前往s5":0.5,"s4-前往s5":0.1,"s4-概率前往":0.9}
np.random.seed()
episodes_1=sample(MDP=MDP,pi=pi_1,timestep_max=1000,number=1000)
episodes_2=sample(MDP=MDP,pi=pi_2,timestep_max=1000,number=1000)
rho_1=occupancy_measure(episodes=episodes_1,s="s4",a="概率前往",timestep_max=1000,gamma=0.5)
rho_2=occupancy_measure(episodes=episodes_2,s="s4",a="概率前往",timestep_max=1000,gamma=0.5)
print("使用蒙特卡洛采样法计算的占用度量为:\n",rho_1,rho_2)
蒙特卡洛采样法近似估计马尔可夫决策过程MDP中不同策略下的占用度量
最新推荐文章于 2024-08-30 12:42:13 发布