import numpy as np
deftransform(state,action):
y,x=state//size,state%size
if action=='n':
y=y-1if y>0and p[y-1,x]else y
elif action=='e':
x=x+1if x<size-1and p[y,x+1]else x
elif action=='s':
y=y+1if y<size-1and p[y+1,x]else y
else:
x=x-1if x>0and p[y,x-1]else x
r=1if y*size+x==success else0
t=Trueif y*size+x==success elseFalsereturn t,y*size+x,r
defepsilon_greed(qs,state,epsilon=.8):if np.random.rand()>epsilon:
keys=['{}_{}'.format(state,i)for i in actions]
r=[qs.get(k)if qs.get(k)isnotNoneelse0for k in keys ]
r=np.where(r==np.max(r))[0]
r=np.random.choice(r)return actions[r]return np.random.choice(actions)defqlearning(state,action):
count=0while count<100:# assert state not in s_over,'{}_{}'.format(state,s_over)
t,next_s,r=transform(state,action)
max_next_q=max([qs[kstr.format(next_s,a)]for a in actions])
qs[kstr.format(state,action)]+=alpha*(r+gama*max_next_q-qs[kstr.format(state,action)])if t:break
state=next_s
action=epsilon_greed(qs,next_s)
count +=1
size=5
success=14
actions=['n','e','s','w']
s_over=[3,8,10,11,14,22,23,24]
p=np.ones((size,size))for s in s_over:
p[s // size, s % size]=0
p[2,-1]=1print(p)
ep=1000
gama=.8
alpha=0.1
kstr='{}_{}'
qs={kstr.format(s,a):0for s inrange(size*size)for a in actions}
ns={}for i inrange(ep):
state=np.random.randint(size*size)ifnot p[state//size,state%size]:continue
action = epsilon_greed(qs, state)
qlearning(state,action)
qs={k:round(qs[k],3)for k in qs}print(qs)for i inrange(size*size):if p[i//size,i%size]:print(i,":",epsilon_greed(qs,i,-1),end='\t')
0: e 1: s 2: s 4: s
5: e 6: e 7: s 9: s
12: e 13: e 14: e
15: e 16: e 17: n 18: n 19: n
20: n 21: n
import numpy as npdef transform(state,action): y,x=state//size,state%size if action=='n': y=y-1 if y>0 and p[y-1,x] else y elif action=='e': x=x+1 if x<size-1 and p[y,x+1] else x elif action=='s': y=y+1 if y&