qlearning解简易迷宫

import numpy as np

def transform(state,action):
    y,x=state//size,state%size
    if action=='n':
        y=y-1 if y>0 and p[y-1,x] else y
    elif action=='e':
        x=x+1 if x<size-1 and p[y,x+1] else x
    elif action=='s':
        y=y+1 if y<size-1 and p[y+1,x] else y
    else:
        x=x-1 if x>0 and p[y,x-1] else x
    r=1 if y*size+x==success else 0
    t=True if y*size+x==success else False
    return t,y*size+x,r

def epsilon_greed(qs,state,epsilon=.8):
    if np.random.rand()>epsilon:
        keys=['{}_{}'.format(state,i) for i in actions]
        r=[qs.get(k) if qs.get(k) is not None else 0 for k in keys ]
        r=np.where(r==np.max(r))[0]
        r=np.random.choice(r)
        return actions[r]
    return np.random.choice(actions)

def qlearning(state,action):
    count=0
    while count<100:
        # assert state not in s_over,'{}_{}'.format(state,s_over)
        t,next_s,r=transform(state,action)
        max_next_q=max([qs[kstr.format(next_s,a)] for a in actions])
        qs[kstr.format(state,action)]+=alpha*(r+gama*max_next_q-qs[kstr.format(state,action)])
        if t: break
        state=next_s
        action=epsilon_greed(qs,next_s)
        count += 1

size=5
success=14
actions=['n','e','s','w']
s_over=[3,8,10,11,14,22,23,24]

p=np.ones((size,size))
for s in s_over:
    p[s // size, s % size]=0
p[2,-1]=1
print(p)


ep=1000
gama=.8
alpha=0.1
kstr='{}_{}'
qs={kstr.format(s,a):0 for s in range(size*size) for a in actions}
ns={}

for i in range(ep):
    state=np.random.randint(size*size)
    if not p[state//size,state%size]:continue
    action = epsilon_greed(qs, state)
    qlearning(state,action)

qs={k:round(qs[k],3) for k in qs}
print(qs)
for i in range(size*size):
    if p[i//size,i%size]:
        print(i,":",epsilon_greed(qs,i,-1),end='\t')

0 : e		1 : s		2 : s		4 : s	
5 : e		6 : e		7 : s		9 : s	
12 : e		13 : e		14 : e	
15 : e		16 : e		17 : n		18 : n		19 : n	
20 : n		21 : n