Q-learning强化学习

import numpy as np
import pandas as pd
import time

np.random.seed(2) # reproducible
N_STATES = 6
# the length of the 1 dimensional world
ACTIONS = [ 'Left','right']
# available actions
EPSILON = 0.9
# greedy police
ALPHA = 0.1
# learning rate
GAMMA = 0.9
# discount factor
MAX_EPISODES = 13
# maximum episodes
FRESH_TIME = 0.3

def build_q_table(n_states,actions):
   
    table = pd.DataFrame(
    np.zeros((n_states,len(actions))),# q_table ir
    columns=actions,
# actions' s name
   )
# print(table)
# show table
    return table
def choose_action(state,q_table):
# This is how to choose an action
    state_actions = q_table.iloc[state,:]
    if(np.random.uniform() > EPSILON) or ((state_actions == 0).all()):
#
        action_name = np.random.choice(ACTIONS)
    else:
# act greedy
            action_name = state_actions.idxmax()
# replace argmax to idxmax
    return action_name

def get_env_feedback(S,A):
    if A == 'right':
        if S == N_STATES - 2:
            S_ = 'terminal'
            R = 1
        else:
            S_ = S+1
            R=0
    else:
        R=0
        if S == 0:
            S_ = S
           
        else:
           
            S_ = S - 1
    return S_, R


def update_env(S, episode, step_counter):
   
# This is how environment be updated
    env_list = ['-']*(N_STATES-1) + ['T']# ' ---------T ' our environment
   
    if S == 'terminal':
        interaction = 'Episode %s: total_ steps = %s' % (episode+1, step_counter)
       
        print('\r{}'.format(interaction),end='')
       
        time.sleep(2)
        print('\r',end='')
    else:
       
        env_list[S] = 'o'
       
        interaction = ''.join(env_list)
        print('\r{}'.format(interaction), end='')
        time.sleep(FRESH_TIME)


def r1():
# main part of RL 1oop
    q_table = build_q_table(N_STATES,ACTIONS)
    for episode in range(MAX_EPISODES):
       
        step_counter = 0
        S=0
        is_terminated = False
        update_env(S, episode, step_counter )
        while not is_terminated:
            A = choose_action(S, q_table)
            S_,R = get_env_feedback(S, A)# take action & get next state anc
            q_predict = q_table.loc[S, A]
            if S_ !='terminal':
               
                q_target = R + GAMMA * q_table.iloc[S_, :].max()
                # next stat
            else:                   
                q_target = R# next state is terminal
                is_terminated = True# terminate this episode
            q_table.loc[S, A] += ALPHA * (q_target - q_predict) # update
            S=S_
            update_env(S, episode, step_counter+1)
            step_counter += 1
    return q_table
   
   
if __name__=="__main__":
   
    q_table = r1()
    print('\r\nQ-table:\n')
    print(q_table)   

实验结果：