强化学习——QLearning和Sarsa算法及其Python实现

主要是强化学习的课程，作业中涉及到了QLearning和Sarsa算法，特此记录。
宝藏博主的强化学习专栏中包含了这两个算法的讲解，极为清晰，非常推荐。链接：机器学习+深度学习+强化学习。

QLearning

理论讲解

• 【强化学习】Q-Learning算法详解以及Python实现【80行代码】

代码实现

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import time

ALPHA = 0.1
GAMMA = 0.95
EPSILION = 0.9
N_STATE = 20
ACTIONS = ['left', 'right']
MAX_EPISODES = 200
FRESH_TIME = 0.1


#############  1. Define Q table  ##############

def build_q_table(n_state, actions):
    q_table = pd.DataFrame(
    np.zeros((n_state, len(actions))),
    np.arange(n_state),
    actions
    )
    return q_table

#############  2. Define action  ##############

def choose_action(state, q_table):
    #epslion - greedy policy
    state_action = q_table.loc[state,:]
    if np.random.uniform()>EPSILION or (state_action==0).all():
        action_name = np.random.choice(ACTIONS)
    else:
        action_name = state_action.idxmax()
    return action_name

#############  3. Environment feedback  ##############

def get_env_feedback(state, action):
    if action=='right':
        if state == N_STATE-2:
            next_state = 'terminal'
            reward = 1
        else:
            next_state = state+1
            reward = -0.5
    else:
        if state == 0:
            next_state = 0
            
        else:
            next_state = state-1
        reward = -0.5
    return next_state, reward


#############  4. Update environment   ##############

def update_env(state,episode, step_counter):
    env = ['-'] *(N_STATE-1)+['T']
    if state =='terminal':
        print("Episode {}, the total step is {}".format(episode+1, step_counter))
        final_env = ['-'] *(N_STATE-1)+['T']
        return True, step_counter
    else:
        env[state]='*'
        env = ''.join(env)
        print(env)
        time.sleep(FRESH_TIME)
        return False, step_counter
        
#############  5. Agent   #################
'''
Please complete the code for this section.
Return value:
-- q_table : Refer to the function 'build_q_table'
-- step_counter_times : List: the number of total steps for every episode.
'''
def q_learning():
    q_table = build_q_table(N_STATE, ACTIONS)
    step_counter_times = []
    '''
    main loop: 
    '''
    for episode in range(MAX_EPISODES):
        state = 0                                   # Initalize state
        istate_terminal = False                         # Judgment variable: whether to end episode.
        step_counter = 0
        update_env(state, episode, step_counter)
        while not istate_terminal:
            A = choose_action(state, q_table)
            next_state, reward = get_env_feedback(state, A)
            q_predict = q_table.loc[state, A]

            if next_state != 'terminal':
                q_target = reward + GAMMA * q_table.loc[next_state, :].max()
            else:
                q_target = reward
                istate_terminal = True
            q_table.loc[state, A] += ALPHA * (q_target - q_predict)
            state = next_state
            update_env(state, episode, step_counter + 1)
            step_counter += 1
        step_counter_times.append(step_counter)

    return q_table, step_counter_times



def main():
    q_table, step_counter_times= q_learning()
    print("Q table\n{}\n".format(q_table))
    print('end')
    
    plt.plot(step_counter_times,'g-')
    plt.ylabel("steps")
    plt.savefig('QLearning_figure.png')
    
    plt.show()
    print("The step_counter_times is {}".format(step_counter_times))

main() 

• 训练结果图：
  

Sarsa算法

算法原理

• 强化学习——Sarsa算法

代码实现

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import time

ALPHA = 0.1
GAMMA = 0.95
EPSILION = 0.9
N_STATE = 6
ACTIONS = ['left', 'right']
MAX_EPISODES = 200
FRESH_TIME = 0.1

#############  1. Define Q table  ##############

def build_q_table(n_state, actions):
    q_table = pd.DataFrame(
    np.zeros((n_state, len(actions))),
    np.arange(n_state),
    actions
    )
    return q_table

#############  2. Define action  ##############

def choose_action(state, q_table):
    #epslion - greedy policy
    state_action = q_table.loc[state,:]
    if np.random.uniform()>EPSILION or (state_action==0).all():
        action_name = np.random.choice(ACTIONS)
    else:
        action_name = state_action.idxmax()
    return action_name

#############  3. Environment feedback  ##############

def get_env_feedback(state, action):
    if action=='right':
        if state == N_STATE-2:
            next_state = 'terminal'
            reward = 1
        else:
            next_state = state+1
            reward = -0.5
    else:
        if state == 0:
            next_state = 0
            
        else:
            next_state = state-1
        reward = -0.5
    return next_state, reward

#############  4. Update environment   ##############

def update_env(state,episode, step_counter):
    env = ['-'] *(N_STATE-1)+['T']
    if state =='terminal':
        print("Episode {}, the total step is {}".format(episode+1, step_counter))
        final_env = ['-'] *(N_STATE-1)+['T']
        return True, step_counter
    else:
        env[state]='*'
        env = ''.join(env)
        print(env)
        time.sleep(FRESH_TIME)
        return False, step_counter
        
#############  5. Agent   #################
'''
Please complete the code for this section.
Return value:
-- q_table : Refer to the function 'build_q_table'
-- step_counter_times : List: the number of total steps for every episode.
'''
def sarsa_learning():
    q_table = build_q_table(N_STATE, ACTIONS)
    step_counter_times = []

    '''
    main loop: 
    '''
    for episode in range(MAX_EPISODES):
        state = 0
        is_terminal = False
        step_counter = 0
        update_env(state, episode, step_counter)
        while not is_terminal:
            A = choose_action(state, q_table)
            next_state, reward = get_env_feedback(state, A)
            q_predict = q_table.loc[state, A]

            if next_state != 'terminal':
                q_target = reward + GAMMA * q_table.loc[next_state, A]
            else:
                q_target = reward
                is_terminal = True
            q_table.loc[state, A] += ALPHA * (q_target - q_predict)
            state = next_state
            update_env(state, episode, step_counter + 1)
            step_counter += 1
        step_counter_times.append(step_counter)
                
    return q_table, step_counter_times

def main():
    q_table, step_counter_times= sarsa_learning()
    print("Q table\n{}\n".format(q_table))
    print('end')
    
    plt.plot(step_counter_times,'g-')
    plt.ylabel("steps")
    
    plt.savefig('Sarsa_figure.png')
    plt.show()
    print("The step_counter_times is {}".format(step_counter_times))

main() 

• 训练结果图：