2021-10-17 4.13

def create_random_policy(nA):
    A = np.ones(nA, dtype=float) / nA  # 创建随即策略
    
    def policy_fn(observation):  # 策略函数
        return A
    return policy_fn


def create_greedy_policy(Q):
    def policy_fn(state):  # 创建贪婪策略
        A = np.zeros_like(Q[state], dtype=float)
        best_action = np.argmax(Q[state])
        A[best_action] = 1.0
        return A
    return policy_fn


def mc_control_importance_sampling(env, num_episode, behavior_policy, discount_factor=1.0):
    Q = defaultdict(lambda: np.zeros(env.action_space.n))
    C = defaultdict(lambda: np.zeros(env.action_space.n))
    target_policy = create_greedy_policy(Q)  # 初始化目标策略
    for i_episode in range(1, num_episode+1):
        episode = []
        state = env.reset()
        while(True):
            probs = behavior_policy(state)  # 从行为策略中采样得到当前状态的概率
            action = np.random.choice(np.arange(len(probs)), p=probs)  # 按照动作概率选择动作
            next_state, reward, done, _ = env.step(action)  # 执行动作，得到状态，奖励
            episode.append((state, action, reward))
            if done:
                break
            state = next_state
        G = 0.0  # 未来折扣奖励
        W = 1.0  # 重要性权重参数
        for t in range(len(episode))[::-1]:  # 在该经验轨迹中从最后的时间步开始遍历
            state, action, reward = episode[t]  # 获得当前经验轨迹的当前步
            G = discount_factor * G + reward  # 更新累计奖励
            C[state][action] += W  # 更新累计权重
            Q[state][action] += (W / C[state][action]) * (G - Q[state][action])  # 更新动作值函数
            if action != np.argmax(target_policy(state)):
                break
            W = W * 1. / behavior_policy(state)[action]  # 根据行为策略更新重要性权重参数
    return Q, target_policy