【强化学习】python实现简单多臂老虎机Tiger machine

简单多臂老虎机原理

• Agent:玩家自己
• Environment: 老虎机
• State:仅有一个状态且奖励是及时的，没有延迟奖励
  设置左右两个老虎机，初始分数为1000
  此处的学习率意义为：有lr的概率我们对老虎机进行随机选择

lr = 0.1 # 学习率
scoreT1 = scoreT2 = 1000 # 初始分数
TimeT1 = TimeT2 = 0 # 选取次数

将奖励设置为服从一定概率分布的随机变量

# 老虎机1为正态分布初始值为500，上下幅度50
Tiger1 = np.random.normal(500, 50, 1)[0]
# 老虎机2为正态分布初始值为550，上下幅度100
Tiger2 = np.random.normal(550, 100, 1)[0]

每次选择老虎机后需要更新分数

# 更新分数
def update_score(score, round_score, t):
    # 分数为之前所有分数之和/总次数
    avg = (score * (t + 1) + round_score) / (t + 2)
    return avg, t + 1

最后经历一定次数迭代后，我们的分数趋近于正态分布设定的分数。从设定上来看老虎机二的期望更大，最后的价值也更大

总算法流程

import numpy as np
import random
# ===============参数设定===============
lr = 0.1
# 初始分数
scoreT1 = scoreT2 = 1000
TimeT1 = TimeT2 = 0
# ===============老虎机算法实现===============
# 更新分数
def update_score(score, round_score, t):
    # 分数为之前所有分数之和/总次数
    avg = (score * (t + 1) + round_score) / (t + 2)
    return avg, t + 1
for i in range(40):
    # 老虎机1为正态分布初始值为500，上下幅度50
    Tiger1 = np.random.normal(500, 50, 1)[0]
    # 老虎机2为正态分布初始值为550，上下幅度100
    Tiger2 = np.random.normal(550, 100, 1)[0]
    
    print('Epoch {} start: '.format(i + 1))
    print('Tiger1 Score: ', Tiger1)
    print('Tiger2 Score: ', Tiger2)
    
    p = np.random.random()
    print("The machine's choice is : ", end='')
    if p < lr: # 执行一次随机选择
        print('\nRandom! ', end='')
        choice = np.random.random()
        if choice < 0.5:
            print('TigerMachine1 ')
            scoreT1, TimeT1 = update_score(scoreT1, Tiger1, TimeT1)
        else:
            print('TigerMachine2 ')
            scoreT2, TimeT2 = update_score(scoreT2, Tiger2, TimeT2)
    else: # 正常选择分数高的老虎机
        if scoreT1 > scoreT2:
            print('TigerMachine1 ')
            scoreT1, TimeT1 = update_score(scoreT1, Tiger1, TimeT1)
        elif scoreT1 < scoreT2:
            print('TigerMachine2 ')
            scoreT2, TimeT2 = update_score(scoreT2, Tiger2, TimeT2)
        else: # 如果一样高，随机选择其中一个
            print("\nTigerMachine1 and 2 have the same score!")
            choice = np.random.random()
            if choice < 0.5:
                print('TigerMachine1 ')
                scoreT1, TimeT1 = update_score(scoreT1, Tiger1, TimeT1)
            else:
                print('TigerMachine2 ')
                scoreT2, TimeT2 = update_score(scoreT2, Tiger2, TimeT2)

    print('The score is: {} || {}'.format(scoreT1, scoreT2))
    print('================================================')
# ===============老虎机结果展示===============
if scoreT1 > scoreT2:
    print("Machine's choice is : TigerMachine1!")
elif scoreT1 < scoreT2:
    print("Machine's choice is : TigerMachine2!")
else:
    print("Machine is not sure which to choose!")