qpython3 l_【强化学习】python 实现 q-learning 例三（例一改写）

本文作者：hhh5460

例一的代码是函数式编写的，这里用面向对象的方式重新撸了一遍。好处是，更便于理解环境(Env)、个体(Agent)之间的关系。

有缘看到的朋友，自己慢慢体会吧。

0.效果图

1.完整代码

import pandas as pd

import random

import time

import pickle

import pathlib

'''

-o---T

# T 就是宝藏的位置, o 是探索者的位置

作者：hhh5460

时间：20181218

地点：Tai Zi Miao

'''

class Env(object):

'''环境类'''

def __init__(self):

'''初始化'''

self.env = list('-----T')

def update(self, state, delay=0.1):

'''更新环境，并打印'''

env = self.env[:]

env[state] = 'o' # 更新环境

print('\r{}'.format(''.join(env)), end='')

time.sleep(delay)

class Agent(object):

'''个体类'''

def __init__(self, alpha=0.01, gamma=0.9):

'''初始化'''

self.states = range(6)

self.actions = ['left', 'right']

self.rewards = [0,0,0,0,0,1]

self.alpha = alpha

self.gamma = gamma

self.q_table = pd.DataFrame(data=[[0 for _ in self.actions] for _ in self.states],

index=self.states,

columns=self.actions)

def save_policy(self):

'''保存Q table'''

with open('q_table.pickle', 'wb') as f:

# Pickle the 'data' dictionary using the highest protocol available.

pickle.dump(self.q_table, f, pickle.HIGHEST_PROTOCOL)

def load_policy(self):

'''导入Q table'''

with open('q_table.pickle', 'rb') as f:

self.q_table = pickle.load(f)

def choose_action(self, state, epsilon=0.8):

'''选择相应的动作。根据当前状态，随机或贪婪，按照参数epsilon'''

if (random.uniform(0,1) > epsilon) or ((self.q_table.ix[state] == 0).all()): # 探索

action = random.choice(self.get_valid_actions(state))

else:

action = self.q_table.ix[state].idxmax() # 利用(贪婪)

return action

def get_q_values(self, state):

'''取状态state的所有Q value'''

q_values = self.q_table.ix[state, self.get_valid_actions(state)]

return q_values

def update_q_value(self, state, action, next_state_reward, next_state_q_values):

'''更新Q value，根据贝尔曼方程'''

self.q_table.ix[state, action] += self.alpha * (next_state_reward + self.gamma * next_state_q_values.max() - self.q_table.ix[state, action])

def get_valid_actions(self, state):

'''取当前状态下所有的合法动作'''

valid_actions = set(self.actions)

if state == self.states[-1]: # 最后一个状态(位置)，则

valid_actions -= set(['right']) # 不能向右

if state == self.states[0]: # 最前一个状态(位置)，则

valid_actions -= set(['left']) # 不能向左

return list(valid_actions)

def get_next_state(self, state, action):

'''对状态执行动作后，得到下一状态'''

# l,r,n = -1,+1,0

if action == 'right' and state != self.states[-1]: # 除非最后一个状态(位置)，向右就+1

next_state = state + 1

elif action == 'left' and state != self.states[0]: # 除非最前一个状态(位置)，向左就-1

next_state = state -1

else:

next_state = state

return next_state

def learn(self, env=None, episode=1000, epsilon=0.8):

'''q-learning算法'''

print('Agent is learning...')

for _ in range(episode):

current_state = self.states[0]

if env is not None: # 若提供了环境，则更新之！

env.update(current_state)

while current_state != self.states[-1]:

current_action = self.choose_action(current_state, epsilon) # 按一定概率，随机或贪婪地选择

next_state = self.get_next_state(current_state, current_action)

next_state_reward = self.rewards[next_state]

next_state_q_values = self.get_q_values(next_state)

self.update_q_value(current_state, current_action, next_state_reward, next_state_q_values)

current_state = next_state

if env is not None: # 若提供了环境，则更新之！

env.update(current_state)

print('\nok')

def play(self, env=None, delay=0.5):

'''玩游戏，使用策略'''

assert env != None, 'Env must be not None!'

if pathlib.Path("q_table.pickle").exists():

self.load_policy()

else:

print("I need to learn before playing this game.")

self.learn(env, 13)

self.save_policy()

print('Agent is playing...')

current_state = self.states[0]

env.update(current_state, delay)

while current_state != self.states[-1]:

current_action = self.choose_action(current_state, 1.) # 1., 不随机

next_state = self.get_next_state(current_state, current_action)

current_state = next_state

env.update(current_state, delay)

print('\nCongratulations, Agent got it!')

if __name__ == '__main__':

env = Env() # 环境

agent = Agent() # 个体

#agent.learn(env, episode=13) # 先学

#agent.save_policy() # 保存所学

#agent.load_policy() # 导入所学

agent.play(env) # 再玩

深度强化学习(DQN-Deep Q Network)之应用-Flappy Bird

深度强化学习(DQN-Deep Q Network)之应用-Flappy Bird 本文系作者原创,转载请注明出处:https://www.cnblogs.com/further-further-fu ...

机器学习之强化学习概览(Machine Learning for Humans: Reinforcement Learning)

声明:本文翻译自Vishal Maini在Medium平台上发布的的教程的

深度强化学习(Deep Reinforcement Learning)入门：RL base & DQN-DDPG-A3C introduction

转自https://zhuanlan.zhihu.com/p/25239682 过去的一段时间在深度强化学习领域投入了不少精力,工作中也在应用DRL解决业务问题.子曰:温故而知新,在进一步深入研究和应 ...

【转】【强化学习】Deep Q Network(DQN)算法详解