Q例子参考自: http://blog.csdn.net/pi9nc/article/details/27649323
Qlearning 选则策略:
1、 E-greedy
int getNextActionE_Greedy(int state)
{
int selectAction = -1;
boolean random= false;
double maxQ = -50000;
int[] doubleValues = new int[st.actionLength];
for(int i = 0;i < st.actionLength;i++)
{
doubleValues[i] = 0;
}
int maxDV = 0;
double selectPossbility = proRandomNumber.nextDouble();
if(selectPossbility < st.epsilon)
{
selectAction = -1;
random = true;
}
else
{
for(int action = 0;action < st.actionLength;action ++)
{
// 如果发现值相同的多个就记录下来
if(st.qValues[state][action]> maxQ)
{
selectAction = action;
maxQ = st.qValues[state][action];
maxDV = 0;
doubleValues[maxDV] = selectAction;
}
else if(st.qValues[state][action] == maxQ)
{
maxDV++;
doubleValues[maxDV] = action;
}
}
if( maxDV > 0 )
{
int randomIndex =proRandomNumber.nextInt(maxDV+1);
selectAction = doubleValues[randomIndex];
}
}
if(selectAction == -1)
{
selectAction = proRandomNumber.nextInt(st.actionLength);
}
while(!validAction(state,selectAction))
{
selectAction = proRandomNumber.nextInt(st.actionLength);;
}
return selectAction;
}2、softMax
int getNextActionE_SoftMax(int state)
{
int softmaxDiv = 1;
int selectAction = -1;
double[] prob = new double[st.actionLength];
double sumProb = 0;
int action_softmax;
for(action_softmax = 0 ; action_softmax < st.actionLength ; action_softmax ++ )
{
if(validAction(state, action_softmax) == false){
prob[action_softmax] = 0;
continue;
}
else{
double temp =1.0 * (st.qValues[state][action_softmax] / softmaxDiv);
prob[action_softmax] = Math.exp(temp);
sumProb += prob[action_softmax];
}
}
for(action_softmax = 0 ; action_softmax < st.actionLength ; action_softmax ++ ){
if(validAction(state, action_softmax) == false){
continue;
}
prob[action_softmax] = prob[action_softmax] / sumProb;
}
boolean valid = false;
while(valid == false)
{
double rndValue;
double offset;
rndValue =proRandomNumber.nextDouble();
offset = 0;
for(action_softmax = 0 ; action_softmax < st.actionLength ; action_softmax ++ )
{
//好好理解下
if(validAction(state, action_softmax) == false){
continue;
}
if( rndValue > offset && rndValue < (offset + prob[action_softmax]))
selectAction = action_softmax;
offset += prob[action_softmax];
}
//监测行为是否能够到达
if(this.validAction(state, selectAction))
{
valid = true;
}
break;
}
return selectAction;
}验证状态的正确性
boolean validAction(int state,int action)
{
boolean ret = true;
if(action < 0 || action > 5)
ret = false;
if(st.reward[state][action] < 0)
ret = false;
return ret;
}进行训练
void qLearningTrain(long time)
{
int count = 0;
int initState = proRandomNumber.nextInt(st.actionLength);;
do
{
double this_Q;
double max_Q;
double new_Q;
double reward = 0;
//int nextAction = proRandomNumber.nextInt(st.actionLength);
int nextAction = getNextActionE_SoftMax(initState);
// while(!validAction(initState,nextAction)){
// nextAction = proRandomNumber.nextInt(st.actionLength);
// }
reward = st.getReward(initState, nextAction);
this_Q = st.getqValues(initState, nextAction);
max_Q = st.getMaxQValues(nextAction);
// new_Q = reward + (st.alpha) * (max_Q);
new_Q = reward + (st.alpha) * (reward + (st.gamma) * max_Q - this_Q);
st.setqValues(initState, nextAction, new_Q);
initState = nextAction;
}while(initState != 5);
}代码参考 :http://pan.baidu.com/s/1c1VTuyG