import java.util.Random;
/*
* The Qlearner class constructs Qlearner objects, a type of player object, since they
* represent players in a repeated general-player, general-state, general-action,
* general-sum game using a Q-Learning epsilon-greedy algorithm for their choice of action.
*/
public class Qlearner extends Player{
/*
*
*/
private double currentAlpha;
private double[][] QValues;
private final double GAMMA;
private final double ALPHA_DECAY;
private final double EPSILON;
/*
* Since in making its decisions, a Qlearner does not take into account
*
* ImaxQ represents the highest possible utility that can be received by the player in
* any state of the game. All Q values are originally set as their maximum possible
* weight. The Q-Values array holds the estimated utility of, in each state (the first
* dimension), taking a certain action (the second dimension).
*/
public Qlearner(int actions, int states, double discount, double decayRate, double explor, int[][][] rewards){
//pass in or ignore general Player fields
super(states, 1, actions, 1, rewards, null);
int globMaxUtil = UTILITIES[0][0][0];
for(int i = 0; i < NUM_STATES; i++){
for(int j = 0; j < NUM_ACTIONS; j++){
for(int k = 0; k < NUM_STATES; k++){
if(UTILITIES[i][j][k] > globMaxUtil){
globMaxUtil = UTILITIES[i][j][k];
}
}
}
}
//pass in Qlearner-specific field values
currentAlpha = 1;
QValues = new double[NUM_STATES][NUM_ACTIONS];
ALPHA_DECAY = decayRate;
EPSILON = explor;
GAMMA = discount;
//Initialize QValues
double initialQ = globMaxUtil/(1 - GAMMA);
for(int i = 0; i < NUM_STATES; i++){
for(int j = 0; j < NUM_ACTIONS; j++){
QValues[i][j] = initialQ;
}
}
}
/*
* Quick constructor makes a one-state, two-action game with usual gamma, alpha decay,
* and epsilon values.
*/
public Qlearner(double imaxQ){
//Pass in general Player fields
super(1, 1, 2, 1, null, null);
//Pass in Qlearner-specific fields
currentAlpha = 1;
QValues = new double[NUM_STATES][NUM_ACTIONS];
GAMMA = .999;
ALPHA_DECAY = .999;
EPSILON = .1;
//Initialize QValues
double initialQ = imaxQ/(1 - GAMMA);
for(int i = 0; i < NUM_STATES; i++){
for(int j = 0; j < NUM_ACTIONS; j++){
QValues[i][j] = initialQ;
}
}
}
/*
* The move method uses the current state and Q values to determine an action following
* the Q-Learning algorithm with epsilon-greedy exploration.
*/
public int move()
throws IllegalStateException{
if(currentState >= NUM_STATES){
throw new IllegalStateException();
}
//Explore with probability EPSILON
Random r = new Random();
if(r.nextDouble() < EPSILON){
return r.nextInt(NUM_ACTIONS);
}
//Create an array of integers representing the moves sharing the maximum expected utility
double maxQ = this.getMaxQ();
//System.out.println("maxQ: " + maxQ);
int numWithMax = 0;
for(int i = 0; i < NUM_ACTIONS; i++){
if(QValues[currentState][i] == maxQ){
numWithMax++;
}
}
if(numWithMax == 0){
return -1;
}
int[] movesWithMax = new int[numWithMax];
int counter = 0;
for(int i = 0; i < NUM_ACTIONS; i++){
if(QValues[currentState][i] == maxQ){
movesWithMax[counter] = i;
counter++;
}
}
//Return a random utility-maximizing move
return movesWithMax[r.nextInt(numWithMax)];
}
/*
* The update method is meant to update the Q-values following a move, given a certain
* received reward. It also updates the variable fields.
*/
public void update(int action, int reward, int newState){
//Perform an iteration of the Q algorithm
int oldState = currentState;
currentState = newState;
QValues[oldState][action] = (1 - currentAlpha)*QValues[oldState][action] + currentAlpha*(reward + GAMMA*this.getMaxQ());
//Increment fields
currentAlpha *= ALPHA_DECAY;
currentTrial++;
}
/*
* Determines estimate of maximum utility obtainable over actions in the current state.
*/
private double getMaxQ(){
//Repetitively replace max variable with the highest seen QValue of the current state
double max = QValues[currentState][0];
for(int i = 1; i < NUM_ACTIONS; i++){
if(QValues[currentState][i] > max){
max = QValues[currentState][i];
}
}
return max;
}
/*
* Returns true iff the Qlearner has not yet incremented its currentState.
*/
public boolean isNew(){
if(currentState == 0){
return true;
}
return false;
}
/*
* Determines if a given game is compatible with the Qlearner.
*/
public boolean canPlay(Game system){
//Obtain the relevant data about "system"
int[] rules = system.getSpecs();
//Check whether the Qlearner's fields match
if(rules[0] != NUM_STATES){
return false;
}
if(rules[2] != NUM_ACTIONS){
return false;
}
if(rules[4] != currentState){
return false;
}
if(rules[5] != currentTrial){
return false;
}
return true;
}
/*
* Hands the value of the GAMMA field.
*/
public double getGamma(){
return GAMMA;
}
}
|