Sample One Step Look Ahead Controller

The Sample One Step Look-Ahead controller implements a simple controller that evaluates the states reached within one move from the current state. The controller tries all available actions in the current state (call to advance), and evaluates the states found after applying each one of these actions. The action that took to the state with the best reward is the one that will be executed. From Agent.java:

package sampleonesteplookahead;
public class Agent extends AbstractPlayer {

    public Agent(StateObservation stateObs, ElapsedCpuTimer elapsedTimer) {}

    public Types.ACTIONS act(StateObservation stateObs, ElapsedCpuTimer elapsedTimer) {
        Types.ACTIONS bestAction = null;
        double maxQ = Double.NEGATIVE_INFINITY; //Variable to store the max reward (Q) found.
        SimpleStateHeuristic heuristic =  new SimpleStateHeuristic(stateObs);
        for (Types.ACTIONS action : stateObs.getAvailableActions()) { //For all available actions.
            StateObservation stCopy = stateObs.copy();  //Copy the original state (to apply action from it)
            stCopy.advance(action);                     //Apply the action. Object 'stCopy' holds the next state.
            double Q = heuristic.evaluateState(stCopy); //Get the reward for this state.

            //Keep the action with the highest reward.
            if (Q > maxQ) { 
                maxQ = Q;
                bestAction = action;
            }
        }
        //Return the best action found.
        return bestAction;
    }
}

The state evaluation is performed by the class SimpleStateHeuristic, when the method evaluateState is called. The following code shows how this method (from SimpleStateHeuristic.java) evaluates the given state. Note that it uses some of the methods described in the Forward Model, querying for positions of other sprites in the game.

public double evaluateState(StateObservation stateObs) {

    //Position of the avatar:
    Vector2d avatarPosition = stateObs.getAvatarPosition();                                   
    //Positions of all NPCs in the game:
    ArrayList[] npcPositions = stateObs.getNPCPositions(avatarPosition);     
    //Positions of all portals in the game:
    ArrayList[] portalPositions = stateObs.getPortalsPositions(avatarPosition);   

    //First, evaluate win/lose condition, which takes preference.
    double won = 0;
    if (stateObs.getGameWinner() == Types.WINNER.PLAYER_WINS) {
        won = 1000000000;
    } else if (stateObs.getGameWinner() == Types.WINNER.PLAYER_LOSES) {
        return -999999999;
    }

    //Count how many NPCs are in the game, and keep the distance to the closest one.
    double minDistance = Double.POSITIVE_INFINITY;
    int npcCounter = 0;
    if (npcPositions != null) {
        //Each one of the arrays from 'npcPositions' corresponds to a different type of NPC.
        for (ArrayList npcs : npcPositions) {
            if(npcs.size() > 0)
            {
                minDistance = npcs.get(0).sqDist;   //This is the (square) distance to the closest NPC.
                npcCounter += npcs.size();
            }
        }
    }

    //If there are no portals, return an score based on the score plus the information from the NPCs.
    if (portalPositions == null) {
        double score = 0;
        if (npcCounter == 0) {
            score = stateObs.getGameScore() + won*100000000;
        } else {
            score = -minDistance / 100.0 + (-npcCounter) * 100.0 + stateObs.getGameScore() + won*100000000;
        }

        return score;
    }

    //We have portals. Get the number of portals and distance to the closest one.
    double minDistancePortal = Double.POSITIVE_INFINITY;
    Vector2d minObjectPortal = null;
    for (ArrayList portals : portalPositions) {
        if(portals.size() > 0)
        {
            minObjectPortal   =  portals.get(0).position; //This is the closest portal
            minDistancePortal =  portals.get(0).sqDist;   //This is the (square) distance to the closest portal
        }
    }

    //Return the reward of the state based on the score of the game and the portals information.
    double score = 0;
    if (minObjectPortal == null) {
        score = stateObs.getGameScore() + won*100000000;
    }
    else {
        score = stateObs.getGameScore() + won*1000000 - minDistancePortal * 10.0;
    }

    return score;
}