Currently trying to implement a Q table algorithm in my environment created using turtle graphics. When i try running the algorithm which uses Q learning I get an error stating:
File "<ipython-input-1-cf5669494f75>", line 304, in <module>
rl()
File "<ipython-input-1-cf5669494f75>", line 282, in rl
A = choose_action(S, q_table)
File "<ipython-input-1-cf5669494f75>", line 162, in choose_action
state_actions = q_table.iloc[state, :]
File "/Users/himansuodedra/anaconda3/lib/python3.6/site-packages/pandas/core/indexing.py", line 1367, in __getitem__
return self._getitem_tuple(key)
File "/Users/himansuodedra/anaconda3/lib/python3.6/site-packages/pandas/core/indexing.py", line 1737, in _getitem_tuple
self._has_valid_tuple(tup)
File "/Users/himansuodedra/anaconda3/lib/python3.6/site-packages/pandas/core/indexing.py", line 204, in _has_valid_tuple
if not self._has_valid_type(k, i):
File "/Users/himansuodedra/anaconda3/lib/python3.6/site-packages/pandas/core/indexing.py", line 1674, in _has_valid_type
return self._is_valid_list_like(key, axis)
File "/Users/himansuodedra/anaconda3/lib/python3.6/site-packages/pandas/core/indexing.py", line 1723, in _is_valid_list_like
raise IndexingError('Too many indexers')
IndexingError: Too many indexers
I cannot seem to pinpoint the problem. The logic to me looks fine. Also I am able to build the environment thereafter the script gets stuck and i am forced to terminate it. Any help would be great. The code is below:
"""
Reinforcement Learning using table lookup Q-learning method.
An agent "Blue circle" is positioned in a grid and must make its way to the
green square. This is the end goal. Each time the agent should improve its
strategy to reach the final Square. There are two traps the red and the wall
which will reset the agent.
"""
import turtle
import pandas as pd
import numpy as np
import time
np.random.seed(2)
""" Setting Parameters """
#N_STATES = 12 # the size of the 2D world
ACTIONS = ['left', 'right', 'down','up'] # available actions
EPSILON = 0.9 # greedy police (randomness factor)
ALPHA = 0.1 # learning rate
GAMMA = 0.9 # discount factor
MAX_EPISODES = 13 # maximum episodes
FRESH_TIME = 0.3 # fresh time for one move
def isGoal():
if player.xcor() == -25 and player.ycor() == 225:
player.goto(-175,125)
status_func(1)
S_ = 'terminal'
R = 1
interaction = 'Episode %s: total_steps = %s' %(episode+1, step_counter)
print('\r{}'.format(interaction), end='')
time.sleep(2)
print('\r', end='')
return S_, R
else:
pass
def isFire():
if player.xcor() == -25 and player.ycor() == 175:
player.goto(-175,125)
status_func(3)
S_ = 'terminal'
R = -1
interaction = 'Episode %s: total_steps = %s' %(episode+1, step_counter)
print('\r{}'.format(interaction), end='')
time.sleep(2)
print('\r', end='')
return S_, R
else:
pass
def isWall():
if player.xcor() == -125 and player.ycor() == 175:
player.goto(-175,125)
status_func(2)
S_ = 'terminal'
R = -1
interaction = 'Episode %s: total_steps = %s' %(episode+1, step_counter)
print('\r{}'.format(interaction), end='')
time.sleep(2)
print('\r', end='')
return S_, R
else:
pass
""" Player Movement """
playerspeed = 50
""" Create the token object """
player = turtle.Turtle()
player.color("blue")
player.shape("circle")
player.penup()
player.speed(0)
player.setposition(-175,125)
player.setheading(90)
#Move the player left and right
def move_left():
x = player.xcor()
x -= playerspeed
if x < -175:
x = -175
player.setx(x)
isGoal()
isFire()
isWall()
S_ = player.pos()
R = 0
def move_right():
x = player.xcor()
x += playerspeed
if x > -25:
x = -25
player.setx(x)
isGoal()
isFire()
isWall()
S_ = player.pos()
R = 0
def move_up():
y = player.ycor()
y += playerspeed
if y > 225:
y = 225
player.sety(y)
isGoal()
isFire()
isWall()
S_ = player.pos()
R = 0
def move_down():
y = player.ycor()
y -= playerspeed
if y < 125:
y = 125
player.sety(y)
isGoal()
isFire()
isWall()
S_ = player.pos()
R = 0
#Create Keyboard Bindings
turtle.listen()
turtle.onkey(move_left, "Left")
turtle.onkey(move_right, "Right")
turtle.onkey(move_up, "Up")
turtle.onkey(move_down, "Down")
def build_q_table(n_states, actions):
table = pd.DataFrame(
np.zeros((n_states, len(actions))), # q_table initial values
columns=actions, # actions's name
)
# print(table) # show table
return table
def choose_action(state, q_table):
# This is how to choose an action
state_actions = q_table.iloc[state, :]
# act non-greedy or state-action have no value
if (np.random.uniform() > EPSILON) or ((state_actions == 0).all()):
action_name = np.random.choice(ACTIONS)
else: # act greedy
# replace argmax to idxmax as argmax means a different function
action_name = state_actions.idxmax()
return action_name
def get_env_feedback(S, A):
if A == 'right':
move_right()
elif A == 'left':
move_left()
elif A == 'up':
move_up()
else: #down
move_down()
return S_, R
def update_env(S, episode, step_counter):
wn = turtle.Screen()
wn.bgcolor("white")
wn.title("test")
""" Create the Grid """
greg = turtle.Turtle()
greg.speed(0)
def create_square(size,color="black"):
greg.color(color)
greg.pd()
for i in range(4):
greg.fd(size)
greg.lt(90)
greg.pu()
greg.fd(size)
def row(size,color="black"):
for i in range(4):
create_square(size)
def board(size,color="black"):
greg.pu()
greg.goto(-(size*4),(size*4))
for i in range(3):
row(size)
greg.bk(size*4)
greg.rt(90)
greg.fd(size)
greg.lt(90)
def color_square(start_pos,distance_sq, sq_width, color):
greg.pu()
greg.goto(start_pos)
greg.fd(distance_sq)
greg.color(color)
greg.begin_fill()
for i in range(4):
greg.fd(sq_width)
greg.lt(90)
greg.end_fill()
greg.pu()
def initiate_grid():
board(50)
color_square((-200,200),150, 50,color="green")
color_square((-200,150),50, 50,color="black")
color_square((-200,150),150, 50,color="red")
greg.hideturtle()
initiate_grid()
""" Create the token object """
player = turtle.Turtle()
player.color("blue")
player.shape("circle")
player.penup()
player.speed(0)
player.setposition(S)
player.setheading(90)
def rl():
possible_states = {0:(-175,125),
1:(-175,175),
2:(-175,225),
3:(-125,125),
4:(-125,175),
5:(-125,225),
6:(-75,125),
7:(-75,175),
8:(-75,225),
9:(-25,125),
10:(-25,175),
11:(-25,225)}
inv_possible_states = {v:k for k,v in possible_states.items()}
#build the qtable
q_table = build_q_table(len(possible_states),ACTIONS)
for episode in range(MAX_EPISODES):
step_counter = 0
which_state = 0
S = possible_states[which_state]
is_terminated = False
update_env(S,episode,step_counter)
while not is_terminated:
A = choose_action(S, q_table)
# take action & get next state and reward
S_, R = get_env_feedback(S, A)
q_predict = q_table.loc[S, A]
if S_ != 'terminal':
S_ = inv_possible_states[S_]
# next state is not terminal
q_target = R + GAMMA * q_table.iloc[S_, :].max()
else:
q_target = R # next state is terminal
is_terminated = True # terminate this episode
q_table.loc[S, A] += ALPHA * (q_target - q_predict) # update
S = S_ # move to next state
update_env(S, episode, step_counter+1)
step_counter += 1
return q_table
rl()
Short answer: You are confusing the screen coordinates with the 12 states of the environment
Long answer: When A = choose_action(S, q_table) is called and the choose_action method is executed, you are running into problems with the following line of code within that method:
state_actions = q_table.iloc[state, :]
The error IndexingError: Too many indexers is trying to tell you that the value you're trying to access does not exist on the q_table.
If you were to print out the state variable that gets passed into the choose_action function, you'll get this:
(-175, 125)
But that doesn't make sense. If you print entire Q-table before the error happens, you'll see the following values:
left right down up
0 0.0 0.0 0.0 0.0
1 0.0 0.0 0.0 0.0
2 0.0 0.0 0.0 0.0
3 0.0 0.0 0.0 0.0
4 0.0 0.0 0.0 0.0
5 0.0 0.0 0.0 0.0
6 0.0 0.0 0.0 0.0
7 0.0 0.0 0.0 0.0
8 0.0 0.0 0.0 0.0
9 0.0 0.0 0.0 0.0
10 0.0 0.0 0.0 0.0
11 0.0 0.0 0.0 0.0
The values are all zeros because you haven't learned anything yet. But your code is trying to access q_table.iloc[state, :] when state is equal to (-175, 125). That doesn't make any sense!
The value you're passing in to the choose_action method should correspond to one of the twelve states within the environment, represented in the q_table by the integers from 0 to 11.
It seems the problem is being caused from this line:
S = possible_states[which_state]
☝️ That line of code in the rl method is changing the value of S to be (-175, 125). If S is supposed to represent which state of the environment the agent is in, then S should always be an integer between 0 and 11 (inclusively).
You need to make sure that you correctly separate the screen locations that turtle-graphics is drawing from the 12 states of the environment that the agent is exploring. turtle-graphics doesn't know how to draw the environment states as they are stored within q_table, and the q_table doesn't know which states in the environment are associated with the coordinates that turtle-graphics uses to draw the squares.