#refer from 莫煩python

import numpy as np
import pandas as pd
import time

np.random.seed(2) #reproducible

N_STATES = 6 #the length of the 1 dimensional world
ACTIONS = ['left','right'] #available actions
ESPILON = 0.9 #greedy police
ALPHA = 0.1 #learning rate
GAMMA = 0.9 #discount factor
MAX_EPISODES = 13 #maximum episodes
FRESH_TIME = 0.01 #fresh time for one move

"""
    [Build q table]
    table will like:
        left  right
    0   0.0    0.0
    1   0.0    0.0
    2   0.0    0.0
    3   0.0    0.0
    4   0.0    0.0
    5   0.0    0.0
    
    args:
        n_states : distance
        actions : define how can go
    rets:
        using panda build board
"""
def build_q_table(n_states, actions):
    table = pd.DataFrame(
        np.zeros((n_states, len(actions))),  #q_table inital value
        columns = actions,  #action's name
    )

    print(table) #show table
    return table
"""
    [Choose action]
    In this case,we has choose best strategy 90% time
    left  10% is search
    args:
        state : cuurent state
        q_table : record leraning data
    rets:
        action_name : decision how next step
        
"""
def choose_action(state, q_table):
    #This is how to choose an action
    state_actions = q_table.iloc[state, :]

    if (np.random.uniform() > ESPILON) or \
        (state_actions.all()==0):
        #action non-grreedy or state-action
        action_name = np.random.choice(ACTIONS)
    else: #act greedy
        action_name = state_actions.argmax()

    return action_name
"""
    [Get env feeback]
    args:
    rets:
        S_ : next state
        R : reward
"""
def get_env_feedback(S, A):
    # This is how agent will interact with the enviroment
    if A == 'right': #move right
        if S == N_STATES -2: #terminate
            S_ = 'terminal'
            R = 1
        else:
            S_ = S + 1
            R = 0
    else:   #move left
        R = 0
        if S == 0:
            S_ = S #reach the well
        else:
            S_ = S - 1

    return S_, R

"""
    [Update env]
    simulation env and update print
"""
def update_env(S, episode, step_counter):
    # This is how environment be updated
    env_list = ['-']*(N_STATES-1) + ['T']   # '---------T' our environment
    if S == 'terminal':
        interaction = 'Episode %s: total_steps = %s' % (episode+1, step_counter)
        print('\r{}'.format(interaction), end='')
        time.sleep(2)
        print('\r                                ', end='')
    else:
        env_list[S] = 'o'
        interaction = ''.join(env_list)
        print('\r{}'.format(interaction), end='')
        time.sleep(FRESH_TIME)

"""
    [rl]
    start learning function
"""
def rl():
    # main part of RL loop
    q_table = build_q_table(N_STATES, ACTIONS)
    #total just run 13 episodes
    for episode in range(MAX_EPISODES):
        step_counter = 0
        S = 0
        is_terminated = False
        update_env(S, episode, step_counter)
        while not is_terminated:

            A = choose_action(S, q_table)

            S_, R = get_env_feedback(S, A)  # take action & get next state and reward
            q_predict = q_table.loc[S, A]
            if S_ != 'terminal':
                q_target = R + GAMMA * q_table.iloc[S_, :].max()   # next state is not terminal
            else:
                q_target = R     # next state is terminal
                is_terminated = True    # terminate this episode
            q_table.loc[S, A] += ALPHA * (q_target - q_predict)  # update


            S = S_  # move to next state

            update_env(S, episode, step_counter+1)
            step_counter += 1
    return q_table


if __name__ == "__main__":
    q_table = rl()
    print('\r\nQ-table:\n')
    #print(q_table)
