

import numpy as np
import gym
import random
from bokeh.plotting import figure, output_file, show



env = gym.make("FrozenLake-v0")



action_size = env.action_space.n
state_size = env.observation_space.n
qtable = np.zeros((state_size, action_size))
print(qtable)


# In[4]:

total_episodes = 10000        # Total episodes
learning_rate = 0.8           # Learning rate
max_steps = 99                # Max steps per episode
gamma = 0.95                  # Discounting rate

# Exploration parameters
epsilon = 1.0                 # Exploration rate
max_epsilon = 1.0             # Exploration probability at start
min_epsilon = 0.01            # Minimum exploration probability 
decay_rate = 0.001             # Exponential decay rate for exploration prob


#show explord rate data
rate_data = []
#show score in every episode
show_scores = []
# List of rewards
rewards = []

# 2 For life or until learning is stopped
for episode in range(total_episodes):
    # Reset the environment
    state = env.reset()
    step = 0
    done = False
    total_rewards = 0
    rate_data.append(epsilon)
    for step in range(max_steps):
        # 3. Choose an action a in the current world state (s)
        ## First we randomize a number
        exp_exp_tradeoff = random.uniform(0, 1)
        
        ## If this number > greater than epsilon --> exploitation (taking the biggest Q value for this state)
        if exp_exp_tradeoff > epsilon:
            action = np.argmax(qtable[state,:])

        # Else doing a random choice --> exploration
        else:
            action = env.action_space.sample()

        # Take the action (a) and observe the outcome state(s') and reward (r)
        new_state, reward, done, info = env.step(action)

        # Update Q(s,a):= Q(s,a) + lr [R(s,a) + gamma * max Q(s',a') - Q(s,a)]
        # qtable[new_state,:] : all the actions we can take from new state
        qtable[state, action] = qtable[state, action] + learning_rate * (reward + gamma * np.max(qtable[new_state, :]) - qtable[state, action])
        
        total_rewards += reward
        
        # Our new state is state
        state = new_state
        
        # If done (if we're dead) : finish episode
        if done == True:
            show_scores.append(reward)
            break
    if len(show_scores) <= episode:
        show_scores.append(0)
    # Reduce epsilon (because we need less and less exploration)
    epsilon = min_epsilon + (max_epsilon - min_epsilon)*np.exp(-decay_rate*episode) 
    rewards.append(total_rewards)

print ("Score over time: " +  str(sum(rewards)/total_episodes))
print(qtable)

p = figure(title="the explore rate in every episode", width=800, height=800)
p.title.align = "center"
#plot the steps in every episode
p.circle(y=show_scores, x=range(len(show_scores)), color= "navy", legend = "score", alpha=0.1)
#plot the change of export_rate
p.line(y=rate_data, x=range(len(rate_data)), color="red", legend = "explore rate")
p.xaxis.axis_label = "episode"
p.yaxis.axis_label = "explore rate: % / scores"
p.legend.location = "center_right"
output_file("explore_rate.html")
show(p)


all_reward = 0
for episode in range(100):
    state = env.reset()
    rewards = 0

    for _ in range(1000):
        #env.render()
        action = np.argmax(qtable[state,:])
        new_state, reward, done, info = env.step(action)
        rewards += reward
        if done:
            break
        state = new_state
    all_reward += rewards
print("the all reward in 100 episodes is {}".format(all_reward))


