import numpy as np
import tensorflow as tf

import utils as U
from config import args, tf_config
from distributions import make_pd_type


class Policy(object):

    def __init__(self, ob_space, ac_space, batch, n_steps, reuse):
        ob_dim = (batch,) + ob_space.shape
        act_dim = ac_space.shape[0]
        self.ph_obs = tf.placeholder(tf.float32, ob_dim, name='ph_obs')

        with tf.variable_scope('policy', reuse=reuse):
            h1 = U.fc(self.ph_obs, out_dim=64, activation_fn=tf.nn.tanh,
                    init_scale=np.sqrt(2), scope='pi_fc1')
            h2 = U.fc(h1, out_dim=64, activation_fn=tf.nn.tanh,
                    init_scale=np.sqrt(2), scope='pi_fc2')
            pi = U.fc(h2, out_dim=act_dim, activation_fn=None, init_scale=0.01,
                    scope='pi')
            h1 = U.fc(self.ph_obs, out_dim=64, activation_fn=tf.nn.tanh,
                    init_scale=np.sqrt(2), scope='vf_fc1')
            h2 = U.fc(h1, out_dim=64, activation_fn=tf.nn.tanh,
                    init_scale=np.sqrt(2), scope='vf_fc2')
            vf = U.fc(h2, out_dim=1, activation_fn=None, scope='vf')[:, 0]
            logstd = tf.get_variable(name='logstd', shape=[1, act_dim],
                                     initializer=tf.zeros_initializer())
            # concatenate probabilities and logstds
            pd_params = tf.concat([pi, pi * 0.0 + logstd], axis=1)
            self.pd_type = make_pd_type(ac_space)
            self.pd = self.pd_type.pdfromflat(pd_params)

            self.a_out = self.pd.sample()
            self.neglogp = self.pd.get_neglogp(self.a_out)

            self.v_out = vf
            self.pi = pi


class PPO(object):

    def __init__(self, env):
        self.sess = tf.Session(config=tf_config)
        ob_space = env.observation_space
        ac_space = env.action_space
        self.act_policy = Policy(ob_space, ac_space, env.num_envs,
                                 n_steps=1, reuse=False)
        self.train_policy = Policy(ob_space, ac_space, args.minibatch,
                                   n_steps=args.batch_steps, reuse=True)
        self._build_train()
        self.sess.run(tf.global_variables_initializer())

    def _build_train(self):
        # build placeholders
        self.ph_obs_train = self.train_policy.ph_obs
        self.ph_a = self.train_policy.pd_type.get_action_placeholder([None])
        self.ph_adv = tf.placeholder(tf.float32, [None])
        self.ph_r = tf.placeholder(tf.float32, [None])
        self.ph_old_neglogp = tf.placeholder(tf.float32, [None])
        self.ph_old_v = tf.placeholder(tf.float32, [None])
        self.ph_lr = tf.placeholder(tf.float32, [])
        self.ph_clip_range = tf.placeholder(tf.float32, [])

        # build losses
        self.neglogp = self.train_policy.pd.get_neglogp(self.ph_a)
        self.entropy = tf.reduce_mean(self.train_policy.pd.get_entropy())
        v = self.train_policy.v_out
        v_clipped = self.ph_old_v + tf.clip_by_value(
            v - self.ph_old_v, -self.ph_clip_range, self.ph_clip_range)
        v_loss1 = tf.square(v - self.ph_r)
        v_loss2 = tf.square(v_clipped - self.ph_r)
        self.v_loss = 0.5 * tf.reduce_mean(tf.maximum(v_loss1, v_loss2))

        # ratio = tf.exp(self.ph_old_neglogp - self.neglogp)
        old_p = tf.exp(-self.ph_old_neglogp)
        new_p = tf.exp(-self.neglogp)
        ratio = new_p / old_p
        pg_loss1 = -self.ph_adv * ratio
        pg_loss2 = -self.ph_adv * tf.clip_by_value(
            ratio, 1.0 - self.ph_clip_range, 1.0 + self.ph_clip_range)
        self.pg_loss = tf.reduce_mean(tf.maximum(pg_loss1, pg_loss2))
        loss = self.pg_loss + args.v_coef * self.v_loss - \
            args.entropy_coef * self.entropy

        # build train operation
        params = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, 'policy')
        grads = tf.gradients(loss, params)
        if args.max_grad_norm is not None:
            grads, _grad_norm = tf.clip_by_global_norm(
                grads, args.max_grad_norm)
        grads = list(zip(grads, params))
        self.train_op = tf.train.AdamOptimizer(
            learning_rate=self.ph_lr, epsilon=1e-5).apply_gradients(grads)

        # train info
        self.approxkl = 0.5 * tf.reduce_mean(
            tf.square(self.neglogp - self.ph_old_neglogp))
        self.clip_frac = tf.reduce_mean(
            tf.to_float(tf.greater(tf.abs(ratio - 1.0), self.ph_clip_range)))
        self.avg_ratio = tf.reduce_mean(ratio)

    def step(self, obs, *_args, **_kwargs):
        feed_dict = {self.act_policy.ph_obs: obs}
        a, v, neglogp = self.sess.run(
            [self.act_policy.a_out,
             self.act_policy.v_out,
             self.act_policy.neglogp],
            feed_dict=feed_dict)
        return a, v, neglogp

    def get_value(self, obs, *_args, **_kwargs):
        feed_dict = {self.act_policy.ph_obs: obs}
        return self.sess.run(self.act_policy.v_out, feed_dict=feed_dict)

    def train(self, lr, clip_range, obs, returns, masks, actions, values, neglogps,
              advs):
        # advs = returns - values
        advs = (advs - advs.mean()) / (advs.std() + 1e-8)
        feed_dict = {self.ph_obs_train: obs, self.ph_a: actions,
                     self.ph_adv: advs, self.ph_r: returns,
                     self.ph_old_neglogp: neglogps, self.ph_old_v: values,
                     self.ph_lr: lr,
                     self.ph_clip_range: clip_range}
        self.loss_names = ['loss_policy', 'loss_value', 'avg_ratio', 'policy_entropy',
                           'approxkl', 'clipfrac']
        return self.sess.run(
            [self.pg_loss, self.v_loss, self.avg_ratio, self.entropy,
             self.approxkl, self.clip_frac, self.train_op],
            feed_dict=feed_dict)[:-1]

    def save(self, save_path):
        pass

    def load(self, load_path):
        pass
