# this code is only for standard lstm--kong
'''
Build a tweet sentiment analyzer
'''

from __future__ import print_function
import six.moves.cPickle as pickle

from collections import OrderedDict
from scipy import optimize, stats
import sys
import time

import copy
import cPickle as pkl

import numpy
import theano
from theano import config
import theano.tensor as tensor
from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams

from util.data_handler_lstm import DataHandler
from util.data_handler_lstm import TrainProto
from util.data_handler_lstm import TestTrainProto
#from util.data_handler import TestValidProto
from util.data_handler_lstm import TestTestProto



# Set the random number generators' seeds for consistency
SEED = 123
numpy.random.seed(SEED)

def numpy_floatX(data):
    return numpy.asarray(data, dtype=config.floatX)

def zipp(params, tparams):
    """
    When we reload the model. Needed for the GPU stuff.
    """
    for kk, vv in params.items():
        tparams[kk].set_value(vv)


def unzip(zipped):
    """
    When we pickle the model. Needed for the GPU stuff.
    """
    new_params = OrderedDict()
    for kk, vv in zipped.items():
        new_params[kk] = vv.get_value()
    return new_params


def dropout_layer(state_before, use_noise, trng):
    proj = tensor.switch(use_noise,
                         (state_before *
                          trng.binomial(state_before.shape,
                                        p=0.5, n=1,
                                        dtype=state_before.dtype)),
                         state_before * 0.5)
    return proj


def _p(pp, name):
    return '%s_%s' % (pp, name)


def init_params(options):
    """
    Global (not LSTM) parameter. For the embeding and the classifier.
    """
    params = OrderedDict()
    # embedding
    # randn = numpy.random.rand(options['n_words'],
    #                           options['dim_proj'])
    # params['Wemb'] = (0.01 * randn).astype(config.floatX)
    params = get_layer(options['encoder'])[0](options,
                                              params,
                                              prefix=options['encoder'])
    # classifier
    params['U'] = 0.01 * numpy.random.randn(options['dim_proj'],
                                            options['ydim']).astype(config.floatX)
    params['b'] = numpy.zeros((options['ydim'],)).astype(config.floatX)

    return params


def load_params(path, params):
    pp = numpy.load(path)
    for kk, vv in params.items():
        if kk not in pp:
            raise Warning('%s is not in the archive' % kk)
        params[kk] = pp[kk]

    return params


def init_tparams(params):
    tparams = OrderedDict()
    for kk, pp in params.items():
        tparams[kk] = theano.shared(params[kk], name=kk)
    return tparams


def get_layer(name):
    fns = layers[name]
    return fns


def ortho_weight(ndim):
    W = numpy.random.randn(ndim, ndim)
    u, s, v = numpy.linalg.svd(W)
    return u.astype(config.floatX)


def param_init_lstm(options, params, prefix='lstm'):
    """
    Init the LSTM parameter:
    :see: init_params
    """
    W = numpy.concatenate([ortho_weight(options['dim_proj']),
                           ortho_weight(options['dim_proj']),
                           ortho_weight(options['dim_proj']),
                           ortho_weight(options['dim_proj'])], axis=1)
    params[_p(prefix, 'W')] = W
    U = numpy.concatenate([ortho_weight(options['dim_proj']),
                           ortho_weight(options['dim_proj']),
                           ortho_weight(options['dim_proj']),
                           ortho_weight(options['dim_proj'])], axis=1)
    params[_p(prefix, 'U')] = U
    b = numpy.zeros((4 * options['dim_proj'],))
    params[_p(prefix, 'b')] = b.astype(config.floatX)

    return params


def lstm_layer(tparams, state_below, options, prefix='lstm', mask=None):
    nsteps = state_below.shape[0]
    if state_below.ndim == 3:
        n_samples = state_below.shape[1]
    else:
        n_samples = 1

    assert mask is not None

    def _slice(_x, n, dim):
        if _x.ndim == 3:
            return _x[:, :, n * dim:(n + 1) * dim]
        return _x[:, n * dim:(n + 1) * dim]

    def _step(m_, x_, h_, c_):
        preact = tensor.dot(h_, tparams[_p(prefix, 'U')])
        preact += x_

        i = tensor.nnet.sigmoid(_slice(preact, 0, options['dim_proj']))
        f = tensor.nnet.sigmoid(_slice(preact, 1, options['dim_proj']))
        o = tensor.nnet.sigmoid(_slice(preact, 2, options['dim_proj']))
        c = tensor.tanh(_slice(preact, 3, options['dim_proj']))

        c = f * c_ + i * c
        c = m_[:, None] * c + (1. - m_)[:, None] * c_

        h = o * tensor.tanh(c)
        h = m_[:, None] * h + (1. - m_)[:, None] * h_

        rval = [h, c] #k

        return rval

    state_below = (tensor.dot(state_below, tparams[_p(prefix, 'W')]) +
                   tparams[_p(prefix, 'b')])

    dim_proj = options['dim_proj']
    rval, updates = theano.scan(_step,
                                sequences=[mask, state_below],
                                outputs_info=[tensor.alloc(numpy_floatX(0.),
                                                           n_samples,
                                                           dim_proj),
                                              tensor.alloc(numpy_floatX(0.),
                                                           n_samples,
                                                           dim_proj)],
                                name=_p(prefix, '_layers'),
                                n_steps=nsteps)
    return rval


# ff: Feed Forward (normal neural net), only useful to put after lstm
#     before the classifier.
layers = {'lstm': (param_init_lstm, lstm_layer)}

def adam(lr, tparams, grads, x, mask, y, cost):
    """
    Adam: A Method for Stochastic Optimization (Diederik Kingma, Jimmy Ba)
    """
    gshared = [theano.shared(p.get_value() * numpy.float32(0.), name='%s_grad'%k) for k, p in tparams.iteritems()]
    gsup = [(gs, g) for gs, g in zip(gshared, grads)]
    #print '\n\ncheck\n\n'
    f_grad_shared = theano.function([x, mask, y], cost, updates=gsup, allow_input_downcast=True)

    # Magic numbers
    lr0 = 0.0002
    b1 = 0.1
    b2 = 0.001
    e = 1e-8

    updates = []

    i = theano.shared(numpy.float32(0.))
    i_t = i + 1.
    fix1 = 1. - b1**(i_t)
    fix2 = 1. - b2**(i_t)
    lr_t = lr0 * (tensor.sqrt(fix2) / fix1)

    for p, g in zip(tparams.values(), gshared):
        m = theano.shared(p.get_value() * numpy.float32(0.))
        v = theano.shared(p.get_value() * numpy.float32(0.))
        m_t = (b1 * g) + ((1. - b1) * m)
        v_t = (b2 * tensor.sqr(g)) + ((1. - b2) * v)
        g_t = m_t / (tensor.sqrt(v_t) + e)
        p_t = p - (lr_t * g_t)
        updates.append((m, m_t))
        updates.append((v, v_t))
        updates.append((p, p_t))
    updates.append((i, i_t))

    f_update = theano.function([lr], [], updates=updates, on_unused_input='ignore')

    return f_grad_shared, f_update

def sgd(lr, tparams, grads, x, mask, y, cost):
    """ Stochastic Gradient Descent
    :note: A more complicated version of sgd then needed.  This is
        done like that for adadelta and rmsprop.
    """
    # New set of shared variable that will contain the gradient
    # for a mini-batch.
    gshared = [theano.shared(p.get_value() * 0., name='%s_grad' % k)
               for k, p in tparams.items()]
    gsup = [(gs, g) for gs, g in zip(gshared, grads)]

    # Function that computes gradients for a mini-batch, but do not
    # updates the weights.
    f_grad_shared = theano.function([x, mask, y], cost, updates=gsup,
                                    name='sgd_f_grad_shared')

    pup = [(p, p - lr * g) for p, g in zip(tparams.values(), gshared)]

    # Function that updates the weights from the previously computed
    # gradient.
    f_update = theano.function([lr], [], updates=pup,
                               name='sgd_f_update')

    return f_grad_shared, f_update


def adadelta(lr, tparams, grads, x, mask, y, cost):
    """
    An adaptive learning rate optimizer
    Parameters
    ----------
    lr : Theano SharedVariable
        Initial learning rate
    tpramas: Theano SharedVariable
        Model parameters
    grads: Theano variable
        Gradients of cost w.r.t to parameres
    x: Theano variable
        Model inputs
    mask: Theano variable
        Sequence mask
    y: Theano variable
        Targets
    cost: Theano variable
        Objective fucntion to minimize
    Notes
    -----
    For more information, see [ADADELTA]_.
    .. [ADADELTA] Matthew D. Zeiler, *ADADELTA: An Adaptive Learning
       Rate Method*, arXiv:1212.5701.
    """

    zipped_grads = [theano.shared(p.get_value() * numpy_floatX(0.),
                                  name='%s_grad' % k)
                    for k, p in tparams.items()]
    running_up2 = [theano.shared(p.get_value() * numpy_floatX(0.),
                                 name='%s_rup2' % k)
                   for k, p in tparams.items()]
    running_grads2 = [theano.shared(p.get_value() * numpy_floatX(0.),
                                    name='%s_rgrad2' % k)
                      for k, p in tparams.items()]

    zgup = [(zg, g) for zg, g in zip(zipped_grads, grads)]
    rg2up = [(rg2, 0.95 * rg2 + 0.05 * (g ** 2))
             for rg2, g in zip(running_grads2, grads)]

    f_grad_shared = theano.function([x, mask, y], cost, updates=zgup + rg2up,
                                    name='adadelta_f_grad_shared')

    updir = [-tensor.sqrt(ru2 + 1e-6) / tensor.sqrt(rg2 + 1e-6) * zg
             for zg, ru2, rg2 in zip(zipped_grads,
                                     running_up2,
                                     running_grads2)]
    ru2up = [(ru2, 0.95 * ru2 + 0.05 * (ud ** 2))
             for ru2, ud in zip(running_up2, updir)]
    param_up = [(p, p + ud) for p, ud in zip(tparams.values(), updir)]

    f_update = theano.function([lr], [], updates=ru2up + param_up,
                               on_unused_input='ignore',
                               name='adadelta_f_update')

    return f_grad_shared, f_update


def rmsprop(lr, tparams, grads, x, mask, y, cost):
    """
    A variant of  SGD that scales the step size by running average of the
    recent step norms.
    Parameters
    ----------
    lr : Theano SharedVariable
        Initial learning rate
    tpramas: Theano SharedVariable
        Model parameters
    grads: Theano variable
        Gradients of cost w.r.t to parameres
    x: Theano variable
        Model inputs
    mask: Theano variable
        Sequence mask
    y: Theano variable
        Targets
    cost: Theano variable
        Objective fucntion to minimize
    Notes
    -----
    For more information, see [Hint2014]_.
    .. [Hint2014] Geoff Hinton, *Neural Networks for Machine Learning*,
       lecture 6a,
       http://cs.toronto.edu/~tijmen/csc321/slides/lecture_slides_lec6.pdf
    """

    zipped_grads = [theano.shared(p.get_value() * numpy_floatX(0.),
                                  name='%s_grad' % k)
                    for k, p in tparams.items()]
    running_grads = [theano.shared(p.get_value() * numpy_floatX(0.),
                                   name='%s_rgrad' % k)
                     for k, p in tparams.items()]
    running_grads2 = [theano.shared(p.get_value() * numpy_floatX(0.),
                                    name='%s_rgrad2' % k)
                      for k, p in tparams.items()]

    zgup = [(zg, g) for zg, g in zip(zipped_grads, grads)]
    rgup = [(rg, 0.95 * rg + 0.05 * g) for rg, g in zip(running_grads, grads)]
    rg2up = [(rg2, 0.95 * rg2 + 0.05 * (g ** 2))
             for rg2, g in zip(running_grads2, grads)]

    f_grad_shared = theano.function([x, mask, y], cost,
                                    updates=zgup + rgup + rg2up,
                                    name='rmsprop_f_grad_shared')

    updir = [theano.shared(p.get_value() * numpy_floatX(0.),
                           name='%s_updir' % k)
             for k, p in tparams.items()]
    updir_new = [(ud, 0.9 * ud - 1e-4 * zg / tensor.sqrt(rg2 - rg ** 2 + 1e-4))
                 for ud, zg, rg, rg2 in zip(updir, zipped_grads, running_grads,
                                            running_grads2)]
    param_up = [(p, p + udn[1])
                for p, udn in zip(tparams.values(), updir_new)]
    f_update = theano.function([lr], [], updates=updir_new + param_up,
                               on_unused_input='ignore',
                               name='rmsprop_f_update')

    return f_grad_shared, f_update


def build_model(tparams, options):
    trng = RandomStreams(SEED)

    # Used for dropout.
    # use_noise = theano.shared(numpy_floatX(0.))
    use_noise = theano.shared(numpy.float32(0.))
    # last_n = options['last_n']


    # x = tensor.matrix('x', dtype='int64')
    # mask = tensor.matrix('mask', dtype=config.floatX)
    # y = tensor.vector('y', dtype='int64')
    x = tensor.tensor3('x', dtype='float32')
    mask = tensor.matrix('mask', dtype='float32')
    n_timesteps = x.shape[0]
    n_samples = x.shape[1]
    ctxdim = x.shape[2]

    y = tensor.matrix('y', dtype='int64')

    ctx = x

    # emb = tparams['Wemb'][x.flatten()].reshape([n_timesteps,
    #                                             n_samples,
    #                                             options['dim_proj']])

    projs = get_layer(options['encoder'])[1](tparams, ctx, options,
                                            prefix=options['encoder'],
                                            mask=mask)
    proj = projs[0] #kong
    myproj = proj
    if options['encoder'] == 'lstm':
        proj = (proj * mask[:, :, None]).sum(axis=0)
        proj = proj / mask.sum(axis=0)[:, None]
    if options['use_dropout']:
        proj = dropout_layer(proj, use_noise, trng)

    pred = tensor.nnet.softmax(tensor.dot(proj, tparams['U']) + tparams['b'])

    f_pred_prob = theano.function([x, mask], pred, name='f_pred_prob')
    f_pred = theano.function([x, mask], pred.argmax(axis=1), name='f_pred')

    off = 1e-8
    if pred.dtype == 'float16':
        off = 1e-6

    cost = -tensor.log(pred[tensor.arange(n_samples), y] + off).mean()

    return use_noise, x, mask, y, f_pred_prob, f_pred, cost, proj,myproj # this proj is the hiddens


# def pred_probs(f_pred_prob, prepare_data, data, iterator, verbose=False):
#     """ If you want to use a trained model, this is useful to compute
#     the probabilities of new examples.
#     """
#     n_samples = len(data[0])
#     probs = numpy.zeros((n_samples, 2)).astype(config.floatX)
#
#     n_done = 0
#
#     for _, valid_index in iterator:
#         x, mask, y = prepare_data([data[0][t] for t in valid_index],
#                                   numpy.array(data[1])[valid_index],
#                                   maxlen=None)
#         pred_probs = f_pred_prob(x, mask)
#         probs[valid_index, :] = pred_probs
#
#         n_done += len(valid_index)
#         if verbose:
#             print('%d/%d samples classified' % (n_done, n_samples))
#
#     return probs
def pred_acc(modelname, batch_size, f_preds, maxlen, data_test_pb, dh_test, test_dataset_size, num_test_batches, last_n, test=True, verbose=False):
    """
    Make predictions for new data
    """
    dh_test.Reset()
    n_examples = test_dataset_size
    preds = numpy.zeros((n_examples,)).astype('float32')
    n_done = 0
    mask = numpy.ones((maxlen, batch_size)).astype('float32')

    for tbidx in xrange(num_test_batches):
        n_done += batch_size
        x, y, n_ex = dh_test.GetBatch(data_test_pb)
        if n_ex != batch_size:
            mask[:,n_ex:] = numpy.zeros((maxlen, batch_size-n_ex)).astype('float32')
        pred_ = f_preds(x,mask)
        if n_ex == batch_size:
            preds[tbidx*batch_size:tbidx*batch_size+batch_size] = pred_[:]
        else:
            preds[tbidx*batch_size:tbidx*batch_size+n_ex] = pred_[0:n_ex]
        if n_ex != batch_size:
            mask[:,n_ex:] = numpy.ones((maxlen, batch_size-n_ex)).astype('float32')

        if verbose:
            print( '%d/%d examples computed'%(n_done,n_examples))

    if test==True:
        fileprefix = 'test_results_last{}_'.format(last_n)
    else:
        fileprefix = 'train_results_last{}_'.format(last_n)
    tempfilename = fileprefix + modelname.split('/')[-1].split('.')[0] + '.txt'
    f = open(tempfilename, 'w')
    vid_idx = 0
    resultstr='{} '.format(vid_idx)
    for i in xrange(n_examples):
        if dh_test.video_ind_[dh_test.frame_indices_[i]] == vid_idx:
            resultstr=resultstr+'{},'.format(int(preds[i]))
        else:
            vid_idx = vid_idx+1
            resultstr=resultstr[:-1]+'\n'
            f.write(resultstr)
            resultstr='{} '.format(vid_idx)
            resultstr=resultstr+'{},'.format(int(preds[i]))
    resultstr=resultstr[:-1]+'\n'
    f.write(resultstr)
    f.close()

    f = open(tempfilename,'r')
    lines = f.readlines()
    f.close()

    pred  = numpy.zeros(len(lines)).astype('int64')
    for i in xrange(len(lines)):
        try:
            s=lines[i].split(' ')[1]
            s=s[0:-1]
            s=s.split(',')
            s = [int(x) for x in s]
            s = numpy.array(s)
            s = stats.mode(s)[0][0]
            pred[i] = int(s)
        except IndexError:
            print( 'One blank index skipped')
            pred[i] = -1

    f = open(data_test_pb.labels_file,'r')
    lines = f.readlines()
    f.close()
    f = open(data_test_pb.num_frames_file,'r')
    framenum = f.readlines()
    f.close()
    truth  = numpy.zeros(len(lines)).astype('int64')
    framel = numpy.zeros(len(lines)).astype('int64')
    for i in xrange(len(lines)):
        s=lines[i][0:-1]
        truth[i] = int(s)
        framel[i]= int(framenum[i][0:-1])
    return (truth==pred).mean()


def pred_error(f_pred, prepare_data, data, iterator, verbose=False):
    """
    Just compute the error
    f_pred: Theano fct computing the prediction
    prepare_data: usual prepare_data for that dataset.
    """
    valid_err = 0
    for _, valid_index in iterator:
        x, mask, y = prepare_data([data[0][t] for t in valid_index],
                                  numpy.array(data[1])[valid_index],
                                  maxlen=None)
        preds = f_pred(x, mask)
        targets = numpy.array(data[1])[valid_index]
        valid_err += (preds == targets).sum()
    valid_err = 1. - numpy_floatX(valid_err) / len(data[0])

    return valid_err


def train_lstm(
    dim_proj=1024,  # word embeding dimension and LSTM number of hidden units.
    patience=10,  # Number of epoch to wait before early stop if no progress
    max_epochs=50,  # The maximum number of epoch to run
    ydim=9,
    dispFreq=100,  # Display to stdout the training progress every N updates
    decay_c=0.,  # Weight decay for the classifier applied to the U weights.
    lrate=0.00001,  # Learning rate for sgd (not used for adadelta and rmsprop)
    #n_words=10000,  # Vocabulary size
    optimizer='rmsprop',  # sgd, adadelta and rmsprop available, sgd very hard to use, not recommanded (probably need momentum and decaying learning rate).
    encoder='lstm',  # TODO: can be removed must be lstm.
    saveto='lstm_model.npz',  # The best model will be saved there
    validFreq=1000,  # Compute the validation error after this number of update.
    saveFreq=2000,  # Save the parameters after every saveFreq updates
    maxlen=10,  # Sequence longer then this get ignored
    batch_size=32,  # The batch size during training.
    valid_batch_size=32,  # The batch size used for validation/test set.
    dataset='volley_lstm',

    # Parameter for extra option
    noise_std=0.,
    use_dropout=True,  # if False slightly faster, but worst test error
                       # This frequently need a bigger model.
    reload_model=False,  # Path to a saved model we want to start from.
    test_size=-1,  # If >0, we keep only this number of test example.
    training_stride=1,
    testing_stride=1,
    fps=30,
last_n = 10,
):

    # Model options
    model_options = locals().copy()
    print("model options", model_options)
    print('Loading data')

    data_pb = TrainProto(batch_size, maxlen, training_stride, dataset, fps)
    dh = DataHandler(data_pb)
    dataset_size = dh.GetDatasetSize()
    num_train_batches = dataset_size / batch_size
    if dataset_size % batch_size != 0:
        num_train_batches += 1

    valid = False  # not None
    test = True  # not None

    data_test_train_pb = TestTrainProto(valid_batch_size, maxlen, testing_stride, dataset, fps)
    dh_test_train = DataHandler(data_test_train_pb)
    test_train_dataset_size = dh_test_train.GetDatasetSize()
    num_test_train_batches = test_train_dataset_size / valid_batch_size
    if test_train_dataset_size % valid_batch_size != 0:
        num_test_train_batches += 1

    data_test_test_pb = TestTestProto(valid_batch_size, maxlen, testing_stride, dataset, fps)
    dh_test_test = DataHandler(data_test_test_pb)
    test_test_dataset_size = dh_test_test.GetDatasetSize()
    num_test_test_batches = test_test_dataset_size / valid_batch_size
    if test_test_dataset_size % valid_batch_size != 0:
        num_test_test_batches += 1


    # train, valid, test = load_data(n_words=n_words, valid_portion=0.05,
    #                               maxlen=maxlen)
    # if test_size > 0:
    #     # The test set is sorted by size, but we want to keep random
    #     # size example.  So we must select a random selection of the
    #     # examples.
    #     idx = numpy.arange(len(test[0]))
    #     numpy.random.shuffle(idx)
    #     idx = idx[:test_size]
    #     test = ([test[0][n] for n in idx], [test[1][n] for n in idx])
    #
    # ydim = numpy.max(train[1]) + 1

    # model_options['ydim'] = ydim

    print('Building model')
    # This create the initial parameters as numpy ndarrays.
    # Dict name (string) -> numpy ndarray
    params = init_params(model_options)

    if reload_model:
        load_params('lstm_model.npz16000.npz', params)

    # This create Theano Shared Variable from the parameters.
    # Dict name (string) -> Theano Tensor Shared Variable
    # params and tparams have different copy of the weights.
    tparams = init_tparams(params)

    # use_noise is for dropout
    (use_noise, x, mask,
     y, f_pred_prob, f_pred, cost, proj, myproj) = build_model(tparams, model_options)  #use_noise, x, mask, y, f_pred_prob, f_pred, cost, proj

    if decay_c > 0.:
        decay_c = theano.shared(numpy_floatX(decay_c), name='decay_c')
        weight_decay = 0.
        weight_decay += (tparams['U'] ** 2).sum()
        weight_decay *= decay_c
        cost += weight_decay

    f_cost = theano.function([x, mask, y], cost, name='f_cost')

    grads = tensor.grad(cost, wrt=list(tparams.values()))
    f_grad = theano.function([x, mask, y], grads, name='f_grad')

    lr = tensor.scalar(name='lr')
    f_grad_shared, f_update = eval(optimizer)(lr, tparams, grads,
                                        x, mask, y, cost)

    print('Optimization')

    # kf_valid = get_minibatches_idx(len(valid[0]), valid_batch_size)
    # kf_test = get_minibatches_idx(len(test[0]), valid_batch_size)
    #
    # print("%d train examples" % len(train[0]))
    # print("%d valid examples" % len(valid[0]))
    # print("%d test examples" % len(test[0]))
    #
    # history_errs = []
    best_p = None
    bad_count = 0
    #
    # if validFreq == -1:
    #     validFreq = len(train[0]) // batch_size
    # if saveFreq == -1:
    #     saveFreq = len(train[0]) // batch_size

    uidx = 0  # the number of update done
    estop = False  # early stop
    start_time = time.time()
    history_errs = []

    try:
        for eidx in range(max_epochs):
            mask = numpy.ones((maxlen, batch_size)).astype('float32')
            print ('Epoch ', eidx)
            n_samples = 0
            if eidx > 0:
                dh.Reset()

            # Get new shuffled index for the training set.
            # kf = get_minibatches_idx(len(train[0]), batch_size, shuffle=True)

            # for _, train_index in kf:
            for tbidx in xrange(num_train_batches):
                uidx += 1
                use_noise.set_value(1.)

                # Select the random examples for this minibatch
                # y = [train[1][t] for t in train_index]
                # x = [train[0][t]for t in train_index]

                # Get the data in numpy.ndarray format
                # This swap the axis!
                # Return something of shape (minibatch maxlen, n samples)
                # x, mask, y = prepare_data(x, y)
                x, y, n_ex = dh.GetBatch(data_pb)
                if n_ex != batch_size:
                    # mask[:, n_ex:] = numpy.zeros((maxlen, batch_size - n_ex)).astype('float32')
                    continue
                n_samples += x.shape[1]

                cost = f_grad_shared(x, mask, y)
                f_update(lrate)

                if numpy.isnan(cost) or numpy.isinf(cost):
                    print('bad cost detected: ', cost)
                    return 1., 1., 1.

                if numpy.mod(uidx, dispFreq) == 0:
                    print('Epoch ', eidx, 'Update ', uidx, 'Cost ', cost)

                if saveto and numpy.mod(uidx, saveFreq) == 0:
                    print('Saving...')

                    if best_p != None:
                        params = copy.copy(best_p)
                    else:
                        params = unzip(tparams)
                    numpy.savez(saveto + str(uidx), history_errs=history_errs, **params)

                    pkl.dump(model_options, open('%s%d.pkl' % (saveto, uidx), 'wb'))
                    print ('Done')

                if numpy.mod(uidx, validFreq) == 0:
                    use_noise.set_value(0.)
                    # train_err = pred_error(f_pred, prepare_data, train, kf)
                    # valid_err = pred_error(f_pred, prepare_data, valid,
                    #                        kf_valid)
                    # test_err = pred_error(f_pred, prepare_data, test, kf_test)

                    # history_errs.append([valid_err, test_err])

                    # if (best_p is None or
                    #     valid_err <= numpy.array(history_errs)[:,
                    #                                            0].min()):
                    #
                    #     best_p = unzip(tparams)
                    #     bad_counter = 0
                    # print('Train ', train_err, 'Valid ', valid_err, 'Test ', test_err)
                    #
                    # if (len(history_errs) > patience and
                    #             valid_err >= numpy.array(history_errs)[:-patience,
                    #                          0].min()):
                    #     bad_counter += 1
                    #     if bad_counter > patience:
                    #         print('Early Stop!')
                    #         estop = True
                    #         break
                    use_noise.set_value(0.)
                    train_err = 0
                    valid_err = 0
                    test_err = 0
                    print('Computing predictions (This will take a while. Set the verbose flag if you want to see the progress)')
                    # train_err = pred_acc(saveto, valid_batch_size, f_preds, maxlen, data_test_train_pb, dh_test_train, test_train_dataset_size, num_test_train_batches, last_n, test=False)
                    # if valid is not None:
                    # valid_err = pred_acc(saveto, valid_batch_size, f_preds, maxlen, data_test_valid_pb, dh_test_valid, test_valid_dataset_size, num_test_valid_batches, last_n, test=True)
                    if test is not None:
                        test_err = pred_acc(saveto, valid_batch_size, f_pred, maxlen, data_test_test_pb, dh_test_test,
                                            test_test_dataset_size, num_test_test_batches, last_n, test=True)

                    # history_errs.append([valid_err, test_err])
                    history_errs.append([test_err])

                    if uidx == 0 or valid_err >= numpy.array(history_errs)[:, 0].max():
                        best_p = unzip(tparams)  # p for min valid err / max valid acc

                    print('Accuracy: Train', train_err, 'Valid', valid_err, 'Test', test_err)



            print('Seen %d samples' % n_samples)

            if estop:
                break

    except KeyboardInterrupt:
        print("Training interupted")

    end_time = time.time()
    if best_p is not None:
        zipp(best_p, tparams)
    else:
        best_p = unzip(tparams)

    use_noise.set_value(0.)
    # kf_train_sorted = get_minibatches_idx(len(train[0]), batch_size)
    # train_err = pred_error(f_pred, prepare_data, train, kf_train_sorted)
    # valid_err = pred_error(f_pred, prepare_data, valid, kf_valid)
    # test_err = pred_error(f_pred, prepare_data, test, kf_test)
    #
    # print( 'Train ', train_err, 'Valid ', valid_err, 'Test ', test_err )
    # if saveto:
    #     numpy.savez(saveto, train_err=train_err,
    #                 valid_err=valid_err, test_err=test_err,
    #                 history_errs=history_errs, **best_p)
    # print('The code run for %d epochs, with %f sec/epochs' % (
    #     (eidx + 1), (end_time - start_time) / (1. * (eidx + 1))))
    # print( ('Training took %.1fs' %
    #         (end_time - start_time)), file=sys.stderr)
    # return train_err, valid_err, test_err


if __name__ == '__main__':
    # See function train for all possible parameter and there definition.
    train_lstm(
    )