#  handlers/main.py  - kla app main webapp request handlers
#
#
__author__ = 'johnw'

import logging, json, html, datetime, pprint
from collections import defaultdict

from flask import (Flask, request, session, g, redirect, url_for,
     abort, render_template)

import settings

log = logging.getLogger('handlers.main')

# instantiate Flask (global) app
kla = Flask('app', **settings.FLASK.app)

def run_dev_server():
    "launch Flask dev server"
    kla.run(**settings.FLASK.run)

# -------- main request handlers --------

parser = None
nodeData = {}

@kla.route('/')
def index():
    #
    #  test parseTree
    from parser.parse import Parser
    from parser.patterns.clientLog import patternDef
    import os
    #
    global parser, nodeData
    parser = Parser(os.path.expanduser("~/Dropbox/Documents/Kontiki 2014/debugging/Lloyds/pDPTu3_AT2k1-20151125-204429/error2.log"), patternDef)
    parser.parse()
    #

    def _getLabels(child):
        return nodeData[child["id"]]["labels"]

    def _getRows(child):
        return nodeData[child["id"]]["rows"]

    def _mergeTableData(child, labelIndexes, labels, rows):
        "merges child labels & row into parent labels & rows"
        childLabels, childRows = _getLabels(child), _getRows(child)
        newLabels = set(childLabels) - set(labels)
        for l in sorted(newLabels):
            labelIndexes[l] = len(labels)
            labels.append(l)
        row = [''] * (len(labels))
        for i, l in enumerate(childLabels):
            row[labelIndexes[l]] = childRows[0][i]
        rows.append(row)

    def _renderParseTree(name, level, path):
        "recursive parseTree-to-jsTree data renderer"
        labelIndexes = dict(timestamp=0, threadID=1, threadName=2, module=3)
        children, labels, row, rows = [], ["timestamp", "threadID", "threadName", "module"], ['', '', '', ''], []
        metaPath = path.strip('.')
        if isinstance(level, dict):
            # first pass for leaf values
            rows = [row]
            keys = set(level.keys())
            for k in sorted(keys):
                v = level[k]
                if isinstance(v, (dict, list)):
                    continue
                else:
                    if k in labelIndexes:
                        row[labelIndexes[k]] = v
                    else:
                        labelIndexes[k] = len(labels)
                        labels.append(k)
                        row.append(v)
                    keys.remove(k)
            # second pass for non-leafs
            for k in sorted(keys):
                v = level[k]
                child = _renderParseTree(k, v, path + '.' + k)
                if len(row) == 0 and len(_getRows(child)) == 1:
                    # no leaf-values at this level, accumulate any immediate child singleton rows into my row vector
                    rows = []
                    _mergeTableData(child, labelIndexes, labels, rows)
                children.append(child)
            #
        else: # list
            #print("**", type(level), level)
            for j, v in enumerate(level):
                # list top-level values always dicts
                elementPath = "%s.%03d" % (path, j)
                #print(j, elementPath, v)
                label = parser.nodeMeta.get(elementPath.strip('.') + ".index", v.get('timestamp'))
                child = _renderParseTree(label, v, elementPath)
                if len(_getRows(child)) == 1:
                    # no leaf-values at this level, accumulate any immediate child singleton rows into my row vector
                    _mergeTableData(child, labelIndexes, labels, rows)
                children.append(child)

        # build node for this level
        id = "n_%d" % len(nodeData)
        node = dict(text=name, id=id)
        if children:
            node['children'] = children
        # extract line bounds from node metadata
        startLineNo, endLineNo = parser.nodeMeta.get(metaPath + ".startLineNo", 0), parser.nodeMeta.get(metaPath + "endLineNo", 0)
        # startLineNo = min(row[0] for row in rows)
        # endLineNo = max(row[1] for row in rows)
        # # strip line-no fields  todo: there has to be a cleaner way than this, make lineNo metadata some side structure perhaps??
        # labels = labels[2:]
        # rows = [row[2:] for row in rows]
        # save table data in side dict for later ajax calls
        nodeData[id] = dict(labels=labels, rows=rows, startLineNo=startLineNo, endLineNo=endLineNo)
        #
        return node

    # top-level jsTree roots are children of the top-level parseTree
    renderedTree = _renderParseTree('top', parser.parseTree, '')['children']

    return render_template("index.html",
                           renderedParseTree=json.dumps(renderedTree))

# jsTree data format
#     'core' : {
#         'data' : [
#                 {
#                   id          : "string" // will be autogenerated if omitted
#                   text        : "string" // node text
#                   icon        : "string" // string for custom
#                   state       : {
#                     opened    : boolean  // is the node open
#                     disabled  : boolean  // is the node disabled
#                     selected  : boolean  // is the node selected
#                   },
#                   children    : []  // array of strings or objects
#                   li_attr     : {}  // attributes for the generated LI node
#                   a_attr      : {}  // attributes for the generated A node
#                 }#         ]
#     }
# });

@kla.route('/node/<nodeID>')
def nodeDataTable(nodeID):
    "ajax call for table data for IDed node"
    nd = nodeData.get(nodeID, dict(labels=[], rows=[], startLineNo=0, endLineNo=0))
    return render_template("ajax/node_table.html",
                           upperFirst=lambda x: x[0].upper() + x[1:],
                           **nd)

@kla.route('/clientlog')
def getClientLog():
    "ajax: returns current client log as plain text"
    return '<pre id="log-pre" style="font-size:10px">' + html.escape(parser.log) + '</pre>'

# ELK stack experiments
from elasticsearch import Elasticsearch
es = Elasticsearch(
    [
        'http://localhost:9200/',
        #'http://tc1-elk.esjc.kontiki.com:9200/',
    ],
)

def loadELKStuff(es):
    # grab index, type & host info
    es_mapping = es.indices.get_mapping()
    indices = sorted(i for i in es_mapping.keys() if i.startswith('logstash-'))
    # as a first hack, extract available dates from index names of the form logstash-2015.01.01
    dates = [i.split('-')[-1].replace('.', '/') for i in indices]
    #
    srch = es.search(body={
        "size": 0,
        "aggs": {
            "logtypes": {
                "terms": { "field": "type.raw" }
            },
            "hosts": {
                "terms": { "field": "host.raw" }
            }
        }})

    logs = {}
    for doc in srch['aggregations']['logtypes']['buckets']:
        logs[doc['key']] = doc['doc_count']
    hosts = {}
    for doc in srch['aggregations']['hosts']['buckets']:
        hosts[doc['key']] = doc['doc_count']

    return indices, dates, sorted(logs), sorted(hosts)

indices, dates, logs, hosts = loadELKStuff(es)

@kla.route('/accesslogs')
def accessLogs():
    "server log-access page"
    return render_template("log_access.html",
                           indices=indices,
                           logs=logs,
                           hosts=hosts,
                           fromDate=dates[0],
                           toDate=dates[-1]
                           )

@kla.route('/serverlogs', methods=['POST'])
def serverLogs():
    "ajax: return selected server logs html"
    # extract log selection
    logs = request.values["logs"].split("+")
    hosts = request.values["hosts"].split("+")
    interval = int(request.values["interval"])
    date = request.values["datetime"]
    dt = datetime.timedelta(minutes=interval)
    fromDate = datetime.datetime.strptime(date, "%Y/%m/%d %H:%M")
    toDate = fromDate + dt
    # determine available log/host/timewindow crosses
    logtable = defaultdict(list)
    for log in logs:
        if log:
            srch = es.search(body={
                "size": 0,
                "aggs": {
                    "logtypes": {
                        "filter": {
                            "bool": {
                                "must": [
                                    { "term": { "type": log }},
                                    { "terms": { "host.raw": hosts }},
                                    { "range": {
                                        "@timestamp": {
                                            "gte": fromDate.isoformat(),
                                            "lt": toDate.isoformat()
                                        }
                                    }}
                                ]
                            }
                        },
                        "aggs": {
                            "hosts": {
                                "terms": { "field": "host.raw" }
                            }
                        }
                    }
                }
            })
            # pick up hosts with entries for this log in given time-window
            if srch['aggregations']['logtypes']['doc_count'] > 0:
                for doc in srch['aggregations']['logtypes']['hosts']['buckets']:
                    host = doc['key']
                    if host in hosts: # todo: wtf, terms: { "host": hosts } fails to filter on hosts list??
                        logtable[log].append(doc['key'])

    # display hosts within log-types
    return render_template("ajax/log_frames.html",
                           interval=interval,
                           fromDate=date,
                           toDate=toDate,
                           filter=request.values["filter"],
                           logs=logtable
                           )

@kla.route('/serverlog/<log>/<host>')
def getServerLog(log, host):
    "ajax: returns selected server log as plain text"
    
    interval = int(request.args.get("i"))
    date = request.args.get("d")
    dt = datetime.timedelta(minutes=interval)
    fromDate = datetime.datetime.strptime(date, "%Y/%m/%d %H:%M")
    toDate = fromDate + dt
    filter = request.args.get("f")

    filterTerms = [
        { "term": { "type" : log }},
        { "term": { "host.raw" : host }},
        { "range": {
            "@timestamp": {
                "gte": fromDate.isoformat(),
                "lt": toDate.isoformat()
            }
        }}
    ]
    if False and filter:
        filterTerms.append({ "regexp": { "log_message": filter.lower() }})
    body = {
    	"size": 100,
        "query": {
            "filtered": {
                "filter": {
                    "bool" : {
                        "must" : filterTerms
                    }
                }
            }
        },
        "sort": [ "@timestamp" ],
        "fields": [ "message" ]
    }
    #pprint.pprint(body)

    srch = es.search(body=body)

    lines = ""
    for h in srch["hits"]["hits"]:
        line = h["fields"]["message"][0]
        if not filter or filter in line:
            lines += line + '\n'

    return '<pre id="log-pre" style="font-size:10px">' + html.escape(lines) + '</pre>'

