from __future__ import division
import os
from nltk import word_tokenize
from nltk import sent_tokenize
from nltk import FreqDist
import math
import re
import collections
import codecs
from bs4 import BeautifulSoup
from nltk.tag.stanford import StanfordNERTagger
#import nltk.tag.stanford as st
from split_smartly import textsplit, isdate, islocation
import os
from unidecode import unidecode


import numpy as np
import csv
from collections import Counter
import sys
csv.field_size_limit(sys.maxsize)


############   Set variables to local machine   #############
inputfile = '/Users/colin/Documents/ResumeParsing/Resume-Parsing/Current/HeaderSplitData.csv'
outputfile = '/Users/colin/Documents/ResumeParsing/Resume-Parsing/Data/Experience.csv'
dataloc = '/Users/colin/Documents/ResumeParsing/Resume-Parsing/Data/OutsideSources/'

positlist = dataloc + "position_list.txt"
statelist = dataloc + "states.csv"
#############################################################

months = "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"
seasons = "Spring", "Summer", "Fall", "Autumn", "Winter"

def isdatehead(txt):
    mos = sum([i in txt for i in months])
    years = len(re.findall(re.compile('20..'), txt))
    seas = sum([i in txt for i in seasons])
    presents = len(re.findall('Present|present', txt))
    #return (ismonth or isyear or isseason or ispresent)
    return (mos + years + seas + presents)/len(txt.split(' ')) > 0.5




def extract_exp(soup): #Handles experience section of ONE resume
    arr_lines = soup.findAll("text")

    #Create array of array of different work experience
    final_arr = []; bullet_arr = []
    temp_str = ""; bulletstr = ""
    contig=False
    
    for lines in arr_lines:
        # print str(lines)
        if '<b>' in str(lines) or '<i>' in str(lines) or \
        ((isdate(str(lines)) or any([islocation(x) for x in str(lines).split(" ")])) and contig):
            temp_str = temp_str + lines.prettify() + "\n"
            contig=True
            if bulletstr:
                bullet_arr.append(bulletstr)
                bulletstr = ""
        else:
            bulletstr = bulletstr + lines.prettify() + "\n"
            contig=False
            if temp_str:
                final_arr.append(temp_str)
                temp_str = ""

    return (final_arr, bullet_arr)

def extract_info(new_val,inp_arr):#Main logic to extract different elements given an array of mixed items of experience
    inp_arr = [x for x in inp_arr if x!='']
    items = len(inp_arr)
    done = []

    #Find position. If found, deleting it from the array
    #Read glassdoor position list
    pos = open(positlist,"r").read().split("\n")
    pos_list = []
    for x in pos:
        arr = x.split(" ")
        [pos_list.append(b) for b in arr]

    pos_list = set(pos_list)
    ind = 0;
    flag = 0;
    for item in inp_arr:
        words = item.split(" ")
        if any([word for word in words if word in pos]):
            new_val["position"] = item
            # inp_arr.remove(item)
            done.append(ind)
            flag =1
            break;
        if (flag ==1):
            break
        ind +=1

    #Find date. If found, deleting it from the array
    ind = 0;
    for item in inp_arr:
        date = re.findall(r'(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec|January|February|March|April|June|July|August|September|October|November|December|Sept)\s|\b(19|20)\d{2}\b', item)
        if(date):
            new_val["date"] = item
            # inp_arr.remove(item)
            done.append(ind)
            break;
        ind +=1

    #Finding location. If found, deleting it from the array
    
    #Read states
#    f = open(statelist).read()
#    lines = f.split("\n")
#    states = []

#    for line in lines:
#        pair = line.split(",")
#        states.append(pair[0])
#        states.append(pair[1])

    
    flag = 0
    ind = 0;
    for item in inp_arr:
        if any([islocation(x) for x in str(item).split(" ")]):
                new_val["location"] = item
                done.append(ind)
                flag =1
                break;
        #for st in states:
        #    if(len(st) ==2): #If abbreviation
        #        if " "+st in item or "," +st in item or ", "+st in item:
        #            new_val["location"] = item
        #            # inp_arr.remove(item)
        #            done.append(ind)
        #            flag =1
        #            break;
        #    else:
        #        if st in item:
        #            new_val["location"] = item
        #            # inp_arr.remove(item)
        #            done.append(ind)
        #            flag =1
        #            break;
        if flag ==1:
            break
        ind +=1
        
    # Identify company names
    prob_company = "LLC|L.L.C|Industries|, Inc|,Inc| INC|Network|Store|University|Institute|Services|Campaign|Partners|Firm|Academy|Agency|Tavern|Restaurant|Consultants|Associates|Club|Bank |&amp|Department|Office | and |Magazine|Realtors|School|Factory|Organization|Committee"
    for item in inp_arr:    
        comp = re.findall(prob_company, item)
        if(comp):
            new_val["company"] = item
            break;
    
    if (items - len(set(done)) == 1): #Only one element is unmarked
        for i in range(0,items):
            if i not in done:
                new_val["company"] = inp_arr[i]
                done.append(i)
                break



def arrange_exp_data(atuple, numbullets):
    filename = atuple[0]
    titleinfo = atuple[1]
    bullets = ['<li>' + bul + '</li>' for bul in atuple[2]]
    
    # Create a separate dictionary for bullet points. Must all be the same length (pad with empty '')
    bvarnames = ['bullet' + str(x) for x in range(1,numbullets+1)]
    vals = bullets + [''] * (numbullets - len(bullets))
    bdict = {key: val for key,val in zip(bvarnames, vals)}
    
    new_val = {"filename":filename, "company" : "UNK", "location" : "UNK", "position" : "UNK", "date" : "UNK", 
               "all" : str(titleinfo), "bullets": str(bullets)}  
    
    new_val.update(bdict)
    
    extract_info(new_val, titleinfo)
    return(new_val)
    
    

def clean_and_arrange(arr):
    ret_resume = []
    for resume in arr:
        cleaned_entry = []
        for exp_str in resume:
            b = re.split("\n|\s\s\s",exp_str) #split in three or more space or \n
            pure_b = []
            for lines in b:

                if "<b>" not in lines and "</b>" not in  lines and "<text" not in  lines and "<i>" not in  lines and "</i>" not in  lines and "/text>" not in  lines:
                    if(lines) and len(unidecode(lines).strip()) > 1: #Dont enter empty lines and single character lines
                        #Here calling the function written by Colin
                        print(lines)
                        splitted_arr = textsplit(lines)
                        for x in splitted_arr:
                            # pure_b.append(lines.encode("utf-8").strip(" ,.-!"))
                            pure_b.append(unidecode(x).strip(" ,.-!"))
            cleaned_entry.append(pure_b)
        ret_resume.append(cleaned_entry)
    return ret_resume

### Code starts from here ###
with open(inputfile, 'rbU') as f:
    reader = csv.reader(f)
    data = [row for row in reader]
datanp = np.array(data)

expsoups = [BeautifulSoup(row) for row in datanp[:,4]]
allexpdata = np.array([extract_exp(soup) for soup in expsoups])
exptitles = [x[0] for x in allexpdata]
expbullets = [x[1] for x in allexpdata]
filename = [row for row in datanp[:,0]]

bulltext = []
for ares in expbullets:
    res = []
    restext = [BeautifulSoup(x).findAll("text") for x in ares]
    for t in restext:
        temp = filter(None, re.split(u"\u2022|\uf0b7|\uf097", ''.join([x.text.replace('\n', '') for x in t])))
        res.append([unidecode(x).strip() for x in temp if len(unidecode(x).strip()) > 1]) 
        
    bulltext.append(res)
maxbull = len(max(bulltext,key=len))

exp_array = clean_and_arrange(exptitles) 
    
# Create an array of tuples: (filename, work header array, bullet array) for each job
# Multiple lines for each resume if it has multiple jobs
flattened = []
for f, exparr, bull in zip(filename, exp_array, bulltext):
    flattened.append([(f, x, b) for x,b in zip(exparr, bull)])
flattened = [item for sublist in flattened for item in sublist]

fin_arr = [arrange_exp_data(flat, maxbull) for flat in flattened]


finalkeys = ["filename", "company", "location", "position", "date", 
               "all", "bullets", "bullet1", "bullet2", "bullet3"]
csvarray = [{key:value for key,value in x.items() if key in finalkeys} for x in fin_arr]
    
#keys = fin_arr[0].keys()
with open(outputfile, 'wb') as csvfile:
    dict_writer = csv.DictWriter(csvfile, finalkeys)
    dict_writer.writeheader()
    dict_writer.writerows(csvarray)

