import sqlite3
import pickle
import cPickle
import subprocess
from sklearn.feature_extraction.text import TfidfVectorizer
import os 

# my imports
import table_funs as tf
from bhistory import ChromeHistory
from clustering import Clustering

def to_strings(ser):
    '''
    
    INPUT: pandas series object ,
    OUTPUT: list of strings


    '''
    ls_strs=[]
    for ser_entry in ser.values:

	    if type(ser_entry) == str or type(ser_entry) == unicode:
	    	ser_entry=''.join([i for i in ser_entry if (ord(i) < 127) & (ord(i) > 0)])
	    	ls_strs.append(ser_entry)

	    else:
	    	try:
	    		ls_strs.append(str(ser_entry))
	    	except:
	    		ls_strs.append('')

    return ls_strs

def get_string_for_file(path):
    with open(path) as f:
        return f.read()
    raise ValueError('Unable to parse file')


def backend(path_to_chrome):

    '''

    INPUT: chrome DB path 
    OUTPUT: rendered html content with results 
    of KMeans clustering

    Will move through the stages of the data pipline as part of 
    flask application back end.

    This function is called by flask_app.py after user has uploaded
    data and clicked Submit. 

    1) to webscrape, 2) parse text, 3) run tdidf, 4) run Kmeans, 
	5) prepare results, 6) create html file for display. 

    '''

    #create tables if necessary (only necessary when first user
    # comes to site)
    if not os.path.exists(os.path.join(os.getcwd(), 'sqlite', 'DATA.DB')):
        try: 
            os.mkdir('sqlite')
        except OSError as e:
            print e 
        print 'you should '
        tf.data()
        tf.urls()

    #get finished pandas dataframe from chrome DB
    print 'the found path to chrome is', path_to_chrome
    user_df=ChromeHistory(path_to_chrome).finallist
    print 'We found ', user_df.shape[0], ' rows'

    #add all websites for this user to sqlite data table
    tf.insert_userdf(user_df)

    #subset number of rows to limit webscrape time
    user_df=user_df.ix[0:250,:]

    #create a list of URLs visited for this user: Save it
    urls=to_strings(user_df['url'])
    with open('pkls/urls.pkl', 'wb') as pi: 
        pickle.dump(urls, pi)

    #create variable titles to use later for clustering output
    titles=to_strings(user_df['title'])
    

    #note; ec2 doesn't like request.get objects being pickled so we are
    #using this updated script instead.
    #url --> url - parsed text  - dict
    os.system('python2 url_urltext.py')

    #lets load the url: parsed text tuple pair list we just created
    #into current name space
    with open('pkls/url_text.pkl', 'rb') as p:
        urlstexts=cPickle.load(p)

    #now that we have url_txt back in name space, lets 
    #put it in sqlite (keeping urls unique)
    tf.addurl(urlstexts)

    #create list of texts
    urlstexts=[item[1] for item in urlstexts]
    print len(urlstexts)
    print len(titles)
    assert len(urlstexts)==len(titles)

    #build tdidf model with texts
    tdidf=TfidfVectorizer(stop_words='english', \
    strip_accents ="ascii", max_features=2000).fit(urlstexts)
    print 'tdidf'

    #for clusters 3,5,7, build model, and write a lit of cluster output
    #files (one file for each cluster, varying number of clusters)
    for n in [6,12,18]:
        cl=Clustering(tdidf, urls, titles, n)
        cl.run_clustering()
        cl.url_sample(n_per_cl=20)

    #load result output
    template_html=open('templates/result_screen.html').read()

    #combine with clustering results
    template_html=template_html.encode('ascii') % (
        open('cluster_output/clusteroutput'+str(6)+'.txt').read(),
        open('cluster_output/clusteroutput'+str(12)+'.txt').read(),  
        open('cluster_output/clusteroutput'+str(18)+'.txt').read()
        )

    #return html content to flask page
    return template_html
