#!/usr/bin/env python2

'''
Simple Parallel Concurrency script to 
get beautiful soup objects from response get
objects

READS: cPickle list of of urls 
WRITES: cPickle list of of url: cleanedtext pairs
OUT: None

'''

from bs4 import BeautifulSoup
import cPickle
import html5lib
import requests 
import pickle
from multiprocessing import Pool
from joblib import Parallel, delayed

import string
import re
import pdb

def cleantext(soup):
   
    exclude = set(string.punctuation)
    lines=[]

    for line in soup.stripped_strings:
        if (len(re.findall('\s?var\s', line))==0) & \
            (len(re.findall('[{}]', line))==0) & \
            (len(re.findall('[<>]', line))==0) & \
            (len(re.findall('[a-z]', line))>0) & \
            (len(re.findall('[^\s]=[^\s]', line))==0) & \
            (len(re.findall('^[a-zA-Z]', line))>0) & \
            (len(re.findall('[a-zA-Z]{15,}', line))==0) & \
            (len(line)>3):

            #remove nonsense characters
            line =''.join([i for i in line if (ord(i) < 128) & (ord(i) > 0)])

            # replace punctuation with space
            line = ''.join(ch if ch not in exclude else \
                ' ' for ch in line.lower() )

            #remove newlines, tabs, etc
            nums=[m.start(0) for m in re.finditer('[\\r\\n\\t]', line)]
            line=''.join([c for co,c in enumerate(line) if co not in nums])

            lines.append(line)
    return ' '.join(lines)


def f(url):
    print url
    try:
        if url[0]%2==0: 
            print url[1]

        response=requests.get(url[1], timeout=4)
        print response
	#print len(response.content)
        if len (response.content ) < 1000000:
   
	    soup=BeautifulSoup(response.content, 'html5lib')
            #print soup.text
            return cleantext(soup)
        else:
	    return ''
    except:
        return ''

if __name__=='__main__':

    with open('pkls/urls.pkl', 'rb') as p:
        urllist=cPickle.load(p)     

    print urllist
    print "you gave me ", len(urllist), 'urls and now I will get response.get objects'
   
    #texts= Parallel(n_jobs=10)(delayed(f)(i) for i in zip(urllist, range(len(urllist))))
    texts=[]


    ziptext=zip(range(len(urllist)), urllist)
    for i in range(len(urllist)):
        texts.append(f(ziptext[i]))

    print len(texts), 'texts acquired responses successfully'

    print len (texts), len(urllist)
    print texts
    #combine url with soup
    assert len(urllist)==len(texts)
    url_texts=zip(urllist, texts)

    # #writing our data now
    with open('pkls/url_text.pkl', 'wb') as p:
        cPickle.dump(url_texts, p)

    print len(texts), 'texts acquired responses \
    successfully'


    
