#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Feb 13 09:13:10 2020

@author: kanchana
"""

from bs4 import BeautifulSoup
from urllib.request import Request, urlopen
import re



stack = []
for i in range(20):   # Number of pages plus one 
    i = i+1
    #print(i)
    #output = ''
    #url = "https://www.glassdoor.com/Job/data-science-jobs-SRCH_KE0,12_IP{}.htm".format(i)
    # loop through multiple pages 
    url='https://www.glassdoor.com/Job/california-data-science-jobs-SRCH_IL.0,10_IS2280_KO11,23_IP{}.htm'.format(i)
    #print(url)
    
    #request url
    req = Request(url, headers={'User-Agent': 'Mozilla/5.0'})
    
    webpage = urlopen(req).read()
    soup = BeautifulSoup(webpage,features="lxml")
    data = soup.findAll('a',attrs={'class':'jobLink jobInfoItem jobTitle'})
    for link in data:
        #print('https://www.glassdoor.com' + link.get('href'))
        stack.append('https://www.glassdoor.com' + link.get('href'))
    
fulltext = ""
count=1

for s in range(len(stack)):
    try:
        u = stack[s]
        print(count)
        count += 1
        r = Request(u, headers={'User-Agent': 'Mozilla/5.0'})
        wp = urlopen(r).read()
        soup = BeautifulSoup(wp,features="lxml")
        # to remove all styles, tags and scripts
        for script in soup(["script", "style"]):
            script.extract()
        
        text = soup.get_text()
        fulltext +=  text  
        fulltext.append('\n')
    except Exception:
        pass

   
      
text_file = open("Output_full_text.txt", "w")
text_file.write("%s" % fulltext)
text_file.close()

# remove redundant lines
  
with open("Output_full_text.txt", "r") as f:
    
    with open("Output_trim_text.txt", "w") as output:
        for line in f:
            output.write(re.sub(r'^Glassdoor.*\n?','',line, flags=re.MULTILINE))
            #output.write(re.sub(r'Glassdoor is committed.*\n?','',line, flags=re.MULTILINE))
    

