from bs4 import BeautifulSoup
import requests
import pandas as pd

from threading import Thread
from multiprocessing import Queue Process
import functools
import time
import itertools
from datetime import datetime


def timer(func):
    @functools.wraps(func)
    def wrapper(*args, **kwargs):
        start = time.perf_counter()
        value = func(*args, **kwargs)
        end = time.perf_counter()
        duration = end - start
        print("{} completed in {:.4f} seconds".format(func.__name__, duration))
        return value
    return wrapper


class JobsAPI:
    
    JOB_COUNT = 0
    RESULTS_PER_PAGE = 25
    CSV_FILE = "{}_jobs_{}.csv"
    XLSX_FILE = "{}_jobs_{}.xlsx"
    TIME_FORMAT = "%d-%m-%Y-%H-%M-%S"

    JOBSERVE_BASE_URL = 'https://www.jobserve.com'
    JOBSERVE_RESULTS_PER_PAGE = 20
    JOBSERVE_URL = "https://www.jobserve.com/gb/en/JobListingBasic.aspx?shid=70C465132748773534BA"
    JOBSERVE_DYNAMIC_URL = JOBSERVE_URL + "&page={}"

    INDEED_BASE_URL = 'https://www.indeed.co.uk'
    INDEED_DYNAMIC_URL = INDEED_BASE_URL + "/jobs?as_and={}&as_phr=&as_any=&as_not=&as_ttl=&as_cmp=&jt=all&st=&as_src" \
                                           "=&salary=&radius=25&l=&fromage=3&limit=1000&sort=date&psf=advsrch&from" \
                                           "=advancedsearch"

    REED_BASE_URL = "https://www.reed.co.uk"
    REED_URL_PAGES = REED_BASE_URL + "/jobs/{}-jobs?salaryfrom={}&datecreatedoffset=LastThreeDays"
    REED_DYNAMIC_URL = REED_BASE_URL + "/jobs/{}-jobs?pageno={}&salaryfrom={}&datecreatedoffset=LastThreeDays"

    TOTAL_JOBS_BASE_URL = "https://www.totaljobs.com"
    TOTAL_JOBS_URL_PAGES = TOTAL_JOBS_BASE_URL + "/jobs/{}?postedwithin=3&salary={}&salarytypeid=1"
    TOTAL_JOBS_DYNAMIC_URL = TOTAL_JOBS_URL_PAGES + "&page={}"
    TOTAL_JOBS_RESULTS_PER_PAGE = 20
    HEADERS = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) '
                             'Chrome/79.0.3945.117 Safari/537.36'}

    @timer
    def __init__(self, keyword, starting_salary=45000):
        self.reed_results_queue = Queue()
        self.indeed_results_queue = Queue()
        self.total_jobs_results_queue = Queue()
        # self.jobserve_results_queue = Queue()
        self.keyword = keyword.lower()
        if len(self.keyword.split()) > 1:
            self.keyword = '+'.join(self.keyword.split())
            self.search_str = keyword.lower()
        else:
            self.search_str = self.keyword
        self.starting_salary = starting_salary
        self.df = self.initialise_pandas_df()

        self.reed_job_links = self.initialise_reed_links()
        self.indeed_job_links = self.initialise_indeed_links()
        self.total_jobs_links = self.initialise_total_jobs_links()
        # self.jobserve_job_links = self.initialise_jobserve_links()

        self.total_links = len(self.reed_job_links) + len(self.indeed_job_links) \
                           + len(self.total_jobs_links)  # + len(self.jobserve_job_links)

        indeed = Process(target=self.get_indeed_jobs, args=())
        reed = Process(target=self.get_reed_jobs, args=())
        total_jobs = Process(target=self.get_total_jobs, args=())

        indeed.start()
        reed.start()
        total_jobs.start()
        # jobserve_jobs.start()

        self.reed_job_links = self.reed_results_queue.get()
        self.indeed_job_links = self.indeed_results_queue.get()
        self.total_jobs_links = self.total_jobs_results_queue.get()
        # self.jobserve_job_links = self.jobserve_results_queue.get()

        frames = [self.reed_job_links, self.indeed_job_links, self.total_jobs_links, ]  # self.jobserve_job_links ]
        self.final_results = pd.concat(frames)

        indeed.join()
        reed.join()
        total_jobs.join()
        # jobserve_jobs.join()

        self.write_to_disk()
        self.get_job_counts()

    @timer
    def write_to_disk(self):
        self.final_results.to_excel(self.XLSX_FILE.format(self.keyword, datetime.utcnow().strftime(self.TIME_FORMAT)),
                                    index=False)
        self.final_results.to_csv(self.CSV_FILE.format(self.keyword, datetime.utcnow().strftime(self.TIME_FORMAT)),
                                  index=False, encoding='utf-8')

    @timer
    def get_job_counts(self):
        print("Total jobs found: {}; Total relevant jobs: {}".format(self.total_links, self.JOB_COUNT))

    @staticmethod
    def initialise_pandas_df():
        columns = ['Title', 'Salary', 'Location', 'Url', 'Advertiser', 'Type', ]
        df = pd.DataFrame(columns=columns)
        return df

    def append_to_pandas_df(self, title, salary, location, job_link, advertiser, job_type, description):
        if self.search_str in str(title).lower() or self.search_str in str(description).lower():
            self.JOB_COUNT += 1
            self.df = self.df.append({'Title': title, 'Salary': salary, 'Location': location,
                                      'Url': job_link, 'Advertiser': advertiser, 'Type': job_type},
                                     ignore_index=True)

    def get_pagination_url(self, url_pagination):
        return url_pagination.format(self.keyword, self.starting_salary)

    def make_soup(self, url_link):
        return BeautifulSoup(requests.get(url_link, headers=self.HEADERS).text, 'lxml')

    @timer
    def get_jobserve_page_count(self):
        jobserve_soup = self.make_soup(self.JOBSERVE_URL)
        results_count = int(jobserve_soup.find('span', attrs={'class': 'resultnumber'}).text)
        if results_count:
            if results_count < self.JOBSERVE_RESULTS_PER_PAGE:
                return int(results_count)
            else:
                return int(round(results_count / self.JOBSERVE_RESULTS_PER_PAGE))

    @timer
    def get_reed_page_count(self):
        reed_soup = self.make_soup(self.get_pagination_url(self.REED_URL_PAGES))
        results_count = int(reed_soup.find('span', attrs={'class': 'count'}).text.strip())
        if results_count:
            if results_count < self.RESULTS_PER_PAGE:
                return int(results_count)
            else:
                return int(round(results_count / self.RESULTS_PER_PAGE))

    @timer
    def get_total_jobs_page_count(self):
        total_jobs_pagination_url = self.get_pagination_url(self.TOTAL_JOBS_URL_PAGES)
        total_jobs_soup = self.make_soup(total_jobs_pagination_url)
        results_count = int(total_jobs_soup.find('div', attrs={'class': 'page-title'}).span.text.strip())
        if results_count:
            if results_count < self.TOTAL_JOBS_RESULTS_PER_PAGE:
                return int(results_count)
            else:
                return int(round(results_count / self.TOTAL_JOBS_RESULTS_PER_PAGE))

    @timer
    def initialise_indeed_links(self):
        url = self.INDEED_DYNAMIC_URL.format(self.keyword)
        soup = self.make_soup(url)
        hrefs = soup.find_all('div', attrs={'class': 'title'})
        result = [self.INDEED_BASE_URL + div.a['href'] for div in hrefs]  # or div.find('a')['href']
        # self.indeed_results_queue.put(result)
        return result

    @timer
    def initialise_jobserve_links(self):
        urls = (self.JOBSERVE_DYNAMIC_URL.format(page) for page in range(1, self.get_jobserve_page_count() + 1))
        soups = (self.make_soup(url) for url in urls)
        hrefs = (soup.find_all('div', attrs={'class': 'jobListHeaderPanel'}) for soup in soups)
        flat_hrefs_list = list(itertools.chain.from_iterable(hrefs))
        result = [self.JOBSERVE_BASE_URL + div.a['href'] for div in flat_hrefs_list]
        return result

    @timer
    def initialise_reed_links(self):
        urls = (self.REED_DYNAMIC_URL.format(self.keyword, page, self.starting_salary)
                for page in range(1, self.get_reed_page_count() + 1))
        soups = (self.make_soup(url) for url in urls)
        hrefs = (soup.find_all('h3', attrs={'class': 'title'}) for soup in soups)
        flat_hrefs_list = list(itertools.chain.from_iterable(hrefs))
        result = [self.REED_BASE_URL + div.a['href'] for div in flat_hrefs_list]
        return result

    @timer
    def initialise_total_jobs_links(self):
        urls = (self.TOTAL_JOBS_DYNAMIC_URL.format(self.keyword, self.starting_salary, page_no)
                for page_no in range(1, self.get_total_jobs_page_count() + 1))
        total_soups = (self.make_soup(link) for link in urls)
        hrefs = (total_soup.find_all('div', {'class': 'job-title'}) for total_soup in total_soups)
        flat_hrefs_list = list(itertools.chain.from_iterable(hrefs))
        result = [div.a['href'] for div in flat_hrefs_list]
        return result

    @timer
    def get_indeed_jobs(self):
        for job_link in self.indeed_job_links:
            location, job_type, salary, advertiser = (None,) * 4
            soup = self.make_soup(job_link)
            description = soup.find('div', {'id': "jobDescriptionText", 'class': "jobsearch-jobDescriptionText"}).text
            title = soup.find('div', {'class': 'jobsearch-JobInfoHeader-title-container'}).text

            location_type_salary_container = \
                soup.find_all('span', {'class': 'jobsearch-JobMetadataHeader-iconLabel'})
            container_is_not_empty = location_type_salary_container is not None
            container_len = len(location_type_salary_container)
            location = location_type_salary_container[0].text \
                if container_is_not_empty and container_len >= 1 else None
            if container_is_not_empty and container_len >= 2:
                is_salary = '£' in location_type_salary_container[1].text
                job_type = location_type_salary_container[1].text if not is_salary else None
                if container_len >= 3:
                    salary = location_type_salary_container[2].text if len(
                        location_type_salary_container) >= 2 else None
            advertiser = soup.find('div', {'class': "icl-u-lg-mr--sm icl-u-xs-mr--xs"}).text
            self.append_to_pandas_df(title, salary, location, job_link, advertiser, job_type, description)
        self.indeed_results_queue.put(self.df)

    @timer
    def get_reed_jobs(self):
        for job_link in self.reed_job_links:
            soup = self.make_soup(job_link)
            description = soup.find('span', {'itemprop': "description"}).text
            title = soup.find('h1').text
            salary_tag = soup.find('span', {'data-qa': "salaryLbl"})
            advertiser_tag = soup.find('span', {'itemprop': "name"})
            location_tag = soup.find('span', {'itemprop': "addressLocality"})
            job_type_tag = soup.find('span', {'itemprop': "employmentType"})

            salary = soup.find('span', {'data-qa': "salaryLbl"}).text if salary_tag is not None else None
            advertiser = soup.find('span', {'itemprop': "name"}).text if advertiser_tag is not None else None
            location = soup.find('span', {'itemprop': "addressLocality"}).text if location_tag is not None else None
            job_type = soup.find('span', {'itemprop': "employmentType"}).text if job_type_tag is not None else None
            self.append_to_pandas_df(title, salary, location, job_link, advertiser, job_type, description)
        self.reed_results_queue.put(self.df)

    @timer
    def get_total_jobs(self):
        for job_link in self.total_jobs_links:
            total_soup = self.make_soup(job_link)
            description = total_soup.find('div', {'class': "job-description"}).text
            title = total_soup.find('h1', {'class': 'brand-font'}).text.strip()
            salary_tag = total_soup.find('li', {'class': "salary icon"})
            advertiser_tag = total_soup.find('a', {'id': "companyJobsLink"})
            location_tag_type_1 = total_soup.find('li', {'class': "location icon"})
            location_tag_type_2 = total_soup.find('div', {'class': "col-xs-12 col-sm-7 travelTime-locationText"})
            location1 = location_tag_type_1.div.text if location_tag_type_1 is not None else None
            location2 = location_tag_type_2.ul.text if location_tag_type_2 is not None else None
            location_options = [location1, location2]
            job_type_tag = total_soup.find('li', {'class': "job-type icon"})

            salary = salary_tag.text.strip() if salary_tag is not None else None
            advertiser = advertiser_tag.text.strip() if advertiser_tag is not None else None
            location = next(item for item in location_options if item is not None)
            job_type = job_type_tag.div.text if job_type_tag is not None else None
            self.append_to_pandas_df(title, salary, location, job_link, advertiser, job_type, description)
        self.total_jobs_results_queue.put(self.df)

    @timer
    def get_jobserve_jobs(self):
        for job_link in self.jobserve_job_links:
            soup = self.make_soup(job_link)
            description = soup.find('div', {'class': "md_skills"}).text
            title = soup.find('h1', {'id': 'positiontitle'}).text
            salary_tag = soup.find('span', {'id': "md_rate"})
            advertiser_tag = soup.find('div', {'id': "recruitername"}).span
            location_tag = soup.find('span', {'id': "md_location"})
            job_type_tag = soup.find('span', {'id': "td_job_type"})

            salary = salary_tag.text if salary_tag is not None else None
            advertiser = advertiser_tag.text if advertiser_tag is not None else None
            location = location_tag.text if location_tag is not None else None
            job_type = job_type_tag.text if job_type_tag is not None else None
            self.append_to_pandas_df(title, salary, location, job_link, advertiser, job_type, description)
        self.jobserve_results_queue.put(self.df)


if __name__ == "__main__":
    JobsAPI(keyword='python', starting_salary=70000)
