# -*- coding: utf-8 -*-
#    Copyright (c) 2017 Feng Shuo
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0

from nginx_log_parse import NginxLogParse
from config import CrawlerLimit
from collections import defaultdict
import time
import concurrent.futures
import re


class AntiCrawler(NginxLogParse):
    """
    Detect out crawler by regex according to RequestInfo value
    """
    __slots__ = ('crawler_limits',              # crawler_limits: list of CrawlerLimit, type: CrawlerLimit list
                 '_crawler_limits',             # _crawler_limits: ^^^
                 'white_list',                  # white_list: ip while list, type: list
                 '_white_list',                 # _white_list: ^^^
                 '_crawler_keywords_dict',      # crawler_keywords_dict: define crawler regex group, type: dict
                 'crawler_keywords_dict',       # _crawler_keywords_dict: ^^^
                 'blk_conf',                    # blk_conf: black ips config file, type: str
                 '_blk_conf',)                  # _blk_conf: ^^^

    def __init__(self):
        """
        Attribute described in __slots__
        """
        super(AntiCrawler, self).__init__()
        self._crawler_limits = [CrawlerLimit(-500, 150, 1800), CrawlerLimit(-1000, 300, 3600),
                                CrawlerLimit(-3000, 900, 10400)]
        self._white_list = ['54.233', '172', '192']

        # For now, we just support Path and Useragent(user browser) crawler filter
        self._crawler_keywords_dict = {
            'Path': [],
            'Useragent': [],
        }

        self._blk_conf = '1.conf'
        # self._blk_conf = '/etc/nginx/conf.d/blockips.conf'

    @property
    def crawler_limits(self):
        return self._crawler_limits

    @crawler_limits.setter
    def crawler_limits(self, new_crawler_limits=None):
        self._crawler_limits = new_crawler_limits

    @property
    def white_list(self):
        return self._white_list

    @white_list.setter
    def white_list(self, new_white_list):
        self._white_list = new_white_list

    @property
    def crawler_keywords_dict(self):
        return self._crawler_keywords_dict

    @crawler_keywords_dict.setter
    def crawler_keywords_dict(self, new_crawler_keywords_dict):
        self._crawler_keywords_dict = new_crawler_keywords_dict

    @property
    def blk_conf(self):
        return self._blk_conf

    @blk_conf.setter
    def blk_conf(self, blk_conf):
        self._blk_conf = blk_conf

    def request_time_gap(self, start_request_log=None, end_request_log=None):
        """
        time gap between first and last request in logs
        :param start_request_log:
        start request log representative start request line/row in nginx access.log
        :type start_request_log:
        ``str``
        :param end_request_log:
        end request log representative end request line/row in nginx access.log
        :type end_request_log:
        ``str``
        """
        try:
            start_time = time.strptime(self.ngx_log_to_requestinfo(start_request_log).Datetime,
                                       "%d/%b/%Y:%H:%M:%S +0000")
            end_time = time.strptime(self.ngx_log_to_requestinfo(end_request_log).Datetime, "%d/%b/%Y:%H:%M:%S +0000")
            time_gap = time.mktime(end_time) - time.mktime(start_time)
        except Exception as e:
            print "Failed get nginx time gap by logs due to %s" % e
        else:
            return int(time_gap)

    def crawlers_re(self):
        """
        Define the crawler regex list pattern by keywords in nginx access.log
        :return filter_dict: generate by crawler_keywords_dict user provided
        :rtype filter_dict: ``defaultdict``
        """
        filter_dict = defaultdict(set)
        for key, crawler_keyword_list in self.crawler_keywords_dict.iteritems():
            for crawler_keyword in crawler_keyword_list:
                filter_dict[key].add(r'(.*)?%s(.*)?' % crawler_keyword)
        return filter_dict

    def request_info_filter(self, request_info=None, filter_dict=None, filter_option=None):
        """
        Request info parameter patterns match crawler keywords regex provided by filter_option
        :param request_info:
        :type  request_info: ``namedtupe RequestInfo``
        :param filter_dict: filter_dict generated by self.crawlers_re()
        :type  filter_dict: ``defaultdict``
        :param filter_option:
        filter option:
            1. None: filter request info with Path, Useragent
            2. ['Path' or 'Useragent']: filter request info based on either one keyword in this list
        :type   filter_option: ``list``
        :return request_info.IP
        :rtype ``str``
        """
        ret = 0
        try:
            if filter_dict is not None and request_info is not None:
                if filter_option is None:
                    filter_option = self.crawler_keywords_dict.keys()
                for key in filter_option:
                    if key == 'Path':
                        filter_string = request_info.Path
                    else:
                        filter_string = request_info.Useragent
                    for re_value in filter_dict[key]:
                        if re.match(re_value, filter_string):
                            ret += 1
                if ret == len(filter_option):
                    return request_info.IP
                else:
                    return None
        except Exception as e:
            print "Failed to filter ip due to %s" % e

    def logs_filter_by_crawler_limit(self, crawler_limit=None, filter_option=None):
        """
        Filter logs with crawler_limit, firstly, logs should transfer into RequestInfo type, then use
        request_info_filter to filter crawrler ip by filter_dict which generate by self.crawlers_re().
        :param crawler_limit:
        :type  crawler_limit: ``CrawlerLimit``
        :param filter_option:
        filter option:
            1. None: filter request info with Path, Useragent
            2. ['Path' or 'Useragent']: filter request info based on either one keyword in this list
        :type filter_option: ``list``
        :return ip_list:
        :rtype  ip_list: ``list``
        """
        ip_list = []
        try:
            if isinstance(crawler_limit, CrawlerLimit):
                ip_count_dict = {}
                logs = self.get_ngx_logs(crawler_limit.msgs_limit)
                start_request_log, end_request_log = logs[0], logs[-1]
                time_gap = self.request_time_gap(start_request_log, end_request_log)

                if time_gap < crawler_limit.tm_gap_limit:
                    for log in logs:
                        request_info = self.ngx_log_to_requestinfo(log)
                        crawler_ip = self.request_info_filter(request_info, self.crawlers_re(), filter_option)
                        if crawler_ip is not None:
                            if not ip_count_dict.has_key(crawler_ip):
                                ip_count_dict[crawler_ip] = 0
                            ip_count_dict[crawler_ip] += 1
                    for ip, count in ip_count_dict.iteritems():
                        if count > crawler_limit.rate_limit:
                            ip_list.append(ip)
                    return ip_list
                else:
                    print "Time gap for request is larger than limit, then quit!"
        except Exception as e:
            print "Failed filter log due to %s" % e

    def con_logs_filter_by_crawler_limit(self, crawler_limits=None, filter_option=None):
        """
        Concurrent run filter crawler ip by crawler_limits
        :param crawler_limits:
        :type crawler_limits: ``CrawlerLimit list``
        :param filter_option:
        described aboved
        :type filter_option:
        described aboved
        :return list(set(raw_crawler_ip_list))
        :rtype ``list``
        """
        # this crawler_ip_list should delete ip in white_list and blackips.conf

        raw_crawler_ip_list = []
        try:
            with concurrent.futures.ThreadPoolExecutor(max_workers=len(self.crawler_limits)) as executor:
                rts = [executor.submit(self.logs_filter_by_crawler_limit, crawler_limit, filter_option) for crawler_limit
                       in crawler_limits]
                for rt in concurrent.futures.as_completed(rts):
                    raw_crawler_ip_list += rt.result()
            return list(set(raw_crawler_ip_list))
        except Exception as e:
            print "Failed to get raw_crawler_ip_list by concurrent run function logs_filter_by_crawler_limit " \
                  "due to %s" % e

    def crawler_ip_list(self, ip_list=None):
        """
        Removed ip already in blackips.conf and white_list
        :param ip_list: ip_set generate by con_logs_filter_by_crawler_limit/
        :type  ip_list: ``list``
        :return crawler_ip_list:
        :rtype  crawler_ip_list: ``list``
        """
        ip_black_list = []
        try:
            with open(self.blk_conf, 'r+') as fd:
                for line in fd.readlines():
                    ip_black_list += [ip for ip in ip_list if re.match(r"^deny\s%s;\n$" % ip, line)]
                crawler_ip_list = [ip for ip in ip_list if ip not in ip_black_list + self.white_list]
            return crawler_ip_list
        except Exception as e:
            print "Failed get crawler_ip_list due to %s" % e

    def black_ip_conf(self, black_ip_list=None):
        try:
            if black_ip_list is not None:
                with open(self.blk_conf, 'a+') as fd:
                    black_ip_confs = ["deny %s;\n" % ip for ip in black_ip_list]
                    for new_ip_blk_conf in black_ip_confs:
                        fd.write(new_ip_blk_conf)
        except Exception as e:
            print "Failed add black ip conf to %s due to %s" % (self.blk_conf, e)


# TODO should move to test
# from config import CrawlerLimit
# from anti_crawler import AntiCrawler
# a = AntiCrawler()
# a.ngx_log = '/Users/leo/web1-nginx-log/access.log-20170101'
# logs = a.get_ngx_logs(100)
# req_info = a.ngx_log_to_requestinfo(logs[0])
# a.crawlers_re()
# a.crawler_keywords_dict['Path'] = ['/airplanes/4349566766/']
# a.crawler_keywords_dict['Useragent'] = ['Mozilla/5.0']
# a.request_info_filter(req_info, a.crawlers_re())
# a.request_info_filter(req_info, a.crawlers_re(), ['Useragent'])
# a.logs_filter_by_crawler_limit(CrawlerLimit(-500, 150, 1800))
# a.con_logs_filter_by_crawler_limit(a.crawler_limits)
# a.blk_conf = '/Users/leo/web1-nginx-log/1.conf'
# a.black_ip_conf(['1.1.1.1'])
# ret = 0
# if a.crawlers_re() is not None and req_info is not None:
#   for key in ['Path']:
#       if key == 'Path':
#            filter_string = req_info.Path
#       else:
#           filter_string = request_info.Useragent
#       print filter_string
#       for re_value in a.crawlers_re()[key]:
#           if re.match(re_value, filter_string):
#               ret += 1
#       print ret