#!/usr/bin/python
# -*- coding: utf-8 -*-

import socket
import re
from bs4 import BeautifulSoup
import requests
import ipwhois
from pprint import pprint


def get_url(url):
    try:
        res = requests.get(url)
    except requests.exceptions.ConnectionError:
        raise requests.exceptions.ConnectionError("DNS lookup failures")
    else:
        if res.status_code != 200:
            raise requests.exceptions.ConnectionError(
                "the {}, answer with {} error".format(url, res.status_code))

        return res


def get_ip(name):
    attempts = 5
    ip = "undefined"
    while attempts:
        try:
            data = socket.gethostbyname_ex(name)
            ip = data[2][0]
            break
        except (socket.herror, socket.gaierror):
            attempts -= 1

    return ip


def get_who_is_and_country(ip):
    try:
        ip_obj = ipwhois.IPWhois(ip)
        who_is = ip_obj.lookup(retry_count=5)
        return str(who_is), who_is['asn_country_code']
    except ipwhois.exceptions.IPDefinedError:
        return "Private-Use Networks", "undefined"
    except ipwhois.exceptions.WhoisLookupError:
        return "undefined", "undefined"


def gather():
    base_url = "http://mirror1.malwaredomains.com/files/"

    res = get_url(base_url)
    soup = BeautifulSoup(res.content, "lxml")
    for link in soup.findAll("a"):
        date = re.findall(r'^[0-9]{4}[0-9]{2}[0-9]{2}\.txt', link.text)
        if not date:
            continue

        url = "{}{}".format(base_url, date[0])
        res = get_url(url)
        for line in res.iter_lines():
            line = line.split("\t")
            if len(line) < 3:
                break
            elif len(line) == 4:
                attack_type = 'undefined'
            else:
                attack_type = line[3]

            host = line[2]
            ip_address = get_ip(host)
            if ip_address == "undefined":
                who_is, country = "undefined", "undefined"
            else:
                who_is, country = get_who_is_and_country(ip_address)

            doc = {
                'IP': ip_address,
                'SourceInfo': url,
                'Type': attack_type,
                'Country': country,
                'Domain': host,
                'URL': host,
                'WhoIsInfo': who_is,
            }

            pprint(doc)

if __name__ == '__main__':
    gather()
