#!/usr/bin/env python
# encoding: UTF-8


__author__ = "Mridul, Parth, Ritwick"
__credits__ = ["Mridul", "Parth", "Ritwick"]
#__license__ = "GPLv3"
__version__ = "1.0"

#######################################################

import argparse
import json
import mmap
import tempfile
from argparse import RawTextHelpFormatter
from pathlib import Path

import censys.certificates
import validators
import sys
import socket
import whois
import dns.resolver
import collections
import re
import os
import random

from censys.base import CensysException
from censys.ipv4 import CensysIPv4
from colorama import Style, Fore
from dns import reversename
import requests
import time
import shodan
import pprint
# import pdb

#######################################################

# remove insecure https warning
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)

#######################################################

message = """
___________              .__                _________              

""".format(__version__)


# ######################  Global Variables ##################  #

pp = pprint.PrettyPrinter(indent=4)
KEYS_FILE = Path(__file__).parent.absolute().joinpath('api_keys.txt')
DEBUG = True


# ######################  Censys.io  ######################### #

def print_short(res):
    """
    Build Short results - Censys.io
    :param res: results, type: `dict`
    :return:
    """
    max_title_len = 50
    # public_info = []
    title_head = 'Title: '
    cut = '[...]'
    http_title = res.get('80.http.get.title', 'N/A')
    cert_name = res.get('443.https.tls.certificate.parsed.subject.common_name', '')
    cert_alt = res.get('443.https.tls.certificate.parsed.extensions.subject_alt_name.dns_names',
                       '')
    as_name = res.get('autonomous_system.name', 'N/A')
    as_num = res.get('autonomous_system.asn', '')
    loc = '%s / %s' % (res.get('location.country_code', 'N/A'), res.get('location.city', 'N/A'))
    os = res.get('metadata.os', 'N/A')
    tags = res.get('tags', '')
    ip = res.get('ip', 'N/A')

    http_title = http_title.replace('\n', '\\n')
    http_title = http_title.replace('\r', '\\r')

    # cleanup of list values
    if isinstance(cert_name, list):
        if len(cert_name) > 1:
            cert_name = cert_name[0] + "+"
        else:
            cert_name = cert_name[0]
    if isinstance(cert_alt, list):
        if len(cert_alt) > 1:
            cert_alt = cert_alt[0] + "+"
        else:
            cert_alt = cert_alt[0]

    # encoding to UTF-8
    http_title = str(http_title.encode('UTF-8'), errors='ignore')
    cert_name = str(cert_name.encode('UTF-8'), errors='ignore')
    cert_alt = str(cert_alt.encode('UTF-8'), errors='ignore')
    tags = ', '.join([str(t.encode('UTF-8'), errors='ignore') for t in tags])
    as_name = str(as_name.encode('UTF-8'), errors='ignore')
    os = str(os.encode('UTF-8'), errors='ignore')
    loc = str(loc.encode('UTF-8'), errors='ignore')

    if cert_alt != '' and cert_alt != cert_name:
        cert_name = cert_name + ' + ' + cert_alt

    # shorten title if too long
    if len(http_title) > (max_title_len - len(title_head) - 1):
        http_title = http_title[:max_title_len - len(title_head) - len(cut) - 1] + cut

    ret = (ip.ljust(16) +
           ((title_head + '%s') % http_title).ljust(max_title_len) +
           ('SSL: %s' % cert_name).ljust(50) +
           ('AS: %s (%s)' % (as_name, as_num)).ljust(40) +
           ('Loc: %s' % loc).ljust(30) +
           ('OS: %s' % os).ljust(15) +
           ('Tags: %s' % tags))
    print(ret)

    return ret


def CensysPublicReport(engine, public_info, output_basename):
    """ Censys Public Scan report """
    if output_basename:
        output1 = output_basename + ".txt"
        output2 = output_basename + ".md"
        output3 = output_basename + ".xml"
        output4 = output_basename + ".html"

        with open(output1, 'a') as txt, open(output2, 'a') as md, \
                open(output3, 'a') as xml, open(output4, 'a') as html:
            txt.write("[+] {} results\n".format(engine))
            txt.write("-------------------------\n")
            md.write("---\n\n")
            md.write("## {} results\n".format(engine))
            xml.write("<{}Results>\n".format(engine))
            html.write("<h3>{} results</h3>\n".format(engine))

            txt.write("\n")
            md.write("\n")

            txt.write("Censys Public Scan:\n")
            md.write("### Censys\n\n")
            xml.write("<Censys>\n")
            html.write("<h4>Censys</h4>\n<ul>\n")

            for publicscan in public_info:
                txt.write("%s\n" % publicscan)
                # md.write("* {}\n".format(publicscan))
                # xml.write("<Public_Censys>{}</Public_Censys>\n".format(publicscan))
                # html.write("<li>{}</li>\n" % publicscan)

            html.write("</ul>\n")
            xml.write("</Censys>\n")
            txt.write("\n")
            md.write("\n")


def CensysPublicScan(api_id, api_sec, output_basename, args, report_buckets,
                     match, public_info, custom_filter_fields=None):
    """ Censys.io Public Scan - Censys.io """
    if (args.mode == 'censys' and (
            args.Limit != float('inf') or args.asn != None or args.report != None or
            args.html != False or args.http_server != None or args.tags != None or
            args.cert_org != None or args.cert_host != None or args.count != False or
            args.html_body != None or args.html_title != None or args.country != None)):
        q, s = BuildQuery(api_id, api_sec, args)
        # count the number of results
        try:
            count = q.report(s, "updated_at")['metadata']['count']
        except CensysException as e:
            print(e.message)
            sys.exit(-1)

        if args.report:
            try:
                r = q.report(s, args.report, report_buckets)
            except CensysException as e:
                print(e.message)
                sys.exit(-1)
            sys.stderr.write("Number of results: %d\n" % count)
            print_report(r, args.report, public_info, output_basename)
            sys.exit(0)

        if args.count:
            print("Number of results: %d\n" % count)
            sys.exit(0)
        else:
            sys.stderr.write("Number of results: %d\n" % count)

        # prepare temp dir for html files
        if args.html:
            htmldir = tempfile.mkdtemp()
            open(htmldir + "/README", "w").write(
                "html body dumped via command:" + ' '.join(sys.argv))
            print("HTML body dumped to %s" % htmldir)

        public_info2 = []
        # search the API 
        if args.filter:
            filter_fields = args.filter.split(',')
        else:
            filter_fields = custom_filter_fields
        r = q.search(s, fields=filter_fields)
        i = 0
        for e in r:
            if i >= float(args.Limit):
                break
            if args.verbose:
                pp.pprint(q.view(e['ip']))
            elif args.filter:
                print(e)  # todo: by default dump raw JSON if filters are used
            else:
                public_info = print_short(e)
                public_info2.extend(public_info)
                if args.html:
                    dump_html_to_file(htmldir, e)
                if match != 'None':
                    print_match(q.view(e['ip']), match)
                i += 1

        return True
    return False


def print_match(res, m):
    """ finding matching results - Censys.io """
    for k in list(res.keys()):
        json_find(res[k], k, list(), m)
    print()


def print_report(res, key, public_info, output_basename):
    """ Format the result output - Censys.io """
    r = res['results']
    print("count".ljust(10) + "\t" + key.split(".")[-1])
    for e in r:
        print(("%d" % e['doc_count']).ljust(10) + "\t" + str(e['key']).ljust(30))
        public_info = ("%d" % e['doc_count']).ljust(10) + "\t" + str(e['key']).ljust(30)

    CensysPublicReport('censys', public_info, output_basename)


def build_query_string(args):
    """ Build the Query String - Censys.io """
    if len(args.arguments) == 0:
        s = '*'
    else:
        s = "(" + args.arguments[0] + ")"
    if args.tags:
        if ',' in args.tags:
            tags_l = args.tags.split(',')
            tags_q = " AND tags:" + " AND tags:".join(tags_l)
        else:
            tags_q = " AND tags:%s" % args.tags
        s += tags_q
    if args.asn:
        s += " AND autonomous_system.asn:%s" % args.asn
    if args.cert_org:
        s += " AND 443.https.tls.certificate.parsed.subject.organization:%s" % args.cert_org
    if args.cert_issuer:
        s += " AND 443.https.tls.certificate.parsed.issuer.organization:%s" % args.cert_issuer
    if args.cert_host:
        s += " AND 443.https.tls.certificate.parsed.subject.common_name:%s" % args.cert_host
    if args.country:
        s += " AND location.country_code:%s" % args.country
    if args.http_server:
        s += " AND 80.http.get.headers.server:%s" % args.http_server
    if args.html_title:
        if " " in args.html_title:
            title = "\"%s\"" % args.html_title
        else:
            title = args.html_title
        s += " AND 80.http.get.title:%s" % title
    if args.html_body:
        if " " in args.html_body:
            body = "\"%s\"" % args.html_body
        else:
            body = args.html_body
        s += " AND 80.http.get.body:%s" % body
    if args.debug:
        print('Query: %s' % s)
    return s


def is_contained(a, b):
    """ returns true if b is contained inside a """
    if type(a) == type(b):
        m = re.search(b, a, re.UNICODE + re.IGNORECASE)
        if m:
            return True
        else:
            return False


def BuildQuery(api_id, api_sec, pargs):
    """ uild The Query - Censys.io """
    # build up query
    q = CensysIPv4(api_id, api_sec)
    s = build_query_string(pargs)
    return q, s


def print_res(path, match, val):
    """ Printing Results - Censys.io """
    sep = ' '
    pre = '[...]'
    post = pre
    pos = match.lower().index(val.lower())
    if len(match) >= 80:
        if pos < 35:
            pre = ''
        match_c = Style.DIM + pre + match[pos - 35:pos] + Fore.RED + Style.BRIGHT + match[
                                                                                    pos:pos + len(
                                                                                        val)] + \
                  Style.RESET_ALL + Style.DIM + match[
                                                pos + len(val):pos + 35] + post + Style.RESET_ALL
        match = pre + match[pos - 35:pos + 35] + post
    else:
        match_c = Style.DIM + match[:pos] + Fore.RED + Style.BRIGHT + match[pos:pos + len(val)] + \
                  Style.RESET_ALL + Style.DIM + match[pos + len(val):] + Style.RESET_ALL

    match_c = match_c.replace('\n', '\\n')
    match_c = match_c.replace('\r', '\\r')
    match = match.replace('\n', '\\n')
    match = match.replace('\r', '\\r')

    if len(path) >= 60:
        sep = '\n\t'
    if sys.stdout.isatty():
        print("  %s:%s%s" % (path, sep, match_c))
    else:
        print("  %s:%s%s" % (path, sep, match))


def append_if_new(l, e):
    if e not in l:
        return l + [e]
    else:
        return l


def json_find(obj, k, visited, val):
    """
    Searching JSON - Censys.io
    recursively find values in dict 'obj' that macthes 'val'
    store the keys to access the matching value in 'path'
    """
    if visited is None:
        visited = list()

    # case of sub-dict : recursivity
    if isinstance(obj, dict):
        visited = append_if_new(visited, k)
        # visited = visited + [k]
        for key in list(obj.keys()):
            visited = json_find(obj[key], key, visited, val)

    # case of list : check all members
    elif isinstance(obj, list):
        for e in obj:
            if is_contained(e, val):
                print_res('.'.join(visited + [k]), e, val)

    # finally easiest case, leaf
    elif is_contained(obj, val):
        print_res('.'.join(visited + [k]), obj, val)

    # remove nodes already visited before returning
    if k in visited:
        visited.pop()
    return visited


def dump_html_to_file(d, rec):
    """ Dump HTML to file - Censys.io """
    html = rec.get('80.http.get.body')
    if html:
        filename = "%s/%s.html" % (d, rec['ip'])
        open(filename, "w").write(html.encode('UTF-8', errors='ignore'))


def DomainSearchCensys(domain_name, api_id, api_sec, output_basename, domains):
    """ Domain Searching with Censys.io """
    if domain_name != None:
        subdomains = CensysSearch(domain_name, api_id, api_sec)
        domains.extend(subdomains)
        SubdomainsReport('Censys', subdomains, output_basename)
        return True

    return False

#######################################################


def checkFile():
    """ Check if file exists """
    is_true = os.path.isfile(KEYS_FILE)
    if is_true != True:
        return False
    if os.path.getsize(KEYS_FILE) == 0:
        return False
    else:
        return True


def checkUser(engine):
    """ Check if engine exists in API key file """
    chk = checkFile()
    if not chk:
        return False
    else:
        with open(KEYS_FILE) as f:
            s = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)
            if s.find(engine) != -1:
                return True
            else:
                return False


# ################# API KEYS ################## #

def readFileContents():
    """ Read API Keys from file """
    with open(KEYS_FILE) as f:
        lines = f.read().splitlines()
        print(
            "|       Engine       |                API Keys ID              |              API Secret Keys            |"
        )
        print(
            "|========================================================================================================|"
        )
        for line in lines:
            print("|", line.split(":")[0], " " * (17 - len(line.split(":")[0])), "|",
                  line.split(":")[1], " " * (38 - len(line.split(":")[1])), "|",
                  line.split(":")[2], " " * (38 - len(line.split(":")[2])), "|")


def createFileAndStoreAPIKeys(engine):
    """ Create file and Store the API Keys """
    f = open(KEYS_FILE, "w+")
    api_id = input("[*] please provide the new %s API ID: " % engine)
    api_sec = input("[*] please provide the new %s API Secret: " % engine)
    f.write(engine + ":" + api_id + ":" + api_sec)
    f.close()
    return ("stored")


def updateAPIKeys(engine):
    """ Update API Keys """
    ckhstored = checkUser(engine)
    api_id = input("[*] please provide the new %s API ID: " % engine)
    api_sec = input("[*] please provide the new %s API Secret: " % engine)
    with open(KEYS_FILE, "r+") as op:
        lines = op.read().splitlines()
        if ckhstored == True:
            for line in lines:
                if line != line.split(":")[0]:
                    with open(KEYS_FILE, "w+") as f:
                        f.write(line)

            for line in lines:
                if line != line.split(":")[0]:
                    with open(KEYS_FILE, "w+") as f1:
                        f1.write(engine + ":" + api_id + ":" + api_sec)
            return 'y'

        if ckhstored == False:
            print("[!] user does not exist in file")
            return 'n'


# ############### Check Domain, IP, Hostname  ###################### #

def CheckDomain(value):
    """ Validate Domain name """
    if not validators.domain(value):
        raise argparse.ArgumentTypeError('Invalid {} domain.'.format(value))
    return value


def CheckDomainOrIP(value):
    """ Verify domain/ip  """
    if not validators.domain(value) and not validators.ip_address.ipv4(value):
        raise argparse.ArgumentTypeError('Invalid domain or ip address ({}).'.format(value))
    return value


def VerifyHostname(value):
    """ Get Domain ip address """
    try:
        ip = socket.gethostbyname(value)
        return ip
    except Exception as e:
        return False


# #####################  Whois, DNS  ########################### #

def WhoisQuery(value):
    whoisData = collections.OrderedDict()
    whoisData["name"] = ["-", "Name:"]
    whoisData["org"] = ["-", "Organization:"]
    whoisData["address"] = ["-", "Address:"]
    whoisData["city"] = ["-", "City:"]
    whoisData["zipcode"] = ["-", "Zip code:"]
    whoisData["country"] = ["-", "Country:"]
    whoisData["emails"] = ["-", "Emails:"]
    whoisData["registrar"] = ["-", "Registrar:"]
    whoisData["whois_server"] = ["-", "Whois Server:"]
    whoisData["updated_date"] = ["-", "Updated Date:"]
    whoisData["expiration_date"] = ["-", "Expiration Date:"]
    whoisData["creation_date"] = ["-", "Creation Date:"]
    whoisData["name_servers"] = ["-", "Name Servers:"]

    domain = whois.whois(value)

    for rec in whoisData:
        if domain[rec]:
            if isinstance(domain[rec], list):
                if rec == 'name_servers':
                    whoisData[rec][0] = []
                    for val in domain[rec]:
                        whoisData[rec][0].append(val + ":" + VerifyHostname(val))
                else:
                    whoisData[rec][0] = []
                    for val in domain[rec]:
                        whoisData[rec][0].append(val)
            else:
                whoisData[rec][0] = str(domain[rec])

    return whoisData


def _query(value, dnsserver, record):
    myresolver = dns.resolver.Resolver()
    myresolver.nameservers = [dnsserver]

    try:
        answers = myresolver.query(value, record)
        for answer in answers:
            # TODO check: this for loop is returning on first loop
            if record == 'NS':
                return answer.to_text() + ":" + VerifyHostname(answer.to_text())
            elif record == 'MX':
                domain_name = re.search(' (.*)\.', answer.to_text(), re.IGNORECASE).group(1)
                return answer.to_text() + ":" + VerifyHostname(domain_name)
            else:
                return answer.to_text()
    except Exception:
        return '-'

    return None  # dnsData?


def DnsQuery(value, dnsserver, record=None):
    """ Perform DNS queries  """
    dnsData = {
        "A": [],
        "CNAME": [],
        "HINFO": [],
        "MX": [],
        "NS": [],
        "PTR": [],
        "SOA": [],
        "TXT": [],
        "SPF": [],
        "SRV": [],
        "RP": []
    }

    if record is None:
        for record in dnsData:
            dnsData[record].append(_query(value, dnsserver, record))
    else:
        dnsData[record].append(_query(value, dnsserver, record))

    return dnsData


def ReverseIPQuery(value, dnsserver):
    """ IP DNS Reverse lookup """
    try:
        revname = reversename.from_address(value)
        myresolver = dns.resolver.Resolver()
        myresolver.nameservers = [dnsserver]
        return str(myresolver.query(revname, 'PTR')[0]).rstrip('.')
    except Exception as e:
        print(e)
        return ''


# ##########  Parse the Results to get usefull data ############## #

def CleanHTML(results):
    """ Clean HTML tags """
    res = results
    res = re.sub('<em>', '', res)
    res = re.sub('<b>', '', res)
    res = re.sub('</b>', '', res)
    res = re.sub('</em>', '', res)
    res = re.sub('%2f', ' ', res)
    res = re.sub('%3a', ' ', res)
    res = re.sub('<strong>', '', res)
    res = re.sub('</strong>', '', res)
    res = re.sub('<wbr>', '', res)
    res = re.sub('</wbr>', '', res)
    res = re.sub('&lt;', '', res)

    return res


def GetEmails(results, value):
    """ Extract Emails """
    res = results
    res = CleanHTML(res)

    # todo this regex needs improvement per case/service
    temp = re.compile(
        '[a-zA-Z0-9.\-_+#~!$&\',;=:]+' +
        '@' +
        '[a-zA-Z0-9.-]*' +
        value)

    emails = temp.findall(res)
    final = []
    for email in emails:
        final.append(email.lower())

    return sorted(set(final))


def GetHostnames(results, value):
    """ Extract Hostnames """
    res = results
    res = CleanHTML(res)

    temp = re.compile('[a-zA-Z0-9.-]*\.' + value)
    hostnames = temp.findall(res)
    final = []
    for host in hostnames:
        final.append(host.lower())

    return sorted(set(final))


def GetHostnamesAll(results):
    """ Get All Hostnames """
    res = results
    temp = re.compile('<cite>(.*?)</cite>')
    hostname = temp.findall(res)
    vhosts = []

    for x in hostname:
        r = ''
        if x.count(':'):
            r = x.split(':')[1].split('/')[2]
        else:
            r = x.split("/")[0]
        vhosts.append(r)

    return sorted(set(vhosts))


def GetDNSDumpsterHostnames(value, results):
    tbl_regex = re.compile('<a name="hostanchor"><\/a>Host Records.*?<table.*?>(.*?)</table>',
                           re.S)
    link_regex = re.compile('<td class="col-md-4">(.*?)<br>', re.S)
    subdomains = []

    try:
        results_tbl = tbl_regex.findall(results)[0]
    except IndexError:
        results_tbl = ''
    links_list = link_regex.findall(results_tbl)
    links = list(set(links_list))

    for link in links:
        subdomain = link.strip()
        if not subdomain.endswith(value):
            continue
        if subdomain and subdomain not in subdomains and subdomain != value:
            subdomains.append(subdomain.strip())

    return sorted(set(subdomains))


def SpyseGetDomains(data):
    """
    :type data: dict
    :rtype: list
    """
    domains = []
    try:
        domains = [i['name'] for i in data['data']['items']]
    except IndexError:
        print("Unrecognised data returned by Spyse")

    return domains


# ################## User Agent #################### #

def PickRandomUA(value):
    """ Pick random User-Agent string """
    if value:
        secure_random = random.SystemRandom()
        return secure_random.choice(value)

    return None


def PickRandomTimeout(value):
    if value:
        secure_random = random.SystemRandom()
        return secure_random.choice(value)

    return 5


# #######################  Common search  ########################### #

def CommonSearch(value, urltemplate, quantity, step, limit, uas, proxies, timeouts):
    counter = 0
    results = ""

    while counter <= limit:
        try:
            url = urltemplate.format(quantity=quantity, counter=counter, value=value)
            s = requests.Session()
            r = s.get(url, verify=False, headers={'User-Agent': PickRandomUA(uas)},
                      proxies=proxies)
            if r.status_code != 200:
                print("[-] Something is going wrong (status code: {})".format(r.status_code))
                return [], []
            results += r.text
        except Exception as e:
            print(e)

        time.sleep(PickRandomTimeout(timeouts))
        counter += step

    return GetEmails(results, value), GetHostnames(results, value)


def CommonSearch2(value, urltemplate, step, limit, uas, proxies, timeouts):
    counter = 1
    results = ""

    while counter <= limit:
        try:
            url = urltemplate.format(counter=counter, value=value)
            s = requests.Session()
            r = s.get(url, verify=False, headers={'User-Agent': PickRandomUA(uas)},
                      proxies=proxies)
            if r.status_code != 200:
                print("[-] Something is going wrong (status code: {})".format(r.status_code))
                return [], []
            results += r.text
        except Exception as e:
            print(e)

        time.sleep(PickRandomTimeout(timeouts))
        counter += step

    return GetEmails(results, value), GetHostnames(results, value)


# ######################  Search Engines & Services ######################### #

def GoogleSearch(value, limit, uas, proxies, timeouts):
    quantity = 100
    step = 100
    url = "https://www.google.com/search?num={quantity}&start={counter}&hl=en&meta=&q=%40%22{value}%22"

    return CommonSearch(value, url, quantity, step, limit, uas, proxies, timeouts)


def BingSearch(value, limit, uas, proxies, timeouts):
    quantity = 50
    step = 50
    url = "https://www.bing.com/search?q=%40{value}&count={quantity}&first={counter}"

    return CommonSearch(value, url, quantity, step, limit, uas, proxies, timeouts)


def AskSearch(value, limit, uas, proxies, timeouts):
    step = 1
    url = "https://www.ask.com/web?q=%40%22{value}%22&page={counter}"

    return CommonSearch2(value, url, step, limit, uas, proxies, timeouts)


# todo captcha
def DogpileSearch(value, limit, uas, proxies, timeouts):
    step = 15
    url = "https://www.dogpile.com/search/web?qsi={counter}&q=%40{value}"

    return CommonSearch2(value, url, step, limit, uas, proxies, timeouts)


def YahooSearch(value, limit, uas, proxies, timeouts):
    step = 10
    url = "https://search.yahoo.com/search?p=%40{value}&b={counter}&pz=10"

    return CommonSearch2(value, url, step, limit, uas, proxies, timeouts)


def YandexSearch(value, limit, uas, proxies, timeouts):
    server = "yandex.com"
    quantity = 50
    step = 50
    results = ""
    page = 0
    counter = 0

    while counter <= limit:
        try:
            url = "https://" + server + "/search/?text=%22%40" + value + "%22&numdoc=" \
                  + str(quantity) + "&p=" + str(page) + "&lr=10418"
            s = requests.Session()
            r = s.get(url, verify=False, headers={'User-Agent': PickRandomUA(uas)},
                      proxies=proxies)
            if r.status_code != 200:
                print("[-] Something is going wrong (status code: {})".format(r.status_code))
                return [], []
            results += r.text
        except Exception as e:
            print(e)

        time.sleep(PickRandomTimeout(timeouts))
        counter += step
        page += 1

    return GetEmails(results, value), GetHostnames(results, value)


def CrtSearch(value, uas, proxies):
    server = "crt.sh"
    results = ""

    try:
        url = "https://" + server + "/?q=" + value
        s = requests.Session()
        r = s.get(url, verify=False, headers={'User-Agent': PickRandomUA(uas)}, proxies=proxies)
        if r.status_code != 200:
            print("[-] Something is going wrong (status code: {})".format(r.status_code))
            return [], []
        results += r.text
    except Exception as e:
        print(e)

    return GetHostnames(results, value)


def DNSDumpsterSearch(targetip, uas, proxies):
    server = "dnsdumpster.com"
    results = ""
    timeout = 25
    subdomains = None

    try:
        url = "https://" + server + "/"
        s = requests.Session()
        myheaders = {'User-Agent': PickRandomUA(uas), 'Referer': 'https://dnsdumpster.com'}
        r = s.get(url, verify=False, headers=myheaders, proxies=proxies, timeout=timeout)
        if r.status_code != 200:
            print("[-] Something is going wrong (status code: {})".format(r.status_code))
            return [], []

        # get csrf token
        csrf_regex = re.compile('name="csrfmiddlewaretoken" value="(.*?)"', re.S)
        try:
            token = csrf_regex.findall(r.text)[0]
        except IndexError:
            print("[-] CSRF Token not found")
            return None

        params = {'csrfmiddlewaretoken': token, 'targetip': targetip}
        pr = s.post(url, verify=False, headers=myheaders, proxies=proxies, data=params,
                    timeout=timeout)
        if pr.status_code != 200:
            print("[-] Something is going wrong (status code: {})".format(pr.status_code))
            return [], []

        subdomains = GetDNSDumpsterHostnames(targetip, pr.text)

    except Exception as e:
        print(e)

    return subdomains


# todo pgp.mit.edu lookup service seems down
def PGPSearch(value, uas, proxies):
    server = "pgp.mit.edu"
    results = ""

    try:
        url = "https://" + server + "/pks/lookup?search=" + value + "&op=index"
        s = requests.Session()
        r = s.get(url, verify=False, headers={'User-Agent': PickRandomUA(uas)}, proxies=proxies)
        if r.status_code != 200:
            print("[-] Something is going wrong (status code: {})".format(r.status_code))
            return [], []
        results += r.text
    except Exception as e:
        print(e)

    return GetEmails(results, value), GetHostnames(results, value)


def NetcraftSearch(value, uas, proxies):
    server = "searchdns.netcraft.com"
    results = ""

    try:
        url = "https://" + server + "?restriction=site+ends+with&host=" + value
        s = requests.Session()
        r = s.get(url, verify=False, headers={'User-Agent': PickRandomUA(uas)}, proxies=proxies)
        if r.status_code != 200:
            print("[-] Something is going wrong (status code: {})".format(r.status_code))
            return [], []
        results += r.text
    except Exception as e:
        print(e)

    return GetHostnames(results, value)


def VTSearch(value, uas, proxies):
    server = "www.virustotal.com"
    results = ""

    try:
        url = "https://" + server + "/en/domain/" + value + "/information/"
        s = requests.Session()
        r = s.get(url, verify=False, headers={'User-Agent': PickRandomUA(uas)}, proxies=proxies)
        if r.status_code != 200:
            print("[-] Something is going wrong (status code: {})".format(r.status_code))
            return [], []
        results += r.text
    except Exception as e:
        print(e)

    return GetHostnames(results, value)


def SpyseSearch(value, apikey, limit=100, proxies=None):
    server = "api.spyse.com"
    data = {}

    try:
        url = f'https://{server}/v2/data/domain/subdomain?limit={limit}&domain={value}'
        headers = {
            'accept': 'application/json',
            'Authorization': f'Bearer {apikey}',
        }
        s = requests.Session()
        r = s.get(url, verify=False, headers=headers, proxies=proxies)
        if DEBUG: print(r.text)
        if r.status_code != 200:
            print(f'[-] Something is going wrong - status code: {r.status_code}')
        else:
            data = json.loads(r.text)
    except (ValueError, TypeError):
        print("Unrecognised data returned by spyse api")
    except NameError as e:
        print("Error: Insufficient data passed to SpyseSearch")
        print(e)
    except Exception as e:
        print(e)

    return data


def CensysSearch(value, api_id, api_secret):
    try:
        censys_certificates = censys.certificates.CensysCertificates(api_id=api_id,
                                                                     api_secret=api_secret)
        certificate_query = 'parsed.names: %s' % value
        certificates_search_results = censys_certificates.search(certificate_query,
                                                                 fields=['parsed.names'])
        subdomains = []
        for search_result in certificates_search_results:
            subdomains.extend(search_result['parsed.names'])
        return set(subdomains)
    except censys.base.CensysUnauthorizedException:
        sys.stderr.write('[-] Your Censys credentials look invalid.\n')
        exit(1)
    except censys.base.CensysRateLimitExceededException:
        sys.stderr.write('[-] Looks like you exceeded your Censys account limits rate. Exiting\n')
        exit(1)


def GoogleSearchEngine(value, site, limit, uas, proxies, timeouts):
    """ google search in site """
    server = "www.google.com"
    quantity = 100
    counter = 0
    step = 100
    results = ""

    while counter <= limit:
        try:
            url = "https://" + server + "/search?num=" + str(quantity) + "&start=" + str(
                counter) + "&hl=en&meta=&q=site%3A" + site + "%20%40%22" + value + "%22"
            s = requests.Session()
            r = s.get(url, verify=False, headers={'User-Agent': PickRandomUA(uas)},
                      proxies=proxies)
            if r.status_code != 200:
                print("[-] Something is going wrong (status code: {})".format(r.status_code))
                return [], []
            results += r.text
        except Exception as e:
            print(e)

        time.sleep(PickRandomTimeout(timeouts))
        counter += step

    return GetEmails(results, value), GetHostnames(results, value)


def BingVHostsSearch(value, limit, uas, proxies, timeouts):
    """ Bing Virtual Hosts Search """
    server = "www.bing.com"
    quantity = 50
    step = 50
    counter = 0
    results = ""
    vhosts = []

    while counter <= limit:
        try:
            url = "https://" + server + "/search?q=ip%3A" + value + "&go=&count=" + str(
                quantity) + "&FORM=QBHL&qs=n&first=" + str(counter)
            s = requests.Session()
            r = s.get(url, verify=False, headers={'User-Agent': PickRandomUA(uas)},
                      proxies=proxies)
            if r.status_code != 200:
                print("[-] Something is going wrong (status code: {})".format(r.status_code))
                return [], []
            results += r.text
        except Exception as e:
            print(e)

        time.sleep(PickRandomTimeout(timeouts))
        counter += step

    all_hostnames = GetHostnamesAll(results)

    for x in all_hostnames:
        x = re.sub(r'[[\<\/?]*[\w]*>]*', '', x)
        x = re.sub('<', '', x)
        x = re.sub('>', '', x)
        vhosts.append(x)

    return sorted(set(vhosts))


# todo if the API is used (valid key), the rtype is `dict` else
# the search is done via the website and the rtype is a `set` of vhosts
def ShodanSearch(api_key, domain, value, uas, proxies, timeouts, limit=0):

    api = shodan.Shodan(api_key)
    server = "shodan.io"
    counter = 0
    quantity = 10
    step = 10
    vhosts = []
    results = ''

    # Warning: Shodan api needs a payment plan!
    # Wrap the request in a try/ except block to catch errors
    try:
        # Search by hostname
        query = 'hostname:' + domain
        return api.search(query)

    except shodan.APIError as e:
        print('Error: %s' % e)

    while counter <= limit:
        try:
            url = "https://" + server + "/search?q=ip%3A" + value + "&go=&count=" + str(
                quantity) + "&FORM=QBHL&qs=n&first=" + str(counter)
            s = requests.Session()
            r = s.get(url, verify=False, headers={'User-Agent': PickRandomUA(uas)},
                      proxies=proxies)
            if r.status_code != 200:
                print("[-] Something is going wrong (status code: {})".format(r.status_code))
                return [], []
            results += r.text
        except Exception as e:
            print(e)

        time.sleep(PickRandomTimeout(timeouts))
        counter += step

    all_hostnames = GetHostnamesAll(results)

    for x in all_hostnames:
        x = re.sub(r'[[\<\/?]*[\w]*>]*', '', x)
        x = re.sub('<', '', x)
        x = re.sub('>', '', x)
        vhosts.append(x)

    return sorted(set(vhosts))


# ##################  Reports  ######################## #

def Report(engine, emails, hostnames, output_basename):
    """ Emails & Hostnames Console report """
    print()
    print("Emails:")
    for email in emails:
        print(email)

    print()
    print("Hostnames:")
    for host in hostnames:
        print(host)
    print()

    if output_basename:
        output1 = output_basename + ".txt"
        output2 = output_basename + ".md"
        output3 = output_basename + ".xml"
        output4 = output_basename + ".html"

        with open(output1, 'a') as txt, open(output2, 'a') as md, \
                open(output3, 'a') as xml, open(output4, 'a') as html:
            txt.write("[+] {} results\n".format(engine))
            txt.write("-------------------------\n")
            md.write("---\n\n")
            md.write("## {} results\n".format(engine))
            xml.write("<{}Results>\n".format(engine))
            html.write("<h3>{} results</h3>\n".format(engine))

            txt.write("\n")
            md.write("\n")

            txt.write("Emails:\n")
            md.write("### Emails\n\n")
            xml.write("<Emails>\n")
            html.write("<h4>Emails</h4>\n<ul>\n")

            for email in emails:
                txt.write("{}\n".format(email))
                md.write("* {}\n".format(email))
                xml.write("<email>{}</email>\n".format(email))
                html.write("<li>{}</li>\n".format(email))

            html.write("</ul>\n")
            xml.write("</Emails>\n")
            txt.write("\n")
            md.write("\n")

            txt.write("Hostnames:\n")
            md.write("### Hostnames\n\n")
            xml.write("<Hostnames>\n")
            html.write("<h4>Hostnames</h4>\n<ul>\n")

            for host in hostnames:
                txt.write("{}\n".format(host))
                md.write("* {}\n".format(host))
                xml.write("<hostname>{}</hostname>\n".format(host))
                html.write("<li>{}</li>\n".format(host))

            html.write("</ul>\n")
            xml.write("</Hostnames>\n")
            txt.write("\n")
            md.write("\n")
            xml.write("</{}Results>\n".format(engine))


def SubdomainsReport(engine, subdomains, output_basename):
    """ Subdomains Console report """
    assert type(subdomains) in (list, tuple, dict)  # should be list
    if len(subdomains) == 0:
        print('[-] Did not find any subdomain')
        return

    print('')
    print('[*] Found %d subdomains' % (len(subdomains)))
    print('')
    for subdomain in subdomains:
        print(subdomain)
    print('')

    if output_basename:
        output1 = output_basename + ".txt"
        output2 = output_basename + ".md"
        output3 = output_basename + ".xml"
        output4 = output_basename + ".html"

        with open(output1, 'a') as txt, open(output2, 'a') as md, open(output3, 'a') as xml, open(
                output4, 'a') as html:
            txt.write("[+] {} results\n".format(engine))
            txt.write("-------------------------\n")
            md.write("---\n\n")
            md.write("## {} results\n".format(engine))
            xml.write("<{}Results>\n".format(engine))
            html.write("<h3>{} results</h3>\n".format(engine))

            txt.write("\n")
            md.write("\n")

            txt.write("Subdomains:\n")
            md.write("### Subdomains\n\n")
            xml.write("<Subdomains>\n")
            html.write("<h4>Subdomains</h4>\n<ul>\n")

            for subdomain in subdomains:
                txt.write("{}\n".format(subdomain))
                md.write("* {}\n".format(subdomain))
                xml.write("<Subdomain>{}</Subdomains>\n".format(subdomain))
                html.write("<li>{}</li>\n".format(subdomain))

            html.write("</ul>\n")
            xml.write("</Subdomains>\n")
            txt.write("\n")
            md.write("\n")


def ShodanReport(results, output_basename):
    """ Shodan Console Report """
    engine = "Shodan"
    if len(results) == 0:
        print('[-] Did not find any results')
        return

    if type(results) in (set, list, tuple):
        print('Shodan Report is not available for the current results')
        print('Results found: %s' % len(results))
        for r in results:
            print(r)
        return

    print()
    print('Results found: %s' % results['total'])
    print('------------------')
    print()

    # pdb.set_trace()

    for result in results['matches']:

        # Print host info
        print('IP: %s' % result['ip_str'])
        print('-------------------')
        print('Hostnames: ' + ','.join(result['hostnames']))
        print('Organization: %s' % result.get('org', 'n/a'))
        print('Operating System: %s' % result.get('os', 'n/a'))
        print('Port: %s' % result['port'])

        # Print banner
        banner = result.get('data')
        if banner is not None:
            print('Banner:')
            print(result['data'])
            print()

    # Output to file
    if output_basename:
        output1 = output_basename + ".txt"
        output2 = output_basename + ".md"
        output3 = output_basename + ".xml"
        output4 = output_basename + ".html"

        with open(output1, 'a') as txt, open(output2, 'a') as md, open(output3, 'a') as xml, open(
                output4, 'a') as html:
            txt.write("[+] {} results\n".format(engine))
            txt.write("-------------------------\n")
            md.write("---\n\n")
            md.write("## {} results\n".format(engine))
            xml.write("<{}Results>\n".format(engine))
            html.write("<h3>{} results</h3>\n".format(engine))

            txt.write("\n")
            md.write("\n")

            for result in results['matches']:
                ip = result['ip_str']
                hostnames = ','.join(result['hostnames'])
                organization = result.get('org', 'n/a')
                os = result.get('os', 'n/a')
                port = result['port']
                banner = result['data']

                # Print IP
                PrintField("IP", ip, txt, md, xml, html)

                # Print Hostnames
                PrintField("Hostnames", hostnames, txt, md, xml, html)

                # Print Organization
                PrintField("Organization", organization, txt, md, xml, html)

                # Print Operating System
                PrintField("OS", os, txt, md, xml, html)

                # Print Port
                PrintField("Port", port, txt, md, xml, html)

                # Print Banner
                PrintField("Banner", banner, txt, md, xml, html)

            xml.write("</{}Results>\n".format(engine))


def PrintField(label, value, txt, md, xml, html):
    """ Print field to output files """
    txt.write("{}:\n".format(label))
    md.write("### {}\n\n".format(label))
    xml.write("<{}>\n".format(label))
    html.write("<h4>{}</h4>\n<ul>\n".format(label))

    txt.write("{}\n".format(value))
    md.write("* {}\n".format(value))
    xml.write("<{}>{}</{}>\n".format(label, value, label))
    html.write("<li>{}</li>\n".format(value))

    html.write("</ul>\n")
    xml.write("</{}>\n".format(label))
    txt.write("\n")
    md.write("\n")


def HostnamesReport(engine, hostnames, output_basename):
    """ Hostnames Console report """
    assert type(hostnames) in (list, tuple, dict)  # should be list

    print()
    print("Hostnames:")
    for host in hostnames:
        print(host)
    print()

    if output_basename:
        output1 = output_basename + ".txt"
        output2 = output_basename + ".md"
        output3 = output_basename + ".xml"
        output4 = output_basename + ".html"

        with open(output1, 'a') as txt, open(output2, 'a') as md, open(output3, 'a') as xml, open(
                output4, 'a') as html:
            txt.write("[+] {} results\n".format(engine))
            txt.write("-------------------------\n")
            md.write("---\n\n")
            md.write("## {} results\n".format(engine))
            xml.write("<{}Results>\n".format(engine))
            html.write("<h3>{} results</h3>\n".format(engine))

            txt.write("\n")
            md.write("\n")

            txt.write("Hostnames:\n")
            md.write("### Hostnames\n\n")
            xml.write("<Hostnames>\n")
            html.write("<h4>Hostnames</h4>\n<ul>\n")

            for host in hostnames:
                txt.write("{}\n".format(host))
                md.write("* {}\n".format(host))
                xml.write("<hostname>{}</hostname>\n".format(host))
                html.write("<li>{}</li>\n".format(host))

            html.write("</ul>\n")
            xml.write("</Hostnames>\n")
            txt.write("\n")
            md.write("\n")
            xml.write("</{}Results>\n".format(engine))


def InfoReport(mode, limit, dnsserver, proxy, domain, ip, uas, output_basename):
    """ Information Console report """
    print("[+] Information gathering: {}".format(mode))
    print("[+] Looking into first {} search engines results".format(limit))
    print("[+] Using DNS server: {}".format(dnsserver))
    if proxy:
        print("[+] Using Proxy server: {}".format(proxy))
    print("[+] Target: {}:{}".format(domain, ip))
    print("[+] User-agent strings: {}".format(uas))
    print()

    if output_basename:
        output1 = output_basename + ".txt"
        output2 = output_basename + ".md"
        output3 = output_basename + ".xml"
        output4 = output_basename + ".html"

        with open(output1, 'w') as txt, open(output2, 'w') as md, open(output3, 'w') as xml, open(
                output4, 'w') as html:
            txt.write("{}\n".format(message))
            md.write("```\n")
            md.write("{}\n".format(message))
            md.write("```\n\n")
            xml.write('<!--\n{}\n-->\n'.format(message))
            xml.write('<?xml version="1.0" encoding="UTF-8"?>\n')
            xml.write("<report>\n")
            xml.write("<information>\n")
            html.write("<!DOCTYPE html><html><head><title>GasMasK Report</title></head><body>\n")
            html.write("<pre>{}</pre>\n</br>\n<ul>\n".format(message))

            txt.write("[+] Information gathering: {}\n".format(",".join(mode)))
            md.write("---\n\n")
            md.write("* Information gathering: {}\n".format(",".join(mode)))
            html.write("<li>Information gathering: <b>{}</b></li>\n".format(",".join(mode)))
            xml.write("<InformationGathering>\n")

            for m in mode:
                xml.write("<query>{}</query>\n".format(m))
            xml.write("</InformationGathering>\n")

            txt.write("[+] Looking into first {} search engine results\n".format(limit))
            md.write("* Looking into first {} search engine results\n".format(limit))
            xml.write("<SearchEngineResults>{}</SearchEngineResults>\n".format(limit))
            html.write("<li>Search Engine Results: <b>{}</b></li>\n".format(limit))

            txt.write("[+] Using DNS server: {}\n".format(dnsserver))
            md.write("* Using DNS server: {}\n".format(dnsserver))
            xml.write("<DNSServer>{}</DNSServer>\n".format(dnsserver))
            html.write("<li>Using DNS server: <b>{}</b></li>\n".format(dnsserver))

            if proxy:
                txt.write("[+] Using Proxy server: {}\n".format(proxy))
                md.write("* Using Proxy server: {}\n".format(proxy))
                xml.write("<ProxyServer>{}</ProxyServer>\n".format(proxy))
                html.write("<li>Using Proxy server: <b>{}</b></li>\n".format(proxy))

            txt.write("[+] Target: {}:{}\n".format(domain, ip))
            md.write("* Target: {}:{}\n".format(domain, ip))
            xml.write("<Target>\n")
            xml.write("<Domain>{}</Domain>\n".format(domain))
            xml.write("<IP>{}</IP>\n".format(ip))
            xml.write("</Target>\n")
            html.write("<li>Target: <b>{}:{}</b></li>\n".format(domain, ip))

            txt.write("[+] User-agent strings: {}\n".format(uas))
            md.write("* User-agent strings: {}\n".format(uas))
            xml.write("<UserAgentStrings>{}</UserAgentStrings>\n".format(uas))
            html.write("<li>User-agent strings: <b>{}</b></li>\n".format(uas))

            txt.write("\n")
            md.write("\n")
            xml.write("</information>\n")
            html.write("</ul>\n")


def WhoisReport(data, output_basename):
    """ Whois Console report"""
    for key, value in data.items():
        if isinstance(value[0], list):
            print()
            print(value[1])
            for val in value[0]:
                print(val)
            print()
        else:
            print(value[1] + " " + value[0])
    print()

    if output_basename:
        output1 = output_basename + ".txt"
        output2 = output_basename + ".md"
        output3 = output_basename + ".xml"
        output4 = output_basename + ".html"

        with open(output1, 'a') as txt, open(output2, 'a') as md, open(output3, 'a') as xml, open(
                output4, 'a') as html:
            txt.write("[+] Whois lookup\n")
            txt.write("----------------\n")
            md.write("---\n\n")
            md.write("## Whois lookup\n\n")
            xml.write("<Whois>\n")
            html.write("<h3>Whois lookup</h3>\n<ul>\n")

            for key, value in data.items():
                if isinstance(value[0], list):
                    txt.write("\n")
                    md.write("\n")

                    txt.write("{}\n".format(value[1]))
                    md.write("* {}\n".format(value[1]))
                    xml.write("<{}>\n".format(key))
                    html.write("<li>{}</li>\n<ul>\n".format(value[1]))

                    for val in value[0]:
                        txt.write("{}\n".format(val))
                        md.write("  * {}\n".format(val))
                        xml.write("<data>{}</data>\n".format(val))
                        html.write("<li><b>{}</b></li>\n".format(val))

                    xml.write("</{}>\n".format(key))
                    html.write("</ul>\n")

                    txt.write("\n")
                    md.write("\n")
                else:
                    txt.write("{} {}\n".format(value[1], value[0]))
                    md.write("* {} {}\n".format(value[1], value[0]))
                    xml.write("<{}>{}</{}>\n".format(key, value[0], key))
                    html.write("<li>{} <b>{}</b></li>\n".format(value[1], value[0]))

            txt.write("\n")
            md.write("\n")
            xml.write("</Whois>\n")
            html.write("</ul>\n")


def DNSReport(data, output_basename):
    """ DNS Console report """
    for key, value in data.items():
        if len(value) == 1:
            print(key + " DNS record: " + value[0])
        else:
            print()
            print(key + " DNS record: ")
            for val in value:
                print(val)
            print()
    print()

    if output_basename:
        output1 = output_basename + ".txt"
        output2 = output_basename + ".md"
        output3 = output_basename + ".xml"
        output4 = output_basename + ".html"

        with open(output1, 'a') as txt, open(output2, 'a') as md, open(output3, 'a') as xml, open(
                output4, 'a') as html:
            txt.write("[+] DNS queries\n")
            txt.write("---------------\n")
            md.write("---\n\n")
            md.write("## DNS queries\n\n")
            xml.write("<DNSQueries>\n")
            html.write("<h3>DNS queries</h3>\n<ul>\n")

            for key, value in data.items():
                if len(value) == 1:
                    txt.write("{} DNS record: {}\n".format(key, value[0]))
                    md.write("* {} DNS record: {}\n".format(key, value[0]))
                    xml.write("<{}>{}</{}>\n".format(key, value[0], key))
                    html.write("<li>{} DNS record: <b>{}</b></li>\n".format(key, value[0]))

                else:
                    txt.write("\n")
                    md.write("\n")

                    txt.write("{} DNS record:\n".format(key))
                    md.write("* {} DNS record:\n".format(key))
                    xml.write("<{}>\n".format(key))
                    html.write("<li>{} DNS record:</li>\n<ul>\n".format(key))

                    for val in value:
                        txt.write("{}\n".format(val))
                        md.write("  * {}\n".format(val))
                        xml.write("<data>{}</data>\n".format(val))
                        html.write("<li><b>{}</b></li>\n".format(val))

                    html.write("</ul>\n")
                    md.write("</{}>\n".format(key))
                    txt.write("\n")
                    md.write("\n")

            txt.write("\n")
            md.write("\n")
            xml.write("</DNSQueries>\n")
            html.write("</ul>\n")


def ReverseDNSReport(ip, data, output_basename):
    """ Reverse DNS Console report """
    if data:
        print(ip + ":" + data)
    print()

    if output_basename:
        output1 = output_basename + ".txt"
        output2 = output_basename + ".md"
        output3 = output_basename + ".xml"
        output4 = output_basename + ".html"

        with open(output1, 'a') as txt, open(output2, 'a') as md, open(output3, 'a') as xml, open(
                output4, 'a') as html:
            txt.write("[+] Reverse DNS Lookup\n")
            txt.write("----------------------\n")
            md.write("---\n\n")
            md.write("## Reverse DNS Lookup\n\n")
            xml.write("<ReverseDNSLookup>\n")
            html.write("<h3>Reverse DNS Lookup</h3>\n")

            if data:
                txt.write("{}:{}\n".format(ip, data))
                md.write("* {}:{}\n".format(ip, data))
                xml.write("<IP>{}</IP>\n".format(ip))
                xml.write("<Domain>{}</Domain>\n".format(data))
                html.write("<ul><li>{}:{}</li></ul>\n".format(ip, data))

            txt.write("\n")
            md.write("\n")
            xml.write("</ReverseDNSLookup>\n")


def VHostsReport(data, output_basename):
    """ VHosts Console report """
    for host in data:
        print(host)
    print()

    if output_basename:
        output1 = output_basename + ".txt"
        output2 = output_basename + ".md"
        output3 = output_basename + ".xml"
        output4 = output_basename + ".html"

        with open(output1, 'a') as txt, open(output2, 'a') as md, open(output3, 'a') as xml, open(
                output4, 'a') as html:
            txt.write("[+] Bing Virtual Hosts\n")
            txt.write("----------------------\n")
            md.write("---\n\n")
            md.write("## Bing Virtual Hosts\n\n")
            xml.write("<BingVHosts>\n")
            html.write("<h3>Bing Virtual Hosts</h3>\n<ul>\n")

            for host in data:
                txt.write("{}\n".format(host))
                md.write("* {}\n".format(host))
                xml.write("<host>{}</host>\n".format(host))
                html.write("<li>{}</li>\n".format(host))

            txt.write("\n")
            md.write("\n")
            xml.write("</BingVHosts>\n")
            html.write("</ul>\n")


def FinalReport(info, output_basename):
    """ All major formats final report """
    print()
    print("[+] Search engines results - Final Report")
    print("-----------------------------------------")

    if info['all_emails']:
        print()
        print("Emails:")
        print()
        for email in info['all_emails']:
            print(email)

    if info['all_hosts']:
        print()
        print("Hostnames:")
        print()
        for host in info['all_hosts']:
            print(host)

    if info['domains']:
        print()
        print("Subdomains:")
        print()
        for domain in info['domains']:
            print(domain)
        print()

    if output_basename:
        output1 = output_basename + ".txt"
        output2 = output_basename + ".md"
        output3 = output_basename + ".xml"
        output4 = output_basename + ".html"

        with open(output1, 'a') as txt, open(output2, 'a') as md, open(output3, 'a') as xml, open(
                output4, 'a') as html:
            txt.write("[+] Search engines results - Final Report\n")
            txt.write("-----------------------------------------\n")
            md.write("---\n\n")
            md.write("## Search engines results - Final Report\n")
            xml.write("<FinalReport>\n")
            html.write("<h3>Search engines results - Final Report</h3>\n")

            txt.write("\n")
            md.write("\n")

            txt.write("Emails:\n")
            md.write("### Emails\n\n")
            xml.write("<Emails>\n")
            html.write("<h4>Emails</h4>\n<ul>\n")

            for email in info['all_emails']:
                txt.write("{}\n".format(email))
                md.write("* {}\n".format(email))
                xml.write("<email>{}</email>\n".format(email))
                html.write("<li>{}</li>\n".format(email))

            html.write("</ul>\n")
            xml.write("</Emails>\n")
            txt.write("\n")
            md.write("\n")

            txt.write("Hostnames:\n")
            md.write("### Hostnames\n\n")
            xml.write("<Hostnames>\n")
            html.write("<h4>Hostnames</h4>\n<ul>\n")

            for host in info['all_hosts']:
                txt.write("{}\n".format(host))
                md.write("* {}\n".format(host))
                xml.write("<hostname>{}</hostname>\n".format(host))
                html.write("<li>{}</li>\n".format(host))

            html.write("</ul>\n")
            xml.write("</Hostnames>\n")
            txt.write("\n")
            md.write("\n")

            txt.write("Subdomains:\n")
            md.write("### Subdomains\n\n")
            xml.write("<Subdomains>\n")
            html.write("<h4>Subdomains</h4>\n<ul>\n")

            for host in info['domains']:
                txt.write("{}\n".format(host))
                md.write("* {}\n".format(host))
                xml.write("<Subdomain>{}</Subdomain>\n".format(host))
                html.write("<li>{}</li>\n".format(host))

            html.write("</ul>\n")
            xml.write("</Subdomain>\n")
            txt.write("\n")
            md.write("\n")
            xml.write("</FinalReport>\n")

#######################################################


def _get_key(service_name):
    """
    Parses the api keys file and returns the corresponding key
    Doesn't work for censys. Works only for services with one key

    :param service_name: The online service e.g spyse
    :type service_name: :class: `str`
    """
    key = None
    if checkFile():
        with open(KEYS_FILE, 'r') as fin:
            lines = fin.readlines()
            for l in lines:
                if l.lower().startswith(service_name) and ':' in l:
                    try:
                        key = l.strip(' \r\n').split(':')[1]
                    except IndexError:
                        print(KEYS_FILE
                              + " doesnt follow the correct format name:value")

    return key

#######################################################


def MainFunc():
    print(message)

    report_buckets = 50
    info = {
        'all_emails': [],
        'all_hosts': [],
        'domains': [],
        'public': []
    }
    uas = []

    user_agent_strings_file = 'common-ua.txt'
    timeouts = [5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]
    report_buckets = 10
    modes = ['basic', 'nongoogle', 'whois', 'dns', 'revdns', 'vhosts', 'google', 'bing', 'yahoo',
             'ask', 'dogpile', 'yandex', 'linkedin', 'twitter', 'youtube', 'reddit',
             'github', 'instagram', 'crt', 'pgp', 'netcraft', 'virustotal', 'dnsdump', 'shodan',
             'censys', 'spyse']

    parser = argparse.ArgumentParser(formatter_class=RawTextHelpFormatter)
    parser.add_argument("-d", '--domain', action="store", metavar='DOMAIN', dest='domain',
                        default=None, type=CheckDomain, help="Domain to search.")
    parser.add_argument("-s", '--server', action="store", metavar='NAMESERVER', dest='dnsserver',
                        default='8.8.8.8', type=CheckDomainOrIP, help="DNS server to use.")
    parser.add_argument('-x', '--proxy', action="store", metavar='PROXY', dest='proxy',
                        default=None, type=str, help="Use a proxy server when retrieving "
                                                     "results from search engines (eg. "
                                                     "'-x http://127.0.0.1:8080')")
    parser.add_argument("-l", '--limit', action="store", metavar='LIMIT', dest='limit', type=int,
                        default=100,
                        help="Limit the number of search engine results (default: 100).")
    parser.add_argument("-i", '--info', action="store", metavar='MODE', dest='mode', type=str,
                        default='basic',
                        help="Limit information gathering (" + ','.join(modes) + ").")
    parser.add_argument('-o', '--output', action='store', metavar='BASENAME', dest='basename',
                        type=str, default=None, help='Output in the four major formats at once '
                                                     '(markdown, txt, xml and html).')
    parser.add_argument('-k', '--shodan-key', action='store', metavar='API-KEY', dest='shodankey',
                        type=str, default=None,
                        help='API key to use with Shodan search (MODE="shodan")')
    parser.add_argument('-e', '--spyse-key', action='store', metavar='SPYSE_API_KEY',
                        dest='spysekey', type=str, default=None)
    # censys.io
    parser.add_argument('-m', '--match', default=None,
                        help='Highlight a string within an existing query result')
    parser.add_argument('-f', '--filter', default=None,
                        help="Filter the JSON keys to display for each result, use value 'help' "
                             "for interesting fields")
    parser.add_argument('--count', action='store_true', help='Print the count result and exit')
    parser.add_argument('-R', '--report', default=None,
                        help="Stats on given field (use value 'help' for listing interesting "
                             "fields)'")
    parser.add_argument('-B', '--report_bucket', default=report_buckets,
                        help='Bucket len in report mode (default: %s)' % report_buckets)
    # query filter shortcuts - censys.io
    parser.add_argument('-1', '--censys_api_id', action='store', metavar='CENSYS_API_ID',
                        dest='censys_api_id', type=str, default=None,
                        help='Provide the authentication ID for the censys.io search engine')
    parser.add_argument('-2', '--censys_api_secret', action='store', metavar='CENSYS_API_SECRET',
                        dest='censys_api_secret', type=str, default=None,
                        help='Provide the secret hash for the censys.io search engine')
    parser.add_argument('-r', '--read_api_keys', action='store_true',
                        help="Read the API Keys stored in api_keys.txt file. (e.g. '-i censys -r')")
    parser.add_argument('-u', '--update_api_keys', action='store_true',
                        help="Update the API Keys stored in api_keys.txt file. (e.g. '-i censys -u')")
    parser.add_argument('-a', '--asn', metavar='ASN', dest='asn', type=str, default=None,
                        help='Filter with ASN (e.g 5408 for GR-NET AS)')
    parser.add_argument('-c', '--country', metavar='COUNTRY', dest='country', type=str,
                        default=None,
                        help='Filter with country')
    parser.add_argument('-O', '--cert-org', metavar='CERT_ORG', dest='cert_org', type=str,
                        default=None,
                        help='Certificate issued to organization')
    parser.add_argument('-I', '--cert-issuer', metavar='CERT_ISSUER', dest='cert_issuer', type=str,
                        default=None,
                        help='Certificate issued by organization')
    parser.add_argument('-z', '--cert-host', metavar='CERT_HOST', dest='cert_host', type=str,
                        default=None,
                        help='hostname Certificate is issued to')
    parser.add_argument('-S', '--http-server', metavar='HTTP_SERVER', dest='http_server', type=str,
                        default=None,
                        help='Server header')
    parser.add_argument('-t', '--html-title', metavar='HTML_TITLE', dest='html_title', type=str,
                        default=None,
                        help='Filter on html page title')
    parser.add_argument('-b', '--html-body', metavar='HTML_BODY', dest='html_body', type=str,
                        default=None,
                        help='Filter on html body content')
    parser.add_argument('-T', '--tags', default=None,
                        help='Filter on specific tags. e.g: -T tag1,tag2,... (use keyword \'list\''
                             ' to list usual tags')
    parser.add_argument('-L', '--Limit', default=float('inf'), help='Limit to N results')
    parser.add_argument('-D', '--debug', action='store_true', help='Debug information')
    parser.add_argument('-v', '--verbose', action='store_true', help='Print raw JSON records')
    parser.add_argument('-H', '--html', action='store_true',
                        help='Renders html elements in a browser')
    parser.add_argument('arguments', metavar='arguments', nargs='*', help='Censys query')

    args = parser.parse_args()

    match = str(args.match)

    # TODO tags/fields missing:
    # fire help before doing any request
    # if args.tags in ['list', 'help']:
    #     pp.pprint(tags_available)
    #     sys.exit(0)
    # if args.report in ['list', 'help']:
    #     pp.pprint(report_fields)
    #     sys.exit(0)
    # if args.filter in ['list', 'help']:
    #     pp.pprint(filter_fields)
    #     sys.exit(0)

    if args.report_bucket:
        report_buckets = args.report_bucket

    if len(sys.argv) == 1:
        parser.print_help()
        sys.exit()

    # args = parser.parse_args()
    info['domain'] = args.domain
    info['proxies'] = {}

    #######################################################

    # Load User-Agents strings from file #
    if os.path.isfile(user_agent_strings_file):
        uas = [line.strip() for line in open(user_agent_strings_file)]
    else:
        print("[-] An error occured while loading user-agent strings from file")
        sys.exit()

    output_basename = None
    if args.basename is not None:
        output_basename = args.basename

    #######################################################

    # information #
    info['mode'] = [x.strip() for x in args.mode.lower().split(',')]
    info['limit'] = args.limit
    info['dnsserver'] = args.dnsserver
    info['ip'] = VerifyHostname(info['domain'])

    if args.proxy:
        print("[+] Proxy will ONLY be used during search engines searches")
        info['proxies'] = {
            'http': args.proxy,
            'https': args.proxy,
        }

    InfoReport(info['mode'], info['limit'], info['dnsserver'], args.proxy, info['domain'],
               info['ip'], len(uas), output_basename)

    #######################################################

    # Whois query report #
    if any(i in ['whois', 'basic', 'nongoogle'] for i in info['mode']):
        print("[+] Whois lookup")
        print("----------------")
        info['whois'] = WhoisQuery(info['domain'])
        WhoisReport(info['whois'], output_basename)

    #######################################################

    # DNS records report #
    if any(i in ['dns', 'basic', 'nongoogle'] for i in info['mode']):
        print("[+] DNS queries")
        print("---------------")
        info['dns'] = DnsQuery(info['domain'], info['dnsserver'])
        DNSReport(info['dns'], output_basename)

    #######################################################

    # IP Reverse DNS lookup report #
    if any(i in ['revdns', 'basic', 'nongoogle'] for i in info['mode']):
        print("[+] Reverse DNS Lookup")
        print("----------------------")
        info['revdns'] = ReverseIPQuery(info['ip'], info['dnsserver'])
        ReverseDNSReport(info['ip'], info['revdns'], output_basename)

    #######################################################

    # Bing Virtual Hosts search results report ##
    if any(i in ['vhosts', 'basic', 'nongoogle'] for i in info['mode']):
        print("[+] Bing Virtual Hosts")
        print("----------------------")
        info['bingvhosts'] = BingVHostsSearch(info['ip'], info['limit'], uas,
                                              info['proxies'], timeouts)
        VHostsReport(info['bingvhosts'], output_basename)

    #######################################################

    # Google search #
    if any(i in ['google'] for i in info['mode']):
        print("[+] Searching in Google..")
        temp1, temp2 = GoogleSearch(info['domain'], info['limit'], uas, info['proxies'], timeouts)
        info['all_emails'].extend(temp1)
        info['all_hosts'].extend(temp2)
        Report("Google", temp1, temp2, output_basename)

    #######################################################

    # Bing search #
    if any(i in ['bing', 'nongoogle'] for i in info['mode']):
        print("[+] Searching in Bing..")
        temp1, temp2 = BingSearch(info['domain'], info['limit'], uas, info['proxies'], timeouts)
        info['all_emails'].extend(temp1)
        info['all_hosts'].extend(temp2)
        Report("Bing", temp1, temp2, output_basename)

    #######################################################

    # Yahoo search #
    if any(i in ['yahoo', 'nongoogle'] for i in info['mode']):
        print("[+] Searching in Yahoo..")
        temp1, temp2 = YahooSearch(info['domain'], info['limit'], uas, info['proxies'], timeouts)
        info['all_emails'].extend(temp1)
        info['all_hosts'].extend(temp2)
        Report("Yahoo", temp1, temp2, output_basename)

    #######################################################

    # Shodan search #
    if any(i in ['shodan'] for i in info['mode']):

        # pdb.set_trace()

        if args.shodankey is None:
            print("[-] API key required for the Shodan search: '-k API-KEY, --shodan-key API-KEY'")
            sys.exit()
        print("[+] Searching in Shodan..")
        print("-------------------")

        results = ShodanSearch(args.shodankey, info['domain'], info['ip'], uas, info['proxies'],
                               timeouts)
        ShodanReport(results, output_basename)

    #######################################################

    # ASK search #
    if any(i in ['ask', 'nongoogle'] for i in info['mode']):
        print("[+] Searching in ASK..")
        temp1, temp2 = AskSearch(info['domain'], 5, uas, info['proxies'], timeouts)  # 5 pages
        info['all_emails'].extend(temp1)
        info['all_hosts'].extend(temp2)
        Report("ASK", temp1, temp2, output_basename)

    #######################################################

    # Dogpile search #
    if any(i in ['dogpile', 'nongoogle'] for i in info['mode']):
        print("[+] Searching in Dogpile..")
        temp1, temp2 = DogpileSearch(info['domain'], info['limit'], uas, info['proxies'], timeouts)
        info['all_emails'].extend(temp1)
        info['all_hosts'].extend(temp2)
        Report("Dogpile", temp1, temp2, output_basename)

    #######################################################

    # Yandex search #
    if any(i in ['yandex', 'nongoogle'] for i in info['mode']):
        print("[+] Searching in Yandex..")
        temp1, temp2 = YandexSearch(info['domain'], info['limit'], uas, info['proxies'], timeouts)
        info['all_emails'].extend(temp1)
        info['all_hosts'].extend(temp2)
        Report("Yandex", temp1, temp2, output_basename)

    #######################################################

    # crt search #
    if any(i in ['crt', 'nongoogle'] for i in info['mode']):
        print("[+] Searching in Crt..")
        temp = CrtSearch(info['domain'], uas, info['proxies'])
        info['all_hosts'].extend(temp)
        HostnamesReport("CRT", temp, output_basename)

    #######################################################

    # dnsdumpster search #
    if any(i in ['dnsdump', 'nongoogle'] for i in info['mode']):
        print("[+] Searching in DNSdumpster..")
        temp = DNSDumpsterSearch(info['domain'], uas, info['proxies'])
        info['all_hosts'].extend(temp)
        HostnamesReport("DNSdumpster", temp, output_basename)

    #######################################################

    # PGP search #
    if any(i in ['pgp', 'nongoogle'] for i in info['mode']):
        print("[+] Searching in PGP..")
        temp1, temp2 = PGPSearch(info['domain'], uas, info['proxies'])
        info['all_emails'].extend(temp1)
        info['all_hosts'].extend(temp2)
        Report("PGP", temp1, temp2, output_basename)

    #######################################################

    # netcraft search #
    if any(i in ['netcraft', 'nongoogle'] for i in info['mode']):
        print("[+] Searching in Netcraft..")
        temp = NetcraftSearch(info['domain'], uas, info['proxies'])
        info['all_hosts'].extend(temp)
        HostnamesReport("Netcraft", temp, output_basename)

    #######################################################

    # virustotal search #
    if any(i in ['virustotal', 'nongoogle'] for i in info['mode']):
        print("[+] Searching in VirusTotal..")
        temp = VTSearch(info['domain'], uas, info['proxies'])
        info['all_hosts'].extend(temp)
        HostnamesReport("VirusTotal", temp, output_basename)

    #######################################################

    # spyse search #
    if any(i in ['spyse', 'nongoogle'] for i in info['mode']):
        spysekey = args.spysekey or _get_key('spyse')
        if not spysekey:
            print("Api Key for spyse was neither provided in cmd line "
                  "nor located inside %s file" % KEYS_FILE)
        else:
            print("[+] Searching in Spyse..")
            temp = SpyseSearch(info['domain'], spysekey, args.limit, info['proxies'])
            subdomains = SpyseGetDomains(temp)
            info['all_hosts'].extend(subdomains)
            SubdomainsReport("SpyseSearch", subdomains, output_basename)

    # LinkedIn search #
    if any(i in ['linkedin'] for i in info['mode']):
        print("[+] Searching in LinkedIn..")
        temp1, temp2 = GoogleSearchEngine(info['domain'], 'linkedin.com', info['limit'], uas,
                                          info['proxies'], timeouts)
        info['all_emails'].extend(temp1)
        info['all_hosts'].extend(temp2)
        Report("LinkedIn", temp1, temp2, output_basename)

    #######################################################

    # Twitter search #
    if any(i in ['twitter'] for i in info['mode']):
        print("[+] Searching in Twitter..")
        temp1, temp2 = GoogleSearchEngine(info['domain'], "twitter.com", info['limit'], uas,
                                          info['proxies'], timeouts)
        info['all_emails'].extend(temp1)
        info['all_hosts'].extend(temp2)
        Report("Twitter", temp1, temp2, output_basename)

    #######################################################

    # Youtube search #
    if any(i in ['youtube'] for i in info['mode']):
        print("[+] Searching in Youtube..")
        temp1, temp2 = GoogleSearchEngine(info['domain'], "youtube.com", info['limit'], uas,
                                          info['proxies'], timeouts)
        info['all_emails'].extend(temp1)
        info['all_hosts'].extend(temp2)
        Report("Youtube", temp1, temp2, output_basename)

    #######################################################

    # Reddit search #
    if any(i in ['reddit'] for i in info['mode']):
        print("[+] Searching in Reddit..")
        temp1, temp2 = GoogleSearchEngine(info['domain'], "reddit.com", info['limit'], uas,
                                          info['proxies'], timeouts)
        info['all_emails'].extend(temp1)
        info['all_hosts'].extend(temp2)
        Report("Reddit", temp1, temp2, output_basename)

    #######################################################

    # Github search #
    if any(i in ['github'] for i in info['mode']):
        print("[+] Searching in Github..")
        temp1, temp2 = GoogleSearchEngine(info['domain'], "github.com", info['limit'], uas,
                                          info['proxies'], timeouts)
        info['all_emails'].extend(temp1)
        info['all_hosts'].extend(temp2)
        Report("Github", temp1, temp2, output_basename)

    #######################################################

    # Instagram search #
    if any(i in ['instagram'] for i in info['mode']):
        print("[+] Searching in Instagram..")
        temp1, temp2 = GoogleSearchEngine(info['domain'], "instagram.com", info['limit'], uas,
                                          info['proxies'],
                                          timeouts)
        info['all_emails'].extend(temp1)
        info['all_hosts'].extend(temp2)
        Report("Instagram", temp1, temp2, output_basename)

    #######################################################

    # Censys.io search #
    if any(i in ['censys'] for i in info['mode']):
        api_id = args.censys_api_id
        api_secret = args.censys_api_secret
        if api_id is not None and api_secret is not None:
            print("[+] Searching in Censys.io..\n")
            res1 = DomainSearchCensys(info['domain'], api_id, api_secret,
                                      output_basename, info['domains'])
            res2 = CensysPublicScan(api_id, api_secret, output_basename, args,
                                    report_buckets, match, info['public'])
            if res1 == False and res2 == False:
                print(
                    "Please use the available censys.io options in order to perform scanning. "
                    "For more information use the '--help' option")
            print()
        else:
            chkstored = checkFile()
            flag = 0
            if chkstored == False:
                chkanswer = input(
                    "[!] API Keys not provided. Would you like to store your API keys ? [y/n]: ")
                if chkanswer == 'y':
                    stored = createFileAndStoreAPIKeys('censys')
                    if stored == "stored":
                        print()
                        readFileContents()
                        print()
                        answer1 = input(
                            "[*] would you like to continue searching with censys.io ? [y/n] ")
                        print()
                        if answer1 == 'n':
                            flag = 1
                            print("[*] Exiting...")
                            exit(0)
                        if answer1 == 'y':
                            with open(KEYS_FILE) as f:
                                lines = f.read().splitlines()
                                print("[+] Searching in Censys.io..")
                                print()
                                for line in lines:
                                    res1 = DomainSearchCensys(
                                        info['domain'], line.split(":")[1], line.split(":")[2],
                                        output_basename, info['domains'])
                                    res2 = CensysPublicScan(
                                        line.split(":")[1], line.split(":")[2], output_basename,
                                        args, report_buckets, match, info['public'])
                                    if res1 == False and res2 == False:
                                        print('Please use the available censys.io options in order'
                                              'to perform scanning. For more information use the '
                                              '\'--help\' option')
                                    print()
                                    flag = 1
                    else:
                        print("[x] API keys has not been stored..")
                        print("[*] Exiting...")
                        exit(0)
                if chkanswer == 'n':
                    print(
                        "[!] Please provide the API keys in the command line to continue searching")
                    print("[*] Exiting....")
                    exit(0)

            if chkstored is True:
                if args.update_api_keys is True and args.mode == 'censys' and flag != 1:
                    keysupdate = updateAPIKeys('censys')
                    if keysupdate == 'n':
                        print("[x] the keys have not been updated")
                        print("[*] Exiting...")
                        exit(0)
                        print()
                    else:
                        print("[!] the keys have been successfully updated!")
                        answer1 = input(
                            "[*] would you like to continue searching with censys.io ? [y/n] ")
                        print()
                        if answer1 == 'y':
                            with open(KEYS_FILE) as f:
                                lines = f.read().splitlines()
                                print("[+] Searching in Censys.io..")
                                print()
                                for line in lines:
                                    res1 = DomainSearchCensys(
                                        info['domain'], line.split(":")[1], line.split(":")[2],
                                        output_basename, info['domains'])
                                    res2 = CensysPublicScan(
                                        line.split(":")[1], line.split(":")[2], output_basename,
                                        args, report_buckets, match, info['public'])
                                    if res1 == False and res2 == False:
                                        print("Please use the available censys.io options in order"
                                              " to perform scanning. For more information use "
                                              " the '--help' option\n")
                        else:
                            print("[*] Exiting...")
                            exit(0)
                else:
                    if args.read_api_keys is not True:
                        with open(KEYS_FILE) as f:
                            lines = f.read().splitlines()
                            print("[+] Searching in Censys.io..")
                            print()
                            for line in lines:
                                res1 = DomainSearchCensys(
                                    info['domain'], line.split(":")[1], line.split(":")[2],
                                    output_basename, info['domains'])
                                res2 = CensysPublicScan(
                                    line.split(":")[1], line.split(":")[2], output_basename, args,
                                    report_buckets, match, info['public'])
                                if res1 == False and res2 == False:
                                    print("Please use the available censys.io options in order to "
                                          "perform scanning. For more information use the '--help'"
                                          " option\n")

                if args.read_api_keys is True and args.mode == 'censys' and flag != 1:
                    print()
                    readFileContents()
                    print()
                    answer1 = input(
                        "[*] would you like to continue searching with censys.io ? [y/n] ")
                    print()
                    if answer1 == 'y':
                        with open(KEYS_FILE) as f:
                            lines = f.read().splitlines()
                            print("[+] Searching in Censys.io..")
                            print()
                            for line in lines:
                                res1 = DomainSearchCensys(
                                    info['domain'], line.split(":")[1], line.split(":")[2],
                                    output_basename, info['domains'])
                                res2 = CensysPublicScan(
                                    line.split(":")[1], line.split(":")[2], output_basename, args,
                                    report_buckets, match, info['public'])
                                if res1 is False and res2 is False:
                                    print(
                                        "Please use the available censys.io options in order to "
                                        "perform scanning. For more information use the "
                                        "'--help' option\n")
                    else:
                        print("[*] Exiting...")
                        exit(0)

    #######################################################

    # Search Results Final Report #
    info['all_emails'] = sorted(set(info['all_emails']))
    info['all_hosts'] = sorted(set(info['all_hosts']))
    info['domains'] = sorted(set(info['domains']))
    FinalReport(info, output_basename)

    #######################################################

    # Close tags for xml and html #
    if output_basename:
        output = output_basename + ".xml"
        output1 = output_basename + ".html"

        with open(output, 'a') as xml, open(output1, 'a') as html:
            xml.write("</report>\n")
            html.write("</body></html>\n")


#######################################################

if __name__ == '__main__':

    try:
        MainFunc()
    except KeyboardInterrupt:
        print("Search interrupted by user..")
    except Exception as e:
        if DEBUG:
            print(e)
        sys.exit()

#######################################################
