from bs4 import BeautifulSoup
import requests
import re


def main():
    URL = "https://www.malware-traffic-analysis.net/2017/"
    index = "index.html"
    r = requests.get(URL + index)
    if r.status_code == 200:
        soup = BeautifulSoup(r.content, 'lxml')
        content = soup.find('div', attrs = {'class':'content'})
        for ul in content.findAll('ul'):
            for li in ul.findAll('li'):
                anchors = list(li.findAll('a'))
                anchors = list(map(lambda x: (URL + x['href'], x.string), anchors))
                download_zip(anchors)


def download_zip(anchors):
    res = requests.get(anchors[0][0])
    if res.status_code == 200:
        soup = BeautifulSoup(res.content, 'lxml')
        try:
            tag = soup.find('div', attrs = {'class':'blog_entry'}).find(text = "ASSOCIATED FILES:").findNext('ul').findAll('li')[-1].find('a', attrs = {'class':'menu_link'})
            if tag:
                if not any(x in tag.string for x in ["IOC", "example", "pcap", "eml"]) and re.search('ransomware', tag.string, re.IGNORECASE):
                    link = re.sub(r"index[0-9]*\.html", tag['href'], anchors[0][0])
                    text = tag.string
                    data = requests.get(link)
                    with open("zip/ransomware/" + text, 'wb') as f:
                        f.write(data.content)
                    print(link + "\t" + text)
            else:
                print("No anchor tag for %s" % anchors[0][0])
        except Exception as err:
            print(str(err) + "\t" + anchors[0][0])


if __name__ == "__main__":
    main()
