from datetime import datetime
import random
import requests
import bs4
from urllib.parse import urldefrag, urljoin, urlparse
import hashlib

root_url= "http://www.securedmz.com"
index_url=root_url + '/'
domain="www.securedmz.com"

proxy = {'http': 'http://127.0.0.1:8118',
      'https': 'http://127.0.0.1:8118',
      }
filepath = 'agents.txt'
file_content = open(filepath,encoding='utf-8',errors='ignore')
file_content_parsed = file_content.read().split("\n")
header_index = random.randint(0,len(file_content_parsed)-2)
useragent={'User-Agent':file_content_parsed[header_index]}


def get_page_urls():
	response= requests.get(index_url)
	soup = bs4.BeautifulSoup(response.text,"html.parser")
	links = [a.attrs.get('href') for a in soup.select('a[href]')]
	links = [urldefrag(link)[0] for link in links]
	links = [link for link in links if link]
	links = [link if bool(urlparse(link).netloc) else urljoin(index_url, link) \
		for link in links]
	if domain:
		links = [link for link in links if urlparse(link).netloc == domain]
	return links


def hash_generator(link):
	byte_array_link=link.encode('utf-8',errors="ignore")
	hashed_link=hashlib.md5(byte_array_link)
	return hashed_link

def crawler(link):
	sess = requests.session()
	response = sess.get(link,proxies=proxy,headers=useragent)
	return len(response.text)	

def storing_data(link,hash_link,length):
	with open("data.txt",'a') as out:
		out.write(link+" "+hash_link+" "+str(length)+" "+str(datetime.now())+'\n')

def main():
	a=get_page_urls()
	for link in a:
		hashed_link=hash_generator(link)
		length=crawler(link)
		#print(link,hashed_link.hexdigest(),length)
		storing_data(link, hashed_link.hexdigest() , length)	
		
if __name__=="__main__":
	main()
