import os

#Each website we crawl is in separate folder
def create_project_dir(directory):
	if not os.path.exists(directory):
		print('creating project '+ directory)
		os.makedirs(directory)

#create  queue and crawl file
def create_data_files(project_name, base_url):
	queue = project_name + '/queue.txt'
	crawled = project_name + '/crawled.txt'
	if not os.path.isfile(queue):
		write_file(queue, base_url)
	if not os.path.isfile(crawled):
		write_file(crawled, '')

#create and write in a file
def write_file(path,data):
	f = open(path,'w')
	f.write(data)
	f.close()

#Append data onto existing file
def append_to_file(path, data):
	with open(path, 'a') as file:
		file.write(data + '\n')

#delete the contents of a file
def delete_file_contents(path):
	with open(path, 'w'):
		pass #overwrite the existing file

# Read a file and make a set of items
def file_to_set(file_name):
	results = set()
	with open(file_name, 'rt') as f:
		for line in f:
			results.add(line.replace('\n', ''))
	return results

# convert a set to file
def set_to_file(links,file):
	delete_file_contents(file)
	for link in sorted(links):
		append_to_file(file,link)
	


#TESTING
#create_project_dir('new')
#create_data_files('new','https://github.com/buckyroberts/Spider')