#This file is meant to clean the URLs generated by MMGAN Algorithm.
import random
def separate_url(URL):
    flag = 0
    protocol = URL[0:6]
    #print protocol
    remainingURL = URL[7:]
    #print remainingURL
    split = remainingURL.split('/')
    #print split
    try:
        host = split[0]
        if host=='':
            flag=1
            raise
        else:
            #print "All OK"
            flag=0

    except:
        print "No host...Do not add this URL"

    return flag

#
with open('EbayThreshold5NP.txt','r') as fileptr:
    URLs = fileptr.readlines()
    f = open('EbayThreshold5NP_Cleaned.txt','a')
    for i in range(len(URLs)):
        flag = separate_url(URLs[i])
        if flag==0:
            #Add this
            if len(URLs[i]) > 7:
                f.writelines(URLs[i])

# dataSampleFile = open('randomDataSamples.txt','a')
# for i in range(0,7000):
#     randomLines = random.choice(open('grepOutput.txt','r').readlines())
#     dataSampleFile.writelines(randomLines)

# Append Random Data samples to the Outresponse CSV File



