import networkx as nx
import pandas as pd
import matplotlib.pyplot as plt


def addEdges(edges,df,key1,key2,filterSet = None):
    def addEdge(data):
        if filterSet is None or data[key1] in filterSet or data[key2] in filterSet:
            edges.append((data[key1],data[key2]))
    df.apply(addEdge,axis=1)


G  = nx.read_adjlist('levenshtein_first.txt')
df = pd.read_csv('first_url.csv')
t = df[['sha1','top']].drop_duplicates().groupby('sha1',as_index=False).count()
shaSet = set(t[t.top==1].sha1)
df = df[df.sha1.map(lambda x:x in shaSet)]

hostSet = set(G.nodes())
edges = []
addEdges(edges,df,'sha1','clean_url',hostSet)
G.add_edges_from(edges)

index = 0
totalSha =0
outf = open('graph.txt','a+')
for largest_components in sorted(nx.connected_components(G), key=len, reverse=False)[:-1]:
    g = G.subgraph(largest_components)
    nodeSet = set(g.nodes) & shaSet
    hostSet = set(g.nodes) - shaSet
    totalSha += len(nodeSet)
    if len(nodeSet) > 1:
        index += 1
        for node in nodeSet:
        #    print("%s,first_domain_%d\n" % (node, index))
            outf.write("%s,first_domain_%d\n" % (node, index))

print(len(nodeSet))
print(totalSha)

# node_colors =  []
# labels = {}
# index = 1
# for node in g.nodes:
#     if node in shaSet:
#         node_colors.append('g')
#         labels[node] = index
#         index+=1
#         print(index,node)
#     else:
#         node_colors.append('r')
#         labels[node] = 0

# nx.draw_networkx(g,labels=labels,node_color=node_colors)
# plt.show()

