# Downloads pages, computes total time

import csv
import glob
import os
import os.path
import shlex
import socket
import subprocess
import sys
import time
import urllib2

def load_page_file(file_name):
    """Loads list of page files from uri"""
    list_file = open(file_name, 'r')
    reader = csv.reader(list_file)
    site_list = []
    for line in reader:
        if len(line) == 0:
            continue
        site = line[0]
        site_list.append(site.strip())
    list_file.close()
    return site_list

def time_page(url):
    """Load a single page and measure the time for that URL"""
    # Start recording time
    print "Loading %s"%url

    start = time.time()
    raw = "curl -silent -m 10 %s >> data_dump 2>&1"%url
    command = shlex.split(raw)
    proc = subprocess.Popen(raw, shell=True)
    proc.wait()
    stop = time.time()


    total_time = stop - start
    return total_time

def measure_list_total(url_list):
    """Loop over a list, load each, sum times """

    total_time = 0
    page_time_list = []
    for page in url_list:
        page_time = time_page(page)
        total_time += page_time
        page_time_list.append(page_time)

    print total_time
    return page_time_list

def measure_type(url_list):
    total_direct = 0
    total_cdn = 0
    type_list = []

    for url in url_list:
        page_type = determine_page_type(url)
        type_list.append(page_type)
        if page_type == "direct":
            total_direct += 1
        elif page_type == "cdn":
            total_cdn += 1
    
    print "CDN: %d"%total_cdn
    print "Direct: %d"%total_direct
    
    return type_list

def determine_page_type(url):
    """Determine if a page is CDN or not"""
    # First, lets clean the url:
    if url.startswith("https"):
        c_url = url[8:]
    else:
        c_url = url[7:]
    host = c_url.split('/')[0]
    #print "[%s] ..."%host
    addr = socket.gethostbyname(host)
    try:
        rev_name = socket.gethostbyaddr(addr)
    except:
        return "direct"

    cdn_id = ['akamai', 'edgecast', 'cdn']
    #print url, rev_name[0]
    for cdn in cdn_id:
        if cdn in rev_name[0]:
            return "cdn"
    return "direct"
   
def save_total_list(total_list, original_name, save_directory):
    base_name = os.path.basename(original_name)
    new_name = base_name.split('.')[0] + "-result.csv"
    if not os.path.isdir(save_directory):
        os.mkdir(save_directory)
    with open(save_directory + "/" + new_name, 'w') as res_file:
        writer = csv.writer(res_file)
        for line in total_list:
            writer.writerow(line)

    print "Done!"

def process_dir(directory):
    """Get data and stuff from each site in the directory """
    contents = glob.glob(directory + "/*.csv")
    for list_file in contents:
        site_list = load_page_file(list_file)
        time_list = measure_list_total(site_list)
        type_list = measure_type(site_list)
        total_list = zip(site_list, time_list, type_list)
        save_total_list(total_list, list_file, directory+"_results") 

if __name__ == "__main__":
    #list_file = sys.argv[1]
    #site_list = load_page_file(list_file) 
    #time_list = measure_list_total(site_list)
    #type_list = measure_type(site_list)

    #total_list = zip(site_list, time_list, type_list)
    #for item in total_list:
    #    print item
    #save_total_list(total_list, sys.argv[1])
    process_dir(sys.argv[1])

