#!/usr/bin/env python

import sys ,string, os, re
from urlparse import urlparse, parse_qsl, urljoin
from optparse import OptionParser

_usage = "Usage: %prog -f FILE [option]"

parser = OptionParser(usage = _usage)
parser.add_option("-f", "--file", dest="file", type="string", action="store", 
                  help="path to log file")
parser.add_option("-d", "--delimiter", dest="delimiter", type="string", action="store", default=" ",
                  help="field delimiter(default ' ')")
parser.add_option("-r", "--reverse", dest="reverse", action="store_true", default=False,
                  help="reverse the result of comparisons")
parser.add_option("--sort-min", dest="sort_min", action="store_true", default=False,
                  help="sort by min")
parser.add_option("--sort-max", dest="sort_max", action="store_true", default=False,
                  help="sort by max")
parser.add_option("--sort-avg", dest="sort_avg", action="store_true", default=False,
                  help="sort by avg")
parser.add_option("--sort-method", dest="sort_method", action="store_true", default=False,
                  help="sort by method")
parser.add_option("--no-headers", dest="header", action="store_false", default=True,
                  help="print no header line at all")

(options, _) = parser.parse_args()

if options.file is None:
    parser.print_help()
    exit(0)

if not os.path.exists(options.file):
    exit("File does not exist: '%s'" % options.file)

if options.sort_max:
    sort_key = "max"
elif options.sort_min:
    sort_key = "min"
elif options.sort_avg:
    sort_key = "avg"
elif options.sort_method:
    sort_key = "method"
else:
    sort_key = "count"

uri_data = {}

def gen_uri(uri):
    list = []
    for query in parse_qsl(urlparse(uri).query):
        list.append(query[0] + "=" + "xxx")

    list.sort()

    if len(list) == 0:
        return urlparse(uri).path
    else:
        return urlparse(uri).path + "?" + "&".join(list)

def gen_key(uri, method):
    return method + "_" + uri
    
def count(key, uri):
    if not uri_data.has_key(key):
        uri_data[key] = {"count": 0, "min": None, "max": 0, "avg": 0, "method": method, "uri": uri}

    uri_data[key]["count"] += 1

def calc(key, res_time):
    try:
        res_time = float(res_time)
    except ValueError as e:
        res_time = 0

    if uri_data[key]["max"] < res_time:
        uri_data[key]["max"] = res_time
        
    if uri_data[key]["min"] >= res_time or uri_data[key]["min"] is None:
        uri_data[key]["min"] = res_time

    uri_data[key]["avg"] += res_time

is_calc = False

f = open(options.file)
flush_counter = 0
for line in f:
    lines = re.split(options.delimiter, string.strip(line))
    if lines[0] == "":
        continue

    if len(lines) == 3:
        method = lines[2]
    else:
        method = "GET"
        
    uri = gen_uri(string.strip(lines[0]))
    key = gen_key(uri, method)
    
    count(key, uri)

    if len(lines) > 1:
        is_calc = True
        calc(key, lines[1])

    flush_counter += 1
    if flush_counter >= 1000:
        f.flush()
        flush_counter = 0
    
    if len(uri_data) >= 5000:
        print "Too many uri(5000 or less)"
        sys.exit(1)

f.close()

if options.header:
    sys.stdout.write("Line of text: method, count")

    if is_calc:
        sys.stdout.write(", min")
        sys.stdout.write(", max")
        sys.stdout.write(", avg")
    
    sys.stdout.write(", uri")

    print
    print

if is_calc:
    for key in uri_data.keys():
        uri_data[key]["avg"] = uri_data[key]["avg"] / uri_data[key]["count"]
    
for data in sorted(uri_data.items(),key=lambda x:x[1][sort_key], reverse=options.reverse):
    output_line = data[1]["method"] + "\t" + str(data[1]["count"])

    if is_calc:
        try:
            output_line += ("\t{0:.3f}".format(data[1]["min"]) +
                            "\t{0:.3f}".format(data[1]["max"]) +
                            "\t{0:.3f}".format(data[1]["avg"]))
        except ValueError as e:
            continue

    sys.stdout.write(output_line + "\t" + data[1]["uri"])

    print
