#!/usr/bin/env python
import sys
from collections import Counter

if __name__ == '__main__':
    # Assumes a tokenized file
    char_len = 0
    word_lens = []
    words = []
    longest_len = -1
    shortest_len = 1000000
    with open(sys.argv[1], 'r') as f:
        for line in f:
            line = line.strip()
            char_len += len(line)
            wrds = line.split(' ')
            words.extend(wrds)
            sent_len = len(wrds)
            word_lens.append(sent_len)
            if sent_len > longest_len:
                longest_len = sent_len
            elif sent_len < shortest_len:
                shortest_len = sent_len

    n_sents = len(word_lens)

    print('Number of sentences: %d' % n_sents)
    print('Longest sentence length: %d' % longest_len)
    print('Shortest sentence length: %d' % shortest_len)
    print('Average # of words: %d' % (sum(word_lens) // n_sents))
    print('Average # of chars: %d' % (char_len // n_sents))
    
    histogram = Counter(word_lens).most_common()
    w_hist = Counter(words).most_common(20)
    print(w_hist)
    cumul = 0.
    for l, cnt in histogram:
        percent = 100 * (cnt / n_sents)
        cumul += percent
        print('%3d-word => %3.3f%% (%4d sentences)\t(Cumulative coverage: %3.3f%%)' % (l, percent, cnt, cumul))
        if cumul > 90.:
            break


