# -*- coding: utf-8 -*-

import codecs
import os

#1. 读取文件
#['aa', 'aaa-bbb-sds'] => ['aa', 'aaa', 'bbb', 'sds']
def word_split(words):
    new_list = []
    for word in words:
        if '-' not in word:
            new_list.append(word)
        else:
            lst = word.split('-')
            new_list.extend(lst)
    return new_list


def read_file(file_path):
    f = codecs.open(file_path, 'r', "utf-8") #打开文件
    lines = f.readlines()
    word_list = []
    for line in lines:
        line = line.strip()
        words = line.split(" ") #用空格分割
        words = word_split(words) #用-分割
        word_list.extend(words)
    return word_list

def get_file_from_folder(folder_path):
    file_paths = []
    for root, dirs, files in os.walk(folder_path):
        for file in files:
            file_path = os.path.join(root, file)
            file_paths.append(file_path)
    return file_paths

#读取多文件里的单词
def read_files(file_paths):
    final_words = []
    for path in file_paths:
        final_words.extend(read_file(path))
    return final_words


#2.获取格式化之后的单词
def format_word(word):
    fmt = 'abcdefghijklmnopqrstuvwxyz-'
    for char in word:
        if char not in fmt:
            word = word.replace(char, '')
    return word.lower()

def format_words(words):
    word_list = []
    for word in words:
        wd = format_word(word)
        if wd:
            word_list.append(wd)
    return word_list

#3. 统计单词数目
# {'aa':4, 'bb':1}
def statictcs_words(words):
    s_word_dict = {}
    for word in words:
        if s_word_dict.has_key(word):
            s_word_dict[word] = s_word_dict[word] + 1
        else:
            s_word_dict[word] = 1
    #排序
    sorted_dict = sorted(s_word_dict.iteritems(), key=lambda d: d[1], reverse=True)
    return sorted_dict

# 3.2、为单词添加所占比例
def add_rate(volcaulay_list,total_count):
    current_count=0
    word_add_rate=[]
    for val in volcaulay_list:
        tem=[]
        num=val[1]
        current_count = current_count + num
        word_rate = (float(current_count) / total_count) * 100
        tem.append(val)
        tem.append(word_rate)
        word_add_rate.append(tem)
    return word_add_rate

# 3.3 截取固定比例
def intercept_word(word_list,start_and_end):
    intercepted_word=[]
    for val in word_list:
        if val[1] > start_and_end[0]:
            if val[1] < start_and_end[1]:
                intercepted_word.append(val)
    return intercepted_word
#4.输出成csv
def print_to_csv(volcaulay_list, to_file_path):
    nfile = open(to_file_path,'w+')
    for val in volcaulay_list:
        nfile.write("%s,%s,%0.2f\n" % (val[0][0], str(val[0][1]),val[1]))
    nfile.close()



def main():

    #设置是否截取单词
    is_intercept=True

    #1. 读取文本
    words = read_files(get_file_from_folder('data1'))
    print '获取了未格式化的单词 %d 个' % (len(words))

    #2. 清洗文本
    f_words = format_words(words)
    total_word_count = len(f_words)
    print '获取了已经格式化的单词 %d 个' %(len(f_words))
    
    #3. 统计单词和排序
    word_list = statictcs_words(f_words)

    # 3.2为单词添加所占比例
    word_list_rate = add_rate(word_list, total_word_count)

    # 3.3 截取固定比例
    start_and_end = [50, 70] #截取这一部分的单词
    if is_intercept:
        intercepted_list=intercept_word(word_list_rate,start_and_end)
    else:
        intercepted_list=word_list_rate


    #4. 输出文件
    print_to_csv(intercepted_list, 'output/test1.csv')


if __name__ == "__main__":
    main()