# count_words.py

"""
Let's write a python module to analyze a given text file containing a book for its vocabulary frequency and display the most frequent words to the user in the terminal. Remember there isn't any "perfect" way to identify a word in english (acronymns, mr/ms, contractions, etc), try to find rules that work best.

Find a book on Project Gutenberg and navigate to the plain-text version. You can then send a request to that url using the requests library to get the text into Python. You can also save the file into the same folder as the .py file and open it using with.

import requests
response = requests.get('https://www.gutenberg.org/files/62897/62897-0.txt')
response.encoding = 'utf-8' # set encoding to utf-8
print(response.text)

We can also download a file of english words and place it next our .py file and load it like so:

with open('the_raven.txt', 'r', encoding='utf-8') as file:
    text = file.read()
print(text)

Take the following steps to build up our dictionary. The result should look something like {'a': 3, 'the': 4}

    Make everything lowercase, strip punctuation, split into a list of words.
    Your dictionary will have words as keys and counts as values. If a word isn't in your dictionary yet, add it with a count of 1. If it is, increment its count.
    Print the most frequent top 10 out with their counts. You can do that with the code below.

# word_dict is a dictionary where the key is the word and the value is the count
words = list(word_dict.items()) # .items() returns a list of tuples
words.sort(key=lambda tup: tup[1], reverse=True)  # sort largest to smallest, based on count
for i in range(min(10, len(words))):  # print the top 10 words, or all of them, whichever is smaller
    print(words[i])


"""
import requests
import re
import os.path


def getFile():
    response = requests.get("https://www.gutenberg.org/files/62897/62897-0.txt")
    response.encoding = "utf-8"  # set encoding to utf-8
    with open("words", "w") as file:
        for line in response.text:
            file.write(line)


def sortAndCount(d):
    words = list(d.items())
    words.sort(key=lambda tup: tup[1], reverse=True)
    return words


def genWordDict(fname):
    d = dict()

    with open(fname, "r", encoding="utf-8") as file:
        for line in file:
            words = re.sub("[^\w ]", "", line).strip().lower().split(" ")
            for w in words:
                if w in d:
                    d[w] += 1
                else:
                    d[w] = 1
    return d


def main():
    fname = "words"
    if not os.path.isfile(fname):
        getFile()
    d = genWordDict(fname)
    words = sortAndCount(d)
    print("The top ten words are: \n")
    for i in range(min(10, len(words))):
        print(words[i][0], ":", words[i][1])


main()