__author__ = 'Kunal'
'''
This script generates a text file (in tab separated format) with columns containing wired.com article's text,
  article link and article id.
'''

# -------------------------- Constants Section Begin -------------------------------------#

# File handler for data file generated by Kimono API
input_file = open('C:\Users\Owner\Downloads\data_tab.txt', 'r')

# File handler for the intermediate data file.
# intermediate_file = open('C:\Users\Owner\Downloads\\intermediate_crawl_file.txt', 'w')

# Article metadata file
initial_data_file = open('C:\Users\Owner\Desktop\Data\\article_metadata_wired.txt', 'r')  # File to get article ids

# File handler for the final data file.
crawl_file = open('C:\Users\Owner\Downloads\\final_crawl_file.txt', 'w')

# Declare a intermediate map to store all the crawl for 5000 documents in the form of key-value pairs
# where key is url of article and value is text body of article.
intermediate_crawl_map = {}

# Final map to store the crawled text. Key is article_id, and value for a key is a tuple (a,b) where
# a is the url of article and b is the text body crawled.
final_crawl_map = {}


# -------------------------- Constants Section End ---------------------------------------#


# -------------------------- Functions Section Begin -------------------------------------#

'''
Input: None
Output: Generated the intermediate tab delimited text file and its map containing the wired.com crawls
'''


def generate_intermediate_crawl_map():
    for line in input_file:
        line = line.strip()  # Remove trailing newline
        line = line.split('\t')  # Split the two columns
        if len(line) == 2:  # Make sure there are two columns
            text = line[0]  # First column is text if the crawled article
            url = line[1]  # Second column is url of crawled article
            if url not in intermediate_crawl_map:
                intermediate_crawl_map[
                    url] = ""  # Init the starting string as empty one. Then we append all text rows one by one
                intermediate_crawl_map[url] = intermediate_crawl_map[
                                                  url] + text  # Update the text value for given url. Keep appending new text rows
            else:  # The crawl link is in map, so just append the text row to already existing text value.
                intermediate_crawl_map[url] = intermediate_crawl_map[
                                                  url] + text  # Update the text value for given url. Keep appending new text rows

    print ("The size of crawl map is: " + str(len(intermediate_crawl_map)))

    '''
    # Done reading the file and creating the map of crawled text. Now write it to a text file as url, text format. 
    for key in intermediate_crawl_map:
        text = intermediate_crawl_map[key]
        intermediate_file.write(str(key) + "\t" + str(text) + "\n")

    # Close output file handlers
    input_file.close()
    intermediate_file.close()
    '''
    return intermediate_crawl_map


'''
Input: None
Output: Final text tab delimited file in the format of article_id, url, text, length of text format
'''


def generate_crawl_file():
    intermediate_map = generate_intermediate_crawl_map()
    for line in initial_data_file:
        line = line.strip()  # Remove trailing newline
        line = line.split('\t')  # Split the columns
        article_id = line[0]  # First column of read file is the article id
        article_link = line[-1]  # Last column of read file the article link
        if article_link in intermediate_map:
            article_text = intermediate_map[article_link]
            link_text_tuple = (article_link, article_text)
            final_crawl_map[article_id] = link_text_tuple

    # Finish populating the map, now write it to final file.
    for key in final_crawl_map:
        id = key
        value = final_crawl_map[key]
        url = value[0]
        text = value[1]
        length = len(text.split(' '))
        # Write to the file
        crawl_file.write(id + "\t" + "\t" + str(length) + "\t" + url + "\t" + text + "\n")

    # Close the file handlers
    initial_data_file.close()
    crawl_file.close()

    return


'''
Input: None
Output: Required formatted text crawl file
'''


def main():
    generate_crawl_file()


# -------------------------- Functions Section End ---------------------------------------#


if __name__ == "__main__":
    main()
