# -*- encoding: utf-8 -*-

import pandas as pd
from bs4 import BeautifulSoup
from nltk.corpus import stopwords
import re
import string

dataframes = {
    "cooking": pd.read_csv("../data/cooking.csv"),
    "crypto": pd.read_csv("../data/crypto.csv"),
    "robotics": pd.read_csv("../data/robotics.csv"),
    "biology": pd.read_csv("../data/biology.csv"),
    "travel": pd.read_csv("../data/travel.csv"),
    "diy": pd.read_csv("../data/diy.csv"),
    "test": pd.read_csv("../data/test.csv")
}
uri_re = r'(?i)\b((?:https?://|www\d{0,3}[.]|[a-z0-9.\-]+[.][a-z]{2,4}/)(?:[^\s()<>]+|\(([^\s()<>]+|(\([^\s()<>]+\)))*\))+(?:\(([^\s()<>]+|(\([^\s()<>]+\)))*\)|[^\s`!()\[\]{};:\'".,<>?«»“”‘’]))'

def stripTagsAndUris(x):
    if x:
        # BeautifulSoup on content
        soup = BeautifulSoup(x, "html.parser")
        # Stripping all <code> tags with their content if any
        if soup.code:
            soup.code.decompose()
        # Get all the text out of the html
        text =  soup.get_text()
        # Returning text stripping out all uris
        return re.sub(uri_re, "", text)
    else:
        return ""

for df in dataframes.values():
    df["content"] = df["content"].map(stripTagsAndUris)

def removePunctuation(x):
    # Lowercasing all words
#    x = x.lower()
    # Removing non ASCII chars
    x = re.sub(r'[^\x00-\x7f]',r' ',x)
    # Removing number
    x = re.sub('[0-9]',' ',x)
    # Removing (replacing with empty spaces actually) all the punctuations
    return re.sub("["+string.punctuation+"]", " ", x)

for df in dataframes.values():
    df["title"] = df["title"].map(removePunctuation)
    df["content"] = df["content"].map(removePunctuation)

stops = set(stopwords.words("english"))
def removeStopwords(x):
    # Removing all the stopwords
    filtered_words = [word for word in x.split() if word not in stops]
    return " ".join(filtered_words)

#for df in dataframes.values():
#    df["title"] = df["title"].map(removeStopwords)
#    df["content"] = df["content"].map(removeStopwords)

#for df in dataframes.values():
#    df["title"] = df["title"].map(removeStopwords)
#    df["content"] = df["content"].map(removeStopwords)


#for df in dataframes.values():
    # From a string sequence of tags to a list of tags
#    df["tags"] = df["tags"].map(lambda x: x.replace)        

for name, df in dataframes.items():
    # Saving to file
    df.to_csv("../input_raw/" +name + "_not_lowercase.csv", index=False)
