import numpy as np
import re
from nltk.corpus import stopwords
stop = stopwords.words("english")

def tokenizer(text):
    text = re.sub("<[^>]*>", "", text)
    emoticons = re.findall("(?::|;|=)(?:-)?(?:\)|\(|D|P", text.lower())
    text = re.sub("[\W]+", " ", text.lower()) + " ".join(emoticons).replace("-", "")
    tokenized = [w for w in text.split() if w not in stop]
    return tokenized


def stream_docs(path):
    with open(path, "r") as csv:
        next(csv)
        for line in csv:
            text, label = line[:-5], (line[-4:-1])
            yield text, label


next(stream_docs(path = "/Users/ilyaperepelitsa/quant/MLpy/practica/movie_data1.csv"))


def get_minibatch(doc_stream, size):
    docs, y = [], []
    try:
        for _ in range(size):
            text, label = next(doc_stream)
            docs.append(text)
            y.append(label)
    except StopIteration:
        return None, None
    return docs, y


from sklearn.feature_extraction.text import HashingVectorizer
from sklearn.linear_model import SGDClassifier

vect = HashingVectorizer(decode_error = "ignore",
                         n_features = 2**21,
                         preprocessor = None,
                         tokenizer = tokenizer)
clf = SGDClassifier(loss = "log", random_state = 1, n_iter = 1)
doc_stream = stream_docs(path = "/Users/ilyaperepelitsa/quant/MLpy/practica/movie_data1.csv")
for i in doc_stream:
    print(i[1])

import pyprind
pbar = pyprind.ProgBar(45)
classes = np.array([0, 1])
for _ in range(45):
    X_train, y_train = get_minibatch(doc_stream, size = 1000)
    if not X_train:
        continue
    X_train = vect.transform(X_train)
    clf.partial_fit(X_train, y_train, classes = classes)
    pbar.update()

X_test, y_test = get_minibatch(doc_stream, size = 5000)
X_test = vect.transform(X_test)
print(X_test)
