
# coding: utf-8

# In[13]:


import csv
import numpy as np
from sklearn.feature_extraction.text import TfidfVectorizer
from wordcloud import WordCloud
import matplotlib.pyplot as plt
from random import shuffle

tech_companies = {
    "AAPL": True,
    "MSFT": True,
    "FB": True,
    "GOOG": True,
    "GOOGL": True,
    "INTC": True,
    "V": True,
    "CSCO": True,
    "MA": True,
    "NVDA": True,
    "ORCL": True,
    "IBM": True,
    "ADBE": True,
    "TXN": True,
    "AVGO": True,
    "ACN": True,
    "PYPL": True,
    "QCOM": True,
    "CRM": True,
    "MU": True,
    "AMAT": True,
    "ATVI": True,
    "ADP": True,
    "CTSH": True,
    "INTU": True,
    "EBAY": True,
    "EA": True,
    "HPQ": True,
    "TEL": True,
    "LRCX": True,
    "ADI": True,
    "FIS": True,
    "FISV": True,
    "DXC": True,
    "NOW": True,
    "HPE": True,
    "WDC": True,
    "RHT": True,
    "APH": True,
    "ADSK": True,
    "GLW": True,
    "WP": True,
    "MCHP": True,
    "PAYX": True,
    "TWTR": True,
    "SWKS": True,
    "XLNX": True,
    "FLT": True,
    "GPN": True,
    "WDAY": True,
"MSI": True,
    "KLAC": True,
    "PANW": True,
    "MXIM": True,
    "NTAP": True,
    "STX": True,
    "SYMC": True,
    "DVMT": True,
    "SPLK": True,
    "MELI": True,
    "TSS": True,
    "SQ": True,
    "CTXS": True,
    "ANSS": True,
    "ANET": True,
    "SNPS": True,
    "BR": True,
    "IAC": True,
    "AKAM": True,
    "TTWO": True,
    "ADS": True,
    "CDW": True,
    "IT": True,
    "ON": True,
    "VRSN": True,
    "CA": True,
    "CDNS": True,
    "KEYS": True,
    "LDOS": True,
    "JKHY": True,
    "CGNX": True,
    "QRVO": True,
    "FFIV": True,
    "JNPR": True,
    "PTC": True,
    "SSNC": True,
    "TER": True,
    "IPGP": True,
    "FLEX": True,
    "TRMB": True,
    "AMD": True,
    "WU": True,
    "GRUB": True,
    "VMW": True,
    "GDDY": True,
    "MRVL": True,
    "CDK": True,
    "FTNT": True,
    "MSCC": True,
    "COMM": True,
    "TYL": True,
    "ZBRA": True,
    "FDC": True,
    "ULTI": True,
    "FLIR": True,
    "CSRA": True,
    "ARW": True,
    "MKSI": True,
    "BKI": True,
    "XRX": True,
    "GWRE": True,
    "LOGM": True,
    "NATI": True,
    "Z": True,
    "WEX": True,
    "CY": True,
    "CAVM": True,
    "EPAM": True,
    "AZPN": True,
    "FSLR": True,
    "BAH": True,
    "NTNX": True,
    "DATA": True,
    "PFPT": True,
    "ENTG": True,
    "LFUS": True,
    "FICO": True,
    "DST": True,
    "AVT": True,
    "SNAP": True,
    "PAYC": True,
    "BLKB": True,
    "TDC": True,
    "ARRS": True,
    "ZEN": True,
    "COHR": True,
    "SABR": True,
    "NUAN": True,
    "MPWR": True,
    "JBL": True,
    "OLED": True,
    "TWOU": True,
    "EEFT": True,
    "LITE": True,
    "MMS": True,
    "IDTI": True,
    "VSM": True,
    "CREE": True,
    "RNG": True,
    "DLB": True,
    "VSAT": True,
    "SLAB": True,
    "JCOM": True,
    "HUBS": True,
    "NCR": True,
    "CLGX": True,
    "CIEN": True,
    "CACI": True,
    "SNX": True,
    "CNDT": True,
    "MLNX": True,
    "YELP": True,
    "STMP": True,
    "ELLI": True,
    "SAIC": True,
    "TECD": True,
    "RP": True,
    "ZNGA": True,
    "ACIW": True,
    "MANH": True,
    "FEYE": True,
    "CCMP": True,
    "BDC": True,
    "ZG": True,
    "TWLO": True,
    "CRUS": True,
    "SMTC": True,
    "NEWR": True,
    "SATS": True,
    "IDCC": True,
    "AEIS": True,
    "ENV": True,
    "CVLT": True,
    "CALD": True,
    "ITRI": True,
    "IIVI": True,
    "VRNT": True,
    "VSH": True,
    "ACXM": True,
    "HAWK": True,
    "BOX": True,
    "QLYS": True,
    "ETSY": True,
    "MTCH": True,
    "PEGA": True,
    "ROG": True,
    "VIAV": True,
    "AXE": True,
    "NTCT": True,
    "CVG": True,
    "CSOD": True,
    "PLXS": True,
    "BRKS": True,
    "POWI": True,
    "CARS": True,
    "GTT": True,
    "UBNT": True,
    "SANM": True,
    "EXLS": True,
    "PLT": True,
    "FNSR": True,
    "NTGR": True,
    "TVPT": True,
    "PAY": True,
    "KLIC": True,
    "EGHT": True,
    "PRGS": True,
    "AMBA": True,
    "NOVT": True,
    "OCLR": True,
    "TIVO": True,
    "INFN": True,
    "PSTG": True,
    "TTD": True,
    "MXL": True,
    "BHE": True,
    "PCTY": True,
    "SYNA": True,
    "EPAY": True,
    "SEDG": True,
    "EBIX": True,
    "CSGS": True,
    "RMBS": True,
    "QTWO": True,
    "MANT": True,
    "MEI": True,
    "TTMI": True,
    "FIVN": True,
    "BMI": True,
    "VRTU": True,
    "IMPV": True,
    "COUP": True,
    "P": True,
    "DDD": True,
    "FN": True,
    "BL": True,
    "DIOD": True,
    "EFII": True,
    "INST": True,
    "MB": True,
    "MSTR": True,
    "NSIT": True,
    "OSIS": True,
    "IPHI": True,
    "CATM": True,
    "KN": True,
    "SYKE": True,
    "AMKR": True,
    "XPER": True,
    "BCOR": True,
    "SPSC": True,
    "PLUS": True,
    "FORM": True,
    "EVTC": True,
    "QUOT": True,
    "FARO": True,
    "DBD": True,
    "TYPE": True,
    "HDP": True,
    "MTSC": True,
    "PRO": True,
    "KEM": True,
    "VRNS": True,
    "SSTK": True,
    "VECO": True,
    "RTEC": True,
    "SYNT": True,
    "EVBG": True,
    "FIT": True,
    "MITL": True,
    "SCSC": True,
    "EGOV": True,
    "AVX": True,
    "ACLS": True,
    "CRAY": True,
    "CTS": True,
    "CAMP": True,
    "CASH": True,
    "PRFT": True,
    "CEVA": True,
    "LPSN": True,
    "RPD": True,
    "ACIA": True,
    "ADTN": True,
    "MTSI": True,
    "UCTT": True,
    "WEB": True,
    "CASS": True,
    "TRUE": True,
    "NANO": True,
    "CMTL": True,
    "LSCC": True,
    "APTI": True,
    "SMCI": True,
    "CHUBK": True,
    "ALRM": True,
    "XCRA": True,
    "PLAB": True,
    "WK": True,
    "UEPS": True,
    "UIS": True,
    "XOXO": True,
    "GOGO": True,
    "TTEC": True,
    "ICHR": True,
    "SPWR": True,
    "EIGI": True,
    "PSDO": True,
    "SNCR": True,
    "AAOI": True,
    "QADA": True,
    "EVRI": True,
    "SGH": True,
    "KE": True,
    "SYX": True,
    "APPF": True,
    "CISN": True,
    "TCX": True,
    "GLUU": True,
    "YEXT": True,
    "MLAB": True,
    "HCKT": True,
    "BNFT": True,
    "VDSI": True,
    "DAKT": True,
    "PKE": True,
    "PDFS": True,
    "CHUBA": True,
    "HLIT": True,
    "UPLD": True,
    "CALX": True,
    "CNXN": True,
    "AOSL": True,
    "ATEN": True,
    "RBBN": True,
    "MGI": True,
    "MOBL": True,
    "PI": True,
    "NPTN": True,
    "QTNA": True,
    "MEET": True,
    "KODK": True,
    "INAP": True,
    "VERI": True,
    "DHX": True,
    "RUBI": True
}

tech_desc = {}

with open('company_descriptions.csv', 'r') as csvfile:
  reader = csv.reader(csvfile, delimiter=',')
  for row in reader:
    if row[0] in tech_companies:
      tech_desc[row[0]] = row[1]

def build_vectorizer(max_features, stop_words, max_df=0.8, min_df=10, norm='l2'):
    """Returns a TfidfVectorizer object
    
    Params: {max_features: Integer,
             max_df: Float,
             min_df: Float,
             norm: String,
             stop_words: String}
    Returns: TfidfVectorizer
    """
    # YOUR CODE HERE
    return TfidfVectorizer(min_df=min_df, max_df=max_df, max_features=max_features, stop_words=stop_words, norm=norm)

n_feats = 5000
doc_by_vocab = np.empty([len(tech_desc.keys()), n_feats])
tfidf_vec = build_vectorizer(n_feats, "english")
doc_by_vocab = tfidf_vec.fit_transform([v for v in tech_desc.values()]).toarray()
index_to_vocab = {i:v for i, v in enumerate(tfidf_vec.get_feature_names())}

term_scores = np.sum(doc_by_vocab, axis=0)
largest = np.argsort(term_scores)[::-1]
words = [index_to_vocab[idx] for idx in largest]
print(words[:10])
new_text = [((index_to_vocab[idx] + " ") * int(largest[idx])).strip() for idx in largest]
new_words = ' '.join(new_text).split(' ')
shuffle(new_words)

wordcloud = WordCloud(max_font_size=40).generate(' '.join(new_words))
plt.figure()
plt.imshow(wordcloud, interpolation="bilinear")
plt.axis("off")
plt.savefig('top_tech_words.png', format='png', dpi=1000)
plt.show()

#
#print("Num docs = " + str(len(tech_desc.keys())))
#co_occur = np.dot(doc_by_vocab.T, doc_by_vocab)
#co_occur / 
#largest = np.dstack(np.unravel_index(np.argsort(co_occur.ravel()), co_occur.shape))[::-1][0]
#words = [(index_to_vocab[t[0]], index_to_vocab[t[1]]) for t in largest]
#print(words[:10])

