import cPickle
from multiprocessing import Pool
from alchemyapi import AlchemyAPI


def get_keywords(ziptext):
    '''
    IN: One long string with many words in it (from website text in this context)
    OUT: One long string with many words in it, but only including relevant keywords
    '''
    n= ziptext[0]
    text=ziptext[1]
    
    #poor man's status monitor
    if n%300==0: print "just collected entry number: ", n
        
    #get keywords
    alchemyapi = AlchemyAPI()
    keywords=alchemyapi.keywords("text", text)
    
    step1=[keywords['keywords'][m]['text'].split() for m in   \
    range(len(keywords['keywords']))]
    
    #paste strings back together again
    lswords= ' '.join([item for sublist in step1 for item in sublist])

    return lswords


if __name__ == '__main__':

    with open('pkls/url_text.pkl', 'rb') as p:
        urlstexts=cPickle.load(p)

    print "you gave me ", len(urlstexts), 'urls' \
    ' and now I will get API relevance texts'

    p = Pool(20)
    urlstexts_relevant=p.map(get_keywords, zip(range(len(urlstexts)), urlstexts))
    

    #writing our data now
    with open('pkls/urlstexts_relevant.pkl', 'wb') as p:
        cPickle.dump(urlstexts_relevant, p)

    print len(urlstexts_relevant), 'responses acquired responses \
    successfully'
