#!/usr/bin/env python2

'''
Simple Parallel Concurrency script to 
get response.get objects from urls

READS: cPickle list of urls (url.pkl)
WRITES: cPickle list of of url: response tuple pairs
OUT: None

'''


import requests
from multiprocessing import Pool
import cPickle
from joblib import Parallel, delayed

def f(url):
    

    try:
        if url[1]%100==0: print url[1]
        req=requests.get(url[0])
        if req is not None:
            print type(req)
	    return req 
    except:
	print type(url)
        print url
        return ['d']

if __name__=='__main__':

    with open('pkls/urls.pkl', 'rb') as p:
        urllist=cPickle.load(p)     
    print urllist
    print "you gave me ", len(urllist), 'urls' \
    ' and now I will get response.get objects'

    #p = Pool(20)
    #responses=p.map(f, zip(urllist, range(len(urllist))))
    
    Parallel(n_jobs=8)(delayed(f)(i) for i in zip(urllist, range(len(urllist))))


    #combine url with response
    assert len(urllist)==len(responses)
    url_response=zip(urllist, responses)

    #writing our data now
    with open('pkls/url_resp.pkl', 'wb') as p:
        cPickle.dump(url_response, p)

    print len(urllist), 'responses acquired responses \
    successfully'
    
