# coding: utf-8

import urllib2
import urlparse
from config import user_agents


def download(url, user_agent='wswp', proxy=None, num_retries=2):
    print 'Downloading: ', url
    headers = {'User-agents': user_agent}
    request = urllib2.Request(url, headers=headers)
    opener = urllib2.build_opener()

    if proxy:
        proxy_params = {urlparse.urlparse(url).scheme: proxy}
        opener.add_handler(urllib2.ProxyHandler(proxy_params))

    try:
        html = opener.urlopen(request).read()
    except urllib2.URLError as e:
        print 'Download error: ', e.reason
        html = None

        if num_retries > 0:
            if hasattr(e, 'code') and 500 <= e.code < 600:
                # recursively retry 5xx HTTP errors
                return download(url, user_agent, proxy, num_retries-1)

    return html
