from __future__ import with_statement

import sys, traceback, re, struct, logging, random, StringIO
import calendar, time, rfc822
import socks, socket, asynchat
import urllib, urllib2, urlparse
import httplib, httplib2
import cookielib
import simplejson
import itertools

from httplib import HTTPConnection
from httplib import NotConnected

import primitives.funcs
import proxy_settings
from Events import EventMixin
from callbacks import callsback

try:
    sentinel
except NameError:
    class Sentinel(object):
        def __repr__(self):
            return "<Sentinel (%r backup) %#x>" % (__file__, id(self))
    sentinel = Sentinel()

log = logging.getLogger('util.net')

default_chunksize = 1024 * 4

def get_ips_s(hostname = ''):
    '''
    returns the ip addresses of the given hostname, default is '' (localhost)
    @param hostname: hostname to get the ips of
    @return ips as list of string
    '''
    # gethostbyname_ex returns tuple: (hostname, aliaslist, ipaddr_list)
    return socket.gethostbyname_ex(hostname or socket.gethostname())[2]

def get_ips(hostname = ''):
    '''
    returns the ip addresses of the given hostname, default is '' (localhost)
    @param hostname: hostname to get the ips of
    @return ips as list of int
    '''
    return [socket.inet_aton(ip) for ip in get_ips_s(hostname)]

myips = get_ips

def myip():
    'Returns the IP of this machine to the outside world.'
    return myips()[0]

def ip_from_bytes(bytes):
    """
    Converts a long int to a dotted XX.XX.XX.XX quad string.
    thanks U{http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/66517}
    """


    return socket.inet_ntoa(bytes)

class FileChunker(object):
    'asynchat producer to return chunks of a file'

    def __init__(self, fileobj, chunksize = default_chunksize, close_when_done = False,
                 progress_cb = lambda bytes: None, bytecounter = None):

        self.fileobj         = fileobj
        self.chunksize       = chunksize
        self.close_when_done = close_when_done
        if bytecounter is None:
            bytecounter = fileobj.tell
        self.total           = bytecounter()
        self.progress_cb     = progress_cb
        self.cancelled       = False

    def more(self):
        try:
            data_read = self.fileobj.read(self.chunksize)
        except ValueError:
            try: self.fileobj.close() #make sure it's good and dead
            except: pass
            return '' #same as what happens at end of file
        sz = len(data_read)

        if sz == 0 and self.close_when_done:
            self.fileobj.close()

        self.total += sz
        self.progress_cb(self.total)

        return data_read

    @classmethod
    def tofile(cls, sourcefile, outfile, progress_callback = lambda *a: None,
               bytecounter = None):
        gen = cls.tofile_gen(sourcefile, outfile, progress_callback, bytecounter)
        gen.next()
        try:
            gen.next()
        except StopIteration:
            pass

    @classmethod
    def tofile_gen(cls, sourcefile, outfile, progress_callback = lambda *a: None,
               bytecounter = None):
        fc = cls(sourcefile, close_when_done = True, bytecounter = bytecounter)
        yield fc
        chunk = fc.more()
        bytes_written = 0

        # localize for speed
        write, tell, more = outfile.write, outfile.tell, fc.more

        while chunk and not fc.cancelled:
            write(chunk)
            bytes_written += len(chunk)
            progress_callback(tell())
            chunk = more()

        outfile.close()

class NoneFileChunker(FileChunker):
    def more(self):
        return super(NoneFileChunker, self).more() or None

def httpjoin(base, path, keepquery=False):
    if path.startswith('http'):
        # path is already absolute
        return path
    else:
        joined = urlparse.urljoin(base, path)
        if not keepquery:
            parsed = list(urlparse.urlparse(joined))
            parsed[4] = ''
            return urlparse.urlunparse(parsed)
        else:
            return joined

class UrlQuery(str):
    @classmethod
    def form_encoder(cls):
        return WebFormData

    @classmethod
    def parse(cls, url, parse_query = True, utf8=False):
        '''
        Return a mapping from a URL string containing the following keys:

        scheme://netloc/path;parameters?query#fragment

        If parse_query is True, "key=value&done" becomes
        {'key':'value', 'done':True} instead. Otherwise query is just a string.
        '''
        scheme, netloc, path, params, query, fragment = urlparse.urlparse(url)

        if parse_query:
            query = cls.form_encoder().parse(query, utf8=utf8)

        return dict(scheme = scheme, netloc = netloc, path = path,
                    params = params, query = query, fragment = fragment)

    @classmethod
    def unparse(cls, scheme = '', netloc = '', path = '', params = '', query = None, fragment = ''):
        if query is None:
            query = {}

        if isinstance(query, dict):
            query = cls.form_encoder()(query)

        return urlparse.urlunparse((scheme, netloc, path, params, query, fragment))

    def __new__(cls, link, d={}, **kwargs):
        '''
        Splat **kwargs (and d) as URL parameters into the given url.

        The url must not already contain parameters (but may end with a
        question mark.)
        '''
        if not (d or kwargs):
            return str.__new__(cls, link)

        if link.endswith('?'):
            link = link[:-1]

        if '?' in link:
            joiner = '&' if (d or kwargs) else ''
        else:
            joiner = '?'

        return str.__new__(cls, ''.join([link, joiner, cls.form_encoder()(d = d, **kwargs)]))

class WebFormData(str):
    @classmethod
    def encoder(cls):
        return urllib.urlencode

    def __new__(cls, d={}, **kwargs):
        if d and kwargs:
            kwargs.update(d)
        else:
            kwargs = kwargs or d
        base = cls.encoder()(kwargs)
        return str.__new__(cls, base)

    @classmethod
    def parse(cls, s, utf8=False):
        querymap = {}
        if not s:
            return querymap
        encoding = 'utf8url' if utf8 else 'url'
        for elem in s.split('&'):
            if '=' in elem:
                key, value = elem.split('=', 1)
                querymap[key] = value.decode(encoding)
            else:
                querymap[elem] = True

        return querymap

class WebFormObjectData(WebFormData):
    @classmethod
    def encoder(cls):
        return urlencode_object

class UrlQueryObject(UrlQuery):
    @classmethod
    def form_encoder(cls):
        return WebFormObjectData

def urlencode_object(query):
    return urllib.urlencode(param(query))

def param(a):
    s = list()
    def add(key, value):
        s.append((key, value))
    for prefix in a:
        buildParams( prefix, a[prefix], add );
    return s

def buildParams(prefix, obj, add):
    if isinstance(obj, dict) and obj:
        for k,v in obj.items():
            buildParams( prefix + "[" + k +  "]", v, add );
    elif isinstance(obj, list):
        for k, v in enumerate(obj):
            if not isinstance(v, (list,dict)):
                k = ''
            buildParams( prefix + "[" + str(k) +  "]", v, add );
    else:
        add(prefix, obj)

def int_to_ip(s, byteorder='<'):
    '''
    turns an int (or string as int) into a dotted IP address.
    ex: int_to_ip('580916604') --> '124.21.160.34'

    default byteorder is little-endian (for msn)
    '''
    return '.'.join(str(ord(c)) for c in struct.pack(byteorder+'I', int(s)))


# matches two or more spaces in a row
spacify_pattern = re.compile('( {2,})')

def spacify_repl(m):
    l = len(m.group())

    if l == 2:
        return '  ' # "word joiner" unicode character is &#2060; but we're not using that anymore.
    else:
        # for more than two spaces use <SPACE><series of nbsp;s><SPACE>
        return ' ' + ''.join(['&nbsp;'] * (l-2)) + ' '

def spacify(s):
    'Turns consecutive spaces into a series of &nbsp; entities.'

    return spacify_pattern.sub(spacify_repl, s)

#
# a url for matching regexes
#
urlregex = re.compile(r'([A-Za-z][A-Za-z0-9+.-]{1,120}:[A-Za-z0-9/]'
                       '(([A-Za-z0-9$_.+!*,;/?:@&~=-])|%[A-Fa-f0-9]{2}){1,333}'
                       "(#([a-zA-Z0-9][a-zA-Z0-9$_.+!*,;/?:@&~=%-']{0,1000}))?)"
                       '[^\. <]')


#
# thanks http://en.wikipedia.org/wiki/List_of_Internet_top-level_domains
#
TLDs = \
['arpa', 'root', 'aero', 'asia', 'biz', 'com', 'coop', 'edu', 'gov', 'info', 'int',
 'museum', 'name', 'net', 'org', 'pro', 'ac', 'ad', 'ae', 'af', 'ag', 'ai', 'al', 'am',
 'an', 'ao', 'aq', 'ar', 'as', 'at', 'au', 'aw', 'ax', 'az', 'ba', 'bb', 'bd',
 'be', 'bf', 'bg', 'bh', 'bi', 'bj', 'bm', 'bn', 'bo', 'br', 'bs', 'bt', 'bv',
 'bw', 'by', 'bz', 'ca', 'cc', 'cd', 'cf', 'cg', 'ch', 'ci', 'ck', 'cl', 'cm',
 'cn', 'co', 'cr', 'cu', 'cv', 'cx', 'cy', 'cz', 'de', 'dj', 'dk', 'dm', 'do',
 'dz', 'ec', 'ee', 'eg', 'er', 'es', 'et', 'eu', 'fi', 'fj', 'fk', 'fm', 'fo',
 'fr', 'ga', 'gb', 'gd', 'ge', 'gf', 'gg', 'gh', 'gi', 'gl', 'gm', 'gn', 'gp',
 'gq', 'gr', 'gs', 'gt', 'gu', 'gw', 'gy', 'hk', 'hm', 'hn', 'hr', 'ht', 'hu',
 'id', 'ie', 'il', 'im', 'in', 'io', 'iq', 'ir', 'is', 'it', 'je', 'jm', 'jo',
 'jp', 'ke', 'kg', 'kh', 'ki', 'km', 'kn', 'kp', 'kr', 'kw', 'ky', 'kz', 'la',
 'lb', 'lc', 'li', 'lk', 'lr', 'ls', 'lt', 'lu', 'lv', 'ly', 'ma', 'mc', 'md',
 'me', 'mg', 'mh', 'mk', 'ml', 'mm', 'mn', 'mo', 'mp', 'mq', 'mr', 'ms', 'mt',
 'mu', 'mv', 'mw', 'mx', 'my', 'mz', 'na', 'nc', 'ne', 'nf', 'ng', 'ni', 'nl',
 'no', 'np', 'nr', 'nu', 'nz', 'om', 'pa', 'pe', 'pf', 'pg', 'ph', 'pk', 'pl',
 'pm', 'pn', 'pr', 'ps', 'pt', 'pw', 'py', 'qa', 're', 'ro', 'rs', 'ru', 'rw',
 'sa', 'sb', 'sc', 'sd', 'se', 'sg', 'sh', 'si', 'sj', 'sk', 'sl', 'sm', 'sn',
 'so', 'sr', 'st', 'su', 'sv', 'sy', 'sz', 'tc', 'td', 'tf', 'tg', 'th', 'tj',
 'tk', 'tl', 'tm', 'tn', 'to', 'tp', 'tr', 'tt', 'tv', 'tw', 'tz', 'ua', 'ug',
 'uk', 'um', 'us', 'uy', 'uz', 'va', 'vc', 've', 'vg', 'vi', 'vn', 'vu', 'wf',
 'ws', 'ye', 'yt', 'yu', 'za', 'zm', 'zw'
]


domains = '(?:%s)' % '|'.join(TLDs)

'''
 the one true email regex(tm)

 complicated case and simple case:
     this_email-user.name+mylabel@bbc.co.uk
     name@domain
'''

email_regex_string = r'(?:([a-zA-Z0-9_][a-zA-Z0-9_\-\.]*)(\+[a-zA-Z0-9_\-\.]+)?@((?:[a-zA-Z0-9\-_]+\.?)*[a-zA-Z]{1,4}))'

email_regex             = re.compile(email_regex_string)
email_wholestring_regex = re.compile('^' + email_regex_string + '$')

is_email = primitives.funcs.ischeck(lambda s:bool(email_wholestring_regex.match(s)))

class EmailAddress(tuple):
    def __new__(cls, addr, default_domain=sentinel):

        try:
            name, label, domain = parse_email(addr)
        except:
            if default_domain is sentinel:
                raise
            else:
                name, label, domain = parse_email(addr + '@' + default_domain)

        return tuple.__new__(cls, (name, label, domain.lower()))

    @property
    def name(self):
        return self[0]

    @property
    def label(self):
        return self[1]

    @property
    def domain(self):
        return self[2]

    def __str__(self):
        if self.label:
            return '%s+%s@%s' % self
        else:
            return '%s@%s' % (self.name, self.domain)

    def __repr__(self):
        return '<EmailAddress %s>' % (self,)

def parse_email(s):
    match = email_wholestring_regex.match(s)

    if match is None:
        raise ValueError('Not a valid email address: %r', s)

    user, lbl, dom = match.groups()
    if lbl:
        lbl = lbl.strip('+')
    else:
        lbl = ''
    return user, lbl, dom


protocols = 'ftp|https?|gopher|msnim|icq|telnet|nntp|aim|file|svn|svn+(?:\w)+'

# for these TLDs, only a few have ever been allowed to be registered.
single_letter_rule_tlds = frozenset(('net', 'com', 'org'))
allowed_single_letter_domains = frozenset(('i.net', 'q.com', 'q.net', 'x.com', 'x.org', 'z.com'))

linkify_url_pattern = re.compile(
  # thanks textile
  r'''(?=[a-zA-Z0-9])                             # Must start correctly
      ((?:                                        # Match the leading part (proto://hostname, or just hostname)
          (?:(?P<protocol>%s)                     #     protocol
          ://                                     #     ://
          (?:                                     #     Optional 'username:password@'
              (?P<username>\w+)                   #         username
              (?::(?P<password>\w+))?             #         optional :password
              @                                   #         @
          )?)?                                    #
          (?P<hostname>                           # hostname (sub.example.com). single-letter
          (?:[iqxz]|(?:[-\w\x7F-\xFF]+))          # domains are not allowed, except those listed:
          (?:\.[\w\x7F-\xFF][-\w\x7F-\xFF]*)*)    # http://en.wikipedia.org/wiki/Single-letter_second-level_domains
      )?                                          #
      (?::(?P<port>\d+))?                         # Optional port number
      (?P<selector>
        (?:                                       # Rest of the URL, optional
          /?                                      #     Start with '/'
          [^.!,?;:"<>\[\]{}\s\x7F-\xFF]+          #     Can't start with these
          (?:                                     #
              [.!,?;:]+                           #     One or more of these
              [^.!,?;:"<>{}\s\x7F-\xFF]+          #     Can't finish with these
              #'"                                 #     # or ' or "
          )*)                                     #
      )?)                                         #
   ''' % protocols, re.VERBOSE)


def isurl(text):
    m = linkify_url_pattern.match(text)
    if not m: return False

    protocol, host = m.group('protocol'), m.group('hostname')

    if host is not None:
        # only allow links without protocols (i.e., www.links.com)
        # if the TLD is one of the allowed ones
        if protocol is None:
            myTLDs = (host.split('.') if '.' in host else [host])
            if len(myTLDs) < 2 or myTLDs[-1] not in TLDs:
                return False

    return True

class LinkAccumulator(object):
    def __init__(self, s=None):
        self.links = []
        self.spans = []

        if s is not None:
            linkify_url_pattern.sub(self.repl, s)

    def repl(self, m):
        url, protocol, after = _url_from_match(m)
        if url is None:
            return ''
        href = ('http://' + url) if protocol is None else url
        self.links.append(href)
        self.spans.append(m.span())
        return ''

    def __iter__(self):
        return itertools.izip(self.links, self.spans)

def find_links(text):
    return LinkAccumulator(text).links

def _url_from_match(m):
    protocol, host = m.group('protocol'), m.group('hostname')
    url = m.group()

    # fix urls (surrounded by parens)
    after = ''
    if url.endswith(')') and '(' not in url:
        url = url[:-1]
        after = ')'

    if host is not None:
        myTLDs = (host.split('.') if '.' in host else [host])
        # only allow links without protocols (i.e., www.links.com)
        # if the TLD is one of the allowed ones
        if protocol is None:
            if len(myTLDs) < 2 or myTLDs[-1] not in TLDs:
                return None, None, None

        if len(myTLDs) >= 2:
            # don't allow single letter second level domains unless they are in the list
            # of allowed ones above
            second_level_domain = myTLDs[-2]
            top_level_domain = myTLDs[-1]
            if (len(second_level_domain) == 1
                and '.'.join((second_level_domain, top_level_domain)) not in allowed_single_letter_domains
                and top_level_domain in single_letter_rule_tlds):
                # "cancel" the replacement
                return None, None, None

        return url, protocol, after

    return None, None, None

def _dolinkify(text):
    def repl(m):
        url, protocol, after = _url_from_match(m)

        if url is None:
            i, j = m.span()
            return text[i:j]

        href = ('http://' + url) if protocol is None else url

        return '<a href="%s">%s</a>' % (href, url) + after # TODO: add 'target="_blank"' ?

    #text = email_regex.sub(r'<a href="mailto:\1">\1</a>', text)
    text = linkify_url_pattern.sub(repl, text)
    return text

def linkify(text):
    if isinstance(text, unicode):
        return _linkify(text.encode('utf-8')).decode('utf-8')
    else:
        return _linkify(text)

def _linkify(text):
    # Linkify URL and emails.

    # If there is no html, do a simple search and replace.
    if not re.search(r'''<.*>''', text):
        return _dolinkify(text)

    # Else split the text into an array at <>.
    else:
        lines = []
        prev_line = ''
        for line in re.split('(<.*?>)', text):
            if not re.match('<.*?>', line) and not prev_line.startswith('<a'):
                line = _dolinkify(line)

            prev_line = line
            lines.append(line)

        return ''.join(lines)

class QueueableMixin(object):
    def __init__(self):
        object.__init__(self)
        self._on_queue = False

    def queue(self):
        # Adds this object to its socket's queue
        # if it's not already there.
        if not self._on_queue:
            self._queue()
            self._on_queue = True

    def unqueue(self):
        if self._on_queue:
            self._unqueue()
            self._on_queue = False

class ProducerQueuable(QueueableMixin):
    def __init__(self, sck):
        QueueableMixin.__init__(self)
        self.sck = sck

    def _queue(self):
        self.sck.push_with_producer(self)
    def _unqueue(self):
        try:
            self.sck.producer_fifo.remove(self)
        except ValueError:
            pass

class RoundRobinProducer(ProducerQueuable):
    def __init__(self, sck):
        ProducerQueuable.__init__(self, sck)
        self.list = []

    def add(self, prod):
        # Adds a producer to our list, and checks that it has
        # a callable "more" attribute.
        try:
            if not callable(prod.more):
                raise AssertionError('Producers must have a "more" method')
        except:
            traceback.print_exc()
            raise

        self.unqueue()
        self.list.append(prod)
        self.queue()

    def more(self):
        # If this is getting called, we *must* be on the queue
        self._on_queue = True

        d = None
        l = self.list
        prod = None
        while (not d) and l:
            prod = l.pop(0)
            d = prod.more()

            # if we got data, loop will break
            # if not, the "bad" producer is gone

        # end loop

        if d:
            # that producer is still good -- put it on the end.
            l.append(prod)
        else:
            # Didn't get any data. We're going to be removed
            # from the socket FIFO

#            if prod is None and not self.list:
#                print 'List of producers was empty, returning None'
#            else:
#                print 'Didn\'t get any data from %r' % prod

            self.unqueue()
            if self.list:
                self.queue()

        return d


class PriorityProducer(ProducerQueuable):
    def __init__(self, sck):
        ProducerQueuable.__init__(self, sck)

        self.high = []
        self.mid = []
        self.low = []

    def add(self, prod, pri='mid'):
        assert callable(prod.more)
        assert pri in ('high', 'mid', 'low')
        self.unqueue()
        getattr(self, pri).append(prod)
        self.queue()

    def more(self):
        # If this is getting called, we *must* be on the queue
        self._on_queue = True

        d = None
        for l in (self.high, self.mid, self.low):
            if not l: continue

            while not d and l:
                prod = l.pop(0)
                d = prod.more()

            if d:
                # Put the producer back where we got it
                l.insert(0, prod)
                break

#        if not d:
#            self.unqueue()

        return d

class HTTPConnProgress(HTTPConnection):
    'Subclass of HTTPConnection which sends a file object, reporting progress.'

    def send_file_cb(self, fileobj, progress_cb, blocksize = default_chunksize, progressDelta = 0):
        'Sends the contents of fileobj (a .read-able object) to server.'

        if self.sock is None:
            if self.auto_open:
                self.connect()
            else:
                raise NotConnected()

        if self.debuglevel > 0:
            print "sending contents of", fileobj

        try:
            read    = fileobj.read
            sendall = self.sock.sendall

            chunk   = read(blocksize)
            total   = 0

            while chunk:
                total += len(chunk)
                sendall(chunk)
                progress_cb(total - progressDelta)
                chunk = read(blocksize)

        except socket.error, v:
            if v[0] == 32:      # Broken pipe
                self.close()
            raise

class SocketEventMixin(EventMixin):
    events = EventMixin.events | set(("connected",
                                  "connection_failed",
                                  "socket_error",
                                  "socket_closed",
                                  ))

    def post_connect_error(self, e=None):
        self.event("socket_error")
        self.post_connect_disconnect()

    def post_connect_expt(self):
        self.event("socket_error")
        self.post_connect_disconnect()

    def post_connect_disconnect(self):
        self.close()
        self.event("socket_closed")

    def post_connect_close(self):
        self.close()
        self.event("socket_closed")

    def reassign(self):
        self.handle_expt = self.post_connect_expt
        self.handle_error = self.post_connect_error
        self.handle_close = self.post_connect_close
        self.do_disconnect = self.post_connect_disconnect

def build_cookie(name, value,
                   version = 0,
                   domain = sentinel,
                   port = sentinel,
                   path = sentinel,
                   secure = False,
                   expires = None,
                   discard = False,
                   comment = None,
                   comment_url = None,
                   rest = {'httponly' : None},
                   rfc2109 = False):

    if domain is sentinel:
        domain = None
        domain_specified = False
        domain_initial_dot = False
    else:
        domain_specified = True
        domain_initial_dot = domain.startswith('.')

    if port is sentinel:
        port = None
        port_specified = False
    else:
        port_specified = True

    if path is sentinel:
        path = None
        path_specified = False
    else:
        path_specified = True
    return cookielib.Cookie(**locals())


def GetSocketType():
    d = GetProxyInfo()
    if d:
        #socks.setdefaultproxy(**GetProxyInfo())
        return socks.socksocket
    else:
        return socket.socket

NONE = 'NONPROX'
SYSDEFAULT = 'SYSPROX'
CUSTOM = "SETPROX"

def GetProxyInfo():
    ps = proxy_settings

    try:
        pd = ps.get_proxy_dict()
    except Exception, e:
        print >>sys.stderr, 'No proxies because: %r' % e
        pd = {}

    get = pd.get
    proxytype= get('proxytype')
    port     = get('port')

    try:
        port = int(port)
    except:
        port = None

    addr     = get('addr')
    username = get('username')
    password = get('password')
    override = get('override')
    rdns     = get('rdns', False)

    try:
        override = int(override)
    except:
        if override not in (SYSDEFAULT, CUSTOM, NONE):
            override = SYSDEFAULT
    else:
        # Hack for old proxy code that only had 2 options
        if override:
            override = CUSTOM
        else:
            override = SYSDEFAULT

    if override == NONE:
        return {}
    elif override == SYSDEFAULT:
        px = urllib._getproxies()
        if not px: return {}

        url = px.get('http', None)
        if url is None: return {}

        url = urlparse.urlparse(url)

        addr = url.hostname or ''
        if not addr: return {}

        port = url.port or 80
        username = url.username or username or None
        password = url.password or password or None
        proxytype = 'http'

#    d = {}
#    if all((addr, port)):
#        d.update(addr=addr, port=port, proxytype=proxytype)
#
#        if all((username, password)):
#            d.update(username=username, password=password)

    if all((type, port, addr)):
        proxytype=getattr(socks, ('proxy_type_%s'%proxytype).upper(), None)
        return dict(addr=addr, port=port, username=username, password=password, proxytype=proxytype, rdns = rdns)
    else:
        return {}

def GetProxyInfoHttp2():
    i = GetProxyInfo()
    if not i:
        return None
    return httplib2.ProxyInfo(proxy_type = i['proxytype'],
                              proxy_host = i['addr'],
                              proxy_port = i['port'],
                              proxy_user = i['username'],
                              proxy_pass = i['password'],
                              proxy_rdns = i.get('rdns', False),
                              )

def getproxies_digsby():
    '''
    A replacement for urllib's getproxies that returns the digsby app settings.
    the return value is a dictionary with key:val as:
      'http' : 'http://user:pass@proxyhost:proxyport'

    the dictionary can be empty, indicating that there are no proxy settings.
    '''

    pinfo = GetProxyInfo()

    proxies = {}
    if pinfo.get('username', None) and pinfo.get('password', None):
        unpw = '%s:%s@' % (pinfo['username'], pinfo['password'])
    else:
        unpw = ''

    if pinfo.get('port', None):
        port = ':' + str(pinfo['port'])
    else:
        port = ''

    host = pinfo.get('addr', None)
    if not host:
        return proxies # empty dict

    all = unpw + host + port

    proxies = urllib.OneProxy()
    proxies._proxyServer = all

    if pinfo['proxytype'] != socks.PROXY_TYPE_HTTP:
        proxy_url = ('socks%d://' % (4 if pinfo['proxytype'] == socks.PROXY_TYPE_SOCKS4 else 5)) + all
        return dict(socks=proxy_url, http=proxy_url, https=proxy_url)

    proxies['https'] = 'http://' + all
    proxies['http'] = 'http://' + all
    proxies['ftp'] = 'http://' + all

    return proxies


class SocksProxyHandler(urllib2.ProxyHandler):
    '''
    Handles SOCKS4/5 proxies as well as HTTP proxies.
    '''
    handler_order = 100
    def proxy_open(self, req, type):
        try:
            req._proxied
        except AttributeError:
            proxyinfo = self.proxies.get(type, '')
            proxytype = urllib2._parse_proxy(proxyinfo)[0]
            if proxytype is None:
                req._proxied = False
                return urllib2.ProxyHandler.proxy_open(self, req, type)
            else:
                req._proxytype = proxytype
                req._proxied = True

                if proxytype == 'http' and type != 'https': # Http proxy
                    return urllib2.ProxyHandler.proxy_open(self, req, type)
                else:
                    return None
        else:
            # Already proxied. skip it.
            return None

    def socks4_open(self, req):
        return self.socks_open(req, 4)

    def socks5_open(self, req):
        return self.socks_open(req, 5)

    def socks_open(self, req, sockstype):

        orig_url_type, __, __, orighostport = urllib2._parse_proxy(req.get_full_url())
        req.set_proxy(orighostport, orig_url_type)

        endpoint = req.get_host()
        if ':' in endpoint:
            host, port = endpoint.rsplit(':', 1)
            port = int(port)
        else:
            host, port = endpoint, 80
        req._proxied = True

        return self.parent.open(req)

try:
    import ssl
except ImportError:
    pass
else:
    class SocksHttpsOpener(urllib2.HTTPSHandler):
        handler_order = 101
        def https_open(self, req):
            if getattr(req, '_proxied', False) and getattr(req, '_proxytype', None) is not None:
                return urllib2.HTTPSHandler.do_open(self, SocksHttpsConnection, req)
            else:
                return urllib2.HTTPSHandler.https_open(self, req)

    class SocksHttpsConnection(httplib.HTTPSConnection):
        _sockettype = socks.socksocket
        def connect(self):
            "Connect to a host on a given (SSL) port."

            pd = urllib.getproxies().get('https', None)
            if pd is None:
                sockstype = ''
            else:
                sockstype, user, password, hostport = urllib2._parse_proxy(pd)


            assert ':' in hostport # if we don't have a port we're screwed
            host, port = hostport.rsplit(':', 1)
            port = int(port)

            sock = self._sockettype(socket.AF_INET, socket.SOCK_STREAM)
            sock.setproxy(proxytype=getattr(socks, 'PROXY_TYPE_%s' % sockstype.upper()), addr=host, port=port, rdns=True, username=user, password=password)
            sock.connect((self.host, self.port))
            self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file)


class SocksHttpOpener(urllib2.HTTPHandler):
    handler_order = 101
    def http_open(self, req):
        proxytype = getattr(req, '_proxytype', None)
        if getattr(req, '_proxied', False) and proxytype not in ('http', None):
            return urllib2.HTTPHandler.do_open(self, SocksConnection, req)
        else:
            return urllib2.HTTPHandler.http_open(self, req)


class SocksConnection(httplib.HTTPConnection):
    _sockettype = socks.socksocket
    def connect(self):
        #- Parse proxies
        pd = urllib.getproxies().get('http', None)
        if pd is None:
            sockstype = ''
        else:
            sockstype, user, password, hostport = urllib2._parse_proxy(pd)

        if 'socks' not in sockstype:
            return httplib.HTTPConnection.connect(self)

        assert ':' in hostport # if we don't have a port we're screwed
        host, port = hostport.rsplit(':', 1)
        port = int(port)


        for res in socket.getaddrinfo(self.host, self.port, 0, socket.SOCK_STREAM):

            af, socktype, proto, canonname, sa = res
            try:
                self.sock = self._sockettype(af, socktype, proto)
                self.sock.setproxy(proxytype=getattr(socks, 'PROXY_TYPE_%s' % sockstype.upper()), addr=host, port=port, rdns=False, username=user, password=password)
                #- The rest is the same as superclass

                if self.debuglevel > 0:
                    print "connect: (%s, %s)" % (self.host, self.port)
                self.sock.connect(sa)
            except socket.error, msg:
                if self.debuglevel > 0:
                    print 'connect fail:', (self.host, self.port)
                if self.sock:
                    self.sock.close()
                self.sock = None
                continue
            break
        if not self.sock:
            raise socket.error, msg

class DigsbyHttpProxyPasswordManager(urllib2.HTTPPasswordMgr):
    def find_user_password(self, realm, uri):
        pi = GetProxyInfo()
        return (pi['username'] or None), (pi['password'] or None)

if not hasattr(urllib, '_getproxies'):
    urllib._getproxies, urllib.getproxies = urllib.getproxies, getproxies_digsby

    urllib2.UnknownHandler.handler_order = sys.maxint # This should be last, no matter what


    # BaseHandler comes first in the superclass list, but it doesn't take any arguments.
    # This causes problems when initializing the class with an argument.
    urllib2.ProxyDigestAuthHandler.__bases__ = urllib2.ProxyDigestAuthHandler.__bases__[::-1]
    urllib2.ProxyBasicAuthHandler.handler_order = 499 # Make sure it comes before the 'default' error handler

    httplib2.ProxyInfo.get_default_proxy = staticmethod(GetProxyInfoHttp2)

def GetDefaultHandlers():
    handlers = [SocksProxyHandler, SocksHttpOpener]
    httpsopener = globals().get('SocksHttpsOpener', None)
    if httpsopener is not None:
        handlers.append(httpsopener)

    pwdmgr = DigsbyHttpProxyPasswordManager()

    for auth_handler_type in (urllib2.ProxyBasicAuthHandler, urllib2.ProxyDigestAuthHandler):
        handlers.append(auth_handler_type(pwdmgr))

    return handlers

def build_opener(*a, **k):
    if 'default_classes' not in k:
        k['default_classes'] = GetDefaultHandlers() + urllib2.default_opener_classes

    return urllib2.build_opener(*a, **k)

opener = urllib2.build_opener(*GetDefaultHandlers())
#    for handler in opener.handlers:
#        handler._debuglevel = 1
urllib2.install_opener(opener)


_hostprog = re.compile('^//([^/?]*)(.*)$')
def splithost(url):
    """splithost('//host[:port]/path') --> 'host[:port]', '/path'."""
    match = _hostprog.match(url)
    if match:
        groups = match.group(1, 2)
        # Check if we're throwing out a slash accidentally. if so, put it back and return.
        if groups[0] == '':
            return groups[0], '/' + groups[1]
        else:
            return groups
    return None, url

urllib.splithost = urllib2.splithost = splithost # urllib2 imports it "from" urllib so we have to replace its copy as well

def _HTTPError__repr(self):
    if not hasattr(self, 'content'):
        try:
            self.content = self.read()
            self.close()
        except Exception, e:
            self._error = e
            self.content = "error reading body: %r" % e
            self.read = lambda: ''
        else:
            self._stringio = StringIO.StringIO(self.content)
            self.read = self._stringio.read

    etxt = self.content
    return '<HTTPError headers = %r, body = %r>' % (str(getattr(self, 'hdrs', {})), etxt)

urllib2.HTTPError.__repr__ = _HTTPError__repr

def httpok(_code):
    return getattr(_code, 'status', _code)//100 == 2

class SimpleProducer(asynchat.simple_producer):
    def more(self):
        data = asynchat.simple_producer.more(self)
        if data == '':
            data = None

        return data

class CallbackProducerMixin(object):
    '''
    Simple mixin for producer classes that calls callback.success() when all data has been read from it (via more()).

    Must be listed before the producer type in the inheritance list, for method resolution to work correctly.
    '''
    def __init__(self):
        bases = self.__class__.__bases__
        found_self = False
        self._siblingClass = None
        for base in bases:
            if base is CallbackProducerMixin:
                found_self = True
            else:
                if hasattr(base, 'more') and found_self:
                    self._siblingClass = base
                    break

        if self._siblingClass is None:
            raise AssertionError("This mix-in requires there is a sibling class with a 'more' method. "
                                 "Additionally, CallbackProducerMixin must be *before* that class in the inheritance list "
                                 "(for method resolution reasons).")

    @callsback
    def set_callback(self, callback=None):
        self._callback = callback

    def more(self):
        if not hasattr(self, '_siblingClass'):
            result = None
        else:
            result = self._siblingClass.more(self)

        if result is None:
            if getattr(self, '_callback', None) is not None:
                self._callback.success()
            if getattr(self, '_callback', None) is not None:
                del self._callback
        return result

class SimpleCallbackProducer(CallbackProducerMixin, SimpleProducer):
    '''
    Subclass of asynchat.simple_producer that calls self._callback.success() when all data has
    been exhausted. Set callback after instantiation with set_callback() method.

    SimpleCallbackProducer(data, buffer_size=512)
    '''
    def __init__(self, data):
        SimpleProducer.__init__(self, data)
        CallbackProducerMixin.__init__(self)

## Add a 'remove' method to asynchat's fifo

def _fifo_remove(self, val):
    '''
    returns True if value was found + removed, false otherwise.
    '''
    # self.list is a deque
    try:
        self.list.remove(val)
    except Exception:
        return False
    else:
        return True

asynchat.fifo.remove = _fifo_remove

@callsback
def producer_cb(data, callback=None):
    '''
    producer(data, success=callable)

    Facade for SimpleCallbackProducer.
    '''
    prod = SimpleCallbackProducer(data)
    prod.set_callback(callback=callback)
    return prod

class GeneratorProducer(object):
    def __init__(self, gen):
        self.gen = gen

    def more(self):
        if self.gen is None:
            return None

        try:
            return self.gen.next()
        except StopIteration:
            self.gen = None
            return None

# from tr.im:
_short_domains = frozenset((
                  '2big.at', '2me.tw', '3.ly', 'a.gd', 'a2n.eu', 'abbrr.com',
                  'adjix.com', 'arunaurl.com', 'beam.to', 'bit.ly',
                  'bitly.com', 'bkite.com', 'blip.fm', 'bloat.me',
                  'budurl.com', 'burnurl.com', 'canurl.com', 'chilp.it',
                  'cli.gs', 'decenturl.com', 'digg.com', 'digs.by', 'dn.vc',
                  'doiop.com', 'durl.us', 'dwarfurl.com', 'easyuri.com',
                  'easyurl.net', 'ff.im', 'fon.gs', 'fyiurl.com', 'ginx.com',
                  'goo.gl', 'go2.me', 'hex.io', 'hopurl.com', 'hurl.ws', 'icanhaz.com',
                  'idek.net', 'is.gd', 'ix.it', 'jijr.com', 'jmp2.net',
                  'knol.me', 'krz.ch', 'kurl.us', 'last.fm', 'lin.cr',
                  'lnk.in', 'makeitbrief.com', 'memurl.com', 'micurl.com',
                  'minu.ws', 'moourl.com', 'myturl.com', 'notlong.com', 'ow.ly',
                  'pic.im', 'pikchur.com', 'ping.fm', 'piurl.com', 'poprl.com',
                  'qurlyq.com', 'r.im', 'refurl.com', 'rubyurl.com', 'rurl.org',
                  'rurl.us', 's7y.us', 'sai.ly', 'sbt.sh', 'shorl.com'
                  'short.ie', 'short.to', 'shortna.me', 'shrinkify.com',
                  'shw.com', 'si9.org', 'skocz.pl', 'smalur.com', 'sn.im',
                  'snipr.com', 'snipurl.com', 'snurl.com', 'spedr.com',
                  'starturl.com', 'three.ly', 'timesurl.at', 'tiny.cc', 'tiny.pl',
                  'tinyarro.ws', 'tinylink.co.za', 'tinyuri.ca', 'tinyurl.com',
                  'tnij.org', 'tr.im', 'turo.us', 'twitclicks.com', 'twitpic.com',
                  'twt.fm', 'twurl.cc', 'twurl.nl', 'u.nu', 'ub0.cc', 'uris.jp',
                  'urlb.at', 'urlcut.com', 'urlenco.de', 'urlhawk.com',
                  'urltea.com', 'vieurl.com', 'w3t.org', 'x.se', 'xaddr.com',
                  'xr.com', 'xrl.us', 'yep.it', 'zi.ma', 'zombieurl.com', 'zz.gd'))

def is_short_url(url, domains = _short_domains):
    parsed = urlparse.urlparse(url)
    if parsed.netloc in domains:
        return True

    return False

def get_snurl(url):
    return get_short_url(url, 'snurl')

def get_isgd(url):
    return get_short_url(url, 'isgd')

def get_tinyurl(url):
    return get_short_url(url, 'tinyurl')

class UrlShortenerException(Exception):
    pass

from .lrucache import LRU
_short_url_cache = LRU(10)
def cache_shortened_url(url, short_url):
    if short_url:
        _short_url_cache[short_url] = url

class UrlShortener(object):
    endpoint = None

    def build_request_url(self, url):
        return UrlQuery(self.endpoint, d = self.get_args(url.encode('utf-8')))

    def shorten(self, url):
        try:
            resp = urllib2.urlopen(self.build_request_url(url))
        except urllib2.HTTPError, e:
            resp = e

        short_url = self.process_response(resp)
        cache_shortened_url(url, short_url)
        return short_url

    def shorten_async(self, url, success, error=None):
        def async_success(req, resp):
            try:
                ret = self.process_response(resp)
            except Exception as e:
                if error is not None: error(e)
            else:
                cache_shortened_url(url, ret)
                success(ret)

        def async_error(req=None, resp=None):
            print req
            print resp
            if error is not None:
                error(None) # TODO: async interface for errors?

        import common.asynchttp as asynchttp
        asynchttp.httpopen(self.build_request_url(url), success=async_success, error=async_error)

    def get_args(self, url):
        raise NotImplementedError

    def process_response(self, resp):
        if resp.code != 200:
            body = resp.read()
            raise UrlShortenerException(body)

        ret = resp.read()
        return ret

class ResponseIsResultShortener(UrlShortener):
    def process_response(self, resp):
        ret = UrlShortener.process_response(self, resp)
        if not isurl(ret):
            raise UrlShortenerException(body)
        return ret

class isgd_shortener(ResponseIsResultShortener):
    endpoint = 'http://is.gd/api.php'
    def get_args(self, url):
        return dict(longurl=url)

class tinyurl_shortener(ResponseIsResultShortener):
    endpoint = 'http://tinyurl.com/api-create.php'
    def get_args(self, url):
        return dict(url=url)

class threely_shortener(UrlShortener):
    endpoint = 'http://3.ly/'
    def get_args(self, url):
        return dict(api = 'em5893833',
                    u = url)

    def process_response(self, resp):
        ret = UrlShortener.process_response(self, resp)
        if not ret.startswith(self.endpoint):
            raise UrlShortenerException(ret)

        return ret

class snipr_shortener(UrlShortener):
    endpoint = 'http://snipr.com/site/snip'
    def get_args(self, url):
        return dict(r='simple', link=url.encode('url'))
    def process_response(self, resp):
        ret = UrlShortener.process_response(self, resp)
        if not ret.startswith('http'):
            raise UrlShortenerException('bad url: %r' % ret, ret)
        return ret

class shortname_shortener(UrlShortener):
    endpoint = 'http://shortna.me/hash/'
    def get_args(self, url):
        return dict(snURL=url, api=0)

    def process_response(self, resp):
        ret = UrlShortener.process_response(self, resp)
        import lxml.html as HTML
        doc = HTML.fromstring(ret)
        links = doc.findall('a')
        for link in links:
            href = link.attrib.get('href')
            if href is not None and href.startswith('http://shortna.me/') and href != 'http://shortna.me':
                return href

        raise UrlShortenerException('short link not found in %r' % ret, ret)

# not currently used
class digsby_shortener(UrlShortener):
    endpoint = 'https://accounts.digsby.com/api/shorturl'
    def get_args(self, url):
        import common
        import hashlib
        username = common.profile.username.encode('utf8')
        password = hashlib.sha256(common.profile.password.encode('utf8')).digest()

        return {'user':username, 'pass':password, 'link' : url}

    def process_response(self, httpresp):
        ret = UrlShortener.process_response(self, httpresp)

        resp = simplejson.loads(ret)
        if resp['shorter']['status'] == 'error':
            raise UrlShortenerException(resp['shorter']['errormsg'])

        elif resp['shorter']['status'] == 'ok':
            import common
            url = resp['shorter']['shortURL']
            to_add = common.pref('urlshorteners.digsby.append_text', type = unicode, default = u'')
            return url + to_add

class bitly_shortener(UrlShortener):
    login = 'digsby'
    api_key = 'R_1fdb0bb8ce9af01f9939c2ffdf391dc8'
    endpoint = 'http://api.bit.ly/shorten'

    def __init__(self, login=None, api_key=None):
        if login is not None:
            self.login = login
        if api_key is not None:
            self.api_key = api_key

    def get_args(self, url):
        return dict(longUrl=url, version='2.0.1', login=self.login, apiKey=self.api_key)

    def process_response(self, resp):
        ret = UrlShortener.process_response(self, resp)

        try:
            info = simplejson.loads(ret)
        except Exception:
            raise UrlShortenerException('expected JSON')
        else:
            if info['errorCode'] == 0:
                return self.extract_shorturl(info)
            else:
                raise UrlShortenerException(info['errorMessage'])

    def extract_shorturl(self, info):
        return info['results'].values()[0]['shortUrl']


class digsby_bitly_shortener(bitly_shortener):
    def extract_shorturl(self, info):
        return "http://digs.by/" + info['results'].values()[0]['userHash']


# TODO: add bit.ly, cli.gs, tr.im
_shorteners = {
   #'snipr'    : snipr_shortener,
   #'snurl'    : snipr_shortener,
   #'snipurl'  : snipr_shortener,      # has a new API we didn't implement yet
    'isgd'     : isgd_shortener,
    'tinyurl'  : tinyurl_shortener,
    'tiny'     : tinyurl_shortener,
    'threely'  : threely_shortener,
    '3ly'      : threely_shortener,
    'shortname': shortname_shortener,
    'digsby'   : digsby_bitly_shortener,
    }

def get_short_url(url, provider=None, choices = None):
    """
    Gets a shortened url from 'provider' through their api.

    Intended to be used with threaded:
    threaded(get_short_url)(url, 'tinyurl', success=func, error=func)

    @param url: The URL to be snipped
    @param provider: The shortening service to use.
    """
    if choices is None:
        choices = list(_shorteners.keys())
    choices = choices[:]
    random.shuffle(choices)

    if provider is not None:
        choices.append(provider)
    else:
        import common
        choices.append(common.pref("url_shortener.default", type = basestring, default = 'digsby'))

    e = None
    while choices:
        try:
            provider = choices.pop()
            shortener = _shorteners.get(provider)
            if shortener is None:
                raise Exception("UrlShortener provider %r not found", provider)

            return shortener().shorten(url)
#        except UrlShortenerException, e:
#            log.error('error getting short URL from %r: %r', provider, e)
#            raise e
        except Exception, e:
            log.error('error getting short URL from %r: %r', provider, e)
            shortener = provider = None

    if e is None:
        e = Exception('No shorteners found!')
    # none of them worked
    raise e


def wget(url, data=None):
    '''
    return urllib2.urlopen(url, data).read()
    '''
    from contextlib import closing
    import urllib2
    with closing(urllib2.urlopen(url, data=data)) as web:
        return web.read()

def long_url_from_cache(shorturl):
    try:
        return _short_url_cache[shorturl]
    except KeyError:
        return None

def unshorten_url(url, cb):
    longurl = long_url_from_cache(url)
    if url is not None:
        return cb(longurl)

    requrl = UrlQuery('http://untiny.me/api/1.0/extract',
                      url=url, format='json')

    def success(req, resp):
        json = resp.read()
        unshortened_url = simplejson.loads(json)['org_url']
        cb(unshortened_url)

    def error(req, resp):
        pass

    import common.asynchttp as asynchttp
    return asynchttp.httpopen(requrl, success=success, error=error)

def timestamp_to_http_date(ts):
    return rfc822.formatdate(timeval=ts)

def http_date_to_timestamp(date_str):
    if date_str is None:
        return None
    return calendar.timegm(rfc822.parsedate(date_str))

def user_agent():
    return 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2.10) Gecko/20100914 Firefox/3.6.15'

if __name__ == '__main__':
    print get_snurl('http://www.google.com')


