i am zww
'''
Created on 2012-6-7

@author: diracfang
'''

from BeautifulSoup import BeautifulSoup
import httplib2
import random
import re
import urllib2
import urlparse

class RandomSpider(object):
    """spider for get some random page url from website\
    
    currently available sites:
    ifanr, engadget, 163, sina, sohu
    
    you can specify sites manually using self.specify_sites(site_names)
    """
    _site_pattern = {
#                      'www.ifanr.com': 'http://www.ifanr.com/\d+',
#                      'cn.engadget.com': 'http://cn.engadget.com/\d+',
                      'www.dapenti.com/blog':'http://www.dapenti.com/blog/more.asp?name=xilei&id=\d+',
#                      'news.sina.com.cn': 'http://news.sina.com.cn/(pl|[cswm])/.+',
#                      'news.sohu.com': 'http://news.sohu.com/\d+',
                      }
#                      'cn.engadget.com': '/\d+',
    
    def __init__(self):
        for i in self._site_pattern:
            print self._site_pattern[i]
            self._site_pattern[i] = re.compile(self._site_pattern[i])
        
        self._sites = ['http://%s' % x for x in self._site_pattern.keys()]
        
            
    def _get_page(self, url):
        try:
            page = urllib2.urlopen(url).read()
        except:
            page = None
        
        return page
    
    def _choose_site(self):
        return random.choice(self._site_pattern.keys())
    
    def _get_index_page(self):
        url = 'http://%s' % self.chosen_site
        page = None
        
        for i in range(10):
            try:
                page = self._get_page(url)
            except Exception, e:
                print e
            else:
                break
        
        return url, page
    
    def _is_required_pattern(self, index_url, link):
        is_required_pattern = False
        if self._is_same_domain(index_url, link):
            domain = urlparse.urlparse(index_url)[1]
            pattern = self._patterns[domain]
            path = urlparse.urlparse(link)[2]
#            print path,
            if pattern.match(path):
                is_required_pattern = True
#                print True
#            else:
#                print False
        
        return is_required_pattern
    
    def _get_links(self, url, page):
        soup = BeautifulSoup(page)
        links = []
        a = soup.findAll('a')
        for tag in a:
            link = tag.get('href', None)
            print '1', link
            link1 = urlparse.urljoin(url, link)
            print '2', link1
            link2 = httplib2.iri2uri(link)
            print '2', link2
            
            pattern = self._site_pattern[self.chosen_site]
            if pattern.match(link):
                print link
#            path = urlparse.urlparse(link)[2]
##            print path,
#            if pattern.match(path):
#                is_required_pattern = True
            
#            link = urlparse.urljoin(index_url, link)
#            link = httplib2.iri2uri(link)
#            if link and self._is_required_pattern(index_url, link):
#                links.append(link)
        
#        for tag in a:
#            link = tag.get('href', None)
#            link = urlparse.urljoin(index_url, link)
#            link = httplib2.iri2uri(link)
#            if link and self._is_required_pattern(index_url, link):
#                links.append(link)
#        links_set = set(links)
#        links = list(links_set)
                
        return links
    
    def _is_same_domain(self, link1, link2):
        domain1 = urlparse.urlparse(link1)[1]
        domain2 = urlparse.urlparse(link2)[1]
        if domain1 == domain2:
            return True
        else:
            return False
        
    def _is_link_valid(self, possible_link):
        page = self._get_page(possible_link)
        if page:
            return True
        else:
            return False
            
    def get_valid_url(self):
        valid_link = None
        self.chosen_site = self._choose_site()
        
        for i in range(10):
            url, page = self._get_index_page()
            if url and page:
                break
            
        if page:
            links = self._get_links(url, page)
        else:
            links = []
            
        if links:
            for i in range(10):
                possible_link = random.choice(links)
                if self._is_link_valid(possible_link):
                    valid_link = possible_link
                    break
        
        return valid_link


if __name__ == '__main__':
    s = RandomSpider()
    print s._sites
#    s.specify_sites(['ifanr'])
    for i in range(1):
        url = s.get_valid_url()
        if url:
            print 'chosen: %s' % url
        else:
            print 'not found'
        
