#!/usr/bin/python
# -*- coding: utf-8 -*-
import urllib2
import socks
import os,sys
import requests
import feedparser
import datetime
import chardet
import socket
import traceback
from urlparse import urlparse,ParseResult
from pyquery import PyQuery as pq
from sockshandler import SocksiPyHandler
from envelopes import Envelope

mymailaddr = None
amazonaddr = None
mypassword = None
mysmtpsvr = None


class PySpider:
    def __init__(self,enableproxy=False, proxyhost='127.0.0.1', proxyport=1080, tmpdir='/tmp/spider',feeds=["http://chinadigitaltimes.net/chinese/category/level-2-article/feed/"],mymailaddr='usr@example.com',amazonaddr='someusr@kindle.cn',mypassword='pswd',mysmtpsvr='my.smtp.com'):
        self.useproxy=enableproxy
        self.switchdict = {
                'text/html': lambda : 'html',
                'text/css' : lambda  : 'css',
                'image/jpeg' : lambda : 'jpg',
                'text/xml' : lambda  : 'xml',
                'image/png' : lambda : 'png',
                'application/x-png' : lambda : 'png',
                'application/x-bmp' : lambda : 'bmp',
                'application/x-javascript': lambda :'js',
                'application/rss+xml' : lambda : 'xml',
                '*/*':lambda :''
                }
        self.feeds = feeds
        self.mymailaddr = mymailaddr
        self.amazonaddr = amazonaddr
        self.mypassword = mypassword
        self.mysmtpsvr = mysmtpsvr
        self.tmpdir = unicode(tmpdir)   
        if(not os.path.exists(tmpdir)):
            os.mkdir(tmpdir)

        if(enableproxy):
            self.opener = urllib2.build_opener(SocksiPyHandler(socks.SOCKS5, proxyhost, proxyport))
     
    def run(self):
        title, posts = self.parse_rss()
        htmlf = 'index.html'
        htmlf = os.path.join(self.tmpdir, htmlf)
        f = open(htmlf, 'wb')

        string = u'<html>\n<title>' + title + u'</title>\n<body>\n'
        for arctitle in posts:
            htm = './' + self.get_filename(MIME='text/html',url=posts[arctitle])
            string += u'<p><a href=' + u'"' + htm + u'"' + u'>' + title + u'</a></p>\n'
        string += u'</body>\n</html>'
        f.write(string.encode('utf-8'))
        f.close()
        for k in posts:
            self.savePage(posts[k])

        cmdline = 'ebook-convert ' + htmlf + ' "' + os.path.join(self.tmpdir, title + '.mobi"')
        print cmdline
        os.system(cmdline.encode('utf-8'))
        return os.path.join(self.tmpdir, title + '.mobi')
    #env = Envelope(from_addr=mymailaddr,to_addr=amazonaddr, subject='convert')
    #env.add_attachment(os.path.join(self.tmpdir, title + '.rtf'))
    #print "Sending Mail From " + mymailaddr + " To " +amazonaddr
    #env.send(mysmtpsvr,login=mymailaddr, password=mypassword, tls=True)


    def parse_rss(self):
        posts = {}
        #title url
        for feed in self.feeds:
            try:
                rss,code,mime = self.http_get(feed)
                rss = feedparser.parse(rss)
                for entry in rss.entries:
                    url = entry.link
                    try:
                        url = unicode(url)
                    except:
                        print "string " + url + " already a utf8"
                    posts[entry.title] = url
                print rss.channel.title.encode('utf-8')
                return rss.channel.title + " Generated at " + str(datetime.datetime.now), posts
            except:
                print "Error downloading rss " + feed
		print traceback.print_exc()
		raise sys.exc_info()[0]
    def http_get(self, url):
            MIME = '*/*'
            unquoteurl = urllib2.unquote(url.encode('utf-8'))
            scheme,netloc,url,params,query,fragment = urlparse(unquoteurl)
	    netloc=urllib2.quote(netloc)
            url = urllib2.quote(url)
            url = ParseResult( scheme,netloc,url,params,query,fragment ).geturl()
            retries = 30
            i = 0
            while(True):
                try:
                    if(self.useproxy):
			print 'using proxy'
                        response = self.opener.open(url,timeout=5)
                        print("GET "  + urllib2.unquote(response.geturl().encode()) + " " + str(response.code))
                        if('content-type' in response.headers):
                            MIME = response.headers['content-type'].split(';')[0]
			print response
                        return response.read(), response.code, MIME
                    else:
                        response = requests.get(url)
                        print("GET "  + urllib2.unquote(str(response.url)) + " " + str(response.status_code))
                        if('content-type' in response.headers):
                            MIME = response.headers['content-type'].split(';')[0]
                        return response.content, response.status_code, MIME
                except:
                    if(i > retries):
                        print traceback.print_exc()
                        raise sys.exc_info()[0]
                    print "timeout 5000ms"
                    i += 1

    def get_filename(self,MIME,url):
        if(url.endswith('/')):
            url = url[0:len(url) - 1]
            splits = url.split('/')
            try:
                return splits[len(splits) -1].split('?')[0] + '.' + self.switchdict[MIME]()
            except KeyError:
                return splits[len(splits) -1].split('?')[0]
        else:
            splits = url.split('/')
            return splits[len(splits) -1].split('?')[0]


    def saveContent(self, isPage=False, content=None, targeturl='http://www.example.com/dir/example.txt'):
        #example : we have a file http://example.com/dir/somefile.txt
        #rooturl = http://example.com/
        #url - rooturl = dir/somefile.txt
        pos = u'./'
        MIME = None
        if(content is None):
            try:
                content,code,MIME = self.http_get(targeturl)
            except:
                print "FAIL over 30 times , giving up"
                return "ERROR", targeturl
        elif(isPage is True):
            MIME = u'text/html'
        try:
            filename = self.get_filename(MIME,  urllib2.unquote(targeturl.encode('utf-8')).decode('utf-8'))
        except UnicodeEncodeError:
            splits = filename
        if(isPage):
            pos = os.path.join(pos, filename)
            self.filesdir = urllib2.unquote(filename.split('.')[0]) + u'_files'
            if(not os.path.exists(os.path.join(self.tmpdir, self.filesdir))):
                os.mkdir(os.path.join(self.tmpdir, self.filesdir))
        else:
            pos = os.path.join(os.path.join(pos, self.filesdir), filename)
        try:
            ofile = open(os.path.join(self.tmpdir, pos),'wb')
            ofile.write(content)
            ofile.close()
        except IOError:
            print "Unable to save " + targeturl
        return content, pos

    def get_ext_by_MIME_type(self,MIME):
        return self.switchdict[MIME]()


    def savePage(self, targeturl, recursive = False):
        #download the main page first
        #go through the head part of html and download all the css style files and all js files
        #go through the html and download all the pictures
        #requirement: keep directory hiratchy
        content,pos = self.saveContent(isPage=True, targeturl=targeturl)
        html = pq(content)
        head = html('head')
        body = html('body')

        #go through <script src.....>
        if(html('link') is not None):
            for css in html('link'):
                if('href' in css.attrib and 'type' in css.attrib):
                    ctx,dir = self.saveContent(targeturl=css.attrib['href'])
                    if(type(dir) is unicode):
                        dir = dir.encode('utf-8')
                    css.attrib['href'] = urllib2.quote(dir)

        if(html('img') is not None):
            for img in html('img'):
                if('src' in img.attrib):
                    targets = img.attrib['src'].split(',')
                    dirs = []
                    for element in targets:
                        ctx, dir = self.saveContent(targeturl=element)
                        if(type(dir) is unicode):
                            dir = dir.encode('utf-8')
                        dirs.append(dir)
                    attr = ""
                    for dir in dirs:
                        dir += ','
                        attr += dir
                    attr = attr[0:len(attr) - 1]
                    img.attrib['src'] = urllib2.quote(attr)

        if(html('script' is not None)):
            for script in html('script'):
                if('src' in script.attrib): #if tag has a src, then download the script
                    ctx,dir = self.saveContent(targeturl=script.attrib['src'])
                    if(type(dir) is unicode):
                        dir = dir.encode('utf-8')
                    script.attrib['src'] = urllib2.quote(dir)

        self.saveContent(isPage=True,content=str(html),targeturl=targeturl)
        #for files like http://example.com/get/file/name/
        #if we can get things from this url (There is a file pointing to here)
        #rename name to name.html, in case something other happens
        #like : IOError: [Errno 21] Is a directory: '/tmp/spider/6/03/ngocn%e5%8e%9f%e5%88%9b-%e4%bd%a0%e4%bb%ac%e8%bf%87%e7%9a%84%e6%98%af%e5%a5%b3%e7%94%9f%e8%8a%82-%e8%bf%98%e6%98%af%e6%80%a7%e9%aa%9a%e6%89%b0%e8%8a%82%ef%bc%9f/feed/'

def usage():
    print "PySpider.py -mymailaddr mymail -mysmtpsvr smtp_svraddr -amazonaddr amazon_addr -mypassword pass"

if(__name__=='__main__'):
    print len(sys.argv)

    if(len(sys.argv) < 9):
        print "wrong args"
        usage()
       # exit(-1)
    for arg in ['-mysmtpsvr','-mymailaddr','-mypassword','-amazonaddr']:
        if(arg not in sys.argv):
            print "Wrong arguments"
        usage()
       # exit(-1)

 #   for i in range(0, len(sys.argv)):
  #      if(i > 0 and i % 2 == 1):
  #          argname = sys.argv[i]
  #      arg = sys.argv[i + 1]
  #      if(argname == '-mymailaddr'):
  #          mymailaddr = arg
  #      elif(argname == '-mysmtpsvr'):
  #          mysmtpsvr = arg
  #      elif(argname == '-amazonaddr'):
  #          amazonaddr = arg
  #      elif(argname == '-mypassword'):
  #          mypassword = arg

    print mymailaddr
    print mysmtpsvr
    print amazonaddr
    print mypassword

    #spider.savePage('http://chinadigitaltimes.net/chinese/2016/03/%E3%80%90%E5%BC%82%E9%97%BB%E8%A7%82%E6%AD%A2%E3%80%91%E5%85%B1%E9%9D%92%E5%9B%A2%E4%B8%AD%E5%A4%AE%EF%BC%9A%E6%88%91%E6%81%8B%E7%88%B1%E4%BA%86-%E5%92%8C%E4%BB%96/')
    PySpider(mymailaddr=mymailaddr, mysmtpsvr=mysmtpsvr, amazonaddr=amazonaddr, mypassword=mypassword).run()
