#!/usr/bin/env python
#-*- coding: UTF-8 -*-

__author__ = 'helljump'


# TODO: парсинг выдачи пауком


import logging
from grab.spider import Spider, Task
from grab.error import DataNotFound
from urlparse import urljoin
from PyQt4 import QtCore
from PyQt4 import QtGui
from PyQt4 import QtWebKit
import startup
from dbobj import Document
from ui.spider_ui import Ui_Dialog
from ui.webbuilder_ui import Ui_Dialog as Ui_WBDialog
import transaction
import os
from utils import MyProgressDialog
import types


log = logging.getLogger(__name__)


class Worker(QtCore.QThread):

    def __init__(self, bot, parent):
        QtCore.QThread.__init__(self, parent)
        self.bot = bot

    def run(self):
        try:
            self.bot.run()
        except:
            log.exception('Spider exception')


class Handler(QtCore.QObject):

    new_doc = QtCore.pyqtSignal(Document)
    spider_stop = QtCore.pyqtSignal()


class SimpleSpider(Spider):

    def __init__(self, start_url, links, titles, texts, *args, **kwargs):
        super(SimpleSpider, self).__init__(*args, **kwargs)
        self.start_url = start_url.lower()
        self.titles = titles
        self.texts = texts
        self.links = links
        self.initial_urls = [start_url]
        self.handler = Handler()
        self.handler.spider_stop.connect(self.stop)
        #a = urlparse(start_url.lower())
        #b = a.netloc.split('.')
        #self.base = '.'.join(b[-2:])
        #self.base = a.netloc

    def prepare(self):
        self.skip = set()
        self.found = list()

    def task_initial(self, grab, task):
        for link in self.links:
            for elem in grab.doc.select(link):
                try:
                    u = urljoin(self.start_url, elem.attr('href'))
                    if u in self.skip or not u.lower().startswith(self.start_url):
                        continue
                    self.skip.add(u)
                    yield Task('initial', url=u)
                    yield Task('page', url=u)
                except DataNotFound:
                    log.info('bad href')
                except:
                    log.exception('spider initial task except')

    def task_page(self, grab, task):
        try:
            for sel in self.titles:

                #print "------------------->", sel
    
                assert type(sel) == unicode
                title = grab.doc.select(sel)
                

                if title:
                    break
            else:
                return
            for sel in self.texts:
                assert type(sel) == unicode
                text = grab.doc.select(sel)
                if text:
                    doc = Document(
                        title=unicode(title[0].text()),
                        text=text[0].html().replace('&#13;', '')
                    )
                    self.handler.new_doc.emit(doc)
                    break
        except:
            log.exception('spider page except')


def mousePressEvent(self, e):
    QtWebKit.QWebView.mousePressEvent(self, e)
    pos = e.pos()
    page = self.page()
    frame = page.frameAt(pos)
    self.qpoint = frame.scrollPosition()
    content = frame.hitTestContent(pos)
    if not content.linkElement().isNull():
        elem = content.linkElement()
    elif not content.enclosingBlockElement().isNull():
        elem = content.enclosingBlockElement()
    else:
        elem = content.element()
    self.qrect = elem.geometry()
    self.repaint()
    egg = unicode("/" + self.dompath(elem))
    self.titleChanged.emit(egg.lower())


def paintEvent(self, e):
    QtWebKit.QWebView.paintEvent(self, e)
    if not self.qrect:
        return
    qp = QtGui.QPainter()
    qp.begin(self)
    pen = QtGui.QPen(QtCore.Qt.red, 2, QtCore.Qt.SolidLine)
    xp1, yp1, xp2, yp2 = self.qrect.getRect()
    xp1 -= self.qpoint.x()
    yp1 -= self.qpoint.y()
    qp.setPen(pen)
    qp.drawRect(xp1, yp1, xp2, yp2)
    qp.end()
    self.qrect = None


def dompath(self, elem):
    egg = elem.tagName()
    attrs = ["@%s=\"%s\"" % (name, elem.attribute(name)) for name in elem.attributeNames() if name in ["id", "class"]]
    if attrs:
        egg += "[%s]" % " and ".join(attrs)
    if elem.tagName():
        egg = self.dompath(elem.parent()) + "/" + egg
    return egg


class WBDialog(QtGui.QDialog, Ui_WBDialog):

    def __init__(self, parent=None):
        super(WBDialog, self).__init__(parent)
        self.setupUi(self)
        self.webView.page().setLinkDelegationPolicy(QtWebKit.QWebPage.DelegateAllLinks)
        self.webView.linkClicked.connect(self.alt_click)
        self.webView.qrect = None
        self.webView.qpoint = None
        self.webView.mousePressEvent = types.MethodType(mousePressEvent, self.webView)
        self.webView.paintEvent = types.MethodType(paintEvent, self.webView)
        self.webView.dompath = types.MethodType(dompath, self.webView)
        if startup.CFG.contains('wbdialog/geometry'):
            self.restoreGeometry(startup.CFG.value('wbdialog/geometry').toByteArray())

    def closeEvent(self, evt):
        startup.CFG.setValue('wbdialog/geometry', self.saveGeometry())
        evt.accept()

    @QtCore.pyqtSlot(QtCore.QString)
    def on_webView_titleChanged(self, v):
        if unicode(v).startswith('//'):
            self.xpath_le.text = unicode(v)

    @QtCore.pyqtSlot()
    def on_xpath_le_clicked(self):
        self.setResult(1)
        self.hide()

    @QtCore.pyqtSlot()
    def on_url_le_clicked(self):
        self.webView.load(QtCore.QUrl(self.url_le.text))

    @QtCore.pyqtSlot(QtCore.QUrl)
    def on_webView_urlChanged(self, url):
        self.url_le.text = url.toString()

    def alt_click(self, url):
        modifiers = QtGui.QApplication.keyboardModifiers()
        if modifiers == QtCore.Qt.AltModifier:
            self.webView.load(url)


class Dialog(QtGui.QDialog, Ui_Dialog):

    def __init__(self, project, parent=None):
        super(Dialog, self).__init__(parent)
        self.setupUi(self)
        self.prefs = allprefs = project.root()['prefs']
        myprefs = allprefs.get('spider', {})
        self.start_le.setText(myprefs.get('start', 'http://www.ru/rus/index.php'))
        self.links_te.setPlainText(myprefs.get('links', '//a'))
        self.titles_te.setPlainText(myprefs.get('titles', '//h3'))
        self.texts_te.setPlainText(myprefs.get('texts', '//p'))
        self.threads_sb.setValue(myprefs.get('threads', 3))
        self.debug_cb.setChecked(myprefs.get('debug', False))

    def select_xpath(self, control):
        dlg = WBDialog(self)
        dlg.url_le.text = self.start_le.text()
        rc = dlg.exec_()
        if not rc:
            return
        egg = dlg.xpath_le.text
        if egg.isEmpty():
            return
        control.appendPlainText(egg)

    @QtCore.pyqtSlot()
    def on_link_tb_clicked(self):
        log.debug('link tb')
        self.select_xpath(self.links_te)

    @QtCore.pyqtSlot()
    def on_title_tb_clicked(self):
        log.debug('title tb')
        self.select_xpath(self.titles_te)

    @QtCore.pyqtSlot()
    def on_text_tb_clicked(self):
        log.debug('text tb')
        self.select_xpath(self.texts_te)

    def accept(self):
        start = unicode(self.start_le.text()).strip()
        links = unicode(self.links_te.toPlainText()).strip()
        titles = unicode(self.titles_te.toPlainText()).strip()
        texts = unicode(self.texts_te.toPlainText()).strip()
        threads = self.threads_sb.value()
        debug = self.debug_cb.isChecked()
        if not start or not links or not titles or not texts:
            QtGui.QMessageBox.warning(self, u'Паук', u'Для запуска паука Вам необходимо заполнить все поля.')
            return
        self.prefs['spider'] = {
            'start': start,
            'links': links,
            'titles': titles,
            'texts': texts,
            'threads': threads,
            'debug': debug,
        }
        transaction.commit()
        self.setResult(1)
        self.hide()


def tonormlist(s):
    n = []
    for e in s.split('\n'):
        n.append(e.strip())#.lower())
    return n


def process(prj, parent=None):

    dlg = Dialog(prj, parent)
    rc = dlg.exec_()
    if not rc:
        return

    bot = SimpleSpider(
        dlg.prefs['spider']['start'],
        tonormlist(dlg.prefs['spider']['links']),
        tonormlist(dlg.prefs['spider']['titles']),
        tonormlist(dlg.prefs['spider']['texts']),
        thread_number=dlg.prefs['spider']['threads']
    )

    bot.setup_grab(hammer_mode=True)  # , lowercased_tree=True)
    if dlg.prefs['spider']['debug']:
        bot.setup_grab(log_dir=startup.DUMPDIR)

    conn = prj.con()
    prefs = conn.root()['prefs']
    docs = conn.root()['documents']
    proxy_file = prefs.get('proxy_file', '')
    if proxy_file and os.path.isfile(proxy_file):
        bot.load_proxylist(proxy_file, 'text_file', auto_init=True, auto_change=True)

    pdlg = MyProgressDialog(u'Паук', u'Обход структуры', u'Отмена', parent=parent)

    def new_doc(doc):
        if __name__ != '__main__':
            docs.appendSorted(doc)
            transaction.commit()
        else:
            log.debug('spider fake append %s', doc.title)
            if len(docs) > 100:
                bot.handler.spider_stop.emit()
        pdlg.set_text(doc.title)

    def finished():
        bot.handler.new_doc.disconnect()
        bot.handler.spider_stop.emit()
        #conn.close()
        pdlg.close()

    bot.handler.new_doc.connect(new_doc)
    pdlg.canceled.connect(bot.handler.spider_stop)
    th = Worker(bot, parent)
    th.finished.connect(finished)
    th.start()
    #pdlg.show()
    pdlg.exec_()


if __name__ == '__main__':
    logging.basicConfig(level=logging.DEBUG)
    root = startup.CONN.root()
    prj = root["projects"][0]
    prj.open()
    process(prj)
    prj.close()
