#!/usr/bin/env python
#-*- coding: UTF-8 -*-

__author__ = 'helljump'

from PyQt4 import QtGui
from PyQt4 import QtCore
from PyQt4 import Qsci
import logging
import startup
from datetime import datetime
import time
import os
import re
import transaction
import random
from utils import MyProgressDialog
from persistent.dict import PersistentDict
import codecs
import unicodecsv as csv
from dbobj import Document
from ui.unique_prefs_ui import Ui_Dialog as Ui_PrefsDialog
from ui.unique_table_ui import Ui_Dialog as Ui_TableDialog
from ui.unique_text_ui import Ui_Dialog as Ui_TextDialog
from dbobj import Template
import sys
from Levenshtein import *
from difflib import SequenceMatcher


log = logging.getLogger(__name__)


STRIP = re.compile('<[^>]+>|[\s\.,\?!]+|&\w+;', re.I | re.S | re.M | re.U)
UNIQLIMIT = 100
JUNK = " .,!?:-+"


class Model(QtCore.QAbstractTableModel):

    def __init__(self, docs, found, parent=None):
        super(Model, self).__init__(parent)
        self.labels = [u'Документ', u'Документ', u'Схожесть']
        self.docs = docs
        self.found = found

    def rowCount(self, parent=QtCore.QModelIndex()):
        return len(self.found)

    def columnCount(self, parent=QtCore.QModelIndex()):
        return len(self.labels)

    def headerData(self, section, orientation, role):
        egg = QtCore.QVariant()
        if orientation == QtCore.Qt.Horizontal and role == QtCore.Qt.DisplayRole:
            egg = QtCore.QVariant(self.labels[section])
        elif orientation == QtCore.Qt.Vertical and role == QtCore.Qt.DisplayRole:
            egg = QtCore.QVariant(section + 1)
        return egg

    def flags(self, index):
        egg = QtCore.Qt.ItemIsSelectable | QtCore.Qt.ItemIsEnabled
        return egg

    def data(self, index, role=QtCore.Qt.DisplayRole):
        if not index.isValid():
            return None
        item = self.found[index.row()]
        ndx = index.column()
        egg = QtCore.QVariant()
        if role == QtCore.Qt.DisplayRole:
            if ndx == 0:
                egg = self.docs[item[0]].title
            elif ndx == 1:
                egg = self.docs[item[1]].title
            elif ndx == 2:
                egg = "%i%%" % (item[2] * 100)
        elif role == QtCore.Qt.UserRole + 1:
            return item
        return egg


class TextDialog(QtGui.QDialog, Ui_TextDialog):
    def __init__(self, doc1, doc2, parent=None):
        super(TextDialog, self).__init__(parent)
        self.setupUi(self)
        if startup.CFG.contains('unique/text/geometry'):
            self.restoreGeometry(startup.CFG.value('unique/text/geometry').toByteArray())
        self.text1_te.verticalScrollBar().valueChanged.connect(self.text2_te.verticalScrollBar().setValue)
        self.text2_te.verticalScrollBar().valueChanged.connect(self.text1_te.verticalScrollBar().setValue)
        self.text1_te.setPlainText(doc1.text)
        self.text2_te.setPlainText(doc2.text)
        self.remove = None

    @QtCore.pyqtSlot()
    def on_del1_pb_clicked(self):
        log.debug('del1')
        self.remove = 0
        self.hide()

    @QtCore.pyqtSlot()
    def on_del2_pb_clicked(self):
        log.debug('del2')
        self.remove = 1
        self.hide()

    def closeEvent(self, evt):
        startup.CFG.setValue('unique/text/geometry', self.saveGeometry())
        evt.accept()


class TableDialog(QtGui.QDialog, Ui_TableDialog):
    def __init__(self, found, project, parent=None):
        super(TableDialog, self).__init__(parent)
        self.setupUi(self)
        self.project = project
        self.found = found
        self.table_tv.setModel(Model(project.root()['documents'], found, self))
        self.table_tv.horizontalHeader().resizeSection(0, 350)
        self.table_tv.horizontalHeader().resizeSection(1, 350)
        if startup.CFG.contains('unique/geometry'):
            self.restoreGeometry(startup.CFG.value('unique/geometry').toByteArray())
        self.parent = parent
        self.cache = set()

    @QtCore.pyqtSlot()
    def on_actionClear_triggered(self):
        log.debug('clear all')
        rc = QtGui.QMessageBox.question(self, u'Очистка', u'Удалить дубли документов?', QtGui.QMessageBox.Yes | QtGui.QMessageBox.No)
        if rc == QtGui.QMessageBox.No:
            return
        docs = self.project.root()['documents']
        pdlg = MyProgressDialog(u'Очистка', u'Сортировка дублей', u'Отмена', 0, len(self.found), parent=self)
        pdlg.show()
        rows = self.cache
        for i, row in enumerate(self.found):
            rows.add(row[1])
            pdlg.set_value(i)
            QtGui.qApp.processEvents()
            if pdlg.wasCanceled():
                return
        rows = list(rows)
        rows.sort(reverse=True)
        pdlg.set_range(0, len(rows))
        pdlg.set_value(0)
        pdlg.set_text(u'Удаление')
        for i, row in enumerate(rows):
            del docs[row]
            pdlg.set_value(i)
            QtGui.qApp.processEvents()
            if pdlg.wasCanceled():
                transaction.abort()
                return
        transaction.commit()
        self.cache = set()
        pdlg.close()
        self.setResult(1)
        self.hide()

    @QtCore.pyqtSlot(QtCore.QModelIndex)
    def on_table_tv_doubleClicked(self, index):
        log.debug('open docs')
        data = self.table_tv.model().data(index, QtCore.Qt.UserRole + 1)
        docs = self.project.root()['documents']
        dlg = TextDialog(docs[data[0]], docs[data[1]], self)
        dlg.exec_()
        if dlg.remove is None:
            return
        pdlg = MyProgressDialog(u'Обработка', u'Удаление дублей', u'Отмена', 0, 0, parent=self)
        pdlg.show()
        delrow = data[dlg.remove]
        self.cache.add(delrow)
        del self.found[index.row()]
        self.table_tv.model().layoutChanged.emit()
        pdlg.close()

    def closeEvent(self, evt):
        if self.parent is not None:
            if self.cache:
                docs = self.project.root()['documents']
                for row in sorted(list(self.cache), reverse=True):
                    del docs[row]
                transaction.commit()
                self.parent.model.layoutChanged.emit()
        startup.CFG.setValue('unique/geometry', self.saveGeometry())
        evt.accept()


class PrefsDialog(QtGui.QDialog, Ui_PrefsDialog):
    def __init__(self, project, parent=None):
        super(PrefsDialog, self).__init__(parent)
        self.setupUi(self)
        self.project = project
        root = project.root()
        prefs = root.get('prefs', {}).get('unique', {})
        self.unique_sl.setValue(prefs.get('procent', 98))

    def accept(self):
        root = self.project.root()
        root['prefs']['unique'] = {
            'procent': self.unique_sl.value(),
        }
        transaction.commit()
        self.setResult(1)
        self.hide()


def junker(x):
    return x in JUNK


def process(prj, parent=None):
    dlg = PrefsDialog(prj, parent)
    rc = dlg.exec_()
    if not rc:
        return
    thres = float(dlg.unique_sl.value()) / 100

    docs = prj.root()['documents']
    pdlg = MyProgressDialog(u'Обработка', u'Расчет уникальности', u'Отмена', 0, len(docs), parent=parent)
    pdlg.show()
    found = []
    for i, doc in enumerate(docs):
        assert isinstance(doc.text, unicode)
        matcher = SequenceMatcher(junker, '', STRIP.sub(' ', doc.text))
        pdlg.set_value(i)
        for j in range(i, len(docs)):
            cmpdoc = docs[j]
            if doc is cmpdoc:
                continue
            if not hasattr(cmpdoc, '_v_strip'):
                cmpdoc._v_strip = STRIP.sub(' ', cmpdoc.text)
            matcher.set_seq1(cmpdoc._v_strip)
            rc = matcher.quick_ratio()
            if rc > thres:
                found.append([i, j, rc])
            if j % 10 == 0:
                QtGui.qApp.processEvents()
            if pdlg.wasCanceled():
                return
    pdlg.close()
    if not found:
        QtGui.QMessageBox.information(parent, u'Очистка', u'Дублей с указанным порогом уникальности не найдено.')
        return
    dlg = TableDialog(found, prj, parent)
    rc = dlg.exec_()
    if rc:
        if parent is not None:
            parent.model.layoutChanged.emit()


if __name__ == '__main__':

    '''
    s = SequenceMatcher(lambda x: x in " .,!?",
        'Изба моя, приветь меня Бутерброд моя, приветь меня, Побалуй давней лаской отчей. Погрей у своего огня,',
        'Изба моя, приветь меня Бутерброд моя, приветь меня, Побалуй давней лаской отчей. Пугрей у своего огня,')
    print s.ratio()
    print s.quick_ratio()
    print s.real_quick_ratio()
    '''

    logging.basicConfig(level=logging.DEBUG)
    root = startup.CONN.root()
    prj = root["projects"][0]
    prj.open()
    process(prj)
    prj.close()

