#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Author  : Leslee
# @Email    : leelovesc@gmail.com
# @Time    : 2019.10.24 11:18
from __future__ import absolute_import,unicode_literals

import marshal
import os
import re
import sys
import tempfile
from hashlib import md5
import logging
import threading
import time
from math import log
from multiprocessing.pool import Pool

from ._compat import *

# 判断操作系统
if os.name == 'nt':
    from shutil import move as _replace_file
else:
    _replace_file = os.rename

_get_abs_path = lambda path:os.path.normpath(os.path.join(os.getcwd(),path))
# 默认词典
DEFAULT_DICT = None
DEFAULT_DICT_NAME = "dict.txt"
# 使用这个Handler可以向类似与sys.stdout或者sys.stderr的任何文件对象(file object)输出信息。
log_console = logging.StreamHandler(sys.stderr)
default_logger = logging.getLogger(__name__)
default_logger.setLevel(logging.DEBUG)
default_logger.addHandler(log_console)

DICT_WRITING = {}

pool = None

# re.U是指unicode匹配
re_userdict = re.compile('^(.+?)( [0-9]+)?( [a-z]+)?$',re.U)

re_eng = re.compile('[a-zA-Z0-9]',re.U)
# 所有不是空格的字符，都会被re_han捕获
# 精准分词模式的正则
re_han_default = re.compile("([\u4E00-\u9FA5a-zA-Z0-9+#&\._]+)",re.U)
re_skip_default = re.compile("(\r\n|\s)",re.U)
# 全局模式的正则
re_han_cut_all = re.compile("([\u4E00-\u9FA5]+)", re.U)#只保留汉字
re_skip_cut_all = re.compile("[^a-zA-Z0-9+#\n]", re.U)

def setLogLevel(log_level):
    global logger
    default_logger.setLevel(log_level)

class Tokenizer(object):

    def __init__(self,dictionary=DEFAULT_DICT):
        self.lock = threading.RLock() # 需要进一步了解多线程编程
        if dictionary == DEFAULT_DICT:
            self.dictionary = dictionary
        else:
            self.dictionary = _get_abs_path(dictionary)
        self.FREQ = {}
        self.total = 0
        self.user_word_tag_tab = {}
        self.initialized = False
        self.tmp_dir = None
        self.cache_file = None
# __repr__和__str__这两个方法都是用于显示的，__str__是面向用户的，而__repr__面向程序员。
# https://blog.csdn.net/luckytanggu/article/details/53649156
    def __repr__(self):
        return '<Tokenizer dictionary=%r>' % self.dictionary
 # 生成前缀集合，存储词语及其前缀。
    def gen_pfdict(self,f):
        lfreq = {} # 词条：词频
        ltotal = 0 # 记录总数
        f_name = resolve_filename(f)
        for lineno,line in enumerate(f,1):
            try:
                line = line.strip().decode("utf-8")
                word,freq = line.split(' ')[:2]
                freq = int(freq)
                lfreq[word] = freq
                ltotal += freq
                # 构建前缀集合
                for ch in range(len(word)):
                    wfrag = word[:ch+1]
                    if wfrag not in lfreq:
                        lfreq[wfrag] = 0
            except ValueError:
                raise ValueError('invalid dictionary entry in %s at Line %s: %s'
                                 % (f_name, lineno, line))
        f.close()
        return lfreq,ltotal
    # 初始化
    def initialize(self,dictionary=None):
        if dictionary:
            abs_path = _get_abs_path(dictionary)
            if self.dictionary == abs_path and self.initialized:
                return
            else:
                self.dictionary = abs_path
                self.initialized = False
        else:
            abs_path = self.dictionary

        with self.lock:
            try:
                with DICT_WRITING[abs_path]:
                    pass
            except KeyError:
                pass
            if self.initialized:
                return

            default_logger.debug("Building prefix dict from %s ..." % (abs_path or 'the default dictionary'))
            t1 = time.time()
            if self.cache_file:
                cache_file = self.cache_file
            # 默认词典
            elif abs_path == DEFAULT_DICT:
                cache_file = "jieba.cache"
            # 用户自定义词典
            else:
                cache_file = "jieba.u%s.cache" % md5(
                    abs_path.encode('utf-8','replace')).hexdigest()
            cache_file = os.path.join(self.tmp_dir or tempfile.gettempdir(),cache_file)
            # 防止self.cache的绝对路径？
            tmpdir = os.path.dirname(cache_file)

            load_from_cache_fail = True
            if os.path.isfile(cache_file) and (abs_path==DEFAULT_DICT or
                os.path.getmtime(cache_file)>os.path.getmtime(abs_path)):
                default_logger.debug(
                    "Loading model from cache %s" % cache_file)
                try:
                    with open(cache_file,'rb') as cf:
                        self.FREQ,self.total = marshal.load(cf)
                    load_from_cache_fail = False
                except Exception:
                    load_from_cache_fail = True
            if load_from_cache_fail:
                wlock = DICT_WRITING.get(abs_path,threading.RLock())
                DICT_WRITING[abs_path] = wlock
                with wlock:
                    self.FREQ,self.total = self.gen_pfdict(self.get_dict_file())
                    default_logger.debug("Dumping model to file cache %s" % cache_file)
                    try:
                        # 防止在不同的文件系统中移动
                        fd,fpath = tempfile.mkstemp(dir=tmpdir)
                        with os.fdopen(fd,'wb') as temp_cache_file:
                            marshal.dump((self.FREQ,self.total),temp_cache_file)
                        _replace_file(fpath,cache_file)
                    except Exception:
                        default_logger.exception("Dump cache file failed.")
                try:
                    del  DICT_WRITING[abs_path]
                except KeyError:
                    pass
            self.initialized = True
            default_logger.debug("Loading model cost %.3f seconds." % (time.time() - t1))
            default_logger.debug("Prefix dict has been built successfully.")

    def check_initialized(self):
        if not self.initialized:
            self.initialize()
    # 动态规划，计算最大概率的切分组合
    def calc(self,sentence,DAG,route):
        N = len(sentence)
        # 最大路径的值所在的路径
        route[N] = (0,0)
        # 概率值取对数，连乘变成连加
        logtotal = log(self.total)
        # 从后向前遍历句子，计算最大概率
        for idx in range(N-1,-1,-1):
            # 求解最大概率对数路径
            # route[idx] = max([(概率对数，词语末字位置) for x in DAG[idx]])
            # 以idx：(最大对数概率，词语末字位置)保存
            # route[x+1][0] 表示词路径[x+1,N-1]的最大概率对数
            route[idx] = max((log(self.FREQ.get(sentence[idx:x+1]) or 1)-
                              logtotal + route[x+1][0],x) for x in DAG[idx])
    # 构造DAG
    def get_DAG(self,sentence):
        self.check_initialized()
        DAG = {}
        N = len(sentence)
        for k in range(N):
            tmplist = []# 从字开始能在FREQ中的匹配到的词末尾位置所在的list
            i = k
            # 取传入词的值。如当k=0时，对应的字
            frag = sentence[k]
            # 当该字在词典中的时候，赋值给tmplist构建可能的词组合list
            while i<N and frag in self.FREQ:
                # 查找结尾下标，加入到tmplist
                if self.FREQ[frag]:
                    tmplist.append(i)
                i+=1 # 继续向后找
                frag = sentence[k:i+1]
            # 查询不到，给最小词频
            if not tmplist:
                tmplist.append(k)
            DAG[k] = tmplist
        return DAG

    def __cut_all(self,sentence):
        dag = self.get_DAG(sentence)
        old_j = -1
        # dag中，key是句子中字所在的index，value是个list，通过dict.txt构成。
        for k,L in iteritems(dag):
            # 单字成词
            if len(L) == 1 and k>old_j:
                yield sentence[k:L[0]+1]
                old_j = L[0]
            else:
                for j in L:
                    if j>k:
                        yield sentence[k:j+1]
                        old_j = j

    def __cut_DAG_NO_HMM(self,sentence):
        DAG = self.get_DAG(sentence)
        route = {}
        self.calc(sentence,DAG,route)
        x = 0
        N = len(sentence)
        buf = ''
        while x <N:
            y = route[x][1]+1
            l_word = sentence[x:y]
            if re_eng.match(l_word) and len(l_word) ==1:
                buf += l_word
                x = y
            else:
                if buf:
                    yield buf
                    buf = ''
                yield l_word
                x = y
        if buf:
            yield buf
            buf = ''

    def __cut_DAG(self,sentence):
        DAG = self.get_DAG(sentence)
        route = {}
        self.calc(sentence,DAG,route)
        x  = 0
        buf = ''
        N = len(sentence)
        while x < N:
            y = route[x][1] + 1
            l_word = sentence[x:y]
            # buf收集连续的单字，组成字符串，调用HMM进行分词。
            if y -x ==1:
                buf += l_word
            else:
                if buf:
                    if len(buf) == 1:
                        yield buf
                        buf = ''
                    else:
                        if not self.FREQ.get(buf):
                            recognized = finalseg.cut(buf)
                            for t in recognized:
                                yield t
                        else:
                            for elem in buf:
                                yield elem
                        buf = ''
                yield l_word
            x = y

        if buf:
            if(len(buf)==1):
                yield buf
            elif not self.FREQ.get(buf):
                recognized = finalseg.cut(buf)
                for t in recognized:
                    yield t
            else:
                for elem in buf:
                    yield elem

    # 主函数
    def cut(self,sentence,cut_all=False,HMM=True):

        sentence = strdecode(sentence)
        # 正则
        if cut_all:
            re_han = re_han_cut_all
            re_skip = re_skip_cut_all
        else:
            re_han = re_han_default
            re_skip = re_skip_default
        # 不同模式下的分词方法
        if cut_all:
            cut_block = self.__cut_all
        elif HMM:
            cut_block = self.__cut_DAG
        else:
            cut_block = self.__cut_DAG_NO_HMM
        # 先用正则切分句子
        blocks = re_han.split(sentence)
        for blk in blocks:
            if not blk:
                continue
            if re_han.match(blk):
                for word in cut_block(blk):
                    yield word
            else:
                tmp = re_skip.split(blk) # 使用re_skip对blk重新切分
                for x in tmp:
                    if re_skip.match(x):
                        yield x
                    elif not cut_all:
                        for xx in x:
                            yield xx
                    else:
                        yield x

    def cut_for_search(self,sentence,HMM=True):
        # 搜索引擎分词，
        words = self.cut(sentence,HMM=HMM)
        for w in words:
            if len(w) >2:
                for i in xrange(len(w)-1):
                    gram2 = w[i:i+2]
                    if self.FREQ.get(gram2):
                        yield gram2
            if len(w) >3:
                for i in range(len(w)-2):
                    gram3 = w[i:i+3]
                    if self.FREQ.get(gram3):
                        yield gram3
            yield w

    def lcut(self,*args,**kwargs):
        return list(self.cut(*args,**kwargs))

    def lcut_for_search(self,*args,**kwargs):
        return list(self.cut_for_search(*args,**kwargs))

    _lcut = lcut
    _lcut_for_search = lcut_for_search

    def _lcut_no_hmm(self,sentence):
        return self.lcut(sentence,False,False)

    def _lcut_all(self,sentence):
        return self.lcut(sentence,True)

    def _lcut_for_search_no_hmm(self,sentence):
        return self.lcut_for_search(sentence,False)

    def get_dict_file(self):
        if self.dictionary == DEFAULT_DICT:
            return get_module_res(DEFAULT_DICT_NAME)
        else:
            return open(self.dictionary,'rb')

    def load_userdict(self,f):

        self.check_initialized()
        if isinstance(f,string_types):
            f_name = f
            f = open(f,'rb')
        else:
            f_name = resolve_filename(f)
        for lineno,ln in enumerate(f,1):
            line = ln.strip()
            if not isinstance(line,text_type):
                try:
                    line = line.decode('utf-8').lstrip('\ufeff')
                except UnicodeDecodeError:
                    raise ValueError('dictionary file %s must be utf-8' % f_name)
            if not line:
                continue

            word,freq,tag = re_userdict.match(line).groups()
            if freq is not None:
                freq = freq.strip()
            if tag is not  None:
                tag = tag.strip()
            self.add_word(word,freq,tag)

    def add_word(self,word,freq=None,tag=None):

        self.check_initialized()
        word = strdecode(word)
        freq = int(freq) if freq is not None else self.suggest_freq(word,False)
        self.FREQ[word] = freq
        self.total += freq
        if tag:
            self.user_word_tag_tab[word] = tag
        for ch in xrange(len(word)):
            wfrag = word[:ch + 1]
            if wfrag not in self.FREQ:
                self.FREQ[wfrag] = 0
        if freq == 0:
            finalseg.add_force_split(word)

    def del_word(self, word):
        self.add_word(word,0)

    def suggest_freq(self,segment,tune=False):

        self.check_initialized()
        ftotal = float(self.total)
        freq = 1

        if isinstance(segment,string_types):
            word = segment
            for seg in self.cut(word,HMM=False):
                freq *= self.FREQ.get(seg,1) / ftotal
            freq = max(int(freq*self.total)+1,self.FREQ.get(word,1))
        else:
            segment = tuple(map(strdecode,segment))
            word = ''.join(segment)
            for seg in segment:
                freq *= self.FREQ.get(seg,1) / ftotal
            freq = min(int(freq*self.total),self.FREQ.get(word,0))
        if tune:
            self.add_word(word,freq)
        return freq

    def tokenize(self,unicode_sentence,mode="default",HMM=True):
        # 生成元祖(word,start,end)
        if not isinstance(unicode_sentence,text_type):
            raise ValueError("jieba: the input parameter should be unicode.")
        start = 0
        if mode =='default':
            for w in self.cut(unicode_sentence,HMM=HMM):
                width = len(w)
                yield (w,start,start+width)
                start += width
        # 细分
        else:
            for w in self.cut(unicode_sentence,HMM=HMM):
                width = len(w)
                if len(w) > 2:
                    for i in range(len(w)-1):
                        gram2 = w[i:i+2]
                        if self.FREQ.get(gram2):
                            yield (gram2,start+i,start+i+2)
                if len(w) >3:
                    for i in range(len(w)-2):
                        gram3 = w[i:i+3]
                        if self.FREQ.get(gram3):
                            yield (gram3,start+i,start+i+3)
                yield (w,start,start+width)
                start += width

    def set_dictionary(self,dictionary_path):
        with self.lock:
            abs_path = _get_abs_path(dictionary_path)
            if not os.path.isfile(abs_path):
                raise Exception("jieba: file does not exist:"+abs_path)
            self.dictionary = abs_path
            self.initialized = False


dt = Tokenizer()

# 全局函数
get_FREQ = lambda k ,d=None:dt.FREQ.get(k,d)
add_word = dt.add_word
calc = dt.calc
cut = dt.cut
lcut = dt.lcut
cut_for_search = dt.cut_for_search
lcut_cut_for_search = dt.lcut_for_search
del_word = dt.del_word
get_DAG = dt.get_DAG
get_dict_file = dt.get_dict_file
initialize = dt.initialize
load_userdict = dt.load_userdict
set_dictionary = dt.set_dictionary
suggest_freq = dt.suggest_freq
tokenizer = dt.tokenize
user_word_tag_tab = dt.user_word_tag_tab

def _lcut_all(s):
    return dt._lcut_all(s)

def _lcut(s):
    return dt._lcut(s)

def _lcut_no_hmm(s):
    return dt._lcut_no_hmm(s)

def _lcut_all(s):
    return dt._lcut_all(s)

def _lcut_for_search(s):
    return dt._lcut_for_search(s)

def _lcut_for_search_no_hmm(s):
    return dt._lcut_for_search_no_hmm(s)

def _pcut(sentence,cut_all=False,HMM=True):
    parts = strdecode(sentence).splitlines(True)
    if cut_all:
        result = pool.map(_lcut_all,parts)
    elif HMM:
        result = pool.map(_lcut,parts)
    else:
        result = pool.map(_lcut_no_hmm, parts)
    for r in result:
        for w in r:
            yield w

def _pcut_for_search(sentence,HMM=True):
    parts = strdecode(sentence).splitlines(True)
    if HMM:
        result = pool.map(_lcut_for_search,parts)
    else:
        result = pool.map(_lcut_for_search_no_hmm,parts)
    for r in result:
        for w in r:
            yield w
# 并行化
def enable_parallel(processnum=None):
    global pool,dt,cut,cut_for_search
    from multiprocessing import cpu_count
    if os.name == 'nt':
        raise NotImplementedError(
            "jieba: parallel mode only supports posix system")
    else:
        from multiprocessing import pool
    dt.check_initialized()
    if processnum is None:
        processnum = cpu_count()
    pool = Pool(processnum)
    cut = _pcut
    cut_for_search = _pcut_for_search

def disable_parallel():
    global pool,dt,cut,cut_for_search
    if pool:
        pool.close()
        pool = None
    cut = dt.cut
    cut_for_search = dt.cut_for_search
