#!/usr/bin/env python
#coding=utf-8

# @file crawl_ccf.py
# @brief crawl_ccf
# @author x565178035,x565178035@126.com
# @version 1.0
# @date 2017-12-15 10:44

import sys

reload(sys)
sys.setdefaultencoding("utf-8")

import requests
from lxml import etree
# Django specific settings
import os
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings")
import django
django.setup()
# Import your models for use in your script
from db.models import *

def crawl_table(url):
    res=requests.get(url)
    selector=etree.HTML(res.text)
    rank=["A","B","C","A","B","C"]
    tables=selector.xpath('//table[contains(@class,"tjb")]')
    cls=selector.xpath('//span[contains(@id,"pt8081field")]/text()')[0]
    for i in range(6):
        each_table=etree.tostring(tables[i]).replace('&#13;','')
        each_table=etree.fromstring(each_table)
        for each_publication in each_table.xpath("//tr")[1:]:
            each_publication=etree.fromstring(etree.tostring(each_publication))
            if len(each_publication.xpath('//td[2]/p')):
                abbv=each_publication.xpath('//td[2]/p')[0].xpath('string(.)').replace(u'\xa0','')
            else:
                abbv=''
            #  name=each_publication.xpath('//td[3]/p')[0].xpath('string(.)').replace(u'\xa0','')
            name=each_publication.xpath('//td[3]/p/text()')
            name=map(lambda e:e.replace('\n','').replace(u'\xa0','').strip(),name)
            name=" ".join(name)
            pub_house=each_publication.xpath('//td[4]/p')[0].xpath('string(.)').replace(u'\xa0','')
            r=rank[i]
            try:
                pub=Publication(
                        name=name,
                        abbv=abbv if abbv != "" else None,
                        pub_house=pub_house,
                        rank=r,
                        type='journal' if i<3 else 'conference',
                        cls=cls)
                pub.save()
            except django.db.utils.IntegrityError:
                print pub

def crawl_ccf(url):
    res=requests.get(url)
    selector=etree.HTML(res.text)
    table_urls=selector.xpath('//tr/td/a[contains(@href,"biaodan")]/@href')
    base_url="http://history.ccf.org.cn"
    for each in table_urls:
        crawl_table(base_url+each)

if __name__ == '__main__':
    #  crawl_table("http://history.ccf.org.cn/sites/ccf/biaodan.jsp?contentId=2903028135780")
    crawl_ccf("http://history.ccf.org.cn/sites/ccf/paiming.jsp")

