#!/usr/bin/env python
# -*- coding: utf-8 -*-

import _env
from pprint import pprint as pp

from lib._db import get_db
from html_parser import Bs4HtmlParser
from spider import LagouCrawler


db = get_db('htmldb')
col = getattr(db, 'lagou_html')    # collection


def test_get_db():
    o = col.find_one({'_id': 1234})
    html = o['html']
    # print html
    pp(o)


def test_get_html():
    import chardet
    _id = 46167
    o = col.find_one({'_id': _id})
    p = Bs4HtmlParser('', o['html'])
    print(p.html)
    t = p.bs.find('p', class_='msg')
    text = t.get_text()
    print(text)


def count_how_many_block_html():
    cnt = 0
    for html_doc in col.find(snapshot=True):
        html = html_doc['html']
        if LagouCrawler.is_block_html(html, False):
            cnt += 1
    return cnt


def count_how_many_check_html():
    cnt = 0
    for html_doc in col.find(snapshot=True):
        html = html_doc['html']
        if LagouCrawler.is_check_html(html, False):
            cnt += 1
    return cnt

if __name__ == '__main__':
    # test_get_html()
    # print(count_how_many_block_html())
    print(count_how_many_check_html())

#!/usr/bin/env python
# -*- coding: utf-8 -*-

import _env
from pprint import pprint as pp

from lib._db import get_db
from html_parser import Bs4HtmlParser
from spider import LagouCrawler


db = get_db('htmldb')
col = getattr(db, 'lagou_html')    # collection


def test_get_db():
    o = col.find_one({'_id': 1234})
    html = o['html']
    # print html
    pp(o)


def test_get_html():
    import chardet
    _id = 46167
    o = col.find_one({'_id': _id})
    p = Bs4HtmlParser('', o['html'])
    print(p.html)
    t = p.bs.find('p', class_='msg')
    text = t.get_text()
    print(text)


def count_how_many_block_html():
    cnt = 0
    for html_doc in col.find(modifiers={"$snapshot": True}):
        html = html_doc['html']
        if LagouCrawler.is_block_html(html, False):
            cnt += 1
    return cnt


def count_how_many_check_html():
    print(col.count())
    cnt = 0
    _id_list = []
    for html_doc in col.find(modifiers={"$snapshot": True}):
        html = html_doc['html']
        if LagouCrawler.is_check_html(html, False):
            cnt += 1
            _id_list.append(html_doc._id)
    return cnt
    print(col.count())
    # col.remove({'_id':{'$in':_id_list}})


if __name__ == '__main__':
    # test_get_html()
    # print(count_how_many_block_html())
    # print(count_how_many_check_html())
    print(col.count())

modify handle db