#!/usr/bin/env python
# -*- encoding: utf-8 -*-
# Created on 2017-08-20 14:18:42
# Project: Tripadvisor

from pyspider.libs.base_handler import *
import pymongo

class Handler(BaseHandler):
    crawl_config = {
    }
    
    client = pymongo.MongoClient("localhost")
    db = client["trip"]

    @every(minutes=24 * 60)
    def on_start(self):
        self.crawl('https://www.tripadvisor.cn/Attractions-g186338-Activities-c47-t163-London_England.html', callback=self.index_page)

    @config(age=10 * 24 * 60 * 60)
    def index_page(self, response):
        for each in response.doc('.listing_title > a').items():
            self.crawl(each.attr.href, callback=self.detail_page)
        next = response.doc("#FILTERED_LIST > div.al_border.deckTools.btm > div > div > a").attr.href
        self.crawl(next,callback=self.index_page)
     
    #代码需要精简
    @config(priority=2)
    def detail_page(self, response):
        url = response.url
        name = response.doc('#HEADING').text(),
        rating = response.doc('#taplc_location_detail_header_attractions_0 > div.rating_and_popularity > span.header_rating').text(),
        address = response.doc('#taplc_location_detail_header_attractions_0 > div.prw_rup.prw_common_atf_header_bl.headerBL > div > div.blEntry.address.clickable.colCnt2').text(),
        phone = response.doc('#taplc_location_detail_header_attractions_0 > div.prw_rup.prw_common_atf_header_bl.headerBL > div > div.blEntry.phone > span:nth-child(2)').text(),
        duration = response.doc('#taplc_attraction_detail_listing_0 > div.section.hours > div.detail_section.duration').text(),
        
        return {
            "url":url,
            "name": name,
            "rating": rating,
            "address": address,
            "phone": phone,
            "duration": duration,
        }
    
    def on_result(self,result):
        if result:
            self.save_to_mongo(result)
    
    def save_to_mongo(self,result):
        if self.db["london"].insert(result):
            print("save to mongo",result)
 
