from pymongo import MongoClient




global links


client = MongoClient('localhost', 27017)
db = client.Medias

links = db.links

#, { "_id":0,"liste_links": 1}

#cursor = links.find({"name" : "http://chokomag.com/"},{ "_id" : 0 ,"liste_links" : 1})
cursor = links.find({"name":"http://www.leparisien.fr/"},{"_id":0,"name":1,"liste_links":1})


for document in cursor:
    links = document


liste_links = links['liste_links']
start_links = links['name']

print(start_links)
global liste_links

global start_links



from scrapy.spiders import CrawlSpider, Rule
from scrapy.contrib.linkextractors import LinkExtractor
from scrapy.selector import HtmlXPathSelector
from mediascrap.items import NewsItem
import datetime





class LeparisienSpider(CrawlSpider):
    """
    A spider crawls domains (in accordance with some  rules we will define
    to collect all the pages that we wish to extract our LemondeItems instances
    from. Most of this crawling logic is provided by Scrapy in the CrawlSpider class, so we
    can extend this class when writing our first spider.
    """

    name = "leparisien"
    #allowed_domains =[start_links[8:21]]
    start_urls =liste_links
    rules = [
        #site which should be saved
        Rule(
            LinkExtractor(
                allow = ['/(\d+|\w+)']),
                'parse_page')]


    def parse_page(self,response):
        hxs = HtmlXPathSelector(response)


        #For rtl //div[@id="article-section"]/p//text()
        #For lemonde //div[@id="articleBody"]/p//text()
        #For liberation //div[@class="article-body read-left-padding"]/p//text()
        #For ma ville //div[@class="elmt-detail article"]/p//text()
        #For sud ouest //div[@class="entry-content"]/p//text()

        title = ''.join(hxs.select('//h1[@itemprop="headline"]//text()').extract()).strip()
        date_article = ''.join(hxs.select('//time[@itemprop="datePublished"]').extract()).strip()
        body = ''.join(hxs.select('//div[@itemprop="articleBody"]/p//text()').extract()).strip()
        comments = ''.join(hxs.select('//span[@class="contReac bulleReacCouleur bgCouleur"]//text()').extract()).strip()





        item = NewsItem()

        if len(body)> 0 :
            item['site'] = start_links
            if len(title)> 0 :
                item['title']= title
            item['body'] = body
            item['url'] = response.url

            if len(date_article)>0 :
                item['date_article'] = date_article[16:41]

            if len(comments)> 0 :
                item['comments']= comments
            else :
                item['comments'] = 0

            item['timeOfScrap'] = datetime.datetime.now()

            return item


        else :
            pass
