# -*- coding: utf-8 -*-
import scrapy
from scrapy_splash import SplashRequest
import pdb
import time
import json
from scrapy.spidermiddlewares.httperror import HttpError
from twisted.internet.error import DNSLookupError
from twisted.internet.error import TimeoutError, TCPTimedOutError
from scrapy.http import FormRequest
from collections import OrderedDict
import pandas

class MakaandataPySpider(scrapy.Spider):
    name = "makaandata.py"
    allowed_domains = ["makaan.com"]
    
    def __init__(self, *args, **kwargs):
        self.start_url=[]
        placeid=['65682',
        '57207',
        '50273',
        '54361',
        '51149',
        '65792',
        '50159',
        '50157',
        '50160',
        '77668',
        '54337',
        '51635',
        '50168',
        '51143',
        '51998',
        '52351',
        '50387',
        '64073',
        '57245',
        '62326',
        '50369',
        '50172',
        '50174',
        '50338',
        '50180',
        '50175',
        '54278',
        '50347',
        '59133',
        '78126',
        '51156',
        '51181',
        '56697',
        '52335',
        '50177',
        '50183',
        '52713',
        '57355',
        '50181',
        '62339',
        '50362',
        '50359',
        '50317',
        '63340',
        '50309',
        '65359',
        '62763',
        '52236',
        '51451',
        '62519',
        '96876',
        '65605',
        '67617',
        '76514',
        '68276',
        '86606',
        '83202',
        '89339',
        '93098',
        '95339',
        '51188',
        '50184',
        '53153',
        '51126',
        '50188',
        '51204',
        '52703',
        '50392',
        '53149',
        '63628',
        '50164',
        '51501',
        '52306',
        '51161',
        '52718',
        '51641',
        '52296',
        '52298',
        '52297',
        '52362',
        '52361',
        '52697',
        '50176',
        '51087',
        '52695',
        '52694',
        '53111',
        '50167',
        '51129',
        '63868',
        '120895',
        '121125',
        '121154',
        '121156',
        '121323',
        '50270',
        '51453',
        '51097',
        '51503',
        '51449',
        '63945',
        '65402',
        '120920',
        '120953',
        '121140',
        '51152',
        '50186',
        '50185',
        '68601',
        '51219',
        '50187',
        '51160',
        '51499',
        '51113',
        '70645',
        '50271',
        '52333',
        '51078',
        '121498',
        '53156',
        '53143',
        '53155',
        '53142',
        '53146',
        '53154',
        '53150',
        '53147',
        '53151',
        '50161',
        '51656',
        '63835',
        '51502',
        '50169',
        '53344',
        '51447',
        '53345',
        '65137',
        '121182',
        '91282',
        '66667',
        '104567',
        '104380',
        '104940',
        '105032',
        '109525',
        '112420',
        '107436',
        '108634',
        '108896',
        '115129',
        '50273',
        '57207',
        '51638',
        '50159',
        '50160',
        '50158',
        '64349',
        '69123',
        '50157',
        '77668',
        '50387',
        '54337',
        '50162',
        '51635',
        '51998',
        '57248',
        '50168',
        '50390',
        '59114',
        '62326',
        '50172',
        '50175',
        '50180',
        '50338',
        '50369',
        '51077',
        '51377',
        '51625',
        '50179',
        '78126',
        '50183',
        '50177',
        '50181',
        '51181',
        '56697',
        '51156',
        '57355',
        '50339',
        '56696',
        '62339',
        '50317',
        '51169',
        '54301',
        '50362',
        '54290',
        '50394',
        '57421',
        '62763',
        '52720',
        '62519',
        '60795',
        '68943',
        '64072',
        '83202',
        '76200',
        '90640',
        '71716',
        '83504',
        '76127',
        '95339',
        '50392',
        '51126',
        '52703',
        '51188',
        '50188',
        '53149',
        '53153',
        '50184',
        '51204',
        '63628',
        '50164',
        '51501',
        '52718',
        '52306',
        '51161',
        '52296',
        '52298',
        '52297',
        '51641',
        '52361',
        '52362',
        '52697',
        '51087',
        '50176',
        '52694',
        '52695',
        '51129',
        '63868',
        '53111',
        '50167',
        '120895',
        '121125',
        '121154',
        '121156',
        '121323',
        '50270',
        '51453',
        '51449',
        '51503',
        '51097',
        '63945',
        '65402',
        '120920',
        '120953',
        '121140',
        '50186',
        '50185',
        '51160',
        '51113',
        '68601',
        '51499',
        '70645',
        '51219',
        '51152',
        '50187',
        '51078',
        '50271',
        '52333',
        '121498',
        '53143',
        '53151',
        '53146',
        '53156',
        '53150',
        '53142',
        '53154',
        '53147',
        '53155',
        '50161',
        '63835',
        '50169',
        '51502',
        '51656',
        '53344',
        '51447',
        '53345',
        '65137',
        '121182',
        '91282',
        '66667',
        '104940',
        '105032',
        '109525',
        '112420',
        '107436',
        '108634',
        '108896',
        '115129',
        '120472',
        '120496',
        '51513',
        '50158',
        '50273',
        '50157',
        '59089',
        '59089',
        '57207',
        '50160',
        '50273',
        '50273',
        '97946',
        '51998',
        '51998',
        '51635',
        '121255',
        '54337',
        '60726',
        '121255',
        '50387',
        '57245',
        '50369',
        '50180',
        '50369',
        '50172',
        '58801',
        '50172',
        '56681',
        '50180',
        '50172',
        '56670',
        '50373',
        '56697',
        '121121',
        '121121',
        '66408',
        '121121',
        '121121',
        '121121',
        '121121',
        '50183',
        '51451',
        '50362',
        '50362',
        '52236',
        '57407',
        '57395',
        '65947',
        '50394',
        '57227',
        '50204',
        '62810',
        '62810',
        '62810',
        '62810',
        '62810',
        '110391',
        '106093',
        '99845',
        '62810',
        '62810',
        '52703',
        '51188',
        '51204',
        '51188',
        '50188',
        '50184',
        '52703',
        '50188',
        '50392',
        '51204',
        '50164',
        '50164',
        '50164',
        '50164',
        '50164',
        '50164',
        '50164',
        '50164',
        '52718',
        '50164',
        '51641',
        '51641',
        '51641',
        '51641',
        '51641',
        '51641',
        '51641',
        '52361',
        '51641',
        '51641',
        '52697',
        '52697',
        '52697',
        '52697',
        '52697',
        '52697',
        '52697',
        '52697',
        '52697',
        '52697',
        '51087',
        '51087',
        '51087',
        '50176',
        '50176',
        '52694',
        '50176',
        '51087',
        '51087',
        '50176',
        '53111',
        '50167',
        '51129',
        '51129',
        '50167',
        '53111',
        '50167',
        '50167',
        '50167',
        '53111',
        '50270',
        '51097',
        '51503',
        '51449',
        '50270',
        '51503',
        '51503',
        '51503',
        '50270',
        '51453',
        '51113',
        '50186',
        '51219',
        '68601',
        '50185',
        '68601',
        '50185',
        '50186',
        '50185',
        '50185',
        '52333',
        '52333',
        '52333',
        '52333',
        '50271',
        '51078',
        '51078',
        '51078',
        '51078',
        '51078',
        '53156',
        '53156',
        '53143',
        '53156',
        '53143',
        '53150',
        '53143',
        '53142',
        '53156',
        '53155',
        '51447',
        '50161',
        '50161',
        '51502',
        '63835',
        '50161',
        '50161',
        '50161',
        '63835',
        '50161',
        '66667',
        '66667',
        '91282',
        '91282',
        '91282',
        '66667',
        '91282',
        '91282',
        '91282',
        '66667',
        '109525']
        placename=['Magadi Road',
        'Benson Town',
        'Frazer Town',
        'Richmond Road',
        'Cnningham Road',
        'Horamav Banjara Lyot',
        'lsoor',
        'Ashok Nagar',
        'Richmond Town',
        'ISRO Colony',
        'Whitefield',
        'Harlr',
        'Sarjapr Road',
        'Hoskote',
        'Varthr',
        'Avalahalli Off Sarjapr Road',
        'HSR Layot',
        'Mallathahalli',
        'Channasandra',
        'Varthr Lake',
        'Horamav',
        'Yelahanka',
        'Devanahalli',
        'Jakkr',
        'Thanisandra',
        'Hebbal',
        'Hennr Road',
        'Chikballapr',
        'Doddaballapr Road',
        'Chikkasandra',
        'Nagarbhavi',
        'Rajaji Nagar',
        'Dasarahalli on Tmkr Road',
        'Chamarajpet',
        'Jalahalli',
        'Yeshwantpr',
        'Krbarahalli on Magadi Road',
        'Basapra',
        'Malleswaram',
        'Mariyappanapalya',
        'Jigani',
        'Anekal City',
        'Koramangala',
        'Chandapra Anekal Road',
        'Marsr',
        'Hosa Road Jnction',
        'Bannerghatta Main Road',
        'Kanakapra Road Beyond Nice Ring Road',
        'Hosa Road',
        'Mico Layot',
        'Anekal Road',
        'Bkkasagar Jigani',
        'White Field',
        'Horamav Agara',
        'Hennr Main Road',
        'Koppa Gate Nisarga Road',
        'Sarjapr Road',
        'TC Palya Road',
        'Anekal Main',
        '4th Cross Road',
        'Gottigere',
        'Bannerghatta',
        'JP Nagar Phase 8',
        'Bilekahalli',
        'JP Nagar Phase 7',
        'Anjanapra',
        'Hlimav',
        'BTM Layot',
        'JP Nagar Phase 4',
        'HSR Layot Silk Board',
        'Marathahalli',
        'Domlr',
        'Jeevan Bima Nagar',
        'Mrgesh Palya',
        'Kodihalli on Old Airport Road',
        'Sarjapr',
        'Sarjapr Road Till Wipro',
        'Sarjapr Road Wipro To Railway Crossing',
        'Sarjapr Road Post Railway Crossing',
        'Volagerekallahalli',
        'Carmelaram',
        'Kannr on Thanisandra Main Road',
        'Hennr',
        'Narayanapra on Hennr Main Road',
        'Chikkagbbi on Hennr Main Road',
        'Kvemp Layot on Hennr Main Road',
        'Bdigere Cross',
        'KR Pram',
        'Ramamrthy Nagar',
        'Kannamangala',
        'Nimbekaipra',
        'Railway Colony',
        'Priyadarshini Layot',
        'Bovi Colony',
        'BSV Reddy Layot',
        'Bellandr',
        'Mahadevapra',
        'Banaswadi',
        'Kalyan Nagar',
        'Doddanekndi',
        'Devarabeesanahalli',
        'Marthi Sevanagar',
        'A Narayanapra',
        'Sbbannayapalya',
        'Belr Nagasandra',
        'Chandapra',
        'Electronic City Phase 1',
        'Begr',
        'Electronic City Phase 2',
        'Attibele',
        'Hosr',
        'Bommanahalli',
        'Bommasandra',
        'Kdl',
        'Singasandra',
        'Rajarajeshwari Nagar',
        'Kmbalgod',
        'Kengeri',
        'Mahaveer Nagar',
        'Talaghattapra',
        'Sbramanyapra',
        'Konanaknte',
        'Kmaraswamy Layot',
        'JP Nagar Phase 1',
        'JP Nagar Phase 9',
        'JP Nagar Phase 5',
        'JP Nagar Phase 2',
        'JP Nagar Phase 6',
        'Whitefield Hope Farm Jnction',
        'ITPL',
        'Hoodi',
        'Kadgodi',
        'Brookefield',
        'Sai Baba Ashram',
        'Ramagondanahalli',
        'Immadihalli',
        'Nallrhalli Whitefield',
        'Sadaramangala Indstrial Area',
        'Kasavanahalli Off Sarjapr Road',
        'Jnnasandra',
        'Bettahalsoor',
        'Yelahanka Airforce Base',
        'Shettigere',
        'Telecom Society',
        'East Colony',
        'Palanahalli',
        'Basaveshwara Nagar Yelahanka',
        'Ambedkar Colony',
        'Chikka Bommasandra',
        'Sri Balaji Layot',
        'Frazer Town',
        'Benson Town',
        'Vasanth Nagar',
        'lsoor',
        'Richmond Town',
        'Cox Town',
        'MG Road',
        'Kalena Agrahara',
        'Ashok Nagar',
        'ISRO Colony',
        'HSR Layot',
        'Whitefield',
        'Indira Nagar',
        'Harlr',
        'Varthr',
        'Cooke Town',
        'Sarjapr Road',
        'Kndalahalli',
        'Bdigere',
        'Varthr Lake',
        'Yelahanka',
        'Hebbal',
        'Thanisandra',
        'Jakkr',
        'Horamav',
        'Sahakar Nagar',
        'Kastri Nagar',
        'Vidyaranyapra',
        'Sanjay Nagar',
        'Chikkasandra',
        'Yeshwantpr',
        'Jalahalli',
        'Malleswaram',
        'Rajaji Nagar',
        'Dasarahalli on Tmkr Road',
        'Nagarbhavi',
        'Basapra',
        'Vijay Nagar',
        'Chandra Layot',
        'Mariyappanapalya',
        'Koramangala',
        'Ejipra',
        'JP Nagar',
        'Jigani',
        'Arekere',
        'Jayanagar',
        'Tavarakere',
        'Bannerghatta Main Road',
        'Adgodi',
        'Mico Layot',
        'BTM 2nd Stage',
        'bannerghatta road',
        'HAL OLD AIRPORT RD',
        'Sarjapr Road',
        'RMV 2nd Stage',
        'Indiranagar HAL 2nd Stage',
        'B Narayanapra',
        'BTM 1st Stage',
        'Hosr Main Road',
        '4th Cross Road',
        'BTM Layot',
        'Bilekahalli',
        'Hlimav',
        'Gottigere',
        'JP Nagar Phase 7',
        'JP Nagar Phase 4',
        'JP Nagar Phase 8',
        'Bannerghatta',
        'Anjanapra',
        'HSR Layot Silk Board',
        'Marathahalli',
        'Domlr',
        'Kodihalli on Old Airport Road',
        'Jeevan Bima Nagar',
        'Mrgesh Palya',
        'Sarjapr Road Till Wipro',
        'Sarjapr Road Wipro To Railway Crossing',
        'Sarjapr Road Post Railway Crossing',
        'Sarjapr',
        'Carmelaram',
        'Volagerekallahalli',
        'Kannr on Thanisandra Main Road',
        'Narayanapra on Hennr Main Road',
        'Hennr',
        'Kvemp Layot on Hennr Main Road',
        'Chikkagbbi on Hennr Main Road',
        'Ramamrthy Nagar',
        'Kannamangala',
        'Bdigere Cross',
        'KR Pram',
        'Nimbekaipra',
        'Railway Colony',
        'Priyadarshini Layot',
        'Bovi Colony',
        'BSV Reddy Layot',
        'Bellandr',
        'Mahadevapra',
        'Doddanekndi',
        'Kalyan Nagar',
        'Banaswadi',
        'Devarabeesanahalli',
        'Marthi Sevanagar',
        'A Narayanapra',
        'Sbbannayapalya',
        'Belr Nagasandra',
        'Electronic City Phase 1',
        'Begr',
        'Bommanahalli',
        'Kdl',
        'Electronic City Phase 2',
        'Bommasandra',
        'Singasandra',
        'Attibele',
        'Chandapra',
        'Hosr',
        'Kengeri',
        'Rajarajeshwari Nagar',
        'Kmbalgod',
        'Mahaveer Nagar',
        'Sbramanyapra',
        'JP Nagar Phase 6',
        'JP Nagar Phase 1',
        'Talaghattapra',
        'JP Nagar Phase 5',
        'Kmaraswamy Layot',
        'JP Nagar Phase 9',
        'JP Nagar Phase 2',
        'Konanaknte',
        'Whitefield Hope Farm Jnction',
        'Hoodi',
        'Brookefield',
        'Kadgodi',
        'ITPL',
        'Sai Baba Ashram',
        'Ramagondanahalli',
        'Immadihalli',
        'Nallrhalli Whitefield',
        'Sadaramangala Indstrial Area',
        'Kasavanahalli Off Sarjapr Road',
        'Jnnasandra',
        'Shettigere',
        'Telecom Society',
        'East Colony',
        'Palanahalli',
        'Basaveshwara Nagar Yelahanka',
        'Ambedkar Colony',
        'Chikka Bommasandra',
        'Sri Balaji Layot',
        'Doddajala',
        'Sggatta',
        'Qeens Terraces',
        'Prva Parkcox Town',
        'Spaces Meridian Tiara',
        'Royal Anmol',
        'Sraj Kiran',
        'Skhashaya Apartment',
        'Tre Winds',
        'Richmond',
        'Melody',
        'Sham Mansion',
        'Rio',
        'Alaya',
        'Bhvana Nivaath',
        'Maple Sqare Block B',
        'Pblicity 1',
        'Icon',
        'Hiranyagarbha Orts',
        'Homes 2',
        'London',
        'Sri Thimmaramhai Swamy Nilaya',
        'Trendsqare Precioso',
        'Galaxy',
        'Crosswinds',
        'Calista',
        'Mont Zion',
        'Sai Springs',
        'Villas In Sky',
        'ltima Smarthomes',
        'Crest IRS',
        'Samyama Jyothi Enclave',
        'Veda',
        'Oasis',
        'Site 3',
        'Bilder 1',
        'Abhyam Residency',
        'Phase 1',
        'Homes 2',
        'Homes 182',
        'Bilder',
        'Sri annaprneshwari apartments',
        'Sbhiksha',
        'CK Adiithya',
        'Aire',
        'Sratar',
        'Gangothri SLV Meadows',
        'Ganesh Prasad Residency',
        'Park Avene',
        'Devine Apartment',
        'Site 2',
        'Akrti Vasantha',
        'Paradise',
        'Greens Signatre Villows',
        'Presco Pride',
        'CBRE Indraprastha Ascentado',
        'Serenity',
        'City Sec 7 Rosewoodenclave',
        'City Sec 6 Palm Wood Enclave',
        'Rahl Complex 3',
        'Doblement',
        'Navaneetha Apartments',
        'Vrshabadri Towers',
        'Balaji Ashirvaad Elite',
        'Felicity',
        'Elite',
        'Essence',
        'Imperia',
        'Epic',
        'Premier Krishna',
        'Ble Diamond',
        'Highland III',
        'Sannidhi',
        'Sri Krappa',
        'Silver Mist',
        'Breeze',
        'Elan',
        'Mathrsree Orchid Homes',
        'Mathrsree Residency',
        'SS Sreshta',
        'Genesis',
        'Sankaramangalam',
        'Greens',
        'Smart Neighborhoods',
        'The Conty Address',
        'JEM',
        '18 and Oak',
        'Flamenco',
        'One More Light',
        'White Waters',
        'Altis',
        'Villa Project In Sarjapr',
        'Oracle Heaven',
        'dyaan 1',
        'Acropolis',
        'Devin Sri Samrddh Paradise',
        'Iris',
        'Lake Front',
        'New Town',
        'Paradise Enclave',
        'Nikoo Homes 2',
        'Skycity',
        'Preethi Woods',
        'The Wisdom Tree Commnity',
        'pavan Phase 1',
        'Villa Martin',
        'Sol and Soil',
        'Green Admire',
        'Xelix',
        'SLV Paradise',
        'Prakash Hibiscs',
        'Asentado',
        'Bena Vista Phase 2',
        'Sree Sai Heritage',
        'Serenity',
        'GK Residency',
        'Elite',
        'Central Park',
        'Shree Choyal Bhavan',
        'Sri Banashankridevi Prasanna',
        'Signatre',
        'O2 Homes',
        'Alpha',
        'Elite',
        'Sdaamini residency',
        'Srikanth Vista',
        'Residency',
        'Green Mist',
        'East',
        'AR Splendor Park',
        'Mthsree Village Homes',
        'Casablanca',
        'Palm Woods',
        'GM Ambitios Enclave',
        'Royal Garden Plots Sector 2',
        'Spencer Annex',
        'Symphony',
        'Sai Ashraya',
        'Trqoise',
        'Sothbrook',
        'Golden Heights',
        'Manjnatha Classic',
        'Donenahalli 1',
        'Krbarapalya 2',
        'Krbarapalya',
        'Kalpa',
        'Elite',
        'Skyspreme',
        'Satellite',
        'Annapoorneshwari Residency',
        'Sri Krishna Gardens Phase III',
        'Sri Krishna Gardens Phase IV',
        'Misty Charm',
        'Forest Edge',
        'Valley Pearl',
        'Park Sqare',
        'Sai Kteer',
        'Montrose',
        'Kota Hills',
        'Shivaganga Omkar',
        'Forest View',
        'Pioneer Park Avene',
        'Splendor',
        'Parth Pride',
        'Misty Meadows',
        'Gangothri Residency',
        'Divine',
        'The Connected Living',
        'Lots',
        'Samrddhi',
        'Air',
        'Jyothi Woods',
        'Santhrpthi',
        'Greens',
        'Platina',
        'Poetree',
        'Pride',
        'Sankalpa Villas',
        'Krishna Viva',
        'Natre Lake',
        'Golden Days',
        'Bloomingdales',
        'Magadi Road']
        # term=raw_input("Enter your locality:")

        # for x, y in zip(placename,placeid):
        #     if x==term:
        #         for i in range(1,25):
        #             url='https://www.makaan.com/bangalore-property/'+x+'-flats-for-sale-'+y+'?page='+str(i)
        #             self.start_url.append(url)

        for x, y in zip(placename, placeid):
            # pdb.set_trace()
            for i in range(1,25):
                url='https://www.makaan.com/bangalore-property/'+x+'-flats-for-sale-'+y+'?page='+str(i)
                self.start_url.append(url)
                # pdb.set_trace()
              
           
        super(MakaandataPySpider, self).__init__(*args, **kwargs)
    def start_requests(self):
        for url in self.start_url:
            yield SplashRequest(url, callback=self.parse)

    def parse(self, response):
        # pdb.set_trace()
        list1=response.css('a.projName ::attr(href)').extract()
        for i in list1:
            yield scrapy.Request(i, callback=self.parse_details)
            # pdb.set_trace()

    def parse_details(self,response):
        print("---------------------------------------------------------------------------------------------")
        # pdb.set_trace()
        item={
        'name':response.css('div.imp-data-wrap >div.type-col >h1>span ::text').extract(),
        'mini_locality':response.css('div.imp-data-wrap >div.type-col>div.loc-wrap >a.loclink>span::text').extract(),
        'location':response.css('div.imp-data-wrap >div.type-col>div.loc-wrap >a.a-grey-link>span::text').extract(),
        'lat_lng':response.css('div.imp-data-wrap >div.type-col>div >meta ::attr(content)').extract(),
        'price_range':response.css('div.price-wrap.project-price >span.price ::text').extract(),
        'pricePerSqft':response.css('div.imp-data-wrap >div.i-row.clearfix >div >div.rate-wrap >span ::text').extract_first(),
        'Apartments':response.css('div.imp-data-wrap >div.i-row.clearfix >div.i-rcol >div >table >tbody >tr >td#Apartment ::text').extract(),
        'Area':response.css('div.imp-data-wrap >div.i-row.clearfix >div.i-rcol >div >table >tbody >tr >td#Area ::text').extract(),
        'Status':response.css('div.imp-data-wrap >div.i-row.clearfix >div.i-rcol >div >table >tbody >tr >td#Status ::text').extract(),
        'About_project':response.css('div.desc.clearfix ::text').extract(),
        # Availability=response.css('div.kd-wrap.js-list-wrap >table >tbody >tr >td#Availability ::text').extract(),
        'Possessiondate':response.css('div.kd-wrap.js-list-wrap >table >tbody >tr >td.val ::text')[1].extract(),
        'Total_acres':response.css('div.kd-wrap.js-list-wrap >table >tbody >tr >td.val ::text')[2].extract(),
        'Lounched_date':response.css('div.kd-wrap.js-list-wrap >table >tbody >tr >td.val ::text')[3].extract(),
        'Amenities':response.css('div.amenities-wrap.js-list-wrap >div.m-scroll-wrap ::text').extract(),
        'Specification':response.css('div.specifictions-wrap.js-list-wrap >div ::text').extract(),
        'Home_loans':response.css('div.banklogo-wrap >div ::attr(data-background-image)').extract(),
        'builder':response.css('div.builder-wrap >h2::text').extract(),
        'floor_plan_value':response.css('div.upl-dtls ::text').extract()}
        # pdb.set_trace()
        yield item

