#coding=utf-8
'''
Created on 2017��8��23��

@author: LIli
'''
import urllib2
import urllib
import codecs
from bs4 import BeautifulSoup
import csv
import re
import sys
reload(sys)
sys.setdefaultencoding('utf-8')


page = 1
url = 'http://sh.fang.lianjia.com/loupan/pg/pg' + str(page)
user_agent = 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'
headers = { 'User-Agent' : user_agent }
try:
    request = urllib2.Request(url,headers = headers)
    response = urllib2.urlopen(request)
    the_page = response.read().decode('utf-8')
    soup = BeautifulSoup(the_page,"lxml")
    list0=[]
    list1=[]
    list2=[]
    list3=[]
    list4=[]
    list5=[]
 
    for tag in soup.find_all(name="div", attrs={"class": re.compile("col-1")}):
        ta1 = tag.find(name="a", attrs={"target": re.compile("_blank")})
        #添加城市
        list0.append(r"上海")
        #名字
        list1.append(ta1.string)
        #面积
        ta2 = tag.find(name="a", attrs={"class": re.compile("area")})
        if ta2 != None:
            list2.append(ta2.string)
        else:
            list2.append(0)
        #状态
        ta3 = tag.find(name="span", attrs={"class": re.compile("status label")})
        list3.append(ta3.string)
       #户型
        ta4 = tag.find(name="span", attrs={"class": re.compile("propertype label")})
        list4.append(ta4.string)
        
    for tag in soup.find_all(name="div", attrs={"class": re.compile("col-2")}):
        #单价
        ta5 = tag.find(name="span", attrs={"class": re.compile("num")})
        if ta5 != None:
             list5.append(ta5.string)
        else:
            list5.append(0)
      
       
    csvfile = file(r'C:\Users\LIli\Desktop\lianjia.csv', 'ab+')
    csvfile.write(codecs.BOM_UTF8)
    writer = csv.writer(csvfile)    
    data = []
    for i in range(0,len(list0)):
        data.append((list0[i],list1[i], list2[i], list3[i], list4[i], list5[i]))
        writer.writerows(data)
    csvfile.close()
    print "done"
    
except urllib2.URLError, e:
    if hasattr(e,"code"):
        print e.code
    if hasattr(e,"reason"):
        print e.reason
        
    