python获取链家网数据

在使用python抓取并分析数据—链家网基础上修改，增加了房屋单价等信息，本来想把数据抓下来分析每个小区价格最低的房间，但是没什么用，链家自己做的查询已经足够好了。

# coding='uft-8'

import sys
reload(sys)
sys.setdefaultencoding('utf-8')

import requests
import time
from bs4 import BeautifulSoup
import re
import pandas as pd


url = 'http://bj.lianjia.com/ershoufang/'
page = 'pg'
headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
'Accept':'text/html;q=0.9,*/*;q=0.8',
'Accept-Charset':'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
'Accept-Encoding':'gzip',
'Connection':'close',
'Referer':'http://www.baidu.com/link?url=_andhfsjjjKRgEWkj7i9cFmYYGsisrnm2A-TN3XZDQXxvGsM9k9ZZSnikW2Yds4s&wd=&eqid=c3435a7d00006bd600000003582bfd1f'
}


# get html
def get_html(page_num):

    a = (url+page+str(page_num)+'/')
    r = requests.get(url=a, headers=headers)
    html = r.content
    time.sleep(1)
    return html


# get house info
def get_info(html):

    lj = BeautifulSoup(html, 'html.parser')

    # unit price
    price = lj.find_all('div', attrs={'class': 'unitPrice'})
    up = []
    for a in price:
        unitPrice = a.span.string
        up.append(unitPrice)

    # total price
    price = lj.find_all('div', attrs={'class': 'priceInfo'})
    tp = []
    for a in price:
        totalPrice = a.span.string
        tp.append(totalPrice)

    # house info
    houseInfo = lj.find_all('div', attrs={'class': 'houseInfo'})
    hi = []
    for b in houseInfo:
        house = b.get_text()
        hi.append(house)

    # follow
    followInfo = lj.find_all('div', attrs={'class': 'followInfo'})
    fi = []
    for c in followInfo:
        follow = c.get_text()
        fi.append(follow)

    # description
    des = re.findall(r'jpg" alt="(.*?)"', html)
    de = []
    for i in des:
        de.append(i)

    # house id
    id = re.findall(r'data-houseid="(.*?)"', html)
    hid = []
    for i in id:
        hid.append(i)

    # make list
    house = pd.DataFrame({'unitprice': up, 'totalprice': tp, 'houseinfo': hi, 'followinfo': fi, 'description': des, 'houseid': hid})
    houseinfo_split = pd.DataFrame((x.split('|') for x in house.houseinfo), index=house.index, columns=['xiaoqu', 'huxing', 'mianji', 'chaoxiang', 'zhuangxiu', 'dianti'])
    house = pd.merge(house, houseinfo_split, right_index=True, left_index=True)

    return house


# save data
def save_data(house_data):
    house_data.to_csv('LJinfo.csv')


# main
frames = []
list_id = []

try:
    for num in xrange(1,1200):
        print 'now loading page %d' % num
        html = get_html(num)
        data = get_info(html)

        for id in data['houseid']:
            if id not in list_id:
                list_id.append(id)
            else:
                print 'same data'
                break

        frames.append(data)
except:
    print 'error'


result = pd.concat(frames)
save_data(result)

生成的csv效果：