python 安居客爬取商铺信息
个人接到一个安居客爬虫, 安居客数据都在html里面, 暂时没有太严重的反爬措施
技术交流:18611372505
封装获取 html方法
requests.packages.urllib3.disable_warnings()
session = requests.session()
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36'
}
def get_html(url):
'''得到页面的etree对象'''
html_obj = requests.get(url, headers=headers, verify=False)
html_obj = html_obj.content.decode() # 解决乱码问题
tree = etree.HTML(html_obj) # 转化为页面的etree对象
return tree
使用 etree 解析html
def get_data(tree):
"""获取一页的房子数据"""
# 建立字典
info_dicts = defaultdict(list)
div_list = tree.xpath('//div[@class="list-left"]/div[@class="list-item"]')
for div in div_list:
# 名称
title = div.xpath('.//div[@class="item-info"]/div/span/text()')[0]
info_dicts['名称'].append(title)
# 详细地址
mr5 = div.xpath('.//div[@class="item-info"]/p[2]//span/text()')[0]
info_dicts['位置'].append(''.join(mr5))
# 行政区
descript = div.xpath('.//div[@class="item-info"]/p[1]/span/text()')[0]
info_dicts['行政区'].append(''.join(descript))
# 面积
area = div.xpath('.//div[@class="item-area"]/p[1]/span[1]/text()')[0]
info_dicts['面积(平方)'].append(area)
# 租金
price_monthly = div.xpath('.//div[@class="item-price"]/div[1]//span/text()')
info_dicts['租金'].append(''.join(price_monthly)) if price_monthly else info_dicts['租金'].append('面议')
# 日租金
price_daily = div.xpath('.//div[@class="item-price"]/div[2]//span/text()')
info_dicts['日租金'].append(''.join(price_daily)) if price_daily else info_dicts['日租金'].append('面议')
data = pd.DataFrame(info_dicts)
#获取下一页的url链接
aNxt = tree.xpath('//a[@class="aNxt"]/@href')
if aNxt:
return data, aNxt
else:
return data, False
循环访问然后添加到pandas.DataFrame,然后导入xls文件
# 主程序部分
data = pd.DataFrame()
url = 'https://bj.sydc.anjuke.com/sp-zu/'
while True:
print("开始抓取链接: %s " % url)
tree = get_html(url)
an_data, aNxt = get_data(tree)
data = data.append(an_data, ignore_index=True)
if aNxt:
url = aNxt[0]
else:
break
time.sleep(random.randint(15, 27))
# 持久化存储
data.to_excel('安居客爬虫.xls', index=False)
完整代码
import random
from collections import defaultdict
import requests
import pandas as pd
from lxml import etree
import re
import time
requests.packages.urllib3.disable_warnings()
session = requests.session()
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36'
}
def get_html(url):
'''得到页面的etree对象'''
html_obj = requests.get(url, headers=headers, verify=False)
html_obj = html_obj.content.decode() # 解决乱码问题
tree = etree.HTML(html_obj) # 转化为页面的etree对象
return tree
def get_data(tree):
"""获取一页的房子数据"""
# 建立字典
info_dicts = defaultdict(list)
div_list = tree.xpath('//div[@class="list-left"]/div[@class="list-item"]')
for div in div_list:
# 名称
title = div.xpath('.//div[@class="item-info"]/div/span/text()')[0]
info_dicts['名称'].append(title)
# 详细地址
mr5 = div.xpath('.//div[@class="item-info"]/p[2]//span/text()')[0]
info_dicts['位置'].append(''.join(mr5))
# 行政区
descript = div.xpath('.//div[@class="item-info"]/p[1]/span/text()')[0]
info_dicts['行政区'].append(''.join(descript))
# 面积
area = div.xpath('.//div[@class="item-area"]/p[1]/span[1]/text()')[0]
info_dicts['面积(平方)'].append(area)
# 租金
price_monthly = div.xpath('.//div[@class="item-price"]/div[1]//span/text()')
info_dicts['租金'].append(''.join(price_monthly)) if price_monthly else info_dicts['租金'].append('面议')
# 日租金
price_daily = div.xpath('.//div[@class="item-price"]/div[2]//span/text()')
info_dicts['日租金'].append(''.join(price_daily)) if price_daily else info_dicts['日租金'].append('面议')
data = pd.DataFrame(info_dicts)
# 获取下一页链接
aNxt = tree.xpath('//a[@class="aNxt"]/@href')
if aNxt:
return data, aNxt
else:
return data, False
# 主程序部分
data = pd.DataFrame()
url = 'https://bj.sydc.anjuke.com/sp-zu/'
while True:
print("开始抓取链接: %s " % url)
tree = get_html(url)
an_data, aNxt = get_data(tree)
data = data.append(an_data, ignore_index=True)
if aNxt:
url = aNxt[0]
else:
break
time.sleep(random.randint(15, 27))
# 持久化存储
data.to_excel('安居客爬虫.xls', index=False)
##结果输出