import requests
import logging
import parsel
import pandas as pd
headers = {
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.87 Safari/537.36 SE 2.X MetaSr 1.0',
}
logging.basicConfig(level=logging.INFO,format='%(asctime)s - %(levelname)s: %(message)s')
TOTAL_PAGE = 11
# 通用方法
def scrape_page(url):
logging.info('正在批量采集 %s', url)
try:
response = requests.get(url,headers=headers)
if response.status_code == 200:
return response.text
logging.error('抓取时获取无效状态码 %s', response.status_code, url)
except requests.RequestException:
logging.error('抓取时发生错误 %s', url, exc_info=True)
# 列表页
def scrape_index(page):
index_url = f'https://cd.zu.anjuke.com/fangyuan/dujiangyan/p{page}/'
return scrape_page(index_url)
# 详情页
def parse_index(html):
selector = parsel.Selector(text=html)
divs = selector.css('.maincontent .list-content .zu-itemmod')
items = []
for div in divs:
title = div.css('.zu-info h3 a b::text').get() # 标题
loupan = div.css('.zu-info .details-item a::text').get() # 楼盘
jiaqian = div.css('.zu-side p strong b::text').get() # 价钱
mianji = div.css('.zu-info .details-item b::text').getall()[-1] # 面积
item = [title, loupan, jiaqian, mianji]
items.append(item)
return items
if __name__ == '__main__':
all_details = []
for page in range(1, TOTAL_PAGE + 1):
index_html = scrape_index(page)
detail_urls = parse_index(index_html)
all_details.extend(detail_urls)
df = pd.DataFrame(all_details, columns=['标题', '楼盘', '价格(元/月)', '面积'])
df.to_excel('都江堰租房情况(安居客).xlsx', index=False)
001—同步爬取安居客租房信息
于 2024-02-17 19:17:57 首次发布