001—同步爬取安居客租房信息

import requests
import logging
import parsel
import pandas as pd

headers = {
    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.87 Safari/537.36 SE 2.X MetaSr 1.0',

}

logging.basicConfig(level=logging.INFO,format='%(asctime)s - %(levelname)s: %(message)s')
TOTAL_PAGE = 11
# 通用方法
def scrape_page(url):
    logging.info('正在批量采集 %s', url)
    try:
        response = requests.get(url,headers=headers)
        if response.status_code == 200:
            return response.text
        logging.error('抓取时获取无效状态码 %s', response.status_code, url)
    except requests.RequestException:
        logging.error('抓取时发生错误 %s', url, exc_info=True)
# 列表页
def scrape_index(page):
    index_url = f'https://cd.zu.anjuke.com/fangyuan/dujiangyan/p{page}/'
    return scrape_page(index_url)

# 详情页
def parse_index(html):
    selector = parsel.Selector(text=html)
    divs = selector.css('.maincontent .list-content .zu-itemmod')
    items = []
    for div in divs:
        title = div.css('.zu-info h3 a b::text').get()  # 标题
        loupan = div.css('.zu-info .details-item a::text').get()  # 楼盘
        jiaqian = div.css('.zu-side p strong b::text').get()  # 价钱
        mianji = div.css('.zu-info .details-item b::text').getall()[-1] # 面积
        item = [title, loupan, jiaqian, mianji]
        items.append(item)
    return items

if __name__ == '__main__':
    all_details = []
    for page in range(1, TOTAL_PAGE + 1):
        index_html = scrape_index(page)
        detail_urls = parse_index(index_html)
        all_details.extend(detail_urls)
    df = pd.DataFrame(all_details, columns=['标题', '楼盘', '价格(元/月)', '面积'])
    df.to_excel('都江堰租房情况（安居客）.xlsx', index=False)