day01爬虫作业

day01爬虫作业

爬取贝壳100页房源信息并存入csv文件

def data_home(str_data=str):
    # 1. 获取网页数据
    response = requests.get(f'https://cd.zu.ke.com/zufang{str_data}/#contentList')
    # 2. 解析网页数据
    soup = BeautifulSoup(response.text, 'lxml')
    all_house = soup.select('.content__list>.content__list--item')
    data = []
    for house in all_house:
        # 名字
        name = house.select_one('.twoline').text.strip()
        # 价格
        price = house.select_one('.content__list--item-price').text

        # 地址-面积-户型
        p = house.select_one('.content__list--item--des')
        message = sub(r'\s+', '', p.text)
        data.append([name, price, message])
    writer = csv.writer(open('files/home_data.csv', 'a', newline='', encoding='utf-8'))
    if str_data=='':
        writer.writerow(['户型朝向', '房租', '地址面积'])
    writer.writerows(data)


str1 = '/pg'
if __name__ == '__main__':
    for x in range(1, 101):
        time.sleep(1)
        if x == 1:
            data_home(str_data='')
        else:
            str_data = str1 + str(x)

            data_home(str_data)