2023年网络爬虫实训（第七天）

任务1：完成链家二手房xpath解析案例。完成房源总价和单价的数据提取。完成房源示例图片的提取。并汇总到实验三中。实验报告可以稍晚提交。

import random
import requests
import json
from lxml import etree


class LHome:
    """链家二手房房源"""
    def __init__(self, city='hz'):
        # 定义请求地址
        self.url = "https://" + city + ".lianjia.com/ershoufang/pg{}"
        # 定义请求头
        self.header = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36"}
        # 定义代理池
        self.proxies = [{"HTTP": "27.42.168.46:55481"}, {"HTTP": "121.13.252.61:41564"}, {"HTTP": "183.236.232.160:8080"}]
        # 数据源
        self.data = []

    def get_response(self, url):
        """请求数据获取响应"""
        random_index = random.randint(0, len(self.proxies)-1)
        return requests.get(url, headers=self.header, proxies=self.proxies[random_index]).content.decode('utf-8')

    def parse_data(self, html):
        """解析数据"""
        # 创建xpath查找对象
        x_finder = etree.HTML(html)
        # 先找到数据所在的  基准节点
        all_li_nodes = x_finder.xpath("//div[@class='leftContent']/ul[@class='sellListContent']/li[contains(@class, 'LOGCLICKDATA')]")
        # print(f"数据基准节点li的个数{len(all_li_nodes)}")

        # 遍历所有的li节点
        for li in all_li_nodes:
            # 建立模型存储数据
            model = dict()
            # 存储房源名称
            model['二手房房源名称'] = li.xpath('.//div[@class="title"]/a/text()')[0]
            # 存储房源所属小区名称
            model['二手房房源所属小区名称'] = li.xpath('.//div[@class="positionInfo"]/a[1]/text()')[0]
            # 存储房源小区位置
            model['二手房房源小区位置'] = li.xpath('.//div[@class="positionInfo"]/a[2]/text()')[0]
            # 存储房源总价
            model['二手房房源总价'] = li.xpath('.//div[@class="totalPrice totalPrice2"]/span/text()')[0] + '万'
            # 存储房源单价
            model['二手房房源单价'] = li.xpath('.//div[@class="unitPrice"]/span/text()')[0]

            # 存储房源具体信息
            house_info = li.xpath('.//div[@class="houseInfo"]/text()')[0]
            # 处理house_info 4室2厅 | 164.47平米 | 南 | 精装 | 中楼层(共5层)  | 暂无数据
            split_result = house_info.split('|')
            model['二手房房源房屋格局'] = split_result[0].strip()
            model['二手房房源房屋面积'] = split_result[1].strip()
            model['二手房房源房屋朝向'] = '朝' + split_result[2].strip()
            model['二手房房源房屋装修'] = split_result[3].strip()
            model['二手房房源楼层位置'] = split_result[4].strip()

            # 存储房源的详情链接
            model['二手房房源详情链接'] = li.xpath('.//div[@class="title"]/a/@href')[0]
            # 存储房源的示例图片
            model['二手房房源示例图片'] = li.xpath('.//a/img[2]/@data-original')[0]
            print(model)

            # 4.2 将存储好的数据放到数据源中
            self.data.append(model)

    def write_to_json(self):
        """写入文件"""
        # 1.定义字典
        temp = {"House_Information": self.data}  # self.data数据源为一个列表[]，里面放字典model{}
        # 2.将字典对象temp转化为json字符串
        content = json.dumps(temp, ensure_ascii=False, indent=4)
        # 3.写入文件
        with open('./链家二手房/二手房信息.json', 'w', encoding='utf-8') as f:
            f.write(content)

    def run(self, page_index=2):
        for index in range(1, page_index):
            # 拼接地址
            url = self.url.format(index)
            # print(url)
            # 发送请求
            html_list = self.get_response(url)
            # 解析列表页数据
            self.parse_data(html_list)
        # 写入json文件中
        self.write_to_json()


if __name__ == '__main__':
    page = int(input('请输入要爬取的页数：\n'))
    LHome(city='changzhou').run(page_index=page)
    print(f'已成功将链家二手房房源信息全部保存到：二手房信息.json 中,请查看"链家二手房"文件夹！')

import random
import requests
import json
from lxml import etree


class LHome:
    """链家二手房房源"""
    def __init__(self, city='hz'):
        # 定义请求地址
        self.url = "https://" + city + ".lianjia.com/ershoufang/pg{}"
        # 定义请求头
        self.header = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36"}
        # 定义代理池
        self.proxies = [{"HTTP": "27.42.168.46:55481"}, {"HTTP": "121.13.252.61:41564"}, {"HTTP": "183.236.232.160:8080"}]
        # 数据源
        self.data = []

    def get_response(self, url):
        """请求数据获取响应"""
        random_index = random.randint(0, len(self.proxies)-1)
        return requests.get(url, headers=self.header, proxies=self.proxies[random_index]).content.decode('utf-8')

    def parse_data(self, html):
        """解析数据"""
        # 创建xpath查找对象
        x_finder = etree.HTML(html)
        # 先找到数据所在的  基准节点
        all_li_nodes = x_finder.xpath("//div[@class='leftContent']/ul[@class='sellListContent']/li[contains(@class, 'LOGCLICKDATA')]")
        # print(f"数据基准节点li的个数{len(all_li_nodes)}")

        # 遍历所有的li节点
        for li in all_li_nodes:
            # 建立模型存储数据
            model = dict()
            # 存储房源名称
            model['二手房房源名称'] = li.xpath('.//div[@class="title"]/a/text()')[0]
            # 存储房源所属小区名称
            model['二手房房源所属小区名称'] = li.xpath('.//div[@class="positionInfo"]/a[1]/text()')[0]
            # 存储房源小区位置
            model['二手房房源小区位置'] = li.xpath('.//div[@class="positionInfo"]/a[2]/text()')[0]
            # 存储房源总价
            model['二手房房源总价'] = li.xpath('.//div[@class="totalPrice totalPrice2"]/span/text()')[0] + '万'
            # 存储房源单价
            model['二手房房源单价'] = li.xpath('.//div[@class="unitPrice"]/span/text()')[0]

            # 存储房源具体信息
            house_info = li.xpath('.//div[@class="houseInfo"]/text()')[0]
            # 处理house_info 4室2厅 | 164.47平米 | 南 | 精装 | 中楼层(共5层)  | 暂无数据
            split_result = house_info.split('|')
            model['二手房房源房屋格局'] = split_result[0].strip()
            model['二手房房源房屋面积'] = split_result[1].strip()
            model['二手房房源房屋朝向'] = '朝' + split_result[2].strip()
            model['二手房房源房屋装修'] = split_result[3].strip()
            model['二手房房源楼层位置'] = split_result[4].strip()

            # 存储房源的详情链接
            model['二手房房源详情链接'] = li.xpath('.//div[@class="title"]/a/@href')[0]
            # 存储房源的示例图片
            model['二手房房源示例图片'] = li.xpath('.//a/img[2]/@data-original')[0]
            print(model)

            # 4.2 将存储好的数据放到数据源中
            self.data.append(model)

    def write_to_json(self):
        """写入文件"""
        # 1.定义字典
        temp = {"House_Information": self.data}  # self.data数据源为一个列表[]，里面放字典model{}
        # 2.将字典对象temp转化为json字符串
        content = json.dumps(temp, ensure_ascii=False, indent=4)
        # 3.写入文件
        with open('./链家二手房/二手房信息.json', 'w', encoding='utf-8') as f:
            f.write(content)

    def run(self, page_index=2):
        for index in range(1, page_index):
            # 拼接地址
            url = self.url.format(index)
            # print(url)
            # 发送请求
            html_list = self.get_response(url)
            # 解析列表页数据
            self.parse_data(html_list)
        # 写入json文件中
        self.write_to_json()


if __name__ == '__main__':
    page = int(input('请输入要爬取的页数：\n'))
    LHome(city='changzhou').run(page_index=page)
    print(f'已成功将链家二手房房源信息全部保存到：二手房信息.json 中,请查看"链家二手房"文件夹！')