Python网络爬虫--项目实战--scrapy爬取贝壳网

一、完整代码

bk.py

import json

import scrapy

from ScrapyAdvanced.items import HouseItem


class BkSpider(scrapy.Spider):
    name = 'bk'
    allowed_domains = ['bj.ke.com']
    start_urls = ['https://bj.ke.com/ershoufang/']

    def parse(self, response):

        # house_list_position = response.xpath("//div[@class='address']/div/div/a/text()").extract()
        # # 存在脏数据  一个空的脏数据 一个 正常数据
        # house_list_info = response.xpath("//div[@class='address']/div[@class='houseInfo']/text()").extract()
        house_list_address = response.xpath("//div[@class='address']")
        house_list_titles = response.xpath("//div[@class='title']/a/text()").extract()
        for i in range(len(house_list_address)):
            title = house_list_titles[i]

            house = house_list_address[i]
            position = house.xpath("./div/div/a/text()").extract_first()
            info = "".join(house.xpath("./div[@class='houseInfo']/text()").extract()).replace("\n", "").replace(" ","")
            tags = ";".join(house.xpath("./div[@class='tag']/span/text()").extract())
            total_price = house.xpath("./div[@class='priceInfo']/div[@class='totalPrice']/span/text()").extract_first() + "万"
            unit_price = house.xpath("./div[@class='priceInfo']/div[@class='unitPrice']/span/text()").extract_first()

            house_item = HouseItem()
            house_item["title"] = title
            house_item["position"] = position
            house_item["info"] = info
            house_item["tags"] = tags
            house_item["total_price"] = total_price
            house_item["unit_price"] = unit_price

            yield house_item

        page_info = json.loads(response.xpath("//div[@class='page-box house-lst-page-box']/@page-data").extract_first())
        page_url = response.xpath("//div[@class='page-box house-lst-page-box']/@page-url").extract_first()
        total_page = page_info.get("totalPage")
        current_page = page_info.get("curPage")
        # 当前页面不是总页面的时候， 也就是说 不是最后一页， 去请求下一页
        if current_page != total_page:
            url = page_url.replace("{page}", str(current_page+1) + "/")
            yield response.follow(url=url, callback=self.parse)

        # 当前页是第一页，发送请求后面所有的页
        # if current_page == 1:
        #     for i in range(2, total_page):
        #         url = page_url.replace("{page}", str(i))
        #         yield response.follow(url=url, callback=self.parse)

items.py

class HouseItem(scrapy.Item):

    title = scrapy.Field()
    position = scrapy.Field()
    info = scrapy.Field()
    tags = scrapy.Field()
    total_price = scrapy.Field()
    unit_price = scrapy.Field()

pipelines.py

class HousePipeline():

    def process_item(self, item, spider):

        if isinstance(item, HouseItem) and isinstance(spider, BkSpider):
            # 存储
            with open("house_list.csv", "a", encoding="utf-8") as house_file:
                csv_writer = csv.writer(house_file)
                row = [it[1] for it in item.items()]
                csv_writer.writerow(row)
		return item