利用scrapy爬取链家网小区数据

爬取的起始网址https://bj.lianjia.com/xiaoqu/pg1cro22/

1.创建项目

scrapy startproject lianjiaSpider
cd lianjiaSpider
scrapy genspider lj lianjia.com

执行完了之后会生成这几个目录

2.编写items.py

# -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/items.html

import scrapy


class LianjiaspiderItem(scrapy.Item):
    # define the fields for your item here like:
    name = scrapy.Field()#名称
    volume = scrapy.Field()#最近30天的成交量
    rent = scrapy.Field()#出租量
    news = scrapy.Field()#相关信息
    href = scrapy.Field()#详情页连接
    build_time = scrapy.Field()#修建时间
    style = scrapy.Field()#建筑类型
    square_meter = scrapy.Field()#多少钱每平米
    property_fee = scrapy.Field()#物业费
    build_number = scrapy.Field()#楼栋总数
    house_number = scrapy.Field()#楼栋总数

3.修改相关 的设置（settins.py）

4.编写爬虫程序lj.py

# -*- coding: utf-8 -*-
import scrapy
from ..items import LianjiaspiderItem
import copy

class LjSpider(scrapy.Spider):
    name = 'lj'#爬虫名字
    allowed_domains = ['lianjia.com']#爬虫允许爬取的网站域名
    start_urls = ['https://bj.lianjia.com/xiaoqu/pg{}cro22/'.format(i)for i in range(1,31)]#爬虫开始爬取的url

    def parse(self, response):#列表页
        item=LianjiaspiderItem()
        li_list=response.xpath('//ul[@class="listContent"]/li')
        for li in li_list:#提取数据
            item['name']=li.xpath('.//div[@class="title"]/a/text()').extract_first()
            item['volume']=li.xpath('.//div[@class="houseInfo"]/a/text()').extract_first()
            item['rent']=li.xpath('.//div[@class="houseInfo"]/a[2]/text()').extract_first()
            item['news']=li.xpath('.//div[@class="positionInfo"]/a//text()').extract()
            item['href']=li.xpath('.//div[@class="title"]/a/@href').extract_first()
            yield scrapy.Request(item['href'],callback=self.details,#发送请求进入详情页面
                                 meta={'item':copy.deepcopy(item)})#将列表页提取的数据带到详情页面，完善信息的提取
    def details(self,response):#详情页
        item=response.meta['item']#在详情页面提取数据
        item['build_time']=response.xpath('//div[@class="xiaoquInfo"]/div[@class="xiaoquInfoItem"][1]/span[2]/text()').extract_first()
        item['style']=response.xpath('//div[@class="xiaoquInfo"]/div[@class="xiaoquInfoItem"][2]/span[2]/text()').extract_first()
        item['square_meter']=response.xpath('//div[@class="xiaoquPrice clear"]/div/span[1]/text()').extract_first()+'元/㎡'
        item['property_fee']=response.xpath('//div[@class="xiaoquInfo"]/div[@class="xiaoquInfoItem"][3]/span[2]/text()').extract_first()
        item['build_number']=response.xpath('//div[@class="xiaoquInfo"]/div[@class="xiaoquInfoItem"][6]/span[2]/text()').extract_first()
        item['house_number']=response.xpath('//div[@class="xiaoquInfo"]/div[@class="xiaoquInfoItem"][7]/span[2]/text()').extract_first()
        yield item#返回所有提取到的信息

5.在pipelines.py中保存数据

# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html

import csv
class LianjiaspiderPipeline(object):
    def process_item(self, item, spider):
        print('正在保存：'+item['name'])
        #保存数据
        with open('house.csv','a+',newline='')as f:
            w=csv.writer(f)
            arges=[item['name'],item['volume'],item['rent'],item['news'],
                       item['build_time'],item['style'],item['square_meter'],
                       item['property_fee'],item['build_number'],item['house_number']]
            w.writerow(arges)

6.创建run.py 文件

import os
os.system('scrapy crawl lj --nolog')

7.然后运行run.py文件

该项目百度网盘提取连接

https://pan.baidu.com/s/1x1R_2FaX7R6lGELQ2yPrDg 

提取码：eayt