Scrapy分布式爬虫 - 房天下案例
一、项目准备与创建
1、项目准备
- 服务器(项目+Redis调度) + MongoDB +Redis
- 本机(项目)
2、项目创建
>scrapy startproject fangtianxia
>cd fangtianxia
>scrapy genspider fang fang.com
改写settings.py中的robot协议为False
修改spiders文件:
fang.py
from scrapy_redis.spiders import RedisSpider
class FangSpider(RedisSpider)
redis_key = 'fang:urls'
二、编写Scrapy项目
1、分析网页结构
主页:
中间页:
以第一个北京为例:
新房链接
二手房链接
最终页:
新房网页:
二手房网页:
2、fang.py
# -*- coding: utf-8 -*-
import scrapy
from scrapy_redis.spiders import RedisSpider
import urllib.parse
from ..items import FangtianxiaItem, FangDetailsItem, SecDetailsItem
class FangSpider(RedisSpider):
name = 'fang'
allowed_domains = ['fang.com']
# start_urls = ['http://fang.com/']
redis_key = 'fang:urls'
# lpush fangLurls https://www.fang.com/SoufunFamily.html
def parse(self, response):
trs = response.css('#c02 tr')
province = ''
for tr in trs:
item = FangtianxiaItem()
item['pro'] = tr.css('td>strong::text').get()
# 如果有省份,则把省份保存下来,便于传递给后面的无省份的地点
if item['pro']:
if item['pro'].strip() == '':
item['pro'] = province
# 无省份,则将该市的省份被赋值给前面保存的省份
else:
province = item['pro']
if item['pro'] == '其他':
continue
item['city'] = tr.css('td>a::text').getall()
item['href'] = tr.css('td>a::attr(href)').getall()
# 获取每个城市,每个a标签
for city, href in zip(item['city'], item['href']):
yield scrapy.Request(href, callback=self.parse_city,
meta={'province': province, 'city': city})
def parse_city(self, response):
new_house_url = response.css('a#dsy_D04_01::attr(href)').get()
if new_house_url:
yield scrapy.Request(new_house_url, meta=response.meta, callback=self.parse_new_house)
sec_house_url = response.css('a#dsy_D05_01::attr(href)').get()
if sec_house_url:
yield scrapy.Request(sec_house_url, meta=response.meta, callback=self.parse_sec_house)
print('response.url', response.url)
print('response meta', response.meta)
print()
def parse_new_house(self, response):
lis = response.css('.nhouse_list ul>li[id]')
province = response.meta.get('province')
city = response.meta.get('city')
print('省(新):', province)
print('市(新):', city)
for li in lis:
item = FangDetailsItem()
item['pro'] = province
item['city'] = city
# 类似于get方法
item['name'] = li.css('.nlc_details .nlcd_name>a::text').re('\S+')
print('房源:', item['name'])
# 户型
item['room'] = li.css('.house_type a::text').getall()
print('户型:', item['room'])
# 面积
# 即将开盘要特殊处理
item['area'] = li.css('.house_type::text').re('[\d~平米]+')
if item['area'] is None:
item['area'] = '即将开盘'
print('面积::', item['area'])
# 地址
site = li.css('.relative_message a::attr(title)').re(r'[(.*?)]')
if site is None:
item['place'] = li.css('.relative_message a::text').re('[\S~]+')
else:
item['place'] = site + li.css('.relative_message a::text').re('[\S~]+')
print('地址:', item['place'])
# 单价
# 特殊标签,如:https://tj.newhouse.fang.com/house/s/
# 价格待定 往期40000元/㎡,价格在i标签中
price = li.css('.nhouse_price span::text').get()
if price is None:
price = li.css('.nhouse_price i::text').get()
unit = li.css('.nhouse_price em::text').get()
# 价格待定,没单位
if unit is None:
unit = ''
item['price'] = price + unit
print('单价:', item['price'])
# 标签 - 在售
on_sale = li.css('.fangyuan span::text').get()
# 标签 - 其他
label1 = ','.join(li.css('.fangyuan a[href]::text').getall())
label2 = ','.join(li.css('.fangyuan a.scSerch em::text').getall())
if label2 is None:
label2 = '无'
item['label'] = on_sale + ',' + label1 + ',' + label2
print('标签:', item['label'])
# 房源地址
item['origin_url'] = li.css('.nlc_details .nlcd_name>a::attr(href)').get()
print('房源网址:', item['origin_url'])
print()
yield item
base_url = 'https://newhouse.fang.com/'
next_page = response.css('a.next::attr(href)').get()
if next_page:
url = urllib.parse.urljoin(base_url, next_page)
yield scrapy.Request(url, meta=response.meta, callback=self.parse_new_house)
def parse_sec_house(self, response):
# 注意,dl有广告,不匹配,要加上属性
dls = response.css('.shop_list dl[dataflag=bg]')
province = response.meta.get('province')
city = response.meta.get('city')
print('省(二手):', province)
print('市(二手):', city)
for dl in dls:
item = SecDetailsItem()
item['pro'] = province
item['city'] = city
item['title'] = dl.css('h4.clearfix a span::text').re('\S+')
print('房源:', item['title'])
item['house_info'] = dl.css('p.tel_shop::text').re('\w+')
item['house_info'][1] += '㎡'
print('房源信息:', item['house_info'])
item['community'] = dl.css('p.add_shop a::text').re('\w+')
print('社区名:', item['community'])
item['address'] = dl.css('p.add_shop span::text').get()
print('地址:', item['address'])
item['label'] = dl.css('p.clearfix.label span::text').re('\w+')
if not item['label']:
item['label'] = '无'
print('标签:', ','.join(item['label']))
area_price = dl.css('dd.price_right span:nth-last-child(1)::text').get()
item['price'] = area_price
print('价格:', item['price'])
price = dl.css('dd.price_right span.red b::text').get()
unit = dl.css('dd.price_right span.red::text').re('\w+')
item['sum_price'] = price + unit[0]
print('总价:', item['sum_price'])
item['origin_url'] = dl.css('h4.clearfix a::attr(href)').get()
print('房源页面:', item['origin_url'])
print()
yield item
base_url = 'https://cq.esf.fang.com/'
next_page = response.css('.page_box p>a:nth-child(1)::attr(href)').get()
if next_page:
url = urllib.parse.urljoin(base_url, next_page)
yield scrapy.Request(url, meta=response.meta, callback=self.parse_sec_house)
3、items.py
import scrapy
class FangtianxiaItem(scrapy.Item):
# define the fields for your item here like:
# 省份
pro = scrapy.Field()
# 城市
city = scrapy.Field()
# 城市链接
href = scrapy.Field()
class FangDetailsItem(scrapy.Item):
# define the fields for your item here like:
# 省份
pro = scrapy.Field()
# 城市
city = scrapy.Field()
# 房源名
name = scrapy.Field()
# 户型
room = scrapy.Field()
# 面积
area = scrapy.Field()
# 地址
place = scrapy.Field()
# 单价
price = scrapy.Field()
# 标签
label = scrapy.Field()
# 房源网址
origin_url = scrapy.Field()
class SecDetailsItem(scrapy.Item):
# define the fields for your item here like:
# 省份
pro = scrapy.Field()
# 城市
city = scrapy.Field()
# 房源
title = scrapy.Field()
# 房源信息
house_info = scrapy.Field()
# 社区名字
community = scrapy.Field()
# 地址
address = scrapy.Field()
# 标签
label = scrapy.Field()
# 单价 平米
price = scrapy.Field()
# 总价
sum_price = scrapy.Field()
# 房源网址
origin_url = scrapy.Field()
4、pipelines.py
主动设置保存到mongoDB,而Redis的保存默认在scrapy_redis包的默认配置中,自动下载,无需更改。
import pymongo
from .items import FangDetailsItem, SecDetailsItem
class FangtianxiaPipeline(object):
def __init__(self):
self.client = pymongo.MongoClient(
host='39.97.118.3',
port=27017,
)
# 创建数据库house_info
self.collection = self.client.house_infos
# 创建新房集合new_house
self.new_house = self.collection.new_house
# 创建二手房集合sec_house
self.sec_house = self.collection.sec_house
def process_item(self, item, spider):
# 新房item的子类
if isinstance(item, FangDetailsItem):
self.new_house.insert(dict(item))
# 二手房item的子类
if isinstance(item, SecDetailsItem):
self.sec_house.insert(dict(item))
return item
def close_spider(self, spider):
self.client.close()
5、settings.py
## 1. 更改
# 不遵循robot协议
ROBOTSTXT_OBEY = False
# 配置管道文件,使其生效
ITEM_PIPELINES = {
'scrapy_redis.pipelines.RedisPipeline': 400,
'fangtianxia.pipelines.FangtianxiaPipeline': 401,
}
## 2. 添加
# 远程服务器ip
REDIS_HOST = 'XX.XX.XX.XX'
REDIS_PORT = 6379
# 去重组件,redis_scrapy实现一个去重方案
DUPEFILTER_CLASS = "scrapy_redis.dupefilter.RFPDupeFilter"
# SCHEDULER 修改调度器,把任务分发的逻辑发给服务器(默认的调度器在本地进行调度)
SCHEDULER = "scrapy_redis.scheduler.Scheduler"
# 开启数据持久化(把爬取后的数据保存下来)
SCHEDULER_PERSIST = True
三、服务器操作
1、更改配置
Redis开放绑定端口
sudo vi /etc/redis/redis.conf
# bind 127.0.0.1
bind 0.0.0.0
# 修改默认端口
port 6379
# 重启服务
sudo service redis-server restart
MongoDB开放绑定ip
sudo vim /etc/mongodb.conf
# 只监听本地接口。注释掉听在所有接口。
bind_ip = 127.0.0.1 # 注释
bind_ip = 0.0.0.0 # 添加
# 重启
sudo service mongod restart
2、更改项目中的ip
将本地fangtianxia项目拷贝一份到服务器
更改fangtainxia项目中settings.py文件中的REDIS_HOST 为服务器本机ip
REDIS_HOST = '127.0.0.1'
四、启动项目
同时启动本地和服务器
scrapy crawl fang
向redis中投入初始url
# 先清空redis
flushdb
# 再投入url
lpush fang:urls https://www.fang.com/SoufunFamily.html
本机和服务器项目都开始跑
本地:
服务器:
可以随时ctrl+c停止运行,下一次再次启动项目时,则无需再投入初始url,项目便会自动运行。服务器和本地两台机器,由Redis分别分配url获取数据。
mongoDB:
Redis:
如果有多台机器,可以将项目分发到多个机器上运行,从而实现更高效的分布式爬取。