【Scrapy LinkExtractor的使用2】爬全站

        LinkExtractor可以看做是专门过滤返回网页内容中网页链接的正则表达式筛选工具，它的另一个功能是读取筛选的链接。参数用来对筛选的链接进行限制。

       在爬虫.py文件中就应该显示的是rule表，可以同时定义两个链接筛选器，一个用来爬取并访问下一页链接，一个用来爬取并访问列表页中的内容页。

       下边是一个完整的爬虫爬取全站实例：爬虫.py

# -*- coding: utf-8 -*-
import scrapy
from scrapy.spiders import CrawlSpider, Rule
from scrapy.linkextractors import LinkExtractor

class InurlSpider(CrawlSpider):
    name = 'Inurl'
    allowed_domains = ['sh.ziroom.com']
    start_urls = ['http://sh.ziroom.com/z/?qwd=%E4%B8%8A%E5%A4%A7']
    link1=LinkExtractor(allow=r'.*sh.ziroom.com/z/.*',restrict_css='div.Z_pages a.next')
    link=LinkExtractor(allow=r'.*sh.ziroom.com/x/.*',restrict_css='div.info-box h5.title a')
    rules = (
        # follow:是指爬取了之后，是否还继续从该页面提取链接，然后继续爬下去
        #Rule(link1,callback='parse_item', follow=True),
        Rule(link,callback='parse_item', follow=True),
        Rule(link1,callback='parse_info', follow=True),
    )
    def parse_item(self, response):
        print('-' * 100)
        print(response.url)
    def parse_info(self, response):
        print('-' * 100)
        print("下一页" * 100)
        print(response.url)

启动文件：

import scrapy
from scrapy.crawler import CrawlerProcess
from scrapy.utils.project import get_project_settings
process = CrawlerProcess(get_project_settings())
process.crawl('Inurl')#爬虫名称
process.start() # the script will block here until the crawling is finished