Scrapy CrawlSpider抓取数据

本文主要是对CrawlSpider爬虫的应用示例

数据爬取对象：中华网科技类新闻

url：https://tech.china.com/articles/

1. 创建项目

scrapy startproject zhonghuawang

2. 创建CrawlSpider爬虫

cd zhonghuawang
scrapy genspider -t crawl china tech.china.com

3. 页面分析

列表页：

通过F12开发者工具，你会发现所有的信息都包含在div[@class="m2left topborder"]这个标签里面，下面的每个字标签div[@class="con_item"]里面就包含每条新闻的信息，包括链接，通过观察，你会发现每条新闻的详情页链接的规律了

这样我们就可以写详情页的链接提取规则了，加之之后我们要在详情页里面提取需要的数据，则肯定需要有回调函数（callback）去处理

Rule(LinkExtractor(allow=r'/article/\d+/\d+.html'), callback='parse_item', follow=False)

接下来，我们再分析一下怎么翻页，通过点击下一页就很容易发现规律了

然后我们再通过F12开发者工具去查看下面翻页页面的信息

 

那么我们下一条Rule规则页就出来了，但它只需要提取链接，并不需要做其它的操作，所以不需要回调函数

Rule(LinkExtractor(allow=r'/articles/index[0-9_]+.html'))

完整的Rule规则如下：

rules = (
        Rule(LinkExtractor(allow=r'/articles/index[0-9_]+.html')),
        Rule(LinkExtractor(allow=r'/article/\d+/\d+.html'), callback='parse_item', follow=False)
    )

详情页：

我们需要提取的信息是：标题，链接，文章内容，发表时间，来源以及站点名称，确定好需要提取的哪些信息字段后，我们就可以写items.py文件了

import scrapy

class ZhonghuawangItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    # 标题
    title = scrapy.Field()
    # 链接
    url = scrapy.Field()
    # 正文
    content = scrapy.Field()
    # 发布时间
    datetime = scrapy.Field()
    # 来源
    source = scrapy.Field()
    # 站点名称
    website = scrapy.Field()

接下来就是通过F12开发者工具去查看需要提取的信息分别在哪个标签下，然后使用xpath规则进行提取

def parse_item(self, response):
        item = ZhonghuawangItem()
        # 标题
        item['title'] = response.xpath('//h1[@id="chan_newsTitle"]/text()').extract()[0]
        # 链接
        item['url'] = response.url
        # 内容
        # item['content'] = response.xpath('//div[@id="chan_newsDetail"]/p/text()').extract()[0]
        item['content'] = ''.join(response.xpath('//div[@id="chan_newsDetail"]//text()').extract()).strip()
        # 发布时间
        item['datetime'] = ' '.join(response.xpath('//div[@id="chan_newsInfo"]/text()').extract()[1].strip().split(' ')[:2])
        # 来源
        # item['source'] = response.xpath('//div[@id="chan_newsInfo"]/text()').extract()[1].strip().split(' ')[-1][3:]
        item['source'] = response.xpath('//div[@id="chan_newsInfo"]/text()').re_first('来源：(.*)').strip()
        # 站点名称
        item['website'] = '中华网'
        yield item

4. 保存信息

对需要的信息提取完成后，最后就写入到json文件进行保存（你可以保存到数据库中），那么我们就要写pipelines.py文件了

import json

class ZhonghuawangPipeline(object):
    def __init__(self):
        self.filename = open('zhonghuawnag.json', 'w')

    def process_item(self, item, spider):
        text = json.dumps(dict(item), ensure_ascii=False) + '\n'
        self.filename.write(text)
        return item

    def close_spider(self, spider):
        self.filename.close()

5. 修改settings.py文件

#不遵守robots.txt协议
ROBOTSTXT_OBEY = False

#不使用Cookies
COOKIES_ENABLED = False

#设置请求报头
DEFAULT_REQUEST_HEADERS = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.62 Safari/537.36',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
  # 'Accept-Language': 'en',
}

#开启piprlines
ITEM_PIPELINES = {
   'zhonghuawang.pipelines.ZhonghuawangPipeline': 300,
}

6. 运行代码即可

scrapy crawl china

 

完整代码如下：

items.py

import scrapy


class ZhonghuawangItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    # 标题
    title = scrapy.Field()
    # 链接
    url = scrapy.Field()
    # 正文
    content = scrapy.Field()
    # 发布时间
    datetime = scrapy.Field()
    # 来源
    source = scrapy.Field()
    # 站点名称
    website = scrapy.Field()

 

china.py

from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from zhonghuawang.items import ZhonghuawangItem


class ChinaSpider(CrawlSpider):
    name = 'china'
    allowed_domains = ['tech.china.com']
    start_urls = ['http://tech.china.com/articles/']

    rules = (
        Rule(LinkExtractor(allow=r'/articles/index[0-9_]+.html')),
        Rule(LinkExtractor(allow=r'/article/\d+/\d+.html'), callback='parse_item', follow=False)
    )

    def parse_item(self, response):
        item = ZhonghuawangItem()
        # 标题
        item['title'] = response.xpath('//h1[@id="chan_newsTitle"]/text()').extract()[0]
        # 链接
        item['url'] = response.url
        # 内容
        # item['content'] = response.xpath('//div[@id="chan_newsDetail"]/p/text()').extract()[0]
        item['content'] = ''.join(response.xpath('//div[@id="chan_newsDetail"]//text()').extract()).strip()
        # 发布时间
        item['datetime'] = ' '.join(response.xpath('//div[@id="chan_newsInfo"]/text()').extract()[1].strip().split(' ')[:2])
        # 来源
        # item['source'] = response.xpath('//div[@id="chan_newsInfo"]/text()').extract()[1].strip().split(' ')[-1][3:]
        item['source'] = response.xpath('//div[@id="chan_newsInfo"]/text()').re_first('来源：(.*)').strip()
        # 站点名称
        item['website'] = '中华网'
        yield item

pipelines.py

from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from zhonghuawang.items import ZhonghuawangItem


class ChinaSpider(CrawlSpider):
    name = 'china'
    allowed_domains = ['tech.china.com']
    start_urls = ['http://tech.china.com/articles/']

    rules = (
        Rule(LinkExtractor(allow=r'/articles/index[0-9_]+.html')),
        Rule(LinkExtractor(allow=r'/article/\d+/\d+.html'), callback='parse_item', follow=False)
    )

    def parse_item(self, response):
        item = ZhonghuawangItem()
        # 标题
        item['title'] = response.xpath('//h1[@id="chan_newsTitle"]/text()').extract()[0]
        # 链接
        item['url'] = response.url
        # 内容
        # item['content'] = response.xpath('//div[@id="chan_newsDetail"]/p/text()').extract()[0]
        item['content'] = ''.join(response.xpath('//div[@id="chan_newsDetail"]//text()').extract()).strip()
        # 发布时间
        item['datetime'] = ' '.join(response.xpath('//div[@id="chan_newsInfo"]/text()').extract()[1].strip().split(' ')[:2])
        # 来源
        # item['source'] = response.xpath('//div[@id="chan_newsInfo"]/text()').extract()[1].strip().split(' ')[-1][3:]
        item['source'] = response.xpath('//div[@id="chan_newsInfo"]/text()').re_first('来源：(.*)').strip()
        # 站点名称
        item['website'] = '中华网'
        yield item

settings.py

#不遵守robots.txt协议
ROBOTSTXT_OBEY = False

#不使用Cookies
COOKIES_ENABLED = False

#设置请求报头
DEFAULT_REQUEST_HEADERS = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.62 Safari/537.36',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
  # 'Accept-Language': 'en',
}

#开启piprlines
ITEM_PIPELINES = {
   'zhonghuawang.pipelines.ZhonghuawangPipeline': 300,
}