Scrapy 框架 CrawlSpider 全站数据爬取-CSDN博客

CrawlSpider 全站数据爬取

创建 crawlSpider 爬虫文件

scrapy genspider -t crawl chouti www.xxx.com

import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule


class CrawSpider(CrawlSpider):
    name = 'craw'
    # allowed_domains = ['www.xxx.com']
    start_urls = ['https://dig.***.com/r/scoff/hot/1']
    #连接提取器：可以根据指定条件提取连接
    link = LinkExtractor(allow=r'/r/scoff/hot/\d+')
    # link1 = LinkExtractor(allow=r'/pic/$')  针对于第一页的 url 不同的 页面使用

    rules = (
        #规则解析器：将连接提取器提取到的连接对应的页面进行指定规则的数据解析
        Rule(link, callback='parse_item', follow=True),
        #参数follow=True：将连接提取器继续作用到连接提取器提取到的连接所有对应的页面中
        # Rule(link1, callback='parse_item', follow=False),
    )

    def parse_item(self, response):
        print(response)

对于简介与详情不是一个 item 的存储

# -*- coding: utf-8 -*-
import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule

from tenPro.items import TenproItem, TenproItem_detail


class TenSpider(CrawlSpider):
    name = 'ten'
    # allowed_domains = ['www.ccc.com']
    start_urls = ['https://hr.****.com/position.php?&start=#a0']
    rules = (
        Rule(LinkExtractor(allow=r'&start=\d+#a'), callback='parse_item', follow=True),
        Rule(LinkExtractor(allow=r'position_detail.php\?id ='), callback='parse_detail', follow=True),
    )

    def parse_item(self, response):
        # 岗位名称和类别
        tr_list = response.xpath(
            '//table[@class="tablelist"]/tr[@class="odd"] | //table[@class="tablelist"]/tr[@class="even"]')
        for tr in tr_list:
            title = tr.xpath('./td[1]/a/text()').extract_first()
            kind = tr.xpath('./td[2]/text()').extract_first()
            item = TenproItem()
            item['title'] = title
            item['kind'] = kind
            yield item

    def parse_detail(self, response):
        desc = response.xpath('//ul[@class="squareli"]//text()').extract()
        desc = ''.join(desc)
        item = TenproItem_detail()
        item['desc'] = desc

        yield item

import scrapy

class TenproItem(scrapy.Item):
    # define the fields for your item here like:
    title = scrapy.Field()
    kind = scrapy.Field()
    # pass
class TenproItem_detail(scrapy.Item):
    desc = scrapy.Field()

# 分别进行存储  利用数据库的 多表联查  或数据解析 
class TenproPipeline(object):
    def process_item(self, item, spider):
        desc = None
        if item.__class__.__name__ == 'TenproItem_detail':
            desc = item['desc']
        else:
            title = item['title']
            kind = item['kind']
        print(item)
        return item