通过相对路径获取下一页的链接

# -*- coding: utf-8 -*-
import scrapy
import re
from urllib.parse import urljoin

from ..items import ScrapyItem


class DocsScrapySpider(scrapy.Spider):
    name = 'docs.scrapy'
    allowed_domains = ['docs.scrapy.org']
    start_urls = ['https://docs.scrapy.org/en/latest/index.html']

    def parse(self, response):
        docs = response.xpath('//div[@class="document"]').extract_first()
        print(docs)
        scrapy_docs = ScrapyItem()
        scrapy_docs["docs_scrapy"] = docs
        yield scrapy_docs

        next_page = response.xpath('//a[@rel="next"]/@href').extract_first()
        if next_page:
            next_page = urljoin(response.url, next_page)
            print(next_page)
            yield response.follow(next_page, self.parse)

urljoin

1. 引入urllib.parse模块下的urljoin，
2. 获取当前页的链接response.url
3. 将当前页的地址和下一页的相对路径地址拼接，从而获取下一页的链接

转载于:https://www.cnblogs.com/chenliang0309/p/10030723.html