Scrapy 学习笔记 - 一个练手任务 ，爬取起点的全部小说名

import scrapy

class myFirstSpider(scrapy.Spider):
    name = "myFirst_Spider_Task"
    #allowed_domains = ["cmis.bjedu.cn"]
    start_urls = ["https://www.qidian.com/all",] #第一个网址
    def parse(self, response):
        
        selector = response.xpath("/html/body/div[2]/div[5]/div[2]/div[2]/div/ul") #本页的小说元素集合
        if len(selector) > 0:
            selector = selector[0]
            for sub in selector.xpath("./li/div[2]/h4/a"):
                href = sub.xpath("./@href")[0].extract() #小说跳转链接
                name = sub.xpath("./text()")[0].extract() #小说名字
                print href,name
        #下一个网址
        next_url = response.xpath("/html/body/div[2]/div[5]/div[2]/div[3]/div/div/ul/li")[-1].xpath("./a/@href")[0].extract()
        if len(next_url) > 20: #这里简单判断结束条件
            yield response.follow(next_url,callback=self.parse)