scrapy的分页(翻页处理)

import scrapy
from bs4 import BeautifulSoup


class BookSpiderSpider(scrapy.Spider):
    name = 'book_spider'
    allowed_domains = ['17k.com']
    start_urls = ['https://www.17k.com/all/book/2_0_0_0_0_0_0_0_1.html']

	"""
	后边会解释这个start_requests方法是肿么肥事
	"""

    def start_requests(self):
        for url in self.start_urls:
            yield scrapy.Request(
                url=url,
                callback=self.parse
            )

    def parse(self, response, **kwargs):
        print(response.url)
        soup = BeautifulSoup(response.text, 'lxml')
        trs = soup.find('div', attrs={'class': 'alltable'}).find('tbody').find_all('tr')[1:]
        for tr in trs:
            book_type = tr.find('td', attrs={'class': 'td2'}).find('a').text
            book_name = tr.find('td', attrs={'class': 'td3'}).find('a').text
            book_words = tr.find('td', attrs={'class': 'td5'}).text
            book_author = tr.find('td', attrs={'class': 'td6'}).find('a').text
            print(book_type, book_name, book_words, book_author)
            #
            break
        """
        下面这个是xpath解析的方式
        """
        # trs = response.xpath("//div[@class='alltable']/table/tbody/tr")[1:]
        # for tr in trs:
        #     type = tr.xpath("./td[2]/a/text()").extract_first()
        #     name = tr.xpath("./td[3]/span/a/text()").extract_first()
        #     words = tr.xpath("./td[5]/text()").extract_first()
        #     author = tr.xpath("./td[6]/a/text()").extract_first()
        #     print(type, name, words, author)

        
        """
        1 找到下一页的url,请求到下一页 分页逻辑 这个逻辑是最简单的那种
        线性的一直爬
        """
        # next_page_url = soup.find('a', text='下一页')['href']
        # if 'javascript' not in next_page_url:
        #     yield scrapy.Request(
        #         url=response.urljoin(next_page_url),
        #         method='get',
        #         callback=self.parse
        #     )

        # 2 残暴的分页逻辑
        """
        拿到所有的url,就发送请求 会经过引擎、调度器(集合 队列) 完成这里的去重工作，交给下载器,
        再返回给爬虫,
        但是那个start_url还是会重复,重复的原因是继承了Spider这个类
        在于父类的方法有dont_filter=True这个东西
        所以这里要重写Spider类里面的start_requests的方法,默认就过滤了
        这样就解决了start_url重复的问题
        """
        # 
        # 
        a_list = soup.find('div', attrs={'class': 'page'}).find_all('a')
        for a in a_list:
            if 'javascript' not in a['href']:
                yield scrapy.Request(
                    url=response.urljoin(a['href']),
                    method='get',
                    callback=self.parse
                )