百度贴吧-crawlspider版本

# -*- coding: utf-8 -*-
import scrapy
import urllib
import requests


class TbSpider(scrapy.Spider):
    name = 'tb'
    allowed_domains = ['tieba.baidu.com']
    start_urls = ['https://tieba.baidu.com/f?kw=%C0%EE%D2%E3&fr=ala0&tpl=5&pn=0&']

    def parse(self, response):
        #  分组
        li_list = response.xpath("//ul[@class='threads_list']/li")
        for li in li_list:
            item = dict()
            item["img"] = li.xpath(".//img/@src").extract()
            item["title"] = li.xpath(".//div[@class='ti_title']/text()").extract_first()
            item["href"] = li.xpath("./a/@href").extract_first()
            if item["href"] is not None:
                item["href"] = urllib.parse.urljoin(response.url, item["href"])
                yield scrapy.Request(
                    item["href"],
                    callback=self.parse_detail,
                    meta={"item": item},
                )

        # 翻页
        next_url = response.xpath("//a/span[@class='pager_next_icon']").extract_first()
        if next_url is not None:
            next_url = urllib.parse.urljoin(response.url, next_url)
            yield scrapy.Request(
                next_url,
                callback=self.parse,
            )

    def parse_detail(self, response):
        item = response.meta["item"]
        next_url = response.xpath("//a/span[@class='pager_next_icon']").extract_first()
        if next_url is not None:
            next_url = urllib.parse.urljoin(response.url, next_url)
            yield scrapy.Request(
                next_url,
                callback=self.parse_detail,
                meta={"item": item},
            )
        else:
            item["img"] = [requests.utils.unquote(i).split("src")[-1] for i in item["img"]]
            print(item)