手机版百度贴吧scrapy爬取

我大LOL吧

'http://tieba.baidu.com/mo/m?word=LOL&tn6=bdISP&tn4=bdKSW&tn7=bdPSB&lp=1050&sub4=%E8%BF%9B%E5%90%A7'

这网页地址不用多说了吧
然后右键检查
记得打开toggle device toolbar

# -*- coding: utf-8 -*-
import scrapy,urllib,requests
from urllib.parse import urljoin

class TbSpider(scrapy.Spider):
    name = 'tb'
    allowed_domains = ['baidu.com']
    start_urls = ['http://tieba.baidu.com/mo/q---1E96CACB0B10B0286B51C743CA288B8A:FG=1-sz@320_240,-1-3-0--2--wapp_1545138808991_550/m?kw=%E6%9D%8E%E6%AF%85&lp=5011&lm=&pinf=1_1_0&pn=10']

    def parse(self, response):
        div_list = response.xpath('//div[contains(@class,"i")]')
        for div in div_list:
            item = {}
            item["img"] = []
            item["href"] = div.xpath('./a/@href').extract_first()
            item["title"] = div.xpath('./a/text()').extract_first()
            if item["href"] is not None:
                item["href"] = urllib.parse.urljoin(response.url,item["href"])
                yield scrapy.Request(
                    item["href"],
                    callback=self.parse_detail,
                    meta={"item":item}
                )
            else:
                print('握草')


        next_url = response.xpath('//a[text()="下一页"]/@href').extract_first()
        print('未处理的下一页'+str(next_url))
        if next_url is not None:
            next_url = urllib.parse.urljoin(response.url,next_url)
            yield scrapy.Request(
                next_url,
                callback=self.parse
            )
            print('处理后的下一页：'+str(next_url))
        else:
            #print(response.text)
            print('麻痹的没搜到')

    def parse_detail(self,response):
        item = response.meta["item"]
        item["img"].extend(response.xpath('//img[@class="BDE_Image"]/@src').extract())
        next_url = response.xpath('//a[text()="下一页"]/@href').extract_first()
        if next_url is not None:
            next_url = urllib.parse.urljoin(response.url,next_url)
            yield scrapy.Request(
                next_url,
                meta={"item":item},
                callback=self.parse_detail
            )

        else:
            item["img"] = [requests.utils.unquote(i.split('src=')[-1]) for i in item["img"]]
            print(item)
            yield item

print很多都是查错的。。。不想加注释了。。。，其他文件也懒得复制。。。2333

over