记录一个爬虫出现的问题

爬取数据时,每次爬取出来的值都是一样的,先贴上代码

import scrapy
from jdbook.items import JdbookItem
import json


class MyjdSpider(scrapy.Spider):
    name = 'myjd'
    allowed_domains = ['jd.com', 'p.3.cn']
    start_urls = ['https://book.jd.com/booksort.html']

    def parse(self, response):
        # 1.先判断页面中的元素是否在骨骼文件中
        # 在骨骼文件中后进行遍历
        big_list = response.xpath('//*[@id="booksort"]/div[2]/dl/dt/a')
        for group in big_list[0:1]:
            # 获取大标签的名字和标题
            big_name = group.xpath('./text()').extract_first()
            big_link = 'https:' + group.xpath('./@href').extract_first()
            # 获取所有的小标签 小标签是兄弟节点
            small_list = group.xpath('../following-sibling::dd[1]/em')
            for small_group in small_list[0:1]:
                my_item = JdbookItem()
                my_item['bigname'] = big_name
                my_item['biglink'] = big_link
                my_item['smallname'] = small_group.xpath('./a/text()').extract_first()
                my_item['smalllink'] = 'https:' + small_group.xpath('./a/@href').extract_first()
                yield scrapy.Request(url=my_item['smalllink'], callback=self.book_message, meta={'data': my_item})

    def book_message(self, response):

        # 获取图书列表信息
        book_list = response.xpath('//*[@id="plist"]/ul/li/div/div[3]/a')
        item = response.meta['data']

        for book in book_list[0:10]:
            # item = JdbookItem()
            # 数据传递
            item['name'] = book.xpath('./em/text()').extract_first().strip()
            item['auth'] = book.xpath('./../../div[4]/span[1]/span/a/text()').extract_first()
            sku_id = book.xpath('./../../@data-sku').extract_first()
            url = 'https://p.3.cn/prices/mgets?skuIds=J_' + sku_id
            item['bookurl'] = 'https:' + book.xpath('./../../div[1]/a/@href').extract_first()
            yield scrapy.Request(url=url, callback=self.get_price, meta={'data': item})

    def get_price(self, response):
        item = response.meta['data']
        item['price'] = json.loads(response.body.decode())[0]['op']
        print(item)
   

此时出现的问题,是每次爬取出的图书的名字和作者,都是该页面最后一个图书的信息,分析了很久,终于找到了原因:
在parse函数中生成JdbookItem的实例对象时,每一个小的二级分类才能生成出一个对象,那么将这个数据传递到book_message函数时,在赋值时会导致重复赋值,最后的结果就是被最后一个图书的信息覆盖掉
解决方案:book_message函数重新创建一个JdbookItem实例对象,然后将之前的信息赋值给这个对象,就可以解决

    def book_message(self, response):

        # 获取图书列表信息
        book_list = response.xpath('//*[@id="plist"]/ul/li/div/div[3]/a')
        item2 = response.meta['data']

        for book in book_list[0:4]:
            item = JdbookItem()
            # 数据传递
            item['bigname'] = item2['bigname']
            item['biglink'] = item2['biglink']
            item['smallname'] = item2['smallname']
            item['smalllink'] = item2['smalllink']

            item['name'] = book.xpath('./em/text()').extract_first().strip()
            item['auth'] = book.xpath('./../../div[4]/span[1]/span/a/text()').extract_first()
            sku_id = book.xpath('./../../@data-sku').extract_first()
            url = 'https://p.3.cn/prices/mgets?skuIds=J_' + sku_id
            item['bookurl'] = 'https:' + book.xpath('./../../div[1]/a/@href').extract_first()
            yield scrapy.Request(url=url, callback=self.get_price, meta={'data': item})

贴一个scrapy使用debug模式运行的方法:

from scrapy.cmdline import execute
import os
import sys

#添加当前项目的绝对地址
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
#执行 scrapy 内置的函数方法execute，  使用 crawl 爬取并调试，最后一个参数是爬虫文件名
execute(['scrapy', 'crawl', 'myjd'])

右键debug运行这个文件就可以了 注意要放在爬虫的主目录下