爬取数据时,每次爬取出来的值都是一样的,先贴上代码
import scrapy
from jdbook.items import JdbookItem
import json
class MyjdSpider(scrapy.Spider):
name = 'myjd'
allowed_domains = ['jd.com', 'p.3.cn']
start_urls = ['https://book.jd.com/booksort.html']
def parse(self, response):
# 1.先判断页面中的元素是否在骨骼文件中
# 在骨骼文件中后进行遍历
big_list = response.xpath('//*[@id="booksort"]/div[2]/dl/dt/a')
for group in big_list[0:1]:
# 获取大标签的名字和标题
big_name = group.xpath('./text()').extract_first()
big_link = 'https:' + group.xpath('./@href').extract_first()
# 获取所有的小标签 小标签是兄弟节点
small_list = group.xpath('../following-sibling::dd[1]/em')
for small_group in small_list[0:1]:
my_item = JdbookItem()
my_item['bigname'] = big_name
my_item['biglink'] = big_link
my_item['smallname'] = small_group.xpath('./a/text()').extract_first()
my_item['smalllink'] = 'https:' + small_group.xpath('./a/@href').extract_first()
yield scrapy.Request(url=my_item['smalllink'], callback=self.book_message, meta={'data': my_item})
def book_message(self, response):
# 获取图书列表信息
book_list = response.xpath('//*[@id="plist"]/ul/li/div/div[3]/a')
item = response.meta['data']
for book in book_list[0:10]:
# item = JdbookItem()
# 数据传递
item['name'] = book.xpath('./em/text()').extract_first().strip()
item['auth'] = book.xpath('./../../div[4]/span[1]/span/a/text()').extract_first()
sku_id = book.xpath('./../../@data-sku').extract_first()
url = 'https://p.3.cn/prices/mgets?skuIds=J_' + sku_id
item['bookurl'] = 'https:' + book.xpath('./../../div[1]/a/@href').extract_first()
yield scrapy.Request(url=url, callback=self.get_price, meta={'data': item})
def get_price(self, response):
item = response.meta['data']
item['price'] = json.loads(response.body.decode())[0]['op']
print(item)
此时出现的问题,是每次爬取出的图书的名字和作者,都是该页面最后一个图书的信息,分析了很久,终于找到了原因:
在parse函数中生成JdbookItem
的实例对象时,每一个小的二级分类才能生成出一个对象,那么将这个数据传递到book_message
函数时,在赋值时会导致重复赋值,最后的结果就是被最后一个图书的信息覆盖掉
解决方案:book_message
函数重新创建一个JdbookItem
实例对象,然后将之前的信息赋值给这个对象,就可以解决
def book_message(self, response):
# 获取图书列表信息
book_list = response.xpath('//*[@id="plist"]/ul/li/div/div[3]/a')
item2 = response.meta['data']
for book in book_list[0:4]:
item = JdbookItem()
# 数据传递
item['bigname'] = item2['bigname']
item['biglink'] = item2['biglink']
item['smallname'] = item2['smallname']
item['smalllink'] = item2['smalllink']
item['name'] = book.xpath('./em/text()').extract_first().strip()
item['auth'] = book.xpath('./../../div[4]/span[1]/span/a/text()').extract_first()
sku_id = book.xpath('./../../@data-sku').extract_first()
url = 'https://p.3.cn/prices/mgets?skuIds=J_' + sku_id
item['bookurl'] = 'https:' + book.xpath('./../../div[1]/a/@href').extract_first()
yield scrapy.Request(url=url, callback=self.get_price, meta={'data': item})
贴一个scrapy
使用debug
模式运行的方法:
from scrapy.cmdline import execute
import os
import sys
#添加当前项目的绝对地址
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
#执行 scrapy 内置的函数方法execute, 使用 crawl 爬取并调试,最后一个参数是爬虫文件名
execute(['scrapy', 'crawl', 'myjd'])
右键debug运行这个文件就可以了 注意要放在爬虫的主目录下