Scrapy框架在写入MongoDB数据库时出现KeyError: 'xxx does not support field: _id'解决方法

最新推荐文章于 2022-08-09 21:00:00 发布

Last_xuan1

最新推荐文章于 2022-08-09 21:00:00 发布

阅读量1.8k

点赞数 1

分类专栏： # 基础知识&错误文章标签： mongodb scrapy 解决方法 KeyError

本文链接：https://blog.csdn.net/qq_43391383/article/details/87094991

版权

基础知识&错误专栏收录该内容

7 篇文章 1 订阅

订阅专栏

在items.py里面加上

_id = scrapy.Field()

在主体里面提取信息时，尽量避免_id产生重复，可以配合random库一起使用

这里是上一篇的一个例子

# -*- coding: utf-8 -*-
import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from scrapy.selector import Selector
from hujiangwangxiao.items import HujiangwangxiaoItem
import random

class HujiangSpider(CrawlSpider):
    name = 'hujiang'
    allowed_domains = ['hujiang.com']
    start_urls = ['https://class.hujiang.com/?utm_campaign=%E6%B2%AA%E6%B1%9F%E7%BD%91%E6%A0%A1%E5%93%81%E4%B8%93&utm_adgroup=%E6%A0%87%E9%A2%98&utm_source=baidu_pz&utm_medium=cpc&sem_source=sem_baidu_pz&utm_adkeyword=%E6%B2%AA%E6%B1%9F%E7%BD%91%E6%A0%A1']

    rules = (
        Rule(LinkExtractor(restrict_xpaths='//div[@class="cate-list"]//a',unique=True), callback='parse_item', follow=False),
    )

    def parse_item(self, response):
        item = HujiangwangxiaoItem()
        selector = Selector(response)
        course_text = selector.xpath('//div[@class="course-txt-box"]')
        for content in course_text:
            title = content.xpath('div[1]/a[1]/text()').extract_first().strip()
            good_info = content.xpath('a[1]/span/text()').extract_first()
            summary = content.xpath('a[1]/text()').extract_first().strip()
            now_price = content.xpath('p/span[1]/span[2]/text()').extract_first()
            orgin_price = content.xpath('p/span[2]/text()').extract_first()
            if orgin_price:
                orgin_price = orgin_price.strip()
            else :
                pass
            end_time = content.xpath('div[2]/span[1]/text()').extract_first().split('总')[0]
            coures_nums = content.xpath('div[2]/span[1]/text()').extract_first().split('总')[1]
            teachers = content.xpath('div[2]/span[2]/text()').extract_first()

            item['_id'] = title + str(random.randint(1,1000))
            item['title'] = title
            item['good_info'] = good_info
            item['summary'] = summary
            item['now_price'] = now_price
            item['orgin_price'] = orgin_price
            item['end_time'] = end_time
            item['coures_nums'] = '总' + coures_nums
            item['teachers'] = teachers
            yield item