# Define here the models for your scraped items## See documentation in:# https://docs.scrapy.org/en/latest/topics/items.htmlimport scrapy
classQidiantopItem(scrapy.Item):# define the fields for your item here like:# name = scrapy.Field()
bid = scrapy.Field()
name = scrapy.Field()
author = scrapy.Field()type= scrapy.Field()
status = scrapy.Field()
new_chapter = scrapy.Field()
new_chapter_date = scrapy.Field()
detail_url = scrapy.Field()
img_url = scrapy.Field()
info = scrapy.Field()
author_url = scrapy.Field()pass
pipelines
# Define your item pipelines here## Don't forget to add your pipeline to the ITEM_PIPELINES setting# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html# useful for handling different item types with a single interfacefrom itemadapter import ItemAdapter
from pymongo import MongoClient
from qiDianTop.items import QidiantopItem
classQidiantopPipeline:# spider对象创建时 调用defopen_spider(self, spider):print("创建mongo连接")
mongoClient = MongoClient(host="xxxxx", port=123)# 验证账号密码
mongoClient.test.authenticate("xx","xx")# 选择账号权限对应的数据库
self.mongo_data_base = mongoClient["test"]# 因为是测试 所以每次创建连接时 参数特定集合的数据
self.mongo_data_base.drop_collection("qiDian")pass# spide对象销毁时 抵用defclose_spider(self, spider):passdefprocess_item(self, item, spider):ifisinstance(item, QidiantopItem):
self.mongo_data_base["qiDian"].insert_one(dict(item))return item