异步保存使应用程序无需等待存储操作完成即可执行后续命令,进而提高执行效率。scrapy通过twisted框架实现,twisted为事件驱动的异步框架。
定义数据结构(items.py)
#定义了两个用于数据库存储的数据结构
import scrapy
class WebItem(scrapy.Item):
# define the fields for your item here like:
# name = scrapy.Field()
fl_id = scrapy.Field()
fl_title = scrapy.Field()
fl_content = scrapy.Field()
fl_region = scrapy.Field()
class AppendixItem(scrapy.Item):
ap_flid = scrapy.Field()
ap_title = scrapy.Field()
ap_savename = scrapy.Field()
编写用于数据库存储的pipline(piplines.py)
目标:
- 实现数据库异步保存
- 自定义文件路径
- 使用spider中确定的文件名保存
import re
from twisted.enterprise import adbapi
from scrapy.pipelines.files import FilesPipeline
from .items import WebItem,AppendixItem
class DbPipeLine(object):
#数据库存储管道,具有异步存储和多类item处理能力
def __init__(self, db_config):
# 建立数据库连接池
self.dbpool = adbapi.ConnectionPool(
db_config['driver'],
host=db_config['host'],
port=db_config['port'],
user=db_config['user'],
password=db_config['password'],
db=db_config['db'],
charset=db_config['charset'] # 此处为utf8,不是utf-8,也没有逗号
)
@classmethod
def from_crawler(cls, crawler):
#重写,以获取settings模块相关信息,每次创建该pipeline时,就会调用此方法
# cls:代表自身类DbPipeLine
# crawler:为爬虫项目类,包含spider,settings等属性
db_config = crawler.settings['DB_CONFIG'] #DB_CONFIG为数据库登录信息
return cls(db_config) # 相当 于DbPipeLine(db_config )
def process_item(self, item, spider):
#固定函数名,用于处理spider传递的item类型数据
#runInteraction:可获取连接池中数据库的游标cursor,并将游标和数值传递给回调函数
#result:接收返回的操作结果
#通过if语句实现多类型item的不同操作
if isinstance(item,WebItem):
result = self.dbpool.runInteraction(self.insert_fl, item)
result.addErrback(self.print_erro)
elif isinstance(item,AppendixItem):
result = self.dbpool.runInteraction(self.insert_appendix, item)
result.addErrback(self.print_erro)
return item
def insert_fl(self, cursor, item):
#定义sql插入语句,并留坑
sql = "INSERT INTO fl(" \
"fl_id," \
"fl_title," \
"fl_content," \
"fl_region)" \
"VALUE(%s,%s,%s,%s)"
args = (
item['fl_id'],
item['fl_title'],
item['fl_content'],
item['fl_region'],
)
cursor.execute(sql, args)
def insert_appendix(self, cursor, item):
#定义sql插入语句,并留坑
sql = "INSERT INTO appendix(" \
"ap_id," \
"ap_flid," \
"ap_title," \
"ap_savename") \
"VALUE(null,%s,%s,%s)" #ap_id为自动增加,设null。
args = (
item['ap_flid'],
item['ap_title'],
item['ap_savename'],
)
cursor.execute(sql, args)
def print_erro(self, failure):
# 打印错误信息
print(failure)
def close_spider(self, spiders):
# 关闭链接池
self.dbpool.close()
设置数据库登陆信息,启用pipline(settings)
settings.py
解除Configure item pipelines的注释
添加数据库信息
ITEM_PIPELINES = {
'myProject.pipelines.MyFilesPipeline': 1,
'myProject.pipelines.DbPipeLine': 10, #优先级低于下载pipline
#'scrapy.pipelines.files.FilesPipeline':1
}
DB_CONFIG = {
'driver':'pymysql', #数据库驱动
'host':'localhost', #IP地址
'port':3306, #端口,默认为3306,实际项目建议更改
'user':'root', #通常为root,实际项目应更改
'password':'xxxxxxxxxx', #密码
'db':'userDataBase', #选定数据库名
'charset':'utf8' #数据库编码,此处不是utf-8。最后一项结尾没有逗号
}