scrapy异步写入mysql_scrapy 异步写入mysql和 ItemLoader的基本使用和

scrapy 异步写入mysql和 ItemLoader的基本使用和

ItemLoader的基本使用

使用itemloader能够提高scrapy在解析数据时的便利性和提升代码的可维护性，itemloader和Field的使用可以更好的提升代码的可读性。

class QuanshuSpider(scrapy.Spider):

name = 'quanshu'

# allowed_domains = ['quanshu.92kaifa.com']

start_urls = ['http://quanshu.92kaifa.com/']

def parse(self, response, **kwargs):

article_list = response.xpath('//ul[@class="b-all-content cf"]/li')

for article in article_list:

# title = scrapy.Field()

# image_url = scrapy.Field()

# type = scrapy.Field()

# author = scrapy.Field()

# id = scrapy.Field()

item_loader = ItemLoader(item=BookItem(), response=response)

item_loader.add_value('title', article.xpath('a[@class="msgBorder"]/@title').get())

item_loader.add_value('image_url', article.xpath('a[@class="msgBorder"]/img/@src').get())

item_loader.add_value('type', article.xpath('a[3]/text()').get())

item_loader.add_value('id', article.xpath('a[@class="msgBorder"]/@href').get())

item_loader.add_value('author', article.xpath('a[4]/text()').get())

item = item_loader.load_item()

yield item

spider文件

import scrapy

import re

from scrapy.loader.processors import TakeFirst, MapCompose

class BookItem(scrapy.Item):

title = scrapy.Field(output_processor=TakeFirst())

image_url = scrapy.Field(output_processor=TakeFirst())

type = scrapy.Field(output_processor=TakeFirst())

author = scrapy.Field(output_processor=TakeFirst())

id = scrapy.Field(

input_processor=MapCompose(lambda x: re.findall('\d+', x)),

output_processor=TakeFirst()

)

item文件

scrapy异步写入mysql

from twisted.enterprise import adbapi

import MySQLdb

import MySQLdb.cursors

class BookPipeline:

def __init__(self, **kwargs):

self.db_pool: adbapi.ConnectionPool = kwargs.get('db_pool')

@classmethod

def from_settings(cls, settings):

# MYSQL_CONFIG = {

# 'host': 'localhost',

# 'user': 'root',

# 'passwd': 'qwe123',

# 'db': 'scrapy_test',

# 'charset': 'utf8',

# }

config = settings['MYSQL_CONFIG']

config['cursorclass'] = MySQLdb.cursors.DictCursor

config['use_unicode'] = True

db_pool = adbapi.ConnectionPool('MySQLdb', **config)

return cls(db_pool=db_pool)

def process_item(self, item, spider):

instance = self.db_pool.runInteraction(self.insert, item)

instance.addErrback(self.handle_err, item, spider)

return item

def insert(self, cursor, item):

sql = f"""

INSERT INTO books

(

`title`,

`image_url`,

`type`,

`author`,

`id`

) VALUES

(

"{item['title']}",

"{item['image_url']}",

"{item['type']}",

"{item['author']}",

"{item['id']}"

)

"""

cursor.execute(sql)

def handle_err(self, error, item, spider):

"""处理异常"""

print(error)

结合mysqldb和twisted一起可以让写入mysql数据库的操作异步化，最后在setting里面打开这个pipline即可。