Scrpay之Pipeline同步/异步方式保存数据库

Scrpay之Pipeline同步方式保存数据库

示例代码

class MysqlPipeline(object):

    #采用同步的机制写入mysql
    def __init__(self):
        self.conn = MySQLdb.connect('192.168.0.106', 'root', 'root', 'article_spider', charset="utf8", use_unicode=True)
        self.cursor = self.conn.cursor()

    def process_item(self, item, spider):
        insert_sql = """
            insert into jobbole_article(title, url, create_date, fav_nums)
            VALUES (%s, %s, %s, %s)
        """
        self.cursor.execute(insert_sql, (item["title"], item["url"], item["create_date"], item["fav_nums"]))
        self.conn.commit()
        
    def close_spider(self, spider):
	 self.cursor.close()
	 self.con.close()

Scrapy之Pipeline异步方式保存数据库

示例代码

settings.py：
MYSQL_HOST = "127.0.0.1"
MYSQL_DBNAME = "article_spider"
MYSQL_USER = "root"
MYSQL_PASSWORD = "root"

SQL_DATETIME_FORMAT = "%Y-%m-%d %H:%M:%S"
SQL_DATE_FORMAT = "%Y-%m-%d"


pipelines.py:
from twisted.enterprise import adbapi 
class MysqlTwistedPipline(object):

    def __init__(self, dbpool):
        self.dbpool = dbpool
        
    @classmethod
    def from_settings(cls, settings):
    	# 从settings中获取数据库配置
        dbparms = dict(
            host = settings["MYSQL_HOST"],
            db = settings["MYSQL_DBNAME"],
            user = settings["MYSQL_USER"],
            passwd = settings["MYSQL_PASSWORD"],
            charset='utf8',
            cursorclass=MySQLdb.cursors.DictCursor,
            use_unicode=True,
        )        
        # dict中的键名称需要和传入的库对应的方法的参数名完全相同        
        # ConnectionPool()的第一个参数是库名称，第二个参数是需要传入的参数
        dbpool = adbapi.ConnectionPool("MySQLdb", **dbparms)
        
        return cls(dbpool)

    def process_item(self, item, spider):
        #使用twisted将mysql插入变成异步执行
        query = self.dbpool.runInteraction(self.do_insert, item)
        query.addErrback(self.handle_error, item, spider) #处理异常

    def handle_error(self, failure, item, spider):
        #处理异步插入的异常
        print (failure)

    def do_insert(self, cursor, item):
        # 执行具体的插入
        # 将sql保存语句写在对应的item对象中
        # 这样写可以直接调用item.get_sql(),根据不同的item获得不同的sql语句
        insert_sql, params = item.get_insert_sql()
        cursor.execute(insert_sql, params)