python 通过异步存储到数据库 mysql

通过异步存储到数据库

from twisted.enterprise import adbapi

class MysqlTwistedPipline(object):

def __init__(self, dbpool):
    self.dbpool = dbpool
@classmethod
def from_settings(cls, settings):
    data_info = dict(
        password=settings['MYSQL_PASSWORD'],
        user=settings['MYSQL_USER'],
        database=settings['MYSQL_DATABASE'],
        host=settings['MYSQL_HOST'],
        charset='utf8',
        cursorclass=pymysql.cursors.DictCursor,
        use_unicode=True,
    )
    dbpool = adbapi.ConnectionPool('pymysql', **data_info)
    return cls(dbpool)

def process_item(self, item, spider):
    # 使用twisted将mysql变成异步插入
    query = self.dbpool.runInteraction(self.do_inster, item)
    query.addErrback(self.handle_err)  # 处理异常

def handle_err(self, failure):
    # 处理异步插入的异常
    print(failure)

def do_inster(self, cursor, item):
    insert_sql = '''
            insert into bole(title,date,url,md5_url)VALUES
            (%s,%s,%s,%s)
            '''
    cursor.execute(insert_sql, (item['title'], item['date'],
                                item['url'], item['md5_url']))