scrapy MysqlPipeline 同步和异步

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53

import MySQLdb import MySQLdb.cursors     class  MysqlPipeline(object):      #采用同步的机制写入mysql      def __init__(self):          self.conn = MySQLdb.connect( '192.168.0.106' ,  'root' ,  'root' ,  'article_spider' , charset= "utf8" , use_unicode=True)          self.cursor = self.conn.cursor()        def process_item(self, item, spider):          insert_sql =  "" "              insert into jobbole_article(title, url, create_date, fav_nums)              VALUES (%s, %s, %s, %s)          "" "          self.cursor.execute(insert_sql, (item[ "title" ], item[ "url" ], item[ "create_date" ], item[ "fav_nums" ]))          self.conn.commit()     class  MysqlTwistedPipline(object):      def __init__(self, dbpool):          self.dbpool = dbpool        @classmethod      def from_settings(cls, settings):          dbparms = dict(              host = settings[ "MYSQL_HOST" ],              db = settings[ "MYSQL_DBNAME" ],              user = settings[ "MYSQL_USER" ],              passwd = settings[ "MYSQL_PASSWORD" ],              charset= 'utf8' ,              cursorclass=MySQLdb.cursors.DictCursor,              use_unicode=True,          )          dbpool = adbapi.ConnectionPool( "MySQLdb" , **dbparms)            return  cls(dbpool)        def process_item(self, item, spider):          #使用twisted将mysql插入变成异步执行          query = self.dbpool.runInteraction(self.do_insert, item)          query.addErrback(self.handle_error, item, spider) #处理异常        def handle_error(self, failure, item, spider):          # 处理异步插入的异常          print (failure)        def do_insert(self, cursor, item):          #执行具体的插入          #根据不同的item 构建不同的sql语句并插入到mysql中          insert_sql, params = item.get_insert_sql()          print (insert_sql, params)          cursor.execute(insert_sql, params)

      本文转自ning1022 51CTO博客，原文链接：http://blog.51cto.com/ning1022/1925593 ，如需转载请自行联系原作者