异步保存数据到mysql或mssql 学习笔记

 

from twisted.enterprise import adbapi
class NewtqPipeline:
    #def process_item(self, item, spider):
        #print(item['city'],item['day'],item['gw'],item['dw'],item['tq'])#以上两行为直接打印出啦
    def __init__(self,mssql_config):
        self.adbpool = adbapi.ConnectionPool(
            mssql_config['DRIVER'],#易错
            server = mssql_config['SERVER'],
            user = mssql_config['USER'],
            password = mssql_config['PASSWORD'],
            database = mssql_config['DATABASE'],
            port = mssql_config['PORT'],
            charset = 'utf8'
        )
    @classmethod
    def from_crawler(cls,crawler):
        mssql_config = crawler.settings['MSSQL_CONFIG']
        return cls(mssql_config)#易错哦
    def process_item(self, item, spider):
        result = self.adbpool.runInteraction(self.insert_parse,item)
        result.addErrback(self.insert_error)
        return item

    def insert_parse(self,cursor,item):
        sql = "insert into tianqi(rq,tq,tem,dw,city) values(%s,%s,%s,%s,%s)"
        args = (item['day'], item['tq'], item['gw'], item['dw'], item['city'])
        cursor.execute(sql,args)

    def insert_error(self, failure):
        print("--" * 30)
        print(failure)
    def spider_close(self, spider):
        self.adbpool.close()