python，scrapy爬虫sql之爬取数据存储到mysql的piplelines.py配置

# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html


import MySQLdb

DBKWARGS={'db':'test','user':'root', 'passwd':'',
    'host':'localhost','use_unicode':True, 'charset':'utf8'}

class TutorialPipeline(object):

    def __init__(self):
        try:
            self.con = MySQLdb.connect(**DBKWARGS)
        except Exception,e:
            print "Connect db error:",e
        
    def process_item(self, item, spider):
        cur = self.con.cursor()
        sql = "insert into dmoz_book values(%s,%s,%s)"   ###数据库名
        lis = (''.join(item["title"]),''.join(item["link"]), ''.join(item["desc"]))   ###提取内容标题、链接、描述
        try:
            cur.execute(sql,lis)
        except Exception,e:
            print "Insert error:",e
            self.con.rollback()
        else:
            self.con.commit()
        cur.close()
        return item

    def __del__(self):
        try:
            self.con.close()
        except Exception,e:
            print "Close db error",e

当然也可以简写：

import MySQLdb

#这段代码可以写在settings.py文件中
# database connection parameters
#DBKWARGS={'db':'ippool','user':'root', 'passwd':'toor','host':'localhost','use_unicode':True, 'charset':'utf8'}

class CollectipsPipeline(object):

    def process_item(self, item, spider):

        DBKWARGS = spider.settings.get('DBKWARGS')
        con = MySQLdb.connect(**DBKWARGS)
        cur = con.cursor()
        sql = ("insert into proxy(IP,PORT,TYPE,POSITION,SPEED,LAST_CHECK_TIME) "
            "values(%s,%s,%s,%s,%s,%s)")
        lis = (item['IP'],item['PORT'],item['TYPE'],item['POSITION'],item['SPEED'],
            item['LAST_CHECK_TIME'])
        try:
            cur.execute(sql,lis)
        except Exception,e:
            print "Insert error:",e
            con.rollback()
        else:
            con.commit()
        cur.close()
        con.close()
        return item