pipelines mysql_scrapy基础知识之将item 通过pipeline保存数据到mysql mongoDB：

pipelines.py

class xxPipeline(object):

def process_item(self, item, spider):

con=pymysql.connect(host='localhost,user='',passwd='',db='',charset='utf8')

cur=con.cursor()

sql=("insert into 表名(字段)”"values(%s)")

lis=(item['字段'])

cur.execute(sql,lis)

con.commit()

cur.close()

con.close()

return item

在settings.py中配置

ITEM_PIPELINES = {

’MySpider.pipelines.xxPipeline': 300,

}

需要在mysql中创建数据库以及数据表字段

pipelines.py

import pymongo

class MyspiderPipeline(object):

def __init__(self):

# pymongo.MongoClient(host, port) 创建MongoDB链接

client = pymongo.MongoClient(host="localhost",port=27017)

# 指向指定的数据库

db_name = client["dbname"]

table_name=db_name["tablename"]

# 获取数据库里存放数据的表名

self.post = table_namedef process_item(self, item, spider):

data = dict(item)

# 向指定的表里添加数据

self.post.insert(data)

return item

settings.py配置

DEFAULT_REQUEST_HEADERS = {

'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:53.0) Gecko/20100101 Firefox/53.0'}