scrapy数据入库(MySQL)
采集的内容在管道中插入到MySQL中:
class BookPipeline:
def __init__(self):
# connection database
self.connect = pymysql.connect(host='127.0.0.1', user='root', passwd='123456',
db='books') # 后面三个依次是数据库连接名、数据库密码、数据库名称
# get cursor
self.cursor = self.connect.cursor()
print("连接数据库成功")
def process_item(self, item, spider):
# sql语句
insert_sql = "insert into book1(tit,content) VALUES (%s,%s)"
print(item)
# 执行插入数据到数据库操作
self.cursor.execute(insert_sql, (item['tit'],
item['content']))
# 提交,不进行提交无法保存到数据库
self.connect.commit()
def close_spider(self, spider):
# 关闭游标和连接
self.cursor.close()
self.connect.close()