#错误示例:
try:
insert_sql = 'insert into mydb(url, title_)'
insert_value = (item['detail_urls'], item['title'])
self.cursor.execute(insert_sql, insert_value)
self.mydb.commit()
except Exception as e:
print(e)
self.mydb.rollback()
return item
#正确示例:
try:
insert_sql = 'insert into mydb(url, title_) values(%s,%s)'
insert_value = (item['detail_urls'], item['title'])
self.cursor.execute(insert_sql, insert_value)
self.mydb.commit()
except Exception as e:
print(e)
self.mydb.rollback()
return item
要注意在mysql语句插入时候的语法,最好分开写,不要写成一条
insert_sql = 'insert into mydb(url, title_) values(%s,%s)'
#最后奉上完整版的基于python中scrapy进行管道存储到mysql中的关键代码:
class MysqlPipeline:
def open_spider(self, spider):
print('开始爬虫...')
self.mydb = mysql.connector.connect(
host='localhost',
user='root',
passwd='root',
port=3306,
charset='utf8',
database='python'
)
def process_item(self, item, spider):
self.cursor = self.mydb.cursor()
try:
insert_sql = 'insert into mydb(url, title_) values (%s,%s)'
insert_value = (item['detail_urls'], item['title'])
self.cursor.execute(insert_sql, insert_value)
self.mydb.commit()
except Exception as e:
print(e)
self.mydb.rollback()
return item
def close_spider(self, spider):
self.cursor.close()
self.mydb.close()
print('爬虫完成...')