class MysqlPipeline(object):
def __init__(self, host, database, user, password, port):
self.host = host
self.database = database
self.user = user
self.password = password
self.port = port
@classmethod
def from_crawler(cls, crawler):
print('22222222222222222222')
return cls(
host=crawler.settings.get('MYSQL_HOST'),
user=crawler.settings.get('MYSQL_USER'),
password=crawler.settings.get('MYSQL_PASSWORD'),
database=crawler.settings.get('MYSQL_DATABASE'),
port=crawler.settings.get('MYSQL_PORT'),
)
def open_spider(self, spider):
print('333333333333')
self.db = pymysql.connect(self.host, self.user, self.password, self.database, port=self.port, charset='utf8')
self.cursor = self.db.cursor()
print('55555555555555')
def process_item(self, item, spider):
print('444444444444444444444444')
print('item = %s==================' % item)
data = dict(item)
keys = ', '.join(data.keys())
values = ', '.join(['%s'] * len(data))
sql = 'insert into jsda(%s) values(%s)' % (keys, values)
self.cursor.execute(sql, tuple(data.values()))
self.db.commit()
return item
def close_spider(self, spider):
print('66666666666666')
self.db.close()
scrapy存储到MySQL数据库
最新推荐文章于 2022-10-28 00:44:28 发布