class PostgresqlPipeline(object):
def __init__(self):
self.connection = psycopg2.connect(host='localhost', user='openpg', password='openpgpwd', database='postgres',
port='5432')
self.connection.autocommit = True
def open_spider(self, spider):
self.cursor = self.connection.cursor()
sql_create = 'create table if not exists jsda(id serial primary key,sequence_number varchar,drug_name varchar,enterprise varchar,batch_number varchar,specification varchar,source varchar,standard varchar,result varchar,defective_item varchar,certification_body varchar,url varchar,file_urls varchar,files varchar,placeholder1 varchar,' \
'placeholder2 varchar,placeholder3 varchar,placeholder4 varchar,placeholder5 varchar,placeholder6 varchar,placeholder7 varchar,placeholder8 varchar,placeholder9 varchar,placeholder10 varchar,placeholder11 varchar,placeholder12 varchar,placeholder13 varchar,placeholder14 varchar,placeholder15 varchar,placeholder16 varchar,placeholder17 varchar,placeholder18 varchar)'
self.cursor.execute(sql_create)
def close_spider(self, spider):
self.cursor.close()
self.connection.close()
def process_item(self, item, spider):
# print('========================1111')
data = dict(item)
# print('data = %s' % item)
keys = ','.join(data.keys())
values = ','.join(['%s'] * len(data))
sql = 'insert into jsda(%s)values(%s)' % (keys, values)
self.cursor.execute(sql, tuple(data.values()))
return item
爬取存储到postgresql数据库
最新推荐文章于 2022-10-27 17:12:48 发布