全item存mysql存在就跳过 不插入

-- coding: utf-8 --

Define your item pipelines here

Don’t forget to add your pipeline to the ITEM_PIPELINES setting

See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html

import pymysql

class XpcPipeline(object):
def init(self):
self.conn = None
self.cur = None

def open_spider(self, spider):
    # 数据库连接
    self.conn = pymysql.connect(
        host='127.0.0.1',
        port=3306,
        db='xinpianchang',
        user='root',
        password='123456',
        charset='utf8mb4',
    )
    # 游标
    self.cur = self.conn.cursor()

def close_spider(self, spider):
    # 关闭游标和连接
    self.cur.close()
    self.conn.close()

def process_item(self, item, spider):
    # 键
    cols = item.keys()
    # 值
    # print(item.values(), type(item.values()), '```````````````````````````````````````````````````')
    # 'name': '導演 Birdy Nio'}) <class 'collections.abc.ValuesView'>


    # print([item.values()], type([item.values()]), '======================================')
    # 'name': '導演 Birdy Nio'})] <class 'list'>

    # print(list(item.values()), type(list(item.values())), '++++++++++++++++++++++++')
    # ', 74395, '1008', 6, '台湾\xa0\xa0', '导演'] <class 'list'> ++++++++++++++++++++++++

    print(type(item),'1111111111111111111111111111111111111')

    values = list(item.values())
    sql = "insert into `{}` ({}) values ({}) on duplicate key update {}".format(
        item.table_name,
        ','.join(['`%s`' % col for col in cols]),
        ','.join(['%s'] * len(values)),
        ','.join(['`{}`=%s'.format(col) for col in cols])  # %s用下面的values*2填

    )
    self.cur.execute(sql, values * 2)
    self.conn.commit()
    return item