python爬虫 mysql重复_用scrpay写爬虫，同样的代码-o保存成CSV时一切正常，但同时写入Mysql数据库时很多数据重复且不齐全...

用scrpay写爬虫用大众点评练手，同样的代码-o保存成CSV时一切正常，但同时写入Mysql数据库时很多数据重复且不齐全。

csv文件一切正常。

Mysql里面的数据库一塌糊涂。

同一段代码，完全没有头绪，请高手帮忙。

class MySQLStorePipeline(object):

"""docstring for MySQLstor"""

def __init__(self):

self.dbpool = adbapi.ConnectionPool('MySQLdb',

host = 'localhost',

db = 'dianping',

user = 'root',

passwd = 'root',

cursorclass = MySQLdb.cursors.DictCursor,

charset = 'utf8',

use_unicode = True

)

def process_item(self, item, spider):

#print spider

# run db query in thread pool

query = self.dbpool.runInteraction(self._conditional_insert, item)

query.addErrback(self.handle_error)

return item

def _conditional_insert(self, tx, item):

if item.get('user_id'):

tx.execute(\

"insert into testtable_gz (city, store_name, store_id, book, group_buy, branch, average_spend, style, store_area, store_addr, store_url, comment_url, store_phone, user_id, star, taste, environment, service, comment, comment_date, user_url)\

values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)",

(item['city'],

item['store_name'],

item['store_id'],

item['book'],

item['group_buy'],

item['branch'],

item['average_spend'],

item['style'],

item['store_area'],

item['store_addr'],

item['store_url'],

item['comment_url'],

item['store_phone'],

item['user_id'],

item['star'],

item['taste'],

item['environment'],

item['service'],

item['comment'],

item['comment_date'],

item['user_url']

))

def handle_error(self, e):

log.err(e)

这是Pinelines.py代码。