我想将the tutorial spider的输出保存到MySQL,但是我很难理解为什么在项管道中出现以下错误:ImportError: No module named MySQLdb
Exception AttributeError: "'QuotePipeline' object has no attribute 'dbpool'"
使用Ubuntu16.04和Python3.5.2。在
如果有人能指出我在这方面出了什么问题,我们将不胜感激,非常感谢!在
以下是相关代码:
蜘蛛/引号.py
^{pr2}$
/项目.pyimport scrapy
class QuoteItem(scrapy.Item):
# define the fields for your item here like:
text = scrapy.Field()
author = scrapy.Field()
tags = scrapy.Field()
pass
/管道.pyfrom twisted.enterprise import adbapi
from scrapy.utils.project import get_project_settings
settings = get_project_settings()
class QuotePipeline(object):
# The table you items.QuoteItem class map to, my table is named quotes
insert_sql = """insert into quotes (%s) values ( %s )"""
def __init__(self):
dbargs = settings.get('DB_CONNECT')
db_server = settings.get('DB_SERVER')
dbpool = adbapi.ConnectionPool(db_server, **dbargs)
self.dbpool = dbpool
def __del__(self):
self.dbpool.close()
def process_item(self, item, spider):
self.insert_data(item, self.insert_sql)
return item
def insert_data(self, item, insert):
keys = item.keys()
fields = u','.join(keys)
qm = u','.join([u'%s'] * len(keys))
sql = insert % (fields, qm)
data = [item[k] for k in keys]
return self.dbpool.runOperation(sql, data)
/设置.pyBOT_NAME = 'tutorial'
SPIDER_MODULES = ['tutorial.spiders']
NEWSPIDER_MODULE = 'tutorial.spiders'
DB_SERVER = 'MySQLdb'
DB_CONNECT = {
'db': 'scrapy',
'user': 'username',
'passwd': 'password',
'host': 'ip.of.the.server',
'charset': 'utf8',
'use_unicode': True,
}
# Obey robots.txt rules
ROBOTSTXT_OBEY = True
# Configure item pipelines
# See http://scrapy.readthedocs.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
'tutorial.pipelines.QuotePipeline': 500,
}
SQL架构CREATE DATABASE `scrapy` /*!40100 DEFAULT CHARACTER SET utf8mb4 */
CREATE TABLE `quotes` (
`id` mediumint(6) NOT NULL AUTO_INCREMENT,
`text` text NOT NULL,
`author` varchar(255) NOT NULL,
`tags` varchar(255) NOT NULL,
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4