使用scrapy框架写的当当网数据爬取,数据能爬下来,但是导入不进去数据库中
大神,又来打扰你了
1.数据库名称:
dd:数据库名
ddw 表名
2.问题报错
3.代码:
3.1 items文件代码:
import scrapy
class DangdangItem(scrapy.Item):
title=scrapy.Field()
link=scrapy.Field()
comment =scrapy.Field()
3.2 ddd文件代码:
import scrapy
from dangdang.items import DangdangItem
class DdSpider(scrapy.Spider):
name = 'ddd'
allowed_domains = ['dangdang.com']
start_urls = ['http://category.dangdang.com/pg1-cid4010275.html']
def parse(self, response):
item=DangdangItem()
item['title']=response.xpath('//p[@class="name"]/a/@title').extract()
item['link']=response.xpath('//p[@class="name"]/a/@href').extract()
item['comment']=response.xpath('//p[@class="star"]/a/text()').extract()
yield item
3.3pipeline文件代码
import pymysql
from dangdang.items import DangdangItem
class DangdangPipeline(object):
def process_item(self, item, spider):
conn=pymysql.connect(host="127.0.0.1",user="root",passwd="562390",db="dd")
cursor=conn.cursor()
for i in range(0,len(item["title"])):
title=item["title"][i]
link=item["link"][i]
comment=item["comment"][i]
sql="insert into ddw(title,link,comment)values('"+title+"','"+link+"','"+comment+"')"
conn.query(sql)
conn.close()
return item