scrapy框架爬取当当网商品

记录一次成功爬虫经验

创建爬虫项目和爬虫文件

爬虫文件的代码

import scrapy
from dangdang.items import DangdangItem
from scrapy.http import Request

class DdSpider(scrapy.Spider):
    name = 'dd'
    allowed_domains = ['dangdang.com']
    start_urls = ['http://search.dangdang.com/?key=%C1%AC%D2%C2%C8%B9&act=input&page_index=1']

    def parse(self, response):
        item = DangdangItem()
        item["title"] = response.xpath('//p[@class="name"]/a/@title').extract()
        item["link"] = response.xpath('//p[@class="name"]/a/@href').extract()
        item["comment"] = response.xpath('//p[@class="star"]/a/text()').extract()
        yield item
        for i in range(90, 92):
            url = 'http://search.dangdang.com/?key=%C1%AC%D2%C2%C8%B9&act=input&page_index=' + str(i)
            yield Request(url,callback=self.parse)

items.py

import scrapy


class DangdangItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    title = scrapy.Field()#标题
    link = scrapy.Field()#链接
    comment = scrapy.Field()#评论数

pipe;ones.py

import pymysql


class DangdangPipeline(object):
    def process_item(self, item, spider):
        conn = pymysql.connect(host="127.0.0.1",user="root",passwd="root",db="dd")
        for i in range(0, len(item["title"])):
            title = item["title"][i]
            link = item["link"][i]
            comment = item["comment"][i]
            # print(title + ":" + link + ":" + comment)
            sql = "insert into goods(title,link,comment) values('" + title + "','" + link + "','" + comment +"')"#sql语句
            print(sql)
            try:
                conn.query(sql)#执行sql语句
            except Exception as err:
                print(err)
        conn.close()
        return item