scrapy爬虫项目实例三（起点图书信息万条数据）

=

=

学习练习起点排行榜的图书信息
起点rank官网

第一步要先点击各个分榜
第二步在每个榜单里点击十四个分栏
第三步才是点击每个图书的详情页进行爬取
以及翻页操作
一共9个分榜
14个分栏
5页每页二十个数据
最后存入csv的数据应为12600条数据

=

=

使用scrapy框架来写
先创建

scrapy startproject qidianPro
scrapy genspider qidian

先写items

# Define here the models for your scraped items
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/items.html

import scrapy


class QidianproItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    # 书名
    name = scrapy.Field()
    # 作者
    author = scrapy.Field()
    # 简介
    information = scrapy.Field()

然后编写spider文件

import scrapy

from qidianPro.items import QidianproItem


class QidianSpider(scrapy.Spider):
    name = 'qidian'
    # allowed_domains = ['www.qidian.com']
    start_urls = ['https://www.qidian.com/rank']

    # 起点排行榜
    def parse(self, response):
        li_list_one = response.xpath('/html/body/div[1]/div[5]/div[1]/ul[2]/li')
        for li in li_list_one:
            url_one = 'https:' + li.xpath('./a/@href').extract_first()
            # print(url_one) # 第一次地址：分榜地址，共九个。
            # print("\n") # success
            yield scrapy.Request(
                url_one,
                callback=self.parse_one,
                meta={'url': url_one}
            )

    # 起点排行榜分榜
    def parse_one(self, response):
        url = response.meta['url']
        # print(url) #success
        # print("\n")
        # https://www.qidian.com/rank/yuepiao?style=1&chn=21&page=1
        # //*[@id="-1"]/div[1]/div[5]/div[2]/div[1]/div/div/div[2]/p/a[2]
        # //*[@id="-1"]/div[1]/div[5]/div[2]/div[1]/div/div/div/div[2]/p/a[1]
        # //*[@id="-1"]/div[1]/div[5]/div[2]/div[1]/div/div/div/div[2]/p/a[2]

        # 经过验证发现，有两个分榜的页面结构不同，xpath要有两个找寻条件，其中不一样的为月票榜和打赏榜，他两也不一样，所以要三个判断。
        # 因为打赏榜没有次级分类，所以就进行抓取了
        li_list_two = response.xpath('//*[@id="-1"]/div[1]/div[5]/div[2]/div[1]/div/div/div/div[2]/p/a|//*['
                                     '@id="-1"]/div[1]/div[5]/div[2]/div[1]/div/div/div[2]/p/a')
        # print(li_list_two)#success

        for li in li_list_two:
            num = li.xpath('./@data-chanid').extract_first()
            if int(num) > 0:
                url_two = url + "&chn={}&page=1".format(num)
                # print(url_two) #第二次地址，为各个分榜下分类的地址，9分榜14分类共应124个
                # print("\n") # success

                yield scrapy.Request(
                    url=url_two,
                    callback=self.parse_two,
                    meta={
                        'url': url_two,
                        'page': 1
                    }
                )

    # 起点排行榜分榜的分栏
    def parse_two(self, response):
        # print(response) # success

        url = response.meta['url']
        page = response.meta['page']
        li_list_three = response.xpath('//*[@id="rank-view-list"]/div/ul/li')
        for li in li_list_three:
            # 分栏每本图书的详情页
            url_three = 'https:' + li.xpath('./div[1]/a/@href').extract_first()

            # 抓取详情页
            yield scrapy.Request(
                url=url_three,
                callback=self.parse_three
            )

        #     下一页抓取
        if response.xpath('//*[@id="page-container"]/div/ul/li[7]/a/@class').extract_first() == 'lbf-pagination-next ':
            # 下一页地址
            page += 1
            url_four = url[:-1] + page
            yield scrapy.Request(
                url=url_four,
                callback=self.parse_two,
                meta={
                    'url': url_four,
                    'page': page
                }
            )

    # 图书的详情页
    def parse_three(self, response):
        item = QidianproItem()
        # print(response) #success

        item['name'] = response.xpath('/html/body/div/div[6]/div[1]/div[2]/h1/em/text()').extract_first()

        item['author'] = response.xpath('/html/body/div/div[6]/div[1]/div[2]/h1/span/a/text()').extract_first()

        item['information'] = response.xpath(
            '/html/body/div/div[6]/div[4]/div[1]/div[1]/div[1]/p/text()').extract_first()

        yield item

最后下载piplines文件

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html


# useful for handling different item types with a single interface
from itemadapter import ItemAdapter
import csv

class QidianproPipeline:
    def open_spider(self, spider):
        self.f = open('./qidian.csv', 'w', encoding='utf-8', newline="")
        self.csvf = csv.DictWriter(self.f, fieldnames=['name', 'author', 'information'])
        self.csvf.writeheader()

    def process_item(self, item, spider):
        self.csvf.writerows([dict(item)])
        print(item)
        # print(type(item))
        return item

    def close_spider(self, spider):
        self.f.close()

生成csv文件