=
=
学习练习起点排行榜的图书信息
起点rank官网
第一步要先点击各个分榜
第二步在每个榜单里点击十四个分栏
第三步才是点击每个图书的详情页进行爬取
以及翻页操作
一共9个分榜
14个分栏
5页每页二十个数据
最后存入csv的数据应为12600条数据
=
=
使用scrapy框架来写
先创建
scrapy startproject qidianPro
scrapy genspider qidian
先写items
# Define here the models for your scraped items
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/items.html
import scrapy
class QidianproItem(scrapy.Item):
# define the fields for your item here like:
# name = scrapy.Field()
# 书名
name = scrapy.Field()
# 作者
author = scrapy.Field()
# 简介
information = scrapy.Field()
然后编写spider文件
import scrapy
from qidianPro.items import QidianproItem
class QidianSpider(scrapy.Spider):
name = 'qidian'
# allowed_domains = ['www.qidian.com']
start_urls = ['https://www.qidian.com/rank']
# 起点排行榜
def parse(self, response):
li_list_one = response.xpath('/html/body/div[1]/div[5]/div[1]/ul[2]/li')
for li in li_list_one:
url_one = 'https:' + li.xpath('./a/@href').extract_first()
# print(url_one) # 第一次地址:分榜地址,共九个。
# print("\n") # success
yield scrapy.Request(
url_one,
callback=self.parse_one,
meta={'url': url_one}
)
# 起点排行榜分榜
def parse_one(self, response):
url = response.meta['url']
# print(url) #success
# print("\n")
# https://www.qidian.com/rank/yuepiao?style=1&chn=21&page=1
# //*[@id="-1"]/div[1]/div[5]/div[2]/div[1]/div/div/div[2]/p/a[2]
# //*[@id="-1"]/div[1]/div[5]/div[2]/div[1]/div/div/div/div[2]/p/a[1]
# //*[@id="-1"]/div[1]/div[5]/div[2]/div[1]/div/div/div/div[2]/p/a[2]
# 经过验证发现,有两个分榜的页面结构不同,xpath要有两个找寻条件,其中不一样的为月票榜和打赏榜,他两也不一样,所以要三个判断。
# 因为打赏榜没有次级分类,所以就进行抓取了
li_list_two = response.xpath('//*[@id="-1"]/div[1]/div[5]/div[2]/div[1]/div/div/div/div[2]/p/a|//*['
'@id="-1"]/div[1]/div[5]/div[2]/div[1]/div/div/div[2]/p/a')
# print(li_list_two)#success
for li in li_list_two:
num = li.xpath('./@data-chanid').extract_first()
if int(num) > 0:
url_two = url + "&chn={}&page=1".format(num)
# print(url_two) #第二次地址,为各个分榜下分类的地址,9分榜14分类共应124个
# print("\n") # success
yield scrapy.Request(
url=url_two,
callback=self.parse_two,
meta={
'url': url_two,
'page': 1
}
)
# 起点排行榜分榜的分栏
def parse_two(self, response):
# print(response) # success
url = response.meta['url']
page = response.meta['page']
li_list_three = response.xpath('//*[@id="rank-view-list"]/div/ul/li')
for li in li_list_three:
# 分栏每本图书的详情页
url_three = 'https:' + li.xpath('./div[1]/a/@href').extract_first()
# 抓取详情页
yield scrapy.Request(
url=url_three,
callback=self.parse_three
)
# 下一页抓取
if response.xpath('//*[@id="page-container"]/div/ul/li[7]/a/@class').extract_first() == 'lbf-pagination-next ':
# 下一页地址
page += 1
url_four = url[:-1] + page
yield scrapy.Request(
url=url_four,
callback=self.parse_two,
meta={
'url': url_four,
'page': page
}
)
# 图书的详情页
def parse_three(self, response):
item = QidianproItem()
# print(response) #success
item['name'] = response.xpath('/html/body/div/div[6]/div[1]/div[2]/h1/em/text()').extract_first()
item['author'] = response.xpath('/html/body/div/div[6]/div[1]/div[2]/h1/span/a/text()').extract_first()
item['information'] = response.xpath(
'/html/body/div/div[6]/div[4]/div[1]/div[1]/div[1]/p/text()').extract_first()
yield item
最后下载piplines文件
# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
# useful for handling different item types with a single interface
from itemadapter import ItemAdapter
import csv
class QidianproPipeline:
def open_spider(self, spider):
self.f = open('./qidian.csv', 'w', encoding='utf-8', newline="")
self.csvf = csv.DictWriter(self.f, fieldnames=['name', 'author', 'information'])
self.csvf.writeheader()
def process_item(self, item, spider):
self.csvf.writerows([dict(item)])
print(item)
# print(type(item))
return item
def close_spider(self, spider):
self.f.close()
生成csv文件