# -*- coding: utf-8 -*-
import scrapy
from ..items import BookListItem,BookDetailItem
class YunqiSpider(scrapy.Spider):
name = 'yunqi'
allowed_domains = ['qq.com']
start_urls = ['http://yunqi.qq.com/bk/so2/n10p1']
def parse(self, response):
# 找到小说div
divs = response.xpath('//div[@class="book"]')
# for循环遍历小说
for novel in divs:
# 小说链接
novel_link = novel.xpath('a/@href').extract_first('')
img_src = novel.xpath('a/img/@src').extract_first('')
novel_id = novel.xpath('div/h3/a/@id').extract_first('')
novel_title = novel.xpath('div/h3/a/text()').extract_first('')
info = novel.xpath('div/dl/dd//text()').extract()
novel_auth = info[0]
novel_categray = info[1]
novel_status = info[2]
novel_date = info[3]
novel_numbers= info[4]
# 创建对象
item = BookListItem(novel_link=novel_link,img_src=[img_src],novel_id=novel_id,novel_title=novel_title,novel_auth=novel_auth,novel_categray=novel_categray,novel_status=novel_status,novel_date=novel_date,novel_numbers=novel_numbers)
yield item
# 进入详情页面
yield scrapy.Request(
url=novel_link,
callback=self.parse_detail,
meta={'novel_id':novel_id}
)
# 找到下一页s,继续爬取下一页
pages = response.xpath('//div[@id="pageHtml2"]/a')
next_link = pages[-1].xpath('@href').extract_first('')
if next_link:
yield scrapy.Request(
url=next_link
)
def parse_detail(self, response):
# 小说id
novel_id = response.meta.get('novel_id')
# 小说标签
tags = response.xpath('//div[@class="tags"]/text()').extract_first('').split(':')[-1].strip('\r\n ')
# 小说信息
infos = response.xpath('//div[@id="novelInfo"]/table/tr/td//text()').extract()
# 小说总点击
novelAllClick = infos[3].split(':')[-1]
# 月点击
novelMonClick = infos[6].split(':')[-1]
# 周点击
novelWeekClick = infos[9].split(':')[-1]
# 总人气
novelAllPopular = infos[4].split(':')[-1]
# 月人气
novelMonPopular = infos[7].split(':')[-1]
# 周人气
novelWeekPopular = infos[10].split(':')[-1]
# 总推荐
novelAllComm = infos[5].split(':')[-1]
# 月推荐
novelMonComm = infos[8].split(':')[-1]
# 周推荐
novelWeekComm = infos[11].split(':')[-1]
# 评论数
CommentNums = response.xpath('//span[@id="novelInfo_commentCount"]/text()').extract_first('')
# 创建item对象
item = BookDetailItem(novel_id=novel_id,tags=tags,novelAllClick=novelAllClick,novelMonClick=novelMonClick,novelWeekClick=novelWeekClick,novelAllPopular=novelAllPopular,novelMonPopular=novelMonPopular,novelWeekPopular=novelWeekPopular,novelAllComm=novelAllComm,novelMonComm=novelMonComm,novelWeekComm=novelWeekComm,CommentNums=CommentNums)
yield item
import scrapy
from ..items import BookListItem,BookDetailItem
class YunqiSpider(scrapy.Spider):
name = 'yunqi'
allowed_domains = ['qq.com']
start_urls = ['http://yunqi.qq.com/bk/so2/n10p1']
def parse(self, response):
# 找到小说div
divs = response.xpath('//div[@class="book"]')
# for循环遍历小说
for novel in divs:
# 小说链接
novel_link = novel.xpath('a/@href').extract_first('')
img_src = novel.xpath('a/img/@src').extract_first('')
novel_id = novel.xpath('div/h3/a/@id').extract_first('')
novel_title = novel.xpath('div/h3/a/text()').extract_first('')
info = novel.xpath('div/dl/dd//text()').extract()
novel_auth = info[0]
novel_categray = info[1]
novel_status = info[2]
novel_date = info[3]
novel_numbers= info[4]
# 创建对象
item = BookListItem(novel_link=novel_link,img_src=[img_src],novel_id=novel_id,novel_title=novel_title,novel_auth=novel_auth,novel_categray=novel_categray,novel_status=novel_status,novel_date=novel_date,novel_numbers=novel_numbers)
yield item
# 进入详情页面
yield scrapy.Request(
url=novel_link,
callback=self.parse_detail,
meta={'novel_id':novel_id}
)
# 找到下一页s,继续爬取下一页
pages = response.xpath('//div[@id="pageHtml2"]/a')
next_link = pages[-1].xpath('@href').extract_first('')
if next_link:
yield scrapy.Request(
url=next_link
)
def parse_detail(self, response):
# 小说id
novel_id = response.meta.get('novel_id')
# 小说标签
tags = response.xpath('//div[@class="tags"]/text()').extract_first('').split(':')[-1].strip('\r\n ')
# 小说信息
infos = response.xpath('//div[@id="novelInfo"]/table/tr/td//text()').extract()
# 小说总点击
novelAllClick = infos[3].split(':')[-1]
# 月点击
novelMonClick = infos[6].split(':')[-1]
# 周点击
novelWeekClick = infos[9].split(':')[-1]
# 总人气
novelAllPopular = infos[4].split(':')[-1]
# 月人气
novelMonPopular = infos[7].split(':')[-1]
# 周人气
novelWeekPopular = infos[10].split(':')[-1]
# 总推荐
novelAllComm = infos[5].split(':')[-1]
# 月推荐
novelMonComm = infos[8].split(':')[-1]
# 周推荐
novelWeekComm = infos[11].split(':')[-1]
# 评论数
CommentNums = response.xpath('//span[@id="novelInfo_commentCount"]/text()').extract_first('')
# 创建item对象
item = BookDetailItem(novel_id=novel_id,tags=tags,novelAllClick=novelAllClick,novelMonClick=novelMonClick,novelWeekClick=novelWeekClick,novelAllPopular=novelAllPopular,novelMonPopular=novelMonPopular,novelWeekPopular=novelWeekPopular,novelAllComm=novelAllComm,novelMonComm=novelMonComm,novelWeekComm=novelWeekComm,CommentNums=CommentNums)
yield item