# -*- coding: utf-8 -*-
import scrapy
from selenium import webdriver
from wangyiPeo.items import WangyipeoItem
class WangyiSpider(scrapy.Spider):
name = 'wangyi'
# allowed_domains = ['www.xxx.com']
start_urls = ['https://news.163.com']
model_urls = []
bro = webdriver.Chrome(executable_path=r'chromedriver.exe')
def parse(self, response):
#解析出5个板块对应的url
li_list = response.xpath('//*[@id="index2016_wrap"]/div[1]/div[2]/div[2]/div[2]/div[2]/div/ul/li')
model_index = [3,4,6,7,8]
for index in model_index:
#li依次表示的是5个板块对应的li标签
li = li_list[index]
#5个板块对应的url
model_url = li.xpath('./a/@href').extract_first()
self.model_urls.append(model_url)
#对每一个板块的url进行手动请求的发送
yield scrapy.Request(model_url,callback=self.parse_model)
def parse_model(self,response):#用作于解析每一个板块对应页面数据中的新闻标题和新闻详情页的url
#该方法中获取的response对象是没有包含动态加载出的新闻数据(是一个不满足需求的response)
div_list = response.xpath('/html/body/div/div[3]/div[4]/div[1]/div/div/ul/li/div/div')#1+5+n
for div in div_list:
title = div.xpath('./div/div[1]/h3/a/text()').extract_first()
detail_url = div.xpath('./a/@href').extract_first()
item = WangyipeoItem()
item['title'] = title
yield scrapy.Request(detail_url,callback=self.parse_new_detail,meta={'item':item})
def parse_new_detail(self,response):#解析新闻内容
item = response.meta['item']
content = response.xpath('//*[@id="endText"]//text()').extract()
content = ''.join(content)
item['content'] = content
yield item
#改方法只会在整个程序结束时执行一次
def closed(self,spider):
self.bro.quit()
下载中间件:
# -*- coding: utf-8 -*-
# Define here the models for your spider middleware
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
from scrapy import signals
from scrapy.http import HtmlResponse
from time import sleep
class WangyipeoDownloaderMiddleware(object):
#参数:
#reuqest:拦截到请求对应的响应对象
#response:拦截到所有的响应对象(1+5+n)
#spider:爬虫类实例化的对象,可以实现爬虫类和中间件类的数据交互
def process_response(self, request, response, spider):
#拦截到5个板块对应的响应对象,将其替换成5个符合需求的新的响应对象进行返回
#1.找出5个板块对应的5个不符合需求的响应对象
if request.url in spider.model_urls:
#就是满足需求的五个板块对应的响应对象
#url:响应对象对应的请求对象的url
#body:响应数据,可以由selenium中的page_source返回
bro = spider.bro
bro.get(request.url)
sleep(3)
page_text = bro.page_source #包含了动态加载的新闻数据
new_response = HtmlResponse(url=request.url,body=page_text,encoding='utf-8',request=request)
return new_response
else:
return response