直接上代码
- spiders
class ExampleSpider(Spider):
name = 'qiushi'
def __init__(self):
self.lit = []
def start_requests(self):
url_str = "https://www.qiushibaike.com/text/page/3/"
yield Request(url=url_str,callback=self.parse,meta={"page":"0"},dont_filter=True)
def parse(self, response):
a = re.search("page",response.url)
url_strs =response.url
self.lit.append(url_strs)
content_block = response.xpath('//div[@class="article block untagged mb15 typs_hot"]')
for i in content_block:
url = i.xpath('//a[@class="contentHerf"]/@href').extract()
for j in url:
url_str = urljoin('https://www.qiushibaike.com',j)
yield Request(url=url_str,callback=self.parse_detail,meta={"page":"0"})
if a:
yield Request(url=self.lit[0],callback=self.parse,meta={'page':"2"},dont_filter=True)
self.lit = []
def parse_detail(self,response):
author = response.xpath('//div[@class="author clearfix"]/a[2]/h2/text()').extract()
content = response.xpath('//*[@id="single-next-link"]/div/text()').extract()
haoxiao_num = response.xpath('//div[@class="stats"]/span[@class="stats-vote"]/i/text()').extract()
pinglun_num = response.xpath('//div[@class="stats"]/span[@class="stats-comments"]/i/text()').extract()
item = QiushibaikeItem()
item["author"] = "".join(author)
item["content"] = "".join(content)
item["haoxiao_num"] = "".join(haoxiao_num)
item["pinglun_num"] = "".join(pinglun_num)
yield item
- items:
import scrapy
class QiushibaikeItem(scrapy.Item):
author = scrapy.Field()
content = scrapy.Field()
haoxiao_num = scrapy.Field()
pinglun_num = scrapy.Field()
- 中间件
from selenium import webdriver
from selenium.webdriver.firefox.options import Options as FOptions
import time
from scrapy.http import HtmlResponse
import re
class SeleniumMiddlewares(object):
def __init__(self):
self.options = FOptions()
#self.options.add_argument("-headless")
self.browser = webdriver.Firefox(executable_path="/home/hello/Downloads/geckodriver",firefox_options=self.options)
def process_request(self,request,spider):
if int(request.meta["page"]) >0:
self.browser.get(request.url)
self.browser.execute_script("window.scrollTo(0,document.body.scrollHeight)")
time.sleep(1)
button = self.browser.find_element_by_xpath('//span[@class="next"]')
button.click()
time.sleep(2)
else:
self.browser.get(request.url)
time.sleep(2)
return HtmlResponse(url=self.browser.current_url, body=self.browser.page_source, encoding="utf-8",
request=request)
- pipelines
import sqlite3
class QiushibaikePipeline(object):
def __init__(self):
self.conn = sqlite3.connect("qiushibaike.db")
self.cursor = self.conn.cursor()
self.cursor.execute("create table IF NOT EXISTS qiushi(author varchar(200),content varchar(500),haoxiao_num varchar(100),pinglun_num varchar(100))")
def process_item(self, item, spider):
self.cursor.execute("insert into qiushi values('%s','%s','%s','%s')"%(item["author"],item["content"],item["haoxiao_num"],item["pinglun_num"]))
self.conn.commit()
return item