pyspider爬取豆瓣延禧攻略短评和回应

#!/usr/bin/env python
# -*- encoding: utf-8 -*-
# Created on 2018-08-30 15:08:10
# Project: yanxigonglve_douban

from pyspider.libs.base_handler import *
from lxml import etree

class Handler(BaseHandler):
    crawl_config = {
    }

    @every(minutes=24 * 60)
    def on_start(self):
        self.crawl('https://movie.douban.com/subject/26999852/discussion/', callback=self.index_page,validate_cert=False)

    @config(age=10 * 24 * 60 * 60)
    def index_page(self, response):
        # 利用xpath获取短评的链接，主要目的是为了得到详情页面的回应
        s = etree.HTML(response.text)
        href = s.xpath('//*[@id="posts-table"]/tr/td[1]/a/@href')
        print(href)
        for i in href:
            self.crawl(i, callback=self.detail_page,validate_cert=False)
        # 获取下一页的链接
        next = response.doc('.next > a').attr.href
        self.crawl(next, callback=self.index_page,validate_cert=False)
        
    @config(priority=2)
    def detail_page(self, response):
        return {
            "url": response.url,
            "title": response.doc('title').text(),
            'content': response.doc('.post .comment-item').text()
        }