【学习笔记】scrapy示例

爬取视频和评论

# -*- coding: utf-8 -*-
import scrapy
import re
from scrapy import Request
import json


def strip(s):
    if s:
        return s.strip()
    return ''


class DiscoverySpider(scrapy.Spider):
    name = 'discovery'
    allowed_domains = ['xinpianchang.com', 'openapi-vtom.vmovier.com']
    start_urls = ['https://www.xinpianchang.com/channel/index/type-/sort-like/duration_type-0/resolution_type-/page-1']

    def parse(self, response):
        pid_list = response.xpath('//ul[@class="video-list"]/li/@data-articleid').extract()
        url = 'https://www.xinpianchang.com/a%s?from=ArticleList'
        for pid in pid_list:
            request = response.follow(url % pid, self.parse_post)
            request.meta['pid'] = pid
            yield request

    def parse_post(self, response):
        pid = response.meta['pid']
        post = dict()
        post['pid'] = pid
        post['title'] = response.xpath('//div[@class="title-wrap"]/h3/text()').get()
        cates = [c.strip() for c in response.xpath('//span[contains(@class,"cate")]//text()').extract()]
        post['category'] = [cate for cate in cates if cate != '-' and cate != '']
        post['update_time'] = response.xpath('//span[contains(@class,"update-time")]/i//text()').get()
        post['play_counts'] = response.xpath('//i[contains(@class,"play-counts")]/@data-curplaycounts').get()
        post['like_counts'] = response.xpath('//span[contains(@class,"like-counts")]/@data-counts').get()
        post['description'] = strip(response.xpath('//p[contains(@class,"desc")]/text()').get())
        vid, = re.findall('var vid = \"(\w+)\";', response.text)
        video_url = 'https://openapi-vtom.vmovier.com/v3/video/%s?expand=resource&usage=xpc_web&appKey=61a2f329348b3bf77'
        request = Request(video_url % vid, callback=self.parse_video)
        request.meta['post'] = post
        yield request

        comment_url = 'https://app.xinpianchang.com/comments?resource_id=%s&type=article&page=1&per_page=24'
        request = Request(comment_url % pid, callback=self.parse_comment)
        request.meta['pid'] = pid
        yield request

    def parse_video(self, response):
        post = response.meta['post']
        result = json.loads(response.text)
        post['video'] = result['data']['resource']['default']['url']
        post['cover'] = result['data']['video']['cover']
        yield post

    def parse_comment(self, response):
        result = json.loads(response.text)
        li = result['data']['list']
        for item in li:
            comment = dict()
            comment['uname'] = item['userInfo']['username']
            comment['content'] = item['content']
            yield comment

        next_page = result['data']['next_page_url']
        if next_page:
            next_page_url = 'https://app.xinpianchang.com/%s' % next_page
            yield response.follow(next_page_url, self.parse_comment)