zhihu reptile

# -*- coding: utf-8 -*-
import json

import scrapy
from scrapy import Request

from testzh.items import TestzhItem


class ZhihuSpider(scrapy.Spider):
    name = 'zhihu'
    allowed_domains = ['zhihu.com']
    start_urls = [
        'https://www.zhihu.com/api/v4/members/excited-vczh/followers?include=data[*].answer_count,articles_count,gender,follower_count,is_followed,is_following,badge[?(type=best_answerer)].topics&offset=40&limit=20']

    user_url = 'https://www.zhihu.com/api/v4/members/{user}?include=allow_message,is_followed,is_following,is_org,is_blocking,employments,answer_count,follower_count,articles_count,gender,badge[?(type=best_answerer)].topics'

    def parse(self, response):
        results = json.loads(response.text)
        if 'data' in results.keys():
            for i in results.get('data'):
                yield Request(self.user_url.format(user=i.get('url_token')), self.parse_follow)
        if 'paging' in results.keys() and results.get('paging').get("is_end") == False:
            next_page = results.get('paging').get("next")
            yield Request(next_page, self.parse)

    def parse_follow(self, response):
        item = TestzhItem()
        results = json.loads(response.text)
        for i in item.fields:
            if i in results.keys():
                item[i] = results.get(i)
        yield item