# -*- coding: utf-8 -*-
import scrapy
from selenium import webdriver
from urllib import parse
import time
import json
from scrapy import Request, FormRequest
from SumSpider.items import ZhihuUserItem
# start_url = "https://www.zhihu.com/api/v4/members/zhu-yun-bei-79?include=allow_message%2Cis_followed%2Cis_following%2Cis_org%2Cis_blocking%2Cemployments%2Canswer_count%2Cfollower_count%2Carticles_count%2Cgender%2Cbadge%5B%3F(type%3Dbest_answerer)%5D.topics"
# start_url = "https://www.zhihu.com/api/v4/members/excited-vczh/followees?include=data%5B*%5D.answer_count%2Carticles_count%2Cgender%2Cfollower_count%2Cis_followed%2Cis_following%2Cbadge%5B%3F(type%3Dbest_answerer)%5D.topics&offset=40&limit=20"
class ZhihuSpider(scrapy.Spider):
name = 'zhihu'
allowed_domains = ['www.zhihu.com']
user = "excited-vczh"
# 每个关注的人的数据url (json格式)
user_url = "https://www.zhihu.com/api/v4/members/{user}?include={include}"
user_query = "allow_message%2Cis_followed%2Cis_following%2Cis_org%2Cis_blocking%2Cemployments%2Canswer_count%2Cfollower_count%2Carticles_count%2Cgender%2Cbadge%5B%3F(type%3Dbest_answerer)%5D.topics"
# 每页关注的人的列表url
follows_url = "https://www.zhihu.com/api/v4/members/{user}/followees?include={include}&offset={offset}&limit={limit}"
follows_query = "data%5B*%5D.answer_count%2Carticles_count%2Cgender%2Cfollower_count%2Cis_followed%2Cis_following%2Cbadge%5B%3F(type%3Dbest_answerer)%5D.topics"
# 每页关注我的人的列表url
followers_url = "https://www.zhihu.com/api/v4/members/{user}/followers?include={include}&offset={offset}&limit={limit}"
followers_query = "data%5B*%5D.answer_count%2Carticles_count%2Cgender%2Cfollower_count%2Cis_followed%2Cis_following%2Cbadge%5B%3F(type%3Dbest_answerer)%5D.topics"
#入口
def start_requests(self):
yield Request(url=self.user_url.format(user=self.user, include=self.user_query), callback=self.parse_user)
yield Request(url=self.follows_url.format(user=self.user, include=self.follows_query, offset=0, limit=20), callback=self.parse_follows)
yield Request(url=self.followers_url.format(user=self.user, include=self.followers_query, offset=0, limit=20),callback=self.parse_followers)
# 解析每个人的数据
def parse_user(self, response):
item = ZhihuUserItem()
res_dict = json.loads(response.text)
for field in item.fields:
if field in res_dict.keys():
item[field] = res_dict.get(field)
yield item
yield Request(url=self.follows_url.format(user=res_dict.get("url_token"), include=self.follows_query, offset=0, limit=20), callback=self.parse_follows)
yield Request(url=self.followers_url.format(user=res_dict.get("url_token"), include=self.followers_query, offset=0, limit=20), callback=self.parse_followers)
# 关注的人的列表 (拿到url_token 和 翻页)
def parse_follows(self, response):
res_dict = json.loads(response.text)
if "data" in res_dict.keys():
for i in res_dict.get("data"):
if "url_token" in i.keys():
# print(i.get("url_token"))
yield Request(url=self.user_url.format(user=i.get("url_token"), include=self.user_query), callback=self.parse_user)
if "paging" in res_dict.keys() and res_dict.get("paging").get("is_end") is False:
next_page = res_dict.get("paging").get("next")
yield Request(url=next_page, callback=self.parse_follows)
# 关注我的人的列表 (拿到url_token 和 翻页)
def parse_followers(self, response):
res_dict = json.loads(response.text)
if "data" in res_dict.keys():
for i in res_dict.get("data"):
if "url_token" in i.keys():
# print(i.get("url_token"))
yield Request(url=self.user_url.format(user=i.get("url_token"), include=self.user_query), callback=self.parse_user)
if "paging" in res_dict.keys() and res_dict.get("paging").get("is_end") is False:
next_page = res_dict.get("paging").get("next")
yield Request(url=next_page, callback=self.parse_followers)
转载于:https://www.cnblogs.com/lilied/p/8352857.html