知乎中的关注列表是动态生成的,每次下拉刷新出20个,这样的话直接得到的html页面中并没有包含全部的关注人。
这里就要用到动态抓取的技术,用requests中的session模块模拟浏览器向对应的链接以post的形式发送数据。
import http.cookiejar
import requests
import re
import json
import math
from bs4 import BeautifulSoup
agent="Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) " \
"Chrome/46.0.2490.76 Mobile Safari/537.36"
headers={
'User-Agent':agent
}
session=requests.session()
session.cookies=http.cookiejar.LWPCookieJar(filename="cookies")
session.cookies.load(ignore_discard=True)
res=session.get('https://www.zhihu.com/people/zui-jiu-qing-niu-4/followees',headers=headers)
bs=BeautifulSoup(res.text,'html.parser')
fonum=bs.find("div",{'class':'zu-main-sidebar'}).find('a',{'class':'item'}).find('strong').text
fonum=int(fonum)
print(fonum)
def get_xsrf():
'''_xsrf 是一个动态变化的参数'''
index_url = 'http://www.zhihu.com'
# 获取登录时需要用到的_xsrf
index_page = session.get(index_url, headers=headers)
html = index_page.text
pattern = r'name="_xsrf" value="(.*?)"'
# 这里的_xsrf 返回的是一个list
_xsrf = re.findall(pattern, html)
return _xsrf[0]
begin=0
end=math.ceil(fonum/20)
num=1
for x in range(0,end):
beginnum = str(x * 20)
#print(x)
postdata={'method':'next',
'params':'{"offset":'+beginnum+',"order_by":"created","hash_id":"29d75b4013b4631aaf7fe5848f3f6113"}',
'_xsrf':str(get_xsrf())}
ress=session.post('https://www.zhihu.com/node/ProfileFolloweesListV2',data=postdata,headers=headers)
jsons=json.loads(ress.content.decode('utf-8'))
for a in jsons['msg']:
#print(a)
abs=BeautifulSoup(a,'html.parser')
print(num)
print("用户:"+abs.find('a',{'class':'zg-link author-link'}).text)
homepage=abs.find('a',{'class':'zg-link author-link'})['href']
name=homepage[29:]
print(name)
print("用户主页:"+homepage)
normals=abs.find_all(a,{'class':'zg-link-gray-normal'})
print(abs.find('a',{'href':'/people/'+name+'/followers'}).text)
print(abs.find('a',{'href':'/people/'+name+'/asks'}).text)
print(abs.find('a', {'href': '/people/' + name + '/answers'}).text)
print(abs.find('a', {'href': '/people/' +name,'class': 'zg-link-gray-normal'}).text)
num=num+1
print("================================================================================================")