抓取动态网页

知乎中的关注列表是动态生成的，每次下拉刷新出20个，这样的话直接得到的html页面中并没有包含全部的关注人。

这里就要用到动态抓取的技术，用requests中的session模块模拟浏览器向对应的链接以post的形式发送数据。

import http.cookiejar
import requests
import re
import json
import math
from bs4 import BeautifulSoup

agent="Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) " \
      "Chrome/46.0.2490.76 Mobile Safari/537.36"
headers={
    'User-Agent':agent
}
session=requests.session()
session.cookies=http.cookiejar.LWPCookieJar(filename="cookies")
session.cookies.load(ignore_discard=True)
res=session.get('https://www.zhihu.com/people/zui-jiu-qing-niu-4/followees',headers=headers)
bs=BeautifulSoup(res.text,'html.parser')
fonum=bs.find("div",{'class':'zu-main-sidebar'}).find('a',{'class':'item'}).find('strong').text
fonum=int(fonum)
print(fonum)


def get_xsrf():
    '''_xsrf 是一个动态变化的参数'''
    index_url = 'http://www.zhihu.com'
    # 获取登录时需要用到的_xsrf
    index_page = session.get(index_url, headers=headers)
    html = index_page.text
    pattern = r'name="_xsrf" value="(.*?)"'
    # 这里的_xsrf 返回的是一个list
    _xsrf = re.findall(pattern, html)
    return _xsrf[0]

begin=0
end=math.ceil(fonum/20)
num=1
for x in range(0,end):
    beginnum = str(x * 20)
    #print(x)
    postdata={'method':'next',
          'params':'{"offset":'+beginnum+',"order_by":"created","hash_id":"29d75b4013b4631aaf7fe5848f3f6113"}',
          '_xsrf':str(get_xsrf())}
    ress=session.post('https://www.zhihu.com/node/ProfileFolloweesListV2',data=postdata,headers=headers)
    jsons=json.loads(ress.content.decode('utf-8'))
    for a in jsons['msg']:
        #print(a)
        abs=BeautifulSoup(a,'html.parser')
        print(num)
        print("用户:"+abs.find('a',{'class':'zg-link author-link'}).text)
        homepage=abs.find('a',{'class':'zg-link author-link'})['href']
        name=homepage[29:]
        print(name)
        print("用户主页:"+homepage)
        normals=abs.find_all(a,{'class':'zg-link-gray-normal'})
        print(abs.find('a',{'href':'/people/'+name+'/followers'}).text)
        print(abs.find('a',{'href':'/people/'+name+'/asks'}).text)
        print(abs.find('a', {'href': '/people/' + name + '/answers'}).text)
        print(abs.find('a', {'href': '/people/' +name,'class': 'zg-link-gray-normal'}).text)
        num=num+1
        print("================================================================================================")