1,微博网页版爬取难度较大,本次爬取手机版评论数据:m.weibo.cn;
2,评论网页为动态的,查看方式network----XHR----请求对应的url
注:url中的id,mid与微博地址中的后缀一致,评论往下滑,20条之后,会出现新的url,新的url中多了max_id,该参数类似分页参数,在preview的data下面有,该max_id即下一条url中的max_id
import requests
import json
import pprint
import pandas as pd
import re
#服务器返回的内容用变量接收
response=requests.get('https://m.weibo.cn/comments/hotflow?id=4664928428097689&mid=4664928428097689&max_id_type=0')
print(response.text)
print(json.loads(response.text))
pprint.pprint(json.loads(response.text))
#字典数据
data=json.loads(response.text)
pprint.pprint(data)
users=data['data']['data']
lname=[]
ltext=[]
for user in users:
#print(user['user']['id'])
#print(user['user']['screen_name'])
#print(user['text'])
text=user['text']
text=re.sub(r'<[^>]*>','',user['text']) #去除图片
name=user['user']['screen_name']
print(name)
print(text)
lname.append(name)
ltext.append(text)
df=pd.DataFrame({'name':lname,'comment':ltext})
df=df[['name','comment']]
df.to_excel(r'D:\comment.xlsx')