先爬取10页淘宝的评论信息
mport requests
import json
import pandas as pd
# 获取网页信息
def getHTMLText(url):
try:
# 设置请求头
headers = {'user-agent': 'Mozilla/5.0'}
r = requests.get(url, headers=headers, timeout=30)
r.raise_for_status()
r.encoding = r.apparent_encoding
return r.text # 返回到json数据页面
except:
print("发生异常,获取页面失败")
print(r.status_code) # 查看访问状态
# 将HTML信息放到列表中
def fillComList(cList, cJson):
try:
# 去掉多余字符
start_Loc = cJson.find('{')
cJson = cJson[start_Loc:-2]
cJson = json.loads(cJson) # 字典数据
pagelen = len(cJson['comments'])
for j in range(pagelen):
cList.append(cJson['comments'][j]['content'])
# data = {'评论内容': cList} # 用字典封装数据
# print(data)
# print(cList)
except:
print("数据封装异常")
ret