def getInitCursorValue() :
isEnd = False
#...
try:
resp = requests.get((f"https://store.steampowered.com/appreviews/{appid}?cursor={nextCursor}&language=schinese&day_range=365&review_type=all&purchase_type=all&filter=recent"),timeout=5).json()
except Exception as e:
print(f"request failed:请求评论失败,尝试重新拉取...{reloadDataNum}")
resp = ""
isEnd = True
reloadDataNum = reloadDataNum + 1
time.sleep(1)
if isEnd :
return isEnd
reloadDataNum = 0 #reset
print(f"request completed , data list len ={len(listAllContent)}")
if lastedNum == len(listAllContent):
totalEndNum = totalEndNum + 1
print(f"request failed:请求不到更多评论...{totalEndNum}")
if totalEndNum >= reloadDataNumMax :
print(f"结束请求...")
isEnd = True
reloadDataNum = 5
time.sleep(1)
return isEnd
else:
lastedNum = len(listAllContent)
totalEndNum = 0
cursor = resp["cursor"]
cursor = cursor.replace("+","%2B")
nextCursor = cursor
html = resp["html"] #本页评论数据
soup= BeautifulSoup(html,"lxml")
#请求成功
#解析 用户名
listName = []
listRecommend = []
listTime = []
listComment = []
for business in soup.find_all('div', class_="persona_name"):
for bb in business.find_all('a') :
listName.append(bb.string)
#解析 是否推荐
for business in soup.find_all('div', class_="title ellipsis"):
listRecommend.append(business.string)
#解析 游戏时长
for business in soup.find_all('div', class_="hours ellipsis"):
text1 = business.text.replace("\r\n\t\t\t\t\t\t","")
text2 = text1.replace("\t\t\t\t\t\t\t\t\t\t\t","")
listTime.append(text2)
#解析 游戏评论
for business in soup.find_all('div', class_="content"):
text1 = business.text.replace("\r\n\t\t\t\t\t","")
text2 = text1.replace("\t\t\t\t\t\n","")
listComment.append(text2)
# all data
for number in range(0,len(listComment)) :
list1 = []
list1.append(listName[number])
list1.append(listRecommend[number])
list1.append(listTime[number])
list1.append(listComment[number])
listAllContent.append(list1)
return isEnd
URL解析
请求的URL格式:https://store.steampowered.com/appreviews/{appid}?cursor={nextCursor}&language=schinese&day_range=365&review_type=all&purchase_type=all&filter=recent
appid:应用ID
nextCursor:用来索引下一页评论的下标,初始值等于*,下一页的值从请求的页面中可以拿到,这个值的键为"cursor"。
因此通过这个URL可以一直动态的请求到所有的评论数据。
if __name__ == '__main__':
getGameName()
while(len(listAllContent)<(int)(maxDataSize)):
checkExit = getInitCursorValue()
time.sleep(0.5)
if checkExit == True :
if reloadDataNum >= 5 :
print("连接超时 5 次,结束")
break
fun2(listAllContent)
time.sleep(1)
测试:拉取PUBG100条评论
google play,appstore获取比较简单,实现见下面工程。
工程链接:Ackerman1998/Craw-SteamComment (github.com)
python版本:3.7.0