python3 爬虫爬取华为应用市场 APP应用评论（一）爬取全部评论——学霸君

最新推荐文章于 2024-09-26 11:01:02 发布

chari克里

最新推荐文章于 2024-09-26 11:01:02 发布

阅读量3.1k

点赞数 3

文章标签： python json 大数据

本文链接：https://blog.csdn.net/weixin_45615730/article/details/108859415

版权

python3 爬虫爬取华为应用市场 APP应用评论（一）爬取全部评论——学霸君

import requests
import re
import requests
from bs4 import  BeautifulSoup as bs
import json
import csv
import re

def getHTMLText(url):
    header = {
        'referer': 'https://appgallery.huawei.com/',
        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.81 Safari/537.36 SE 2.X MetaSr 1.0',
        'Connection': 'keep-alive'
    }
    try:
        r = requests.get(url, timeout=30,headers=header)
        r.raise_for_status()
        r.encoding = r.apparent_encoding
        return r.content
    except:
        return ""

def main():
    depth = 5
    infoList = []
    for i in range(depth):
        try:
            url = 'https://web-drcn.hispace.dbankcloud.cn/uowap/index?method=internal.user.commenList3&serviceType=20&reqPageNum='+str(i)+'&maxResults=25&appid=C10207269&version=10.0.0&zone=&locale=zh_CN'
            html = getHTMLText(url).decode("utf-8")
            result=json.loads(html)    
            for i in range(len(result['list'])):
                #print(result['list'][i]['accountName'])
                comment = result['list'][i]['commentInfo']
                #print(comment)
                infoList.append(comment)
            #print(result)
            #parsePage(infoList, html)
            
        except:
            continue
    #printGoodsList(infoList)
    print(infoList)
    # 把评论数据保存到文件中
    with open('xuebajunComment.txt', 'a', encoding='utf-8') as f:
        f.write(json.dumps(infoList, ensure_ascii=False) + '\n')

main()