爬取手机评论信息

1.获取到json数据

import requests


#https://club.jd.com/comment/productPageComments.action?callback=fetchJSON_comment98&productId=100016944073&score=0&sortType=5&page=2&pageSize=10&isShadowSku=0&rid=0&fold=1
url = "https://club.jd.com/comment/productPageComments.action?callback=fetchJSON_comment98&productId=100016944073&score=0&sortType=5&pageSize=10&isShadowSku=0&fold=1"

headers ={
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.54 Safari/537.36'
}
param ={
    'page':0
}
response = requests.get(url=url,headers=headers,params=param)
content = response.content.decode('GBK')
print(content)
with open('one.json','w',encoding='utf-8') as x:
  x.write(content)
print(response.encoding)

2.提取数据

import json
import jsonpath
obj = json.load(open('one.json','r',encoding='utf-8'))
res = jsonpath.jsonpath(obj,'$..content')
# print(obj)
print(res)

2.1结果 

 3.词云

3.1结果

4.优化 

import requests
import json
import jsonpath
import time

#https://club.jd.com/comment/productPageComments.action?callback=fetchJSON_comment98&productId=100016944073&score=0&sortType=5&page=2&pageSize=10&isShadowSku=0&rid=0&fold=1

def response(number):
    url = "https://club.jd.com/comment/productPageComments.action?callback=fetchJSON_comment98&productId=10042487383595&score=0&sortType=5&pageSize=10&isShadowSku=0&rid=0&fold=1&"

    headers = {
        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.54 Safari/537.36'
    }
    param = {
        'page': number
    }
    response = requests.get(url=url, headers=headers, params=param)
    content = response.content.decode('GBK')
    return content

def download(result,number):

    print(result)
    with open('xdn'+ str(number)+ '.json','w',encoding='utf-8') as x:
      x.write(result)


    # print(response.encoding)
if __name__ == '__main__':
   p1 = int(input("起始页："))
   p2 = int(input("结束页："))
   for number in range(p1,p2):
        result = response(number)
        download(result,number)
        time.sleep(3)