导入数据模块,并输入要查询下载的产品编码及下载评论页数)
import requests
import pandas as pd
import json
from lxml import etree
c_num = input(r"请输入要下载评论的商品编号:")
num = input(r"请输入要下载几页评论:")
爬取网页,并用Json读取数据
url =f"https://club.jd.com/comment/productPageComments.action?callback=fetchJSON_comment98&productId={c_num}&score=0&sortType=5&page={int(num)-1}&pageSize=10&isShadowSku=0&rid=0&fold=1"
headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.92 Safari/537.36'}
response = requests.get(url=url,headers=headers).text
response = response.replace("fetchJSON_comment98(","")
response = response.replace(');','')
dt = json.loads(response)
获取每个类表中每个元素的字典
dt_list = dt['comments']
cont = [i["content"] for i in dt_list]
color = [j["productColor"] for j in dt_list]
size = [b["productSize"] for b in dt_list]
数据保存
date = pd.DataFrame({"评价":cont,"颜色":color,"型号":size})
date.index = date.index + 1
date.to_excel("e:/京东评论.xlsx")
完整代码
import requests
import pandas as pd
import json
from lxml import etree
c_num = input(r"请输入要下载评论的商品编号:")
num = input(r"请输入要下载几页评论:")
url =f"https://club.jd.com/comment/productPageComments.action?callback=fetchJSON_comment98&productId={c_num}&score=0&sortType=5&page={int(num)-1}&pageSize=10&isShadowSku=0&rid=0&fold=1"
headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.92 Safari/537.36'}
response = requests.get(url=url,headers=headers).text
response = response.replace("fetchJSON_comment98(","")
response = response.replace(');','')
dt = json.loads(response)
dt_list = dt['comments']
cont = [i["content"] for i in dt_list]
color = [j["productColor"] for j in dt_list]
size = [b["productSize"] for b in dt_list]
#数据存储
date = pd.DataFrame({"评价":cont,"颜色":color,"型号":size})
date.index = date.index + 1
date.to_excel("e:/京东评论.xlsx")
总结
网页中URL地址栏中的不是真是url
需要在网页刷新时找到包含真是url的地址
真是url中含有翻页信息及产品编码
访问网页返回Json数据,其中包含多余符号,不能直接进行Json读取,需要将多余的替换成空
Json读取数据获得是列表