Python爬虫之京东评论数据爬取
记一次爬取京东产品评论时攻克的难点:
1、json数据解析时,容易出现decode报错
2、京东PC端评论数据模块反爬特别严重,几乎是请求几次到几十次就会被封IP
解决方法:
1、采用python的demjson库解析json评论数据,同时在每次请求时加上time.sleep(n),避免请求过于频繁
2、尝试多次,发现PC的评论数据反爬确实不是作者可以攻破的,作者尝试过time.sleep(),代理IP等,最终还是无效;最后采用请求移动端的接口,再加上time.sleep(),数据可以正常爬取
附上完整代码:
import json,demjson
import pandas as pd
import time
import requests
import datetime
from sqlalchemy import create_engine
import pymysql
from numpy import *
import re,urllib.request
from lxml import etree
def jd_comment_new():
url1 = 'https://search.jd.com/search?keyword=%E9%9B%86%E6%88%90%E7%81%B6&qrst=1&wq=%E9%9B%86%E6%88%90%E7%81%B6&stock=1&ev=exbrand_%E7%81%AB%E6%98%9F%E4%BA%BA%EF%BC%88marssenger%EF%BC%89%5E&page=1&s=61&click=0'
url = 'https://search.jd.com/search?keyword=%E9%9B%86%E6%88%90%E7%81%B6&qrst=1&wq=%E9%9B%86%E6%88%90%E7%81%B6&stock=1&ev=exbrand_%E7%81%AB%E6%98%9F%E4%BA%BA%EF%BC%88marssenger%EF%BC%89%5E&page={}&s=61&click=0'
headers = {
'Cookie':'抓取你的cookie',
'User-Agent':'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Mobile Safari/537.36',
'referer':'https://item.m.jd.com/',
}
re=requests.get(url1,headers=headers)
time.sleep(2)
html1=etree.HTML(re.text)
page_num=html1.xpath('//*[@id="J_topPage"]/span/i/text()')[0] #获取最大页数
# print(page_num)
page_size=2*int(page_num)
for page in range(1,page_size+1):
resp = requests.get(url.format(page),headers=headers)
time.sleep(2)
htmllist=etree.HTML(resp.text)
# print(htmllist)
ul=htmllist.xpath('//*[@id="J_goodsList"]/ul/li') #商品列表
# print(ul)
for li in ul:
# print(li,'\n')
data_sku=li.xpath('./@data-sku')[0] #获取商品ID
# sku_name=li.xpath('./div/div[3]/a/em/text()[2]')[0]
# print(sku_name)
# print(data_sku)
url2='https://wq.jd.com/commodity/comment/getcommentlist?&version=v2&fold=1&pagesize=10&sceneval=2&score=0&sku={}&sorttype=5&page={}'
maxpage=100
for page1 in range(maxpage):
# print(page1)
res=requests.get(url=url2.format(data_sku,page1),headers=headers)
# try:
# res.encoding='utf-8'
# except:
# res.encoding='GB18030'
time.sleep(2)
# print(res.text.strip())
js = demjson.decode(res.text.strip()[10:-1])['result']['comments']
# print(js)
# if not js:
# print(page1)
# # break
content=[];creationTime=[];referenceId=[]
#评论内容 #评论时间 #产品ID
for i in js:
# print(i['content'],'\n')
content.append(i['content'])
creationTime.append(i['creationTime'])
referenceId.append(i['referenceId'])
# print(content)
dic={}
dic['content']=content
# print(type(dic))
dic['creationTime']=creationTime
dic['referenceId']=referenceId
# print(dic)
df=pd.DataFrame(dic)
print(df)
jd_comment_new()
本人也是最近才开始摸索python爬虫,对python爬虫中的很多框架和技术都不是很熟悉,欢迎大家来交流爬虫心得。