怎么使用:
- 本地要新建相应txt,xlsx文件
- 自己要准备好一些代理ip
- 如果要搜自己指定的关键字的话,pararm里的关键字(keyword)要改一下,还有get_X_sign里也要对应改一下,就是关键字要改成,‘%E4%BA%BA%E7%94%9F%E6%84%9F%E6%82%9F’同样的形式
- 时间久了的话,一些参数会过期,重新抓包获取参数
其他:
参考自:
https://blog.csdn.net/m0_46639364/article/details/108012594
代码:
import time
import requests
import re
import urllib3
from selenium.webdriver import Chrome
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
import json
from openpyxl import load_workbook
import hashlib
class xhs_class(object):
def __init__(self):
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
self.url = 'https://www.xiaohongshu.com/fe_api/burdock/weixin/v2/search/notes?'
self.key_dict=dict()
with open('小红书url.txt', 'r', encoding='utf-8')as f:
r = f.read().replace('\ufeff', '')
self.old_list = r.split('\n')
#print(self.old_list)
options = Options()
options.add_argument('--headless')
self.driver = Chrome(options=options)
def get_X_sign(self):
for i in range(1,6):
par = {
'keyword': '美食',
'sortBy': 'general',
'page': i,
'pageSize': '20',
'prependNoteIds':'',
'needGifCover': 'true',
}
qwq= '/fe_api/burdock/weixin/v2/search/notes?keyword='+'%E4%BA%BA%E7%94%9F%E6%84%9F%E6%82%9F'+'&sortBy=general&page='+str(i)+'&pageSize=20&prependNoteIds=&needGifCover=true'
s=qwq+'WSUDD'
#print (s)
md5 = hashlib.md5()
md5.update(s.encode("utf-8"))
X_sign='X'+md5.hexdigest()
#print(X_sign)
self.key_dict[str(i)]=X_sign
print(self.key_dict)
def get_ip(self,num):
res=requests.get('http://webapi.http.zhimacangku.com/getip?num=200&type=1&pro=&city=0&yys=0&port=1&time=1&ts=0&ys=0&cs=0&lb=1&sb=0&pb=4&mr=1®ions=')
#print(res.text)
http_dict=dict()
a=res.text.split('\r\n')
a.pop()
for i, item in enumerate(a):
lst=a[i].split(':')
key=lst[0]
value=lst[1]
http_dict[key]=value
target=0
for key,value in http_dict.items():
proxyHost = key
proxyPort = value
proxyMeta = "http://%(host)s:%(port)s" % {
"host" : proxyHost,
"port" : proxyPort,
}
self.proxies = {
"http" : proxyMeta,
"https" : proxyMeta
}
target+=1
if target==num:
break
return self.proxies
def get_detail_url(self):
self.get_X_sign()
self.num=0
for key, value in self.key_dict.items():
headers = {
'Host': 'www.xiaohongshu.com',
'Connection': 'keep-alive',
'Authorization': 'wxmp.c3657635-30c6-421c-88e8-d76d1676ae12',
'Device-Fingerprint': 'WHJMrwNw1k/Hg/nCupWeBd8sBskqluBSRBmzIRRI207qeIoxRiCuJhSk7OZmoaFLmYPM6m5bBIGFGAOhuNZ6DRl6kLx8sRQcTdCW1tldyDzmauSxIJm5Txg==1487582755342',
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36 MicroMessenger/7.0.9.501 NetType/WIFI MiniProgramEnv/Windows WindowsWechat',
'X-Sign': value,
'content-type': 'application/json',
'Referer': 'https://servicewechat.com/wxb296433268a1c654/60/page-frame.html',
'Accept-Encoding':'gzip, deflate, br',
}
params = {
'keyword': '人生感悟',
'sortBy': 'general',
'page': key,
'pageSize': '20',
'prependNoteIds':'',
'needGifCover': 'true',
}
res= requests.get(self.url,headers=headers,params=params,verify=False,proxies=self.get_ip(self.num))
self.num+=1
#print (self.get_ip(self.num))
res2=res.text
#print(res)
#print(res.status_code)
res_dict = json.loads(res2)
#print(res_dict)
notes = res_dict['data']['notes']
for note in notes:
id = note['id']
#print(id)
self.detail_url = 'https://www.xiaohongshu.com/discovery/item/' + id
#print(self.detail_url)
if self.detail_url in self.old_list:
print('链接已存在。')
continue
else:
with open('小红书url.txt', 'a', encoding='utf-8')as w:
w.write('\n')
w.write(self.detail_url)
self.get_detail()
self.store_data()
continue
def get_detail(self):
proxy = self.get_ip(self.num)
proxy=proxy['http']
# 设置代理IP
ops = webdriver.ChromeOptions()
ops.add_argument('--proxy-server=%s' % proxy)
self.driver= webdriver.Chrome(chrome_options=ops)
ops.add_argument('--headless')
self.driver.get(self.detail_url)
self.num+=1
#print (self.chrome)
time.sleep(1.5)
try:
self.pub_time = str(self.driver.find_element_by_xpath('//div[@class="publish-date"]/span').text).replace(
'发布于', '')
print(self.pub_time)
except:
self.pub_time = ' '
try:
self.author = self.driver.find_element_by_class_name('name-detail').text
print(self.author)
except:
self.author = ' '
try:
self.title = self.driver.find_element_by_class_name('title').text
print(self.title)
except:
self.title=' '
try:
self.content = self.driver.find_element_by_class_name('content').text
print(self.content)
except:
self.content=' '
def store_data(self):
wb = load_workbook('数据表.xlsx')
sheet = wb.get_sheet_by_name("Sheet1")
excel_head=['作者','发布时间','标题','内容']
for i, item in enumerate(excel_head):
sheet.cell(row = 1,column=i+1,value=item)
data=[self.author,self.pub_time,self.title,self.content]
print(data)
sheet.append(data)
wb.save('数据表.xlsx')
print('保存成功'+str(self.num))
if __name__ == '__main__':
xhs = xhs_class()
xhs.get_detail_url()