shein网站反爬逐渐升级，最近又新增了几个参数校验，现在需要破解的参数有anti-in, smdeviceid, armortoken， x-gw-auth,部分网页可能不需要每个参数都校验，但想获取全量数据，不破解这些参数是不行的。
列表页部分代码：
def treat_page(self, region, link, page):
base_url = f'https://{region.lower()}.shein.com/api/productList/info/get?'
if 'pdsearch' in link:
routeid_pattern = re.compile('pdsearch/(.*?)/')
else:
routeid_pattern = re.compile('-(\d+).html')
routeid = re.search(routeid_pattern, link).group(1)
if region in ['BR']:
_lange = 'pt-br'
elif region in ['PH', 'SG', 'US', 'MY']:
_lange = 'en'
elif region in ['TH']:
_lange = 'th'
else:
raise
param = {
'_ver': '1.1.8',
'_lang': _lange,
'type': 'selection' if 'RecommendSelection' in link else 'search' if 'search' in link else 'entity',
'routeId': routeid,
'page': page,
'reqSheinClub': 'true',
'isPaid': '0',
'source': 'sort',
'sourceStatus': '1',
'sort': '7',
'requestType': 'firstLoad' if page == 1 else 'pageChange'
}
try:
ici = re.search(self.ici_pattern, link).group(1)
param['ici'] = ici
except:
pass
try:
srctype = re.search(self.srctype_pattern, link).group(1)
param['srctype'] = srctype
except:
pass
try:
src_tab_page_id = re.search(self.src_tab_page_id_pattern, link).group(1)
param['src_tab_page_id'] = src_tab_page_id
except:
pass
try:
userpath = re.search(self.userpath_pattern, link).group(1)
param['userpath'] = userpath
except:
pass
try:
src_identifier = re.search(self.src_identifier_pattern, link).group(1)
param['src_identifier'] = src_identifier
except:
pass
try:
adp = re.search(self.adp_pattern, link).group(1)
param['adp'] = adp
except:
pass
try:
child_cat_id = re.search(self.child_cat_id_pattern, link).group(1)
param['child_cat_id'] = child_cat_id
except:
pass
try:
src_module = re.search(self.src_module_pattern, link).group(1)
param['src_module'] = src_module
except:
pass
if 'RecommendSelection' in link:
param['sub_type'] = 'RecommendSelection'
param['categoryJump'] = 'true'
param['src_module'] = 'topcat'
page_url = base_url + urlencode(param).replace('%25', '%').replace('%20', '+')
# print('page_url:', page_url)
# headers = self.headers.copy()
if re.findall('page=', link):
referer = link.split('&page')[0] + f'&page={page}'
else:
if link.endswith('html') or link.endswith('/'):
referer = link + f'?page={page}&sort=7&source=sort&sourceStatus=1'
else:
referer = link + f'&page={page}&sort=7&source=sort&sourceStatus=1'
# headers['Referer'] = referer
path = page_url.replace('https://th.shein.com', '')
ua = UserAgent()
ii = 0
while ii < 5:
random_user_agent = ua.random
device = gen_device(region, referer, path, random_user_agent, site_type='web')
# print(device)
headers = {
'accept': 'application/json, text/plain, */*',
'accept-encoding': 'gzip, deflate, br',
'accept-language': 'zh-CN,zh;q=0.9',
"referer": referer,
"anti-in": device.get("msg", {}).get("headers", {}).get("anti-in"),
"smdeviceid": device.get("msg", {}).get("headers", {}).get("smdeviceid"),
"armortoken": device.get("msg", {}).get("headers", {}).get("armortoken"),
"x-gw-auth": device.get("msg", {}).get("headers", {}).get("x-gw-auth"),
"user-agent": random_user_agent,
"x-requested-with": "XMLHttpRequest",
'uber-trace-id': 'ffae1d5fbd3ecdba:ffae1d5fbd3ecdba:0:0',
'x-csrf-token': 'CmT59WLJ-n7-tWFF--_58eJMNbbrPx6qlMFk'
}
try:
# response = requests.get(page_url, headers=headers, proxies=self.proxies, timeout=30)
response = requests.get(page_url, headers=headers, cookies=device.get("msg", {}).get("cookies", {}),
proxies=self.proxies, timeout=20, verify=False)
# print(response.text)
except Exception as e:
ii += 1
# print(f'try {ii} times,{e.args}')
continue
try:
response = response.json()
# print(f'page:{page}', response)
except:
# print(f'try {ii} times, ')
ii += 1
continue
if not response:
ii += 1
# print(f'request page, try {x} times')
continue
else:
print(f'page:{page} success')
break
else:
raise Exception(page_url)
items = response['goods']
urls = []
for item in items:
# 两种构造方式都可？
# url = f'https://{region.lower()}.shein.com/{item["goods_url_name"]}-p-{item["goods_id"]}-cat-{item["cat_id"]}.html'
url = f'https://{region.lower()}.shein.com/{item["goods_url_name"].replace(" ", "-")}-p-{item["goods_id"]}.html' # + urlencode(para) # 2024.7.22详情页链接构造发生变化
urls.append(url)
l = len(urls)
print('urls:', l, urls[:1])
exector = ThreadPoolExecutor(max_workers=40)
for index, url in enumerate(urls):
# self.request_detail(url, region, referer)
# break
exector.submit(self.request_detail, url, region, referer).add_done_callback(self.parse_exception)
exector.shutdown(wait=True)
详情页
import re
import time
import requests
from urllib.parse import urlencode
import urllib3
urllib3.disable_warnings()
from shein_utils import gen_device
from fake_useragent import UserAgent
class SheinDetail:
def __init__(self):
self.product_data_pattern = re.compile('window.gbRawData = (.*?)\n')
self.proxies = {
}
def detail(self, detail_url, region):
"""详情页"""
t = re.search(re.compile('shein.com/(.*?)\.html'), detail_url).group(1)
referer = f'https://m.shein.com/{region.lower()}/{t}.html?ref={region.lower()}&rep=dir&ret=m{region.lower()}' # ref=th&rep=dir&ret=mth' #
p = re.compile('-(\d+)\.html')
goods_id = re.search(p, referer).group(1)
# print(referer)
# print(goods_id)
if region in ['BR']:
_lange = 'pt-br'
elif region in ['PH', 'SG', 'US', 'MY']:
_lange = 'en'
elif region in ['TH']:
_lange = 'th'
else:
raise
currency_map = {
'BR': 'BRL',
'PH': 'PHP',
'TH': 'THB',
'SG': 'SGD',
'MY': 'MYR',
'US': 'USD'
}
url = f"https://m.shein.com/{region.lower()}/api/productInfo/productDetail/get"
params = {
"currency": currency_map[region], # "BRL", #"THB", # SGD",
"fromSpa": "1",
"goods_id": str(goods_id),
"imgRatio": "3-4",
"mallCode": "1",
"showFeedbackRec": "1",
"template": "0",
"version": "1.0.4",
"withI18n": "0",
"_ver": "1.1.8",
"_lang": _lange #"th", #"en"
}
path = url.replace('https://m.shein.com', '') + "?" + urlencode(params)
# print('path:', path)
ua = UserAgent()
x = 0
while x < 15:
random_user_agent = ua.random
device = gen_device(region, referer, path, random_user_agent)
# print(device)
headers = {
"referer": referer,
"anti-in": device.get("msg", {}).get("headers", {}).get("anti-in"),
"smdeviceid": device.get("msg", {}).get("headers", {}).get("smdeviceid"),
"armortoken": device.get("msg", {}).get("headers", {}).get("armortoken"),
"x-gw-auth": device.get("msg", {}).get("headers", {}).get("x-gw-auth"),
"user-agent": random_user_agent,
"x-requested-with": "XMLHttpRequest",
}
# print(headers)
try:
response = requests.get(url, headers=headers, cookies=device.get("msg", {}).get("cookies", {}),
proxies=self.proxies, params=params, timeout=20, verify=False)
except Exception as e:
x += 1
time.sleep(3)
# print(22, e.args)
continue
# print(response.status_code, response.text)
# print(response.status_code == 403)
if response.status_code == 403 or 'goods not exist' in response.text or 'code":"836100"' in response.text:
x += 1
time.sleep(3)
# print(22)
continue
else:
break
else:
# raise Exception(detail_url)
return []
try:
modules = response.json()['modules']
except:
return [] # goods not exist
# print(11, modules)
return modules
if __name__ == '__main__':
s = SheinDetail()
s.detail('https://us.shein.com/SHEIN-Comfortcana-4pcs-Set-Striped-Leopard-Print-Solid-Casual-Slim-Fit-Cami-Top-Summer-p-33543404.html', 'US')
评论页
def comment_page(self, page, product_relation_id, region, store_code, cat_id, sku_id):
if region in ['BR', 'PH', 'SG', 'TH', 'US', 'MY']:
base_url = f'https://m.shein.com/{region.lower()}/bff-api/product/get_goods_review_detail?'
else:
raise
if region in ['BR']:
_lange = 'pt-br'
elif region in ['PH', 'SG', 'US', 'MY']:
_lange = 'en'
elif region in ['TH']:
_lange = 'th'
else:
raise
para = {
'_ver': '1.1.8',
'_lang': _lange,
'comment_rank': '',
'goods_id': '',
'goods_spu': product_relation_id,
'is_picture': '',
'local_site_abt_flag': '',
'local_site_query_flag': '',
'size': '',
'sku': sku_id,
'sort': 'time_desc',
'store_code': store_code,
'tag_id': '',
'tag_rule_id': 'type=B',
'store_comment_flag': '1',
'isLowestPriceProductOfBuyBox': '0',
'mainProductSameGroupId': '',
'page': page,
'cat_id': cat_id,
}
url = base_url + urlencode(para)
x = 0
while x < 20:
try:
response = requests.get(url, headers=self.comment_headers, proxies=self.proxies, timeout=30)
# print(response.text)
except:
x += 1
# print(f'try {x} times') # , detail_url:{detail_url}
antiin = self.gen_antiin()
self.comment_headers['anti-in'] = antiin
continue
try:
res = response.json()
# print('res:', res)
except:
# print(response.text)
x += 1
# print(f'try {x} t imes') # , detail_url:{detail_url}
antiin = self.gen_antiin()
self.comment_headers['anti-in'] = antiin
cookie_jar = response.cookies.items()
cookie_jar = [i[0] + '=' + i[1] for i in cookie_jar]
cookie = ';'.join(cookie_jar)
self.comment_headers['cookie'] = cookie
continue
if not res or res['msg'] != 'ok':
x += 1
# print(f'request comment, try {x} times')
antiin = self.gen_antiin()
self.comment_headers['anti-in'] = antiin
continue
else:
# print(res)
return res
else:
return None
def comment(self, product_relation_id, region, store_code, cat_id, sku_id):
"""
获取最近7天的评价数、好评数
点击进入商品页面
拉到下方评论页面，选择location rating (本地评论）, sort by 最新评论
记录7天内的评论数 （评论有显示时间戳）
记录7天内的好评数（>=4星为好评）
@return:
"""
# print('product_relation_id:', product_relation_id)
comment_num_7, comment_good_num_7 = 0, 0
page = 1
while page < 75:
res = self.comment_page(page, product_relation_id, region, store_code, cat_id, sku_id)
# print('res:', res)
if not res:
return comment_num_7, comment_good_num_7
comments = res['info']['comment_info']
# print('comments:', len(comments))
if not comments:
return comment_num_7, comment_good_num_7
for comment in comments:
comment_time = int(comment['add_time'])
within7 = self.within_gap(comment_time)
if not within7:
return comment_num_7, comment_good_num_7
comment_num_7 += 1
star = int(comment['comment_rank'])
if star >= 4:
comment_good_num_7 += 1
page += 1
这里主要列出了shein爬虫的主干代码，涉及细节部分欢迎私聊。
扫一扫
358
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
