本文旨在交流学习,勿作他用,否则后果自负
环境 linux+pycharm+anaconda
import csv
import requests
from lxml import etree
import re
import random
import json
from usere_agent import UA
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
url = 'https://youpin.mi.com/app/shopv3/pipe'
headers1 = {
'Accept': '*/*',
'Accept-Encoding': 'gzip, deflate, br',
'Accept-Language': 'zh-CN,zh;q=0.9',
'Connection': 'keep-alive',
'Content-Length': '130',
'Content-Type': 'application/x-www-form-urlencoded',
'Host': 'youpin.mi.com',
'Origin': 'https://youpin.mi.com',
'Referer': 'https://youpin.mi.com/',
'User-Agent': UA)
}
headers2 = {
'Accept': '*/*',
'Accept-Encoding': 'gzip, deflate, br',
'Accept-Language': 'zh-CN,zh;q=0.9',
'Connection': 'keep-alive',
'Content-Length': '145',
'Content-Type': 'application/x-www-form-urlencoded',
'Cookie': (你的cookie),
'Origin': 'https://youpin.mi.com',
'Referer': 'https://youpin.mi.com/',
'User-Agent': UA,
}
headers3 = {
'Accept': '*/*',
'Accept-Encoding': 'gzip, deflate, br',
'Accept-Language': 'zh-CN,zh;q=0.9',
'Connection': 'keep-alive',
'Content-Length': '364',
'Content-Type': 'application/x-www-form-urlencoded',
'Host': 'youpin.mi.com',
'Origin': 'https://youpin.mi.com',
'Referer': 'https://youpin.mi.com/detail?gid={}'.format(str(id)),
'User-Agent': UA,
}
data1 = {
'mkbl_data': '{"result": {"model": "Homepage", "action": "GetGroup2ClassInfo", "parameters": {}}}',
}
req = requests.post(url=url, headers=headers1, data=data1, verify=False).json()
groups = req['result']['result']['mkbl_data']['groups']
c_name=[]
c_id=[]
for i in groups:
for j in i:
class1_name = j['class']['name']
ucid1 = j['class']['ucid']
c_name.append(class1_name)
c_id.append(ucid1)
for k in j['sub_class']:
class2_name = k['name']
ucid2 = k['ucid']
for i,j in zip(c_name,c_id):
s = requests.session()
s.headers.update(headers2)
data2 = {
'mkbl_data': '{"uClassList": {"model": "Homepage", "action": "BuildHome", "parameters": {"id": "' + str(
j) + '"}}}'
}
respon = s.post(url=url, data=data2, verify=False).json()
print(respon)
itemdata = respon['result']['uClassList']['mkbl_data']
for j in itemdata:
if 'content' in j:
content_name = j['content']['name']
ucid = j['content']['ucid']
for k in j['mkbl_data']:
try:
gid = k['gid'] ##商品ID
name = k['name'] ##商品名称
summary = k['summary'] ##商品简介
pic_url = k['pic_url'] ##商品图片
price_min = int(k['price_min']) / 100 ##价格
itemurl = k['url'] ##商品链接
print(i,name,summary,pic_url,price_min,itemurl)
with open('/media/liu/_dde_data/project/spider/供应商/xmyp/' + i + '.csv', 'a+') as f:
f_csv = csv.writer(f)
f_csv.writerows([(i,name,summary,pic_url,price_min,itemurl)])
except:
continue