python 爬虫 爬取 小米有品 全网 商品数据

#!coding=utf-8
import requests
import re
import random
import time
import json
from requests.packages.urllib3.exceptions import InsecureRequestWarning
import pandas as pd

requests.packages.urllib3.disable_warnings(InsecureRequestWarning)  ###禁止提醒SSL警告


###格式化时间戳
def timestamp_to_date(time_stamp, format_string="%Y-%m-%d %H:%M:%S"):
    time_array = time.localtime(int(time_stamp))
    str_date = time.strftime(format_string, time_array)
    return str_date

def de_duplication(lst):   ##去重不改变原数据顺序
    de_du = list(set(lst))
    de_du.sort(key=lst.index)
    return de_du


class xm(object):

    ###  获取分类
    def get_categoryList(self):
        url = 'https://youpin.mi.com/app/shopv3/pipe'
        headers = {
            'Accept':'*/*',
            'Accept-Encoding':'gzip, deflate, br',
            'Accept-Language':'zh-CN,zh;q=0.9',
            'Connection':'keep-alive',
            'Content-Length':'130',
            'Content-Type':'application/x-www-form-urlencoded',
            'Host':'youpin.mi.com',
            'Origin':'https://youpin.mi.com',
            'Referer':'https://youpin.mi.com/',
            'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.15 Safari/537.36'
        }
        data={
            'data':'{"result": {"model": "Homepage", "action": "GetGroup2ClassInfo", "parameters": {}}}',
        }
        req = requests.post(url=url, headers=headers,data=data, verify=False).json()
        groups=req['result']['result']['data']['groups']
        df = pd.DataFrame(columns=('一级分类ID', '一级分类', '二级分类ID', '二级分类'))
        x = 0
        for i in groups: