python抓取亚马逊某个类目下Top 100排名的商品

1.获取分页url 列表

2. 获取详情页url列表

3. 获取详情页数据

import requests
from lxml import etree
import urllib3
import time
from Database import Database
import socket
import random
import ssl

ssl._create_default_https_context = ssl._create_unverified_context
urllib3.disable_warnings()
headers = {
   
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36",
    "cookie": 'session-id-time=2082787201l; i18n-prefs=USD; session-id=133-9435810-7669859; ubid-main=132-7428001-1041912; lc-main=en_US; session-token=CoBxJ/ZP/aIvdcJ98VsX3lcm9kEU0420b2piNFZwXRmnua/IblHVfSIvsTOGbwUiTx4blRwxTw7IP0S63dIMzfCfguOPn/pwZWR4Ez00Mte8+0FldtcfSvpCFseldBdh1cKVJkRdB3e8aClgcKEeGgpNVmV6GbPpIgYj5hPQco5uKSy9WkEO0GFfXostLhUj; csm-hit=tb:s-244GWV52C0S66K0HZ4WK|1606551502475&t:1606551506602&adb:adblk_yes',
    "downlink": "10",
    "ect": "4g",
    "rtt": "250",
    "authority": "www.amazon.com",
}
# 分页url 列表
def get_page_url_list(num):
    page_url_list = []
    basic = "https://www.amazon.com/Best-Sellers-Mens-Water-Shoes/zgbs/fashion/679304011/ref=zg_bs_pg_"
    for i in range(num):
        page_url =  basic + str(i+1)
        page_url_list.append(page_url)
    return page_url_list
# 获取详情页url列表
def get_detail_url_list(page_url_list):
    detail_url_list = []
    basic = "https://www.amazon.com"
    exception_detail_url_list = []
    for i in page_url_list:
        try:
            res = requests.get(i, headers=headers, verify=False, timeout=100)
            time.sleep(random.randint(3, 10))
            res.close()
            socket.setdefaulttimeout(30)
            html = etree.HTML(res.text)
            detail_url = html.xpath(
                '//ol[@id="zg-ordered-list"]//li//span[@class="aok-inline-block zg-item"]/a/@href')
            detail_url = [basic + i for i in detail_url]
            detail_url_list.append(