1.获取分页url 列表
2. 获取详情页url列表
3. 获取详情页数据
import requests
from lxml import etree
import urllib3
import time
from Database import Database
import socket
import random
import ssl
ssl._create_default_https_context = ssl._create_unverified_context
urllib3.disable_warnings()
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36",
"cookie": 'session-id-time=2082787201l; i18n-prefs=USD; session-id=133-9435810-7669859; ubid-main=132-7428001-1041912; lc-main=en_US; session-token=CoBxJ/ZP/aIvdcJ98VsX3lcm9kEU0420b2piNFZwXRmnua/IblHVfSIvsTOGbwUiTx4blRwxTw7IP0S63dIMzfCfguOPn/pwZWR4Ez00Mte8+0FldtcfSvpCFseldBdh1cKVJkRdB3e8aClgcKEeGgpNVmV6GbPpIgYj5hPQco5uKSy9WkEO0GFfXostLhUj; csm-hit=tb:s-244GWV52C0S66K0HZ4WK|1606551502475&t:1606551506602&adb:adblk_yes',
"downlink": "10",
"ect": "4g",
"rtt": "250",
"authority": "www.amazon.com",
}
def get_page_url_list(num):
page_url_list = []
basic = "https://www.amazon.com/Best-Sellers-Mens-Water-Shoes/zgbs/fashion/679304011/ref=zg_bs_pg_"
for i in range(num):
page_url = basic + str(i+1)
page_url_list.append(page_url)
return page_url_list
def get_detail_url_list(page_url_list):
detail_url_list = []
basic = "https://www.amazon.com"
exception_detail_url_list = []
for i in page_url_list:
try:
res = requests.get(i, headers=headers, verify=False, timeout=100)
time.sleep(random.randint(3, 10))
res.close()
socket.setdefaulttimeout(30)
html = etree.HTML(res.text)
detail_url = html.xpath(
'//ol[@id="zg-ordered-list"]//li//span[@class="aok-inline-block zg-item"]/a/@href')
detail_url = [basic + i for i in detail_url]
detail_url_list.append(