2020-07-19

import time
from lxml import etreefrom selenium import webdriverdriver_path = r"C:\ChromeDriver\chromedriver.exe"driver = webdriver.Chrome(executable_path=driver_path)base_url="https://www.amazon.com""""option = webdriver.ChromeOptions()option.add_argument("--user-data-dir="+r"C:/Users/HANCHENG/AppData/Local/Google/Chrome/User Data/")driver = webdriver.Chrome(chrome_options=option,executable_path=driver_path)"""url = 'https://www.amazon.com/gp/bestsellers/hi/9628854011/ref=pd_zg_hrsr_hi'"""    next_btn = self.driver.find_element_by_class_name("pager_next ")    if "pager_next pager_next_disabled" in next_btn.get_attribute("class"):        break    else:        next_btn.click()    time.sleep(1)        html = etree.HTML(self.driver.page_source)        title = html.xpath('//h1[@class="name"]/text()')[0].strip()"""driver.get(url)bst_urls=[]while True:    html = etree.HTML(driver.page_source)    titles = html.xpath('//span[@class="aok-inline-block zg-item"]/a[@class="a-link-normal"]/@href')    for title in titles:        new_title=base_url+title        bst_urls.append(new_title)    next_btn = driver.find_elements_by_xpath('//li[@class="a-last"]')    if len(next_btn):        next_btn[0].click()    else:        break    for bst_url in bst_urls:    print(bst_url)    print('='*50)print(len(bst_urls))