# coding:utf-8
from selenium import webdriver
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from lxml import etree
import re
options = webdriver.ChromeOptions()
options.add_argument('--headless')
driver = webdriver.Chrome(options=options)
def search():
try:
print('打开网页中...')
driver.get('https://www.taobao.com/')
driver.maximize_window()
search = WebDriverWait(driver,10).until(EC.presence_of_element_located((By.XPATH,'//*[@id="q"]')))
submit = WebDriverWait(driver,10).until(EC.element_to_be_clickable((By.XPATH,'//*[@id="J_TSearchForm"]/div[1]/button')))
search.send_keys('美食')
submit.click()
print('搜索中...')
# 等待页面关键元素完全加载,以页面底部的总页数为参照
total_page = WebDriverWait(driver,10).until(EC.presence_of_element_located((By.XPATH,'//*[@id="mainsrp-pager"]/div/div/div/div[1]')))
print('搜索完成!')
get_product()
# print(total_page.text)
return total_page.text
except:
print('搜索失败!再来一次...')
search()
def next_page(number):
try:
number_input = WebDriverWait(driver,10).until(EC.presence_of_element_located((By.XPATH,'//*[@id="mainsrp-pager"]/div/div/div/div[2]/input')))
submit = WebDriverWait(driver,10).until(EC.element_to_be_clickable((By.XPATH,'//*[@id="mainsrp-pager"]/div/div/div/div[2]/span[3]')))
number_input.clear()
number_input.send_keys(number)
submit.click()
print('搜索第%s页中...'%(number))
WebDriverWait(driver,20).until(EC.presence_of_element_located((By.XPATH,'//*[@id="mainsrp-itemlist"]/div/div/div[1]/div')))
print('搜索第%s页完成...'%(number))
get_product()
except:
next_page(number)
def get_product():
print('加载商品页面中...')
WebDriverWait(driver,20).until(EC.presence_of_element_located((By.XPATH,'//*[@id="mainsrp-itemlist"]/div/div/div[1]/div')))
print('当前页商品加载完成...')
selector = etree.HTML(driver.page_source)
# print(html)
shop_names = selector.xpath('//*[@id="mainsrp-itemlist"]//div[1]/a/span[2]/text()')
product_names = selector.xpath('//*[@id="mainsrp-itemlist"]//div/div[1]/a/img/@alt')
product_prices = selector.xpath('//*[@id="mainsrp-itemlist"]//div[2]/div[1]/div[1]/strong/text()')
image_urls = selector.xpath('//*[@id="mainsrp-itemlist"]//div/div[1]/a/img/@src')
for item in zip(shop_names,product_names,product_prices,image_urls):
# print(item)
# print(type(item[0])) <class 'lxml.etree._ElementUnicodeResult'>
# print(type(item[1])) <class 'lxml.etree._ElementUnicodeResult'>
# print(type(item[2])) <class 'lxml.etree._ElementUnicodeResult'>
# print(type(item[3])) <class 'lxml.etree._ElementUnicodeResult'>
product = {
'shop_names': item[0],
'product_names': item[1],
'product_prices': '¥' + item[2],
'image_urls': item[3]
}
print(product)
def main():
count_text = search()
count = int(re.compile('(\d+)').search(count_text).group(1))
for i in range(2,count+1):
next_page(i)
if __name__ == '__main__':
main()
爬虫小练习:Chrome无弹窗爬取淘宝美食信息
最新推荐文章于 2024-04-25 18:00:00 发布