day24selenuim的高级用法
1网页滚动爬取京东数据
from selenium.webdriver import Chrome,ChromeOptions
from selenium.webdriver.common.keys import Keys
from bs4 import BeautifulSoup
import time,csv
options=ChromeOptions()
options.add_experimental_option('excludeSwitches',['enable-automation'])
options.add_experimental_option('prefs',{"profile.managed_default_content_settings.images":2})
b=Chrome(chrome_options=options)
b.get('https://www.jd.com')
search=b.find_element_by_css_selector('#key')
search.send_keys('运动鞋')
search.send_keys(Keys.ENTER)
height=500
for i in range(10):
b.execute_script(f'window.scrollTo(0,{height})')
height+=800
time.sleep(1)
soup=BeautifulSoup(b.page_source,'lxml')
all_good=soup.select('.gl-warp>li')
print(len(all_good))
list1=[]
for good in all_good:
name=good.select_one('.p-name.p-name-type-2 em').text
evaluate=good.select_one('.p-commit a').text
shop=good.select_one('.J_im_icon>a').text
price = good.select_one('.p-price i').text
list1.append([name,evaluate,shop,price])
f=open('file/jd鞋子情况.csv','w',encoding='utf-8',newline='')
writer=csv.writer(f)
writer.writerow(['名字','评价','店名','价格'])
writer.writerows(list1)
2显式等待和隐式等待
wait=WebDriverWait(b,10)
"""
等待对象.until(条件)
等待对象.until_not(条件)
"""