selenium学习爬取京东网页数据

程序：

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait
import time
from pyquery import PyQuery as pq
browser = webdriver.Chrome(executable_path="C:/Users/25104/AppData/Local/Google/Chrome/Application/chromedriver.exe")
wait = WebDriverWait(browser, 10)

def get_produces(html):
    doc = pq(html)
    items = doc('#J_goodsList > ul > li').items()
    for item in items:
         product = {
             'price': item.find('.p-price').text(),
             'title': item.find('.p-name').text(),
             'shop': item.find('.p-shop').text(),
         }
         print(product)

def change_page(page_num):
    page_input = wait.until(
        EC.presence_of_element_located((By.XPATH, '//*[@id="J_bottomPage"]/span[2]/input'))
    )
    page_input.send_keys('0')
    time.sleep(1)#写两遍是因为第一次输入会被拉回网页中央
    page_input = wait.until(
        EC.presence_of_element_located((By.XPATH, '//*[@id="J_bottomPage"]/span[2]/input'))
    )
    page_input.clear()
    page_input.send_keys(str(page_num))
    page_confirm_button = wait.until(
        EC.presence_of_element_located((By.XPATH, '//*[@id="J_bottomPage"]/span[2]/a'))
    )
    page_confirm_button.click()
    return browser.page_source

def get_page_source(keyword='switch',maxnum=3):
    url='https://search.jd.com/Search'
    browser.get(url)
    search_input=wait.until(
        EC.presence_of_element_located((By.ID,'key'))
    )
    search_input.send_keys(keyword)
    xpath = "//div[@class='form'][1]/button[1]"
    button = wait.until(
        EC.presence_of_element_located((By.XPATH,xpath))
    )
    button.click()
    for page_num in range(1,maxnum):
        get_produces(change_page(page_num))
    #browser.close()

def main():
    get_page_source('switch')

main()

主要关键是通过
EC.presence_of_element_located()
这个方法来确定元素的位置，一般直接通过谷歌浏览器的审查元素，有copy xpath这个选项，通过By.XPATH可以定位到想要的元素在通过Pyquery整合信息方面，通过css选择器来把想要的关键信息提取出来

这样的结构对应的就是
items = doc(’#J_goodsList > ul > li’).items()
遍历items，再配合上.find方法即可