目标
导入模块,创建浏览器对象
from selenium import webdriver
import csv
import time
# 1. 接收要搜索的商品类型,打开商品搜索页面
pro = input("请输入商品名称:")
driver = webdriver.Chrome()
driver.get("https://www.jd.com/")
发请求获取页面
抓取对象,xpath匹配节点对象
driver.find_element_by_class_name("text")\
.send_keys(pro)
driver.find_element_by_class_name("button")\
.click()
time.sleep(0.5)
i = 1
连续抓取
while True:
# 2. 执行JS脚本,把下拉菜单拉到最底部
driver.execute_script(
'window.scrollTo(0,document.body.scrollHeight)')
time.sleep(1)
r_list = driver.find_elements_by_xpath('//div[@id="J_goodsList"]//li')
#print(r_list[5].text)
for onePro in r_list:
m = onePro.text.split('\n')
# m : ["¥98.5","名称","9条评价",...]
price = m[0]
name = m[1]
commit = m[2]
market = m[3]
L = [name,price,commit,market]
with open("商品.csv","a",newline="",encoding="gb18030")as f:
writer = csv.writer(f)
writer.writerow(L)
print("第%d页抓取完毕" % i)
i += 1
# 点击下一页
if driver.page_source.find("pn-next disabled") == -1:
driver.find_element_by_class_name("pn-next").click()
time.sleep(2)
else:
print("所有商品爬取完毕")
break
结果显示
注意:chromedriver.exe放到anaconda的Scripts目录下