# #使用selenium抓取淘宝商品数据
# # 通过传入关键词 得到要找的关键词商品的网页
# # 解析网页中的商品数据
# # 将解析后的数据保存到mongodb
from selenium import webdriver
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait
from urllib.parse import quote
from pyquery import PyQuery as pq
import pymongo
# ###无界面模式 爬取的时候不会弹出浏览器
# chrome_options = webdriver.ChromeOptions()
# chrome_options.add_argument('--headless')
# browser = webdriver.Chrome(chrome_options=chrome_options)
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--headless')
browser = webdriver.Chrome(chrome_options=chrome_options)
# browser = webdriver.Chrome()
wait = WebDriverWait(browser,50)
def get_page(page):
#https://list.tmall.com/search_product.htm?q=ipad&type=p&vmarket=&spm=875.7931836%2FB.a2227oh.d100&from=mallfp..pc_1_searchbutton
print('正在爬取第',page,'页')
KEYWORD = 'ipad'
try:
url = 'https://list.tmall.com/search_product.htm?q='+quote(KEYWORD)
browser.get(url)
if page>1:
input = wait.until(
EC
使用selenium抓取淘宝商品数据
最新推荐文章于 2024-08-08 21:15:25 发布