使用selenium抓取淘宝商品数据

# #使用selenium抓取淘宝商品数据
# # 通过传入关键词  得到要找的关键词商品的网页
# # 解析网页中的商品数据
# #  将解析后的数据保存到mongodb
from selenium import webdriver
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait
from urllib.parse import quote
from pyquery import  PyQuery as pq
import pymongo


# ###无界面模式 爬取的时候不会弹出浏览器
# chrome_options = webdriver.ChromeOptions()
# chrome_options.add_argument('--headless')
# browser = webdriver.Chrome(chrome_options=chrome_options)

chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--headless')
browser = webdriver.Chrome(chrome_options=chrome_options)
# browser = webdriver.Chrome()
wait = WebDriverWait(browser,50)

def get_page(page):

     #https://list.tmall.com/search_product.htm?q=ipad&type=p&vmarket=&spm=875.7931836%2FB.a2227oh.d100&from=mallfp..pc_1_searchbutton

    print('正在爬取第',page,'页')
    KEYWORD = 'ipad'
    try:
        url = 'https://list.tmall.com/search_product.htm?q='+quote(KEYWORD)
        browser.get(url)
        if page>1:
            input = wait.until(
                EC