利用selenium获取京东商城搜索信息

<! -/* author by:daidai */>
京东商品信息爬取脚本
from selenium.webdriver import Chrome
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from urllib.parse import urlencode
import mongodemo
import time

driver = Chrome()
chrome_options = Options()
chrome_options.add_experimental_option(‘excludeSwitches’, [‘enable-automation’])

keyword = ‘电脑’

京东搜索页的链接url = https://search.jd.com/Search?keyword=电脑&enc=utf-8&wq=电脑&pvid=53766f90843b4166be83aa2139402e65
par = {‘enc’: ‘utf-8’, ‘keyword’: keyword, ‘wq’: keyword}

kw = urlencode(par)

url = 'https://search.jd.com/Search?’ + kw
请求搜索页
driver.get(url)

等待页面加载
driver.implicitly_wait(5)

由于页面有懒加载效果，所以先获取页面的高度
height = driver.execute_script(“return document.body.clientHeight”)

等待页面加载完成
driver.implicitly_wait(7)

由分析知道，页面有懒加载效果，得知一页数据为60条
def get_data():
# 获取包裹商品信息的大标签ul
ul = driver.find_element_by_class_name(‘gl-warp’)
# 商品列表 在标签li class = "gl-item"中， 完整一页一共60个
items = ul.find_elements_by_class_name(‘gl-item’)

if len(items) == 60:
    return items
# 如果没有获取到60条就递归调用，直到获取该页完整的数据
else:
    return get_data()

def parser_data(items):
for item in items:
# 商品的详细信息都在标签div class=p-img 下 它的下面的a标签下是图片的信息
link = item.find_element_by_css_selector(".p-img a").get_attribute(“href”)
# 图片地址
img = item.find_element_by_css_selector(".p-img a img").get_attribute(“src”)
# 因为在图片中也使用了懒加载，所以做判断
if not img:
img = item.find_element_by_css_selector(".p-img a img").get_attribute(“data-lazy-img”)
img = “https:” + img
# 商品价格
price = item.find_element_by_css_selector(".p-price i").text
# 商品的标题
title = item.find_element_by_css_selector(".p-name a em").text
# 店家名
trade_name = item.find_element_by_css_selector(".p-shop a").text
# 评论数
commit = item.find_element_by_css_selector(".p-commit").text
data = {“link”: link, “img”: img, “title”: title, “shop_name”: trade_name, “commit”: commit, “price”: price}
# 存到数据库
mongodemo.insert_data(data)

def get_next_page():
# 获取下一页的数据，获取标签
next_page = driver.find_element_by_partial_link_text(“下一页”)
next_page.click() # 实现点击事件，跳转到下一页
# 滑动屏幕
driver.execute_script("""
window.scrollTo({
top: %s,
behavior: “smooth”
});""" % height)
time.sleep(1) # 等待页面加载
items = get_data()
parser_data(items)

if name == ‘main’:
keyword = input(‘请输入要查找的信息：’)
par = {‘enc’: ‘utf-8’, ‘keyword’: keyword, ‘wq’: keyword}
kw = urlencode(par)
url = 'https://search.jd.com/Search?’ + kw
driver.get(url)
driver.implicitly_wait(10)
height = driver.execute_script(“return document.body.clientHeight”)
driver.implicitly_wait(10)
# 首页的处理 第一页的信息爬取
driver.execute_script("""
window.scrollTo({
top: %s,
behavior: “smooth”
});""" % height)
items = get_data()
parser_data(items)

for i in range(8):
    get_next_page()

time.sleep(30)
driver.close()

利用mongo存储数据

from pymongo import MongoClient

table = None
c = None

def connect_server():
global table, c
c = MongoClient(‘mongodb://root:root@127.0.0.1:27017’) # 获取连接
table = c[‘jd’][‘data’]

def insert_data(data):
if not table:
connect_server()
table.insert(data)

def close():
c.close()