双十一商品优惠打折，用Python爬取商品信息分析发现这里套路真深-CSDN博客

咱们今天就以京东笔记本电脑为例分析一下

首先咱们针对京东商城笔记本的网页进行分析，这回只要在网页源码上分析，就可以获取笔记本价格、标题、评论数、商家名称、商家性质。

爬取代码

from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.by import By
from selenium import webdriver
from bs4 import BeautifulSoup
import pymongo
import time
# 连接数据库
client = pymongo.MongoClient(host='localhost', port=27017)
db = client.JD_products
collection = db.products
# 启动浏览器
browser = webdriver.Chrome()
wait = WebDriverWait(browser, 50)
def to_mongodb(data):
# 存储数据信息
try:
collection.insert(data)
print("Insert The Data Successfully")
except:
print('Insert The Data Failed')
def search():
browser.get('https://www.jd.com/')
try:
# 查找搜索框及搜索按钮，输入信息并点击按钮
input = wait.until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, "#key")))
submit = wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "#search > div > div.form > button")))
input[0].send_keys('笔记本')
submit.click()
# 查找笔记本按钮及销量按钮，依次点击按钮
button_1 = wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "#J_selector > div:nth-child(2) > div > div.sl-value > div.sl-v-list > ul > li:nth-child(1) > a")))
button_1.click()
button_2 = wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "#J_filter > div.f-line.top > div.f-sort > a:nth-child(2)")))
button_2.click()
# 获取总页数
page = wait.until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, '#J_bottomPage > span.p-skip > em:nth-child(1) > b')))
return page[0].text
except TimeoutException:
search()
def next_page(page_number):
try:
# 滑动到网页底部，加载出所有商品信息
browser.execute_script("window.scrollTo(0, document.body.scrollHeight);")
time.sleep(10)
html = browser.page_source
parse_html(html)
# 当网页到达100页时，下一页按钮失效，所以选择结束程序
while page_number == 101:
exit()
# 查找下一页按钮，并点击按钮
button = wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, '#J_bottomPage > span.p-num > a.pn-next > em')))
button.click()
wait.until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, "#J_goodsList > ul > li:nth-child(60)")))
# 判断翻页成功
wait.until(EC.text_to_be_present_in_element((By.CSS_SELECTOR, "#J_bottomPage > span.p-num > a.curr"), str(page_number)))
except TimeoutException:
return next_page(page_number)
def parse_html(html):
"""
解析商品列表网页
"""
data = {}
soup = BeautifulSoup(html, 'html.parser')
goods_info = soup.select('.gl-item')
# 查看当前页商品数量，了解是否还有未加载的商品
quantity = 'item: ' + str(len(goods_info))
print(quantity)
for info in goods_info:
# 获取商品标题信息
title = info.select('.p-name.p-name-type-2 a em')[0].text.strip()
title = title.replace('爱心东东', '')
print("title: ", title)
data['_id'] = title
# 获取商品价格信息
price = info.select('.p-price i')[0].text.strip()
price = int(float(price))
print("price: ", price)
data['price'] = price
# 获取商品的评论数量
commit = info.select('.p-commit strong')[0].text.strip()
commit = commit.replace('条评价', '')
if '万' in commit:
commit = commit.split("万")
commit = int(float(commit[0]) * 10000)
else:
commit = int(float(commit.replace('+', '')))
print("commit: ", commit)
data['commit'] = commit
# 获取商品的商店名称
shop_name = info.select('.p-shop a')
if (len(shop_name)) == 1:
print("shop_name: ", shop_name[0].text.strip())
data['shop_name'] = shop_name[0].text.strip()
else:
print("shop_name: ", '京东')
data['shop_name'] = '京东'
# 获取商品的商店属性
shop_property = info.select('.p-icons i')
if (len(shop_property)) >= 1:
message = shop_property[0].text.strip()
if message == '自营':
print("shop_property: ", message)
data['shop_property'] = message
else:
print("shop_property: ", '非自营')
data['shop_property'] = '非自营'
else:
print("shop_property: ", '非自营')
data['shop_property'] = '非自营'
to_mongodb(data)
print(data)
print("

")
def main():
total = int(search())
print(total)
for i in range(2, total+2):
time.sleep(20)
print("第", i-1, "页：")
next_page(i)
if __name__ == "__main__":
main()
复制代码

虽然一开始就是以笔记本这个关键词去搜索，但是这里还是需要再点击一次笔记本按钮，这是因为直接搜索笔记本会出现平常上课做笔记的那种笔记本，导致会获取无用信息。所以利用京东自身更加详细的归类，得到我们想要的信息。

其中每一个网页有60条商品数据，那么按道理应该有6000条的笔记本商品信息，但是最后却只获取了5992条。

估计两个原因：

1、在MongoDB中商品的标题为主键，商品标题出现重复

2、网页未能加载完所有的商品信息

最后成功获取商品信息

读取MongoDB中数据进行可视化分析部分代码

from pyecharts import Bar
import pandas as pd
import pymongo
client = pymongo.MongoClient('localhost', 27017)
db = client.JD_products
table = db.products
df = pd.DataFrame(list(table.find()))
price_info = df['price']
bins = [0, 2000, 2500, 3000, 3500, 4000, 5000, 6000, 7000, 8000, 9000, 10000, 12000, 14000, 16000, 19000, 200000]
level = ['0-2000', '2000-2500', '2500-3000', '3000-3500', '3500-4000', '4000-5000', '5000-6000', '6000-7000', '7000-8000', '8000-9000', '9000-10000', '10000-12000', '12000-14000', '14000-16000', '16000-19000', '19000以上']
price_stage = pd.cut(price_info, bins=bins, labels=level).value_counts().sort_index()
attr = price_stage.index
v1 = price_stage.values
bar = Bar('笔记本价格分布柱状图', title_pos='center', title_top='10', width=800, height=400)
bar.add('', attr, v1, is_stack=True, xaxis_rotate=30, yaxis_min=0, xaxis_interval=0, is_splitline_show=False, is_label_show=True)
bar.render('笔记本价格分布柱状图.html')
复制代码