#!/usr/bin/env python
# coding=utf-8
# 自动获取京东热卖水果排行榜信息,url地址是通过js加载生成的无法获取
from selenium import webdriver
import time,re,os
from debug_info import Log_info
import openpyxl
from openpyxl import Workbook
from selenium.webdriver import ChromeOptions # 更改Chrome配置达到反爬机制
class Saas(object):
# 初始化工作
def __init__(self):
# self.driver = webdriver.Chrome()
# 反爬机制代码开始,采用此代码在F12控制台输入window.navigator.webdriver结果不是True,而是undefined就成功了
option = ChromeOptions()
option.add_experimental_option('excludeSwitches', ['enable-automation'])
self.driver = webdriver.Chrome(options=option)
# 反爬机制代码结束
self.driver.maximize_window() # 将浏览器最大化
# 本地表格名称,如果存在则加载
self.file_name = "./jd新鲜水果榜.xlsx"
self.wb = Workbook()
if os.path.exists(self.file_name):
print("存在表格")
self.wb = openpyxl.load_workbook(self.file_name)
self.top = ['序号', '名称', '累计购买人数', '近24小时售出(件)', '价格', '获取时间','链接地址']
# 创建表头
# self.ws.append(self.top)
# 转换数字类型
def changeNum(self,num):
if "." in num:
result = float(num)
else:
result = int(num)
return result
def jd(self, logger):
# 先做模拟登陆
self.driver.get('https://ranking.m.jd.com/comLandingPage/comLandingPage?contentId=9294&sku=%27%27&rankType=10')
time.sleep(2)
# target_elem = self.driver.find_element_by_link_text("更多商品")
# js = 'document.getElementsByClassName("hide-vertical-indicator")[0].scrollTop=1000'
# self.driver.execute_script(js)
result_date = []
# patt = re.compile(r'\d+\.*\d+')
patt = re.compile(r'([1-9]+[0-9]*|0)(\.[\d]+)?')
# 获取时间
now_time = time.strftime("%Y-%m-%d", time.localtime())
for i in range(1,5):
logger("数据[%s]获取中..."%i)
result = []
title = self.driver.find_elements_by_xpath('//*[@id="hello"]/div/div/div/div['+str(i+2)+']/div/div/div[3]/div[1]/div[1]/span')[0].text
title = title.replace("自营","")
# 累计购买人数
buy_count = self.driver.find_elements_by_xpath('//*[@id="hello"]/div/div/div/div['+str(i+2)+']/div/div/div[3]/div[1]/div[2]/div[1]/span')[0].text
# 近24小时售出,如果购买人数不存在,则24小时售出会代替其位置
try:
buy_sale = self.driver.find_elements_by_xpath(
'//*[@id="hello"]/div/div/div/div[' + str(i + 2) + ']/div/div/div[3]/div[1]/div[2]/div[2]/span')[0].text
except:
logger("Warning:注意第%s个"%i)
buy_sale = ''
if "累计" in buy_count:
patt_re = patt.findall(buy_count)[0]
patt_result = patt_re[0] + patt_re[1]
if "万" in buy_count:
buy_count = round(float(patt_result)*10000)
else:
buy_count = self.changeNum(patt_result)
elif "24小时" in buy_count:
buy_sale = buy_count
buy_count = "暂无数据"
else:
buy_count = "暂无数据"
print("购买人数:", buy_count)
if "24小时" in buy_sale:
patt_re = patt.findall(buy_sale)[1]
patt_result = patt_re[0] + patt_re[1]
if "万" in buy_sale:
buy_sale = round(float(patt_result)*10000)
else:
buy_sale = int(patt_result)
else:
buy_sale = "暂无数据"
print("近24小时售出:", buy_sale)
# 价格
try:
price = self.driver.find_elements_by_xpath('//*[@id="hello"]/div/div/div/div['+str(i+2)+']/div/div/div[3]/div[2]/div/span')[0].text
price = price.replace("¥","")
price = self.changeNum(price)
except:
price = "暂无定价"
print("价格:",price)
url = "https://item.m.jd.com/product/*.html"
result.append(i)
result.append(title)
result.append(buy_count)
result.append(buy_sale)
result.append(price)
result.append(now_time)
result.append(url)
result_date.append(result)
time.sleep(1)
target_elem = self.driver.find_element_by_xpath("//*[text()='" + str(i) + "']")
self.driver.execute_script("return arguments[0].scrollIntoView();", target_elem) # 用目标元素参考去拖动
time.sleep(1)
self.ws = self.wb.create_sheet(now_time)# 默认插在最后
for row in range(len(result_date)):
for col in range(len(self.top)):
if result_date[row][col]:
value = result_date[row][col]
else:
value = ""
self.ws.cell(row=1, column=col + 1, value=self.top[col])
self.ws.cell(row=row + 2, column=col + 1, value=value)
time.sleep(1)
self.wb.save('jd新鲜水果榜.xlsx')
logger("全部处理完成!")
# def tearDown(self):
# # 退出
# logger("以上数据全部处理完毕!")
# # time.sleep(15)
# # 这里就不退出了
# # self.driver.quit()
if __name__ == "__main__":
# 将输出结果同时输出到屏幕和保存到文件中
logger = Log_info().main()
time.sleep(2)
# 最后再执行自动化方法
saas = Saas()
try:
saas.jd(logger)
except Exception as e:
logger("出现异常:", e)