项目目录
![在这里插入图片描述](https://i-blog.csdnimg.cn/blog_migrate/eaf89eb6aba348327d887b6136902048.png)
注意事项
- 本来是要再做一个界面在运行时显示正在爬取哪个内容
- log 文件本来是要做一个日志文件,不过其实是单机版本其实好像没什么必要(实际上是嫌太麻烦了)
代码演示
lib 文件夹下 main.py 文件
import os,sys
path = os.path.dirname(os.path.dirname(__file__))
sys.path.append(path)
if __name__ == '__main__':
from bin.JingDongProject import jing_dong_project
jing_dong_project()
bin 文件夹下 JingDongProject.py 文件
from PyQt5.QtGui import *
from PyQt5.QtCore import *
from PyQt5.QtWidgets import *
from PyQt5.QtWidgets import QMainWindow, QApplication,QLabel,QTableWidgetItem,QPushButton,QLineEdit,QGridLayout,QWidget,QTableWidget
from core.UI.Start import Ui_Dialog
class Window(QDialog,Ui_Dialog):
def __init__(self):
super().__init__()
self.setupUi(self)
def setup_ui(self):
pass
class jing_dong_project:
def __init__(self):
app = QApplication([])
window = Window()
window.show()
app.exec_()
core 文件夹下 Spider 文件夹下 spider.py 文件
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.edge.EdgeOptions import EdgeOptions
from conf import Settings
from time import sleep
from lxml import etree
import os
class Spider:
def __init__(self, goods_name, paixu, goods_page, comment_page):
self.goods_name = goods_name
self.paixu = paixu
self.goods_page = goods_page
self.comment_page = comment_page
self.run()
def run(self):
self.start_web()
self.search_goods()
self.Paixu()
self.goods_message()
self.driver.close()
def start_web(self):
"""初始化浏览器"""
options = EdgeOptions()
options.add_arguments([r"--headless", r"--disable-gpu"])
self.driver = webdriver.Edge(capabilities=options.to_capabilities())
self.driver.get(Settings.URL)
sleep(Settings.SLEEP_TIME)
def page_read(self):
"""网页源码解析"""
page = self.driver.page_source
self.tree = etree.HTML(page)
def search_goods(self):
"""输入商品名称,并点击搜索"""
name_input = self.driver.find_element_by_id("key")
name_input.send_keys(self.goods_name)
sleep(Settings.SLEEP_TIME)
click = self.driver.find_element_by_xpath('//*[@id="search"]/div/div[2]/button')
self.driver.execute_script('arguments[0].click()', click)
sleep(Settings.SLEEP_TIME)
def goods_message(self):
"""获取商品信息"""
"""
商品页翻页循环
每次循环获取商品价格、名称等信息(这也是一个循环,因为一页有多个商品)
获取一个商品信息后(即一个循环后)
进入商品详情页获取评论,评论翻页也需要循环
"""
page = 1
while page <= int(self.goods_page):
print(page)
self.driver.execute_script('window.scrollTo(0,document.body.scrollHeight)')
sleep(Settings.SLEEP_TIME)
self.goods_price_name()
self.page_read()
page_list_length = len(self.tree.xpath('//*[@id="J_bottomPage"]/span[1]/a'))
next_page = self.driver.find_element_by_xpath('//*[@id="J_bottomPage"]/span[1]/a[{}]'.format(page_list_length))
self.driver.execute_script('arguments[0].click()', next_page)
page += 1
def goods_price_name(self):
"""爬取商品信息"""
self.page_read()
length = len(self.tree.xpath('//*[@id="J_goodsList"]/ul/li'))
count = 1
while count <= length:
print(count)
self.page_read()
if len(self.tree.xpath('//*[@id="J_goodsList"]/ul/li[{}]/div/div[3]/strong/i/text()'.format(count))) != 0