爬取马蜂窝热门游记的标题与简介

最新推荐文章于 2024-06-28 14:57:50 发布

岛上书店_

最新推荐文章于 2024-06-28 14:57:50 发布

阅读量97

点赞数

文章标签： python 爬虫 selenium 测试用例测试工具

本文链接：https://blog.csdn.net/2301_78671272/article/details/133948562

版权

一.工具

selenium,python,谷歌浏览器,xpath插件

二.代码如下

from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
import time

s = Service("chromedriver.exe")
o = webdriver.ChromeOptions()
o.add_argument("start-maximized")
o.add_experimental_option("excludeSwitches", ["enable-automation"])
o.add_experimental_option('useAutomationExtension', False)
o.add_argument("--disable-blink-features")
o.add_argument("--disable-blink-features=AutomationControlled")
# o.add_argument('--headless')
o.add_argument('--no-sandbox')  # fix:DevToolsActivePort file doesn't exist
o.add_argument('--disable-gpu')  # fix:DevToolsActivePort file doesn't exist
o.add_argument('--disable-dev-shm-usage')  # fix:DevToolsActivePort file doesn't exist
o.add_argument('--remote-debugging-port=9222')  # fix:DevToolsActivePort file doesn't
wd = webdriver.Chrome(options=o, service=s)
wd.execute_script("Object.defineProperty(navigator, 'webdriver', {get: () => undefined})")
wd.execute_cdp_cmd('Network.setUserAgentOverride', {"userAgent": 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.53 Safari/537.36'})
print(wd.execute_script("return navigator.userAgent;"))
wd.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {
  "source": """
    Object.defineProperty(navigator, 'webdriver', {
      get: () => undefined
    })
  """
})
wd.get('https://www.mafengwo.cn/')


# 等待页面的加载
time.sleep(3)


# 获取旅游的标题定位
title = wd.find_elements(By.XPATH,"//dl/dt/a[@target='_blank']")
content = wd.find_elements(By.XPATH,"//div[@class='tn-wrapper']/dl/dd/a[@target='_blank']")
# 获取标题内容
for i in range(len(title)):
  print("*"*50)
  print(title[i].text)
  print(content[i].text)


# 循环点击下一页
for j in range(8):
    # 获取下一页
    nextPage = wd.find_elements(By.XPATH, "//div[@id='_j_tn_pagination']/a[@class='pi _j_pageitem']")
    nextPageli = wd.find_elements(By.XPATH, "//div[@id='_j_tn_pagination']/a[@class='pi _j_pageitem']")
    nextPage[j].click()
    time.sleep(3)

    # 重新获取元素
    title = wd.find_elements(By.XPATH, "//dl/dt/a[@target='_blank']")
    content = wd.find_elements(By.XPATH, "//div[@class='tn-wrapper']/dl/dd/a[@target='_blank']")

    # 使用新的元素列表
    for k in range(len(title)):
      print("*" * 50)
      print(title[k].text)
      print(content[k].text)




print("程序结束了")
time.sleep(10)