python DrissionPage 拼多多爬取

打开网页版拼多多

设置本地谷歌路径

path = r'C:\Users\PC\AppData\Local\Google\Chrome\Application\chrome.exe'  # 请改为你电脑内Chrome可执行文件路径
ChromiumOptions().set_browser_path(path).save()

判断是否出现验证

while True:
     if page.eles('xpath://p[text()="点击下方“安全验证”按钮，完成拼多多官方验证"]|//div[text()="拼多多安全验证"]'):
         print('出现验证,请通过验证---')
         page.wait(2)
      else:
         break

完整代码如下

import time

from DrissionPage import WebPage ,ChromiumOptions
from DataRecorder import Recorder

path = r'C:\Users\PC\AppData\Local\Google\Chrome\Application\chrome.exe'  # 请改为你电脑内Chrome可执行文件路径
ChromiumOptions().set_browser_path(path).save()
url = 'https://mobile.yangkeduo.com/search_result.html?search_key=iphone15'
bookName = '拼多多_'+time.strftime("%m-%d-%H-%M", time.localtime()) + ".xlsx"
r = Recorder(bookName)
r.add_data(('url','标题','店铺名'))

def main():

    co = ChromiumOptions().use_system_user_path()

    page = WebPage(driver_or_options=co)
    page.timeout = 1
    page.set.retry_times(5) #连接失败时重连次数
    page.set.retry_interval(3) #连接失败时重连间隔
    page.set.NoneElement_value('')

    page.get(url, show_errmsg=True)
    for item in range(1, 800):
        print(item)
        if page.eles(f'xpath://div[@class="_3glhOBhU"][{item}]//*[contains(text(),"试试搜这些")]'):
            continue
        page.ele(f'xpath://div[@class="_3glhOBhU"][{item}]').click()
        page.wait(2.5)
        while True:
            if page.eles('xpath://p[text()="点击下方“安全验证”按钮，完成拼多多官方验证"]|//div[text()="拼多多安全验证"]'):
                print('出现验证,请通过验证---')
                page.wait(2)
            else:
                break
        info_url = page.url
        title = page.ele('xpath://span[contains(@class,"enable-select")]').text
        shop = page.ele('xpath://div[@style="margin-bottom:.02rem"]').text

        print(info_url, title, shop)
        r.add_data((info_url, title, shop))
        page.back()
        page.wait(1)
        page.run_js("window.scrollBy(0,1000)")
        page.wait(2)

        r.record()


if __name__ == '__main__':
    try:
        main()
    except Exception as e:
        print(e)
        r.record()