Pyppeteer常用技巧

Pyppeteer常用技巧

绕过检测步骤

• 第一步

# 在from pyppeteer import launch 导入前，执行下面代码
from pyppeteer import launcher
launcher.DEFAULT_ARGS.remove("--enable-automation")
from pyppeteer_stealth import stealth   # 先安装pyppeteer_stealth
#####################################################################
# 或者点开源码launcher将‘--enable-automation’ 注释掉
from pyppeteer_stealth import stealth   

• 第二步

# 先进入到搜索页
async def search_page(url,cookie_):
    browser = await launch({'headless': False,'dumpio':True, 'autoClose':False})
    page = await browser.newPage()
    await stealth(page)  # <-- Here
    await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3542.0 Safari/537.36')
    await page.evaluateOnNewDocument('() =>{ Object.defineProperties(navigator,'
                      '{ webdriver:{ get: () => false } }) }')
#     await page.setViewport({"width": 1920, "height": 1080})  # 设置窗口大小
    await page.setViewport({"width": 1100, "height": 768})
    cookie_list = await get_cookie(cookie_)
    for cookie in cookie_list:
        await page.setCookie(cookie)
#     print(cookie_list)
    await page.goto(url)
    # 等待页面加载
    await page.waitForXPath('//div[@class="check_box "]')
    return page

常用方法

• 页面返回

page_text = await page.content()

• 等待页面元素加载

# 等待页面加载,使用xpath方式等待某元素出现
await page.waitForXPath('//div[@class="check_box "]')
# 等待某选择器出现，返回选择器，‘#’ 代表标签id属性,'.'代表标签class属性
await page.waitForSelector('#authImg')

• 输入框输入内容操作

# 使用执行js（evaluate方法）方式清空输入框
await page.evaluate('document.querySelector("#select-key:shenqingh").value=""')
# 输入内容（type方法），第一个参数是输入框选择器属性 ,输入框  #代表id  '.'代表class  第二个参数是输入值
await page.type('#select-key:shenqingh', num)
 # 点击查询
await page.click("#query")

• 截图

# egg 截图某页面上的验证码，根据选择器
yazhengma = await page.waitForSelector('#authImg')
# 根据选择器截图(screenshot方法)，存储以字典形式
await yazhengma.screenshot({'path': 'yazhengma.png'})

• 设置cookie, 和selenium设置cookie类似

# cookie 格式
'''
[{'domain': '.cpquery.cnipa.gov.cn',
  'expiry': 2153740843,
  'httpOnly': False,
  'name': 'UR3ZMlLdcLIE80T',
  'path': '/',
  'secure': False,
  'value': '4ReYPoTfhE8XU8tWBTzIJNqujckzb1ZfkSs3HZNOlhxzOsDfyC8JRlKexOUy.mK9u5bSgRIS8hRnZPIW3C8c_a9YliJfuQX7ZKI7gwK6KGNgnLyKuks2jcwfqYdg5ERFwADXj0gJ6hPvGlO6V2UV_wk_gzqcSOpsBN2jh2kBz5bEIFxfktH5PzKJRnpDBml5u4OoQMRUhzvyVVnDLiSHnqt1I60OhoMrEAuFcjl.eJbal3.cJrnvm9wlRgSz0YRrUw60O2udav7F8PZz3g8qcGBOO76lYR9mVFtJlSNNdewpSDP2G_kEwUlxBVwb65sVHPxRN0C1H.r6qURabLfzzbTpVzZXb8rAvptSmovQttf5YoA'},
 {'domain': '.cpquery.cnipa.gov.cn',
  'httpOnly': False,
  'name': '_gscs_930750436',
  'path': '/',
  'secure': False,
  'value': '38407965zb8ett35|pv:2'},
 {'domain': '.cpquery.cnipa.gov.cn',
  'httpOnly': False,
  'name': 'bg6',
  'path': '/',
  'secure': False,
  'value': '384|A8R2v'},
 {'domain': '.cpquery.cnipa.gov.cn',
  'httpOnly': False,
  'name': 'UR3ZMlLdcLIE83T',
  'path': '/',
  'secure': False,
  'value': '4bg6QmpZtyIgQPzflQesXqaw5Vpns_EpqgO5UM2HwmRPegQCdj847WIMwSm9f.HL.bPxEp0Oab6kHo5v56yQPNb5rEbDoQBuqHWZ8mL8rUU3NIbT6EValIiLcD2DjTSxGBeHwVHM1P9UsYW4fXbtFYtSNRGGx3HLjPu7uT5nVuCyNVTMrQs_QTJhByLyj1bn0LRlrJItt1JgrQ.eEwBAm4Kp.z8akO_uwhobTbLenvVAVxk.kosi_iStsK_5ZjcdIUNKp3kKLUun8TZ_M3PtUTDE..qhL06PgNBZROjfDbDA.kWsLnR1SFHC_fKqeg9DSvvEqU0FBNLcwaSnYpIDe0_jMdpHCakWAE.VpZjQzi39ytpKLVbTO_z0xEaUy133ZzEV'},
 {'domain': '.cpquery.cnipa.gov.cn',
  'httpOnly': False,
  'name': 'JSESSIONID',
  'path': '/',
  'secure': False,
  'value': '8b98d1eba495b3b5712735dcb4d5'},
 {'domain': '.cpquery.cnipa.gov.cn',
  'httpOnly': False,
  'name': 'UR3ZMlLdcLIE83S',
  'path': '/',
  'secure': False,
  'value': 'tiIZjLZOZ9efKrNyHrF9bWq8SkX6mzf3cj7QUhCAMw6nFmofbL_wRL.5nkPFGTUU'},
 {'domain': '.cpquery.cnipa.gov.cn',
  'httpOnly': False,
  'name': '_gscu_930750436',
  'path': '/',
  'secure': False,
  'value': '33768864tauoxr35'},
 {'domain': 'cpquery.cnipa.gov.cn',
  'expiry': 1953769688,
  'httpOnly': False,
  'name': 'UR3ZMlLdcLIE80T',
  'path': '/',
  'secure': False,
  'value': '4JgVnsaGMQMBotHJdfL8wemn_KD7CDyGs2KRIL7PVIW7uokGNPMWxAsZR61g4ps2OjOaLJFaiIQu7CFJEPMkj9fVVKscrkOvhigVsgF4bATZQ07NUMmcaXKvhdCm89Kx5vhSyNVDIHSm.Z.is2tS9HwUG6yPrbKR9UrKZ1tJuWMcINxfK7RfN1fcftR7_POcXmQgF4pMY8GpKCd_M3oRBbw3VDYSxPO.NGGK0VOYHoEPhqJqAEENUEztie.ZdOtDaMpPDiEs1Qrh2fhUk_B9N..bf2md5LsfApiaTdOi9LwJ7Esg.i23xuIiW7cw2cLvc_wG'},
 {'domain': '.cpquery.cnipa.gov.cn',
  'httpOnly': False,
  'name': '_gscbrs_930750436',
  'path': '/',
  'secure': False,
  'value': '1'},
 {'domain': '.cpquery.cnipa.gov.cn',
  'httpOnly': False,
  'name': 'UR3ZMlLdcLIE80S',
  'path': '/',
  'secure': False,
  'value': 'bmK.oV43x2p_uorGA2a8RkbLAERicnQcqpl8YMZ1OiH3RL.Gbh4QAOyqLMMSE3Mk'}]
'''
# pyppeteer 设置cookie，需要遍历上面cookie列表插入
for cookie in cookie_list:
    await page.setCookie(cookie)

# selenium设置 ,也需要一个一个添加
for i in cookie_list:
    bro.add_cookie(i)