pyppeteer使用样例脚本

代码功能介绍:

1. 完成在http://iir.circ.gov.cn/ipq/ipiQuery.html页面的自动化查询
2. 使用百度AI的开放API来识别验证码, 此处已经封装, 不建议直接复制使用.

代码如下:

import asyncio
from pyppeteer import launch
import os
import sys
import inspect
import arrow
import requests
from requests.cookies import RequestsCookieJar
from datetime import datetime
import json
import re
common_dir = os.path.realpath(os.path.abspath(os.path.join(
    os.path.split(inspect.getfile(inspect.currentframe()))[0],
    "../../../")))
if common_dir not in sys.path:
    sys.path.insert(0, common_dir)
# import logging
# logging.basicConfig(level=logging.DEBUG,
#                     format='%(asctime)s - %(filename)s[line:%(lineno)d] - %(levelname)s: %(message)s')
from ai.AIfactory.AIfactory import AIfactory


class AgentQuery():
    def __init__(self, *args, **kwargs):
    	# 由于使用百度ai的API, 此处已经封装
        self.ai = AIfactory.create_ai(ai_company='baidu')(**kwargs)
        self.session = requests.Session()
        self.session.headers = {
            "User-Agent":'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.75 Safari/537.36',
        }
        self.jar = RequestsCookieJar()
        self.host = 'http://iir.circ.gov.cn/ipq/ipiQuery.html'   # 首页面
        self.captcha_url = 'http://iir.circ.gov.cn/ipq/captchacn.svl'  # 验证码URL
        self.base_url = 'http://iir.circ.gov.cn/ipq/ipiqueryru.html'  # 查询后的列表页
        self.baseinfo_url = 'http://iir.circ.gov.cn/ipq/ipiqueryru.do?validate' # 查询预览页面的接口
        self.detail_url = 'http://iir.circ.gov.cn/ipq/ipiqueryru.do?query' # 查询详细的数据接口

async def main():
    result = {}
    obj = AgentQuery(choice='general_basic')
    # headless参数设为False，则变成有头模式
    browser = await launch(
        ignoreHTTPSErrors=True,
        headless=False,
        autoClose=False
    )
    page = await browser.newPage()
    
    # 设置页面视图大小
    await page.setViewport(viewport={'width':1280, 'height':800})
    
    # 是否启用JS，enabled设为False，则无渲染效果
    await page.setJavaScriptEnabled(enabled=True)
    await page.goto(obj.host)
    await page.setUserAgent(obj.session.headers.get('User-Agent'))
    await page.type('input#agencyname','世纪保险经纪股份有限公司上海分公司')
    cookies = await page.cookies()
    for cookie in cookies:
        obj.jar.set(cookie['name'], cookie['value'])
    response2 = obj.session.get(obj.captcha_url, cookies=obj.jar)
    img_path = os.path.join(common_dir,'agencycaptchacn.png')
    with open(img_path, 'wb') as f:
        f.write(response2.content)
    captcha = obj.ai.get_result(img_path)
    print('识别后的验证码是', captcha)
    if captcha is None or len(captcha)!=4:
        await main()
    else:
        await page.type('input#yzm',captcha)
        await page.click('#chaxun')
        await page.waitFor(3000)
        await page.reload()  # 必须项, 否则在browser.pages()中只有两个页面
        page_list = await browser.pages()
        page = page_list[-1]
        if page.url == obj.base_url:
            content = await page.content()
            cookies = await page.cookies()
                
            not_found_info = await page.Jx('//*[@id="interme"]/tr[1]/td')
            if not_found_info:
                not_found_info = await ( await not_found_info[0].getProperty('textContent')).jsonValue()
                if '很抱歉，没有找到符合条件的记录' in not_found_info:
                    return result
            # 以下已经基本信息采集
            agentname = await page.Jx('//*[@id="interme"]/tr[1]/td[2]/a')
            if agentname:
                agentname = await ( await agentname[0].getProperty('textContent')).jsonValue()
            agencycode = await page.Jx('//*[@id="interme"]/tr[1]/td[3]')
            if agencycode:
                agencycode = await ( await agencycode[0].getProperty('textContent')).jsonValue()
            regulatorycode = await page.Jx('//*[@id="interme"]/tr[1]/td[4]')
            if regulatorycode:
                regulatorycode = await ( await regulatorycode[0].getProperty('textContent')).jsonValue()
            registertime = await page.Jx('//*[@id="interme"]/tr[1]/td[5]')
            if registertime:
                registertime = await ( await registertime[0].getProperty('textContent')).jsonValue()
            result.setdefault('agentname', agentname)
            result.setdefault('agencycode', agencycode)
            result.setdefault('regulatorycode', regulatorycode)
            result.setdefault('registertime', registertime)
            return result
            # 以下基于API研究开发, 可以使用,也可以不使用
            org_no = await page.Jx('//*[@id="interme"]/tr[1]/td[2]/a')
            if org_no:
                org_no = await ( await org_no[0].getProperty('href')).jsonValue()
                org_no_list = re.findall(r"\".*\"", str(org_no))
                if org_no_list:
                    org_no = re.sub(r'\"', '', org_no_list[0])
                    for cookie in cookies:
                        obj.jar.set(cookie['name'], cookie['value'])
                    response3 = obj.session.post(
                        obj.detail_url,
                        data = {
                            'org_no': org_no
                        },
                        cookies=obj.jar
                    )
                    print(response3.text)

if __name__=='__main__':
    asyncio.get_event_loop().run_until_complete(main())