selenium实现并发

for循环和多线程 + selenium

实例一

for循环

# -*- coding: utf-8 -*-

"""
Datetime: 2019/6/22
Author: Zhang Yafei
Description: 
"""
import time

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from concurrent.futures import ThreadPoolExecutor
import functools


chrome_options = Options()
chrome_options.add_argument("--headless")
chrome_options.add_argument('--disable-gpu')


def timeit(func):
    """
    装饰器： 判断函数执行时间
    :param func:
    :return:
    """

    @functools.wraps(func)
    def inner(*args, **kwargs):
        start = time.time()
        ret = func(*args, **kwargs)
        end = time.time() - start
        if end < 60:
            print(f'花费时间：\t{round(end, 2)}秒')
        else:
            min, sec = divmod(end, 60)
            print(f'花费时间\t{round(min)}分\t{round(sec, 2)}秒')
        return ret

    return inner


class PolicyUrlDownload(object):
    """ 政策数据下载 """

    def __init__(self, url, pages_num, output_file, a_xpath, headless: bool=True):
        self.url_list = [url.format(page) for page in range(1, pages_num+1)]
        self.output_file = output_file
        self.a_xpath = a_xpath
        if headless:
            self.driver = webdriver.Chrome(options=chrome_options)
        else:
            self.driver = webdriver.Chrome()

    def start(self, page, url):
        with open(self.output_file, mode='a', encoding='utf-8') as file:
            print(f"make request to {url}")
            self.driver.get(url)
            titles = self.driver.find_elements_by_xpath(self.a_xpath)
            for title in titles:
                href = title.get_attribute('href')
                file.write(f'{page}\t{href}\n')
            print(f'{url} download completed')

    def run(self):
        for page, url in enumerate(self.url_list):
            self.start(page+1, url)
        self.driver.close()


@timeit
def main(setting):
    policy_data = PolicyUrlDownload(**setting)
    policy_data.run()


if __name__ == '__main__':
    start_time = time.time()
    print('######################## 开始下载 #########################')

    # 多配置页面地址下载
    settings = [
        {
            'output_file': '药品供应保障综合的管理.txt',
            'url': 'http://cdsip.nhfpc.gov.c