多进程爬取

import requests
from lxml import etree
import re
import time
from multiprocessing import Pool  #导入multiprocessing库的Pool模块

headers = {'user-agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.92 Safari/537.36'}

def get_info(url):
    html = requests.get(url,headers = headers)
    selector = etree.HTML(html.text)
    names = selector.xpath('//*[@class="article block untagged mb15 typs_hot"]/div[1]/a[2]/h2/text()')
    centents = re.findall('<div class="content">.*?<span>(.*?)</span>',html.text,re.S) #第一个正则是为了匹配换行符
    laughs = re.findall('<span class="stats-vote"><i class="number">(\d+)</i>',html.text,re.S)
    comments = re.findall('<i class="number">(\d+)</i> 评论',html.text,re.S)
    for name,centent,laugh,comment in zip(names,centents,laughs,comments):
        info = {
            'name':name,
            'centent':centents,
            'laugh':laughs,
            'comment':comments
        }
        return (info)

if __name__ == '__main__':
    urls = ["https://www.qiushibaike.com/text/page/{}/".format(num)for num in range(0,14)]
    start_1 = time.time()
    for url in urls:
        get_info(url)
    end_1 = time.time()
    print('串行爬取花费时间：' + str(end_1 - start_1))

    start_2 = time.time()
    pool = Pool(processes=2)          #创建进程池，processes为设置的进程个数
    pool.map(get_info,urls)  #利用map()函数运行进程，参数fuc为运行的函数，iterable为迭代参数
    end_2 = time.time()
    print('2个进程：' + str(end_2 - start_2))

    start_3 = time.time()
    pool = Pool(processes=4)  # 创建进程池，processes为设置的进程个数
    pool.map(get_info, urls)  # 利用map()函数运行进程，参数fuc为运行的函数，iterable为迭代参数
    end_3 = time.time()
    print('4个进程：' + str(end_3 - start_3))

输出：

D:\Python\venv\Scripts\python.exe D:/Python/venv/test12.py
串行爬取花费时间：5.043288469314575
2个进程：3.351191759109497
4个进程：2.882164716720581

Process finished with exit code 0

转载于:https://blog.51cto.com/12884584/2358703