Python初级爬虫（利用多任务协程爬取虎牙MM图片）

Python多任务协程下载虎牙直播MM图片

# coding = utf-8
import re
import gevent
from gevent import monkey, pool
import time, random
import urllib.request

monkey.patch_all()


def downloader_img(img_url, img_name):
    """下载图片"""
    req = urllib.request.urlopen(img_url)
    img_content = req.read()
    # 下载图片保存
    with open(r"/home/python/Desktop/01_Python基础/day03/正则表达式/photo/mm%s" % img_name, "wb") as f:
        f.write(img_content)
        time.sleep(random.random())


def main():
    p1 = pool.Pool(10)  # 加入协程
    # 读取文件
    locals_addr = r"/home/python/Desktop/01_Python基础/day03/正则表达式/1.txt"
    url_txt = open(locals_addr, "r", encoding=" utf-8")
    url_content = url_txt.read()
    jpg_lists = re.findall(r"https://.*?\.jpg", url_content)  # 正则匹配网页的有效地址
    num = 0
    list1 = []  # 定义一个列表用来存网址
    # 遍历图片列表
    for j in jpg_lists:
        list1.append(gevent.spawn(downloader_img, j, "%d.jpg" % num))
        num += 1
    gevent.joinall(list1)


if __name__ == '__main__':
    main()

方法
主函数
效果