Python爬取MZT网站：多进程处理

from multiprocessing import Process
import requests
from urllib import request
from bs4 import BeautifulSoup
import re
import os
import time

def run(name):
    print(">>>启动子进程：" + str(os.getpid()))
    count = 1
    headers = {'User-Agent': 'Mozilla/5.0', "referer": "https://www.*****.com/"}
    surl = "https://www.*****.com/"+name
    r = requests.get(surl, headers=headers)
    sum_init = re.findall(r'https://www.*****.com/'+str(name)+'/page/(.*?)/', r.text)
    sum = int(sum_init[3])
    for n in range(1,int(sum)+1):
        print(name+"正在运行！")
        url_1 = "https://www.*****.com/"+name+"/page/"+str(n)
        #获取主页面HTML
        html_1 = requests.get(url_1,headers = headers)
        #得到每组图片的url_2页面
        url_2 = re.findall(r'''href="(.*?)" target="_blank"><img class='lazy''',html_1.text)
        for i in range(len(url_2)):
            try:
                #获取每组图片的总html,html_2
                html_2 = requests.get(url_2[i],headers = headers)
                #获取每组图片的各自的总数量number
                number = re.findall(r''+str(url_2[i])+'/(\d\d)',html_2.text)
                for j in range(1,int(number[0])+1):
                    #构造每张图片所在网址的url,url_3
                    url_3 = url_2[i]+"/"+str(j)
                    #得到每张图片的所在的html,html_3
                    html_3 = requests.get(url_3,headers = headers)
                    #获取每张图片的url,p_url
                    p_url = re.findall(r'''img src="(.*?)"''',html_3.text)
                    print(name+"第"+str(i+1)+"组:"+p_url[0])
                    picture = requests.get(p_url[0], headers=headers)
                    path = "D:/mzt/"+name+"/"+str(n)+"_"+str(i+1)+"_"+str(j)+"_"+str(count)+".jpg"
                    with open(path, 'wb') as f:
                        f.write(picture.content)
                        count = count + 1
            except:
                print("Got a Exception!")

if __name__ == '__main__':
    print(">>>启动主进程：" + str(os.getpid()))
    arry = ["xinggan","japan","taiwan","mm"]
    for i in range(0,4):
        p = Process(target=run, args=(arry[i],))
        p.start()
    #子进程结束后主进程才结束p.join
    p.join()
    print("主进程结束")

Tip：因为mei zi tu 网站反爬虫，所以比普通爬虫多加一些信息headers中多加一点："referer": "https://www.mzt.com/"

Tip：另外熟悉一下多进程处理能力