1, Pool
from multiprocessing import Pool
import os
def worker(arg):
print("begin: %s" % (str(arg)))
if __name__ == "__main__":
po = Pool(10) # 定义进程池,最大进程为10
for i in range(100):
po .apply_async(worker, (i, ))
po.close()
po.join()
当然上述代码为对多线程进行加锁,也可以为了对控制变量读写问题,通过multiprocessing.Lock()对过程进行加锁。
2, Parallel
from joblib import Parallel, delayed
def worker(arg):
print("begin: %s" % (str(arg)))
if __name__ == "__main__":
n_jobs = 15
Parallel(n_jobs=n_jobs)(delayed(woker(i) for i in range(100)))
3, threading
from threading import Thread
def worker(arg):
print("begin: %s" % (str(arg)))
jobs.pop()
if __name__ == "__main__":
n_threads = []
jobs = []
n_jobs = 15
for i in range(100):
while True:
if len(jobs) < n_jobs:
break
thread = Thread(target=worker, args=(i, ))
thread.start()
n_threads.append(thread)
jobs.append(i)
for j in n_threads:
j.join()
注:欢迎指正,欢迎搬砖。