Python中有进程池的概念,相比线程池,更加稳定。在此基础上,进行一个简单封装。
首先写一个工具类ProcessUtil.py
#!/user/bin/env python3
# -*- coding: utf-8 -*-
from multiprocessing.pool import Pool
class TaskAction(object):
def do_in_action(self):
pass
class TaskProcess(object):
def __init__(self, core_size=None):
if core_size is None:
self.pool = Pool()
else:
if type(core_size) is not int:
raise ValueError('core_size can\'t handle type %s ,only int support ' % type(core_size))
self.pool = Pool(core_size)
# 检查task参数有效性
@staticmethod
def _check_task(tasks):
# check tasks type is list ?
if type(tasks) is not list:
raise ValueError('tasks can\'t handle type %s , only list support' % type(tasks))
# check tasks' subset is TaskAction ?
for child_task in tasks:
if not isinstance(child_task, TaskAction):
raise ValueError(
'tasks\' subset can\'t handle type %s , only TaskAction support' % type(child_task))
# 异步执行进程
def execute_task(self, tasks):
apply_results = []
# check params
self._check_task(tasks)
# 添加任务至进程池
for child_task in tasks:
apply_results.append(self.pool.apply_async(child_task.do_in_action))
# 拒绝添加新的Process
self.pool.close()
# 等待所有子进程执行完毕
self.pool.join()
# 封装请求结果
results = []
for res in apply_results:
results.append(res.get())
return results
然后是一个测试case:
#!/user/bin/env python3
# -*- coding: utf-8 -*-
import json
import os
import random
# 进程的试炼
import time
import unittest
from util import DateUtil
from util import ProcessUtil
class ProcessTest(unittest.TestCase):
def setUp(self):
print('============================= %s is ready to start...' % self._testMethodName)
def tearDown(self):
print('============================= %s had finished...' % self._testMethodName)
def test_04(self):
list = []
# 构造任务列
for i in range(100):
user_id = i + 1
access_token = DateUtil.DateParser.parse_stamp_second(DateUtil.DateParser.now()) + user_id + random.randint(
10, 99)
list.append(DemoTest(user_id, access_token))
# 执行任务列
result = ProcessUtil.TaskProcess(3).execute_task(list)
print('All sub processes done.')
print(json.dumps(result))
class DemoTest(ProcessUtil.TaskAction):
def __init__(self, user_id, access_token):
self.user_id = user_id
self.access_token = access_token
def do_in_action(self):
second = random.random() * 3
time.sleep(second)
print('UserId : %s , AccessToken : %s , (%s) ' % (self.user_id, self.access_token, os.getpid()))
return second
if __name__ == '__main__':
unittest.main()
控制台部分打印信息:
UserId : 90 , AccessToken : 1504516412.0 , (15272)
UserId : 88 , AccessToken : 1504516457.0 , (7456)
UserId : 91 , AccessToken : 1504516405.0 , (15272)
UserId : 92 , AccessToken : 1504516425.0 , (7456)
UserId : 89 , AccessToken : 1504516452.0 , (7096)
UserId : 93 , AccessToken : 1504516471.0 , (15272)
UserId : 96 , AccessToken : 1504516435.0 , (15272)
UserId : 97 , AccessToken : 1504516460.0 , (15272)
UserId : 98 , AccessToken : 1504516439.0 , (15272)
UserId : 94 , AccessToken : 1504516455.0 , (7456)
UserId : 95 , AccessToken : 1504516454.0 , (7096)
UserId : 100 , AccessToken : 1504516402.0 , (7456)
UserId : 99 , AccessToken : 1504516426.0 , (15272)
.
----------------------------------------------------------------------
Ran 1 test in 48.824s
从结果上看 进程设置了3个进程 分别在 7456 ,7096,15272 三个pid上运行,提高了效率
最后吐槽下,java中实现线程池好累,要关心池的配置,资源释放,类型校验等等