背景
有一个需求,是celery异步任务中使用多线程,同时需要对总体数量进度的更新。
描述
import threading
from celery import Celery
import celery
app = Celery(
broker='redis://:@127.0.0.1:6379/1',
backend='mongodb://test:{password}@{host}:{port}/test_map?authSource=test_map'.format(
password='test',
host='127.0.0.1',
port=27017,
)
)
lock = threading.Lock()
def run_subtask(celery_task, i):
lock.acquire()
#Error raises here, when update_state calls
# 这里会无法更新
celery_task.update_state(state=celery.states.SUCCESS, meta={'subtask_id': i})
lock.release()
@app.task(bind=True, ignore_result=True)
def get_info(self):
workers = []
for i in range(4):
workers.append(threading.Thread(target=run_subtask, args=(self, i)))
for worker in workers:
worker.start()
for worker in workers:
worker.join()
# ignore_result=True 加上这句话避免 return None在数据库里看到null值
查看数据库结果:
没有出现对应的数据记录
调整
def run_subtask(celery_task, i, task_id):
lock.acquire()
#Error raises here, when update_state calls
# celery_task.update_state(state=states.SUCCESS, meta={'subtask_id': i})
# 之前的方法无法在多线程获取到指定的对象,很诡异的无法更新
celery_task.update_state(task_id=task_id, state=celery.states.SUCCESS, meta={'subtask_id': i})
lock.release()
@celery.task(bind=True, ignore_result=True)
def get_info(self, user):
task_id = self.request.id
for i in range(4):
worker = threading.Thread(target=run_subtask, args=(self, i, task_id))
worker.start()
查看到数据记录:
符合预期