fastapi+celery+redis-使用记录

fastapi + celery 使用记录

1.安装对应的包

pip install -i https://pypi.douban.com/simple fastapi celery aioredis 

1.初始化celery

init_celery.py # 初始化celery py文件

# -*- coding:utf-8 -*-
from celery import Celery, platforms
from multiprocessing import cpu_count
platforms.C_FORCE_ROOT = True
celery_redis = f"redis://{127.0.0.1}:{6379}/{0}"
cpu_count_number = cpu_count()
celery_app = Celery(TRAIN_TASK_NAME, broker=celery_redis, backend=celery_redis)
celery_app.conf.update(
    broker_heartbeat=None,
    worker_max_tasks_per_child=5,  # 每20万个任务就销毁work
    task_track_started=True,  # 生产
    CELERYD_FORCE_EXECV=True,   # 非常重要,有些情况下可以防止死锁
    worker_concurrency=cpu_count_number  # 并发worker数
)

task.py  # celery任务 py文件

import asyncio

from celery.result import AsyncResult
from celery.app.control import Control

# celery_app 这个导包是跟我的项目导的
from algorithm_api.celery_task.init_celery import celery_app


def run_async(coro):
    """运行async函数 fastapi中异步函数居多 需要此方法执行异步函数"""
    return asyncio.run(coro)


@celery_app.task
def run_task():
    return run_async(once_func())


async def once_func():
    number = 0
    while True:
        number += 1
        await asyncio.sleep(1)
        if number >= 10:
            break


# 需要自己打点的写法  bind=True
@celery_app.task(bind=True)
def run_check_task(self, params_id: str, other_id: int):
    """self是celery的参数 需要打点就需要这个参数"""
    return run_async(run_func(self, params_id, other_id))


# 在执行函数中添加打点
async def run_func(self, params_id: str, other_id: int):
    """执行函数"""
    print(f"other_id---------------{other_id}")
    # 添加状态
    self.update_state(state="working")
    # 添加数据 mete是个dict
    self.update_state(meta={"all_count": 10086, "run_cout": 1024})
    # 添加状态/添加数据
    self.update_state(state="working", meta={"all_count": 10086, "run_cout": 1024})
    print(f"params_id---------------{params_id}")
    await asyncio.sleep(3)
    self.update_state(state="complete", meta={"all_count": 10086, "run_cout": 1024})
    return True


# 获取打点数据
async def get_celery_data():
    """
    获取celery中的打点数据
    必须要任务的task_id
    """
    params = {"params_id": "10086", "other_id": 10086}  #
    task = run_check_task.apply_async(kwargs=params)
    celery_task = AsyncResult(id=task.id)
    while True:
        celery_status = celery_task.status
        celery_result = celery_task.result or {}
        print(f"celery_status---------------{celery_status}")
        print(f"celery_result---------------{celery_result}")
        if celery_status == "complete":
            # 获取任务结果 timeout 超时时间
            try:
                result = celery_task.get(timeout=60)
                print(f"result--------------------{result}")
            except Exception as e:
                print(f"error:->->->->{str(e)}")
            break
    # 任务结束 杀掉celery worker
    celery_control = Control(app=celery_app)
    celery_control.revoke(task_id=task.id, terminate=True)
    return True

暂时先写到这 后面学习了再补充

fastapi初始化 aioredis初始化不写了