flask中使用celery做异步任务

需求：

项目中某些接口会执行一些较为耗时的处理，这时候接口的响应速度太慢，会影响系统的并发能力，故使用celery对耗时的代码段进行异步处理，完成后再进行回调

1 安装依赖包

• Flask-Celery == 2.4.3
• Flask-Celery-Helper == 1.1.0
• Flask-Script == 2.0.6 # flask的脚本工具，可以设定在命令行执行的自定义的命令
• celery
• eventlet

2 项目结构

• apps ： 对应项目创建的应用， 可以有多个
• static：存放静态文件
• templates：存放模板
• config.py: 项目配置文件
• manage.py：管理文件，类似于django中的manage.py文件
• mycelery.py：celery初始化文件

3 文件内容

• flask项目应用配置 apps/__init __.py文件

# apps/__init__.py 中利用工厂函数，实现flask应用实例
from flask import Flask
from mycelery import celery
from config import config
from .home import home as home_blueprint


def create_app(config_name):
    app = Flask(__name__)
    # app = Flask(__name__)
    app.config.from_object(config[config_name])
    # 中间部分是你的其他库的初始化部分，省略
    ...
    app.config.from_mapping({"DEBUG": True})  # 设定是否开启DEBUG, 也可在app.run中进行指定

    app.register_blueprint(home_blueprint)

    celery.init_app(app)
    return app

• manage.py管理文件
  该文件调用工厂函数，得到flask实例

# manage.py
from mycelery import celery
import os, sys

BASE_DIR = os.path.abspath(os.path.dirname(__file__))
apps = os.path.join(BASE_DIR, 'apps')
sys.path.insert(0, apps)

from flask_script import Manager
from apps import create_app


app = create_app(os.getenv('FLASK_CONFIG') or 'default')
manager = Manager(app)

if __name__ == '__main__':
    print(os.getenv('FLASK_CONFIG'))
    manager.run()

• mycelery.py文件

# mycelery.py
from flask_celery import Celery
from flask import has_request_context, make_response, request


class CeleryWithContext(Celery):
    def init_app(self, app):
        # super(CeleryWithContext, self).init_app(app)

        task_base = self.Task

        class ContextTask(task_base):

            #: Name of the additional parameter passed to tasks
            #: that contains information about the original Flask request context.
            CONTEXT_ARG_NAME = '_flask_request_context'

            def __call__(self, *args, **kwargs):
                """Execute task code with given arguments."""
                call = lambda: super(ContextTask, self).__call__(*args, **kwargs)

                context = kwargs.pop(self.CONTEXT_ARG_NAME, None)
                if context is None or has_request_context():
                    return call()

                with app.test_request_context(**context):
                    result = call()

                    # process a fake "Response" so that
                    # ``@after_request`` hooks are executed
                    app.process_response(make_response(result or ''))

                return result

            def apply_async(self, args=None, kwargs=None, **rest):
                self._include_request_context(kwargs)
                return super(ContextTask, self).apply_async(args, kwargs, **rest)

            def apply(self, args=None, kwargs=None, **rest):
                self._include_request_context(kwargs)
                return super(ContextTask, self).apply(args, kwargs, **rest)

            def retry(self, args=None, kwargs=None, **rest):
                self._include_request_context(kwargs)
                return super(ContextTask, self).retry(args, kwargs, **rest)

            def _include_request_context(self, kwargs):
                """Includes all the information about current Flask request context
                as an additional argument to the task.
                """
                if not has_request_context():
                    return

                # keys correspond to arguments of :meth:`Flask.test_request_context`
                context = {
                    'path': request.path,
                    'base_url': request.url_root,
                    'method': request.method,
                    'headers': dict(request.headers),
                    'data': request.form
                }
                if '?' in request.url:
                    context['query_string'] = request.url[(request.url.find('?') + 1):]

                kwargs[self.CONTEXT_ARG_NAME] = context

        setattr(ContextTask, 'abstract', True)
        setattr(self, 'Task', ContextTask)


# celery = CeleryWithContext()
celery = Celery()

• config.py文件（也可定义为setting.py文件）

# config.py
class Config(object):
    ...
    # 这块注意看 home.tasks app就对应着我的单个应用，tasks就是下面的文件.
    # 文件名也只能是tasks
    CELERY_IMPORTS = ('home.tasks',)
    CELERY_BROKER_URL = 'redis://hehehehe@localhost'
    # CELERY_RESULT_BACKEND = 'redis://:hehehehe@localhost'
    CELERY_ACCEPT_CONTENT = ['application/json']
    CELERY_TASK_SERIALIZER = 'json'
    CELERY_RESULT_SERIALIZER = 'json'

    @staticmethod
    def init_app(app):
        pass

config = {
    'default': Config
}

• apps/home/views.py文件

# 调用
from . import home
from .tasks import add
from flask import jsonify, request


def func(a, b):
    ...
    add.delay(a, b)
    return


@home.route('/', methods=['POST'])
def index():
    if request.method == 'POST':
        print(123)
        add.delay(3, 7)
    # print(456)
    return jsonify({'w': 1234})

• apps/home/tasks.py文件
  存放任务

from mycelery import celery

import time
@celery.task()
def add(a, b):
    time.sleep(10)
    return str(a + b)

4 启动

• flask启动: python3 manage.py runserver
• celery启动: celery -A manage.celery worker -P eventlet -c 100 -l info
  注： 关于eventlet prefork，自己查文档把

5 运行截图

• celery 启动截图
  

• flask启动截图