【Local/docker-compose】安装Celery并启动beat定时任务-20220823

settings.py（redis由container产生，不引用外部redis）

"""
Django settings for GAGA project.

Generated by 'django-admin startproject' using Django 3.2.7.

For more information on this file, see
https://docs.djangoproject.com/en/3.2/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/3.2/ref/settings/
"""

from pathlib import Path
import os
from django.utils.translation import gettext_lazy as _


# 环境变量(含默认值)
REDIS_LOCATION = os.environ.get('REDIS_LOCATION',"redis://redis:6379/1")  # 务必使用redis://redis:6379/1，否则会报错
#REDIS_LOCATION = os.environ.get('REDIS_LOCATION',"redis://10.41.241.169:32621/1")
# REDIS_LOCATION = os.environ.get('REDIS_LOCATION',"redis://106.52.14.84:32222/1")
# REDIS_PASSWORD = os.environ.get('REDIS_PASSWORD','redis@654321')
DJANGO_DEBUG = os.environ.get('DJANGO_DEBUG', True)
# REDIS_LOCATION = "redis://106.52.14.84:32222/1"
# REDIS_PASSWORD = "redis@654321"


# Build paths inside the project like this: BASE_DIR / 'subdir'.
BASE_DIR = Path(__file__).resolve().parent.parent
LOG_DIR = Path(__file__).resolve().parent.parent / 'log'

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/3.2/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = 'django-insecure-+=99f6x#@pq9f5=riae=8skj)6vu55vptpo9-in#yl*+2qfop@'

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = False
DEBUG = DJANGO_DEBUG

ALLOWED_HOSTS = ['*']





# Application definition

INSTALLED_APPS = [
    'simpleui',
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'GAGA_meeting',
    'django_celery_beat',
]


MIDDLEWARE = [
    'django.middleware.security.SecurityMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.locale.LocaleMiddleware',### 多语言中间件
    'django.middleware.cache.UpdateCacheMiddleware',#redis中间件
    'django.middleware.common.CommonMiddleware',
    'django.middleware.cache.FetchFromCacheMiddleware',#redis中间件
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
]

ROOT_URLCONF = 'GAGA.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [os.path.join(BASE_DIR, 'templates')],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    },
]

WSGI_APPLICATION = 'GAGA.wsgi.application'



# Database
# https://docs.djangoproject.com/en/3.2/ref/settings/#databases

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        #'NAME': BASE_DIR / 'db.sqlite3',
        'NAME': Path(__file__).resolve().parent /'SQL' /'db.sqlite3',
    }
}


# Password validation
# https://docs.djangoproject.com/en/3.2/ref/settings/#auth-password-validators

AUTH_PASSWORD_VALIDATORS = [
    {
        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
    },
]

# 修改中文，Django内部设置zh_Hans方法指向中文 
LANGUAGES = [
    ('zh-hans', _('Chinese')),
    ('en', _('English')),
]

LANGUAGE_CODE = 'zh-hans' 
# LANGUAGE_CODE = 'en-us'

LOCALE_PATHS = (
    os.path.join(BASE_DIR, 'locale'),
)

# 修改中国时区
TIME_ZONE = 'Asia/Shanghai'
USE_TZ = False
# USE_TZ = True
USE_I18N = True
USE_L10N = True



# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/3.2/howto/static-files/
# 通过url直接访问我在项目中的静态文件
STATIC_URL = '/static/'
# 部署静态文件时(pyhtonmanage.pycollectstatic)所有的静态文静聚合的目录
STATIC_ROOT = os.path.join(BASE_DIR, "/static/")
# STATICFILES_DIRS告诉django,首先到STATICFILES_DIRS里面寻找静态文件,其次再到各个app的static文件夹里面找
STATICFILES_DIRS = (
    os.path.join(BASE_DIR, "static"),
)

MEDIA_URL = '/media/'
MEDIA_ROOT = os.path.join(BASE_DIR, 'media')



# Default primary key field type
# https://docs.djangoproject.com/en/3.2/ref/settings/#default-auto-field

DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'



LOGIN_URL = '/user_login/'

DATE_FORMAT = 'Y-m-d'





LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    # 'formatters': {
    #     'simple': { # exact format is not important, this is the minimum information
    #         'format': '%(asctime)s %(name)-12s %(lineno)d %(levelname)-8s %(message)s',
    #     },
    # },
    'handlers': {
        'console': {
            'class': 'logging.StreamHandler',
        },

        'mail_admins': { # Add Handler for mail_admins for `warning` and above
            # 'level': 'ERROR',
            'class': 'django.utils.log.AdminEmailHandler',
        },
        'file': {
            'class': 'logging.FileHandler',
            'filename': os.path.join(LOG_DIR, 'admin.log'),
        },

    },

    'root': {
        'handlers': ['console', 'file','mail_admins'],
        'level': 'INFO',
    },
}



# Racher redis
# CACHES = {
#     "default": {
#         "BACKEND": "django_redis.cache.RedisCache",
#         "LOCATION": "redis://10.41.241.169:32621/1",
#         #"LOCATION": REDIS_LOCATION,
#         "OPTIONS": {
#             "CLIENT_CLASS": "django_redis.client.DefaultClient",
#             #"PASSWORD":"000415",
#             #"PASSWORD":REDIS_PASSWORD,
#             "SOCKET_CONNECT_TIMEOUT": 5,  # in seconds
#             "SOCKET_TIMEOUT": 5,  # r/w timeout in seconds
#         }
#     }
# }



# K8S redis
CACHES = {
    "default": {
        "BACKEND": "django_redis.cache.RedisCache",
        "LOCATION": REDIS_LOCATION,
        "OPTIONS": {
            "CLIENT_CLASS": "django_redis.client.DefaultClient",
            #"PASSWORD":"redis@654321",
            "SOCKET_CONNECT_TIMEOUT": 5,  # in seconds
            "SOCKET_TIMEOUT": 5,  # r/w timeout in seconds
        }
    }
}



# CELERY_BROKER_URL = 'redis://10.41.241.169:32621/1'
#CELERY_BROKER_URL =  "redis://106.52.14.84:32222/1"
CELERY_BROKER_URL = REDIS_LOCATION
# CELERY_RESULT_BACKEND = 'redis://10.41.241.169:32621/1'
#CELERY_RESULT_BACKEND =  "redis://106.52.14.84:32222/1"
CELERY_RESULT_BACKEND = REDIS_LOCATION
CELERY_ACCEPT_CONTENT = ['application/json']
CELERY_RESULT_SERIALIZER = 'json'
CELERY_TASK_SERIALIZER = 'json'
CELERY_TIMEZONE = 'Asia/Shanghai'
CELERYD_MAX_TASKS_PER_CHILD = 10
CELERYD_LOG_FILE = os.path.join(BASE_DIR, "logs", "celery_work.log")
CELERYBEAT_LOG_FILE = os.path.join(BASE_DIR, "logs", "celery_beat.log")

DJANGO_CELERY_BEAT_TZ_AWARE = False

1.Local安装Celery

步骤0：安装好redis，并启用。

参考链接https://blog.csdn.net/m0_46629123/article/details/126377540

步骤1：安装好celery和flower，并启用。

https://blog.csdn.net/m0_46629123/article/details/126405866

步骤2：安装好beat，并启用。

使用pip安装包：

pip install django-celery-beat

将django_celery_beat模块添加到INSTALLED_APP中：

INSTALLED_APPS = [
...
'django_celery_beat',
...
]

数据库迁移

python manage.py makemigrations
python manage.py migrate

intervals填写周期

periodic tasks填写定期任务

修改时区为默认时区，否则会报错（后面再解决报错的问题）：

BUG: TypeError: can’t compare offset-naive and offset-aware datetimes

# 修改中国时区
#TIME_ZONE = 'Asia/Shanghai'
#USE_TZ = False
USE_TZ = True
USE_I18N = True
USE_L10N = True

手动启动beat：

celery -A GAGA beat --scheduler django_celery_beat.schedulers:DatabaseScheduler

beat启动后，定时任务在celery执行

2.docker-compose安装Celery并启动beat定时任务

步骤1：构建dockerfile

步骤2：docker-compose（初版，需debug）

version: "3.2"
services:
  web:
    image: "seasonzhang/gaga_meeting:0.9"
    container_name: gaga_meeting-web
    entrypoint: ["/bin/sh","/code/package/start.sh"]
    # environment:
    #   - server_params=--settings=settings.local
    volumes:
      - C:/Users/Season/Desktop/SQL:/code/package/GAGA/SQL
    ports:
      - "8000:8000"
    depends_on:
      - redis
      - celery
      # - flower
  redis:
    image: "redis:alpine"
    container_name: gaga_meeting-redis
    ports:
      - "6379:6379"
  celery:
    image: "seasonzhang/gaga_meeting:0.9"
    container_name: gaga_meeting-celery
    # volumes:
    #   - .:/code/package/GAGA/SQL
    entrypoint: ["/bin/sh","/code/package/start-celery.sh"]
    depends_on: 
      - redis
  flower:
    image: "seasonzhang/gaga_meeting:0.9"
    container_name: gaga_meeting-flower
    ports:
      - "5555:5555"
    # volumes:
    #   - .:/code/package/GAGA/SQL
    entrypoint: ["/bin/sh","/code/package/start-flower.sh"]
    depends_on: 
      - redis
      - web
  beat:
    image: "seasonzhang/gaga_meeting:0.9"
    container_name: gaga_meeting-beat
    # volumes:
    #   - .:/code/package/GAGA/SQL
    entrypoint: ["/bin/sh","/code/package/start-beat.sh"]
    depends_on: 
      - flower

DJANGO_CELERY_BEAT_TZ_AWARE = False

https://blog.csdn.net/qq_38002009/article/details/101521821
BUG如下：我暂时无法debug gaga_meeting-celery ，装了beat之后celery会报错。需要设定DJANGO_CELERY_BEAT_TZ_AWARE = False

gaga_meeting-celery | [2022-08-23 18:37:05,984: WARNING/MainProcess] 102

gaga_meeting-celery | [2022-08-23 18:37:06,004: INFO/MainProcess] Task GAGA_meeting.tasks.async_action[e8859b2f-e592-4e90-b482-169671a20d56] succeeded in 10.021898800000145s: 101

gaga_meeting-celery | [2022-08-23 18:37:06,005: ERROR/MainProcess] Signal handler <bound method DjangoWorkerFixup.on_task_postrun of <celery.fixups.django.DjangoWorkerFixup object at 0x7f4aa3a94cd0>> raised: DatabaseError("DatabaseWrapper objects created in a thread can only be used in that same thread. The object with alias 'default' was created in thread id 139958587675040 and this is thread id 139958509175024.")

gaga_meeting-celery | Traceback (most recent call last):

gaga_meeting-celery |   File "/usr/local/lib/python3.8/site-packages/celery/utils/dispatch/signal.py", line 288, in send

gaga_meeting-celery |     response = receiver(signal=self, sender=sender, **named)

gaga_meeting-celery |   File "/usr/local/lib/python3.8/site-packages/celery/fixups/django.py", line 172, in on_task_postrun

gaga_meeting-celery |     self.close_database()

gaga_meeting-celery |   File "/usr/local/lib/python3.8/site-packages/celery/fixups/django.py", line 177, in close_database

gaga_meeting-celery |     return self._close_database()

gaga_meeting-celery |   File "/usr/local/lib/python3.8/site-packages/celery/fixups/django.py", line 186, in _close_database

gaga_meeting-celery |     conn.close()

gaga_meeting-celery |   File "/usr/local/lib/python3.8/site-packages/django/utils/asyncio.py", line 26, in inner

gaga_meeting-celery |     return func(*args, **kwargs)

gaga_meeting-celery |   File "/usr/local/lib/python3.8/site-packages/django/db/backends/sqlite3/base.py", line 278, in close

gaga_meeting-celery |     self.validate_thread_sharing()

gaga_meeting-celery |   File "/usr/local/lib/python3.8/site-packages/django/db/backends/base/base.py", line 552, in validate_thread_sharing

gaga_meeting-celery |     raise DatabaseError(

gaga_meeting-celery | django.db.utils.DatabaseError: DatabaseWrapper objects created in a thread can only be used in that same thread. The object with alias 'default' was created in thread id 139958587675040 and this is thread id 139958509175024.

设定DJANGO_CELERY_BEAT_TZ_AWARE = False后，beat可以正常启动。