运维提升之Django篇④

7-1 章节导学

7-2 Django 日志模块

格式器formatter

过滤器filter

日志模块的使用

# logging日志配置
LOG_DIR = os.path.join(BASE_DIR, 'log')
if not os.path.exists(LOG_DIR):
    os.makedirs(LOG_DIR)
    
LOGGING = {
    'version': 1,
    'disable_existing_loggers': True,
    'formatters': {# 日志格式
        'standard': {
            'format': '%(asctime)s [%(threadName)s:%(thread)d] '
                      '[%(pathname)s:%(funcName)s:%(lineno)d] [%(levelname)s]- %(message)s'
        },
        'simple': {
            'format': '%(asctime)s %(message)s'
        }
    },
    'filters': {# 过滤器
        'test':{
            '()': 'ops.TestFilter'
        }
    },
    'handlers': {# 处理器
        'null': {
            'level': 'DEBUG',
            'class': 'logging.NullHandler',
        },
        'error_handler': {# error内容输出到另外的文件
            'level':'DEBUG',
            'class':'logging.handlers.RotatingFileHandler',
            'filename': os.path.join(LOG_DIR,'error.log'),#日志输出文件
            'maxBytes':1024*1024*1,#文件大小
            'backupCount': 5,#备份份数
            'formatter':'standard',#使用哪种formatters日志格式
            'encoding': 'utf8',
        },
        'file_handler': {# 记录到日志文件(需要创建对应的目录，否则会出错)
            'level':'DEBUG',
            'class':'logging.handlers.RotatingFileHandler',
            'filename': os.path.join(LOG_DIR,'service.log'),# 日志输出文件
            'maxBytes':1024*1024*1,#文件大小
            'backupCount': 5,#备份份数
            'formatter':'standard',#使用哪种formatters日志格式
            'encoding': 'utf8',
        },
        'console_handler':{# 输出到控制台
            'level': 'DEBUG',
            'class': 'logging.StreamHandler',
            'formatter': 'standard',
        },
        'statistics_handler':{
            'level':'DEBUG',
            'class':'logging.handlers.RotatingFileHandler',
            'filename': os.path.join(LOG_DIR,'statistics.log'),
            'maxBytes':1024*1024*5,
            'backupCount': 5,
            'formatter':'simple',
            'encoding': 'utf8',
        }
    },
    'loggers': {# logging管理器
        'django': {
            # 'handlers': ['console_handler', 'file_handler', 'error_handler'],
            'handlers': ['console_handler', 'file_handler'],
            'filters': ['test'],
            'level': 'DEBUG'
        },
        'statistics': {
            'handlers': ['statistics_handler'],
            'level': 'DEBUG'
        }
    }
}

7-3 Django Admin模块

python manage.py createsuperuser

在每个app的admin中注册管理

内容列表字段

不显示不可编辑字段

重写save_model函数实现保存

不允许后台修改open_id，可将其屏蔽

# authorization\admin.py

# admin.site.register(App)


@admin.register(App) # 在后端注册 等同于上面
class ApisAppAdmin(admin.ModelAdmin):
	# 展示用户所使用的字段
    fields = ['name', 'application', 'category', 'url', 'publish_date', 'desc']
    # exclude = ['appid'] # 屏蔽该字段

	# appid 可以由后台生成
    def save_model(self, request, obj, form, change):
        src = obj.category + obj.application
        appid = hashlib.md5(src.encode('utf8')).hexdigest()
        obj.appid = appid
        super().save_model(request, obj, form, change)

7-4 Django 缓存模块

# 星座运势
def constellation(request):
    data = []
    if already_authorized(request):
        user = get_user(request)
        constellations = json.loads(user.focus_constellations)
    else:
        constellations = all_constellations
    for c in constellations:
        result = cache.get(c)
        if not result:
            result = thirdparty.juhe.constellation(c)
            timeout = timeutil.get_day_left_in_second()
            cache.set(c, result, timeout)
            logger.info('set cache. key=[%s], value=[%s], timeout=[%d]' % (c, result ,timeout))
        data.append(result)

    response = CommonResponseMixin.wrap_json_response(data=data, code=ReturnCode.SUCCESS)
    return JsonResponse(response, safe=False)

7-5 Django 后台服务部署

很基础

7-6 Django Crontab 定时任务

python manage.py crontab show
python manage.py crontab add

7-7 Django Middleware 中间件

自定义中间件

#!/usr/bin/python
# -*-encoding=utf8 -*-


class TestMiddleware:

    def __init__(self, get_response):
        self.get_response = get_response

    def __call__(self, request):
        print('TestMiddleWare before request.')
        response = self.get_response(request)
        print('TestMiddleWare after request.')
        return response

7-9 综合实践：基于邮件通知的服务监控和告警系统

统计中间件

#!/usr/bin/python
# -*- encoding=utf-8 -*-


import time
import logging

from backend import settings

logger = logging.getLogger('statistics')
logger2 = logging.getLogger('django')


# 统计中间件
class StatisticsMiddleware:
    def __init__(self, get_response):
        self.get_response = get_response
        logger2.info('Build StatisticsMiddleware.')

    def __call__(self, request):
        tick = time.time()
        response = self.get_response(request)
        path = request.path
        full_path = request.get_full_path()
        tock = time.time()
        cost = tock - tick
        content_list = []
        content_list.append('now=[%d]' % tock)
        content_list.append('path=[%s]' % path)
        content_list.append('full_path=[%s]' % full_path)
        content_list.append('cost=[%.6f]' % cost)
        content = settings.STATISTICS_SPLIT_FLAG.join(content_list)
        logger.info(content)

        return response

添加统计中间件

setting 设置logging管理器

LOGGING = {
    'version': 1,
    'disable_existing_loggers': True,
    'formatters': {# 日志格式
        'standard': {
            'format': '%(asctime)s [%(threadName)s:%(thread)d] '
                      '[%(pathname)s:%(funcName)s:%(lineno)d] [%(levelname)s]- %(message)s'
        },
        'simple': {
            'format': '%(asctime)s %(message)s'
        }
    },
    'filters': {# 过滤器
        'test':{
            '()': 'ops.TestFilter'
        }
    },
    'handlers': {# 处理器
        'null': {
            'level': 'DEBUG',
            'class': 'logging.NullHandler',
        },
        'error_handler': {# error内容输出到另外的文件
            'level':'DEBUG',
            'class':'logging.handlers.RotatingFileHandler',
            'filename': os.path.join(LOG_DIR,'error.log'),#日志输出文件
            'maxBytes':1024*1024*1,#文件大小
            'backupCount': 5,#备份份数
            'formatter':'standard',#使用哪种formatters日志格式
            'encoding': 'utf8',
        },
        'file_handler': {# 记录到日志文件(需要创建对应的目录，否则会出错)
            'level':'DEBUG',
            'class':'logging.handlers.RotatingFileHandler',
            'filename': os.path.join(LOG_DIR,'service.log'),# 日志输出文件
            'maxBytes':1024*1024*1,#文件大小
            'backupCount': 5,#备份份数
            'formatter':'standard',#使用哪种formatters日志格式
            'encoding': 'utf8',
        },
        'console_handler':{# 输出到控制台
            'level': 'DEBUG',
            'class': 'logging.StreamHandler',
            'formatter': 'standard',
        },
        'statistics_handler':{
            'level':'DEBUG',
            'class':'logging.handlers.RotatingFileHandler',
            'filename': os.path.join(LOG_DIR,'statistics.log'),
            'maxBytes':1024*1024*5,
            'backupCount': 5,
            'formatter':'simple',
            'encoding': 'utf8',
        }
    },
    'loggers': {# logging管理器
        'django': {
            # 'handlers': ['console_handler', 'file_handler', 'error_handler'],
            'handlers': ['console_handler', 'file_handler'],
            'filters': ['test'],
            'level': 'DEBUG'
        },
        'statistics': {
            'handlers': ['statistics_handler'],
            'level': 'DEBUG'
        }
    }
}

发送邮件实例

#!/usr/bin/python
# -*- encoding=utf-8 -*-


import os
import time
import django
import datetime
import smtplib
import logging

from backend import settings
from email.mime.text import MIMEText


logger = logging.getLogger('django')


def demo():
    message = 'Job log in crontab, now: ' + str(datetime.datetime.now())
    print(message)
    logger.info(message)


# 分析日志的任务
def statistics():
    # 读取统计的log内容
    data_file = os.path.join(settings.BASE_DIR, 'log', 'statistics.log')
    if not os.path.exists(data_file):
        logger.warning("file not exists. file=[%s]" % data_file)
        return

    result = {}
    with open(data_file, 'r') as data_file:
        for line in data_file:
            line = line.strip()
            content = line.split(' ')[2]
            content_list = content.split(settings.STATISTICS_SPLIT_FLAG)
            print(content_list)
            log_time = int(content_list[0].split('=')[1][1:-1])
            print(log_time)
            path = content_list[1].split('=')[1][1:-1]
            print(path)
            full_path = content_list[2].split('=')[1][1:-1]
            cost = float(content_list[3].split('=')[1][1:-1])

            # 记录数据
            # path: value_list
            if path not in result.keys():
                result[path] = []
            result[path].append(cost)

    # 最大值、最小值、平均值
    report_content = []
    for k, v_list in result.items():
        # 请求次数
        count = len(v_list)
        # 最大值
        v_max = max(v_list)
        # 最小值
        v_min = min(v_list)
        # 平均值
        v_avg = sum(v_list) * 1.00 / count
        content = '%-40s COUNT: %d    MAX_TIME: %.4f(s)    MIN_TIME: %.4f(s)    AVG_TIME: %.4f(s)' \
                  % (k, count, v_max, v_min, v_avg)
        print(content)
        report_content.append(content)
    return report_content


def report_by_mail():
    logger.info('Begin statistics data.')
    content = statistics()
    content = '\r\n'.join(content)
    logger.info('End statistics data.')
    receivers = ['why957177569@163.com']
    msg = MIMEText(content, 'plain', 'utf-8')
    msg['FROM'] = '【Django Backend】'
    msg['Subject'] = '【Django Service Performance Monitor】'
    server = smtplib.SMTP_SSL(settings.EMAIL_HOST, settings.EMAIL_PORT)
    server.set_debuglevel(1)
    server.login(settings.EMAIL_HOST_USER, settings.EMAIL_HOST_PASSWORD)
    server.sendmail(settings.EMAIL_FROM, receivers, msg.as_string())
    server.close()
    logger.info('Send monitor Email success.')


if __name__ == '__main__':
    report_by_mail()