6 网站架构---mongo使用

models_mongo.py：创建数据库、集合、model

# coding=utf-8
import os
import uuid
import magic
import urllib
from datetime import datetime
import cropresize2
import short_url
from PIL import Image
from flask import abort, request
from werkzeug.utils import cached_property
from mongoengine import (
    Document as BaseDocument, connect, ValidationError, DoesNotExist,
    QuerySet, MultipleObjectsReturned, IntField, DateTimeField, StringField,
    SequenceField)
from app.ik_global import ik_global
from app.utils.mimes import IMAGE_MIMES, AUDIO_MIMES, VIDEO_MIMES
from app.utils.ik_util import get_file_md5, get_file_path
from config import Config
# ik_global.Logger.info("MONGO_DB_HOST-------------> {}".format(Config.MONGO_DB_HOST))

connect(Config.MONGO_DB_NAME, host=Config.MONGO_DB_HOST, port=int(Config.MONGO_DB_PORT))


class BaseQuerySet(QuerySet):
    def get_or_404(self, *args, **kwargs):
        try:
            return self.get(*args, **kwargs)
        except (MultipleObjectsReturned, DoesNotExist, ValidationError):
            abort(404)


class Document(BaseDocument):
    meta = {'abstract': True,
            'queryset_class': BaseQuerySet
            }


class PasteFile(Document):
    id = SequenceField(primary_key=True)
    filename = StringField(max_length=5000, null=False)
    filehash = StringField(max_length=128, null=False, unique=True)
    filemd5 = StringField(max_length=128, null=False, unique=True)
    uploadtime = DateTimeField(null=False)
    mimetype = StringField(max_length=128, null=False)
    size = IntField(null=False)
    meta = {'collection': 'paste_file'}  # 自定义集合的名字

    def __init__(self, filename='', mimetype='application/octet-stream',
                 size=0, filehash=None, filemd5=None, *args, **kwargs):
        # 初始化父类的__init__方法
        super(PasteFile, self).__init__(filename=filename, mimetype=mimetype,
                                        size=size, filehash=filehash,
                                        filemd5=filemd5, *args, **kwargs)
        self.uploadtime = datetime.now()
        self.mimetype = mimetype
        self.size = int(size)
        self.filehash = filehash if filehash else self._hash_filename(filename)
        self.filename = filename if filename else self.filehash
        self.filemd5 = filemd5

    @staticmethod
    def _hash_filename(filename):
        _, _, suffix = filename.rpartition('.')
        return '%s.%s' % (uuid.uuid4().hex, suffix)

    @cached_property
    def symlink(self):
        return short_url.encode_url(self.id)

    @classmethod
    def get_by_symlink(cls, symlink, code=404):
        id = short_url.decode_url(symlink)
        return cls.objects.get_or_404(id=id)

    @classmethod
    def get_by_filehash(cls, filehash, code=404):
        return cls.objects.get_or_404(filehash=filehash)

    @classmethod
    def get_by_md5(cls, filemd5):
        rs = cls.objects(filemd5=filemd5)
        return rs[0] if rs else None

    @classmethod
    def create_by_upload_file(cls, uploaded_file):
        ik_global.Logger.info("uploaded_file----> {} {} {}".format(uploaded_file,
                                                   uploaded_file.filename, uploaded_file.mimetype))
        rst = cls(uploaded_file.filename, uploaded_file.mimetype, 0)
        ik_global.Logger.info("rst.path----> {}".format(rst.path))
        uploaded_file.save(rst.path)
        with open(rst.path, "rb") as f:
            filemd5 = get_file_md5(f)
            uploaded_file = cls.get_by_md5(filemd5)
            if uploaded_file:
                os.remove(rst.path)
                return uploaded_file
        filestat = os.stat(rst.path)
        rst.size = filestat.st_size
        rst.filemd5 = filemd5
        return rst

    @classmethod
    def create_by_old_paste(cls, filehash):
        filepath = get_file_path(filehash)
        mimetype = magic.from_file(filepath, mime=True)
        filestat = os.stat(filepath)
        size = filestat.st_size

        rst = cls(filehash, mimetype, size, filehash=filehash)
        return rst

    @property
    def path(self):
        return get_file_path(self.filehash)

    def get_url(self, subtype, is_symlink=False):
        hash_or_link = self.symlink if is_symlink else self.filehash
        return 'http://{host}/{subtype}/{hash_or_link}'.format(
            subtype=subtype, host=request.host, hash_or_link=hash_or_link)

    @property
    def url_i(self):
        return self.get_url('i')

    @property
    def url_p(self):
        return self.get_url('p')

    @property
    def url_s(self):
        return self.get_url('s', is_symlink=True)

    @property
    def url_d(self):
        return self.get_url('d')

    @property
    def image_size(self):
        if self.is_image:
            im = Image.open(self.path)
            return im.size
        return (0, 0)

    @property
    def quoteurl(self):
        return urllib.parse.quote(self.url_i)

    @classmethod
    def rsize(cls, old_paste, weight, height):
        assert old_paste.is_image, TypeError('Unsupported Image Type.')

        img = cropresize2.crop_resize(
            Image.open(old_paste.path), (int(weight), int(height)))

        rst = cls(old_paste.filename, old_paste.mimetype, 0)
        img.save(rst.path)
        filestat = os.stat(rst.path)
        rst.size = filestat.st_size
        return rst

    @property
    def is_image(self):
        return self.mimetype in IMAGE_MIMES

    @property
    def is_audio(self):
        return self.mimetype in AUDIO_MIMES

    @property
    def is_video(self):
        return self.mimetype in VIDEO_MIMES

    @property
    def is_pdf(self):
        return self.mimetype == 'application/pdf'

    @property
    def type(self):
        for t in ('image', 'pdf', 'video', 'audio'):
            if getattr(self, 'is_' + t):
                return t
        return 'binary'

ik_file_view.py：定义http接口

import logging
import os
import time
from datetime import datetime
from app.comm import ik_comm
import ipaddress
from flask import abort, request, jsonify, redirect, send_file
from flask_jwt import jwt_required, current_identity
from werkzeug.contrib.cache import SimpleCache
from flask import jsonify
import json
from app.ikdb.model.models_mongo import PasteFile
from app.utils.ik_util import get_file_path, humanize_bytes
from app.utils.status_code import err_resp_json_make, FAIL, succ_resp_json_make
# from app.vpp.vpp import Pttool
from flask import Blueprint
ONE_MONTH = 60 * 60 * 24 * 30

bp = Blueprint('file', __name__,url_prefix="/file")


@bp.route('/rsize/<img_hash>')
def rsize(img_hash):
    w = request.args['w']
    h = request.args['h']

    old_paste = PasteFile.get_by_filehash(img_hash)
    new_paste = PasteFile.rsize(old_paste, w, h)

    return new_paste.url_i


@bp.route('/download/<filehash>', methods=['GET'])
def download(filehash):
    paste_file = PasteFile.get_by_filehash(filehash)

    return send_file(open(paste_file.path, 'rb'),
                     mimetype='application/octet-stream',
                     cache_timeout=ONE_MONTH,
                     as_attachment=True,
                     attachment_filename=paste_file.filename)


@bp.route('/upload', methods=['POST'])
def index():
    if request.method == 'POST':
        uploaded_file = request.files['file'] # werkzeug.datastructures.FileStorage
        w = request.form.get('w')
        h = request.form.get('h')
        if not uploaded_file:
            return abort(400)

        if w and h:
            paste_file = PasteFile.rsize(uploaded_file, w, h)
        else:
            paste_file = PasteFile.create_by_upload_file(uploaded_file)
        paste_file.save() #保存到db

        return jsonify({
            'url_d': paste_file.url_d,
            'url_i': paste_file.url_i,
            'url_s': paste_file.url_s,
            'url_p': paste_file.url_p,
            'filename': paste_file.filename,
            'size': humanize_bytes(paste_file.size),
            'time': str(paste_file.uploadtime),
            'type': paste_file.type,
            'quoteurl': paste_file.quoteurl
        })
    # return render_template('index.html', **locals())
    return err_resp_json_make(FAIL, extra_errmsg="upload fail")


@bp.route('/j', methods=['POST'])
def j():
    uploaded_file = request.files['file']

    if uploaded_file:
        paste_file = PasteFile.create_by_upload_file(uploaded_file)
        paste_file.save()
        width, height = paste_file.image_size

        return jsonify({
            'url': paste_file.url_i,
            'short_url': paste_file.url_s,
            'origin_filename': paste_file.filename,
            'hash': paste_file.filehash,
            'width': width,
            'height': height
        })

    return abort(400)


@bp.route('/preview/<filehash>')
def preview(filehash):
    paste_file = PasteFile.get_by_filehash(filehash)

    if not paste_file:
        filepath = get_file_path(filehash)
        if not(os.path.exists(filepath) and (not os.path.islink(filepath))):
            return abort(404)

        paste_file = PasteFile.create_by_old_paste(filehash)
        paste_file.save()

    # return render_template('success.html', p=paste_file)
    return succ_resp_json_make(extra_errmsg="preview success")


@bp.route('/s/<symlink>')
def s(symlink):
    paste_file = PasteFile.get_by_symlink(symlink)

    return redirect(paste_file.url_p)

app/__init__.py: flask启动时，初始化mongodb,注册蓝图

#!/usr/bin/env python
# -*- coding:utf-8 -*-
# File: ik_agent.py
import sys
from flask import Flask, jsonify
# from flask_jwt import JWT
from config import config
import multiprocessing as mp
from app.ik_global import ik_global, vpp_sig_register
from werkzeug.contrib.cache import SimpleCache
# from app.vpp.vpp import Pttool
from app.comm import ik_comm
from app.comm.agent_process import AgentProcess
from app.log import log
from app.ik_global.ik_global import MP_LIST
# from app.vpp.vpp import Pttool
from app.utils.threads import DaemonThreadTimer
from app.comm.ik_thread_work import monitor_vpp_instance_status
from app.utils.ik_util import show_memory_info
from app.utils.auth import authenticate, identity
from app.utils.threads import set_interval
from app.ikdb.exts import db
from flask_jwt import JWT
from werkzeug.contrib.profiler import ProfilerMiddleware
from werkzeug.wsgi import DispatcherMiddleware
from collections import OrderedDict
# from app.views import json_page

def creat_child_processes():
    # 这个函数用于创建并启动agent的子进程，
    # 子进程处理一些不需要运行在主进程的任务，
    # 以提升主进程对 web 请求的响应性能。
    # miscellaneous_works_init()

    # 子进程退出流程，详见函数 kill_all_child
    # 1 主进程捕获 gunicorn 的退出信号；
    # 2 在主进程信号处理函数中，向各个子进程发送 TERM 信号；
    # 3 子进程收到 TERM 信号后，进程自动退出
    # agent_process.start()
    # p = mp.Process(target=vpp_sig_register, name="recv_vpp_signal_process")
    p = AgentProcess(target=vpp_sig_register, name="recv_vpp_signal_process")
    p.start()

    MP_LIST.append(p)


'''
def check_vpp():
    
    if vpp:
        vpp = ik_comm.connect_vpp()
        ik_global.VPP_INSTANCE = vpp
        # ik_global.Logger.info(ik_global.VPP_INSTANCE)
        ik_comm.trigger_config_restore()
        return True
    return False
'''


def create_app(config_name):

    app = Flask(__name__)
    app.config.from_object(config[config_name])
    app.config.setdefault('JWT_SECRET_KEY', 'secret')
    app.debug = True
    # db init 
    # 方式一
    db.init_app(app=app)# 初始化连接数据库
    # 方式二
    from app.ikdb.exts import connect_db
    # init log
    # log.init_log(app)
    ik_global.Logger = log.create_logger()

    # mongodb init
    from app.ikdb.model import models_mongo
    
    # blueprint regester
    from app.views import ik_user_view
    from app.views import ik_agent_view
    from app.views import ik_vpp_view
    from app.views import ik_file_view
    app.register_blueprint(ik_user_view.bp)
    app.register_blueprint(ik_agent_view.bp)
    app.register_blueprint(ik_vpp_view.bp)
    app.register_blueprint(ik_file_view.bp)

    # ik_global.ik_agent_sig_register()
    # app.register_blueprint(ik_auth_view)

    # open profile
    # app.wsgi_app = ProfilerMiddleware(app.wsgi_app, profile_dir="/tmp")

    # 自动封装请求路由为/j/开头的地址为jsonify格式
    app.wsgi_app = DispatcherMiddleware(app.wsgi_app, OrderedDict((('/j', views.json_page),)))

    ik_global.Logger.info("性能测试工具节点--agent启动成功!")
    ik_global.Logger.info("版本号: {}".format(ik_global.__version__))
    ik_global.Logger.info("agent: {} port: {}".format(config[config_name].LISTEN_ADDR, config[config_name].VPP_AGENT_LISTEN_PORT))
    # ik_global.Logger.info("controller_url: {}{}".format(
    #     config[config_name].CONTROLLER_HOST, config[config_name].CONTROLLER_HOST_PATH))
    # ik_global.Logger.info(
    #     "VPP_CORE_DIR: {}".format(config[config_name].VPP_CORE_DIR))
    # ik_global.Logger.info(
    #     "VPP_PLUGINS_DIR: {}".format(config[config_name].VPP_PLUGINS_DIR))
    ik_global.Logger.info(show_memory_info("after vpp started"))
    return app