Python简单部署脚本工具

目录

│  set_unix.py
│  sync_force.py
│
└─common
    │  create_scripts_normal.py
    │  monitor.py
    │  __init__.py
    │  __main__.py
    │  __out__.py
    │
    └─template
            log.txt
            monitor.txt
            start.txt
            stop.txt

scripts

- common

– template

— log.txt

— monitor.txt

#!/bin/python3
from common import start_process_single, MONITOR_PATH

SERVICE_NAME = '{}'

start_process_single(MONITOR_PATH, '{}', [SERVICE_NAME])

— start.txt

#!/bin/python3
import os

from common import start_process_single, MAIN_PATH

SERVICE_NAME = '{}'
MAIN = os.path.join(MAIN_PATH, f'{SERVICE_NAME}.py')

start_process_single(MAIN, '{}')

— stop.txt

#!/bin/python3
from common import kill_process

SERVICE_NAME = '{}'
MODEL = '{}'
kill_process('monitor', [MODEL, SERVICE_NAME])
kill_process('info_collector.py', [MODEL])

– init.py

from .__main__ import *
from .__out__ import *

– main.py

import os
import re
import sys
import time

from .__out__ import GUNICORN_PATH_OF_WINDOWS, GUNICORN_PATH_OF_LINUX
from .__out__ import PYTHON_PATH_OF_WINDOWS, PYTHON_PATH_OF_LINUX
from .__out__ import get_logger

logger = get_logger(__file__)


def get_python():
    """
    获取python路径
    @return:
    """
    if sys.platform[0: 3] == 'win':
        python = PYTHON_PATH_OF_WINDOWS
    elif sys.platform[0: 5] == 'linux':
        python = PYTHON_PATH_OF_LINUX
    else:
        logger.error('Mac?')
        raise Exception('Mac?')
    check_file(python)
    return python


def get_gunicorn():
    """
    获取gunicorn路径
    @return:
    """
    if sys.platform[0: 3] == 'win':
        gunicorn = GUNICORN_PATH_OF_WINDOWS
    elif sys.platform[0: 5] == 'linux':
        gunicorn = GUNICORN_PATH_OF_LINUX
    else:
        logger.error('Mac?')
        raise Exception('Mac?')
    check_file(gunicorn)
    return gunicorn


def check_file(file: str):
    """
    检查文件
    @param file:
    @return:
    """
    if not os.path.isfile(file):
        logger.error(f'can not found {file}.')
        raise Exception(f'can not found {file}.')


def check_dir(_dir: str):
    """
    检查文件
    @param _dir:
    @return:
    """
    if not os.path.isdir(_dir):
        logger.error(f'can not found {_dir}.')
        raise Exception(f'can not found {_dir}.')


def check_process(name: str, params: list = None, ignores: list = None):
    """
    检查进程
    @param name:
    @param params:
    @param ignores:
    @return:
    """
    if sys.platform[0: 3] == 'win':
        # TODO develope
        raise Exception('Check_process of win is undeveloped.')
    elif sys.platform[0: 5] == 'linux' or sys.platform[0: 3] == 'mac':
        cmd = f'ps aux|grep -v grep|grep -v check|grep {name}'
        if params:
            for _param in params:
                cmd = cmd + f'|grep {_param}'
        if ignores:
            for ignore in ignores:
                cmd = cmd + f'|grep -v {ignore}'
        res = os.popen(cmd)
        is_active = False
        processes = []
        for process in res:
            _process = re.sub(' +', ' ', process)
            processes.append(_process[:-1])
            is_active = True
        return is_active, processes
    else:
        logger.error('Unknow system.')
        raise Exception('Unknow system.')


def start_process(path: str, mode: str = None, params: list = None, log_path=None, nohup=False):
    """
    启动进程
    @param path: 程序路径
    @param mode: 程序模式
    @param params: 参数
    @param log_path: 日志路径
    @param nohup: 是否后台启动
    @return:
    """
    check_file(path)
    python = get_python()
    cmd = f'{python} {path} {mode}'
    if params:
        for param in params:
            cmd = cmd + f' {param}'
    if nohup:
        if sys.platform[0: 3] == 'win':
            os.system(f'nohup {cmd} >{log_path if log_path else "NUL"} 2>&1 &')
        elif sys.platform[0: 5] == 'linux' or sys.platform[0: 3] == 'mac':
            os.system(f'nohup {cmd} >{log_path if log_path else "/dev/null"} 2>&1 &')
        else:
            logger.error('Unknow system.')
            raise Exception('Unknow system.')
    else:
        os.system(cmd)


def start_process_single(path: str, mode: str = None, params: list = None, ignores: list = None, log_path=None):
    """
    启动单例进程
    @param path: 程序路径
    @param mode: 程序模式
    @param params: 参数
    @param ignores: 检查忽略
    @param log_path: 日志路径
    @return:
    """
    process = os.path.split(path)[-1]
    is_active, processes = check_process(process, params, ignores)
    info = f'{process} {mode} ' + ' '.join(params or [])
    if is_active:
        for process in processes:
            logger.warning(f'{info} is already started at {process.split(" ")[1]}.')
        return
    else:
        logger.info(f'{info} is starting...')
        start_process(path, mode, params, log_path, nohup=True)
    time.sleep(1)
    is_active, processes = check_process(process, params, ignores)
    if is_active:
        for process in processes:
            logger.info(f'{info} is started at {process.split(" ")[1]}.')
    else:
        logger.error(f'{info} startup failed.')


def kill_process(name: str, params: list = None, ignores: list = None):
    """
    关闭进程
    @param name: 程序名称/路径
    @param params: 参数
    @param ignores: 程序忽略
    @return:
    """
    is_active, processes = check_process(name, params, ignores)
    info = f'{name} ' + ' '.join(params or [])
    if is_active:
        cmd = f'ps aux|grep -v grep|grep {name}'
        if params:
            for _param in params:
                cmd = cmd + f'|grep {_param}'
        if ignores:
            for ignore in ignores:
                cmd = cmd + f'|grep -v {ignore}'
        cmd = f"{cmd}|tr -s ' '| cut -d ' ' -f 2"
        res = os.popen(cmd)
        for row in res:
            logger.info(f'{info} is active at {row[:-1]}, killing it.')
            os.system(f'kill -9 {row}')
        is_active, processes = check_process(name, params, ignores)
        if is_active:
            for process in processes:
                logger.error(f'{info} is still active at {process.split(" ")[1]}.')
        else:
            logger.info(f'{info} is stoped.')
    else:
        logger.warning(f'{info} is not started.')


def check_gunicorn(name: str, ):
    """
    检查进程
    @param name:
    @return:
    """
    if sys.platform[0: 3] == 'win':
        # TODO develope
        raise Exception('Check_gunicorn of win is undeveloped.')
    elif sys.platform[0: 5] == 'linux' or sys.platform[0: 3] == 'mac':
        cmd = f'pstree -ap|grep -v grep|grep gunicorn|grep {name}'
        res = os.popen(cmd)
        is_active = False
        processes = []
        pids = []
        for process in res:
            _process = re.sub(' +', ' ', process)
            main = re.search(r'gunicorn,(\d+)', process)
            if main:
                pids.append(main[1])
            processes.append(_process[:-1])
            is_active = True
        return is_active, processes, pids
    else:
        logger.error('Unknow system.')
        raise Exception('Unknow system.')


def start_gunicorn(path: str, params: list = None, app='app', restart=False):
    params = params or []
    process_file = os.path.split(path)[-1]
    process_dir = path.split(process_file)[0]
    process_name = process_file.split('.py')[0]
    is_active, processes_of_main, pids = check_gunicorn(process_name)
    paramsstr = ' '.join(params)
    if is_active and restart is False:
        logger.warning(f'{process_name} is already started at {pids[0]}.')
    elif not is_active and restart:
        logger.warning(f'{process_name} is not started .')
    elif is_active and restart:
        cmd = f'kill -HUP {pids}'
        os.system(cmd)
    else:
        gunicorn = get_gunicorn()
        cmd = f'{gunicorn} {process_name}:{app} {paramsstr} --chdir {process_dir}'
        logger.info(cmd)
        os.system(cmd)
        is_active, processes_of_main, pids = check_gunicorn(process_name)
        if is_active:
            logger.info(f'{process_name} is started at {pids}')
        else:
            logger.error(f'{process_name} can not start.')


def kill_gunicorn(path: str):
    """
    关闭进程
    @param path: 程序名称/路径
    @return:
    """
    process_file = os.path.split(path)[-1]
    process_name = process_file.split('.py')[0]
    is_active, processes_of_main, pids = check_gunicorn(process_name)
    if is_active:
        cmd = f'kill -9 {" ".join(pids)}'
        os.system(cmd)
        is_active, processes_of_main, pids = check_gunicorn(process_name)
        if is_active:
            logger.error(f'{process_name} can not stop.')
        else:
            logger.info(f'{process_name} is stoped.')
    else:
        logger.warning(f'{process_name} is not started.')

– out.py

import os
import sys

LOCAL_PATH = os.path.dirname(os.path.abspath(__file__))
MONITOR_PATH = os.path.join(LOCAL_PATH, 'monitor.py')
SCRIPTS_PATH = os.path.dirname(LOCAL_PATH)
MAIN_PATH = os.path.dirname(SCRIPTS_PATH)
TEMPLATE_PATH = os.path.join(LOCAL_PATH, 'template')
sys.path.append(MAIN_PATH)

from config import MAIN_PATH, VENV_PATH, SCRIPTS_PATH, LOGS_PATH
from config import PYTHON_PATH_OF_LINUX, PYTHON_PATH_OF_WINDOWS
from config import GUNICORN_PATH_OF_LINUX, GUNICORN_PATH_OF_WINDOWS
from libs.logger import set_logger, get_logger

if 'monitor.py' not in sys.argv[0]:
    set_logger(os.path.join(LOGS_PATH, f'scripts.log'))

__all__ = [
    'TEMPLATE_PATH',
    'MAIN_PATH',
    'MONITOR_PATH',
    'SCRIPTS_PATH',
    'VENV_PATH',
    'LOGS_PATH',
    'PYTHON_PATH_OF_LINUX',
    'PYTHON_PATH_OF_WINDOWS',
    'GUNICORN_PATH_OF_LINUX',
    'GUNICORN_PATH_OF_WINDOWS',
    'set_logger',
    'get_logger',
]

– create_scripts_normal.py

import os

from .__init__ import SCRIPTS_PATH, TEMPLATE_PATH


def create_monitor_scripts(server_name, mode):
    template_path = os.path.join(TEMPLATE_PATH, 'monitor.txt')
    scripts_path = os.path.join(SCRIPTS_PATH, f'monitor_{server_name}_{mode}_mode.py')
    with open(template_path, 'r') as t:
        template = t.read()
        with open(scripts_path, 'w') as f:
            f.write(template.format(server_name, mode))


def create_start_scripts(server_name, mode):
    template_path = os.path.join(TEMPLATE_PATH, 'start.txt')
    scripts_path = os.path.join(SCRIPTS_PATH, f'start_{server_name}_{mode}_mode.py')
    with open(template_path, 'r') as t:
        template = t.read()
        with open(scripts_path, 'w') as f:
            f.write(template.format(server_name, mode))


def create_stop_scripts(server_name, mode):
    template_path = os.path.join(TEMPLATE_PATH, 'stop.txt')
    scripts_path = os.path.join(SCRIPTS_PATH, f'stop_{server_name}_{mode}_mode.py')
    with open(template_path, 'r') as t:
        template = t.read()
        with open(scripts_path, 'w') as f:
            f.write(template.format(server_name, mode))

– monitor.py

import os
import sys
import time

LOCAL_PATH = os.path.dirname(os.path.abspath(__file__))
SCRIPTS_PATH = os.path.dirname(LOCAL_PATH)
sys.path.append(SCRIPTS_PATH)

from common import check_process
from common import MAIN_PATH, LOGS_PATH, set_logger, get_logger


def monitor(mode, service_name):
    set_logger(os.path.join(LOGS_PATH, f'monitor_{service_name}_{mode}.log'))
    logger = get_logger(__name__)
    path = os.path.join(MAIN_PATH, 'scripts', f'start_{service_name}_{mode}_mode.py')
    while True:
        is_active_of_main, _ = check_process(f'{service_name}.py', params=[mode])
        if is_active_of_main:
            logger.info(f'{service_name}_{mode}_mode is living.')
        else:
            logger.error(f'{service_name}_{mode}_mode has stoped')
            os.system(path)
        time.sleep(300)


if __name__ == '__main__':
    if len(sys.argv) > 2:
        _mode = sys.argv[1].lower()
        monitor(sys.argv[1], sys.argv[2])
    else:
        raise Exception('No parameter.')

- set_unix.py

#!/bin/python3
import os
from common import SCRIPTS_PATH
from common import get_logger

logger = get_logger(__file__)

LOCAL_PATH = SCRIPTS_PATH


def change_line_break(_path, _newline, encoding='utf-8'):
    """
    换行符转换（win<—>linux）
    @param _path: 文件路径
    @param _newline: '\n'：Unix(LF)；'\r\n'：Windows(CRLF)
    @param encoding:
    @return:
    """
    with open(_path, encoding=encoding) as infile:
        lines = infile.readlines()
        with open(_path, 'w', newline=_newline, encoding=encoding) as outfile:
            outfile.writelines(lines)
            logger.info(f'{_path} is changed.')


if __name__ == "__main__":
    newline = '\n'
    for file in os.listdir():
        if file[-2:] == 'py':
            path = os.path.join(LOCAL_PATH, file)
            change_line_break(path, newline)

- sync_force.py

#!/bin/python3
import os
import sys

from common import MAIN_PATH

os.chdir(MAIN_PATH)

os.system(f'git fetch --all')
if len(sys.argv) > 0:
    os.system(f'git reset --hard origin/{sys.argv[1]}')
os.system(f'git pull')