Python 多线程图片备份实践

说明：

- 数据库有10个uploads 文件表，其中1条记录对应有5个图片文件，现需把某个图片文件移到备份目录

要求：

- 备份时保留全路径，如 src/a/.../d/001.jpg => backups/a/.../d/001.jpg

- 有迁移日志

- 按时间建zip压缩包

- sftp远程备份

- 多线程，不同表启动一个线程

步骤：

1. 目录结构

.
├── common
│   ├── config.py
│   ├── db.py
│   ├── models
│   │   ├── TheModel.py
│   │   ├── PhotoOptimizer.py
│   │   ├── __init__.py
│   │   └── models.py
│   └── utils
│       └── utils.py
├── main.py
├── requirements.txt
└── setup.py

2. 用工具sqlacodegen建模

sqlacodegen mysql+pymysql://developer:developer@localhost/temp > models.py

修改模型FileUploadX, 使其继承Base / TheModel，其中:

db.py

# 数据模型 db.py
from flask import Flask
import flask_sqlalchemy
import common.config as Config

app = Flask(__name__)
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
app.config['SQLALCHEMY_DATABASE_URI'] = Config.SQLALCHEMY_DATABASE_URI
db = flask_sqlalchemy.SQLAlchemy(app)
Base = db.Model

models.py 

# coding: utf-8
from sqlalchemy import text, Column, String, Integer, DateTime, BigInteger, Numeric, ForeignKey, SmallInteger, \
    create_engine
from sqlalchemy.dialects.mysql import BIGINT, INTEGER, SMALLINT
import common.config as Config
from common.db import Base
from .OymModel import OymModel

class FileUpload0(Base, TheModel):
...

3. main.py

#!/usr/bin/env python3

import os
import threading
import time
import common.config as Config
from common.models.PhotoOptimizer import *
from common.models.models import *
from common.utils.utils import *
import logging
from atpbar import flush

if __name__ == '__main__':
    try:
        start_time = time.time()

        # 设置日志
        logging.basicConfig(filename=f"{Config.LOG_PATH}/log.txt", level=logging.DEBUG, filemode="w",
                            format="%(asctime)s - %(levelname)s - %(message)s")

        # 文件夹有效性检查
        check()

        # 线程定义
        threads = [
            PhotoOptimizer(0, f"Thread-AomaUpload-0", AomaUpload0),
            PhotoOptimizer(1, f"Thread-AomaUpload-1", AomaUpload1),
            ...
        ]

        for t in threads:
            t.start()

        # 线程同步: 待子线程执行结束之后主线程再终止
        for t in threads:
            t.join()
        flush()

        print('主线程结束了！', threading.current_thread().name)
        t_now = time.time()
        m, s = divmod(t_now - start_time, 60)
        h, m = divmod(m, 60)
        print('一共用时：', "%02d时 %02d分 %02d秒 (%s)" % (h, m, s, t_now - start_time))

        # 压缩文件
        dstFile = ""
        if (not Config.IS_DRY_RUN):
            print('Start to zip')
            dstFile = os.path.dirname(Config.PHOTOS_BAK_PATH) + '/' + Config.PHOTOS_BAK_ZIPFILE % (
                time.strftime("%Y-%m-%d.%H%M%S", time.localtime())) + '.zip'
            zip_file(Config.PHOTOS_BAK_PATH, dstFile)
            print('Zip finished')

        # 远程备份
        if (not Config.IS_DRY_RUN and Config.SFTP_ENABLE):
            print(f"Start to upload Host:{Config.SFTP_HOST}, Path:{Config.SFTP_DST_PATH}")
            upload(dstFile, Config.SFTP_DST_PATH)
            print('Upload finished.')

        # 清除备份文件夹
        if (not Config.IS_DRY_RUN and Config.NEED_CLEAN_UP):
            print(f"Start to clean up")
            cleanup(Config.PHOTOS_BAK_PATH + '/file')
            print('Clean up finished.')

    except RuntimeError as err:
        print("Error: Cannot run threads!", err)

4. 优化器 PhotoOptimizer，分页获取、处理数据记录

import common.config as Config
import json
import threading
import time
import uuid
from .models import *
import logging
from atpbar import atpbar


class PhotoOptimizer(threading.Thread):
    def __init__(self, threadID, name, entityModel):
        threading.Thread.__init__(self)
        self.threadID = threadID
        self.name = name
        self.entityModel = entityModel

    def run(self):
        # print("Thread Start：" + self.name)

        # fiterVars = self.entityModel.fileurl.like('%82630%')
        fiterVars = self.entityModel.pid > 0

        pagination = self.entityModel \
            .query \
            .filter(fiterVars) \
            .order_by(self.entityModel.pid.desc()) \
            .paginate(
            page=1,
            per_page=Config.QUERY_PER_PAGE,
            max_per_page=Config.QUERY_PER_PAGE,
            error_out=False
        )

        processResult = {'move_nums': 0}

        for i in atpbar(range(pagination.pages), name=f"{self.name} Progress"):
            items = self.entityModel \
                .query \
                .filter(fiterVars) \
                .order_by(self.entityModel.pid.desc()) \
                .paginate(
                page=i,
                per_page=Config.QUERY_PER_PAGE,
                max_per_page=Config.QUERY_PER_PAGE,
                error_out=False
            )

            # print(result)
            for model in items.items:
                # print(i, model.pid, model.getMainPhotoPath())
                # logging.info(f"{i}: {model.pid}: {model.getMainPhotoPath()}")
                if model.photoProcess():
                    processResult['move_nums'] += 1

5. 部署

编写 / 生成 requiremens.txt

pip3 install -r requirements.txt

python3 main.py