阿里云OSS本地批量上传脚本工具

 使用需要提前安装 pip install oss2 模块

 上传到目录时，存储到oss上key取目录内文件或者目录列表的相对路径：

  如：D:/demo/ 文件夹，文件夹内有，dir1/f1, dir2/f2, f3这几个文件，则存储到oss上key取dir1/f1,dir2/f2,f3

# -*- coding: utf-8 -*-
# Author: tom 
# Description: ali oss tool for upload local file or dir
# Date: 2020年8月24日
# Usage: python alioss_uploader.py D:\workspace\web "\\resources\\|\\WEB-INF\\|\\wx\\|.*.jsp|.*.swf", first arg is upload path, second arg is exclude pattern

import oss2
import os
import sys
import re
import logging
# import asyncio
from concurrent import futures

console_handler = logging.StreamHandler()
file_handler = logging.FileHandler(filename='./upload.log', mode='w')
log_format = '%(asctime)s %(threadName)s-%(thread)d %(levelname)s: %(message)s'
formatter = logging.Formatter(log_format)
logging.basicConfig(filename='./upload.log', filemode='w', format=log_format
        , datefmt='%Y/%m/%d %H:%M:%S'
        , level=logging.INFO)
console_handler.setFormatter(formatter)
file_handler.setFormatter(formatter)
log = logging.getLogger('alioss_uploader')
log.setLevel(logging.INFO)
log.addHandler(console_handler)
log.addHandler(file_handler)

auth = oss2.Auth('accessKey', 'secretKey')
bucket = oss2.Bucket(auth, 'http://oss-xxx.aliyuncs.com', 'mybucket')
# bucket = oss2.Bucket(auth, 'http://oss-xxx-internal.aliyuncs.com', 'mybucket')

upload_path = None

def upload_to_oss(file):
    with open(file, 'rb') as fileobj:
        try:
            upload_key = file.replace(upload_path, '').replace('\\', '/')[1:]
            log.info(f'Upload: {file}, key: {upload_key}')
            result = bucket.put_object(upload_key, fileobj)
            log.info(f'Upload result: {result.status}')
        except Exception as e:
            log.error(f'Upload Error: {e}')
        
def recursive_upload(path, exclude_regx_pattern, pool):
    path = os.path.abspath(path)
    if exclude_regx_pattern is None or exclude_regx_pattern.search(path) is None:
        if os.path.isfile(path):
            #upload_to_oss(path)
            pool.submit(upload_to_oss, file=path)
        elif os.path.isdir(path):
            base_path = os.path.abspath(path)
            lsdir = os.listdir(path)
            pathnames = [os.path.join(base_path, filename) for filename in lsdir]
            for f in pathnames:
                recursive_upload(f, exclude_regx_pattern, pool)
    else:
        log.info(f'Ignore file: {path}')
        
if __name__ == '__main__':
    exclude_pattern = None
    if len(sys.argv) < 2:
        raise Exception('Argument upload path absent!')
        
    upload_path = os.path.abspath(str(sys.argv[1]))
    
    if len(sys.argv) >= 3 and sys.argv[2] is not None:
         exclude_pattern = sys.argv[2]
         exclude_pattern = re.compile(exclude_pattern)
         
    log.info(f'Oss upload local: {upload_path}, ignore pattern {exclude_pattern}');
    pool = futures.ThreadPoolExecutor(max_workers=16)
    recursive_upload(upload_path, exclude_pattern, pool)
    pool.shutdown()