python脚本实现日志切割异地备份到OSS

脚本作用：

1. 将项目全量日志备份到阿里云OSS。
2. 配合crontab定时任务和OSS的生命周期，使日志归档策略可控。
3. 备份完成之后清空日志，避免了全量日志占用磁盘空间。
4. 判断大日志文件，采用分片上传。

上传到OSS的截图

脚本如下：

# -*- coding: utf-8 -*-

"""
# Install:
# step 1 : yum install python-devel / apt-get install python-dev
# step 2 : pip install oss2
# Modify:
# access_key_id, access_key_secret, bucket_name, endpoint, project, logs_list ( allow /x/y/z/*.xxx )
# Use:
# step 1 : upload this script into server's path : /usr/local/seektruth/logs_backup_oss
# step 2 : exec command 'crontab -e' . put next shell into text
  0 2 */7 * * nohup python -u /usr/local/seektruth/logs_backup_oss/logs_backup_oss.py >> /tmp/oss_backup_oss.log 2>&1 &
"""

import os
import sys
import time
from urllib2 import urlopen
from oss2 import SizedFileAdapter, determine_part_size
from oss2.models import PartInfo
import oss2

access_key_id = 'aaa'
access_key_secret = 'bbb'

# # 上海oss
# bucket_name = 'ccc'
# 内网：
# endpoint = 'https://ddd'
# 外网
# endpoint = 'https://eee'
# -------------------------------------------------
# 北京oss
bucket_name = 'fff'
# 内网
endpoint = 'https://ggg'
# 外网
# endpoint = 'https://hhh'





# 清空文件内容
def clear(log_abs_path):
    with open(log_abs_path, 'w') as f1:
        f1.seek(0)
        f1.truncate()


# 大日志文件分片上传
def multi_upload(objectName, localfile):
    """
    :param objectName: 上传到OSS之后的key
    :param localfile:  本地打日志文件绝对路径


    """
    key = objectName
    filename = localfile

    total_size = os.path.getsize(filename)
    # determine_part_size方法用于确定分片大小。网络好建议调大分片大小
    part_size = determine_part_size(total_size, preferred_size=100 * 1024)

    # 初始化分片。
    upload_id = bucket.init_multipart_upload(key).upload_id
    parts = []

    # 逐个上传分片。
    with open(filename, 'rb') as fileobj:
        part_number = 1
        offset = 0
        while offset < total_size:
            num_to_upload = min(part_size, total_size - offset)
            # 调用SizedFileAdapter(fileobj, size)方法会生成一个新的文件对象，重新计算起始追加位置。
            result = bucket.upload_part(key, upload_id, part_number,
                                        SizedFileAdapter(fileobj, num_to_upload))
            parts.append(PartInfo(part_number, result.etag))

            offset += num_to_upload
            part_number += 1

    # 完成分片上传。
    bucket.complete_multipart_upload(key, upload_id, parts)


# 如果file_list中的文件绝对路径填写的是*.xxx，需要获取所有以xxx后缀为结尾的文件并返回一个list
def get_mulity_files(abs_parenet_dir, suffix):
    """

    :param abs_parenet_dir: 以 suffix为结尾的所有日志文件的父级绝对路径
    :param suffix: 后缀 file log
    :return: List列表，重新组合的包含所有此次重组的日志文件绝对路径列表
    """
    new_compose_file_list = list()
    for file in os.listdir(abs_parenet_dir):
        if os.path.isdir(file):
            pass
        else:
            if file.endswith(suffix):
                new_compose_file_list.append(os.path.join(abs_parenet_dir, file))
    return new_compose_file_list


def backup(project_arg, backup_date_arg, logs_list_arg):
    try:
        # 初始化一个列表用于最终列表
        list_sample = logs_list_arg

        public_ip = urlopen('http://ip.42.pl/raw').read()
        # 如果传入的列表中的元素有包含*.suffix后缀的，重新组合列表
        for file in logs_list_arg:
            if file.split('/')[-1].split('.')[0] == '*':
                abs_parenet_dir = file.rstrip(file.split('/')[-1])
                suffix = file.split('/')[-1].split('.')[-1]
                new_compose_file_list = get_mulity_files(abs_parenet_dir, suffix)
                # 拼接到最终列表
                list_sample.extend(new_compose_file_list)
                # 去除原带有*的元素
                list_sample.remove(file)

        # 遍历最终列表，进行上传备份操作
                for logsfle_abs_path in list_sample:
            objectName = os.path.join(project_arg, public_ip, backup_date_arg, logsfle_abs_path.lstrip('/'))
            localFile = logsfle_abs_path

            # 如果文件大小大于 2G 就分片上传
            if os.path.getsize(logsfle_abs_path) > 1073741824 * 2:
                multi_upload(objectName, localFile)
            else:
                # 小于2G普通上传
                bucket.put_object_from_file(objectName, localFile)
            # 清空日志
            clear(logsfle_abs_path)
    except Exception, e:
        if sys.version[0] == "2":
            print backup_date_arg
            print str(e)
        else:
            print(backup_date_arg)
            print(str(e))


if __name__ == '__main__':
    assert ('linux' in sys.platform), "该脚本目前只能在 Linux 下执行"

    # 初始化bucket对象
    bucket = oss2.Bucket(oss2.Auth(access_key_id, access_key_secret), endpoint, bucket_name)
    # 项目名称（用于拼接oss路径）
    project = 'jjj'
    # 备份日期（用于拼接oss路径）
    backup_date = time.strftime('%Y-%m-%d %H-%M')
    # 需要备份的日志绝对路径列表
    logs_list = [
        '/a/b/c/d.file',
        '/a/bc/c/*.log',
        '/d/e/f/2.txt'


    ]
    # 遍历备份上传到OSS
    backup(project_arg=project, backup_date_arg=backup_date, logs_list_arg=logs_list)