采集Doris的FE和BE 日志到对象存储

import os
import socket
import time

#doris log dir
log_src_dir="/data/doris-decimal/output/fe/log/"
#tar.gz temp dir
log_temp_dir="/data/doris-/temp/log/"
#record upload log
log_file="/data/doris-/temp/log"
#role
role = "fe"
#how log check log
interval_time_sec = 10
#whic bucket to upload
"""
the result is : bucket_url + role + ip + log.tar.gz
run: nohup python3 doris-log-s3.py >> &
"""
bucket_url = "https://yourbucketinfo.cos.ap-beijing.myqcloud.com"

#create file and directory
os.system("touch {}".format(log_file))
os.system("mkdir -p {}".format(log_temp_dir))

def collection_log():
    files = os.listdir(log_src_dir)
    for file in files:
        if not os.path.isdir(file):
           file_name = file.split()[0]
           print(file_name)
           is_upload_cmd = "cat {} |grep {} |wc -l".format(log_file,file_name)
           print(is_upload_cmd)
           is_upload= int(os.popen(is_upload_cmd).read().replace(" ",""))
           print(is_upload)
           # is logfile is not upload  and not a tar.gz file,then tar file and upload to s3
           if is_upload == 0 and  not str(file_name).__contains__("tar.gz"):
               print("start upload")
               #step1:tar file
              tar_cmd = "cd {} && tar -czvf  {}/{}.tar.gz  {}".format(log_src_dir,log_temp_dir,file_name,file_name)
               print(tar_cmd)
               tar_res = os.popen(tar_cmd).read()
               print(tar_res)
               #step2:upload file
               hostname = socket.gethostname()
               time.sleep(2)
               ip = socket.gethostbyname(hostname)
               print(ip)
               curl_cmd = "curl -T  {}/{}.tar.gz  {}/{}/{}/{}.tar.gz ".format(log_temp_dir, file_name,bucket_url,role,ip,file_name)
               print(curl_cmd)
               os.system(curl_cmd)
               #print(curl_res)
               print("{}/{}/{}/{}.tar.gz".format(bucket_url,role,ip,file_name))
               #step3:delete temp file
               rm_cmd = "rm -rf {}/{}.tar.gz".format(log_file, file_name)
               print(rm_cmd)
               rm_res = os.popen(rm_cmd).read()
               # step4:echo log
               print(rm_res)
               time_str = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
               echo_log = "echo '{}  upload finish {}'  >> {}".format(file_name,time_str,log_file)
               print(echo_log)
               os.popen(echo_log)
           else:
               print("file {} have uploaded".format(file_name))
# Press the green button in the gutter to run the script.
if __name__ == '__main__':
    while True:
      collection_log()
      time.sleep(interval_time_sec)