自动检测并上传新文件的脚本

自动检测并上传新文件的脚本

作用: 当检测到特定文件夹下有新的文件添加, 则执行相关指令

用途: 当我把一个新打包好的deb包放文件夹, 程序会自动完成上传到服务器的功能

优点:

1. 通过systemd进行管理, 可以实现开机自启, 在后台启动该程序
2. 在没有新文件添加时保持静默, 减少系统资源占用
3. 在失败后会尝试多次重传
4. log完善, 可以通过systemctl status servive_name.service进行log的查询

import subprocess
import os
import logging
import time
import threading
import queue
from configparser import ConfigParser
import logging.handlers

# 初始化日志配置
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')

# FileHandler for logging to a file
file_handler = logging.FileHandler('upload.log', 'a')  # Append mode
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)

# StreamHandler for logging to stdout (for systemd)
stream_handler = logging.StreamHandler()
stream_handler.setFormatter(formatter)
logger.addHandler(stream_handler)

# Read configuration file
config = ConfigParser()
config.read('config.ini')
folder_to_monitor = os.path.expanduser(config.get('DEFAULT', 'FolderToMonitor'))
remote_location = config.get('DEFAULT', 'RemoteLocation')
max_retries = config.getint('DEFAULT', 'MaxRetries')
retry_delay = config.getint('DEFAULT', 'RetryDelay')
print(remote_location)
# 文件队列
file_queue = queue.Queue()
# 计数器
total_received = 0
total_transferred = 0
total_failed = 0
# 记录失败的文件名
failed_files = []

def scp_file(file_path, remote_location):
	global total_transferred, total_failed, failed_files
	retries = 0
	while retries <= max_retries:
		try:
			command = ["scp", file_path, remote_location]
			result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=60)
			if result.returncode == 0:
				total_transferred += 1
				logger.info(f"File {os.path.basename(file_path)} successfully transferred.")
				return True
			else:
				raise Exception(result.stderr.decode())
		except Exception as e:
			logger.warning(f"Attempt {retries + 1}: Error transferring file {os.path.basename(file_path)}: {e}")
			time.sleep(retry_delay)
			retries += 1
	total_failed += 1
	failed_files.append(os.path.basename(file_path))
	logger.error(f"Failed to transfer file {os.path.basename(file_path)} after {max_retries} retries.")
	return False

def worker():
	while True:
		file_path = file_queue.get()
		if file_path is None:
			file_queue.task_done()
			break  # 结束线程
		logger.info(f"Processing file: {os.path.basename(file_path)}")
		if scp_file(file_path, remote_location):
			os.remove(file_path)  # 仅当上传成功时删除文件
		file_queue.task_done()

def monitor_folder(folder_path, remote_location):
	global total_received
	# 启动工作线程
	threading.Thread(target=worker, daemon=True).start()

	logger.info("Starting folder monitoring...")
	command = ['inotifywait', '-m', '-e', 'close_write', '--format', '%w%f', folder_path]
	with subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) as proc:
		try:
			for line in proc.stdout:
				file_path = line.strip()
				if file_path.endswith('.deb'):
					total_received += 1
					logger.info(f"Detected file: {os.path.basename(file_path)}. Total received: {total_received}")
					file_queue.put(file_path)
		except KeyboardInterrupt:
			logger.info("Stopping folder monitoring.")
			file_queue.put(None)  # 发送结束信号到工作线程
			proc.terminate()
			logger.info(f"Batch Summary: Total Received: {total_received}, Total Transferred: {total_transferred}, Total Failed: {total_failed}.")
			if failed_files:
				logger.info(f"Failed Files: {', '.join(failed_files)}. Check these files for manual processing.")

def print_config_info(config):
	logger.info("Configuration Info:")
	# 打印默认部分
	default_section = config.defaults()
	if default_section:
		logger.info("Section: DEFAULT")
		for key in default_section:
			logger.info(f"  {key} = {default_section[key]}")
	# 打印其他部分
	for section in config.sections():
		logger.info(f"Section: {section}")
		for key in config[section]:
			logger.info(f"  {key} = {config[section][key]}")

if __name__ == "__main__":
	print_config_info(config)  # 输出配置信息
	monitor_folder(folder_to_monitor, remote_location)

使用方法:

1. 将本机的ssh拷贝到服务器(安全考虑, 须提前生成ssh密钥)

   # 生成本机的ssh
   ssh-keygen -t rsa -b 4096 #全部回车, 不需要输入任何内容)
   ssh-copy-id username@ip_addr
   # 输入服务器密码
   

2. 将源码保存为python文件

3. 在同目录下创建config.ini配置文件

[DEFAULT]
FolderToMonitor = /path/to/monitor/
RemoteLocation = root@ip_addr:/path/to/remote/
MaxRetries = 3
RetryDelay = 5

4. 创建service文件, 目录/etc/systemd/system/

[Unit]
Description=Auto Upload Service
After=network.target

[Service]
Type=simple
WorkingDirectory=/path/to/folder/
ExecStart=/usr/bin/python3 /path/to/folder/auto_upload.py
Restart=on-failure
User=[username]
Group=[username]

[Install]
WantedBy=multi-user.target

5. 添加开机自启动

sudo systemctl start service_name.service
sudo systemctl enable service_name.service