import os
import smtplib
import psutil
from email.mime.text import MIMEText
import time
import pytz
def check_gpu_idle(threshold=10.0):
# 获取GPU利用率信息
gpu_info = os.popen('nvidia-smi --query-gpu=utilization.gpu --format=csv,noheader,nounits').read().strip()
# 提取GPU的利用率,只需要看后4块
gpu_usage = gpu_info.split('\n')[4:8]
gpu_idle = []
gpu_idle_flag = False
# 判断GPU是否空闲
for i in range(0, len(gpu_usage)):
if float(gpu_usage[i]) < threshold:
gpu_idle.append('idle')
else:
gpu_idle.append('use')
if 'idle' in gpu_idle:
gpu_idle_flag = True
return gpu_idle, len(gpu_usage), gpu_idle_flag
def send_email(subject, body):
# 设置邮箱参数
sender_email = ''
sender_password = ''
receiver_email = ''
# 构造邮件内容
message = MIMEText(body)
message['Subject'] = subject
message['From'] = sender_email
message['To'] = receiver_email
# 连接到SMTP服务器
with smtplib.SMTP_SSL('smtp.126.com', 465) as server:
server.login(sender_email, sender_password)
# 发送邮件
server.sendmail(sender_email, receiver_email, message.as_string())
def main():
# 设置GPU利用率阈值
gpu_threshold = 10.0
# 设置报告间隔时间(秒)
report_interval = 50 * 60 # 50分钟
while True:
current_time = time.localtime()
# 格林威治时间,比北京时间8小时
hour, minute = current_time.tm_hour, current_time.tm_min
hour = (hour + 8) % 24
# 检查GPU是否空闲
gpu_state, gpu_num , gpu_idle_flag= check_gpu_idle(gpu_threshold)
subject = "GPU Resources"
body = ""
for i in range(0, gpu_num):
body = body + "GPU ID:" + str(i) + " " + gpu_state[i] + "\n"
if gpu_idle_flag:
# 在早晨10点到12点和下午2点到晚上9点之间
if 10 <= hour < 12 or 14 <= hour < 21:
send_email(subject, body)
time.sleep(report_interval)
if __name__ == "__main__":
main()
python白天定时查询服务器GPU空闲情况并发邮件通知
于 2024-01-17 10:33:33 首次发布