折腾了一个晚上,对Shell脚本完全不熟,靠着百度,总算弄完,以后多用自己熟悉的Python,简单又强大
发送邮件请参考:https://blog.csdn.net/zhu6201976/article/details/85809032
[root@api-1 sbin]# cat nginx_logrotate.sh
#! /bin/bash
# 每日0:00统计昨日数据
d=`date -d "-1 day" +%Y%m%d`
logdir="/var/log/nginx"
nginx_pid="/usr/local/nginx/logs/nginx.pid"
cd $logdir
# 重命名昨日数据为当前日期
mv ads_access.log ads_access.log_$d
# 创建昨日错误记录文件
touch ads_access.log_error_$d
# 昨日总请求个数
yesterday_total_request=`cat ads_access.log_$d | wc -l`
# 昨日服务器异常个数
yesterday_5xx_request=`grep ' 5[0-9][0-9] ' ads_access.log_$d | wc -l`
# 昨日数据输出到文件保存
echo $d total_request:$yesterday_total_request 5xx_request:$yesterday_5xx_request >> ads_access.log_error_$d
# 如果服务器异常数据占比超过50%,发送邮件通知
if [ $yesterday_total_request -ne 0 ]
then
yesterday_5xx_request_2=$((yesterday_5xx_request*2))
if [ $yesterday_5xx_request_2 -ge $yesterday_total_request ]
then
echo 服务器异常,5xx错误占比超过50% >> ads_access.log_error_$d
echo 服务器异常,5xx错误占比超过50% |mail -s "服务器预警" 414566019@qq.com
fi
fi
/bin/kill -HUP `cat $nginx_pid`
-------------------------------------------------2019年1月5日18:24:32更新完善 使用python实现-------------------------------------------------------
import time
import datetime
import re
import os
import smtplib
from email.mime.text import MIMEText
from email.header import Header
def main():
total_request_count = 0
request_5xx_count = 0
# 获取当前日期,时间
localtime = time.localtime()
now_date = '%04d-%02d-%02d' % (localtime.tm_year, localtime.tm_mon, localtime.tm_mday)
now_time = '%02d:%02d' % (localtime.tm_hour, localtime.tm_min)
# 换行写入今日日期
if re.match(r'00:00', now_time):
with open('ads_access.log_data', 'a+') as f:
f.write('\n' + now_date + '\t')
# 获取 结束时间 开始时间
end_time = datetime.datetime.now()
start_time = end_time - datetime.timedelta(minutes=5)
# 格式化为0秒
end_time = end_time.strftime('%H:%M:00')
start_time = start_time.strftime('%H:%M:00')
# 将字符串时间转回时间对象
end_time = time.strptime(end_time, '%H:%M:00')
start_time = time.strptime(start_time, '%H:%M:00')
with open('ads_access.log', 'r') as f:
while True:
line = f.readline()
if len(line) == 0:
break
# 判断每一条请求时间是否符在前5分钟内
t = re.search(r':\d{2}:\d{2}:\d{2}', line).group()
t = t[1:]
t = time.strptime(t, '%H:%M:%S')
if start_time <= t <= end_time:
total_request_count += 1
if re.search(r'HTTP/1.1" 5\d\d ', line):
request_5xx_count += 1
# 将前5分钟总访问量,5xx服务器异常量写入文件
start_time = '%02d:%02d:%02d' % (start_time.tm_hour, start_time.tm_min, start_time.tm_sec)
end_time = '%02d:%02d:%02d' % (end_time.tm_hour, end_time.tm_min, end_time.tm_sec)
with open('ads_access.log_data', 'a+') as f:
content = '%s-%s %d:%d\t' % (start_time, end_time, total_request_count, request_5xx_count)
f.write(content)
# 服务器异常,邮件预警
if total_request_count != 0:
if request_5xx_count / total_request_count >= 0.5:
pass
# 方法1:第三方 SMTP 服务,在unbuntu虚拟机测试发邮件正常,服务器连接异常
# mail_host = "smtp.qq.com" # 设置服务器
# mail_user = "414566019@qq.com" # 用户名
# mail_pass = "******" # 使用自己的口令
# sender = '414566019@qq.com'
# receivers = ['414566019@qq.com','zhuhualong@viptaohui.com'] # 接收邮件,可设置为你的QQ邮箱或者其他邮箱
# message = MIMEText('前5分钟总访问量:%d,服务器异常量:%d' % (total_request_count, request_5xx_count), 'plain', 'utf-8')
# message['From'] = Header("47.106.66.177", 'utf-8')
# message['To'] = Header("测试", 'utf-8')
# subject = 'ads_nginx.log预警'
# message['Subject'] = Header(subject, 'utf-8')
# try:
# smtpObj = smtplib.SMTP(timeout=10)
# smtpObj.connect(mail_host, 25) # 25 为 SMTP 端口号
# smtpObj.login(mail_user, mail_pass)
# smtpObj.sendmail(sender, receivers, message.as_string())
# print("邮件发送成功")
# except smtplib.SMTPException:
# print("Error: 无法发送邮件")
# 方法2:使用python执行shell命令发送邮件
os.system(
'echo ads服务器异常,异常占比超50%. | mail -s "47.106.66.177服务器预警" 414566019@qq.com zhuhualong@viptaohui.com xiaochun@viptaohui.com')
if __name__ == '__main__':
main()
2019年4月30日22:51:18更新:日志分割就是执行一条shell指令,移动一个文件而已,所以完全可以用Python实现.
"""
toutiao项目日志文件每日分割python脚本
注意:可将用户自定义shell脚本,Python脚本放置:/usr/local/sbin目录
crontab -e
0 0 * * * /usr/local/bin/python3 /data/wwwroot/toutiao/utils/toutiao_log_file_rotate.py
保存退出即可
查看定时任务:crontab -l
"""
import os
def toutiao_log_rotate():
"""toutiao项目日志文件每日0:00分割"""
log_file_list = [
'baidu_article.log',
'douban_tv_2.log',
'kandian_article.log',
'toutiao_article.log',
'wangyi_article.log',
'dayu_article.log',
'fenghuang_article.log',
'kuaibao_article.log',
'sohu_article.log',
'toutiao_profile_spider.log',
'douban_movie_2.log',
'get_toutiao_comment2.log',
'mogu_ip.log',
'tjg_spider_2.log',
'toutiao_video.log',
]
for log_file in log_file_list:
# 1.移动日志文件
log_dir = log_file.split('.')[0]
shell_command = 'mv /data/wwwroot/toutiao/{} /data/wwwroot/toutiao/logs/{}/{}_`date -d "-1 day" +%Y%m%d`'.format(
log_file, log_dir, log_file)
os.system(shell_command)
# 2.nohup 第一次运行自动创建日志文件,运行中删除日志文件不再重复创建
touch_command = 'touch /data/wwwroot/toutiao/{}'.format(log_file)
os.system(touch_command)
if __name__ == '__main__':
toutiao_log_rotate()