#coding:utf-8
#!/bin/python
#修改日志:
#修改日期 修改人 修改内容
#-------------------------------------------------------------
#2015/07/16 刘远生 第一版上生产,有基本功能
#2015/07/17 刘远生 新增订单号提取功能
#2015/07/22 刘远生 新增功能:把日志文件添加到附件中
# 解决邮件收件人不能超过3人的限制
# 改进展示样式,由CSS+HTML呈现,优化视觉效果
# 在邮件中含超链接,点击链接可下载、预览日志
#2015/07/27 刘远生 修复BUG:邮件发送异常(超时、认证失败等)导致的程序崩溃
# 修复BUG:先修改"最近已扫描的文件列表",而忽略邮件是事报警成功
# 添加程序的日志功能;所有的日志都要记录进文件,ERROR日志还要发送给syslog
import os,getpass,syslog
import time,datetime
import smtplib
from email.mime.text import MIMEText
from email.mime.multipart import MIMEMultipart
TDS_LOG_HOME="/opt/trc/stp" #TDS系统的交易日志路径
LATEST_LOGFILE="/opt/trc/latest.list" #最近已扫描的文件列表
LOG="log" #当前脚本的日志
KEY_WORD=[' ERROR - '] #查找ERROR日志用的关键词
STRIP_WORD=['加锁失败','手机号码不能为空','执行process出错','[STHDSMK1]未发送']
EMAIL_ATTACHMENT=[] #邮件的附件
EMAIL_TO=['abc1@abc.com','abc2@abc.com','abc3@abc.com','abc4@abc.com'] #收件人列表
SMTP_SERVER="smtp.ym.163.com"
EMAIL_FROM="monitor@abc.com"
EMAIL_USERNAME=EMAIL_FROM
EMAIL_PASSWORD="password1234"
EMAIL_SUBJECT="日志报警:互联网支付生产系统"
STRIP_FILE=['swp']
EMAIL_CSS_STYLE="""
<style type="text/css">
table {
background-color: #F2F2F2;
border-spacing: 0px;
border-collapse: collapse;
padding:1em;
margin:2em auto;
width:100%;
max-width:960px;
}
th {
background-color: #48A1CA;
color: #FFF;
font-size: 15px;
text-align:right;
width:12%;
}
td {
width:98%;
}
tr{
line-height:1.4em;
}
td,th {border: 1px solid #FFF;
padding: 0.7em 1em;
}
</style>
"""
def print_and_log(string,level="INFO",Syslog=False):
"""
在屏幕上显示信息,并记录到syslog及日志文件中
"""
#print string #调试时找开
if isinstance(string, str):
#如果string是字符串,且长度大于0
if len(string)>0:
timestamp=time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time()))
print >> LOG_FILE,"[%s] %s - %s" % (timestamp,level,string)
if level=="ERROR" or level=="WARM":
syslog.openlog("tdshome_log_analysis.py")
syslog.syslog(string)
else:
pass
else:
#如果string非字符串,则只记录进log文件
timestamp=time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time()))
print >> LOG_FILE,"[%s] %s - %s" % (timestamp,level,string)
def isListPartInString(string,List):
"""
判断列表内容是有否部分/全部包含在字符串中
"""
result=False
for line in List:
if line in string:
result=True
return result
def filterIrrelevantFile(FileList=[]):
"""
过滤掉不相关的文件
"""
itemList=[]
for line in FileList:
if not isListPartInString(line,STRIP_FILE):
itemList.append(line)
return itemList
def addEmailAttach(f):
"""
给邮件添加附件
"""
attach = MIMEText(open(f,"rb").read(),"base64","utf-8")
attach["Content-Type"]="application/octest-stream"
attach["Content-Disposition"]="attachment;filename=%s" % os.path.basename(f)
attach["Content-ID"]="%s" % os.path.basename(f)
return attach
def send_mail(mail_from,mail_to,text_msg,attachment):
"""
发送邮件
"""
print_and_log("准备发送邮件...")
msg=MIMEMultipart('related')
msgtext=MIMEText(text_msg,"html","utf-8")
#添加邮件内容
msg.attach(msgtext)
print_and_log("添加内容:%s" % text_msg)
#添加附件
for f in attachment:
msg.attach(addEmailAttach(f))
print_and_log("添加附件:%s" % f)
#设置邮件消息头
msg['Subject']=EMAIL_SUBJECT
msg['From']=EMAIL_FROM
print_and_log("添加收件人:%s" % (',').join(EMAIL_TO))
try:
#登录邮件服务器
server=smtplib.SMTP()
server.connect(SMTP_SERVER,"25")
server.login(EMAIL_USERNAME,EMAIL_PASSWORD)
#print msg.as_string()
#发邮件
server.sendmail(EMAIL_FROM,EMAIL_TO,msg.as_string())
server.quit()
print_and_log("邮件发送成功!")
return 0
except Exception,e:
print_and_log(e,level="ERROR")
print_and_log("邮件发送失败!",level="ERROR",Syslog=True)
return 1
def getOrderNumber(LogFile):
"""
用来获取商户号
"""
ORDKEY="[Prdordno]"
of=open(LogFile,"r")
#下面几行用来获取商户号
OrderNumber="unknown"
for ordnum in of.readlines():
if ORDKEY in ordnum:
OrderNumber=ordnum.replace(r']',' ').replace(r'[',' ').split()[-1]
of.close()
return OrderNumber
def getMsgAndTimestamp(line):
"""
分离消息内容与时间戳
"""
msg=line.split(' ')
if msg[1] == 'ERROR':
#格式如:"2015-07-22 08:39:23s,344 ERROR - 交易[Remote0000000031896_600106]执行出错 : 没有找到交易码600106!"
timestamp=msg[0]
message=(' ').join(msg[1:])
else:
#格式如:"10:06:22,397 ERROR - [Remote0000000032172_YSE001]:Lock: 登记锁失败. aRecKey=[15072200000049] : ORA-00001: 违反唯一约束条件 (HRDBNET.PK_PUBLCKREC)"
timestamp=(' ').join(msg[0:2])
message=(' ').join(msg[2:])
return timestamp,message
def writeLogFile(File,String="",List=[]):
#把当前状态写入最后一次状态
try:
of=open(File,"w")
if len(String)>0:
of.writelines(String)
elif len(List)>0:
for line in List:
of.writelines("%s\n" % line)
else:
pass
of.close()
except IOError,e:
print(e)
def getAllLogFile():
"""
获取当天所有的日志文件的列表
"""
#获取时间信息
CurrentYear=time.strftime("%Y",time.localtime())
CurrentMonth=time.strftime("%m",time.localtime())
CurrentDate=time.strftime("%d",time.localtime())
CurrentHour=time.strftime("%H",time.localtime())
Yesterday = datetime.date.today() - datetime.timedelta(days=1)
LastYear =Yesterday.strftime("%Y")
LastMonth =Yesterday.strftime("%m")
LastDate =Yesterday.strftime("%d")
TodayLogFileDir="%s/%s%s/%s" % (TDS_LOG_HOME,CurrentYear,CurrentMonth,CurrentDate)
LastLogFileDir="%s/%s%s/%s" % (TDS_LOG_HOME,LastYear,LastMonth,LastDate)
#获取今天的文件列表
AllLogFile=[]
if os.path.exists(TodayLogFileDir):
for line in os.listdir(TodayLogFileDir):
AllLogFile.append("%s/%s" % (TodayLogFileDir,line))
else:
print_and_log("目录 %s 不存在" % TodayLogFileDir, level="WARM", Syslog=True)
#00:00 ~ 02:00 则仍需扫描前一天的日志
if CurrentHour < 2 :
if os.path.exists(LastLogFileDir):
for line in os.listdir(LastLogFileDir):
AllLogFile.append("%s/%s" % (LastLogFileDir,line))
else:
print_and_log("目录 %s 不存在" % LastLogFileDir, level="WARM", Syslog=True)
else:
pass
return AllLogFile
def getLastScannedFileList():
"""
获取最近一次已扫描的文件的列表
"""
LastScannedFileList=[]
try:
of=open(LATEST_LOGFILE,"r")
for line in of.readlines():
LastScannedFileList.append(line.strip())
of.close()
except IOError,e:
print(e)
return LastScannedFileList
def getUnscannedFileList(AllLogFile=[],LastScannedFileList=[]):
"""
获取未被扫描过的文件的列表
"""
UnscannedFileList=[]
for line in AllLogFile:
if line not in LastScannedFileList:
UnscannedFileList.append(line)
return filterIrrelevantFile(UnscannedFileList)
if __name__ == "__main__":
try:
LOG_FILE=open(LOG,"a")
except IOError,e:
print (e)
print_and_log("=====用户 %s 执行 %s =====" % (getpass.getuser(),os.path.basename(__file__)))
current_logfile=getAllLogFile()
latest_logfile=getLastScannedFileList()
#获取未扫描过的文件列表
new_logfile=getUnscannedFileList(current_logfile, latest_logfile)
#扫描新文件
email_content=EMAIL_CSS_STYLE
for new_file in new_logfile:
try:
of=open(new_file,"r")#匹配结果用
#具体的日志分析工作在这里
LineNumber=1
for line in of.readlines():
for key_word in KEY_WORD:
if key_word in line:#找到关键
#排除不关注的内容
if not isListPartInString(line, STRIP_WORD):
#记录匹配的结果
if new_file not in EMAIL_ATTACHMENT:
EMAIL_ATTACHMENT.append(new_file)
timestamp,message=getMsgAndTimestamp(line)
email_content+="""
<table>
<tr><th>日志文件</th><td><a href="cid:%s" target="_blank">%s</a></td></tr>
<tr><th>行号</th><td>%s</td></tr>
<tr><th>时间</th><td>%s</td></tr>
<tr><th>订单号</th><td>%s</td></tr>
<tr><th>消息内容</th><td>%s</td></tr>
</table>
""" % (os.path.basename(new_file),new_file,LineNumber,timestamp,getOrderNumber(new_file),message)
LineNumber+=1
of.close()
except IOError,e:
print(e)
if len(EMAIL_ATTACHMENT)>0:
#如果有报警则发邮件
send_email_result=send_mail(mail_from=EMAIL_FROM, mail_to=EMAIL_TO, text_msg=email_content, attachment=EMAIL_ATTACHMENT)
if send_email_result==0:
#如果邮件发送成功则修改最近已扫描的文件列表"latest.list"
writeLogFile(File=LATEST_LOGFILE,List=current_logfile)
else:
#如果没有报警则直接修改"latest.list"
writeLogFile(File=LATEST_LOGFILE,List=current_logfile)
print_and_log("非常好,没有查到报警信息!")