一. Python 监控脚本说明
DB 服务器是Redhat 5.5. 系统上的Python 是2.4. 所以这套监控脚本也是基于Python 2.* 版本。 Python 3.0 版本的语法上有一定的出入。
1. 1 几点说明：
（1）Python 严格的区分格式的，所以，格式要对其。
（2）脚本里用到了dns.resolver包，脚本里有下载地址。 安装方法参考dns.resolver包的readme文档。
（3）脚本分成2部分，所有的配置信息，放到了healthcheck_config.py里面。 监控部分代码放在了healthcheck.py里了。 配置文件里设定了警告值，超过这个值就会发送报警信息。
（4）本打算把RMAN 的监控也放到里面的。 但是RMAN 每天只需要运行一次。 而这个监控脚本是10分钟执行一次。 所以RMAN 备份的log 会在单独弄个脚本来处理。
（5）脚本里面调用了一些系统命令。 如ifconfig，这个命令只有root用户能执行，所以这个监控脚本也必须使用root用户来部署。 在调试这个脚本的时候，浪费了很多的时间。
（6）脚本里使用了2种方式来发送警告信息，一个是使用smtp发送到邮箱，另一个是通过短信接口发送到手机。 现在一般公司都有自己的邮件服务器。 所以这些在内网就能直接搞定了。 如果说没有短信接口的话，可以发送到移动的139邮箱，或者qq 邮箱， 这2个邮箱都可以设置短信通知邮件。
（7） 这里把所有要监控的服务器，全部添加到脚本配置文件里。 这样在移植的时候，直接部署过去就可以了。 不需要进行其他的修改。
1.2 Python 的基础知识参考一下链接：
Python 基础语法知识一
http://blog.csdn.net/tianlesoftware/archive/2011/02/11/6179869.aspx
Python 基础语法知识二
http://blog.csdn.net/tianlesoftware/archive/2011/02/12/6181166.aspx
Python 执行 系统命令
http://blog.csdn.net/tianlesoftware/archive/2011/02/17/6192202.aspx
二. 监控脚本
2.1 . 参数文件 healthcheck_config.py
#!/usr/bin/python
# coding=gbk
#################################################
# Created by Tianlesoftware
# 2011/2/23
##################################################
# 注意, 如果需要换行时缩进需要跟原有的代码保持统一
##################################################
#CPU 使用率警戒值
#wa: Time spent waiting for IO. Prior to Linux 2.5.41, shown as zero.
MAX_CPU_USED = 70
MAX_WA = 20
FILTERS = ['/boot', '/dev/shm']
MIN_FREE_MB = 200
#磁盘空间使用率
MAX_USED_PCT = 80
#邮件服务器地址及发送邮件账户和密码
MAIL_FROM='tianlesoftware@vip.qq.com'
SMTP_SERVER='192.168.1.100'
EMAIL_USER='user'
EMAIL_PASSWD='pwd'
#监控的进程信息，以下进程都将被监控
PROCESS = {
'192.168.1.1': ['LISTENER','ora_pmon_wangou','ora_lgwr_wangou','ora_dbw0_wangou','ora_ckpt_wangou','ora_smon_wangou','ora_arc0_wangou'],
'192.168.1.2': ['LISTENER','ora_pmon_wangou','ora_lgwr_wangou','ora_dbw0_wangou','ora_ckpt_wangou','ora_smon_wangou','ora_arc0_wangou'],
'192.168.1.3': ['LISTENER','ora_pmon_xezf','ora_lgwr_xezf','ora_dbw0_xezf','ora_ckpt_xezf','ora_smon_xezf','ora_arc0_xezf'],
'192.168.1.4': ['LISTENER','ora_pmon_xezf','ora_lgwr_xezf','ora_dbw0_xezf','ora_ckpt_xezf','ora_smon_xezf','ora_arc0_xezf'],
'192.168.1.5': ['LISTENER','ora_pmon_XEZF','ora_lgwr_XEZF','ora_dbw0_XEZF','ora_ckpt_XEZF','ora_smon_XEZF','ora_arc0_XEZF'],
}
#监控的文件信息
FILES = {
'192.168.1.1': [
{'file':'/home/oracle_app/admin/XEZF/bdump/alert_XEZF.log', 'type':'ora-log'},
{'file':'/u01/backup/logs/rman.log', 'type':'rman-log'}],
'192.168.1.2': [
{'file':'/u01/scripts/rman_backup.sh.out','type':'rman-log'},
{'file':'/u01/app/oracle/admin/wangou/bdump/alert_wangou.log', 'type':'ora-log'}],
'192.168.1.3': [{'file':'/u01/app/oracle/admin/wangou/bdump/alert_wangou.log', 'type':'ora-log'}],
'192.168.1.4': [{'file':'/u01/app/oracle/admin/xezf/bdump/alert_xezf.log', 'type':'ora-log'}],
'192.168.1.5': [
{'file':'/u01/backup/logs/rman.log','type':'rman-log'},
{'file':'/u01/app/oracle/admin/xezf/bdump/alert_xezf.log', 'type':'ora-log'}]
}
#默认邮箱和手机是DBA的联系方式
DEFAULT_MAILS = ['tianlesoftware@vip.qq.com']
DEFAULT_MOBILES = '13888888888'
#对应服务器的邮件接收人
MAILS = {
'192.168.1.1': ['tianlesoftware@vip.qq.com',' tianlesoftware@gmail.com'],
'192.168.1.2': ['tianlesoftware@vip.qq.com'],
'192.168.1.3': ['tianlesoftware@vip.qq.com'],
'192.168.1.4': ['tianlesoftware@vip.qq.com'],
'192.168.1.5': ['tianlesoftware@vip.qq.com']
}
#对应服务器负责人手机号
MOBILES = {
'192.168.1.1': '13688888888,13888888888',
'192.168.1.2': '13688888888,13888888888',
'192.168.1.3': '13688888888,13888888888',
'192.168.1.4': '13688888888,13888888888',
'192.168.1.5': '13688888888,13888888888'
}
2.2. Python 监控脚本： healthcheck.py
#!/usr/bin/python
# coding=gbk
#################################################
# Created by Tianlesoftware
# 2011/2/23
########################################################
# require dns.resolver
# download url: http://www.dnspython.org/kits/1.6.0/
import os
import time
import datetime
import sys
import smtplib
import dns.resolver
import re
import random
import httplib
import urllib2
import urllib
import hc_config
MAX_CPU_USED = hc_config.MAX_CPU_USED
MAX_WA = hc_config.MAX_WA
FILTERS = hc_config.FILTERS
MIN_FREE_MB = hc_config.MIN_FREE_MB
MAX_USED_PCT = hc_config.MAX_USED_PCT
PROCESS = hc_config.PROCESS
MAILS = hc_config.DEFAULT_MAILS
MOBILES = hc_config.DEFAULT_MOBILES
httplib.HTTPConnection.debuglevel = 1
def uniqify(seq, idfun=None):
# order preserving
if idfun is None:
def idfun(x): return x
seen = {}
result = []
for item in seq:
marker = idfun(item)
# in old Python versions:
# if seen.has_key(marker)
# but in new ones:
if marker in seen: continue
seen[marker] = 1
result.append(item)
return result
def get_ip():
pipe = os.popen('/sbin/ifconfig')
data = pipe.read().strip()
p = re.compile(r'inet addr:([/./d]+)')
r = p.findall(data)
if len(r) > 0:
return r[0]
ip = get_ip()
IP = ip
if hc_config.MAILS.has_key(ip):
MAILS = MAILS + hc_config.MAILS[ip]
MAILS = uniqify(MAILS)
if hc_config.MOBILES.has_key(ip):
MOBILES = MOBILES + ',' + hc_config.MOBILES[ip]
mobiles = MOBILES.split(',')
mobiles = [x.strip() for x in mobiles]
mobiles = uniqify(mobiles)
if len(mobiles) > 0:
MOBILES = mobiles[0]
if len(mobiles) > 1:
for x in mobiles[1:]:
MOBILES = MOBILES + ',' + x
FILES = []
if hc_config.FILES.has_key(ip):
FILES = hc_config.FILES[ip]
########################################################
def load(fname='warn.dat'):
import pickle
fp=open(fname)
d2=fp.read()
fp.close()
o=pickle.loads(d2)
return o
def save(obj,fname='warn.dat'):
# save to file
import pickle
d2 = pickle.dumps(obj)
fp=open(fname, 'w')
fp.write(d2)
fp.close()
########################################################
# for sms
def sendsms(dt,msg):
if len(msg) > 150:
message = msg[0:149]
else:
message = msg
print 'Send to %s %s' % (dt, msg)
f = { 'msg':message }
m = urllib.urlencode(f)
#这里是短信接口的地方，写上自己的短信接口就可以了
url = 'http://.../%s&%s' % (dt,m)
htm = urlget(url)
print htm
def sendsms_s(msg):
sendsms(MOBILES, msg)
def urlget2(url):
request = urllib2.Request(url)
handle = urllib2.urlopen(request)
htm = handle.read()
return htm
def urlget(url):
try:
request = urllib2.Request(url)
handle = urllib2.urlopen(request)
htm = handle.read()
except urllib2.URLError, err:
errno = err[0][0]
errmsg = err[0][1]
print errno, 'AAA', errmsg
if errno == -2:
request = urllib2.Request(url)
handle = urllib2.urlopen(request)
htm = handle.read()
else:
htm = str(err)
except:
htm = str(sys.exc_info())
return htm
########################################################
# Send mail ....
SMTP_SERVER=hc_config.SMTP_SERVER
EMAIL_USER=hc_config.EMAIL_USER
EMAIL_PASSWD=hc_config.EMAIL_PASSWD
def sendmail(fromaddr, toaddr, subject, body):
server=smtplib.SMTP(SMTP_SERVER)
server.login(EMAIL_USER,EMAIL_PASSWD)
#server.set_debuglevel(1)
#server.sendmail(fromaddr, toaddrs, msg)
msg = 'From: %s/nTo: %s/nSubject: %s/n/n%s/n' % (fromaddr, toaddr, subject, body)
server.sendmail(fromaddr, toaddr, msg)
server.quit()
def sendmail_s(fromaddr, toaddrs, subject, body):
for toaddr in toaddrs:
try:
sendmail(fromaddr, toaddr, subject, body)
except:
info = str(sys.exc_info())
print info
######################################
#
# Requirement:
# General:
# FS: free space < 5%，or < 200M
# CPU: Usage > 70%
# Log: dmesg / messages
#文件系统 1K-块 已用 可用 已用% 挂载点
#/dev/mapper/VolGroup00-LogVol00 14855176 3665468 10422940 27% /
#/dev/sda1 101086 10979 84888 12% /boot
#tmpfs 1037848 0 1037848 0% /dev/shm
#/dev/mapper/VolGroup00-Home 51735156 13732272 35374892 28% /home
def get_df_data():
cmd = "df -k | awk '(NF > 1){ printf /"%s//n/", $0 }(NF==1){ printf /"%s /", $0}'"
pipe = os.popen(cmd)
data = pipe.read().strip()
lines = data.split('/n')
mps = []
for line in lines[1:]:
fields = line.split()
mp = {}
mp['mount_point'] = fields[-1].strip()
mp['used'] = int(fields[-2].strip().replace('%',''))
mp['free'] = int(fields[-3].strip())
mp['freeM'] = int(fields[-3].strip())/1024
mps.append(mp)
return mps
#procs -----------memory---------- ---swap-- -----io---- --system-- -----cpu------
#r b swpd free buff cache si so bi bo in cs us sy id wa st
#2 0 0 692900 380608 803308 0 0 1 11 20 1 0 0 99 1 0
#0 0 0 692900 380608 803308 0 0 0 0 1012 94 0 0 100 0 0
def get_vmstat_data():
pipe = os.popen('vmstat -n 1 3')
data = pipe.read().strip()
lines = data.split('/n')
wa = 0
idle = 0
for line in lines[3:]:
fields = line.split()
wa = wa + int(fields[-2].strip())
idle = idle + int(fields[-3].strip())
wa = wa / len(lines[3:])
idle = idle / len(lines[3:])
used = 100-wa-idle
ret = {'wa':wa, 'idle':idle, 'used':used }
return ret
#print 'wa=%d idle=%d used=%d' % (wa, idle, used)
def file_data(fname):
cmd = 'cat "%s"' % (fname)
pipe = os.popen(cmd)
data = pipe.read().strip()
return data
def get_ps_data():
pipe = os.popen('ps -ef')
data = pipe.read().strip()
return data
mail_from =hc_config.MAIL_FROM
vmstat = get_vmstat_data()
mps = get_df_data()
#########################################
# FS: free space < 5%，or < 200M
#########################################
body = ''
ip=get_ip()
print ip
for item in mps:
#print item
mp = item['mount_point']
body = body + str(item) + '/n'
if mp not in FILTERS:
alarm = False
msg = '%s (%s) ' % (ip, mp)
if item['freeM'] < MIN_FREE_MB:
msg = msg + 'freeM(%dM)<%dM ' % (item['freeM'],MIN_FREE_MB)
alarm = True
if item['used'] > MAX_USED_PCT:
msg = '%s (%s) used(%d%%)>%d%%' % (ip,mp,item['used'],MAX_USED_PCT)
alarm = True
if alarm:
sendmail_s(mail_from, MAILS, msg, msg)
sendsms_s(msg);
print msg
# CPU: Usage > 70%
if vmstat['used'] > MAX_CPU_USED:
subject = '%s cpu overload %d%% > %d%% ' % (ip, vmstat['used'], MAX_CPU_USED)
body = 'Host %s cpu overload, vmstat: %s' % (ip, str(vmstat))
sendmail_s(mail_from, MAILS, subject, body)
msg = '%s cpu overload %d%% > %d%% ' % (ip, vmstat['used'], MAX_CPU_USED)
sendsms_s(msg);
if vmstat['wa'] > MAX_WA:
subject = '%s cpu wa overload %d%% > %d%% ' % (ip, vmstat['wa'], MAX_WA)
body = 'Host %s cpu wa overload, vmstat: %s' % (ip, str(vmstat))
sendmail_s(mail_from, MAILS, subject, body)
msg = '%s cpu wa overload %d%% > %d%% ' % (ip, vmstat['wa'], MAX_WA)
sendsms_s(msg);
if PROCESS.has_key(ip):
data = get_ps_data()
process = PROCESS[ip]
for proc in process:
if data.find(proc) == -1:
msg = '%s %s is not running' % (ip, proc)
sendmail_s(mail_from, MAILS, msg, data)
sendsms_s(msg)
def ora_log_handle(data,fname,last_ltime=0):
ltime = last_ltime
p = re.compile(r'((Sun|Mon|Tue|Wed|Thu|Fri|Sat) (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) /d+ /d+:/d+:/d+ /d+)', re.DOTALL)
rr = p.findall(data)
if len(rr) > 0:
start = 0
pos = []
for r in rr:
pp = data.find(r[0],start)
if pp != -1:
pos.append(pp)
llen = len(r[0])
start = pp + llen
fdata = [data[pos[i]:pos[i+1]] for i in range(len(pos)-1)]
fdata.append(data[pos[-1]:])
warns = []
for log_item in fdata:
#print '[%s]' % (log_item)
rr2 = p.findall(log_item)
if len(rr2) > 0:
ts = rr2[0][0]
d2 = time.strptime(ts,'%a %b %d %H:%M:%S %Y')
t = time.mktime(d2)
print t, ltime
if log_item.find('ORA-') != -1 and t > ltime:
warns.append((t, log_item[len(ts):]))
for x in warns:
t = x[0]
line = x[1]
tt = datetime.datetime.fromtimestamp(int(t))
ts = tt.strftime('%m/%d_%H:%M:%S')
msg = '%s %s %s %s' % (ip, fname, ts, line)
sendsms_s(msg)
sendmail_s(mail_from, MAILS, msg, msg)
ltime = int(t)
return ltime
#FILE_HANDLES = { 'rman-log':rman_log_handle, 'ora-log':ora_log_handle }
FILE_HANDLES = {'ora-log':ora_log_handle }
DATA_FILE = 'file.dat'
file_objs = {}
if os.path.exists(DATA_FILE) == True:
file_objs = load(DATA_FILE)
else:
save(file_objs, DATA_FILE)
for file in FILES:
fname = file['file']
ty = file['type']
if os.path.exists(fname) == True and FILE_HANDLES.has_key(ty):
print 'checking ', fname
last_mtime = 0
last_ltime = 0
mtime = os.stat(fname).st_mtime
if file_objs.has_key(fname):
last_mtime = file_objs[fname]['mtime']
last_ltime = file_objs[fname]['ltime']
if mtime > last_mtime:
print fname
fdata = file_data(fname)
handle = FILE_HANDLES[ty]
ltime = handle(fdata,fname,last_ltime)
file_objs[fname] = {'mtime':mtime, 'ltime':ltime}
save(file_objs, DATA_FILE)
#print data
2.3. 添加到crontab
脚本添加到crontab ，每10分钟执行一次，注意，是root用户：
[root@qs-wg-db1 pymonitor]# crontab -l
*/10 * * * * /u01/scripts/pymonitor/hc.py >>/u01/scripts/pymonitor/hc.log 2>1&
Linux Crontab 定时任务 命令详解
http://blog.csdn.net/tianlesoftware/archive/2011/02/17/6192202.aspx
------------------------------------------------------------------------------
Blog： http://blog.csdn.net/tianlesoftware
网上资源： http://tianlesoftware.download.csdn.net
相关视频：http://blog.csdn.net/tianlesoftware/archive/2009/11/27/4886500.aspx
DBA1 群：62697716(满); DBA2 群：62697977(满)
DBA3 群：62697850 DBA 超级群：63306533;
聊天 群：40132017
--加群需要在备注说明Oracle表空间和数据文件的关系，否则拒绝申请
1万+
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
