重点URL访问监控-当天和前一天相同时间段内的访问对比-python

该脚本实现当天的重点URL访问量和头一天相同小时时间段内的访问量对比，超过指定的阈值则报警通知并且记录日志

#! /usr/local/bin/python3

import bz2
import re
import contextlib
import shelve
import datetime
import sys
#enviroment.py为提供一些公共函数的函数库，部分使用到的函数下文给出
import enviroment as eo
'''
def get_log(log_filename,log_format=
'[%(asctime)-15s] [%(levelname)s] [%(filename)s] [%(funcName)s] [%(lineno)d] %(message)s'
,log_level='debug'):
    if log_level=='debug':
        logging.basicConfig(format=log_format,filename=log_filename,level=logging.DEBUG)
def get_local_ip_tag():
    short_ip=(subprocess.check_output(
'/sbin/ifconfig | grep eth0 -A1 | tail -1 | awk \'{print $2}\' | awk -F "." \'{print $4}\''
,shell=True)
.decode().strip())
    return short_ip+'log:'
#下文中的 eo.sms_XX(message) 函数都是对该函数的再包装，提供receivers
def send_message(receivers,message):
    for receiver in receivers:
        warn_url='http_sendmessage_interface?'
        query_args={'username':receivers[receiver],'message':get_local_ip_tag()+message}
        encoded_args=urllib.parse.urlencode(query_args)
        warn_url=warn_url+encoded_args
        response=urllib.request.urlopen(warn_url)
'''

def generate_previous_hour():
    format='%d/%b/%Y:%H'
    previous_hour_time=datetime.datetime.today().timetuple().tm_hour-1
    return previous_hour_time,(datetime.datetime.today()-datetime.timedelta(hours=1)).strftime(format)

def check_hour_logs(logpath,keys,examine_hour):
    result={key:0 for key in keys}
    regex_previous_hour=re.compile(examine_hour)
#由于日志是按照时间顺序，而且时间戳字段line.split(' ')[3]在一个月内可以按照字典顺序比较
#这里可以改进成时间比较。
#cron的本身暂时设定为0点小时不执行，所以不会触发跨月的这个漏洞
#这里不进行时间比较还有跟实际使用脚本相关的其他原因，字典顺序够用。
#搜索指定小时的日志，可以跨越式前进搜索
#这里增量设定为10M，可以根据实际的日志大小进行设定
    step=10*1024*1024
    with open(logpath) as file:
        line=file.readline()
        while line:
            time_line=line.split(' ')[3][1:]
            if time_line>examine_hour:
                file.seek(file.tell()-step-10240)
                #再读一行保证下一行为完整行
                file.readline()
                break
            file.seek(file.tell()+step)
            file.readline()
            line=file.readline().strip()
        for line in file:
            line=line.strip()
            for key in keys:
                regex_key=re.compile(key)
                words=line.split(' ')
#words的第四个字段为时间戳字段、第七个字段为访问URL
                if regex_previous_hour.search(words[3]) and regex_key.search(words[6]):
                    result[key]+=1
    return result

def compare_data(db,previous_hour_time,examine_keys):
    with contextlib.closing(shelve.open(db)) as dba:
        yesterday_key='yesterday'
        today_key='today'
        if yesterday_key in dba:
            try:
                yesterday=dba[yesterday_key]
                today=dba[today_key]
                for key in examine_keys:
#报警阈值为今天的某个URL访问量大于400，并且是昨天相同时间段的两倍
                    if int(today[previous_hour_time][key]) > 2*int(yesterday[previous_hour_time][key]) and int(today[previous_hour_time][key])>400:
                        eo.logging.error('alarm. %s too large.the number of visits is %s',key,today[previous_hour_time][key])
                        message='alarm. {0} too large.the number of visits is {1}'.format(key,today[previous_hour_time][key])
                        eo.sms_XX(message)  
            except KeyError:
                eo.logging.warning('%s',sys.exc_info()[0:2])

def store_data(db,previous_hour_data,previous_hour_time):
    yesterday_key='yesterday'
    today_key='today'
    with contextlib.closing(shelve.open(db,writeback=True)) as dba:
        if today_key not in dba:
            today={}
        else:
            today=dba[today_key]
            today[previous_hour_time]=previous_hour_data
            if int(previous_hour_time)==23:
                dba[yesterday_key]=dba[today_key].copy()

#使用这些函数可以方便的操作db，比如查看可以使用print_all_data函数
#这些方法也可以放在environment.py中
def put_data(filename,key,saved_data):
    with contextlib.closing(shelve.open(filename)) as dba:
        dba[key]=saved_data

def get_data(filename,key):
    with contextlib.closing(shelve.open(filename)) as dba:
        return dba[key]

def print_all_data(filename):
    with contextlib.closing(shelve.open(filename)) as dba:
        for key in dba:
            print(key+':'+str(dba[key]))

def delete_data(filename,key=0):
    with contextlib.closing(shelve.open(filename)) as dba:
        try:
            if key:
                del dba[key]
            else:
                dba.clear()
        except KeyError:
            print('find a KeyError no key:'+key)

def main(log_name,db,examine_keys,my_log):
    eo.get_log(my_log)
    previous_hour_time,previous_hour=generate_previous_hour()
    previous_hour_data=check_hour_logs(log_name,examine_keys,previous_hour)
    store_data(db,previous_hour_data,previous_hour_time)
    compare_data(db,previous_hour_time,examine_keys)

if __name__ == '__main__':
    log_name=''
    basedir=''
    db=basedir+'/examine_important_url_hours.db'
    my_log=basedir+'/run.log'
    examine_keys=[]
    main(log_name,db,examine_keys,my_log)