基于python的open-falcon 多日志多关键字监控插件

最近公司业务运维对日志要求关键字监控，网上的open-logdog是基于go语言开发的插件，编译测试发现有些问题不符合自己公司的要求，迫于时间压力，使用python进行了改造，满足多日志、多关键字、忽略大小写及定时推送数据到接口，主代码如下：

#!/usr/bin/python
# coding=utf-8

import sys
import re
import time
from commands import getstatusoutput as getcmd
import threading
import subprocess
from collections import Counter as counter
import sched
from config import conf

class LogMonitor(object):

    def __init__(self):
        self.logfile = '/tmp/info.log'
        self.tmp = []
        self.data = {}
        self.key = {}
        self.schedule = sched.scheduler(time.time, time.sleep)

        self.type_post = [
            "LogKeys"
        ]
        self.data_post = {
            "metric": conf["metric"],
            "endpoint": "",
            "timestamp": 1,
            "step": conf["timer"],
            "value": 0,
            "counterType": "GAUGE",
            "tags": ""
        }

    def timer(self):
        self.post(inc=conf["timer"])

    def watch_file(self, path, logname, keywords):
        logfile = path + logname
        try:
            popen = subprocess.Popen('tail -f ' + logfile, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            while True:
                line = popen.stdout.readline().strip()
                if re.findall(r'(?i)%s' % keywords, line):
                    rt = map(str.lower, re.findall(r'(?i)%s' % keywords, line))
                    for k in rt:
                        self.tmp.append(k)
                    self.key[logname] = counter(self.tmp)
                    self.data.update(self.key)

        except Exception, e:
            print e

    def collect_data(self):
        try:
            import json
        except:
            import simplejson as json
        data = ""
        timestamp = int(time.time())
        _, hostname = getcmd('hostname')
        _, ip = getcmd(''' ip a|awk '{if($0~/inet .* brd/ && $2~/^10|^192.168|^172.(1[6-9]|2[1-9]|3[0-1])/) {gsub("/.*|^ | $","",$0); print $2;exit}}' ''')
        endpoint = hostname + "-" + ip
        for i in conf.get("files"):
            logname = i.get("logname")
            keywords = i.get("keywords")
            for key in keywords:
                self.data_post["timestamp"] = timestamp
                self.data_post["endpoint"] = endpoint
                self.data_post["tags"] = "log="+logname + ",key=" + key
                if self.data.get(logname):
                    if self.data.get(logname).get(key.lower()):
                        self.data_post["value"] = self.data.get(logname).get(key.lower())
                    else:
                        self.data_post["value"] = 0
                else:
                    self.data_post["value"] = 0


                data_t = json.dumps(self.data_post)
                if data == "":
                    data = "%s" % (data_t)
                else:
                    data = data + ",%s" % data_t
        return data

    def post(self, cmd='', inc=60):
        url = conf.get("agent")
        data = self.collect_data()
        data = "[%s]" % (data)
        cmd = "/usr/bin/curl -X POST -d  '%s' %s" % (data, url)
        getcmd(cmd)
        self.record('info',data)
        self.tmp = []
        self.key = {}
        self.data = {}
        self.schedule.enter(inc, 0, self.post, (cmd, inc))
        self.schedule.run()

    def record(self, level, messages):
        import logging
        logger = logging.getLogger()
        if not logger.handlers:
            logger.setLevel(logging.DEBUG)
            w_log = logging.FileHandler(self.logfile)
            # formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s : %(message)s')
            formatter = logging.Formatter('%(asctime)s-%(levelname)s:  %(message)s')
            w_log.setFormatter(formatter)
            logger.addHandler(w_log)

        if level.lower() == "info":
            logger.info(messages)
        elif level.lower() == "debug":
            logger.debug(messages)
        elif level.lower() == "warn":
            logger.warning(messages)
        elif level.lower() == "error":
            logger.error(messages)
        else:
            logger.info(messages)

    def echo(self, messages, color='green', backgroud=False, raw=False):
        backgroud = '4' if backgroud else '3'

        color_dict = {'red': '1', 'green': '2', 'yellow': '3', 'blue': '4', 'purple': '5', }
        if color in color_dict:
            msgs = '\033[%s%sm%s\033[0m' % (backgroud, color_dict.get(color), messages)
        else:
            msgs = messages

        if raw:
            v = raw_input(msgs).strip()
        else:
            print(msgs)
            v = None
        if color == "red":
            self.record('error', messages)
        elif color == "yellow":
            self.record('warn', messages)
        else:
            self.record('info', messages)

        return v

def main():
   
    u = LogMonitor()
    data = conf.get("files")
    threads = []
    nloops = range(len(data) + 1)
    for v in data:
        keywords = '|'.join(v.get("keywords"))
        path = v.get("path")
        logname = v.get("logname")
        t = threading.Thread(target=u.watch_file, args=(path, logname, keywords))
        threads.append(t)


    t1 = threading.Thread(target=u.timer, args=())
    threads.append(t1)


    for i in nloops:
        threads[i].start()


    for i in nloops:
        threads[i].join()


if __name__ == '__main__':
    try:
        main()
    except BaseException as e:
        print e

配置文件config.py如下：

conf = {
  "metric": "log",
  "timer": 30,
  "agent": "http://127.0.0.1:1988/v1/push",
  "files": [
    {
      "path": "/root/test/tmp/",
      "logname": "host.log",
      "keywords": ["error", "test"]
    },
    {
      "path": "/root/test/tmp/",
      "logname": "host2.log",
      "keywords": ["error", "test"]
    }

  ]
}

日志如下：

2018-01-31 16:36:36,612-INFO:  [{"endpoint": "zww-192.168.138.136", "tags": "log=host.log,key=error", "timestamp": 1517387796, "metric": "log", "value": 0, "counterType": "GAUGE", "step": 30},{"endpoint": "zww-192.168.138.136", "tags": "log=host.log,key=test", "timestamp": 1517387796, "metric": "log", "value": 0, "counterType": "GAUGE", "step": 30},{"endpoint": "zww-192.168.138.136", "tags": "log=host2.log,key=error", "timestamp": 1517387796, "metric": "log", "value": 0, "counterType": "GAUGE", "step": 30},{"endpoint": "zww-192.168.138.136", "tags": "log=host2.log,key=test", "timestamp": 1517387796, "metric": "log", "value": 0, "counterType": "GAUGE", "step": 30}]
2018-01-31 16:37:06,672-INFO:  [{"endpoint": "zww-192.168.138.136", "tags": "log=host.log,key=error", "timestamp": 1517387826, "metric": "log", "value": 0, "counterType": "GAUGE", "step": 30},{"endpoint": "zww-192.168.138.136", "tags": "log=host.log,key=test", "timestamp": 1517387826, "metric": "log", "value": 0, "counterType": "GAUGE", "step": 30},{"endpoint": "zww-192.168.138.136", "tags": "log=host2.log,key=error", "timestamp": 1517387826, "metric": "log", "value": 0, "counterType": "GAUGE", "step": 30},{"endpoint": "zww-192.168.138.136", "tags": "log=host2.log,key=test", "timestamp": 1517387826, "metric": "log", "value": 0, "counterType": "GAUGE", "step": 30}]
2018-01-31 16:37:36,692-INFO:  [{"endpoint": "zww-192.168.138.136", "tags": "log=host.log,key=error", "timestamp": 1517387856, "metric": "log", "value": 0, "counterType": "GAUGE", "step": 30},{"endpoint": "zww-192.168.138.136", "tags": "log=host.log,key=test", "timestamp": 1517387856, "metric": "log", "value": 0, "counterType": "GAUGE", "step": 30},{"endpoint": "zww-192.168.138.136", "tags": "log=host2.log,key=error", "timestamp": 1517387856, "metric": "log", "value": 0, "counterType": "GAUGE", "step": 30},{"endpoint": "zww-192.168.138.136", "tags": "log=host2.log,key=test", "timestamp": 1517387856, "metric": "log", "value": 0, "counterType": "GAUGE", "step": 30}]
2018-01-31 16:38:06,732-INFO:  [{"endpoint": "zww-192.168.138.136", "tags": "log=host.log,key=error", "timestamp": 1517387886, "metric": "log", "value": 0, "counterType": "GAUGE", "step": 30},{"endpoint": "zww-192.168.138.136", "tags": "log=host.log,key=test", "timestamp": 1517387886, "metric": "log", "value": 0, "counterType": "GAUGE", "step": 30},{"endpoint": "zww-192.168.138.136", "tags": "log=host2.log,key=error", "timestamp": 1517387886, "metric": "log", "value": 0, "counterType": "GAUGE", "step": 30},{"endpoint": "zww-192.168.138.136", "tags": "log=host2.log,key=test", "timestamp": 1517387886, "metric": "log", "value": 0, "counterType": "GAUGE", "step": 30}]
2018-01-31 16:38:55,310-INFO:  [{"endpoint": "zww-192.168.138.136", "tags": "log=host.log,key=error", "timestamp": 1517387935, "metric": "log", "value": 0, "counterType": "GAUGE", "step": 30},{"endpoint": "zww-192.168.138.136", "tags": "log=host.log,key=test", "timestamp": 1517387935, "metric": "log", "value": 0, "counterType": "GAUGE", "step": 30},{"endpoint": "zww-192.168.138.136", "tags": "log=host2.log,key=error", "timestamp": 1517387935, "metric": "log", "value": 0, "counterType": "GAUGE", "step": 30},{"endpoint": "zww-192.168.138.136", "tags": "log=host2.log,key=test", "timestamp": 1517387935, "metric": "log", "value": 0, "counterType": "GAUGE", "step": 30}]
2018-01-31 16:39:25,366-INFO:  [{"endpoint": "zww-192.168.138.136", "tags": "log=host.log,key=error", "timestamp": 1517387965, "metric": "log", "value": 0, "counterType": "GAUGE", "step": 30},{"endpoint": "zww-192.168.138.136", "tags": "log=host.log,key=test", "timestamp": 1517387965, "metric": "log", "value": 0, "counterType": "GAUGE", "step": 30},{"endpoint": "zww-192.168.138.136", "tags": "log=host2.log,key=error", "timestamp": 1517387965, "metric": "log", "value": 0, "counterType": "GAUGE", "step": 30},{"endpoint": "zww-192.168.138.136", "tags": "log=host2.log,key=test", "timestamp": 1517387965, "metric": "log", "value": 0, "counterType": "GAUGE", "step": 30}]

因为要适配的系统环境比较多，很多python模块没法直接使用，可以看到我这里是直接使用的curl进行的post，实际情况可以根据自己需求进行更改。

插件功能主要就是进行多日志多关键的过滤并按配置时间进行推送数据（如：timer:30,即每30s推送这期间出现的关键字次数）