python3利用pyinotify实现把日志实时写入数据库或用于监控error告警

安装库pyinotify

pip3 install pyinotify

1. 先使用pyinotify实现一个tail -f 的功能：（来来实时提取日志）```

import pyinotify
import time
import os
import sys

class ProcessTransientFile(pyinotify.ProcessEvent):
    def process_IN_MODIFY(self,event):
        line = file.readline()
        if line:
            print(line, end='')


if __name__ == '__main__':
    filename = sys.argv[1]
    file = open(filename,'r')
    st_results = os.stat(filename)
    st_size = st_results[6]
    file.seek(st_size)

    wm = pyinotify.WatchManager()
    notifier = pyinotify.Notifier(wm)
    wm.watch_transient_file(filename, pyinotify.IN_MODIFY, ProcessTransientFile)
    notifier.loop()

2. 然后通过python pytaif.py /data/logs/nginx/access.log就可以进行日志的实时查看。
   
   这个是实时查看，和tail -f 功能一毛一样一样。

3. 加入想实现的功能，我这里是把日志写入到clickhouse。

def intodb(line):
    click_client = Client("127.0.0.1", "9000", "db", "user", "passwd")
    if '[' in line:
        data = re.match(r'.*]\s(.+)', line)
        dic = json.loads(data.group(1))
        k = str(list((dic.keys()))).replace('[', '(').replace(']', ')').replace("'", '')
        # v = str(list((dic.values())))
        # print(dic)
        sql = 'INSERT INTO db %s VALUES ' % k
        # print(sql)
        try:
            click_client.execute(sql, [dic])
            # print("成功")
        except Exception as e:
            # return (e)
            Dingtalk(log + "插入数据异常请检查:" + e)
            print(e, [dic])

4. 下面是所有源代码 pytaif.py

from clickhouse_driver import Client
import pyinotify
import re,os,json,sys

class ProcessTransientFile(pyinotify.ProcessEvent):
    def process_IN_MODIFY(self,event):
        line = file.readline()
        if line:
            #print(line, end='')
            intodb(line)


def intodb(line):
    click_client = Client("127.0.0.1", "9000", "db", "user", "passwd")
    if '[' in line:
        data = re.match(r'.*]\s(.+)', line)
        dic = json.loads(data.group(1))
        k = str(list((dic.keys()))).replace('[', '(').replace(']', ')').replace("'", '')
        # v = str(list((dic.values())))
        # print(dic)
        sql = 'INSERT INTO client_report_action_flow %s VALUES ' % k
        # print(sql)
        try:
            click_client.execute(sql, [dic])
            # print("成功")
        except Exception as e:
            # return (e)
            Dingtalk(log + "插入数据异常请检查:" + e)
            print(e, [dic])

if __name__ == '__main__':
    filename = sys.argv[1]
    file = open(filename,'r')
    st_results = os.stat(filename)
    st_size = st_results[6]
    file.seek(st_size)

    wm = pyinotify.WatchManager()
    notifier = pyinotify.Notifier(wm)
    wm.watch_transient_file(filename, pyinotify.IN_MODIFY, ProcessTransientFile)
    notifier.loop()

5. 监控error日志钉钉告警输入
   换个方法即可

from dingtalkchatbot.chatbot import DingtalkChatbot

def Dingtalk(line):

    webhook = 'https://oapi.dingtalk.com/robot/send?access_token=***your_access_token***'

    xiaoding = DingtalkChatbot(webhook)

    xiaoding.send_text(msg=line)