nginx错误日志入库小程序

最新推荐文章于 2024-06-20 11:23:27 发布

caixingyunzi

最新推荐文章于 2024-06-20 11:23:27 发布

阅读量300

点赞数

分类专栏： python程序记录（以后回过头来看）文章标签： python nginx_error

本文链接：https://blog.csdn.net/caixingyunzi/article/details/79387787

版权

python程序记录（以后回过头来看）专栏收录该内容

5 篇文章 0 订阅

订阅专栏

nginx错误日志入库小程序

为了独立记录nginx_error日志来自定义一些ip策略阻挡攻击，这里把所有nginx服务器的错误日志入库来进行统一处理，代码如下,然后结合supervisor来使用：

#!/usr/bin/env python
# -*- coding:utf-8 -*-
# __author__ = 'xiaojinsong'
# __time__ = '2018/2/27'



from __future__ import print_function
import pyinotify
import re
import os
import sys
import MySQLdb
import socket


class ProcessTransientFile(pyinotify.ProcessEvent):
    def process_IN_MODIFY(self, event):
        line = file.readline()
        # print(line)
        if line:
            intodb(line)


def nginxLogAnalyzer(line):
    # print(line,end='')
    m = re.match(r'(\d{4}\/\d{2}\/\d{2}\ \d{2}\:\d{2}\:\d{2}) ([^,]*),\s(.*)', line)

    dic1 = {}
    v = m.group(3).split(', ')
    dict1 = dict([(a.split(': ')[0].strip(), a.split(': ')[1].strip()) for a in v])
    if dict1['request']:
        x = dict1['request']
        dict1.pop('request')
        dict1['method'] = x.split(' ')[0].strip('"')
        dict1['request'] = x.split(' ')[1].strip('"')
        dict1['http_version'] = x.split(' ')[2].strip('"')

    dict1['time'] = m.group(1)
    dict1['errortype'] = m.group(2)
    hn = socket.gethostname()
    dict1['nginxIP'] = socket.gethostbyname(hn)
    return dict1


def intodb(line):
    dict1 = nginxLogAnalyzer(line)
    field = []
    cur.execute("describe nginx_error")
    field = cur.fetchall()
    mylist = []
    for fie in field:
        mylist.append(fie[0])
    for key in dict1.keys():
        if key not in mylist:
            try:
                sql = "alter table nginx_error add column %s varchar(30) DEFAULT NULL" % key
                cur.execute(sql)
                conn.commit()
            except Exception, e:
                conn.rollback()
                print(e)
    placeholders = ', '.join(['%s'] * len(dict1))
    columns = ', '.join(dict1.keys())
    sql = "INSERT INTO nginx_error (%s) VALUES (%s)" % (columns, placeholders)
    try:
        cur.execute(sql, dict1.values())
        conn.commit()
    except Exception, e:
        conn.rollback()
        print(e)


if __name__ == '__main__':
    conn = MySQLdb.connect(host='xxx.xxx.xxxx', port=3306, user='nginx', passwd='nginx', db='nginx',
                           charset="utf8")
    # cur = conn.cursor(MySQLdb.cursors.DictCursor)
    cur = conn.cursor()
    cur.execute("SET NAMES utf8")

    filename = sys.argv[1]
    file = open(filename, 'r')
    st_results = os.stat(filename)
    st_size = st_results[6]
    file.seek(st_size)

    wm = pyinotify.WatchManager()
    notifier = pyinotify.Notifier(wm)
    wm.watch_transient_file(filename, pyinotify.IN_MODIFY, ProcessTransientFile)
    notifier.loop()