logstash.conf示例

参考logstash的一个入门资料： http://doc.yonyoucloud.com/doc/logstash-best-practice-cn/index.html

输出ES时创建的索引模板定义：https://www.cnblogs.com/you-you-111/p/9844131.html

https://www.cnblogs.com/cangqinglang/p/12187801.html

 

ongdb_query_log.conf

#输入
input {
  	beats {
		port  => "5044"
		client_inactivity_timeout => 36000
 	}
}

#过滤
filter {
	grok {
		match => {
			"message" =>[
				"%{TIMESTAMP_ISO8601:log_timestamp} %{LOGLEVEL:log_level}  %{INT:time_consuming} %{USERNAME:time_consuming_unit}:.*client/%{IP:client_ip}:%{INT:client_port}.*%{IP:server_ip}:%{INT:server_port}.* -[ |\r\n]%{GREEDYDATA:cypher} - {.*} - .*",
				"%{TIMESTAMP_ISO8601:log_timestamp} %{LOGLEVEL:log_level}  %{INT:time_consuming} %{USERNAME:time_consuming_unit}:.*(ongdb|graph-user1|graph-user2|neo4j|graph-user3|techfin|esg) -[ |\r\n]%{GREEDYDATA:cypher} - {}",
				"%{TIMESTAMP_ISO8601:log_timestamp} %{LOGLEVEL:log_level}  %{INT:time_consuming} %{USERNAME:time_consuming_unit}:.* -[ |\r\n]%{GREEDYDATA:cypher} - {.*} - .*"
			]
		}
#		add_field => ["day", "%{+YYYY.MM.dd}"]
#		add_field => ["received_at", "%{@timestamp}"]
#		add_field => ["received_log", "%{host}"]
#		remove_field => ["host"]
#		add_field => ["received_logstash", "%{host}"]
#		remove_field => ["message","@timestamp","tags","log","input","agent","ecs"]

		
		add_field => ["received_at", "%{@timestamp}"]
		add_field => ["received_from", "%{host}"]
		add_field => ["day", "%{+YYYY.MM.dd}"]
		remove_field => ["message","@timestamp","tags","log","input","agent","ecs","host"]
	}
#	mutate {
#        		convert => ["time_consuming", "int"]
#   	}
#	date {
#		match => [ "log_timestamp", "YYYY-MMM-dd HH:mm:ss.SSS Z" ]
#	}
}
#输出
output {
	elasticsearch {
#        		hosts => "http://10.20.13.130:9200"
        		hosts => "http://10.20.8.155:9200"
		index => "logstash_ongdb_querylog_%{day}"
#		index => "ongdb_querylog"
		template => "/home/ubuntu/ongdbETL/logstash-7.5.1/bin/conf/logstash_ongdb_querylog.json"
            		template_name => "logstash_ongdb_querylog_*"
           		template_overwrite => true
    	}
	stdout {}
}

 filebeat.yml

#运行命令	./filebeat -c filebeat_neo4j_log.yml -e
filebeat.inputs:
- type: log
  enabled: true
  encoding: utf-8
  paths:
    - /home/ongdb/ongdb-enterprise-3.5.22/logs/query.*
  multiline.pattern: '^\d{4}-\d{2}-\d{2}.*'
  multiline.negate: true
  multiline.match: after

output.logstash:
  hosts: ["10.20.4.28:5044"]

logstash_neo4j_querylog.json 

{
  "template": "logstash_ongdb_querylog_*",
  "order": 1,
  "settings": {
    "number_of_replicas": 0,
    "number_of_shards": 1,
    "refresh_interval": "60s",
    "translog": {
      "flush_threshold_size": "256mb"
    },
    "merge": {
      "scheduler": {
        "max_thread_count": "1"
      }
    },
    "index": {
      "routing": {
        "allocation": {
          "total_shards_per_node": "1"
        }
      }
    },
    "analysis": {
      "normalizer": {
        "my_normalizer": {
          "type": "custom",
          "filter": [
            "lowercase",
            "asciifolding"
          ]
        }
      }
    }
  },
  "mappings": {
    "properties": {
      "time_consuming": {
        "index": true,
        "store": true,
        "type": "integer"
      },
      "time_consuming_unit": {
        "index": true,
        "store": true,
        "type": "keyword"
      },
      "client_ip": {
        "index": true,
        "store": true,
        "type": "keyword"
      },
      "client_port": {
        "index": true,
        "store": true,
        "type": "keyword"
      },
      "server_ip": {
        "index": true,
        "store": true,
        "type": "keyword"
      },
      "server_port": {
        "index": true,
        "store": true,
        "type": "keyword"
      },
      "cypher": {
        "index": true,
        "store": true,
        "type": "text"
      },
      "received_from": {
        "index": true,
        "store": true,
        "type": "keyword"
      },
      "received_at": {
        "index": true,
        "store": true,
        "type": "keyword"
      },
      "log_level": {
        "index": true,
        "store": true,
        "type": "keyword"
      },
      "log_timestamp": {
        "index": true,
        "store": true,
        "type": "keyword"
      }
    }
  },
        "aliases": {
            "logstash_neo4j_querylog": {}
        }
}