ES接入Redis
1.安装redis
[root@redis ~]# yum install redis -y
2.配置redis密码
[root@redis ~]# grep "^[a-Z]" /etc/redis.conf |head -4
bind 127.0.0.1 10.0.0.50
requirepass 123456
3.启动redis
[root@redis ~]# systemctl restart redis
[root@redis ~]# systemctl enable redis
4.修改filebeat输出数据至Redis
[root@web01 ~]# cat /etc/filebeat/filebeat.yml
filebeat.inputs:
- type: log
enabled: true
paths:
- /var/log/nginx/access.log
tags: ["access"]
- type: log
enabled: true
paths:
- /var/log/nginx/error.log
tags: ["error"]
output.redis:
hosts: ["10.0.0.50:6379"] #redis地址
password: 123456 #redis密码
timeout: 5 #连接超时时间
db: 0 #写入db0库中
keys: #存储的key名称
- key: "nginx_access"
when.contains:
tags: "access"
- key: "nginx_error"
when.contains:
tags: "error"
启动filebeat并测试是否能存到redis里
systectl restart filebeat
[root@redis ~]# redis-cli
127.0.0.1:6379> keys *
1) "nginx_access"
127.0.0.1:6379> TYPE nginx_access
list
127.0.0.1:6379> LLEN nginx_access
(integer) 112875
127.0.0.1:6379> LRANGE nginx_access 1 2
1) "{\"@timestamp\":\"2020-02-17T13:54:58.449Z\",\"@metadata\":{\"beat\":\"filebeat\",\"type\":\"_doc\",\"version\":\"7.4.0\"},\"input\":{\"type\":\"log\"},\"ecs\":{\"version\":\"1.1.0\"},\"host\":{\"name\":\"web01\"},\"agent\":{\"hostname\":\"web01\",\"id\":\"405d2c76-4639-4263-88dc-87dac3a8f313\",\"version\":\"7.4.0\",\"type\":\"filebeat\",\"ephemeral_id\":\"1a77a34e-0e5e-4ba4-8d4c-0eb21acf5dc0\"},\"log\":{\"file\":{\"path\":\"/var/log/nginx/access.log\"},\"offset\":197},\"message\":\"192.168.37.59 - - [16/Jan/2020:10:21:14 +0800] \\\"GET /favicon.ico HTTP/1.1\\\" 404 187 \\\"http://192.168.37.22/\\\" \\\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36\\\" \\\"-\\\"\",\"tags\":[\"access\"]}"
2) "{\"@timestamp\":\"2020-02-17T13:54:58.449Z\",\"@metadata\":{\"beat\":\"filebeat\",\"type\":\"_doc\",\"version\":\"7.4.0\"},\"log\":{\"offset\":426,\"file\":{\"path\":\"/var/log/nginx/access.log\"}},\"message\":\"192.168.37.59 - - [16/Jan/2020:10:21:14 +0800] \\\"GET / HTTP/1.1\\\" 200 13 \\\"-\\\" \\\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36\\\" \\\"-\\\"\",\"tags\":[\"access\"],\"input\":{\"type\":\"log\"},\"host\":{\"name\":\"web01\"},\"agent\":{\"ephemeral_id\":\"1a77a34e-0e5e-4ba4-8d4c-0eb21acf5dc0\",\"hostname\":\"web01\",\"id\":\"405d2c76-4639-4263-88dc-87dac3a8f313\",\"version\":\"7.4.0\",\"type\":\"filebeat\"},\"ecs\":{\"version\":\"1.1.0\"}}"
127.0.0.1:6379>
5.修改logstash从Redis中读取数据
[root@logstash-node1 ~]# cat /etc/logstash/conf.d/input_redis_nginx_1w_output_es.conf
input {
redis {
host => ["10.0.0.50"] #redis地址
port => "6379"
password => "123456"
data_type => "list"
key => "nginx_access"
db => "0" #从哪个库读
}
redis {
host => ["10.0.0.50"]
port => "6379"
password => "123456"
data_type => "list"
key => "nginx_error"
db => "0"
}
}
filter {
if "access" in [tags][0] {
grok {
match => { "message" => "%{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] \"%{WORD:method} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}\" %{NUMBER:response} (?:%{NUMBER:bytes}|-) %{QS:hostname} (?:%{QS:referrer}|-) (?:%{NOTSPACE:post_args}|-) %{QS:useragent} (?:%{QS:x_forward_for}|-) (?:%{URIHOST:upstream_host}|-) (?:%{NUMBER:upstream_response_code}|-) (?:%{NUMBER:upstream_response_time}|-) (?:%{NUMBER:response_time}|-)" }
}
useragent {
source => "useragent"
target => "useragent"
}
geoip {
source => "clientip"
}
date {
match => ["timestamp","dd/MMM/yyyy:HH:mm:ss Z"]
target => "@timestamp"
timezone => "Asia/Shanghai"
}
mutate {
convert => ["bytes","integer"]
convert => ["response_time", "float"]
remove_field => ["message","agent","tags"]
add_field => { "target_index" => "logstash-redis-nginx-access-%{+YYYY.MM.dd}" }
}
#提取referrer具体的域名/^"http/
if [referrer] =~ /^"http/ {
grok {
match => { "referrer" => '%{URIPROTO}://%{URIHOST:referrer_host}' }
}
}
#提取用户请求资源类型以及资源ID编号
if "oldxu.com" in [referrer_host] {
grok {
match => { "referrer" => '%{URIPROTO}://%{URIHOST}/(%{NOTSPACE:oldxu_type}/%{NOTSPACE:oldxu_res_id})?"' }
}
}
}
else if "error" in [tags][0] {
date {
match => ["timestamp","dd/MMM/yyyy:HH:mm:ss Z"]
target => "@timestamp"
timezone => "Asia/Shanghai"
}
mutate {
add_field => { "target_index" => "logstash-redis-nginx-error-%{+YYYY.MM.dd}" }
}
}
}
output {
elasticsearch {
hosts => ["10.0.0.161:9200","10.0.0.162:9200","10.0.0.163:9200"]
index => "%{[target_index]}"
template_overwrite => true
}
stdout {
codec => rubydebug
}
}
测试:
[root@redis ~]# systemctl restart redis
[root@redis ~]# redis-cli
127.0.0.1:6379> auth 123456
127.0.0.1:6379> keys *
127.0.0.1:6379> type nginx_access
list
127.0.0.1:6379> llen nginx_access