ELK 日志采集增加数据缓冲队列 redis

ELFK架构优化思路

---

在日志类的场景当中建议引入消息队列中间件作为缓冲层对数据进行缓冲 （指标分析/日志分析），使用Redis作为消息队列，起到数据缓冲作用，也就是峰值处理能力

• Logfile ->filebeat阶段是不太可能丢数据的
• Filebeat ->logstash 随着机器的增加，filebeat越来越多，logstash负载变大可能会丢失数据或者卡住（因为处理不过来），这样就可以增加数据的缓冲队列来进行去优化。
• 在filebeat和logstash之间加上可持久化的队列来临时的存储数据

 

 

增加数据缓冲队列

---

配置你的redis 

[root@localhost ~]# yum install redis -y
[root@localhost ~]# vim /etc/redis.conf 
bind 0.0.0.0
requirepass 123456
[root@localhost ~]# systemctl start redis
[root@localhost ~]# netstat -tpln | grep 6379
tcp        0      0 0.0.0.0:6379            0.0.0.0:*               LISTEN      2675/redis-server 0

Filebeat采集的日志都会推送到redis

[root@localhost ~]# cat /etc/filebeat/filebeat.yml
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/nginx/*.log
  tags: ["nginx"]
  fields_under_root: true
  fields:
    project: microservice
    app: product

output.redis:
  hosts: ["192.168.179.102:6379"]
  password: 123456
  key: "filebeat"
  db: 0
  datatype: "list"

[root@localhost ~]# systemctl restart filebeat
[root@localhost ~]# journalctl -u filebeat -f

在你的网页访问一下nginx，同时观察你的redis里面是否有数据了,可以看到日志信息存储到redis里面

[root@localhost ~]# redis-cli -a 123456
127.0.0.1:6379> keys *
(empty list or set)
127.0.0.1:6379> MONITOR
OK
1611112331.115870 [0 192.168.179.102:46876] "AUTH" "123456"
1611112331.116308 [0 192.168.179.102:46876] "PING"
1611112331.116378 [0 192.168.179.102:46876] "INFO"
1611112331.117628 [0 192.168.179.102:46876] "RPUSH" "filebeat" "{\"@timestamp\":\"2021-01-20T03:12:10.111Z\",\"@metadata\":{\"beat\":\"filebeat\",\"type\":\"_doc\",\"version\":\"7.10.1\"},\"input\":{\"type\":\"log\"},\"project\":\"microservice\",\"log\":{\"offset\":0,\"file\":{\"path\":\"/var/log/nginx/access.log\"}},\"message\":\"{ \\\"@timestamp\\\": \\\"2021-01-20T11:12:09+08:00\\\", \\\"remote_addr\\\": \\\"192.168.179.4\\\", \\\"remote_user\\\": \\\"-\\\", \\\"body_bytes_sent\\\": \\\"4833\\\", \\\"request_time\\\": \\\"0.000\\\", \\\"status\\\": \\\"200\\\", \\\"request_uri\\\": \\\"/\\\", \\\"request_method\\\": \\\"GET\\\", \\\"http_referrer\\\": \\\"-\\\", \\\"http_x_forwarded_for\\\": \\\"-\\\", \\\"http_user_agent\\\": \\\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36\\\"}\",\"app\":\"product\",\"agent\":{\"id\":\"e67f736a-42b4-413c-aae1-03d9b61a02ca\",\"name\":\"localhost.localdomain\",\"type\":\"filebeat\",\"version\":\"7.10.1\",\"hostname\":\"localhost.localdomain\",\"ephemeral_id\":\"4d57e989-b9cb-4b4f-909d-05b15381a06f\"},\"tags\":[\"nginx\"],\"ecs\":{\"version\":\"1.6.0\"},\"host\":{\"name\":\"localhost.localdomain\"}}" "{\"@timestamp\":\"2021-01-20T03:12:10.112Z\",\"@metadata\":{\"beat\":\"filebeat\",\"type\":\"_doc\",\"version\":\"7.10.1\"},\"message\":\"2021/01/20 11:12:09 [error] 2107#0: *31 open() \\\"/usr/share/nginx/html/favicon.ico\\\" failed (2: No such file or directory), client: 192.168.179.4, server: _, request: \\\"GET /favicon.ico HTTP/1.1\\\", host: \\\"192.168.179.102\\\", referrer: \\\"http://192.168.179.102/\\\"\",\"tags\":[\"nginx\"],\"input\":{\"type\":\"log\"},\"app\":\"product\",\"host\":{\"name\":\"localhost.localdomain\"},\"log\":{\"offset\":0,\"file\":{\"path\":\"/var/log/nginx/error.log\"}},\"project\":\"microservice\",\"ecs\":{\"version\":\"1.6.0\"},\"agent\":{\"version\":\"7.10.1\",\"hostname\":\"localhost.localdomain\",\"ephemeral_id\":\"4d57e989-b9cb-4b4f-909d-05b15381a06f\",\"id\":\"e67f736a-42b4-413c-aae1-03d9b61a02ca\",\"name\":\"localhost.localdomain\",\"type\":\"filebeat\"}}" "{\"@timestamp\":\"2021-01-20T03:12:10.113Z\",\"@metadata\":{\"beat\":\"filebeat\",\"type\":\"_doc\",\"version\":\"7.10.1\"},\"message\":\"{ \\\"@timestamp\\\": \\\"2021-01-20T11:12:09+08:00\\\", \\\"remote_addr\\\": \\\"192.168.179.4\\\", \\\"remote_user\\\": \\\"-\\\", \\\"body_bytes_sent\\\": \\\"3030\\\", \\\"request_time\\\": \\\"0.000\\\", \\\"status\\\": \\\"200\\\", \\\"request_uri\\\": \\\"/img/centos-logo.png\\\", \\\"request_method\\\": \\\"GET\\\", \\\"http_referrer\\\": \\\"http://192.168.179.102/\\\", \\\"http_x_forwarded_for\\\": \\\"-\\\", \\\"http_user_agent\\\": \\\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36\\\"}\",\"project\":\"microservice\",\"input\":{\"type\":\"log\"},\"host\":{\"name\":\"localhost.localdomain\"},\"log\":{\"offset\":400,\"file\":{\"path\":\"/var/log/nginx/access.log\"}},\"tags\":[\"nginx\"],\"app\":\"product\",\"ecs\":{\"version\":\"1.6.0\"},\"agent\":{\"id\":\"e67f736a-42b4-413c-aae1-03d9b61a02ca\",\"name\":\"localhost.localdomain\",\"type\":\"filebeat\",\"version\":\"7.10.1\",\"hostname\":\"localhost.localdomain\",\"ephemeral_id\":\"4d57e989-b9cb-4b4f-909d-05b15381a06f\"}}" "{\"@timestamp\":\"2021-01-20T03:12:10.113Z\",\"@metadata\":{\"beat\":\"filebeat\",\"type\":\"_doc\",\"version\":\"7.10.1\"},\"agent\":{\"hostname\":\"localhost.localdomain\",\"ephemeral_id\":\"4d57e989-b9cb-4b4f-909d-05b15381a06f\",\"id\":\"e67f736a-42b4-413c-aae1-03d9b61a02ca\",\"name\":\"localhost.localdomain\",\"type\":\"filebeat\",\"version\":\"7.10.1\"},\"input\":{\"type\":\"log\"},\"log\":{\"offset\":841,\"file\":{\"path\":\"/var/log/nginx/access.log\"}},\"tags\":[\"nginx\"],\"project\":\"microservice\",\"app\":\"product\",\"ecs\":{\"version\":\"1.6.0\"},\"host\":{\"name\":\"localhost.localdomain\"},\"message\":\"{ \\\"@timestamp\\\": \\\"2021-01-20T11:12:09+08:00\\\", \\\"remote_addr\\\": \\\"192.168.179.4\\\", \\\"remote_user\\\": \\\"-\\\", \\\"body_bytes_sent\\\": \\\"1801\\\", \\\"request_time\\\": \\\"0.000\\\", \\\"status\\\": \\\"200\\\", \\\"request_uri\\\": \\\"/img/html-background.png\\\", \\\"request_method\\\": \\\"GET\\\", \\\"http_referrer\\\": \\\"http://192.168.179.102/\\\", \\\"http_x_forwarded_for\\\": \\\"-\\\", \\\"http_user_agent\\\": \\\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36\\\"}\"}" "{\"@timestamp\":\"2021-01-20T03:12:10.113Z\",\"@metadata\":{\"beat\":\"filebeat\",\"type\":\"_doc\",\"version\":\"7.10.1\"},\"tags\":[\"nginx\"],\"input\":{\"type\":\"log\"},\"project\":\"microservice\",\"ecs\":{\"version\":\"1.6.0\"},\"log\":{\"offset\":1286,\"file\":{\"path\":\"/var/log/nginx/access.log\"}},\"message\":\"{ \\\"@timestamp\\\": \\\"2021-01-20T11:12:09+08:00\\\", \\\"remote_addr\\\": \\\"192.168.179.4\\\", \\\"remote_user\\\": \\\"-\\\", \\\"body_bytes_sent\\\": \\\"82896\\\", \\\"request_time\\\": \\\"0.000\\\", \\\"status\\\": \\\"200\\\", \\\"request_uri\\\": \\\"/img/header-background.png\\\", \\\"request_method\\\": \\\"GET\\\", \\\"http_referrer\\\": \\\"http://192.168.179.102/\\\", \\\"http_x_forwarded_for\\\": \\\"-\\\", \\\"http_user_agent\\\": \\\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36\\\"}\",\"app\":\"product\",\"host\":{\"name\":\"localhost.localdomain\"},\"agent\":{\"version\":\"7.10.1\",\"hostname\":\"localhost.localdomain\",\"ephemeral_id\":\"4d57e989-b9cb-4b4f-909d-05b15381a06f\",\"id\":\"e67f736a-42b4-413c-aae1-03d9b61a02ca\",\"name\":\"localhost.localdomain\",\"type\":\"filebeat\"}}" "{\"@timestamp\":\"2021-01-20T03:12:10.113Z\",\"@metadata\":{\"beat\":\"filebeat\",\"type\":\"_doc\",\"version\":\"7.10.1\"},\"log\":{\"file\":{\"path\":\"/var/log/nginx/access.log\"},\"offset\":1734},\"message\":\"{ \\\"@timestamp\\\": \\\"2021-01-20T11:12:09+08:00\\\", \\\"remote_addr\\\": \\\"192.168.179.4\\\", \\\"remote_user\\\": \\\"-\\\", \\\"body_bytes_sent\\\": \\\"3650\\\", \\\"request_time\\\": \\\"0.000\\\", \\\"status\\\": \\\"404\\\", \\\"request_uri\\\": \\\"/favicon.ico\\\", \\\"request_method\\\": \\\"GET\\\", \\\"http_referrer\\\": \\\"http://192.168.179.102/\\\", \\\"http_x_forwarded_for\\\": \\\"-\\\", \\\"http_user_agent\\\": \\\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36\\\"}\",\"tags\":[\"nginx\"],\"app\":\"product\",\"host\":{\"name\":\"localhost.localdomain\"},\"input\":{\"type\":\"log\"},\"project\":\"microservice\",\"ecs\":{\"version\":\"1.6.0\"},\"agent\":{\"id\":\"e67f736a-42b4-413c-aae1-03d9b61a02ca\",\"name\":\"localhost.localdomain\",\"type\":\"filebeat\",\"version\":\"7.10.1\",\"hostname\":\"localhost.localdomain\",\"ephemeral_id\":\"4d57e989-b9cb-4b4f-909d-05b15381a06f\"}}"

配置你的logstash

[root@localhost ~]# systemctl restart logstash
[root@localhost ~]# cat /usr/local/logstash/conf.d/test.conf
input {
  redis {
    host => "192.168.179.102"
    port => 6379
    password => "123456"
    key => "filebeat"
    db => 0
    data_type => "list" 
 } 
}

filter {
  json {
    source => "message"
}
  if [app] == "product" {
    mutate { 
      add_field => { 
        "[@metadata][target_index]" => "microservice-product-%{+YYYY.MM}" 
   } 
  }
} else if [app] == "gateway" {
    mutate { 
      add_field => { 
        "[@metadata][target_index]" => "microservice-gateway-%{+YYYY.MM.dd}" 
   } 
  }
} else {
    mutate { 
      add_field => { 
        "[@metadata][target_index]" => "unknown-%{+YYYY}" 
   } 
  } 
 } 
}

output {
  elasticsearch {
    hosts => "192.168.179.102:9200"
    index => "%{[@metadata][target_index]}"
 } 
}


[root@localhost ~]# journalctl  -u logstash -f

 查看结果