ELK-Redis架构接入

最新推荐文章于 2023-04-11 17:30:17 发布

一个不专业的码农

最新推荐文章于 2023-04-11 17:30:17 发布

阅读量366

点赞数

分类专栏： ELK Stack 文章标签： elk

本文链接：https://blog.csdn.net/weixin_45858439/article/details/118942660

版权

ELK Stack 专栏收录该内容

9 篇文章 0 订阅

订阅专栏

ELK-Redis架构接入

在这里插入图片描述

1.安装redis

#0.机器是复用之前的
10.0.0.161   10.0.0.162  10.0.0.163   安装elasticsearch集群
10.0.0.6    安装nginx和filebeat
10.0.0.151  安装mariadb
10.0.0.7    安装redis

#1.安装redis
[root@redis ~]# yum install redis -y

#2.配置redis（设置密码）
[root@redis ~]# grep '^[a-Z]' /etc/redis.conf 
bind 127.0.0.1 10.0.0.7
requirepass 123456
.......(其它配置默认)

#3.启动redis
[root@redis ~]# systemctl enable redis
[root@redis ~]# systemctl start redis

#4.配置filebeat
[root@web01 ~]# cat /etc/filebeat/filebeat.yml
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/nginx/access.log
  tags: ["access"]

- type: log
  enabled: true
  paths:
    - /var/log/nginx/error.log
  tags: ["error"]

output.redis:
  hosts: ["10.0.0.8:6379"]		#redis地址
  password: 123456				#redis密码
  timeout: 5					#连接超时时间
  db: 0							#写入db0库中
  keys:							#存储的key名称
    - key: "nginx_access"
      when.contains:
        tags: "access"
    - key: "nginx_error"
      when.contains:
        tags: "error"


#5.重启filebeat
[root@web01 ~]# systemctl restart filebeat

#6.查看redis
[root@redis ~]# redis-cli
127.0.0.1:6379> auth 123456
OK
127.0.0.1:6379> keys *
1) "nginx_error"
2) "nginx_access"
127.0.0.1:6379> TYPE nginx_access
list
127.0.0.1:6379> LLEN nginx_access
(integer) 12
127.0.0.1:6379> LRANG nginx_access 1 2
(error) ERR unknown command 'LRANG'
127.0.0.1:6379> LRANGE nginx_access 1 2
1) "{\"@timestamp\":\"2021-07-19T12:49:45.718Z\",\"@metadata\":{\"beat\":\"filebeat\",\"type\":\"_doc\",\"version\":\"7.4.0\"},\"log\":{\"offset\":1432,\"file\":{\"path\":\"/var/log/nginx/access.log\"}},\"message\":\"124.161.176.119 - - [10/Nov/2019:00:01:52 +0800] \\\"POST /api3/appover HTTP/1.1\\\" 200 103 \\\"www.oldxu.com\\\" \\\"-\\\" code=B157963E-5BDA-4090-A021-A3D46D2E6BA2&secrect=f0fbb455c7aebc69c5cc39d68c7859fe&time=9441&timestamp=1478707307012&token=12d0f0cfa1efb81e42c321f027bbe752&uid=4384521 \\\"oldxu/5.0.2 (iPhone;iOS 10.1.1; Scale/2.00)\\\" \\\"-\\\" 10.100.136.65:80 200 0.011 0.011\",\"tags\":[\"access\"],\"input\":{\"type\":\"log\"},\"ecs\":{\"version\":\"1.1.0\"},\"host\":{\"name\":\"web01\"},\"agent\":{\"ephemeral_id\":\"50c0e773-eaf7-4115-8c1c-6c40758d4ccf\",\"hostname\":\"web01\",\"id\":\"c2e57138-56c0-4c2f-9cfc-416d8213b66a\",\"version\":\"7.4.0\",\"type\":\"filebeat\"}}"
2) "{\"@timestamp\":\"2021-07-19T12:49:45.718Z\",\"@metadata\":{\"beat\":\"filebeat\",\"type\":\"_doc\",\"version\":\"7.4.0\"},\"ecs\":{\"version\":\"1.1.0\"},\"host\":{\"name\":\"web01\"},\"agent\":{\"type\":\"filebeat\",\"ephemeral_id\":\"50c0e773-eaf7-4115-8c1c-6c40758d4ccf\",\"hostname\":\"web01\",\"id\":\"c2e57138-56c0-4c2f-9cfc-416d8213b66a\",\"version\":\"7.4.0\"},\"log\":{\"file\":{\"path\":\"/var/log/nginx/access.log\"},\"offset\":1790},\"message\":\"124.161.176.119 - - [10/Nov/2019:00:01:52 +0800] \\\"POST /api3/appover HTTP/1.1\\\" 200 103 \\\"www.oldxu.com\\\" \\\"-\\\" code=B157963E-5BDA-4090-A021-A3D46D2E6BA2&secrect=f0fbb455c7aebc69c5cc39d68c7859fe&time=9441&timestamp=1478707307012&token=12d0f0cfa1efb81e42c321f027bbe752&uid=4384521 \\\"oldxu/5.0.2 (iPhone;iOS 10.1.1; Scale/2.00)\\\" \\\"-\\\" 10.100.136.65:80 200 0.011 0.011\",\"tags\":[\"access\"],\"input\":{\"type\":\"log\"}}"
127.0.0.1:6379> LRANGE nginx_error 1 2
1) "{\"@timestamp\":\"2021-07-19T12:46:30.659Z\",\"@metadata\":{\"beat\":\"filebeat\",\"type\":\"_doc\",\"version\":\"7.4.0\"},\"message\":\"2021/07/19 20:42:18 [notice] 7354#7354: nginx/1.20.1\",\"log\":{\"offset\":4267,\"file\":{\"path\":\"/var/log/nginx/error.log\"}},\"tags\":[\"error\"],\"input\":{\"type\":\"log\"},\"host\":{\"name\":\"web01\"},\"agent\":{\"ephemeral_id\":\"50c0e773-eaf7-4115-8c1c-6c40758d4ccf\",\"hostname\":\"web01\",\"id\":\"c2e57138-56c0-4c2f-9cfc-416d8213b66a\",\"version\":\"7.4.0\",\"type\":\"filebeat\"},\"ecs\":{\"version\":\"1.1.0\"}}"
2) "{\"@timestamp\":\"2021-07-19T12:46:30.660Z\",\"@metadata\":{\"beat\":\"filebeat\",\"type\":\"_doc\",\"version\":\"7.4.0\"},\"agent\":{\"type\":\"filebeat\",\"ephemeral_id\":\"50c0e773-eaf7-4115-8c1c-6c40758d4ccf\",\"hostname\":\"web01\",\"id\":\"c2e57138-56c0-4c2f-9cfc-416d8213b66a\",\"version\":\"7.4.0\"},\"log\":{\"file\":{\"path\":\"/var/log/nginx/error.log\"},\"offset\":4320},\"message\":\"2021/07/19 20:42:18 [notice] 7354#7354: built by gcc 4.8.5 20150623 (Red Hat 4.8.5-44) (GCC) \",\"tags\":[\"error\"],\"input\":{\"type\":\"log\"},\"ecs\":{\"version\":\"1.1.0\"},\"host\":{\"name\":\"web01\"}}"
127.0.0.1:6379> 



#7.配置logstash
[root@logstash-node1 ~]# cat /etc/logstash/conf.d/input_redis_nginx_1w_output_es.conf
input {
	redis {
		host => ["10.0.0.7"]
		port => "6379"
		password => "123456"
		data_type => "list"
		key => "nginx_access"
		db => "0"
	}

	redis {
		host => ["10.0.0.7"]
		port => "6379"
		password => "123456"
		data_type => "list"
		key => "nginx_error"
		db => "0"
	}
}

filter {
    if "access" in [tags][0] {
        grok {
            match => { "message" => "%{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] \"%{WORD:method} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}\" %{NUMBER:response} (?:%{NUMBER:bytes}|-) %{QS:hostname} (?:%{QS:referrer}|-) (?:%{NOTSPACE:post_args}|-) %{QS:useragent} (?:%{QS:x_forward_for}|-) (?:%{URIHOST:upstream_host}|-) (?:%{NUMBER:upstream_response_code}|-) (?:%{NUMBER:upstream_response_time}|-) (?:%{NUMBER:response_time}|-)" }
        }

        useragent {
            source => "useragent"
            target => "useragent"
        }
        geoip {
            source => "clientip"
        }
        date {
            match => ["timestamp","dd/MMM/yyyy:HH:mm:ss Z"]
            target => "@timestamp"
            timezone => "Asia/Shanghai"
        }
        mutate {
            convert => ["bytes","integer"]
            convert => ["response_time", "float"]
            remove_field => ["message","agent","tags"]
            add_field => { "target_index" => "logstash-redis-nginx-access-%{+YYYY.MM.dd}" }
    }
           #提取referrer具体的域名/^"http/
        if [referrer] =~ /^"http/ {
            grok {
                match => { "referrer" => '%{URIPROTO}://%{URIHOST:referrer_host}' }
            }
        }

        #提取用户请求资源类型以及资源ID编号
        if "oldxu.com" in [referrer_host] {
            grok {
                match => { "referrer" => '%{URIPROTO}://%{URIHOST}/(%{NOTSPACE:oldxu_type}/%{NOTSPACE:oldxu_res_id})?"' }
            }
        }
    } 
    else if "error" in [tags][0] {
        date {
            match => ["timestamp","dd/MMM/yyyy:HH:mm:ss Z"]
                target => "@timestamp"
                timezone => "Asia/Shanghai"
        }
        mutate {
            add_field => { "target_index" => "logstash-redis-nginx-error-%{+YYYY.MM.dd}" }
        }
    }
}

output {
    elasticsearch {
        hosts => ["10.0.0.161:9200","10.0.0.162:9200","10.0.0.163:9200"]
        index => "%{[target_index]}"
        template_overwrite => true
    }

    stdout {
	codec => rubydebug
    }

}

#8.启动logstash
[root@logstash-node1 ~]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/input_redis_nginx_1w_output_es.conf -r

在这里插入图片描述

一个不专业的码农

关注

0
点赞
踩
0

收藏

觉得还不错? 一键收藏
1
评论
ELK-Redis架构接入

ELK-Redis架构接入1.安装redis#0.机器是复用之前的10.0.0.161 10.0.0.162 10.0.0.163 安装elasticsearch集群10.0.0.6 安装nginx和filebeat10.0.0.151 安装mariadb10.0.0.7 安装redis#1.安装redis[root@redis ~]# yum install redis -y#2.配置redis（设置密码）[root@redis ~]# grep '^[a
复制链接

扫一扫