【fillebeat】filebeat采集nginx-to-redis,logstash推送redis数据到elasticsearch集群

1.安装nginx 

tar xvf nginx-1.20.2.tar.gz
--安装nginx;
cd /esdb/soft/nginx-1.20.2
./configure --prefix=/usr/local/nginx --without-http_rewrite_module --without-http_gzip_module
make && make install

--
ln -s /usr/local/nginx/sbin/nginx /usr/local/sbin/
rm -rf /usr/local/nginx/html/*
*/ --默认业务清理。

--将tomcat的文档拷贝到nginx的html里面。
cp -r /tomcat/tomcat8080/webapps/docs/* /usr/local/nginx/html/
chmod 777 /usr/local/nginx/html
*/

2.nginx参数配置 

vi /usr/local/nginx/conf/nginx.conf

3.启动nginx 

[root@oracle1 nginx-1.20.2]# nginx
[root@oracle1 nginx-1.20.2]# 
[root@oracle1 nginx-1.20.2]# ps -ef |grep nginx
root      69451      1  0 14:46 ?        00:00:00 nginx: master process nginx
nobody    69452  69451  0 14:46 ?        00:00:00 nginx: worker process

--nginx的默认端口是：80;
[root@oracle1 nginx-1.20.2]# netstat -ntap | grep nginx
tcp        0      0 0.0.0.0:80              0.0.0.0:*               LISTEN      69451/nginx: master

4.配置filebeat抽取nginx数据到redis

cp -r /esdb/soft/filebeat /esdb/filebeat-nginx

[root@oracle1 filebeat-nginx]# cat /esdb/filebeat-nginx/sspu-nginx.yml

5.启动filebeat 

nohup /esdb/filebeat-nginx/filebeat -e -c /esdb/filebeat-nginx/sspu-nginx.yml > nohup.nginx &

6.登陆redis;

[root@oracle1 ~]# redis-cli -h 192.168.1.7  
192.168.1.7:6379> keys *
 1) "sspu-nginx"

192.168.1.7:6379> type sspu-nginx
list
--filebeat成功将 nginx的日志数据采集到了 redis。
192.168.1.7:6379> lrange sspu-nginx 0 1
1) "{\"@timestamp\":\"2024-06-30T07:09:54.614Z\",\"@metadata\":{\"beat\":\"filebeat\",\"type\":\"_doc\",\"version\":\"7.9.2\"},\"log\":{\"file\":{\"path\":\"/usr/local/nginx/logs/access-json.log\"},\"offset\":0},\"message\":\"{ \\\"@timestamp\\\": \\\"2024-06-30T14:49:05+08:00\\\", \\\"remote_addr\\\": \\\"192.168.1.2\\\", \\\"http_host\\\": \\\"192.168.1.7\\\",\\\"referer\\\": \\\"-\\\", \\\"scheme\\\": \\\"http\\\", \\\"request\\\": \\\"GET / HTTP/1.1\\\", \\\"request_method\\\": \\\"GET\\\", \\\"request_time\\\": \\\"0.000\\\", \\\"server_protocol\\\": \\\"HTTP/1.1\\\", \\\"uri\\\": \\\"/index.html\\\", \\\"http_host\\\": \\\"192.168.1.7\\\", \\\"domain\\\":\\\"localhost\\\",\\\"hostname\\\":\\\"oracle1\\\",\\\"status\\\": 403, \\\"bytes\\\":555, \\\"agent\\\": \\\"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36\\\", \\\"x_forwarded\\\": \\\"-\\\", \\\"upstr_addr\\\": \\\"-\\\",\\\"upstr_host\\\": \\\"-\\\",\\\"ups_resp_time\\\": \\\"-\\\" }\",\"tags\":[\"sspu-nginx\"],\"input\":{\"type\":\"log\"},\"ecs\":{\"version\":\"1.5.0\"},\"host\":{\"name\":\"oracle1\"},\"agent\":{\"ephemeral_id\":\"9258b711-0435-49d5-b62f-55b817562c85\",\"id\":\"b22f72a4-463d-4370-8316-ca11475146f6\",\"name\":\"oracle1\",\"type\":\"filebeat\",\"version\":\"7.9.2\",\"hostname\":\"oracle1\"}}"
2) "{\"@timestamp\":\"2024-06-30T07:09:54.614Z\",\"@metadata\":{\"beat\":\"filebeat\",\"type\":\"_doc\",\"version\":\"7.9.2\"},\"log\":{\"offset\":567,\"file\":{\"path\":\"/usr/local/nginx/logs/access-json.log\"}},\"message\":\"{ \\\"@timestamp\\\": \\\"2024-06-30T14:49:05+08:00\\\", \\\"remote_addr\\\": \\\"192.168.1.2\\\", \\\"http_host\\\": \\\"192.168.1.7\\\",\\\"referer\\\": \\\"http://192.168.1.7/\\\", \\\"scheme\\\": \\\"http\\\", \\\"request\\\": \\\"GET /favicon.ico HTTP/1.1\\\", \\\"request_method\\\": \\\"GET\\\", \\\"request_time\\\": \\\"0.000\\\", \\\"server_protocol\\\": \\\"HTTP/1.1\\\", \\\"uri\\\": \\\"/favicon.ico\\\", \\\"http_host\\\": \\\"192.168.1.7\\\", \\\"domain\\\":\\\"localhost\\\",\\\"hostname\\\":\\\"oracle1\\\",\\\"status\\\": 404, \\\"bytes\\\":555, \\\"agent\\\": \\\"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36\\\", \\\"x_forwarded\\\": \\\"-\\\", \\\"upstr_addr\\\": \\\"-\\\",\\\"upstr_host\\\": \\\"-\\\",\\\"ups_resp_time\\\": \\\"-\\\" }\",\"tags\":[\"sspu-nginx\"],\"input\":{\"type\":\"log\"},\"ecs\":{\"version\":\"1.5.0\"},\"host\":{\"name\":\"oracle1\"},\"agent\":{\"version\":\"7.9.2\",\"hostname\":\"oracle1\",\"ephemeral_id\":\"9258b711-0435-49d5-b62f-55b817562c85\",\"id\":\"b22f72a4-463d-4370-8316-ca11475146f6\",\"name\":\"oracle1\",\"type\":\"filebeat\"}}"

7.redis-to-logstash-ES;

将redis的数据推送到:logstash;再由logstash将数据推送到ES; 

LOGSTASH配置管道。
--配置多个管道。
mkdir -p /esdb/logstash/app/config/pipelines

vi /esdb/logstash/app/config/pipelines/sspu-nginx.conf


filebeat:nginx-to-redis
filebeat:tomcat-to-redis
logstash:redis-to-elasticsearch 
logstash:tomcat-to-elasticsearch

--配置管道信息
vi /esdb/logstash/app/config/pipelines.yml

8.
--默认读取多管道。

nohup /esdb/logstash/app/bin/logstash &
--无法启动logstash;
[2024-06-30T15:50:43,076][ERROR][logstash.inputs.redis    ] Invalid setting for redis input plugin:

  input {
    redis {
      # This setting must be a ["list", "channel", "pattern_channel"]
      # Expected one of ["list", "channel", "pattern_channel"], got ["1ist"]
      data_type => "1ist"
      ...
    }
  }

LogStash::Error: Don't know how to handle `Java::JavaLang::IllegalStateException` for `PipelineAction::Creat<sspu-nginx>`
          create at org/logstash/execution/ConvergeResultExt.java:129
             add at org/logstash/execution/ConvergeResultExt.java:57
  converge_state at /esdb/logstash/app/logstash-core/lib/logstash/agent.rb:370
[2024-06-30T15:40:51,340][ERROR][logstash.agent           ] An exception happened when converging configuraton {:exception=>LogStash::Error, :message=>"Don't know how to handle `Java::JavaLang::IllegalStateException`for `PipelineAction::Create<sspu-nginx>`"}
[2024-06-30T15:40:51,384][FATAL][logstash.runner          ] An unexpected error occurred! {:error=>#<LogStas::Error: Don't know how to handle `Java::JavaLang::IllegalStateException` for `PipelineAction::Create<sspu-ninx>`>, :backtrace=>["org/logstash/execution/ConvergeResultExt.java:129:in `create'", "org/logstash/executio/ConvergeResultExt.java:57:in `add'", "/esdb/logstash/app/logstash-core/lib/logstash/agent.rb:370:in `block n converge_state'"]}
[2024-06-30T15:40:51,405][ERROR][org.logstash.Logstash    ] java.lang.IllegalStateException: Logstash stoppe processing because of an error: (SystemExit) exit
'

字符书写错误:
l(L)写成了一"1";
重新启动:
nohup /esdb/logstash/app/bin/logstash &

--对客户端发送了认证，但是没有提供密码。
[2024-06-30T16:01:57,709][WARN ][logstash.inputs.redis    ]
[sspu-nginx][3ad6a78b18408d0a9b60bc832cf38c0a7bebc8f08c2d2a921f7d4d068953c212] 
Redis connection problem {:exception=>#<Redis::CommandError: ERR Client sent AUTH, but no password is set>}

--成功启动logstash;
[2024-06-30T16:04:23,821][INFO ][logstash.javapipeline    ][sspu-nginx] Pipeline Java execution initialization time {"seconds"=>1.17}
[2024-06-30T16:04:23,893][INFO ][logstash.inputs.redis    ][sspu-nginx] Registering Redis {:identity=>"redis://@192.168.1.7:6379/0 list:sspu-nginx"}
[2024-06-30T16:04:23,894][INFO ][logstash.inputs.redis    ][sspu-nginx] Registering Redis {:identity=>"redis://@192.168.1.7:6379/0 list:sspu-nginx"}
[2024-06-30T16:04:23,896][INFO ][logstash.inputs.redis    ][sspu-nginx] Registering Redis {:identity=>"redis://@192.168.1.7:6379/0 list:sspu-nginx"}
[2024-06-30T16:04:23,897][INFO ][logstash.inputs.redis    ][sspu-nginx] Registering Redis {:identity=>"redis://@192.168.1.7:6379/0 list:sspu-nginx"}
[2024-06-30T16:04:23,898][INFO ][logstash.inputs.redis    ][sspu-nginx] Registering Redis {:identity=>"redis://@192.168.1.7:6379/0 list:sspu-nginx"}
[2024-06-30T16:04:24,007][INFO ][logstash.javapipeline    ][sspu-nginx] Pipeline started {"pipeline.id"=>"sspu-nginx"}
[2024-06-30T16:04:24,096][INFO ][logstash.agent           ] Pipelines running {:count=>1, :running_pipelines=>[:"sspu-nginx"], :non_running_pipelines=>[]}
[2024-06-30T16:04:24,910][INFO ][logstash.agent           ] Successfully started Logstash API endpoint {:port=>9600}

11.
--检查生成的ES索引。

sspu-nginx-redis-2024.06.30 green open 3  1 11 83.6kb 

12.总结

如果redis没有设置密码，则配置logstash时，注释掉:password 关键字。