elk分析nginx日志和展示

本次使用7.10.2版本的elk

搭建logstach

建议将logstach和nginx部署到一台方便日志收集，如果想分开部署需要添加filebeat

wget https://artifacts.elastic.co/downloads/logstash/logstash-7.10.2-x86_64.rpm

rpm -ivh logstash-7.10.2-x86_64.rpm
cat /etc/logstash/logstash.yml |grep -v '#' |grep -v '^$'
path.data: /var/lib/logstash
path.config: /etc/logstash/conf.d/					打开注释,并加上配置目录路径
http.host: "10.1.1.13"								打开注释,并改为本机IP
path.logs: /var/log/logstash

vim /etc/logstash/conf.d/nginx.conf
input {
  file {
        ## 修改你环境nginx日志路径
        path => "/var/log/nginx/access.log"
        ignore_older => 0 
    codec => json
    }
}

filter {
  geoip {
    #multiLang => "zh-CN"
    target => "geoip"
    source => "client_ip"
    database => "/usr/share/logstash/GeoLite2-City.mmdb"
    add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
    add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ]
    # 去掉显示 geoip 显示的多余信息
    remove_field => ["[geoip][latitude]", "[geoip][longitude]", "[geoip][country_code]", "[geoip][country_code2]", "[geoip][country_code3]", "[geoip][timezone]", "[geoip][continent_code]", "[geoip][region_code]"]
  }
  mutate {
    convert => [ "size", "integer" ]
    convert => [ "status", "integer" ]
    convert => [ "responsetime", "float" ]
    convert => [ "upstreamtime", "float" ]
    convert => [ "[geoip][coordinates]", "float" ]
    # 过滤 filebeat 没用的字段,这里过滤的字段要考虑好输出到es的，否则过滤了就没法做判断
    remove_field => [ "ecs","agent","host","cloud","@version","input","logs_type" ]
  }
  # 根据http_user_agent来自动处理区分用户客户端系统与版本
  useragent {
    source => "http_user_agent"
    target => "ua"
    # 过滤useragent没用的字段
    remove_field => [ "[ua][minor]","[ua][major]","[ua][build]","[ua][patch]","[ua][os_minor]","[ua][os_major]" ]
  }
}
output {
  elasticsearch {
    hosts => "192.168.31.196"
    #user => "elastic"
    #password => "password"
    index => "logstash-nginx-%{+YYYY.MM.dd}"
  }
}

chmod o+r /var/log/messages
cd /usr/share/logstash/bin
nohup ./logstash --path.settings /etc/logstash/ -f /etc/logstash/conf.d/nginx.conf &
tail -f nohup.out

出现这个字样说明启动成功

nginx配置

http {
    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';

    log_format aka_logs
                      '{"@timestamp":"$time_iso8601",'
                      '"host":"$hostname",'
                      '"server_ip":"$server_addr",'
                      '"client_ip":"$remote_addr",'
                      '"xff":"$http_x_forwarded_for",'
                      '"domain":"$host",'
                      '"url":"$uri",'
                      '"referer":"$http_referer",'
                      '"args":"$args",'
                      '"upstreamtime":"$upstream_response_time",'
                      '"responsetime":"$request_time",'
                      '"request_method":"$request_method",'
                      '"status":"$status",'
                      '"size":"$body_bytes_sent",'
                      '"request_body":"$request_body",'
                      '"request_length":"$request_length",'
                      '"protocol":"$server_protocol",'
                      '"upstreamhost":"$upstream_addr",'
                      '"file_dir":"$request_filename",'
                      '"http_user_agent":"$http_user_agent"'
    '}';

    #access_log  /var/log/nginx/access.log  main;
    access_log  /var/log/nginx/access.log  aka_logs;

加入log_format aka_logs配置

elasticsearch安装

tar -xvf elasticsearch-7.10.2-linux-x86_64.tar.gz -C /usr/local/
vim /etc/security/limits.conf
* soft nofile 65535
* hard nofile 65535
sysctl -w vm.max_map_count=655360
echo 'vm.max_map_count=655360' >> /etc/sysctl.conf
sysctl -p
useradd es
passwd es
chown -R es.es elasticsearch-7.10.2
su es
cd /usr/local/elasticsearch-7.10.2/
[es@pass elasticsearch-7.10.2]$ cat ./config/elasticsearch.yml |grep -v '#'
cluster.name: shuhai
node.name: node-1
network.host: ip地址
cluster.initial_master_nodes: ["node-1"]
http.cors.enabled: true
http.cors.allow-origin: "*"

./bin/elasticsearch -d

lsof -i:9200
#确认启动成功

grafana配置

grafana-cli plugins install grafana-piechart-panel
grafana-cli plugins install grafana-worldmap-panel
cd /var/lib/grafana/plugins/
sed -i 's/https:\/\/cartodb-basemaps{s}.global.ssl.fastly.net\/light_all\/{z}\/{x}\/{y}.png/http:\/\/{s}.basemaps.cartocdn.com\/light_all\/{z}\/{x}\/{y}.png/' module.js
sed -i 's/https:\/\/cartodb-basemaps{s}.global.ssl.fastly.net\/light_all\/{z}\/{x}\/{y}.png/http:\/\/{s}.basemaps.cartocdn.com\/light_all\/{z}\/{x}\/{y}.png/' module.js.map
sed -i 's/https:\/\/cartodb-basemaps-{s}.global.ssl.fastly.net\/dark_all\/{z}\/{x}\/{y}.png/http:\/\/{s}.basemaps.cartocdn.com\/dark_all\/{z}\/{x}\/{y}.png/' module.js
sed -i 's/https:\/\/cartodb-basemaps-{s}.global.ssl.fastly.net\/dark_all\/{z}\/{x}\/{y}.png/http:\/\/{s}.basemaps.cartocdn.com\/dark_all\/{z}\/{x}\/{y}.png/' module.js.map
systemctl restart grafana-server

 

 11190模板号