filebeat_elk多机环境入门探测（三）

最新推荐文章于 2022-08-19 11:11:08 发布

weixin_33910759

最新推荐文章于 2022-08-19 11:11:08 发布

阅读量69

点赞数

文章标签：数据库大数据 java

原文链接：https://my.oschina.net/u/1762991/blog/887777

版权

2019独角兽企业重金招聘Python工程师标准>>>

接着上一篇继续https://my.oschina.net/u/1762991/blog/887751（filebeat_elk多机环境入门探测（二））

使用redis作为消息中间件，在filebeat上收集到日志后先放到redis中，logstash的input段中指定redis的位置，这样做的目的就是为了在filebeat收集日志过多时使消息不丢失。

redis版本为redis-2.8.20.tar.gz

在test2上安装redis

自动化安装脚本：
源码编译安装，提前准备好jdk,redis
# environment prepare. NOTICE: jdk need versions
rpm -q gcc > /dev/null
[ $? -ne 0 ] && yum install -y gcc > /dev/null
rpm -q gcc-c++ > /dev/null
[ $? -ne 0 ] && yum install -y gcc-c++ > /dev/null
rpm -q jdk > /dev/null
[ $? -ne 0 ] && yum install -y /usr/local/sr/jdk* > /dev/null
cat > /etc/profile.d/java.sh << EOF
export JAVA_HOME=/usr/java/latest
export PATH=\${JAVA_HOME}/bin:\${PATH}
EOF
source /etc/profile.d/java.sh

# redis source build install
tar xf /usr/local/src/redis-2.8.20.tar.gz -C /usr/local/
cd /usr/local/
cd redis-2.8.20/
make > /dev/null

# configuration redis
mkdir -p /usr/local/redis/{bin,etc,var}
cp -af src/{redis-benchmark,redis-check-aof,redis-check-dump,redis-cli,redis-sentinel,redis-server} /usr/local/redis/bin/
cp -a redis.conf /usr/local/redis/etc/
echo "export PATH=/usr/local/redis/bin:\$PATH" > /etc/profile.d/redis2.8.sh
source /etc/profile.d/redis2.8.sh
sed -i 's@pidfile.*@pidfile /var/run/redis.pid@' /usr/local/redis/etc/redis.conf
sed -i "s@logfile.*@logfile /usr/local/redis/var/redis.log@" /usr/local/redis/etc/redis.conf
sed -i "s@^dir.*@dir /usr/local/redis/var@" /usr/local/redis/etc/redis.conf
sed -i 's@daemonize no@daemonize yes@' /usr/local/redis/etc/redis.conf
[ -z "`grep ^maxmemory /usr/local/redis/etc/redis.conf`" ] && sed -i 's@maxmemory <bytes>@maxmemory <bytes>\nmaxmemory 360000000@' /usr/local/redis/etc/redis.conf

# add chkconfig and start redis
wget http://www.dwhd.org/script/Redis-server-init-CentOS -O /etc/init.d/redis-server
chmod +x /etc/init.d/redis-server
chkconfig --add redis-server
chkconfig redis-server on
service redis-server start
[root@test2 ~]# bash auto_redis_install.sh

修改logstash配置
[root@iptables2 ~]# cat ver6.conf
input {
#    beats {
#        port => 5044
#        type => "syslog"
#    }
    redis {
        host => "192.168.40.103"
        data_type => "list"
        type => "redis-input"
        key => "filebeat"
    }
}

filter {
    if [type] == "filebeat" {
        grok {
            match => [ "message", "%{SYSLOGLINE}" ]
            overwrite => [ "message" ]
        }
    }
    date {
        match => [ "timestamp", "MMM dd HH:mm:ss", "MMM d HH:mm:ss" ]
    }
}

output {
    stdout {
        codec => rubydebug
    }
#    elasticsearch {
#        hosts => "192.168.40.105:9200"
#    }
}

修改filebeat配置
[root@test2 ~]# cat /etc/filebeat/filebeat.yml
##################################################### filebeat #######################################################
filebeat:
prospectors:
    -
      paths:
        - /var/log/messages
      input_type: log
      document_type: messages

    -
      paths:
        - /var/log/secure
      input_type: syslog
      document_type: loginmsg

    -
      paths:
        - /var/log/nginx_access.log
      input_type: log
      document_type: nginxacclog

      multiline:
          pattern: '^[[:space:]]'
          negate: true
          match: after

registry_file: /var/lib/filebeat/registry

##################################################### output #######################################################
#output:
# logstash:
#    hosts: ["192.168.40.83:5044"]
output:
redis:
    host: "192.168.40.103"
    port: 6379
    save_topology: true
    index: "filebeat"
    db: 0
    db_topology: 1
    timeout: 5
    reconnect_interval: 1

##################################################### Logging #######################################################
logging:
files:
rotateeverybytes: 10485760 # = 10MB
重启filebeat

启动logstash
[root@iptables2 ~]# ./logstash-2.3.2/bin/logstash -f ver6.conf
Settings: Default pipeline workers: 2
Pipeline main started
{
    "@timestamp" => "2017-04-18T02:11:42.285Z",
          "beat" => {
        "hostname" => "test2",
            "name" => "test2"
    },
         "count" => 1,
        "fields" => nil,
    "input_type" => "log",
       "message" => "Apr 18 10:11:34 localhost sshd[10402]: Received disconnect from 192.168.40.26: 0: ",
        "offset" => 2839,
        "source" => "/var/log/secure",
          "type" => "loginmsg",
      "@version" => "1"
}
{
    "@timestamp" => "2017-04-18T02:11:42.285Z",
          "beat" => {
        "hostname" => "test2",
            "name" => "test2"
    },
         "count" => 1,
        "fields" => nil,
    "input_type" => "log",
       "message" => "Apr 18 10:11:34 localhost sshd[10402]: pam_unix(sshd:session): session closed for user root",
        "offset" => 2922,
        "source" => "/var/log/secure",
          "type" => "loginmsg",
      "@version" => "1"
}
操作test2回话，产生日志，标准输出中有了

修改logstash配置，把输出放到Elasticsearch中和标准输出中
[root@iptables2 ~]# cat ver6.conf
input {
#    beats {
#        port => 5044
#        type => "syslog"
#    }
    redis {
        host => "192.168.40.103"
        data_type => "list"
        type => "redis-input"
        key => "filebeat"
    }
}

output {
    stdout {
        codec => rubydebug
    }
    elasticsearch {
        hosts => "192.168.40.105:9200"
    }
}
启动logstash，操作产生会话日志
[root@iptables2 ~]# ./logstash-2.3.2/bin/logstash -f ver6.conf
Settings: Default pipeline workers: 2
Pipeline main started
{
    "@timestamp" => "2017-04-18T02:17:42.417Z",
          "beat" => {
        "hostname" => "test2",
            "name" => "test2"
    },
         "count" => 1,
        "fields" => nil,
    "input_type" => "log",
       "message" => "Apr 18 10:17:33 localhost sshd[10456]: Received disconnect from 192.168.40.26: 0: ",
        "offset" => 3218,
        "source" => "/var/log/secure",
          "type" => "loginmsg",
      "@version" => "1"
}
{
    "@timestamp" => "2017-04-18T02:17:42.417Z",
          "beat" => {
        "hostname" => "test2",
            "name" => "test2"
    },
         "count" => 1,
        "fields" => nil,
    "input_type" => "log",
       "message" => "Apr 18 10:17:33 localhost sshd[10456]: pam_unix(sshd:session): session closed for user root",
        "offset" => 3301,
        "source" => "/var/log/secure",
          "type" => "loginmsg",
      "@version" => "1"
}

访问http://192.168.40.105:9200/_search?pretty
有数据

安装kibana，在test5中安装kibana

安装kibana
tar xf /usr/local/src/kibana-4.5.3-linux-x64.tar.gz -C /usr/local
cd /usr/local/
ln -s kibana-4.5.3-linux-x64 kibana
cp kibana/config/kibana.yml kibaba/config/kibana.yml.bak_$(date +%F_%H:%M)
配置kibana.yml
server.port: 5601
server.host: "0.0.0.0"
elasticsearch.url: "http://192.168.40.105:9200"
其它内容不动
启动kibana
./kibana/bin/kibana
访问
http://192.168.40.105:5601/即可

访问http://192.168.40.103/
此时nginx产生了日志
{
    "@timestamp" => "2017-04-18T06:49:25.700Z",
          "beat" => {
        "hostname" => "test2",
            "name" => "test2"
    },
         "count" => 1,
        "fields" => nil,
    "input_type" => "log",
       "message" => "{\"@timestamp\":\"2017-04-18T14:49:24+08:00\",\"slbip\":\"192.168.40.26\",\"clientip\":\"-\",\"serverip\":\"192.168.40.103\",\"size\":\"2340\",\"responsetime\":\"0.751\",\"domain\":\"192.168.40.103\",\"method\":\"GET\",\"requesturi\":\"/\",\"url\":\"/\",\"appversion\":\"-\",\"referer\":\"-\",\"agent\":\"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36\",\"status\":\"200\",\"devicecode\":\"-\"}",
        "offset" => 471022,
        "source" => "/var/log/nginx_access.log",
          "type" => "nginxacclog",
      "@version" => "1"
}
{
    "@timestamp" => "2017-04-18T06:49:40.712Z",
          "beat" => {
        "hostname" => "test2",
            "name" => "test2"
    },
         "count" => 1,
        "fields" => nil,
    "input_type" => "log",
       "message" => "{\"@timestamp\":\"2017-04-18T14:49:40+08:00\",\"slbip\":\"192.168.40.26\",\"clientip\":\"-\",\"serverip\":\"192.168.40.103\",\"size\":\"5\",\"responsetime\":\"0.061\",\"domain\":\"192.168.40.103\",\"method\":\"GET\",\"requesturi\":\"/session\",\"url\":\"/session\",\"appversion\":\"-\",\"referer\":\"-\",\"agent\":\"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36\",\"status\":\"302\",\"devicecode\":\"-\"}\n{\"@timestamp\":\"2017-04-18T14:49:43+08:00\",\"slbip\":\"192.168.40.26\",\"clientip\":\"-\",\"serverip\":\"192.168.40.103\",\"size\":\"8862\",\"responsetime\":\"2.946\",\"domain\":\"192.168.40.103\",\"method\":\"GET\",\"requesturi\":\"/session/\",\"url\":\"/session/\",\"appversion\":\"-\",\"referer\":\"-\",\"agent\":\"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36\",\"status\":\"500\",\"devicecode\":\"-\"}",
        "offset" => 471419,
        "source" => "/var/log/nginx_access.log",
          "type" => "nginxacclog",
      "@version" => "1"
}