CentOS6.5安装elasticsearch+logstash+kibana

部署中心节点

如果有防火墙需要放开这些端口：

· port 80 (for the web interface)

· port 5544 (to receive remote syslog messages)

· port 9200 (so the web interface can access elasticsearch)

安装 java环境和Apache服务

JVM版本建议：

 Java 8 update 20 or later, or Java 7 update 55 or later 

否则elasticsearch启动后报错：elasticsearch dead but subsys locked

 

第一步：执行 rpm -qa|grep jdk 命令查看当前的jdk情况。

第二步：执行 yum -y remove java java-1.7.0-openjdk* 卸载openjdk，在执行这个命令的过程中，会卸载删除一些和openJDK有依赖关系的软件，不用担心，无影响

 

卸载系统自带的openJDK, 重新下载JDK

Please confirm that your Java version is 7 or higher.

#https://www.reucon.com/cdn/java/jdk-7u67-linux-x64.tar.gz

#tar zxvf jdk-7u67-linux-x64.tar.gz

#mv jdk1.7.0_67 /usr/local/

#cd /usr/local/

#ln -s jdk1.7.0_67 jdk

#chown -R root:root jdk/

 

配置环境变量

vim /etc/profile

export JAVA_HOME=/usr/local/jdk   

export JRE_HOME=$JAVA_HOME/jre

export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar:$JRE_HOME/lib:$CLASSPATH

export PATH=$JAVA_HOME/bin:$PATH

export REDIS_HOME=/usr/local/redis

export ES_HOME=/usr/local/elasticsearch

export ES_CLASSPATH=$ES_HOME/config

 

变量生效

source /etc/profile

 

验证版本

java -version

安装ES环境 elasticsearch  

wget https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch-1.6.0.noarch.rpm &&  yum install elasticsearch-1.6.0.noarch.rpm -y

 

安装logstash环境

wget  https://download.elastic.co/logstash/logstash/packages/centos/logstash-1.5.1-1.noarch.rpm && yum install logstash-1.5.1-1.noarch.rpm -y

 

安装kibana环境

wget https://download.elastic.co/kibana/kibana/kibana-4.1.6-linux-x64.tar.gz

tar zxvf kibana-4.1.3-linux-x64.tar.gz –C /usr/local/kibana

 

安装elasticsearch插件

cd /usr/share/elasticsearch/ &&  ./bin/plugin -install mobz/elasticsearch-head && ./bin/plugin -install lukas-vlcek/bigdesk/2.5.0

修改elasticsearch配置文件

vim /etc/elasticsearch/elasticsearch.yml  

cluster.name: elasticsearch 去掉前面”#”号  

http.cors.enabled: true  

启动elasticsearch

/etc/rc.d/init.d/elasticsearch start  

测试elasticsearch

[root@localhost ~]# curl 127.0.0.1:9200  

{

"status" : 200,

"name" : "Agamotto",

"cluster_name" : "elasticsearch",

"version" : {

"number" : "1.4.1",

"build_hash" : "89d3241d670db65f994242c8e8383b169779e2d4",

"build_timestamp" : "2015-11-26T15:49:29Z",

"build_snapshot" : false,  

"lucene_version" : "4.10.2"  

},

"tagline" : "You Know, for Search"  

}  

[root@localhost ~]#  

配置logstash，如下是Logstash的配置文件

[root@localhost conf.d]# vim /etc/logstash/conf.d/logstash_access.conf

input {

file {

    type => "apache"

    path => ["/etc/httpd/logs/*_log","/var/log/nginx/*.log"]

}

file {

    type => "syslog"

    path => [ "/var/log/messages", "/var/log/syslog" ]

}

file {

    type => "nova"

    path => [ "/var/log/nova/*.log" ]

}

file {

    type => "cinder"

    path => [ "/var/log/cinder/*.log" ]

}

file {

    type => "neutron"

    path => [ "/var/log/neutron/*.log" ]

}

file {

    type => "ceilometer"

    path => [ "/var/log/ceilometer/*.log" ]

}

}

filter {

if [type] == "syslog" {

    grok {

      match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }

      add_field => [ "received_at", "%{@timestamp}" ]

      add_field => [ "received_from", "%{host}" ]

    }

    syslog_pri { }

    date {

      match => [ "syslog_timestamp", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]

    }

  }else if [type] == "apache"  {

                if [path] =~ "access" {

                        mutate { replace => { loglevel => "apache_access" } }

                        grok {

                                match => { "message" => "%{COMBINEDAPACHELOG}" }

                                }

                        date {

                                match => [ "timestamp" , "dd/MMM/yyyy:HH:mm:ss Z" ]

        }

        } else if [path] =~ "error" {

                mutate { replace => { loglevel  => "apache_error" } }

                } else {

                        mutate { replace => { loglevel => "apache_random_logs" } }

}

}else {

        grok {

            match => ["message","%{TIMESTAMP_ISO8601:logtime} %{NUMBER:pid} %{WORD:loglevel} %{DATA:process} %{GREEDYDATA:other}"]

    }

  }

}

output {

        stdout { codec => rubydebug }

        elasticsearch_http {

                host => "192.168.215.101"

                port => "9200"

        }

}

 

启动logstash

[root@localhost bin]#/opt/logstash/bin/logstash -f /etc/logstash/conf.d/logstasg_access.conf

 

或者/etc/init.d/logstash start

启动kibana

cd /usr/local/kibana/bin/

./kibana

或者将kibana做成服务后用命令service kibana start启动

Kibana 服务

#!/bin/bash

### BEGIN INIT INFO

# Provides:          kibana

# Default-Start:     2 3 4 5

# Default-Stop:      0 1 6

# Short-Description: Runs kibana daemon

# Description: Runs the kibana daemon as a non-root user

### END INIT INFO

# Process name

NAME=kibana

DESC="Kibana4"

PROG="/etc/init.d/kibana"

# Configure location of Kibana bin

KIBANA_BIN=/usr/local/kibana/bin

# PID Info

PID_FOLDER=/var/run/kibana/

PID_FILE=/var/run/kibana/$NAME.pid

LOCK_FILE=/var/lock/subsys/$NAME

PATH=/bin:/usr/bin:/sbin:/usr/sbin:$KIBANA_BIN

DAEMON=$KIBANA_BIN/$NAME

# Configure User to run daemon process

DAEMON_USER=root

# Configure logging location

KIBANA_LOG=/var/log/kibana.log

# Begin Script

RETVAL=0

if [ `id -u` -ne 0 ]; then

        echo "You need root privileges to run this script"

        exit 1

fi

# Function library

. /etc/init.d/functions

 

start() {

        echo -n "Starting $DESC : "

pid=`pidofproc -p $PID_FILE kibana`

        if [ -n "$pid" ] ; then

                echo "Already running."

                exit 0

        else

        # Start Daemon

if [ ! -d "$PID_FOLDER" ] ; then

                        mkdir $PID_FOLDER

                fi

daemon --user=$DAEMON_USER --pidfile=$PID_FILE $DAEMON 1>"$KIBANA_LOG" 2>&1 &

                sleep 2

                pidofproc node > $PID_FILE

                RETVAL=$?

                [[ $? -eq 0 ]] && success || failure

echo

                [ $RETVAL = 0 ] && touch $LOCK_FILE

                return $RETVAL

        fi

}

reload()

{

    echo "Reload command is not implemented for this service."

    return $RETVAL

}

stop() {

        echo -n "Stopping $DESC : "

        killproc -p $PID_FILE $DAEMON

        RETVAL=$?

echo

        [ $RETVAL = 0 ] && rm -f $PID_FILE $LOCK_FILE

}

 

case "$1" in

  start)

        start

;;

  stop)

        stop

        ;;

  status)

        status -p $PID_FILE $DAEMON

        RETVAL=$?

        ;;

  restart)

        stop

        start

        ;;

  reload)

reload

;;

  *)

# Invalid Arguments, print the following message.

        echo "Usage: $0 {start|stop|status|restart}" >&2

exit 2

        ;;

esac

 

启动apache

/etc/rc.d/init.d/httpd start

http://192.168.213.11:5601  web页面中展示
elk
elk 

删除ES日志存储：

[root@localhost bin]# curl -XDELETE 'http://192.168.213.11:9200/logstash-2015.06.03*'