elasticsearch

[root@server1 elk]# rpm -ivh elasticsearch-2.3.3.rpm
[root@server1 elk]# cd /etc/elasticsearch/
[root@server1 elasticsearch]# vim elasticsearch.yml

cluster.name: my-application
node.name: server1
path.data: /var/lib/elasticsearch/
bootstrap.mlockall: true
network.host: 172.25.20.1
http.port: 9200

[root@server1 ~]# rpm -ivh jdk-8u121-linux-x64.rpm
[root@server1 ~]# /etc/init.d/elasticsearch start

Starting elasticsearch:                                    [  OK  ]

[root@server1 ~]# cat /var/log/elasticsearch/my-application.log

[2018-10-26 09:47:23,233][WARN ][bootstrap                ] These can be adjusted by modifying /etc/security/limits.conf, for example: 
	# allow user 'elasticsearch' mlockall
	elasticsearch soft memlock unlimited
	elasticsearch hard memlock unlimited

[root@server1 ~]# vim /etc/security/limits.conf

elasticsearch soft memlock unlimited
elasticsearch hard memlock unlimited

[root@server1 ~]# /etc/init.d/elasticsearch restart

Stopping elasticsearch:                                    [  OK  ]
Starting elasticsearch:                                    [  OK  ]

在这里插入图片描述
[root@server1 elk]# cd /usr/share/elasticsearch/
[root@server1 elasticsearch]# cd bin/
[root@server1 bin]# ./plugin install file:/root/elk/elasticsearch-head-master.zip
-> Installing from file:/root/elk/elasticsearch-head-master.zip…

Trying file:/root/elk/elasticsearch-head-master.zip ...
Downloading .........DONE
Verifying file:/root/elk/elasticsearch-head-master.zip checksums if available ...
NOTE: Unable to verify checksum for downloaded plugin (unable to find .sha1 or .md5 file to verify)
Installed head into /usr/share/elasticsearch/plugins/head

[root@server1 bin]# cd …
[root@server1 elasticsearch]# cd plugins/
[root@server1 plugins]# cd head/
[root@server1 head]# ls

elasticsearch-head.sublime-project  index.html    plugin-descriptor.properties  src
Gruntfile.js                        LICENCE       README.textile                test
grunt_fileSets.js                   package.json  _site

在这里插入图片描述
在这里插入图片描述
在这里插入图片描述
Server1:
[root@server1 ~]# vim /etc/elasticsearch/elasticsearch.yml

discovery.zen.ping.unicast.hosts: ["server1", "server2", "server3"]

[root@server1 ~]# /etc/init.d/elasticsearch restart

Stopping elasticsearch:                                    [  OK  ]
Starting elasticsearch:                                    [  OK  ]

Server2:
[root@server2 ~]# yum install -y elasticsearch-2.3.3.rpm jdk-8u121-linux-x64.rpm
[root@server2 ~]# vim /etc/security/limits.conf

elasticsearch soft memlock unlimited
elasticsearch hard memlock unlimited

[root@server2 ~]# vim /etc/elasticsearch/elasticsearch.yml

cluster.name: my-application
node.name: server2
path.data: /var/lib/elasticsearch/
bootstrap.mlockall: true
network.host: 172.25.20.2
http.port: 9200
discovery.zen.ping.unicast.hosts: ["server1", "server2", "server3"]

[root@server2 ~]# /etc/init.d/elasticsearch start

Starting elasticsearch:                                    [  OK  ]

Server3:
[root@server3 ~]# yum install -y elasticsearch-2.3.3.rpm jdk-8u121-linux-x64.rpm
[root@server3 ~]# vim /etc/security/limits.conf

elasticsearch soft memlock unlimited
elasticsearch hard memlock unlimited

[root@server3 ~]# vim /etc/elasticsearch/elasticsearch.yml

cluster.name: my-application
node.name: server1
path.data: /var/lib/elasticsearch/
bootstrap.mlockall: true
network.host: 172.25.20.3
http.port: 9200
discovery.zen.ping.unicast.hosts: ["server1", "server2", "server3"]

[root@server3 ~]# /etc/init.d/elasticsearch start

Starting elasticsearch:                                    [  OK  ]

在这里插入图片描述

Server1:
[root@server1 ~]# vim /etc/elasticsearch/elasticsearch.yml

node.data: false
node.master: true

[root@server1 ~]# /etc/init.d/elasticsearch restart

Stopping elasticsearch:                                    [  OK  ]
Starting elasticsearch:                                    [  OK  ]

在这里插入图片描述
Server2:
[root@server2 ~]# vim /etc/elasticsearch/elasticsearch.yml

node.master: false
node.data: true

[root@server2 ~]# /etc/init.d/elasticsearch restart

Stopping elasticsearch:                                    [  OK  ]
Starting elasticsearch:                                    [  OK  ]

Server3:
[root@server3 ~]# vim /etc/elasticsearch/elasticsearch.yml

node.master: false
node.data: true

[root@server3 ~]# /etc/init.d/elasticsearch restart

Stopping elasticsearch:                                    [  OK  ]
Starting elasticsearch:                                    [  OK  ]

在这里插入图片描述

Server1:
[root@server1 elk]# rpm -ivh logstash-2.3.3-1.noarch.rpm

Preparing...                ########################################### [100%]
   1:logstash               ########################################### [100%]

[root@server1 etc]# cd /opt/logstash/
[root@server1 logstash]# bin/logstash -e ‘input { stdin { } } output { stdout {} }’

Settings: Default pipeline workers: 1
Pipeline main started
hello world
2018-10-26T03:14:49.110Z server1 hello world
^CSIGINT received. Shutting down the agent. {:level=>:warn}
stopping pipeline {:id=>"main"}

Pipeline main has been shutdown

[root@server1 logstash]# bin/logstash -e ‘input { stdin { } } output { stdout {codec => rubydebug } elasticsearch { hosts => [“172.25.20.1”] index => “logstash-%{+YYYY.MM.dd}”} }’

Settings: Default pipeline workers: 1
Pipeline main started
hello
{
       "message" => "hello",
      "@version" => "1",
    "@timestamp" => "2018-10-26T03:19:43.566Z",
          "host" => "server1"
}
worls
{
       "message" => "worls",
      "@version" => "1",
    "@timestamp" => "2018-10-26T03:19:49.622Z",
          "host" => "server1"
}

在这里插入图片描述
[root@server1 logstash]# cd /etc/logstash/conf.d/
[root@server1 conf.d]# vim es.conf

input {
	stdin {}
 } 
output {
	stdout {
		codec => rubydebug
	} 
	elasticsearch {
		hosts => ["172.25.20.1"]
		index => "logstash-%{+YYYY.MM.dd}"
	}
}

[root@server1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/es.conf

Settings: Default pipeline workers: 1
Pipeline main started
hello
{
       "message" => "hello",
      "@version" => "1",
    "@timestamp" => "2018-10-26T03:33:26.958Z",
          "host" => "server1"
}
world
{
       "message" => "world",
      "@version" => "1",
    "@timestamp" => "2018-10-26T03:33:31.148Z",
          "host" => "server1"
}
^CSIGINT received. Shutting down the agent. {:level=>:warn}
stopping pipeline {:id=>"main"}

Pipeline main has been shutdown

[root@server1 conf.d]# vim es.conf

input {
	stdin {}
 } 
output {
#	stdout {
#		codec => rubydebug
#	} 
	elasticsearch {
		hosts => ["172.25.20.1"]
		index => "logstash-%{+YYYY.MM.dd}"
	}
	file {
		path => "/tmp/testfile"
		codec => line { format => "custom format: %{message}"}
	}
}

[root@server1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/es.conf

Settings: Default pipeline workers: 1
Pipeline main started
hello
world
^CSIGINT received. Shutting down the agent. {:level=>:warn}
stopping pipeline {:id=>"main"}

Pipeline main has been shutdown

[root@server1 conf.d]# cat /tmp/testfile

custom format: hello
custom format: world

在这里插入图片描述
[root@server1 conf.d]# vim es.conf

input {
        file {
                path => "/var/log/messages"
                start_position => "beginning"
        }
 }
output {
#       stdout {
#               codec => rubydebug
#       }
        elasticsearch {
                hosts => ["172.25.20.1"]
                index => "messages-%{+YYYY.MM.dd}"
        }
#       file {
#               path => "/tmp/testfile"
#               codec => line { format => "custom format: %{message}"}
#       }
}

[root@server1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/es.conf

Settings: Default pipeline workers: 1
Pipeline main started
^CSIGINT received. Shutting down the agent. {:level=>:warn}
stopping pipeline {:id=>"main"}
Pipeline main has been shutdown

在这里插入图片描述

Server2:

[root@server2 ~]# vim /etc/rsyslog.conf

# Provides UDP syslog reception
$ModLoad imudp
$UDPServerRun 514

# Provides TCP syslog reception
$ModLoad imtcp
$InputTCPServerRun 514

*.* @@172.25.20.1:514

[root@server2 ~]# /etc/init.d/rsyslog restart

Shutting down system logger:                               [  OK  ]
Starting system logger:                                    [  OK  ]

Server1:

[root@server1 conf.d]# vim es.conf

input {
#       file {
#               path => "/var/log/messages"
#               start_position => "beginning"
#       }
        syslog {
                port => 514
        }
 }
output {
#       stdout {
#               codec => rubydebug
#       } 
        elasticsearch {
                hosts => ["172.25.20.1"]
                index => "syslog-%{+YYYY.MM.dd}"
        }
#       file {
#               path => "/tmp/testfile"
#               codec => line { format => "custom format: %{message}"}
#       }
}

[root@server1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/es.conf

在这里插入图片描述

[root@server1 conf.d]# vim es.conf

input {
#       file {
#               path => "/var/log/elasticsearch/my-application.log"
#               start_position => "beginning"
#       }
#       syslog {
#               port => 514
#       }
        stdin {
                codec => multiline {
                pattern => "^\["
                negate => "true"
                what => "previous"
                }
        }
 }
#filter {
#multiline {
#                pattern => "^\["
#                negate => "true"
#                what => "previous"
#                }
#}
output {
        stdout {
                codec => rubydebug
        }
#       elasticsearch {
#               hosts => ["172.25.20.1"]
#               index => "es-%{+YYYY.MM.dd}"
#       }
#       file {
#               path => "/tmp/testfile"
#               codec => line { format => "custom format: %{message}"}
#       }
}

[root@server1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/es.conf

Settings: Default pipeline workers: 1
Pipeline main started
hello
hell
[
{
    "@timestamp" => "2018-10-26T06:16:12.501Z",
       "message" => "hello\nhell",
      "@version" => "1",
          "tags" => [
        [0] "multiline"
    ],
          "host" => "server1"
}
^CSIGINT received. Shutting down the agent. {:level=>:warn}
stopping pipeline {:id=>"main"}

Pipeline main has been shutdown

[root@server1 conf.d]# vim es.conf

input {
        file {
                path => "/var/log/elasticsearch/my-application.log"
                start_position => "beginning"
        }
#       syslog {
#               port => 514
#       }
#       stdin {
#               codec => multiline {
#               pattern => "^\["
#               negate => "true"
#               what => "previous"
#               }
#       }
 }
filter {
multiline {
                pattern => "^\["
                negate => "true"
                what => "previous"
                }
}
output {
#       stdout {
#               codec => rubydebug
#       }
        elasticsearch {
                hosts => ["172.25.20.1"]
                index => "es-%{+YYYY.MM.dd}"
        }
#       file {
#               path => "/tmp/testfile"
#               codec => line { format => "custom format: %{message}"}
#       }
}

[root@server1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/es.conf

Settings: Default pipeline workers: 1
Pipeline main started
^CSIGINT received. Shutting down the agent. {:level=>:warn}
stopping pipeline {:id=>"main"}
Pipeline main has been shutdown

在这里插入图片描述

[root@server1 ~]# l.

.              .bash_logout   .cshrc                                     .ssh
..             .bash_profile  .oracle_jre_usage                          .tcshrc
.bash_history  .bashrc        .sincedb_4d10e6fb4e3ffdf0fe00d1ce0f25b0fd  .viminfo

[root@server1 ~]# rm -rf .sincedb_4d10e6fb4e3ffdf0fe00d1ce0f25b0fd

在这里插入图片描述

[root@server1 conf.d]# yum install httpd -y
[root@server1 conf.d]# /etc/init.d/httpd start

Starting httpd: httpd: Could not reliably determine the server's fully qualified domain name, using 172.25.20.1 for ServerName
                                                           [  OK  ]

[root@server1 conf.d]# vim es.conf

input {
        file {
                path => "/var/log/httpd/access_log"
                start_position => "beginning"
        }
#       syslog {
#               port => 514
#       }
#       stdin {
#               codec => multiline {
#               pattern => "^\["
#               negate => "true"
#               what => "previous"
#               }
#       }
 }
#filter {
#multiline {
#               pattern => "^\["
#               negate => "true"
#               what => "previous"
#               }
#}
output {
        stdout {
                codec => rubydebug
        }
#       elasticsearch {
#               hosts => ["172.25.20.1"]
#               index => "apache-%{+YYYY.MM.dd}"
#       }
#       file {
#               path => "/tmp/testfile"
#               codec => line { format => "custom format: %{message}"}
#       }
}

[root@server1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/es.conf

Settings: Default pipeline workers: 1
Pipeline main started
{
       "message" => "172.25.20.250 - - [26/Oct/2018:15:11:00 +0800] \"GET / HTTP/1.1\" 403 3985 \"-\" \"Mozilla/5.0 (X11; Linux x86_64; rv:45.0) Gecko/20100101 Firefox/45.0\"",
      "@version" => "1",
    "@timestamp" => "2018-10-26T07:11:01.598Z",
          "path" => "/var/log/httpd/access_log",
          "host" => "server1"
}

[root@server1 conf.d]# vim test.conf
[root@server1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/test.conf

Settings: Default pipeline workers: 1
Pipeline main started
55.3.244.1 GET /index.html 15824 0.043
{
       "message" => "55.3.244.1 GET /index.html 15824 0.043",
      "@version" => "1",
    "@timestamp" => "2018-10-26T07:16:11.459Z",
          "host" => "server1",
        "client" => "55.3.244.1",
        "method" => "GET",
       "request" => "/index.html",
         "bytes" => "15824",
      "duration" => "0.043"
}
^CSIGINT received. Shutting down the agent. {:level=>:warn}
stopping pipeline {:id=>"main"}

Pipeline main has been shutdown

[root@server1 conf.d]# vim /etc/httpd/conf/httpd.conf

[root@server1 conf.d]# vim /opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-patterns-core-2.0.5/patterns/grok-patterns
[root@server1 conf.d]# vim es.conf

input {
        file {
                path => "/var/log/httpd/access_log"
                start_position => "beginning"
        }
#       syslog {
#               port => 514
#       }
#       stdin {
#               codec => multiline {
#               pattern => "^\["
#               negate => "true"
#               what => "previous"
#               }
#       }
 }
#filter {
#multiline {
#               pattern => "^\["
#               negate => "true"
#               what => "previous"
#               }
#}
filter {
        grok {
                match => { "message" => "%{COMBINEDAPACHELOG}"}
        }
}
output {
        stdout {
                codec => rubydebug
        }
        elasticsearch {
                hosts => ["172.25.20.1"]
                index => "apache-%{+YYYY.MM.dd}"
        }
#       file {
#               path => "/tmp/testfile"
#               codec => line { format => "custom format: %{message}"}
#       }
}

[root@server1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/es.conf

Settings: Default pipeline workers: 1
Pipeline main started
{
        "message" => "172.25.20.250 - - [26/Oct/2018:15:22:35 +0800] \"GET / HTTP/1.1\" 403 3985 \"-\" \"Mozilla/5.0 (X11; Linux x86_64; rv:45.0) Gecko/20100101 Firefox/45.0\"",
       "@version" => "1",
     "@timestamp" => "2018-10-26T07:22:35.549Z",
           "path" => "/var/log/httpd/access_log",
           "host" => "server1",
       "clientip" => "172.25.20.250",
          "ident" => "-",
           "auth" => "-",
      "timestamp" => "26/Oct/2018:15:22:35 +0800",
           "verb" => "GET",
        "request" => "/",
    "httpversion" => "1.1",
       "response" => "403",
          "bytes" => "3985",
       "referrer" => "\"-\"",
          "agent" => "\"Mozilla/5.0 (X11; Linux x86_64; rv:45.0) Gecko/20100101 Firefox/45.0\""
}

[root@server1 elk]# yum install -y nginx-1.8.0-1.el6.ngx.x86_64.rpm
[root@server1 conf.d]# vim nginx.conf

input {
        file {
                path => "/var/log/nginx/access.log"
                start_position => "beginning"
        }
 }
filter {
        grok {
                match => { "message" => "%{COMBINEDAPACHELOG} %{QS:xforward}" }
        }
}
output {
        stdout {
                codec => rubydebug
        }
        elasticsearch {
                hosts => ["172.25.20.1"]
                index => "nginx-%{+YYYY.MM.dd}"
        }
}

[root@server1 conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/nginx.conf

Settings: Default pipeline workers: 1
Pipeline main started
{
        "message" => "172.25.20.250 - - [26/Oct/2018:16:04:39 +0800] \"GET / HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (X11; Linux x86_64; rv:45.0) Gecko/20100101 Firefox/45.0\" \"-\"",
       "@version" => "1",
     "@timestamp" => "2018-10-26T08:04:40.693Z",
           "path" => "/var/log/nginx/access.log",
           "host" => "server1",
       "clientip" => "172.25.20.250",
          "ident" => "-",
           "auth" => "-",
      "timestamp" => "26/Oct/2018:16:04:39 +0800",
           "verb" => "GET",
        "request" => "/",
    "httpversion" => "1.1",
       "response" => "200",
          "bytes" => "612",
       "referrer" => "\"-\"",
          "agent" => "\"Mozilla/5.0 (X11; Linux x86_64; rv:45.0) Gecko/20100101 Firefox/45.0\"",
       "xforward" => "\"-\""
}

在这里插入图片描述

[root@server1 elk]# yum install -y kibana-4.5.1-1.x86_64.rpm
[root@server1 elk]# cd /opt/kibana/config/
[root@server1 config]# vim kibana.yml

elasticsearch.url: "http://172.25.20.1:9200"
kibana.index: ".kibana"

[root@server1 config]# /etc/init.d/kibana start

kibana started

[root@server1 config]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/nginx.conf

Settings: Default pipeline workers: 1
Pipeline main started
{
        "message" => "172.25.20.250 - - [26/Oct/2018:16:15:46 +0800] \"GET / HTTP/1.1\" 304 0 \"-\" \"Mozilla/5.0 (X11; Linux x86_64; rv:45.0) Gecko/20100101 Firefox/45.0\" \"-\"",
       "@version" => "1",
     "@timestamp" => "2018-10-26T08:15:47.465Z",
           "path" => "/var/log/nginx/access.log",
           "host" => "server1",
       "clientip" => "172.25.20.250",
          "ident" => "-",
           "auth" => "-",
      "timestamp" => "26/Oct/2018:16:15:46 +0800",
           "verb" => "GET",
        "request" => "/",
    "httpversion" => "1.1",
       "response" => "304",
          "bytes" => "0",
       "referrer" => "\"-\"",
          "agent" => "\"Mozilla/5.0 (X11; Linux x86_64; rv:45.0) Gecko/20100101 Firefox/45.0\"",
       "xforward" => "\"-\""
}

在这里插入图片描述
在这里插入图片描述
在这里插入图片描述
Server2:

[root@server2 ~]# yum install -y gcc
[root@server2 ~]# tar zxf redis-4.0.8.tar.gz
[root@server2 ~]# cd redis-4.0.8
[root@server2 redis-4.0.8]# make
[root@server2 redis-4.0.8]# make install
[root@server2 redis-4.0.8]# cd utils/
[root@server2 utils]# ./install_server.sh
[root@server2 utils]# vim /etc/redis/6379.conf

bind 0.0.0.0

[root@server2 utils]# /etc/init.d/redis_6379 restart

Stopping ...
Redis stopped
Starting Redis server...

Server1:
[root@server1 conf.d]# vim nginx.conf

input {
        file {
                path => "/var/log/nginx/access.log"
                start_position => "beginning"
        }
filter {
        grok {
                match => { "message" => "%{COMBINEDAPACHELOG} %{QS:xforward}" }
        }
}
output {
        redis {
                host => ["172.25.20.2"]
                port => 6379
                data_type => "list"
                key => "logstashtoredis"
        }
}

[root@server1 conf.d]# ll /var/log/nginx/access.log

-rw-r----- 1 nginx adm 1045 Oct 26 16:15 /var/log/nginx/access.log

[root@server1 conf.d]# chmod 644 /var/log/nginx/access.log
[root@server1 conf.d]# /etc/init.d/logstash restart

Killing logstash (pid 3313) with SIGTERM
Waiting logstash (pid 3313) to die...
Waiting logstash (pid 3313) to die...
logstash stopped.
logstash started.

Server3:
[root@server3 elk]# rpm -ivh logstash-2.3.3-1.noarch.rpm

Preparing...                ########################################### [100%]
   1:logstash               ########################################### [100%]

[root@server3 elk]# cd /etc/logstash/conf.d/
[root@server3 conf.d]# vim redis.conf

input {
        redis {
                host => ["172.25.20.2"]
                port => 6379
                data_type => "list"
                key => "logstashtoredis"
        }
}

output {
        elasticsearch {
                hosts => ["172.25.20.1"]
                index => "nginx-%{+YYYY.MM.dd}"
        }
}

[root@server3 conf.d]# /etc/init.d/logstash restart

Killing logstash (pid 1412) with SIGTERM
Waiting logstash (pid 1412) to die...
Waiting logstash (pid 1412) to die...
logstash stopped.
logstash started.

在这里插入图片描述
在这里插入图片描述
在这里插入图片描述
在这里插入图片描述
在这里插入图片描述

  • 2
    点赞
  • 0
    收藏
    觉得还不错? 一键收藏
  • 1
    评论

“相关推荐”对你有帮助么?

  • 非常没帮助
  • 没帮助
  • 一般
  • 有帮助
  • 非常有帮助
提交
评论 1
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值